EntitAB/Entkube
SHA256
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

database and gitsync issues fixed

Browse Source
This commit is contained in:
Nils Blomgren
2026-06-07 14:54:16 +02:00
parent 8dc46ef031
commit 76b7e55931
72 changed files with 31067 additions and 1508 deletions
Download Patch File Download Diff File Expand all files Collapse all files

244
src/EntKube.Web/Services/KubernetesS3Proxy.cs
View File

@@ -7,21 +7,17 @@ using k8s.Models;
namespace EntKube.Web.Services;
/// <summary>
/// Routes Amazon S3 SDK requests through the Kubernetes API server pod-proxy so that
/// cluster-internal MinIO pods are reachable from EntKube.Web.
/// Routes Amazon S3 SDK requests through the Kubernetes API server service proxy so that
/// cluster-internal MinIO pods are reachable from EntKube.Web (same mechanism as kubectl proxy).
///
/// The core challenge: AWS Signature V4 includes the Host header. The K8s pod proxy
/// rewrites Host to podIP:port when forwarding to the pod, causing a signature mismatch
/// when the SDK signs for the API server hostname.
/// AWS Signature V4 includes the Host header. The proxy handler signs requests for
/// podIP:podPort (what MinIO actually receives) and rewrites the URL to the K8s service-proxy
/// path before forwarding. The signature stays valid because K8s forwards Host=podIP:podPort
/// to the backend pod.
///
/// Solution: sign with Host = podIP:port (what MinIO actually receives), then intercept
/// the request in KubernetesProxyHandler and rewrite the URL to the K8s pod-proxy path.
/// The signature remains valid; K8s routes based on the URL path, not the Host header.
///
/// SDK signs request for: http://podIP:9000/bucket?cors
/// Handler rewrites to: https://k8s-api/api/v1/namespaces/{ns}/pods/{pod}:9000/proxy/bucket?cors
/// K8s forwards to pod: http://podIP:9000/bucket?cors (Host = podIP:9000)
/// MinIO validates: Host matches signed value ✓
/// Multiple service candidates are tried in priority order (headless → ClusterIP).
/// On a 5xx from K8s the handler advances to the next candidate transparently, so the
/// caller never needs to know which service/port ultimately worked.
/// </summary>
public static class KubernetesS3Proxy
{
@@ -48,8 +44,12 @@ public static class KubernetesS3Proxy
/// <summary>
/// Resolves a ready MinIO pod, builds a correctly-signed S3 client that routes
/// through the Kubernetes API server pod proxy, and returns both as a single
/// disposable. Returns null if the endpoint URL cannot be parsed.
/// through the Kubernetes API server service proxy, and returns both as a single
/// disposable. Returns null if the endpoint URL cannot be parsed as a cluster-internal URL.
///
/// The underlying handler tries multiple K8s service candidates in priority order
/// (headless service first, then ClusterIP). On a 5xx from K8s it automatically
/// advances to the next candidate so callers do not need retry logic.
///
/// Usage:
/// using var proxied = await KubernetesS3Proxy.OpenAsync(kubeconfig, endpoint, key, secret);
@@ -74,41 +74,33 @@ public static class KubernetesS3Proxy
(string podName, string podIP) = await FindReadyPodAsync(
k8s, parsed.Value.Namespace, parsed.Value.Service, ct);
// Use the K8s SERVICE proxy instead of the pod proxy.
// Pod proxy requires the API server to make a direct TCP connection to the pod IP,
// which fails with 503 in clusters where the control plane cannot reach pod IPs
// (common in managed/cloud K8s environments).
// Service proxy routes through kube-proxy/iptables — the same path as normal
// in-cluster traffic — and works regardless of control-plane ↔ pod connectivity.
//
// Look up the real service port (e.g. 443 for TLS MinIO, 9000 for plain HTTP).
// The service port in the URL must match a port the service actually exposes.
bool endpointTls = internalEndpoint.StartsWith("https://", StringComparison.OrdinalIgnoreCase);
(int svcPort, bool svcTls) = await GetServicePortAsync(
k8s, parsed.Value.Namespace, parsed.Value.Service, parsed.Value.Port, endpointTls, ct);
string svcSchemePrefix = endpointTls ? "https:" : "";
string k8sBase = k8s.BaseUri.ToString().TrimEnd('/');
string ns = parsed.Value.Namespace;
string svcSchemePrefix = svcTls ? "https:" : "";
// Build an ordered list of proxy-base URLs to try. For MinIO operator the
// headless service ({tenant}-hl) directly exposes the pod port without
// ClusterIP translation; the main service ({tenant}) maps 443→9000.
// The retrying handler will try them in order and skip 5xx responses.
IReadOnlyList<(string SvcName, int SvcPort)> candidates =
await ResolveServiceCandidatesAsync(k8s, ns, parsed.Value.Service, parsed.Value.Port, ct);
// K8s service proxy URL: the scheme prefix tells K8s whether to use TLS when
// connecting to the backend pod (InsecureSkipVerify=true for self-signed certs).
string proxyBase = k8s.BaseUri.ToString().TrimEnd('/')
+ $"/api/v1/namespaces/{parsed.Value.Namespace}/services/{svcSchemePrefix}{parsed.Value.Service}:{svcPort}/proxy";
List<string> proxyCandidates = [..candidates
.Select(c => $"{k8sBase}/api/v1/namespaces/{ns}/services/{svcSchemePrefix}{c.SvcName}:{c.SvcPort}/proxy")];
// AWS Signature V4 is signed for the host MinIO actually receives.
// K8s service proxy creates the backend request to podIP:podPort (from the Endpoints
// resource), so MinIO sees Host = podIP:podPort — same as pod proxy.
string signingScheme = svcTls ? "https" : "http";
// Sign requests for the host MinIO actually sees: podIP:podPort.
string signingScheme = endpointTls ? "https" : "http";
string signingBaseUrl = $"{signingScheme}://{podIP}:{parsed.Value.Port}";
// Custom handler intercepts SDK requests (addressed to podIP:port) and reroutes to K8s.
HttpClient proxyHttpClient = new(new KubernetesProxyHandler(k8s.HttpClient, proxyBase));
HttpClient proxyHttpClient = new(new RetryingKubernetesProxyHandler(k8s.HttpClient, proxyCandidates));
AmazonS3Config config = new()
{
ServiceURL = signingBaseUrl,
ForcePathStyle = true,
AuthenticationRegion = region,
UseHttp = !svcTls, // must match signingBaseUrl scheme
UseHttp = !endpointTls,
Timeout = TimeSpan.FromSeconds(30),
MaxErrorRetry = 0,
HttpClientFactory = new SingletonHttpClientFactory(proxyHttpClient)
@@ -118,39 +110,68 @@ public static class KubernetesS3Proxy
return new ProxiedS3Client(k8s, s3, proxyHttpClient);
}
// ── Service Port Detection ──
// ── Service Candidate Resolution ──
/// <summary>
/// Looks up the K8s service to find the port that maps to the given pod port.
/// Returns (servicePort=443, tls=true) for TLS MinIO (operator default),
/// or (servicePort=podPort, tls=false) if no 443 mapping is found.
/// Returns an ordered list of (serviceName, servicePort) candidates to try as
/// K8s service-proxy targets for the given pod port.
///
/// Priority:
/// 1. Headless service ({svc}-hl) with a port that directly equals podPort
/// 2. Headless service ({svc}-hl) with a port whose targetPort resolves to podPort
/// 3. Main service ({svc}) with a port whose targetPort resolves to podPort
/// 4. Main service ({svc}) with a port that directly equals podPort
/// 5. Main service ({svc}) first port as last resort
/// 6. Hard fallback: ({svc}, endpointTls ? 443 : podPort)
/// </summary>
/// <param name="endpointTls">Whether the stored endpoint uses https:// — determines the
/// service proxy scheme prefix (https: tells K8s to use TLS for the backend connection).
/// Service port 443 does NOT imply TLS; the endpoint scheme is the source of truth.</param>
private static async Task<(int ServicePort, bool Tls)> GetServicePortAsync(
Kubernetes k8s, string ns, string svc, int podPort, bool endpointTls, CancellationToken ct)
private static async Task<IReadOnlyList<(string SvcName, int SvcPort)>> ResolveServiceCandidatesAsync(
Kubernetes k8s, string ns, string svc, int podPort, CancellationToken ct)
{
List<(string, int)> results = [];
// 1 & 2: Headless service.
try
{
V1Service service = await k8s.CoreV1.ReadNamespacedServiceAsync(svc, ns, cancellationToken: ct);
V1Service hl = await k8s.CoreV1.ReadNamespacedServiceAsync($"{svc}-hl", ns, cancellationToken: ct);
IList<V1ServicePort>? hlPorts = hl.Spec?.Ports;
// Find the service port that targets our pod port.
V1ServicePort? match = service.Spec?.Ports?.FirstOrDefault(p =>
p.TargetPort?.Value == podPort.ToString()
|| (p.TargetPort?.Value == null && p.Port == podPort));
if (match is not null)
return (match.Port, endpointTls); // TLS from endpoint scheme, not service port
// Fallback: use the first exposed port.
int first = service.Spec?.Ports?.FirstOrDefault()?.Port ?? podPort;
return (first, endpointTls);
V1ServicePort? direct = hlPorts?.FirstOrDefault(p => p.Port == podPort);
if (direct is not null)
results.Add(($"{svc}-hl", direct.Port));
else
{
V1ServicePort? byTarget = hlPorts?.FirstOrDefault(p => p.TargetPort?.Value == podPort.ToString());
if (byTarget is not null)
results.Add(($"{svc}-hl", byTarget.Port));
}
}
catch
catch { }
// 3, 4 & 5: Main ClusterIP service.
try
{
return (podPort, endpointTls);
V1Service main = await k8s.CoreV1.ReadNamespacedServiceAsync(svc, ns, cancellationToken: ct);
IList<V1ServicePort>? ports = main.Spec?.Ports;
V1ServicePort? byTarget = ports?.FirstOrDefault(p => p.TargetPort?.Value == podPort.ToString());
if (byTarget is not null)
{
results.Add((svc, byTarget.Port));
}
else
{
V1ServicePort? direct = ports?.FirstOrDefault(p => p.Port == podPort);
int fallbackPort = direct?.Port ?? ports?.FirstOrDefault()?.Port ?? podPort;
results.Add((svc, fallbackPort));
}
}
catch { }
// Hard fallback so there is always at least one candidate.
if (results.Count == 0)
results.Add((svc, podPort));
return results;
}
// ── Pod Discovery ──
@@ -187,63 +208,79 @@ public static class KubernetesS3Proxy
return null;
}
// ── Proxy Handler ──
// ── Retrying Proxy Handler ──
/// <summary>
/// Intercepts Amazon SDK requests addressed to http://podIP:port/path and
/// rewrites the URL to the K8s pod-proxy endpoint before forwarding via the
/// K8s-authenticated HttpClient. All headers (including Authorization) are
/// preserved so AWS Signature V4 reaches MinIO intact.
/// Intercepts Amazon SDK requests and rewrites the URL to successive K8s service-proxy
/// candidates until one succeeds (non-5xx) or all are exhausted.
///
/// - Never throws: on total failure the last 5xx response is returned so the AWS SDK
/// can produce a proper AmazonServiceException that callers can catch.
/// - Content bodies are buffered before the first attempt so the stream can be
/// replayed on each retry.
/// - Host is stripped from forwarded headers so the K8s API server does not reject
/// the connection for a hostname mismatch.
/// </summary>
private sealed class KubernetesProxyHandler(HttpClient k8sClient, string proxyBase) : HttpMessageHandler
private sealed class RetryingKubernetesProxyHandler(
HttpClient k8sClient,
IReadOnlyList<string> proxyCandidates) : HttpMessageHandler
{
private readonly string _proxyBase = proxyBase.TrimEnd('/');
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request, CancellationToken cancellationToken)
{
string pathAndQuery = request.RequestUri?.PathAndQuery ?? "/";
string targetUrl = _proxyBase + pathAndQuery;
HttpRequestMessage proxied = new(request.Method, targetUrl);
// Copy all headers except Host.
// - Authorization (AWS Signature V4), X-Amz-Date, etc. must reach MinIO unchanged.
// - Host must NOT be copied: the AWS SDK sets Host=podIP:port for signing, but
// the K8s API server validates Host against its own hostname and closes the
// connection if it doesn't match ("unexpected EOF" / SSL error).
// K8s will set Host=podIP:port itself when forwarding to the pod, which is
// exactly what MinIO needs to validate the signature.
foreach (KeyValuePair<string, IEnumerable<string>> h in request.Headers)
{
if (string.Equals(h.Key, "Host", StringComparison.OrdinalIgnoreCase))
continue;
proxied.Headers.TryAddWithoutValidation(h.Key, h.Value);
}
// Buffer body upfront so it can be resent on each candidate retry.
byte[]? bodyBytes = null;
IEnumerable<KeyValuePair<string, IEnumerable<string>>>? contentHeaders = null;
if (request.Content is not null)
proxied.Content = request.Content;
HttpResponseMessage response = await k8sClient.SendAsync(proxied, cancellationToken);
// For 5xx errors from K8s, read the body so we can surface the real reason
// (e.g. "error trying to reach service: ...", certificate errors, RBAC, etc.)
// rather than a generic "ServiceUnavailable" from the Amazon SDK.
if ((int)response.StatusCode >= 500)
{
string body = await response.Content.ReadAsStringAsync(cancellationToken);
string snippet = body.Length > 600 ? body[..600] : body;
throw new InvalidOperationException(
$"K8s proxy returned {(int)response.StatusCode} for {targetUrl} — {snippet}");
bodyBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken);
contentHeaders = [..request.Content.Headers];
}
return response;
HttpResponseMessage? lastResponse = null;
foreach (string proxyBase in proxyCandidates)
{
string pathAndQuery = request.RequestUri?.PathAndQuery ?? "/";
string targetUrl = proxyBase.TrimEnd('/') + pathAndQuery;
HttpRequestMessage proxied = new(request.Method, targetUrl);
// Copy all headers except Host (K8s API validates Host against its own name).
foreach (KeyValuePair<string, IEnumerable<string>> h in request.Headers)
{
if (string.Equals(h.Key, "Host", StringComparison.OrdinalIgnoreCase))
continue;
proxied.Headers.TryAddWithoutValidation(h.Key, h.Value);
}
if (bodyBytes is not null)
{
ByteArrayContent body = new(bodyBytes);
foreach (KeyValuePair<string, IEnumerable<string>> ch in contentHeaders!)
body.Headers.TryAddWithoutValidation(ch.Key, ch.Value);
proxied.Content = body;
}
HttpResponseMessage response = await k8sClient.SendAsync(proxied, cancellationToken);
if ((int)response.StatusCode < 500)
return response;
// Dispose the previous failing response before moving to the next candidate.
lastResponse?.Dispose();
lastResponse = response;
}
// All candidates exhausted — return the last 5xx response so the AWS SDK
// raises a proper AmazonServiceException (catchable by callers).
return lastResponse!;
}
}
// ── Amazon SDK HttpClientFactory injection ──
private sealed class SingletonHttpClientFactory(HttpClient client) : Amazon.Runtime.HttpClientFactory
private sealed class SingletonHttpClientFactory(HttpClient client) : HttpClientFactory
{
public override HttpClient CreateHttpClient(IClientConfig clientConfig) => client;
public override bool UseSDKHttpClientCaching(IClientConfig clientConfig) => false;
@@ -252,8 +289,7 @@ public static class KubernetesS3Proxy
}
/// <summary>
/// A paired K8s client + proxied S3 client. Dispose to release all three resources
/// in the correct order (S3 → proxyHttpClient → K8s).
/// A paired K8s client + proxied S3 client. Dispose to release all resources.
/// </summary>
public sealed class ProxiedS3Client(Kubernetes k8s, AmazonS3Client s3, HttpClient proxyHttpClient) : IDisposable
{