using System.Text; using Amazon.Runtime; using Amazon.S3; using k8s; using k8s.Models; namespace EntKube.Web.Services; /// /// Routes Amazon S3 SDK requests through the Kubernetes API server service proxy so that /// cluster-internal MinIO pods are reachable from EntKube.Web (same mechanism as kubectl proxy). /// /// AWS Signature V4 includes the Host header. The proxy handler signs requests for /// podIP:podPort (what MinIO actually receives) and rewrites the URL to the K8s service-proxy /// path before forwarding. The signature stays valid because K8s forwards Host=podIP:podPort /// to the backend pod. /// /// Multiple service candidates are tried in priority order (headless → ClusterIP). /// On a 5xx from K8s the handler advances to the next candidate transparently, so the /// caller never needs to know which service/port ultimately worked. /// public static class KubernetesS3Proxy { // ── URL Parsing ── /// /// Parses a cluster-internal service URL of the form /// http(s)://svcName.namespace.svc.cluster.local[:port]. /// Returns null if the URL doesn't match that pattern. /// public static (string Service, string Namespace, int Port)? ParseInternalServiceUrl(string url) { if (!Uri.TryCreate(url, UriKind.Absolute, out Uri? uri)) return null; string[] parts = uri.Host.Split('.'); if (parts.Length < 3 || !parts[2].Equals("svc", StringComparison.OrdinalIgnoreCase)) return null; return (parts[0], parts[1], uri.Port > 0 ? uri.Port : (uri.Scheme == "https" ? 443 : 80)); } // ── Open ── /// /// Resolves a ready MinIO pod, builds a correctly-signed S3 client that routes /// through the Kubernetes API server service proxy, and returns both as a single /// disposable. Returns null if the endpoint URL cannot be parsed as a cluster-internal URL. /// /// The underlying handler tries multiple K8s service candidates in priority order /// (headless service first, then ClusterIP). On a 5xx from K8s it automatically /// advances to the next candidate so callers do not need retry logic. /// /// Usage: /// using var proxied = await KubernetesS3Proxy.OpenAsync(kubeconfig, endpoint, key, secret); /// var result = await proxied.S3.ListBucketsAsync(); /// public static async Task OpenAsync( string kubeconfig, string internalEndpoint, string accessKey, string secretKey, string region = "us-east-1", CancellationToken ct = default) { var parsed = ParseInternalServiceUrl(internalEndpoint); if (parsed is null) return null; using MemoryStream stream = new(Encoding.UTF8.GetBytes(kubeconfig)); KubernetesClientConfiguration k8sCfg = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); Kubernetes k8s = new(k8sCfg); (string podName, string podIP) = await FindReadyPodAsync( k8s, parsed.Value.Namespace, parsed.Value.Service, ct); bool endpointTls = internalEndpoint.StartsWith("https://", StringComparison.OrdinalIgnoreCase); string svcSchemePrefix = endpointTls ? "https:" : ""; string k8sBase = k8s.BaseUri.ToString().TrimEnd('/'); string ns = parsed.Value.Namespace; // Build an ordered list of proxy-base URLs to try. For MinIO operator the // headless service ({tenant}-hl) directly exposes the pod port without // ClusterIP translation; the main service ({tenant}) maps 443→9000. // The retrying handler will try them in order and skip 5xx responses. IReadOnlyList<(string SvcName, int SvcPort)> candidates = await ResolveServiceCandidatesAsync(k8s, ns, parsed.Value.Service, parsed.Value.Port, ct); List proxyCandidates = [..candidates .Select(c => $"{k8sBase}/api/v1/namespaces/{ns}/services/{svcSchemePrefix}{c.SvcName}:{c.SvcPort}/proxy")]; // Sign requests for the host MinIO actually sees: podIP:podPort. string signingScheme = endpointTls ? "https" : "http"; string signingBaseUrl = $"{signingScheme}://{podIP}:{parsed.Value.Port}"; HttpClient proxyHttpClient = new(new RetryingKubernetesProxyHandler(k8s.HttpClient, proxyCandidates)); AmazonS3Config config = new() { ServiceURL = signingBaseUrl, ForcePathStyle = true, AuthenticationRegion = region, UseHttp = !endpointTls, Timeout = TimeSpan.FromSeconds(30), MaxErrorRetry = 0, HttpClientFactory = new SingletonHttpClientFactory(proxyHttpClient) }; AmazonS3Client s3 = new(new BasicAWSCredentials(accessKey, secretKey), config); return new ProxiedS3Client(k8s, s3, proxyHttpClient); } // ── Service Candidate Resolution ── /// /// Returns an ordered list of (serviceName, servicePort) candidates to try as /// K8s service-proxy targets for the given pod port. /// /// Priority: /// 1. Headless service ({svc}-hl) with a port that directly equals podPort /// 2. Headless service ({svc}-hl) with a port whose targetPort resolves to podPort /// 3. Main service ({svc}) with a port whose targetPort resolves to podPort /// 4. Main service ({svc}) with a port that directly equals podPort /// 5. Main service ({svc}) first port as last resort /// 6. Hard fallback: ({svc}, endpointTls ? 443 : podPort) /// private static async Task> ResolveServiceCandidatesAsync( Kubernetes k8s, string ns, string svc, int podPort, CancellationToken ct) { List<(string, int)> results = []; // 1 & 2: Headless service. try { V1Service hl = await k8s.CoreV1.ReadNamespacedServiceAsync($"{svc}-hl", ns, cancellationToken: ct); IList? hlPorts = hl.Spec?.Ports; V1ServicePort? direct = hlPorts?.FirstOrDefault(p => p.Port == podPort); if (direct is not null) results.Add(($"{svc}-hl", direct.Port)); else { V1ServicePort? byTarget = hlPorts?.FirstOrDefault(p => p.TargetPort?.Value == podPort.ToString()); if (byTarget is not null) results.Add(($"{svc}-hl", byTarget.Port)); } } catch { } // 3, 4 & 5: Main ClusterIP service. try { V1Service main = await k8s.CoreV1.ReadNamespacedServiceAsync(svc, ns, cancellationToken: ct); IList? ports = main.Spec?.Ports; V1ServicePort? byTarget = ports?.FirstOrDefault(p => p.TargetPort?.Value == podPort.ToString()); if (byTarget is not null) { results.Add((svc, byTarget.Port)); } else { V1ServicePort? direct = ports?.FirstOrDefault(p => p.Port == podPort); int fallbackPort = direct?.Port ?? ports?.FirstOrDefault()?.Port ?? podPort; results.Add((svc, fallbackPort)); } } catch { } // Hard fallback so there is always at least one candidate. if (results.Count == 0) results.Add((svc, podPort)); return results; } // ── Pod Discovery ── private static async Task<(string PodName, string PodIP)> FindReadyPodAsync( Kubernetes k8s, string ns, string tenantName, CancellationToken ct) { // Primary: MinIO operator labels pods with v1.min.io/tenant=. V1PodList pods = await k8s.CoreV1.ListNamespacedPodAsync( ns, labelSelector: $"v1.min.io/tenant={tenantName}", cancellationToken: ct); var found = PickReadyPodWithIP(pods); if (found is not null) return found.Value; // Fallback: standalone MinIO or different label convention. V1PodList all = await k8s.CoreV1.ListNamespacedPodAsync(ns, cancellationToken: ct); found = PickReadyPodWithIP(all); return found ?? throw new InvalidOperationException( $"No running MinIO pods with a pod IP found in namespace '{ns}'. " + "Ensure MinIO is healthy before accessing bucket management."); } private static (string PodName, string PodIP)? PickReadyPodWithIP(V1PodList podList) { foreach (V1Pod pod in podList.Items) { if (pod.Status?.Phase != "Running") continue; if (string.IsNullOrEmpty(pod.Status.PodIP)) continue; if (pod.Status.ContainerStatuses?.All(cs => cs.Ready) != true) continue; return (pod.Metadata.Name, pod.Status.PodIP); } return null; } // ── Retrying Proxy Handler ── /// /// Intercepts Amazon SDK requests and rewrites the URL to successive K8s service-proxy /// candidates until one succeeds (non-5xx) or all are exhausted. /// /// - Never throws: on total failure the last 5xx response is returned so the AWS SDK /// can produce a proper AmazonServiceException that callers can catch. /// - Content bodies are buffered before the first attempt so the stream can be /// replayed on each retry. /// - Host is stripped from forwarded headers so the K8s API server does not reject /// the connection for a hostname mismatch. /// private sealed class RetryingKubernetesProxyHandler( HttpClient k8sClient, IReadOnlyList proxyCandidates) : HttpMessageHandler { protected override async Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { // Buffer body upfront so it can be resent on each candidate retry. byte[]? bodyBytes = null; IEnumerable>>? contentHeaders = null; if (request.Content is not null) { bodyBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken); contentHeaders = [..request.Content.Headers]; } HttpResponseMessage? lastResponse = null; foreach (string proxyBase in proxyCandidates) { string pathAndQuery = request.RequestUri?.PathAndQuery ?? "/"; string targetUrl = proxyBase.TrimEnd('/') + pathAndQuery; HttpRequestMessage proxied = new(request.Method, targetUrl); // Copy all headers except Host (K8s API validates Host against its own name). foreach (KeyValuePair> h in request.Headers) { if (string.Equals(h.Key, "Host", StringComparison.OrdinalIgnoreCase)) continue; proxied.Headers.TryAddWithoutValidation(h.Key, h.Value); } if (bodyBytes is not null) { ByteArrayContent body = new(bodyBytes); foreach (KeyValuePair> ch in contentHeaders!) body.Headers.TryAddWithoutValidation(ch.Key, ch.Value); proxied.Content = body; } HttpResponseMessage response = await k8sClient.SendAsync(proxied, cancellationToken); if ((int)response.StatusCode < 500) return response; // Dispose the previous failing response before moving to the next candidate. lastResponse?.Dispose(); lastResponse = response; } // All candidates exhausted — return the last 5xx response so the AWS SDK // raises a proper AmazonServiceException (catchable by callers). return lastResponse!; } } // ── Amazon SDK HttpClientFactory injection ── private sealed class SingletonHttpClientFactory(HttpClient client) : HttpClientFactory { public override HttpClient CreateHttpClient(IClientConfig clientConfig) => client; public override bool UseSDKHttpClientCaching(IClientConfig clientConfig) => false; public override bool DisposeHttpClientsAfterUse(IClientConfig clientConfig) => false; } } /// /// A paired K8s client + proxied S3 client. Dispose to release all resources. /// public sealed class ProxiedS3Client(Kubernetes k8s, AmazonS3Client s3, HttpClient proxyHttpClient) : IDisposable { public AmazonS3Client S3 { get; } = s3; public void Dispose() { S3.Dispose(); proxyHttpClient.Dispose(); k8s.Dispose(); } }