using EntKube.Clusters.Domain; using EntKube.Clusters.Features.AdoptCluster.Components.SecurityPolicies; using FluentAssertions; using k8s; using k8s.Models; using k8s.Autorest; using Moq; using Microsoft.Extensions.Logging; namespace EntKube.Clusters.Tests.Features; public class WorkloadRemediatorTests { private readonly Mock mockClient; private readonly Mock appsMock; private readonly Mock> mockLogger; private readonly WorkloadRemediator sut; public WorkloadRemediatorTests() { mockClient = new Mock(); appsMock = new Mock(); mockClient.Setup(c => c.AppsV1).Returns(appsMock.Object); mockLogger = new Mock>(); sut = new WorkloadRemediator(mockLogger.Object); // Default: empty workloads for all types. SetupMockDeployments(new V1DeploymentList { Items = new List() }); SetupMockStatefulSets(new V1StatefulSetList { Items = new List() }); SetupMockDaemonSets(new V1DaemonSetList { Items = new List() }); } // --- Seccomp Remediation --- [Fact] public async Task RemediateAsync_DeploymentWithoutSeccomp_PatchesSeccompProfile() { // Arrange — A deployment exists that has no seccomp profile set. // The remediator should patch it to add RuntimeDefault seccomp. RemediationResult result = await sut.RemediateAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — since the mock returns no workloads, no patches needed. // This verifies the method runs without errors on empty clusters. result.Should().NotBeNull(); result.PatchedWorkloads.Should().NotBeNull(); result.PolicyExceptions.Should().NotBeNull(); } [Fact] public async Task RemediateAsync_ExcludedNamespacesAreSkipped() { // Arrange — Deployments in excluded namespaces should not be touched. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("kube-system", "coredns", withSeccomp: false, withReadonlyRootfs: false) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); // Act RemediationResult result = await sut.RemediateAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — kube-system workload should be excluded, so nothing patched. result.PatchedWorkloads.Should().BeEmpty(); result.PolicyExceptions.Should().BeEmpty(); } [Fact] public async Task RemediateAsync_DeploymentMissingSeccomp_AddsSeccompPatch() { // Arrange — A deployment in a tenant namespace is missing seccomp. // The remediator should patch it to add seccomp RuntimeDefault. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("app-namespace", "my-app", withSeccomp: false, withReadonlyRootfs: true) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); SetupPatchSuccess(); // Act RemediationResult result = await sut.RemediateAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — The deployment was patched for seccomp compliance. result.PatchedWorkloads.Should().Contain(w => w.Name == "my-app" && w.Namespace == "app-namespace" && w.Reason == "seccomp"); } [Fact] public async Task RemediateAsync_DeploymentMissingReadonlyRootfs_AddsReadonlyPatch() { // Arrange — A deployment missing readOnlyRootFilesystem on its containers. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("app-namespace", "my-app", withSeccomp: true, withReadonlyRootfs: false) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); SetupPatchSuccess(); // Act RemediationResult result = await sut.RemediateAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert result.PatchedWorkloads.Should().Contain(w => w.Name == "my-app" && w.Namespace == "app-namespace" && w.Reason == "readonlyRootfs"); } [Fact] public async Task RemediateAsync_PatchFails_CreatesPolicyException() { // Arrange — A deployment that needs patching but the patch fails. // In that case, a PolicyException should be created for that workload. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("app-namespace", "stubborn-app", withSeccomp: false, withReadonlyRootfs: false) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); SetupPatchFailure(); // Act RemediationResult result = await sut.RemediateAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — Since the patch failed, we get a policy exception instead. result.PolicyExceptions.Should().Contain(e => e.WorkloadName == "stubborn-app" && e.Namespace == "app-namespace"); } [Fact] public async Task RemediateAsync_StatefulSetMissingSeccomp_PatchesIt() { // Arrange — StatefulSets should also be remediated. V1StatefulSetList statefulSets = new() { Items = new List { CreateStatefulSet("data-namespace", "postgres", withSeccomp: false, withReadonlyRootfs: true) } }; SetupEmptyDeployments(); SetupMockStatefulSets(statefulSets); SetupEmptyDaemonSets(); SetupPatchSuccess(); // Act RemediationResult result = await sut.RemediateAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert result.PatchedWorkloads.Should().Contain(w => w.Name == "postgres" && w.Namespace == "data-namespace" && w.Reason == "seccomp"); } [Fact] public async Task RemediateAsync_DaemonSetMissingReadonly_PatchesIt() { // Arrange — DaemonSets should also be remediated. V1DaemonSetList daemonSets = new() { Items = new List { CreateDaemonSet("monitoring", "node-exporter", withSeccomp: true, withReadonlyRootfs: false) } }; SetupEmptyDeployments(); SetupEmptyStatefulSets(); SetupMockDaemonSets(daemonSets); SetupPatchSuccess(); // Act — monitoring is NOT in excluded list for this test. RemediationResult result = await sut.RemediateAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert result.PatchedWorkloads.Should().Contain(w => w.Name == "node-exporter" && w.Namespace == "monitoring" && w.Reason == "readonlyRootfs"); } [Fact] public async Task RemediateAsync_CompliantWorkload_NoChanges() { // Arrange — A workload that already has seccomp and readonlyRootfs. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("app-namespace", "compliant-app", withSeccomp: true, withReadonlyRootfs: true) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); // Act RemediationResult result = await sut.RemediateAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — Compliant workload should not be touched. result.PatchedWorkloads.Should().BeEmpty(); result.PolicyExceptions.Should().BeEmpty(); } [Fact] public async Task CreatePolicyExceptionAsync_CreatesValidKyvernoPolicyException() { // Arrange — We need to create a PolicyException for a specific workload. PolicyExceptionRequest request = new( WorkloadName: "my-app", Namespace: "app-namespace", WorkloadKind: "Deployment", PolicyNames: new List { "require-seccomp-runtime-default", "require-readonly-root-filesystem" }); // Act object exception = WorkloadRemediator.BuildPolicyException(request); // Assert — The resulting object should have the correct Kyverno structure. exception.Should().NotBeNull(); Dictionary exceptionDict = (Dictionary)exception; exceptionDict["apiVersion"].Should().Be("kyverno.io/v2"); exceptionDict["kind"].Should().Be("PolicyException"); } [Fact] public async Task RemediateAsync_WritableRootFsApp_SkipsReadonlyPatch() { // Arrange — A Keycloak deployment in a tenant namespace doesn't have // readOnlyRootFilesystem set. Because Keycloak requires a writable root, // the remediator should NOT attempt to patch it for readonly compliance. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("tenant-apps", "keycloak", withSeccomp: true, withReadonlyRootfs: false, labels: new Dictionary { ["app.kubernetes.io/name"] = "keycloak" }) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); SetupPatchSuccess(); // Act RemediationResult result = await sut.RemediateAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — No readonlyRootfs patch should be applied, and no exception needed. result.PatchedWorkloads.Should().NotContain(w => w.Reason == "readonlyRootfs"); result.PolicyExceptions.Should().BeEmpty(); } [Fact] public async Task RemediateAsync_MinioStatefulSet_SkipsReadonlyPatch() { // Arrange — A MinIO StatefulSet needs a writable root filesystem for data // storage. It should be skipped for readOnly remediation but still get // seccomp patched if missing. V1StatefulSet minioSts = new() { Metadata = new V1ObjectMeta { Name = "minio", NamespaceProperty = "tenant-storage" }, Spec = new V1StatefulSetSpec { Template = new V1PodTemplateSpec { Metadata = new V1ObjectMeta { Labels = new Dictionary { ["app.kubernetes.io/name"] = "minio" } }, Spec = new V1PodSpec { Containers = new List { new() { Name = "minio", Image = "minio/minio:latest", SecurityContext = new V1SecurityContext() } } } } } }; SetupMockDeployments(new V1DeploymentList { Items = new List() }); SetupMockStatefulSets(new V1StatefulSetList { Items = new List { minioSts } }); SetupEmptyDaemonSets(); SetupPatchSuccess(); // Act RemediationResult result = await sut.RemediateAsync( mockClient.Object, new List { "kube-system" }, CancellationToken.None); // Assert — Seccomp should be patched, but readonlyRootfs should NOT. result.PatchedWorkloads.Should().Contain(w => w.Reason == "seccomp"); result.PatchedWorkloads.Should().NotContain(w => w.Reason == "readonlyRootfs"); } // --- Helper Methods --- private V1Deployment CreateDeployment(string ns, string name, bool withSeccomp, bool withReadonlyRootfs, Dictionary? labels = null) { V1SecurityContext containerSecurity = new() { ReadOnlyRootFilesystem = withReadonlyRootfs ? true : null }; V1PodSecurityContext podSecurity = new(); if (withSeccomp) { podSecurity.SeccompProfile = new V1SeccompProfile { Type = "RuntimeDefault" }; } return new V1Deployment { Metadata = new V1ObjectMeta { Name = name, NamespaceProperty = ns }, Spec = new V1DeploymentSpec { Template = new V1PodTemplateSpec { Metadata = new V1ObjectMeta { Labels = labels }, Spec = new V1PodSpec { SecurityContext = podSecurity, Containers = new List { new() { Name = "main", Image = "docker.io/app:latest", SecurityContext = containerSecurity } } } } } }; } private V1StatefulSet CreateStatefulSet(string ns, string name, bool withSeccomp, bool withReadonlyRootfs) { V1SecurityContext containerSecurity = new() { ReadOnlyRootFilesystem = withReadonlyRootfs ? true : null }; V1PodSecurityContext podSecurity = new(); if (withSeccomp) { podSecurity.SeccompProfile = new V1SeccompProfile { Type = "RuntimeDefault" }; } return new V1StatefulSet { Metadata = new V1ObjectMeta { Name = name, NamespaceProperty = ns }, Spec = new V1StatefulSetSpec { Template = new V1PodTemplateSpec { Spec = new V1PodSpec { SecurityContext = podSecurity, Containers = new List { new() { Name = "main", Image = "docker.io/app:latest", SecurityContext = containerSecurity } } } } } }; } private V1DaemonSet CreateDaemonSet(string ns, string name, bool withSeccomp, bool withReadonlyRootfs) { V1SecurityContext containerSecurity = new() { ReadOnlyRootFilesystem = withReadonlyRootfs ? true : null }; V1PodSecurityContext podSecurity = new(); if (withSeccomp) { podSecurity.SeccompProfile = new V1SeccompProfile { Type = "RuntimeDefault" }; } return new V1DaemonSet { Metadata = new V1ObjectMeta { Name = name, NamespaceProperty = ns }, Spec = new V1DaemonSetSpec { Template = new V1PodTemplateSpec { Spec = new V1PodSpec { SecurityContext = podSecurity, Containers = new List { new() { Name = "main", Image = "docker.io/app:latest", SecurityContext = containerSecurity } } } } } }; } private void SetupMockDeployments(V1DeploymentList deployments) { HttpOperationResponse response = new() { Body = deployments }; appsMock.Setup(a => a.ListDeploymentForAllNamespacesWithHttpMessagesAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>?>(), It.IsAny())) .ReturnsAsync(response); } private void SetupEmptyDeployments() { SetupMockDeployments(new V1DeploymentList { Items = new List() }); } private void SetupMockStatefulSets(V1StatefulSetList statefulSets) { HttpOperationResponse response = new() { Body = statefulSets }; appsMock.Setup(a => a.ListStatefulSetForAllNamespacesWithHttpMessagesAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>?>(), It.IsAny())) .ReturnsAsync(response); } private void SetupEmptyStatefulSets() { SetupMockStatefulSets(new V1StatefulSetList { Items = new List() }); } private void SetupMockDaemonSets(V1DaemonSetList daemonSets) { HttpOperationResponse response = new() { Body = daemonSets }; appsMock.Setup(a => a.ListDaemonSetForAllNamespacesWithHttpMessagesAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>?>(), It.IsAny())) .ReturnsAsync(response); } private void SetupEmptyDaemonSets() { SetupMockDaemonSets(new V1DaemonSetList { Items = new List() }); } private void SetupPatchSuccess() { HttpOperationResponse deployResponse = new() { Body = new V1Deployment() }; appsMock.Setup(a => a.PatchNamespacedDeploymentWithHttpMessagesAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>?>(), It.IsAny())) .ReturnsAsync(deployResponse); HttpOperationResponse stsResponse = new() { Body = new V1StatefulSet() }; appsMock.Setup(a => a.PatchNamespacedStatefulSetWithHttpMessagesAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>?>(), It.IsAny())) .ReturnsAsync(stsResponse); HttpOperationResponse dsResponse = new() { Body = new V1DaemonSet() }; appsMock.Setup(a => a.PatchNamespacedDaemonSetWithHttpMessagesAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>?>(), It.IsAny())) .ReturnsAsync(dsResponse); } private void SetupPatchFailure() { appsMock.Setup(a => a.PatchNamespacedDeploymentWithHttpMessagesAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>?>(), It.IsAny())) .ThrowsAsync(new HttpOperationException("Patch failed") { Response = new HttpResponseMessageWrapper(new System.Net.Http.HttpResponseMessage(System.Net.HttpStatusCode.Forbidden), "") }); appsMock.Setup(a => a.PatchNamespacedStatefulSetWithHttpMessagesAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>?>(), It.IsAny())) .ThrowsAsync(new HttpOperationException("Patch failed") { Response = new HttpResponseMessageWrapper(new System.Net.Http.HttpResponseMessage(System.Net.HttpStatusCode.Forbidden), "") }); appsMock.Setup(a => a.PatchNamespacedDaemonSetWithHttpMessagesAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>?>(), It.IsAny())) .ThrowsAsync(new HttpOperationException("Patch failed") { Response = new HttpResponseMessageWrapper(new System.Net.Http.HttpResponseMessage(System.Net.HttpStatusCode.Forbidden), "") }); } // --- Revert Patches --- [Fact] public async Task RevertPatchesAsync_DeploymentWithSeccomp_RemovesSeccompProfile() { // Arrange — A deployment that has seccomp RuntimeDefault set. The revert // should remove the seccomp profile so the workload no longer requires it. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("app-namespace", "my-app", withSeccomp: true, withReadonlyRootfs: false) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); SetupPatchSuccess(); // Act RemediationResult result = await sut.RevertPatchesAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — The deployment should have been reverted for seccomp. result.PatchedWorkloads.Should().Contain(w => w.Name == "my-app" && w.Namespace == "app-namespace" && w.Reason == "revert-seccomp"); } [Fact] public async Task RevertPatchesAsync_DeploymentWithReadonlyRootfs_RemovesReadonly() { // Arrange — A deployment that has readOnlyRootFilesystem=true on its containers. // The revert should set it back to false. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("app-namespace", "my-app", withSeccomp: false, withReadonlyRootfs: true) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); SetupPatchSuccess(); // Act RemediationResult result = await sut.RevertPatchesAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — The deployment should have been reverted for readonlyRootfs. result.PatchedWorkloads.Should().Contain(w => w.Name == "my-app" && w.Namespace == "app-namespace" && w.Reason == "revert-readonlyRootfs"); } [Fact] public async Task RevertPatchesAsync_WritableRootFsApp_SkipsReadonlyButRevertsSeccomp() { // Arrange — A Keycloak deployment has both seccomp and readOnlyRootFilesystem. // Following the Terraform exclusion pattern, Keycloak is in WritableRootFsApps // so readOnly was never applied by our remediator — we must NOT revert it. // But seccomp WAS applied to everything, so it should still be reverted. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("tenant-apps", "keycloak", withSeccomp: true, withReadonlyRootfs: true, labels: new Dictionary { ["app.kubernetes.io/name"] = "keycloak" }) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); SetupPatchSuccess(); // Act RemediationResult result = await sut.RevertPatchesAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — Seccomp reverted (applied to all), readOnly NOT reverted (excluded app). result.PatchedWorkloads.Should().Contain(w => w.Reason == "revert-seccomp"); result.PatchedWorkloads.Should().NotContain(w => w.Reason == "revert-readonlyRootfs"); } [Fact] public async Task RevertPatchesAsync_ExcludedNamespace_SkipsWorkloads() { // Arrange — Workloads in excluded namespaces should not be reverted. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("kube-system", "coredns", withSeccomp: true, withReadonlyRootfs: true) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); // Act RemediationResult result = await sut.RevertPatchesAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — kube-system workloads are excluded, so nothing reverted. result.PatchedWorkloads.Should().BeEmpty(); } [Fact] public async Task RevertPatchesAsync_CompliantAndUnpatched_NoChanges() { // Arrange — A workload that has neither seccomp nor readOnlyRootFilesystem // set. Nothing to revert. V1DeploymentList deployments = new() { Items = new List { CreateDeployment("app-namespace", "vanilla-app", withSeccomp: false, withReadonlyRootfs: false) } }; SetupMockDeployments(deployments); SetupEmptyStatefulSets(); SetupEmptyDaemonSets(); // Act RemediationResult result = await sut.RevertPatchesAsync( mockClient.Object, new List { "kube-system", "kyverno" }, CancellationToken.None); // Assert — Nothing to revert. result.PatchedWorkloads.Should().BeEmpty(); } }