using EntKube.Provisioning.Domain; using EntKube.Provisioning.Features.DatabaseStatusReconcile; using EntKube.Provisioning.Features.MongoClusters; using EntKube.Provisioning.Features.PostgresClusters; using EntKube.Provisioning.Features.RedisClusters; using EntKube.Provisioning.Infrastructure; using FluentAssertions; using Moq; namespace EntKube.Provisioning.Tests.Features; /// /// Tests for the DatabaseStatusReconcileHandler, which periodically checks /// Kubernetes CRD status for CNPG, Redis, and MongoDB clusters and transitions /// the domain models from Provisioning → Running (or Degraded) based on live state. /// /// This is the missing piece that caused database clusters to stay stuck in /// "Provisioning" status forever even after Kubernetes reported them as healthy. /// public class DatabaseStatusReconcileHandlerTests { private readonly InMemoryPostgresClusterRepository pgRepository; private readonly InMemoryRedisClusterRepository redisRepository; private readonly InMemoryMongoClusterRepository mongoRepository; private readonly Mock cnpgClient; private readonly Mock redisClient; private readonly Mock mongoClient; private readonly Mock credentialsResolver; private readonly DatabaseStatusReconcileHandler handler; private const string FakeKubeConfig = "apiVersion: v1\nclusters: []"; private const string FakeContextName = "test-context"; public DatabaseStatusReconcileHandlerTests() { pgRepository = new InMemoryPostgresClusterRepository(); redisRepository = new InMemoryRedisClusterRepository(); mongoRepository = new InMemoryMongoClusterRepository(); cnpgClient = new Mock(); redisClient = new Mock(); mongoClient = new Mock(); credentialsResolver = new Mock(); handler = new DatabaseStatusReconcileHandler( pgRepository, redisRepository, mongoRepository, cnpgClient.Object, redisClient.Object, mongoClient.Object, credentialsResolver.Object); } // ─── CNPG PostgreSQL ───────────────────────────────────────────────── [Fact] public async Task ReconcileAsync_CnpgClusterHealthy_TransitionsToRunning() { // Arrange — A freshly provisioned CNPG cluster still in Provisioning state. // Kubernetes reports it as healthy (phase = "Cluster in healthy state"). Guid clusterId = Guid.NewGuid(); PostgresCluster pgCluster = PostgresCluster.Create( Guid.NewGuid(), clusterId, "test-db", "cnpg-system", "16.4", 3, "50Gi"); await pgRepository.AddAsync(pgCluster); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); cnpgClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "test-db", "cnpg-system", It.IsAny())) .ReturnsAsync(new CnpgClusterStatus("Cluster in healthy state", 3, 3, "test-db-1", null, null)); // Act — The reconciler checks the status and transitions accordingly. await handler.ReconcileAsync(CancellationToken.None); // Assert — The cluster should now be Running. PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id); updated!.Status.Should().Be(PostgresClusterStatus.Running); updated.LastReconcileAt.Should().NotBeNull(); } [Fact] public async Task ReconcileAsync_CnpgClusterNotReady_StaysProvisioning() { // Arrange — A CNPG cluster where K8s reports it's still setting up // (not all instances are ready yet). Guid clusterId = Guid.NewGuid(); PostgresCluster pgCluster = PostgresCluster.Create( Guid.NewGuid(), clusterId, "test-db", "cnpg-system", "16.4", 3, "50Gi"); await pgRepository.AddAsync(pgCluster); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); cnpgClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "test-db", "cnpg-system", It.IsAny())) .ReturnsAsync(new CnpgClusterStatus("Setting up primary", 0, 3, null, null, null)); // Act await handler.ReconcileAsync(CancellationToken.None); // Assert — Still provisioning; not all instances are ready. PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id); updated!.Status.Should().Be(PostgresClusterStatus.Provisioning); } [Fact] public async Task ReconcileAsync_CnpgClusterAlreadyRunning_SkipsButChecksForDegradation() { // Arrange — A cluster that's already Running. The reconciler should check // its health and mark it Degraded if it has become unhealthy. Guid clusterId = Guid.NewGuid(); PostgresCluster pgCluster = PostgresCluster.Create( Guid.NewGuid(), clusterId, "healthy-db", "cnpg-system", "16.4", 3, "50Gi"); pgCluster.MarkRunning(); await pgRepository.AddAsync(pgCluster); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); cnpgClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "healthy-db", "cnpg-system", It.IsAny())) .ReturnsAsync(new CnpgClusterStatus("Cluster in healthy state", 2, 3, "healthy-db-1", null, null)); // Act await handler.ReconcileAsync(CancellationToken.None); // Assert — Not all instances ready → mark degraded. PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id); updated!.Status.Should().Be(PostgresClusterStatus.Degraded); } [Fact] public async Task ReconcileAsync_CnpgUpgrading_CompletesUpgradeWhenHealthy() { // Arrange — A cluster that's mid-upgrade. When K8s reports healthy at the // target version, the reconciler should complete the upgrade. Guid clusterId = Guid.NewGuid(); PostgresCluster pgCluster = PostgresCluster.Create( Guid.NewGuid(), clusterId, "upgrade-db", "cnpg-system", "15.6", 3, "50Gi"); pgCluster.MarkRunning(); pgCluster.RequestUpgrade("16.4"); await pgRepository.AddAsync(pgCluster); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); cnpgClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "upgrade-db", "cnpg-system", It.IsAny())) .ReturnsAsync(new CnpgClusterStatus("Cluster in healthy state", 3, 3, "upgrade-db-1", null, null)); // Act await handler.ReconcileAsync(CancellationToken.None); // Assert — Upgrade completed, back to Running. PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id); updated!.Status.Should().Be(PostgresClusterStatus.Running); updated.PostgresVersion.Should().Be("16.4"); } // ─── Redis ─────────────────────────────────────────────────────────── [Fact] public async Task ReconcileAsync_RedisClusterReady_TransitionsToRunning() { // Arrange — A freshly provisioned Redis cluster still in Provisioning state. // Kubernetes reports all replicas are ready. Guid clusterId = Guid.NewGuid(); RedisCluster redis = RedisCluster.Create( Guid.NewGuid(), clusterId, "test-redis", "redis", "7.2.5", 3, "10Gi", true); await redisRepository.AddAsync(redis); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); redisClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "test-redis", "redis", It.IsAny())) .ReturnsAsync(new RedisCrdStatus("Ready", 3, 3, "test-redis-0")); // Act await handler.ReconcileAsync(CancellationToken.None); // Assert — The Redis cluster should now be Running. RedisCluster? updated = await redisRepository.GetByIdAsync(redis.Id); updated!.Status.Should().Be(RedisClusterStatus.Running); updated.LastReconcileAt.Should().NotBeNull(); } [Fact] public async Task ReconcileAsync_RedisClusterNotReady_StaysProvisioning() { // Arrange — Redis operator reports Unknown phase with no ready replicas. Guid clusterId = Guid.NewGuid(); RedisCluster redis = RedisCluster.Create( Guid.NewGuid(), clusterId, "test-redis", "redis", "7.2.5", 3, "10Gi", true); await redisRepository.AddAsync(redis); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); redisClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "test-redis", "redis", It.IsAny())) .ReturnsAsync(new RedisCrdStatus("Unknown", 0, 0, null)); // Act await handler.ReconcileAsync(CancellationToken.None); // Assert — Still provisioning. RedisCluster? updated = await redisRepository.GetByIdAsync(redis.Id); updated!.Status.Should().Be(RedisClusterStatus.Provisioning); } [Fact] public async Task ReconcileAsync_RedisUpgrading_CompletesUpgradeWhenReady() { // Arrange — Redis cluster mid-upgrade. Guid clusterId = Guid.NewGuid(); RedisCluster redis = RedisCluster.Create( Guid.NewGuid(), clusterId, "upgrade-redis", "redis", "7.0.15", 3, "10Gi", true); redis.MarkRunning(); redis.RequestUpgrade("7.2.5"); await redisRepository.AddAsync(redis); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); redisClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "upgrade-redis", "redis", It.IsAny())) .ReturnsAsync(new RedisCrdStatus("Ready", 3, 3, "upgrade-redis-0")); // Act await handler.ReconcileAsync(CancellationToken.None); // Assert — Upgrade completed. RedisCluster? updated = await redisRepository.GetByIdAsync(redis.Id); updated!.Status.Should().Be(RedisClusterStatus.Running); updated.RedisVersion.Should().Be("7.2.5"); } // ─── MongoDB ───────────────────────────────────────────────────────── [Fact] public async Task ReconcileAsync_MongoClusterRunningPhase_TransitionsToRunning() { // Arrange — A freshly provisioned MongoDB cluster still in Provisioning. // The operator reports phase = "Running" with all members ready. Guid clusterId = Guid.NewGuid(); MongoCluster mongo = MongoCluster.Create( Guid.NewGuid(), clusterId, "test-mongo", "mongodb-system", "7.0.12", 3, "50Gi"); await mongoRepository.AddAsync(mongo); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); mongoClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "test-mongo", "mongodb-system", It.IsAny())) .ReturnsAsync(new MongoCrdStatus("Running", 3, 3, "test-mongo-0", null)); // Act await handler.ReconcileAsync(CancellationToken.None); // Assert — The MongoDB cluster should now be Running. MongoCluster? updated = await mongoRepository.GetByIdAsync(mongo.Id); updated!.Status.Should().Be(MongoClusterStatus.Running); updated.LastReconcileAt.Should().NotBeNull(); } [Fact] public async Task ReconcileAsync_MongoClusterFailed_MarksDegraded() { // Arrange — A MongoDB cluster where the operator reports phase = "Failed". // This matches the user's scenario where MongoDB shows "Failed" with 0/3 ready. Guid clusterId = Guid.NewGuid(); MongoCluster mongo = MongoCluster.Create( Guid.NewGuid(), clusterId, "failed-mongo", "mongodb-system", "7.0.12", 3, "50Gi"); await mongoRepository.AddAsync(mongo); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); mongoClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "failed-mongo", "mongodb-system", It.IsAny())) .ReturnsAsync(new MongoCrdStatus("Failed", 0, 3, null, null)); // Act await handler.ReconcileAsync(CancellationToken.None); // Assert — Failed phase → Degraded with message. MongoCluster? updated = await mongoRepository.GetByIdAsync(mongo.Id); updated!.Status.Should().Be(MongoClusterStatus.Degraded); updated.StatusMessage.Should().Contain("Failed"); } [Fact] public async Task ReconcileAsync_MongoClusterNotReady_StaysProvisioning() { // Arrange — MongoDB with pending phase and 0 ready members. Guid clusterId = Guid.NewGuid(); MongoCluster mongo = MongoCluster.Create( Guid.NewGuid(), clusterId, "pending-mongo", "mongodb-system", "7.0.12", 3, "50Gi"); await mongoRepository.AddAsync(mongo); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); mongoClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "pending-mongo", "mongodb-system", It.IsAny())) .ReturnsAsync(new MongoCrdStatus("Pending", 0, 3, null, null)); // Act await handler.ReconcileAsync(CancellationToken.None); // Assert — Still provisioning. MongoCluster? updated = await mongoRepository.GetByIdAsync(mongo.Id); updated!.Status.Should().Be(MongoClusterStatus.Provisioning); } // ─── Credential resolution failures ────────────────────────────────── [Fact] public async Task ReconcileAsync_CredentialResolutionFails_SkipsCluster() { // Arrange — Can't reach the Clusters service. The reconciler should skip // this cluster without crashing and move on to others. Guid clusterId = Guid.NewGuid(); PostgresCluster pgCluster = PostgresCluster.Create( Guid.NewGuid(), clusterId, "unreachable-db", "cnpg-system", "16.4", 3, "50Gi"); await pgRepository.AddAsync(pgCluster); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync((ClusterCredentials?)null); // Act — Should not throw. await handler.ReconcileAsync(CancellationToken.None); // Assert — Status unchanged. PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id); updated!.Status.Should().Be(PostgresClusterStatus.Provisioning); } [Fact] public async Task ReconcileAsync_StatusQueryReturnsNull_DoesNotChangeStatus() { // Arrange — The K8s status query returns null (cluster CR not found). // The reconciler should leave the status unchanged. Guid clusterId = Guid.NewGuid(); PostgresCluster pgCluster = PostgresCluster.Create( Guid.NewGuid(), clusterId, "missing-cr-db", "cnpg-system", "16.4", 3, "50Gi"); await pgRepository.AddAsync(pgCluster); credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny())) .ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName)); cnpgClient.Setup(c => c.GetClusterStatusAsync( FakeKubeConfig, FakeContextName, "missing-cr-db", "cnpg-system", It.IsAny())) .ReturnsAsync((CnpgClusterStatus?)null); // Act await handler.ReconcileAsync(CancellationToken.None); // Assert — Status unchanged. PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id); updated!.Status.Should().Be(PostgresClusterStatus.Provisioning); } }