too much for one commit
This commit is contained in:
Nils Blomgren
@@ -0,0 +1,425 @@
|
||||
using EntKube.Provisioning.Domain;
|
||||
using EntKube.Provisioning.Features.DatabaseStatusReconcile;
|
||||
using EntKube.Provisioning.Features.MongoClusters;
|
||||
using EntKube.Provisioning.Features.PostgresClusters;
|
||||
using EntKube.Provisioning.Features.RedisClusters;
|
||||
using EntKube.Provisioning.Infrastructure;
|
||||
using FluentAssertions;
|
||||
using Moq;
|
||||
|
||||
namespace EntKube.Provisioning.Tests.Features;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for the DatabaseStatusReconcileHandler, which periodically checks
|
||||
/// Kubernetes CRD status for CNPG, Redis, and MongoDB clusters and transitions
|
||||
/// the domain models from Provisioning → Running (or Degraded) based on live state.
|
||||
///
|
||||
/// This is the missing piece that caused database clusters to stay stuck in
|
||||
/// "Provisioning" status forever even after Kubernetes reported them as healthy.
|
||||
/// </summary>
|
||||
public class DatabaseStatusReconcileHandlerTests
|
||||
{
|
||||
private readonly InMemoryPostgresClusterRepository pgRepository;
|
||||
private readonly InMemoryRedisClusterRepository redisRepository;
|
||||
private readonly InMemoryMongoClusterRepository mongoRepository;
|
||||
private readonly Mock<ICnpgClusterClient> cnpgClient;
|
||||
private readonly Mock<IRedisClusterClient> redisClient;
|
||||
private readonly Mock<IMongoClusterClient> mongoClient;
|
||||
private readonly Mock<IClusterCredentialsResolver> credentialsResolver;
|
||||
private readonly DatabaseStatusReconcileHandler handler;
|
||||
|
||||
private const string FakeKubeConfig = "apiVersion: v1\nclusters: []";
|
||||
private const string FakeContextName = "test-context";
|
||||
|
||||
public DatabaseStatusReconcileHandlerTests()
|
||||
{
|
||||
pgRepository = new InMemoryPostgresClusterRepository();
|
||||
redisRepository = new InMemoryRedisClusterRepository();
|
||||
mongoRepository = new InMemoryMongoClusterRepository();
|
||||
cnpgClient = new Mock<ICnpgClusterClient>();
|
||||
redisClient = new Mock<IRedisClusterClient>();
|
||||
mongoClient = new Mock<IMongoClusterClient>();
|
||||
credentialsResolver = new Mock<IClusterCredentialsResolver>();
|
||||
|
||||
handler = new DatabaseStatusReconcileHandler(
|
||||
pgRepository,
|
||||
redisRepository,
|
||||
mongoRepository,
|
||||
cnpgClient.Object,
|
||||
redisClient.Object,
|
||||
mongoClient.Object,
|
||||
credentialsResolver.Object);
|
||||
}
|
||||
|
||||
// ─── CNPG PostgreSQL ─────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_CnpgClusterHealthy_TransitionsToRunning()
|
||||
{
|
||||
// Arrange — A freshly provisioned CNPG cluster still in Provisioning state.
|
||||
// Kubernetes reports it as healthy (phase = "Cluster in healthy state").
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
PostgresCluster pgCluster = PostgresCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "test-db", "cnpg-system", "16.4", 3, "50Gi");
|
||||
|
||||
await pgRepository.AddAsync(pgCluster);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
cnpgClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "test-db", "cnpg-system", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new CnpgClusterStatus("Cluster in healthy state", 3, 3, "test-db-1", null, null));
|
||||
|
||||
// Act — The reconciler checks the status and transitions accordingly.
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — The cluster should now be Running.
|
||||
|
||||
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
||||
updated!.Status.Should().Be(PostgresClusterStatus.Running);
|
||||
updated.LastReconcileAt.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_CnpgClusterNotReady_StaysProvisioning()
|
||||
{
|
||||
// Arrange — A CNPG cluster where K8s reports it's still setting up
|
||||
// (not all instances are ready yet).
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
PostgresCluster pgCluster = PostgresCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "test-db", "cnpg-system", "16.4", 3, "50Gi");
|
||||
|
||||
await pgRepository.AddAsync(pgCluster);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
cnpgClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "test-db", "cnpg-system", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new CnpgClusterStatus("Setting up primary", 0, 3, null, null, null));
|
||||
|
||||
// Act
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — Still provisioning; not all instances are ready.
|
||||
|
||||
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
||||
updated!.Status.Should().Be(PostgresClusterStatus.Provisioning);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_CnpgClusterAlreadyRunning_SkipsButChecksForDegradation()
|
||||
{
|
||||
// Arrange — A cluster that's already Running. The reconciler should check
|
||||
// its health and mark it Degraded if it has become unhealthy.
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
PostgresCluster pgCluster = PostgresCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "healthy-db", "cnpg-system", "16.4", 3, "50Gi");
|
||||
pgCluster.MarkRunning();
|
||||
|
||||
await pgRepository.AddAsync(pgCluster);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
cnpgClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "healthy-db", "cnpg-system", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new CnpgClusterStatus("Cluster in healthy state", 2, 3, "healthy-db-1", null, null));
|
||||
|
||||
// Act
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — Not all instances ready → mark degraded.
|
||||
|
||||
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
||||
updated!.Status.Should().Be(PostgresClusterStatus.Degraded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_CnpgUpgrading_CompletesUpgradeWhenHealthy()
|
||||
{
|
||||
// Arrange — A cluster that's mid-upgrade. When K8s reports healthy at the
|
||||
// target version, the reconciler should complete the upgrade.
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
PostgresCluster pgCluster = PostgresCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "upgrade-db", "cnpg-system", "15.6", 3, "50Gi");
|
||||
pgCluster.MarkRunning();
|
||||
pgCluster.RequestUpgrade("16.4");
|
||||
|
||||
await pgRepository.AddAsync(pgCluster);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
cnpgClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "upgrade-db", "cnpg-system", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new CnpgClusterStatus("Cluster in healthy state", 3, 3, "upgrade-db-1", null, null));
|
||||
|
||||
// Act
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — Upgrade completed, back to Running.
|
||||
|
||||
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
||||
updated!.Status.Should().Be(PostgresClusterStatus.Running);
|
||||
updated.PostgresVersion.Should().Be("16.4");
|
||||
}
|
||||
|
||||
// ─── Redis ───────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_RedisClusterReady_TransitionsToRunning()
|
||||
{
|
||||
// Arrange — A freshly provisioned Redis cluster still in Provisioning state.
|
||||
// Kubernetes reports all replicas are ready.
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
RedisCluster redis = RedisCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "test-redis", "redis", "7.2.5", 3, "10Gi", true);
|
||||
|
||||
await redisRepository.AddAsync(redis);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
redisClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "test-redis", "redis", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new RedisCrdStatus("Ready", 3, 3, "test-redis-0"));
|
||||
|
||||
// Act
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — The Redis cluster should now be Running.
|
||||
|
||||
RedisCluster? updated = await redisRepository.GetByIdAsync(redis.Id);
|
||||
updated!.Status.Should().Be(RedisClusterStatus.Running);
|
||||
updated.LastReconcileAt.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_RedisClusterNotReady_StaysProvisioning()
|
||||
{
|
||||
// Arrange — Redis operator reports Unknown phase with no ready replicas.
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
RedisCluster redis = RedisCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "test-redis", "redis", "7.2.5", 3, "10Gi", true);
|
||||
|
||||
await redisRepository.AddAsync(redis);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
redisClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "test-redis", "redis", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new RedisCrdStatus("Unknown", 0, 0, null));
|
||||
|
||||
// Act
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — Still provisioning.
|
||||
|
||||
RedisCluster? updated = await redisRepository.GetByIdAsync(redis.Id);
|
||||
updated!.Status.Should().Be(RedisClusterStatus.Provisioning);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_RedisUpgrading_CompletesUpgradeWhenReady()
|
||||
{
|
||||
// Arrange — Redis cluster mid-upgrade.
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
RedisCluster redis = RedisCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "upgrade-redis", "redis", "7.0.15", 3, "10Gi", true);
|
||||
redis.MarkRunning();
|
||||
redis.RequestUpgrade("7.2.5");
|
||||
|
||||
await redisRepository.AddAsync(redis);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
redisClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "upgrade-redis", "redis", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new RedisCrdStatus("Ready", 3, 3, "upgrade-redis-0"));
|
||||
|
||||
// Act
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — Upgrade completed.
|
||||
|
||||
RedisCluster? updated = await redisRepository.GetByIdAsync(redis.Id);
|
||||
updated!.Status.Should().Be(RedisClusterStatus.Running);
|
||||
updated.RedisVersion.Should().Be("7.2.5");
|
||||
}
|
||||
|
||||
// ─── MongoDB ─────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_MongoClusterRunningPhase_TransitionsToRunning()
|
||||
{
|
||||
// Arrange — A freshly provisioned MongoDB cluster still in Provisioning.
|
||||
// The operator reports phase = "Running" with all members ready.
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
MongoCluster mongo = MongoCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "test-mongo", "mongodb-system", "7.0.12", 3, "50Gi");
|
||||
|
||||
await mongoRepository.AddAsync(mongo);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
mongoClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "test-mongo", "mongodb-system", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new MongoCrdStatus("Running", 3, 3, "test-mongo-0", null));
|
||||
|
||||
// Act
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — The MongoDB cluster should now be Running.
|
||||
|
||||
MongoCluster? updated = await mongoRepository.GetByIdAsync(mongo.Id);
|
||||
updated!.Status.Should().Be(MongoClusterStatus.Running);
|
||||
updated.LastReconcileAt.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_MongoClusterFailed_MarksDegraded()
|
||||
{
|
||||
// Arrange — A MongoDB cluster where the operator reports phase = "Failed".
|
||||
// This matches the user's scenario where MongoDB shows "Failed" with 0/3 ready.
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
MongoCluster mongo = MongoCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "failed-mongo", "mongodb-system", "7.0.12", 3, "50Gi");
|
||||
|
||||
await mongoRepository.AddAsync(mongo);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
mongoClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "failed-mongo", "mongodb-system", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new MongoCrdStatus("Failed", 0, 3, null, null));
|
||||
|
||||
// Act
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — Failed phase → Degraded with message.
|
||||
|
||||
MongoCluster? updated = await mongoRepository.GetByIdAsync(mongo.Id);
|
||||
updated!.Status.Should().Be(MongoClusterStatus.Degraded);
|
||||
updated.StatusMessage.Should().Contain("Failed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_MongoClusterNotReady_StaysProvisioning()
|
||||
{
|
||||
// Arrange — MongoDB with pending phase and 0 ready members.
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
MongoCluster mongo = MongoCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "pending-mongo", "mongodb-system", "7.0.12", 3, "50Gi");
|
||||
|
||||
await mongoRepository.AddAsync(mongo);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
mongoClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "pending-mongo", "mongodb-system", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new MongoCrdStatus("Pending", 0, 3, null, null));
|
||||
|
||||
// Act
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — Still provisioning.
|
||||
|
||||
MongoCluster? updated = await mongoRepository.GetByIdAsync(mongo.Id);
|
||||
updated!.Status.Should().Be(MongoClusterStatus.Provisioning);
|
||||
}
|
||||
|
||||
// ─── Credential resolution failures ──────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_CredentialResolutionFails_SkipsCluster()
|
||||
{
|
||||
// Arrange — Can't reach the Clusters service. The reconciler should skip
|
||||
// this cluster without crashing and move on to others.
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
PostgresCluster pgCluster = PostgresCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "unreachable-db", "cnpg-system", "16.4", 3, "50Gi");
|
||||
|
||||
await pgRepository.AddAsync(pgCluster);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((ClusterCredentials?)null);
|
||||
|
||||
// Act — Should not throw.
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — Status unchanged.
|
||||
|
||||
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
||||
updated!.Status.Should().Be(PostgresClusterStatus.Provisioning);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReconcileAsync_StatusQueryReturnsNull_DoesNotChangeStatus()
|
||||
{
|
||||
// Arrange — The K8s status query returns null (cluster CR not found).
|
||||
// The reconciler should leave the status unchanged.
|
||||
|
||||
Guid clusterId = Guid.NewGuid();
|
||||
|
||||
PostgresCluster pgCluster = PostgresCluster.Create(
|
||||
Guid.NewGuid(), clusterId, "missing-cr-db", "cnpg-system", "16.4", 3, "50Gi");
|
||||
|
||||
await pgRepository.AddAsync(pgCluster);
|
||||
|
||||
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
||||
|
||||
cnpgClient.Setup(c => c.GetClusterStatusAsync(
|
||||
FakeKubeConfig, FakeContextName, "missing-cr-db", "cnpg-system", It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((CnpgClusterStatus?)null);
|
||||
|
||||
// Act
|
||||
|
||||
await handler.ReconcileAsync(CancellationToken.None);
|
||||
|
||||
// Assert — Status unchanged.
|
||||
|
||||
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
||||
updated!.Status.Should().Be(PostgresClusterStatus.Provisioning);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user