426 lines
17 KiB
C#
426 lines
17 KiB
C#
using EntKube.Provisioning.Domain;
|
|
using EntKube.Provisioning.Features.DatabaseStatusReconcile;
|
|
using EntKube.Provisioning.Features.MongoClusters;
|
|
using EntKube.Provisioning.Features.PostgresClusters;
|
|
using EntKube.Provisioning.Features.RedisClusters;
|
|
using EntKube.Provisioning.Infrastructure;
|
|
using FluentAssertions;
|
|
using Moq;
|
|
|
|
namespace EntKube.Provisioning.Tests.Features;
|
|
|
|
/// <summary>
|
|
/// Tests for the DatabaseStatusReconcileHandler, which periodically checks
|
|
/// Kubernetes CRD status for CNPG, Redis, and MongoDB clusters and transitions
|
|
/// the domain models from Provisioning → Running (or Degraded) based on live state.
|
|
///
|
|
/// This is the missing piece that caused database clusters to stay stuck in
|
|
/// "Provisioning" status forever even after Kubernetes reported them as healthy.
|
|
/// </summary>
|
|
public class DatabaseStatusReconcileHandlerTests
|
|
{
|
|
private readonly InMemoryPostgresClusterRepository pgRepository;
|
|
private readonly InMemoryRedisClusterRepository redisRepository;
|
|
private readonly InMemoryMongoClusterRepository mongoRepository;
|
|
private readonly Mock<ICnpgClusterClient> cnpgClient;
|
|
private readonly Mock<IRedisClusterClient> redisClient;
|
|
private readonly Mock<IMongoClusterClient> mongoClient;
|
|
private readonly Mock<IClusterCredentialsResolver> credentialsResolver;
|
|
private readonly DatabaseStatusReconcileHandler handler;
|
|
|
|
private const string FakeKubeConfig = "apiVersion: v1\nclusters: []";
|
|
private const string FakeContextName = "test-context";
|
|
|
|
public DatabaseStatusReconcileHandlerTests()
|
|
{
|
|
pgRepository = new InMemoryPostgresClusterRepository();
|
|
redisRepository = new InMemoryRedisClusterRepository();
|
|
mongoRepository = new InMemoryMongoClusterRepository();
|
|
cnpgClient = new Mock<ICnpgClusterClient>();
|
|
redisClient = new Mock<IRedisClusterClient>();
|
|
mongoClient = new Mock<IMongoClusterClient>();
|
|
credentialsResolver = new Mock<IClusterCredentialsResolver>();
|
|
|
|
handler = new DatabaseStatusReconcileHandler(
|
|
pgRepository,
|
|
redisRepository,
|
|
mongoRepository,
|
|
cnpgClient.Object,
|
|
redisClient.Object,
|
|
mongoClient.Object,
|
|
credentialsResolver.Object);
|
|
}
|
|
|
|
// ─── CNPG PostgreSQL ─────────────────────────────────────────────────
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_CnpgClusterHealthy_TransitionsToRunning()
|
|
{
|
|
// Arrange — A freshly provisioned CNPG cluster still in Provisioning state.
|
|
// Kubernetes reports it as healthy (phase = "Cluster in healthy state").
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
PostgresCluster pgCluster = PostgresCluster.Create(
|
|
Guid.NewGuid(), clusterId, "test-db", "cnpg-system", "16.4", 3, "50Gi");
|
|
|
|
await pgRepository.AddAsync(pgCluster);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
cnpgClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "test-db", "cnpg-system", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new CnpgClusterStatus("Cluster in healthy state", 3, 3, "test-db-1", null, null));
|
|
|
|
// Act — The reconciler checks the status and transitions accordingly.
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — The cluster should now be Running.
|
|
|
|
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
|
updated!.Status.Should().Be(PostgresClusterStatus.Running);
|
|
updated.LastReconcileAt.Should().NotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_CnpgClusterNotReady_StaysProvisioning()
|
|
{
|
|
// Arrange — A CNPG cluster where K8s reports it's still setting up
|
|
// (not all instances are ready yet).
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
PostgresCluster pgCluster = PostgresCluster.Create(
|
|
Guid.NewGuid(), clusterId, "test-db", "cnpg-system", "16.4", 3, "50Gi");
|
|
|
|
await pgRepository.AddAsync(pgCluster);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
cnpgClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "test-db", "cnpg-system", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new CnpgClusterStatus("Setting up primary", 0, 3, null, null, null));
|
|
|
|
// Act
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — Still provisioning; not all instances are ready.
|
|
|
|
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
|
updated!.Status.Should().Be(PostgresClusterStatus.Provisioning);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_CnpgClusterAlreadyRunning_SkipsButChecksForDegradation()
|
|
{
|
|
// Arrange — A cluster that's already Running. The reconciler should check
|
|
// its health and mark it Degraded if it has become unhealthy.
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
PostgresCluster pgCluster = PostgresCluster.Create(
|
|
Guid.NewGuid(), clusterId, "healthy-db", "cnpg-system", "16.4", 3, "50Gi");
|
|
pgCluster.MarkRunning();
|
|
|
|
await pgRepository.AddAsync(pgCluster);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
cnpgClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "healthy-db", "cnpg-system", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new CnpgClusterStatus("Cluster in healthy state", 2, 3, "healthy-db-1", null, null));
|
|
|
|
// Act
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — Not all instances ready → mark degraded.
|
|
|
|
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
|
updated!.Status.Should().Be(PostgresClusterStatus.Degraded);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_CnpgUpgrading_CompletesUpgradeWhenHealthy()
|
|
{
|
|
// Arrange — A cluster that's mid-upgrade. When K8s reports healthy at the
|
|
// target version, the reconciler should complete the upgrade.
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
PostgresCluster pgCluster = PostgresCluster.Create(
|
|
Guid.NewGuid(), clusterId, "upgrade-db", "cnpg-system", "15.6", 3, "50Gi");
|
|
pgCluster.MarkRunning();
|
|
pgCluster.RequestUpgrade("16.4");
|
|
|
|
await pgRepository.AddAsync(pgCluster);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
cnpgClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "upgrade-db", "cnpg-system", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new CnpgClusterStatus("Cluster in healthy state", 3, 3, "upgrade-db-1", null, null));
|
|
|
|
// Act
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — Upgrade completed, back to Running.
|
|
|
|
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
|
updated!.Status.Should().Be(PostgresClusterStatus.Running);
|
|
updated.PostgresVersion.Should().Be("16.4");
|
|
}
|
|
|
|
// ─── Redis ───────────────────────────────────────────────────────────
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_RedisClusterReady_TransitionsToRunning()
|
|
{
|
|
// Arrange — A freshly provisioned Redis cluster still in Provisioning state.
|
|
// Kubernetes reports all replicas are ready.
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
RedisCluster redis = RedisCluster.Create(
|
|
Guid.NewGuid(), clusterId, "test-redis", "redis", "7.2.5", 3, "10Gi", true);
|
|
|
|
await redisRepository.AddAsync(redis);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
redisClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "test-redis", "redis", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new RedisCrdStatus("Ready", 3, 3, "test-redis-0"));
|
|
|
|
// Act
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — The Redis cluster should now be Running.
|
|
|
|
RedisCluster? updated = await redisRepository.GetByIdAsync(redis.Id);
|
|
updated!.Status.Should().Be(RedisClusterStatus.Running);
|
|
updated.LastReconcileAt.Should().NotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_RedisClusterNotReady_StaysProvisioning()
|
|
{
|
|
// Arrange — Redis operator reports Unknown phase with no ready replicas.
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
RedisCluster redis = RedisCluster.Create(
|
|
Guid.NewGuid(), clusterId, "test-redis", "redis", "7.2.5", 3, "10Gi", true);
|
|
|
|
await redisRepository.AddAsync(redis);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
redisClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "test-redis", "redis", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new RedisCrdStatus("Unknown", 0, 0, null));
|
|
|
|
// Act
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — Still provisioning.
|
|
|
|
RedisCluster? updated = await redisRepository.GetByIdAsync(redis.Id);
|
|
updated!.Status.Should().Be(RedisClusterStatus.Provisioning);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_RedisUpgrading_CompletesUpgradeWhenReady()
|
|
{
|
|
// Arrange — Redis cluster mid-upgrade.
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
RedisCluster redis = RedisCluster.Create(
|
|
Guid.NewGuid(), clusterId, "upgrade-redis", "redis", "7.0.15", 3, "10Gi", true);
|
|
redis.MarkRunning();
|
|
redis.RequestUpgrade("7.2.5");
|
|
|
|
await redisRepository.AddAsync(redis);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
redisClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "upgrade-redis", "redis", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new RedisCrdStatus("Ready", 3, 3, "upgrade-redis-0"));
|
|
|
|
// Act
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — Upgrade completed.
|
|
|
|
RedisCluster? updated = await redisRepository.GetByIdAsync(redis.Id);
|
|
updated!.Status.Should().Be(RedisClusterStatus.Running);
|
|
updated.RedisVersion.Should().Be("7.2.5");
|
|
}
|
|
|
|
// ─── MongoDB ─────────────────────────────────────────────────────────
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_MongoClusterRunningPhase_TransitionsToRunning()
|
|
{
|
|
// Arrange — A freshly provisioned MongoDB cluster still in Provisioning.
|
|
// The operator reports phase = "Running" with all members ready.
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
MongoCluster mongo = MongoCluster.Create(
|
|
Guid.NewGuid(), clusterId, "test-mongo", "mongodb-system", "7.0.12", 3, "50Gi");
|
|
|
|
await mongoRepository.AddAsync(mongo);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
mongoClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "test-mongo", "mongodb-system", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new MongoCrdStatus("Running", 3, 3, "test-mongo-0", null));
|
|
|
|
// Act
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — The MongoDB cluster should now be Running.
|
|
|
|
MongoCluster? updated = await mongoRepository.GetByIdAsync(mongo.Id);
|
|
updated!.Status.Should().Be(MongoClusterStatus.Running);
|
|
updated.LastReconcileAt.Should().NotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_MongoClusterFailed_MarksDegraded()
|
|
{
|
|
// Arrange — A MongoDB cluster where the operator reports phase = "Failed".
|
|
// This matches the user's scenario where MongoDB shows "Failed" with 0/3 ready.
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
MongoCluster mongo = MongoCluster.Create(
|
|
Guid.NewGuid(), clusterId, "failed-mongo", "mongodb-system", "7.0.12", 3, "50Gi");
|
|
|
|
await mongoRepository.AddAsync(mongo);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
mongoClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "failed-mongo", "mongodb-system", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new MongoCrdStatus("Failed", 0, 3, null, null));
|
|
|
|
// Act
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — Failed phase → Degraded with message.
|
|
|
|
MongoCluster? updated = await mongoRepository.GetByIdAsync(mongo.Id);
|
|
updated!.Status.Should().Be(MongoClusterStatus.Degraded);
|
|
updated.StatusMessage.Should().Contain("Failed");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_MongoClusterNotReady_StaysProvisioning()
|
|
{
|
|
// Arrange — MongoDB with pending phase and 0 ready members.
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
MongoCluster mongo = MongoCluster.Create(
|
|
Guid.NewGuid(), clusterId, "pending-mongo", "mongodb-system", "7.0.12", 3, "50Gi");
|
|
|
|
await mongoRepository.AddAsync(mongo);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
mongoClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "pending-mongo", "mongodb-system", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new MongoCrdStatus("Pending", 0, 3, null, null));
|
|
|
|
// Act
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — Still provisioning.
|
|
|
|
MongoCluster? updated = await mongoRepository.GetByIdAsync(mongo.Id);
|
|
updated!.Status.Should().Be(MongoClusterStatus.Provisioning);
|
|
}
|
|
|
|
// ─── Credential resolution failures ──────────────────────────────────
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_CredentialResolutionFails_SkipsCluster()
|
|
{
|
|
// Arrange — Can't reach the Clusters service. The reconciler should skip
|
|
// this cluster without crashing and move on to others.
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
PostgresCluster pgCluster = PostgresCluster.Create(
|
|
Guid.NewGuid(), clusterId, "unreachable-db", "cnpg-system", "16.4", 3, "50Gi");
|
|
|
|
await pgRepository.AddAsync(pgCluster);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync((ClusterCredentials?)null);
|
|
|
|
// Act — Should not throw.
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — Status unchanged.
|
|
|
|
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
|
updated!.Status.Should().Be(PostgresClusterStatus.Provisioning);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReconcileAsync_StatusQueryReturnsNull_DoesNotChangeStatus()
|
|
{
|
|
// Arrange — The K8s status query returns null (cluster CR not found).
|
|
// The reconciler should leave the status unchanged.
|
|
|
|
Guid clusterId = Guid.NewGuid();
|
|
|
|
PostgresCluster pgCluster = PostgresCluster.Create(
|
|
Guid.NewGuid(), clusterId, "missing-cr-db", "cnpg-system", "16.4", 3, "50Gi");
|
|
|
|
await pgRepository.AddAsync(pgCluster);
|
|
|
|
credentialsResolver.Setup(r => r.ResolveAsync(clusterId, It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new ClusterCredentials(FakeKubeConfig, FakeContextName));
|
|
|
|
cnpgClient.Setup(c => c.GetClusterStatusAsync(
|
|
FakeKubeConfig, FakeContextName, "missing-cr-db", "cnpg-system", It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync((CnpgClusterStatus?)null);
|
|
|
|
// Act
|
|
|
|
await handler.ReconcileAsync(CancellationToken.None);
|
|
|
|
// Assert — Status unchanged.
|
|
|
|
PostgresCluster? updated = await pgRepository.GetByIdAsync(pgCluster.Id);
|
|
updated!.Status.Should().Be(PostgresClusterStatus.Provisioning);
|
|
}
|
|
}
|