EntitAB/Entkube
SHA256
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gateway api fixes

Browse Source
This commit is contained in:
Nils Blomgren
2026-05-21 16:57:50 +02:00
parent 8b94fb8826
commit e0f967482e
68 changed files with 44082 additions and 437 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
src/EntKube.Web/Services/OpenStackS3Service.cs
View File

@@ -79,7 +79,7 @@ public class OpenStackS3Service(VaultService vaultService, IHttpClientFactory ht
// Step 2: Authenticate against Keystone to get a scoped token, user ID, and project ID.
(string token, string userId, string projectId) = await AuthenticateKeystoneAsync(connection, password, ct);
(string token, string userId, string projectId, _) = await AuthenticateKeystoneAsync(connection, password, ct);
// Step 3: Create EC2 credentials for S3 access.
@@ -121,7 +121,7 @@ public class OpenStackS3Service(VaultService vaultService, IHttpClientFactory ht
/// Authenticates against OpenStack Keystone v3 using password auth.
/// Returns the scoped auth token and the user ID (needed for EC2 credential creation).
/// </summary>
private async Task<(string Token, string UserId, string ProjectId)> AuthenticateKeystoneAsync(
private async Task<(string Token, string UserId, string ProjectId, string? SwiftEndpoint)> AuthenticateKeystoneAsync(
OpenStackConnection connection, string password, CancellationToken ct)
{
// Build the Keystone v3 auth request body.
@@ -185,10 +185,10 @@ public class OpenStackS3Service(VaultService vaultService, IHttpClientFactory ht
string token = response.Headers.GetValues("X-Subject-Token").First();
// Extract the user ID and project ID from the response body.
// The project ID here is always a UUID, even if we authenticated by project name.
// Extract the user ID, project ID, and Swift endpoint from the response body.
using JsonDocument doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync(ct));
string body = await response.Content.ReadAsStringAsync(ct);
using JsonDocument doc = JsonDocument.Parse(body);
JsonElement tokenElement = doc.RootElement.GetProperty("token");
string userId = tokenElement
@@ -201,7 +201,104 @@ public class OpenStackS3Service(VaultService vaultService, IHttpClientFactory ht
.GetProperty("id")
.GetString()!;
return (token, userId, projectId);
// Find the public Swift (object-store) endpoint from the service catalog.
// This lets us list all containers regardless of which EC2 credential pair was used.
string? swiftEndpoint = null;
if (tokenElement.TryGetProperty("catalog", out JsonElement catalog))
{
foreach (JsonElement service in catalog.EnumerateArray())
{
if (service.TryGetProperty("type", out JsonElement typeEl)
&& typeEl.GetString() == "object-store"
&& service.TryGetProperty("endpoints", out JsonElement endpoints))
{
foreach (JsonElement ep in endpoints.EnumerateArray())
{
if (ep.TryGetProperty("interface", out JsonElement iface)
&& iface.GetString() == "public"
&& ep.TryGetProperty("url", out JsonElement urlEl))
{
swiftEndpoint = urlEl.GetString();
break;
}
}
if (swiftEndpoint is not null)
{
break;
}
}
}
}
return (token, userId, projectId, swiftEndpoint);
}
/// <summary>
/// Lists all containers in the OpenStack project via the Swift REST API.
/// More reliable than S3 ListBuckets against Ceph RADOS Gateway because it
/// goes through the project-scoped object-store endpoint rather than the
/// per-EC2-credential RGW user scope.
/// </summary>
public async Task<List<S3BucketInfo>> ListBucketsViaSwiftAsync(
OpenStackConnection connection, string password, CancellationToken ct = default)
{
(string token, _, _, string? swiftEndpoint) = await AuthenticateKeystoneAsync(connection, password, ct);
if (swiftEndpoint is null)
{
throw new InvalidOperationException(
"No public Swift (object-store) endpoint found in the Keystone service catalog.");
}
// GET <swift_endpoint>?format=json lists all containers owned by this project.
using HttpClient client = httpClientFactory.CreateClient();
string listUrl = swiftEndpoint.TrimEnd('/') + "?format=json";
using HttpRequestMessage listRequest = new(HttpMethod.Get, listUrl);
listRequest.Headers.Add("X-Auth-Token", token);
using HttpResponseMessage listResponse = await client.SendAsync(listRequest, ct);
if (!listResponse.IsSuccessStatusCode)
{
string errorBody = await listResponse.Content.ReadAsStringAsync(ct);
throw new InvalidOperationException(
$"Swift container listing failed ({listResponse.StatusCode}): {errorBody}");
}
string responseBody = await listResponse.Content.ReadAsStringAsync(ct);
if (string.IsNullOrWhiteSpace(responseBody) || responseBody == "[]")
{
return [];
}
using JsonDocument doc = JsonDocument.Parse(responseBody);
return doc.RootElement.EnumerateArray()
.Select(container =>
{
string name = container.TryGetProperty("name", out JsonElement nameEl)
? nameEl.GetString() ?? ""
: "";
DateTime lastModified = default;
if (container.TryGetProperty("last_modified", out JsonElement lmEl)
&& DateTime.TryParse(lmEl.GetString(), out DateTime parsed))
{
lastModified = DateTime.SpecifyKind(parsed, DateTimeKind.Utc);
}
return new S3BucketInfo { Name = name, CreatedAt = lastModified };
})
.Where(b => b.Name.Length > 0)
.ToList();
}
/// <summary>
@@ -335,8 +432,8 @@ public class OpenStackS3Service(VaultService vaultService, IHttpClientFactory ht
ListBucketsResponse response = await s3Client.ListBucketsAsync(ct);
return response.Buckets
.Select(b => new S3BucketInfo { Name = b.BucketName, CreatedAt = b.CreationDate.GetValueOrDefault() })
return (response.Buckets ?? [])
.Select(b => new S3BucketInfo { Name = b.BucketName ?? "", CreatedAt = b.CreationDate.GetValueOrDefault() })
.ToList();
}
@@ -352,36 +449,47 @@ public class OpenStackS3Service(VaultService vaultService, IHttpClientFactory ht
// Empty the bucket first — S3 won't delete non-empty buckets.
// We page through all objects and delete them in batches of up to 1000.
// If the bucket doesn't exist at all we treat it as already gone.
string? continuationToken = null;
do
try
{
ListObjectsV2Request listRequest = new()
{
BucketName = bucketName,
MaxKeys = 1000,
ContinuationToken = continuationToken
};
string? continuationToken = null;
ListObjectsV2Response listResponse = await s3Client.ListObjectsV2Async(listRequest, ct);
if (listResponse.S3Objects.Count > 0)
do
{
DeleteObjectsRequest deleteObjectsRequest = new()
ListObjectsV2Request listRequest = new()
{
BucketName = bucketName,
Objects = listResponse.S3Objects
.Select(o => new KeyVersion { Key = o.Key })
.ToList()
MaxKeys = 1000,
ContinuationToken = continuationToken
};
await s3Client.DeleteObjectsAsync(deleteObjectsRequest, ct);
}
ListObjectsV2Response listResponse = await s3Client.ListObjectsV2Async(listRequest, ct);
continuationToken = listResponse.IsTruncated == true ? listResponse.NextContinuationToken : null;
List<S3Object> objects = listResponse.S3Objects ?? [];
if (objects.Count > 0)
{
DeleteObjectsRequest deleteObjectsRequest = new()
{
BucketName = bucketName,
Objects = objects
.Select(o => new KeyVersion { Key = o.Key ?? "" })
.ToList()
};
await s3Client.DeleteObjectsAsync(deleteObjectsRequest, ct);
}
continuationToken = listResponse.IsTruncated == true ? listResponse.NextContinuationToken : null;
}
while (continuationToken is not null);
}
catch (AmazonS3Exception ex) when (ex.ErrorCode == "NoSuchBucket")
{
// Bucket was already deleted outside of EntKube — nothing left to empty.
return;
}
while (continuationToken is not null);
// Remove the bucket policy before deletion. Ceph RADOS Gateway (used by Cleura)
// may reject DeleteBucket if a policy is still attached.
@@ -395,8 +503,15 @@ public class OpenStackS3Service(VaultService vaultService, IHttpClientFactory ht
// No policy to remove — that's fine, proceed with deletion.
}
DeleteBucketRequest deleteRequest = new() { BucketName = bucketName };
await s3Client.DeleteBucketAsync(deleteRequest, ct);
try
{
DeleteBucketRequest deleteRequest = new() { BucketName = bucketName };
await s3Client.DeleteBucketAsync(deleteRequest, ct);
}
catch (AmazonS3Exception ex) when (ex.ErrorCode == "NoSuchBucket")
{
// Already gone — nothing to do.
}
}
/// <summary>
@@ -544,7 +659,7 @@ public class OpenStackS3Service(VaultService vaultService, IHttpClientFactory ht
// Authenticate and create fresh EC2 credentials.
(string token, string userId, string projectId) = await AuthenticateKeystoneAsync(connection, password, ct);
(string token, string userId, string projectId, _) = await AuthenticateKeystoneAsync(connection, password, ct);
(string newAccessKey, string newSecretKey) = await CreateEc2CredentialsAsync(token, userId, projectId, connection.AuthUrl, ct);
// Verify the new credentials work by listing the bucket.
@@ -574,6 +689,11 @@ public class OpenStackS3Service(VaultService vaultService, IHttpClientFactory ht
/// </summary>
private static AmazonS3Client CreateS3Client(string endpoint, string accessKey, string secretKey, string region)
{
if (string.IsNullOrEmpty(endpoint))
{
throw new InvalidOperationException("S3 endpoint is required but was not configured for this storage link.");
}
AmazonS3Config config = new()
{
ServiceURL = endpoint,