object storage: give each bucket its own credential with two rotatable key slots

Author: MitchDrage

PendingReleaseNotes

@@ -53,3 +53,34 @@ example.ver.1 > example.ver.2:
     the guest while the NVRAM sidecar is copied, so that the captured firmware
     state is consistent with the disk snapshot. Non-UEFI VMs are unaffected and
     continue to snapshot live.
+
+4.22.1.0 > 24.0.0.0:
+  * Per-bucket credentials for object storage. Buckets on Ceph RGW object storage
+    now each get their own credential, and each credential has two key slots that
+    rotate independently, so one key can be replaced while consumers keep working
+    on the other. Every bucket in an account previously shared one credential, so
+    a leaked key exposed all of them and there was no way to rotate it.
+
+    - Requires Ceph Squid (v19) or later, because it is built on RGW accounts.
+      Object storage that cannot provide per-bucket credentials keeps the existing
+      per-account behaviour, and says so in the UI.
+
+    - Existing accounts keep the shared credential until an administrator migrates
+      them, from the Object Storage tab on the account. Migration is per account
+      per object store, and cannot be undone: it adopts the account's existing RGW
+      user as the root of a new RGW account at the gateway. Buckets created before
+      the migration keep working on the shared key until each one is given its own
+      credential.
+
+    - Once every bucket on an object store has its own credential, an administrator
+      can rotate the account's own key, so that the key that was shared with users
+      no longer works.
+
+    - The global setting object.storage.per.bucket.credentials decides how an
+      account is set up the first time it uses an object store. Turning it off
+      leaves already-migrated accounts as they are.
+
+  * listBuckets now applies its objectstorageid parameter. The parameter has been
+    accepted since 4.19.0 but never filtered, so any caller passing it received
+    every bucket. Those callers will now receive only the buckets on that object
+    store.

api/src/main/java/com/cloud/agent/api/to/BucketCredentialTO.java

@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import java.util.List;
+
+/**
+ * A dedicated backend identity provisioned for a single bucket by an object
+ * store provider, together with the key pairs it currently holds.
+ */
+public final class BucketCredentialTO {
+
+    private final String providerCredentialId;
+
+    private final List<BucketKeyTO> keys;
+
+    public BucketCredentialTO(String providerCredentialId, List<BucketKeyTO> keys) {
+        this.providerCredentialId = providerCredentialId;
+        this.keys = keys;
+    }
+
+    public String getProviderCredentialId() {
+        return providerCredentialId;
+    }
+
+    public List<BucketKeyTO> getKeys() {
+        return keys;
+    }
+}

api/src/main/java/com/cloud/agent/api/to/BucketKeyTO.java

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+/**
+ * A single access/secret key pair of a bucket credential, as returned by an
+ * object store provider.
+ */
+public final class BucketKeyTO {
+
+    private final String accessKey;
+
+    private final String secretKey;
+
+    public BucketKeyTO(String accessKey, String secretKey) {
+        this.accessKey = accessKey;
+        this.secretKey = secretKey;
+    }
+
+    public String getAccessKey() {
+        return accessKey;
+    }
+
+    public String getSecretKey() {
+        return secretKey;
+    }
+}

api/src/main/java/com/cloud/agent/api/to/BucketTO.java

@@ -22,14 +22,19 @@ public final class BucketTO {
 
     private String name;
 
+    private String uuid;
+
     private String accessKey;
 
     private String secretKey;
 
     private long accountId;
 
+    private String providerCredentialId;
+
     public BucketTO(Bucket bucket) {
         this.name = bucket.getName();
+        this.uuid = bucket.getUuid();
         this.accessKey = bucket.getAccessKey();
         this.secretKey = bucket.getSecretKey();
         this.accountId = bucket.getAccountId();
@@ -43,6 +48,10 @@ public String getName() {
         return this.name;
     }
 
+    public String getUuid() {
+        return this.uuid;
+    }
+
     public String getAccessKey() {
         return this.accessKey;
     }
@@ -54,4 +63,12 @@ public String getSecretKey() {
     public long getAccountId() {
         return this.accountId;
     }
+
+    public String getProviderCredentialId() {
+        return this.providerCredentialId;
+    }
+
+    public void setProviderCredentialId(String providerCredentialId) {
+        this.providerCredentialId = providerCredentialId;
+    }
 }

api/src/main/java/com/cloud/event/EventTypes.java

@@ -816,6 +816,11 @@ public class EventTypes {
     public static final String EVENT_BUCKET_CREATE = "BUCKET.CREATE";
     public static final String EVENT_BUCKET_DELETE = "BUCKET.DELETE";
     public static final String EVENT_BUCKET_UPDATE = "BUCKET.UPDATE";
+    public static final String EVENT_BUCKET_KEY_ROTATE = "BUCKET.KEY.ROTATE";
+    public static final String EVENT_BUCKET_KEY_REVOKE = "BUCKET.KEY.REVOKE";
+    public static final String EVENT_BUCKET_CREDENTIAL_MIGRATE = "BUCKET.CREDENTIAL.MIGRATE";
+    public static final String EVENT_OBJECT_STORE_ACCOUNT_MIGRATE = "OBJECTSTORE.ACCOUNT.MIGRATE";
+    public static final String EVENT_OBJECT_STORE_ACCOUNT_KEY_ROTATE = "OBJECTSTORE.ACCOUNT.KEY.ROTATE";
 
     // Quota
     public static final String EVENT_QUOTA_TARIFF_CREATE = "QUOTA.TARIFF.CREATE";
@@ -1395,6 +1400,11 @@ public class EventTypes {
         entityEventDetails.put(EVENT_BUCKET_CREATE, Bucket.class);
         entityEventDetails.put(EVENT_BUCKET_UPDATE, Bucket.class);
         entityEventDetails.put(EVENT_BUCKET_DELETE, Bucket.class);
+        entityEventDetails.put(EVENT_BUCKET_KEY_ROTATE, Bucket.class);
+        entityEventDetails.put(EVENT_BUCKET_KEY_REVOKE, Bucket.class);
+        entityEventDetails.put(EVENT_BUCKET_CREDENTIAL_MIGRATE, Bucket.class);
+        entityEventDetails.put(EVENT_OBJECT_STORE_ACCOUNT_MIGRATE, Account.class);
+        entityEventDetails.put(EVENT_OBJECT_STORE_ACCOUNT_KEY_ROTATE, Account.class);
 
         // Quota
         entityEventDetails.put(EVENT_QUOTA_TARIFF_CREATE, QuotaTariff.class);

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -1382,6 +1382,10 @@ public class ApiConstants {
     public static final String BUCKET_LIMIT = "bucketlimit";
     public static final String BUCKET_TOTAL = "buckettotal";
     public static final String OBJECT_STORAGE_ID = "objectstorageid";
+    public static final String KEY_SLOT = "keyslot";
+    public static final String LAST_USED = "lastused";
+    public static final String CREDENTIAL_SCOPE = "credentialscope";
+    public static final String CREDENTIAL_KEYS = "keys";
     public static final String OBJECT_STORAGE = "objectstore";
     public static final String OBJECT_STORAGE_AVAILABLE = "objectstorageavailable";
     public static final String OBJECT_STORAGE_LIMIT = "objectstoragelimit";

api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java

@@ -44,6 +44,7 @@
 import org.apache.cloudstack.api.response.BackupRepositoryResponse;
 import org.apache.cloudstack.api.response.BackupScheduleResponse;
 import org.apache.cloudstack.api.response.BaseRolePermissionResponse;
+import org.apache.cloudstack.api.response.BucketKeyResponse;
 import org.apache.cloudstack.api.response.BucketResponse;
 import org.apache.cloudstack.api.response.CapacityResponse;
 import org.apache.cloudstack.api.response.ClusterResponse;
@@ -163,6 +164,7 @@
 import org.apache.cloudstack.region.Region;
 import org.apache.cloudstack.secstorage.heuristics.Heuristic;
 import org.apache.cloudstack.storage.object.Bucket;
+import org.apache.cloudstack.storage.object.BucketCredentialKey;
 import org.apache.cloudstack.storage.object.ObjectStore;
 import org.apache.cloudstack.storage.sharedfs.SharedFS;
 import org.apache.cloudstack.usage.Usage;
@@ -592,6 +594,8 @@ DirectDownloadCertificateHostStatusResponse createDirectDownloadCertificateProvi
 
     BucketResponse createBucketResponse(Bucket bucket);
 
+    BucketKeyResponse createBucketKeyResponse(BucketCredentialKey key);
+
     ASNRangeResponse createASNumberRangeResponse(ASNumberRange asnRange);
 
     ASNumberResponse createASNumberResponse(ASNumber asn);

api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java

@@ -22,6 +22,7 @@
 import org.apache.cloudstack.api.BaseListCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.api.response.AccountResponse;
 import org.apache.cloudstack.api.response.ObjectStoreResponse;
 
 @APICommand(name = "listObjectStoragePools", description = "Lists object storage pools.", responseObject = ObjectStoreResponse.class, since = "4.19.0",
@@ -43,6 +44,10 @@ public class ListObjectStoragePoolsCmd extends BaseListCmd {
     @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class, description = "the ID of the storage pool")
     private Long id;
 
+    @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class,
+            description = "when given, each store also reports the credential state of this account on it: accountcredentialscope and legacybuckets", since = "24.0.0")
+    private Long accountId;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -56,6 +61,10 @@ public Long getId() {
         return id;
     }
 
+    public Long getAccountId() {
+        return accountId;
+    }
+
     public String getProvider() {
         return provider;
     }

api/src/main/java/org/apache/cloudstack/api/command/admin/storage/MigrateObjectStoreAccountCmd.java

@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.storage;
+
+import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiCommandResourceType;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.AccountResponse;
+import org.apache.cloudstack.api.response.ObjectStoreResponse;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.context.CallContext;
+
+@APICommand(name = "migrateObjectStoreAccount", description = "Migrates an account's identity on an object store into the state its provider requires for per-bucket credentials. On Ceph RGW this creates an RGW account and adopts the account's existing RGW user into it as the account root, which transfers ownership of all its buckets to the RGW account and is permanent. Existing buckets keep working with their current keys; they gain dedicated credentials individually via migrateBucketCredential.",
+        responseObject = SuccessResponse.class, entityType = {Account.class},
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "24.0.0",
+        authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin})
+public class MigrateObjectStoreAccountCmd extends BaseCmd {
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class,
+            required = true, description = "The ID of the account to migrate")
+    private Long accountId;
+
+    @Parameter(name = ApiConstants.OBJECT_STORAGE_ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class,
+            required = true, description = "The ID of the object store on which to migrate the account")
+    private Long objectStoreId;
+
+    /////////////////////////////////////////////////////
+    /////////////////// Accessors ///////////////////////
+    /////////////////////////////////////////////////////
+
+    public Long getAccountId() {
+        return accountId;
+    }
+
+    public Long getObjectStoreId() {
+        return objectStoreId;
+    }
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        Account account = _entityMgr.findById(Account.class, getAccountId());
+        if (account != null) {
+            return account.getId();
+        }
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+
+    @Override
+    public Long getApiResourceId() {
+        return accountId;
+    }
+
+    @Override
+    public ApiCommandResourceType getApiResourceType() {
+        return ApiCommandResourceType.Account;
+    }
+
+    @Override
+    public void execute() {
+        CallContext.current().setEventDetails("Account ID: " + getResourceUuid(ApiConstants.ACCOUNT_ID) + " object store ID: " + getResourceUuid(ApiConstants.OBJECT_STORAGE_ID));
+        boolean result;
+        try {
+            result = _bucketService.migrateObjectStoreAccount(this, CallContext.current().getCallingAccount());
+        } catch (InvalidParameterValueException e) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, e.getMessage());
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Error while migrating account on object store. " + e.getMessage());
+        }
+        if (result) {
+            setResponseObject(new SuccessResponse(getCommandName()));
+        } else {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to migrate account on object store");
+        }
+    }
+}