Skip to content

poc db design identity replacement v1 a bit more simplified #6841

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Miguel Soriano (miguelsorianod) wants to merge 1 commit into
base: main
Choose a base branch
from
+771 −168
Draft

poc db design identity replacement v1 a bit more simplified #6841

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 96 additions & 7 deletions internal/api/coreapi/types_cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ type HCPOpenShiftCluster struct {
ServiceProviderProperties HCPOpenShiftClusterServiceProviderProperties `json:"serviceProviderProperties,omitempty"`
// Written by: Frontend PUT/PATCH Cluster (Create/Update), ClusterIdentitySync
Identity *ManagedServiceIdentity `json:"identity,omitempty"`
// Written by: ClusterDegradedAggregator, ClusterRequirementsValidAggregator
// Written by: ClusterDegradedAggregator, ClusterRequirementsValidAggregator, ManagedIdentitiesStatusAggregator
Status HCPOpenShiftClusterStatus `json:"status"`
}

Expand Down Expand Up @@ -69,6 +69,84 @@ type HCPOpenShiftClusterStatus struct {
// +listType=map
// +listMapKey=type
UserFacingConditions []metav1.Condition `json:"userFacingConditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`

// OperatorIdentities is a distilled rollup of identity replacement for
// Cluster-document readers (Desired vs Active ResourceID per operator).
// The full slot/instance ledger lives on
// ServiceProviderCluster.Spec/Status.ManagedIdentities; see
// ServiceProviderClusterManagedIdentitiesSpec. This field is not mapped to
// ARM ResourceStatus. ARM only exposes UserFacingConditions.
// Written by: ManagedIdentitiesStatusAggregator
OperatorIdentities ClusterOperatorIdentitiesStatus `json:"operatorIdentities,omitempty"`
}

// ClusterOperatorIdentitiesStatus is the Cluster-document rollup of identity
// replacement. Each map entry is one slot (one operator, or the SMI): Desired
// is what the user asked for, Active is what is in service. Instance history,
// role-assignment progress, and overlapping replacements live on
// ServiceProviderCluster.Status.ManagedIdentities.
type ClusterOperatorIdentitiesStatus struct {
// Conditions are per-cluster identity observations. Keep this list small.
// Known types:
// - "Configured": True when every desired slot is Active on its desired
// instance and no instance is still deconfiguring.
// Written by: ManagedIdentitiesStatusAggregator
// +optional
// +patchMergeKey=type
// +patchStrategy=merge
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`

// ControlPlaneOperators is keyed by operator name as stored on
// Cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators.
// Keys are not lowercased; they keep the Cluster map's casing. Lookups must
// use that same name, not a lowercased form and not the identity ResourceID.
// Written by: ManagedIdentitiesStatusAggregator
ControlPlaneOperators map[string]ClusterOperatorIdentityStatus `json:"controlPlaneOperators,omitempty"`
// DataPlaneOperators is keyed by operator name as stored on
// Cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.DataPlaneOperators.
// Keys are not lowercased; they keep the Cluster map's casing. Lookups must
// use that same name, not a lowercased form and not the identity ResourceID.
// Written by: ManagedIdentitiesStatusAggregator
DataPlaneOperators map[string]ClusterOperatorIdentityStatus `json:"dataPlaneOperators,omitempty"`
// ServiceManagedIdentity is the distilled observed state of the cluster's
// service managed identity. Nil when that slot is absent.
// Written by: ManagedIdentitiesStatusAggregator
ServiceManagedIdentity *ClusterOperatorIdentityStatus `json:"serviceManagedIdentity,omitempty"`
}

// ClusterOperatorIdentityStatus is the distilled observed state of one slot
// (one operator or the SMI). Desired* is the latest Cluster ResourceID. Active*
// is what is in service. They differ during replacement.
type ClusterOperatorIdentityStatus struct {
// DesiredResourceID is the latest Cluster-desired identity ResourceID for
// this slot, copied from Spec.ManagedIdentities.
// Written by: ManagedIdentitiesStatusAggregator
DesiredResourceID *azcorearm.ResourceID `json:"desiredResourceID,omitempty"`
// ActiveResourceID is the identity ResourceID currently in service for this
// slot, copied from Status.ManagedIdentities slot ActiveResourceID. Empty
// until an instance is activated.
// Written by: ManagedIdentitiesStatusAggregator
ActiveResourceID *azcorearm.ResourceID `json:"activeResourceID,omitempty"`
// ClientID is the Client ID of the Active identity. Empty until an instance
// is activated and metadata is resolved.
// Written by: ManagedIdentitiesStatusAggregator
ClientID *string `json:"clientID,omitempty"`
// PrincipalID is the Principal ID of the Active identity. Empty until an
// instance is activated and metadata is resolved.
// Written by: ManagedIdentitiesStatusAggregator
PrincipalID *string `json:"principalID,omitempty"`

// Conditions for this slot. Known types:
// - "Configured": True when Active matches Desired and that instance is fully configured.
// Written by: ManagedIdentitiesStatusAggregator
// +optional
// +patchMergeKey=type
// +patchStrategy=merge
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
}

var _ CosmosPersistable = &HCPOpenShiftCluster{}
Expand Down Expand Up @@ -298,13 +376,24 @@ type OperatorsAuthenticationProfile struct {
UserAssignedIdentities UserAssignedIdentitiesProfile `json:"userAssignedIdentities,omitempty"`
}

// UserAssignedIdentitiesProfile represents authentication configuration for
// OpenShift operators using user-assigned managed identities.
// Visibility for the entire struct is "read create".
// UserAssignedIdentitiesProfile is the ARM-visible latest desired ResourceID
// for each operator and the SMI. Visibility for the entire struct is "read create".
// Frontend PUT/PATCH overwrites these maps. Replacement history (old ResourceIDs
// still being deconfigured, overlapping A then B then C changes) lives on
// ServiceProviderCluster Spec/Status ManagedIdentities, not here. See
// ServiceProviderClusterManagedIdentitiesSpec.
type UserAssignedIdentitiesProfile struct {
ControlPlaneOperators map[string]*azcorearm.ResourceID `json:"controlPlaneOperators,omitempty"`
DataPlaneOperators map[string]*azcorearm.ResourceID `json:"dataPlaneOperators,omitempty"`
ServiceManagedIdentity *azcorearm.ResourceID `json:"serviceManagedIdentity,omitempty"`
// ControlPlaneOperators is keyed by operator name (for example
// "cloud-controller-manager"). Keys are not lowercased; they keep the
// casing from the ARM request. Values are identity ResourceIDs.
ControlPlaneOperators map[string]*azcorearm.ResourceID `json:"controlPlaneOperators,omitempty"`
// DataPlaneOperators is keyed by operator name (for example
// "image-registry"). Keys are not lowercased; they keep the casing from
// the ARM request. Values are identity ResourceIDs.
DataPlaneOperators map[string]*azcorearm.ResourceID `json:"dataPlaneOperators,omitempty"`
// ServiceManagedIdentity is the latest desired ResourceID of the cluster's
// service managed identity. Nil when the user has not provided one.
ServiceManagedIdentity *azcorearm.ResourceID `json:"serviceManagedIdentity,omitempty"`
}

// ClusterImageRegistryProfile - OpenShift cluster image registry
Expand Down
176 changes: 15 additions & 161 deletions internal/api/coreapi/types_serviceprovider_cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,13 @@ type ServiceProviderClusterSpec struct {
// BackupScheduleState is the desired backup scheduling state: Enabled or Disabled.
// Default is Enabled. Set to Disabled via Admin API to pause scheduled backups.
BackupScheduleState BackupScheduleState `json:"backupScheduleState,omitempty"`

// ManagedIdentities is the desired identity per operator slot and the SMI.
// A slot is one operator (or the SMI) asking which Azure identity to use.
// See ServiceProviderClusterManagedIdentitiesSpec for the
// slot/generation/instance model.
// Written by: ManagedIdentitiesCoordinator
ManagedIdentities ServiceProviderClusterManagedIdentitiesSpec `json:"managedIdentities,omitempty"`
}

// ServiceProviderClusterSpecVersion contains the desired version information.
Expand Down Expand Up @@ -233,167 +240,14 @@ type ServiceProviderClusterStatus struct {
// the cluster, including deny assignments and the managed resource group.
AzureResources AzureResources `json:"azureResources,omitempty"`

// MSIManagedIdentities tracks resolved ClientID/PrincipalID for
// the Managed Service Identity (MSI) based Azure User-Assigned Managed Identities
// associated to the cluster. Those are the cluster's control plane operators and
// the cluster's service managed identity.
// A cluster's control plane operator is a kubernetes operator associated to
// the cluster that runs in the cluster's control plane. For example,
// the Cluster's CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators
// map contains (and is not limited to) the set of required control plane operators associated to a Cluster.
// The cluster's service managed identity is used to read and modify
// customer-provided Azure resources (for example the cluster subnet),
// subject to the permissions granted to that identity.
// MSI-based user-assigned managed identities are the identities defined in
// the Cluster's `identity` section. Credentials for those identities can be
// obtained from Microsoft's Managed Identities Data Plane service.
// In ARO-HCP environments where Microsoft's Managed Identities Data Plane
// service is unavailable, a fake Managed Identities Data Plane client is
// used. That client always returns the same identity metadata and
// credentials, regardless of which identity is requested. The returned
// values belong to the "MI Mock" identity, so the ClientID and PrincipalID
// stored for each entry here will not match that entry's ResourceID key,
// nor the real ClientID/PrincipalID of the corresponding identity in the
// Cluster's `identity` section.
// Additionally, this also tracks when Azure should next be re-queried for that info.
// Written by: FetchMSIIdentitiesInfo
MSIManagedIdentities ServiceProviderClusterMSIManagedIdentities `json:"msiManagedIdentities,omitempty"`

// DataPlaneOperatorsManagedIdentities tracks resolved ClientID/PrincipalID for
// the Azure User Assigned Managed Identities associated with the cluster's data
// plane operators, plus when Azure should next be re-queried for that info.
// A cluster's data plane operator is a Kubernetes operator associated with the
// cluster that runs in the cluster's data plane.
// For example, the Cluster's CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.DataPlaneOperators map
// contains the set of required data plane operators associated with a Cluster.
// Written by: FetchDataPlaneOperatorsManagedIdentitiesInfoController
DataPlaneOperatorsManagedIdentities ServiceProviderClusterDataPlaneOperatorsManagedIdentities `json:"dataPlaneOperatorsManagedIdentities,omitempty"`
}

// ServiceProviderClusterMSIManagedIdentities holds Managed Service Identity (MSI)
// based identity metadata resolved by FetchMSIIdentitiesInfo and consumed by ClusterIdentitySync to
// populate HCPOpenShiftCluster.Identity.UserAssignedIdentities.
type ServiceProviderClusterMSIManagedIdentities struct {
// EarliestRecheckTime is the earliest time at which the controller
// should re-query Azure for ClientID/PrincipalID of ControlPlaneOperatorsIdentities
// and ServiceManagedIdentity.
// Nil means recheck immediately.
// The same recheck time applies across all entries in ControlPlaneOperatorsIdentities
// and ServiceManagedIdentity.
// This allows the controller to avoid repeatedly hitting an Azure API to
// recheck that the desired state is true.
// Controllers should set this field with substantial jitter: without another
// concern, jitter of 50% is considered normal so that any storms are quickly
// dissipated. Additionally, long recheck times are recommended for resources
// outside of their active phases. Order of at least six hours is, with
// durations up to 24 hours considered normal.
// Written by: FetchMSIIdentitiesInfo
EarliestRecheckTime *metav1.Time `json:"earliestRecheckTime,omitempty"`
// ControlPlaneOperatorsIdentities is a map containing resolved ClientID/PrincipalID
// for Managed Service Identity (MSI) based Azure User-Assigned Managed Identities
// used by the cluster's control plane operators. The key is the fully lowercased
// Azure Resource ID of the identity. Which operators reference each identity is
// tracked on Cluster.CustomerProperties, not here. Multiple operators may share
// one identity entry.
// Written by: FetchMSIIdentitiesInfo
ControlPlaneOperatorsIdentities map[string]*ServiceProviderClusterControlPlaneOperatorIdentity `json:"controlPlaneOperatorsIdentities,omitempty"`
// ServiceManagedIdentity holds resolved ClientID/PrincipalID for the cluster's
// service managed identity.
// Written by: FetchMSIIdentitiesInfo
ServiceManagedIdentity *ServiceProviderClusterServiceManagedIdentity `json:"serviceManagedIdentity,omitempty"`
}

// ServiceProviderClusterControlPlaneOperatorIdentity is the resolved metadata for a
// single Managed Service Identity (MSI) based Azure User-Assigned Managed Identity
// used by one or more control plane operators.
// Which operators reference this identity is tracked on
// Cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.ControlPlaneOperators.
type ServiceProviderClusterControlPlaneOperatorIdentity struct {
// ResourceID is the Azure Resource ID of the Azure User Assigned Managed Identity.
// Its value comes from the Cluster's CustomerProperties.
// The ControlPlaneOperatorsIdentities map key is the fully lowercased form
// of this ID used for lookups.
ResourceID *azcorearm.ResourceID `json:"resourceId,omitempty"`
// ClientID is the Client ID of the Azure User Assigned Managed Identity represented by ResourceID.
// Fetched from Azure and written here by the FetchMSIIdentitiesInfo.
// It may be nil or empty.
ClientID *string `json:"clientId,omitempty"`
// PrincipalID is the Principal ID of the Azure User Assigned Managed Identity represented by ResourceID.
// Fetched from Azure and written here by the FetchMSIIdentitiesInfo.
// It may be nil or empty.
PrincipalID *string `json:"principalId,omitempty"`
}

// ServiceProviderClusterServiceManagedIdentity is the resolved metadata for the
// cluster's service managed identity.
type ServiceProviderClusterServiceManagedIdentity struct {
// ResourceID is the Azure Resource ID of the Azure User Assigned Managed Identity that is associated to the cluster's Service Managed Identity.
// Its value comes from the Cluster's CustomerProperties.
ResourceID *azcorearm.ResourceID `json:"resourceId,omitempty"`
// ClientID is the Client ID of the Azure User Assigned Managed Identity represented by ResourceID.
// Fetched from Azure and written here by the FetchMSIIdentitiesInfo.
// It may be nil or empty.
ClientID *string `json:"clientId,omitempty"`
// PrincipalID is the Principal ID of the Azure User Assigned Managed Identity represented by ResourceID.
// Fetched from Azure and written here by the FetchMSIIdentitiesInfo.
// It may be nil or empty.
PrincipalID *string `json:"principalId,omitempty"`
}

// ServiceProviderClusterDataPlaneOperatorsManagedIdentities holds the resolved
// managed-identity metadata for all data plane operators on a cluster, together
// with a single EarliestRecheckTime that applies to every entry in Identities.
type ServiceProviderClusterDataPlaneOperatorsManagedIdentities struct {
// Identities is a map containing resolved ClientID/PrincipalID for the Azure
// User Assigned Managed Identities associated with the cluster's data plane
// operators. The key is the fully lowercased Azure Resource ID of the
// identity. Which operators reference each identity is tracked on
// Cluster.CustomerProperties, not here. Multiple operators may share one
// identity entry.
// Written by: FetchDataPlaneOperatorsManagedIdentitiesInfoController
Identities map[string]*ServiceProviderClusterDataPlaneOperatorManagedIdentity `json:"identities,omitempty"`
// EarliestRecheckTime is the earliest time at which the controller should
// re-query Azure for ClientID/PrincipalID of Identities. Nil means recheck
// immediately. The same recheck time applies across all elements of Identities.
// This allows the controller to avoid repeatedly hitting an Azure API to
// recheck that the desired state is true.
// Controllers should set this field with substantial jitter: without another
// concern, jitter of 50% is considered normal so that any storms are quickly
// dissipated. Additionally, long recheck times are recommended for resources
// outside of their active phases. Order of at least six hours is, with
// durations up to 24 hours considered normal.
// Written by: FetchDataPlaneOperatorsManagedIdentitiesInfoController
EarliestRecheckTime *metav1.Time `json:"earliestRecheckTime,omitempty"`
}

// ServiceProviderClusterDataPlaneOperatorManagedIdentity contains resolved
// ClientID/PrincipalID for an Azure User Assigned Managed Identity used by one
// or more of a cluster's data plane operators.
// A cluster's data plane operator is a customer operator associated with the cluster that runs in the cluster's data plane.
// Which operators reference this identity is tracked on
// Cluster.CustomerProperties.Platform.OperatorsAuthentication.UserAssignedIdentities.DataPlaneOperators.
type ServiceProviderClusterDataPlaneOperatorManagedIdentity struct {
// ResourceID is the Azure Resource ID of the Azure User Assigned Managed Identity.
// This field is an input: its value is mirrored from the Cluster's CustomerProperties
// (Platform.OperatorsAuthentication.UserAssignedIdentities.DataPlaneOperators) into this status.
// Written by: FetchDataPlaneOperatorsManagedIdentitiesInfoController
ResourceID *azcorearm.ResourceID `json:"resourceID,omitempty"`
// ClientID is the Client ID of the Azure User Assigned Managed Identity represented by ResourceID.
// This field is an output: it is fetched from Azure and written here by the controller.
// Written by: FetchDataPlaneOperatorsManagedIdentitiesInfoController
ClientID *string `json:"clientId,omitempty"`
// PrincipalID is the Principal ID of the Azure User Assigned Managed Identity represented by ResourceID.
// This field is an output: it is fetched from Azure and written here by the controller.
// Written by: FetchDataPlaneOperatorsManagedIdentitiesInfoController
PrincipalID *string `json:"principalId,omitempty"`
// RetrievalError, when non-nil, is the error (truncated to the first 1024 characters) from the
// most recent attempt to retrieve this identity's metadata from Azure. When set, ClientID and
// PrincipalID are nil because the last retrieval attempt failed - either the identity was not
// found in Azure or the Get call returned an error - and any previously resolved values are no
// longer trustworthy. It is nil when the last retrieval succeeded.
// This field is an output: it is written here by the controller.
// Written by: FetchDataPlaneOperatorsManagedIdentitiesInfoController
RetrievalError *string `json:"retrievalError,omitempty"`
// ManagedIdentities is the observed replacement ledger: Active identity per
// slot, shared generations, control-plane / SMI / data-plane instance maps,
// nested conditions, and fetch recheck times. Spec.ManagedIdentities is what
// each slot wants; this Status is what is in service and in flight. See
// ServiceProviderClusterManagedIdentitiesSpec.
// Written by: ManagedIdentitiesCoordinator, FetchMSIIdentitiesInfo,
// FetchDataPlaneOperatorsManagedIdentitiesInfo, ManagedIdentitiesStatusAggregator
ManagedIdentities ServiceProviderClusterManagedIdentitiesStatus `json:"managedIdentities,omitempty"`
}

// AzureResources groups the Azure resource references associated with a cluster.
Expand Down
Loading