Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/actions/deploy-lifecycle-manager-e2e/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,19 @@ runs:
cat legacy-secret-rotation.yaml
kustomize edit add patch --path legacy-secret-rotation.yaml --kind Deployment
popd
- name: Patch SKR Image Pull Secret
if: ${{ matrix.e2e-test == 'skr-image-pull-secret-sync' }}
working-directory: lifecycle-manager
shell: bash
run: |
pushd ${E2E_KUSTOMIZE_DIR}
echo \
"- op: add
path: /spec/template/spec/containers/0/args/-
value: --skr-image-pull-secret=some-special-image-pull-secret" >> skr_image_pull_secret_patch.yaml
cat skr_image_pull_secret_patch.yaml
kustomize edit add patch --path skr_image_pull_secret_patch.yaml --kind Deployment
popd
- name: Create and use maintenance window policy
if: ${{matrix.e2e-test == 'maintenance-windows' ||
matrix.e2e-test == 'maintenance-windows-initial-installation' ||
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/test-e2e.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name: TestSuite E2E

permissions: {}
permissions: { }

on:
workflow_dispatch:
Expand All @@ -9,7 +9,7 @@ on:
description: "With Kubernetes version (format: X.Y.Z)"
required: false
pull_request:
types: [opened, edited, synchronize, reopened, ready_for_review]
types: [ opened, edited, synchronize, reopened, ready_for_review ]

jobs:
wait-for-image-build:
Expand Down Expand Up @@ -79,6 +79,7 @@ jobs:
- oci-reg-cred-secret
- module-transferred-to-another-oci-registry
- simulate-skr-secret-rotation-evict-client-cache
- skr-image-pull-secret-sync

runs-on: ubuntu-latest
timeout-minutes: 20
Expand Down
1 change: 1 addition & 0 deletions .golangci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ linters:
- forcetypeassert
- funlen
- gochecknoglobals
- gocognit
- goconst
- maintidx
- nilnil
Expand Down
2 changes: 1 addition & 1 deletion .run/Launch KLM locally.run.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<configuration default="false" name="Launch KLM locally" type="GoApplicationRunConfiguration" factoryName="Go Application">
<module name="lifecycle-manager" />
<working_directory value="$PROJECT_DIR$" />
<parameters value="--enable-kcp-watcher --skr-watcher-image-tag=1.2.3"/>
<parameters value="--enable-kcp-watcher --skr-watcher-image-tag=1.2.1 --oci-registry-host=http://k3d-kcp-registry.localhost:5000 --leader-elect=false" />
<envs>
<env name="KUBECONFIG" value="$USER_HOME$/.k3d/kcp-local.yaml" />
</envs>
Expand Down
36 changes: 26 additions & 10 deletions api/v1beta2/condition_messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,30 @@ import (
apimetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Extend this list by actual needs.
//nolint:gosec // no secrets defined in these consts
const (
// DeprecatedConditionTypeReady was introduced by a bug. Ths condition needs to be removed on all KymaCRs.
DeprecatedConditionTypeReady KymaConditionType = "Ready"
ConditionTypeModules KymaConditionType = "Modules"
ConditionTypeModuleCatalog KymaConditionType = "ModuleCatalog"
ConditionTypeSKRWebhook KymaConditionType = "SKRWebhook"

ConditionTypeSKRImagePullSecretSync KymaConditionType = "SKRImagePullSecretSync"

// ConditionReason will be set to `Ready` on all Conditions. If the Condition is actual ready,
// can be determined by the state.
ConditionReason KymaConditionReason = "Ready"

ConditionMessageModuleInReadyState = "all modules are in ready state"
ConditionMessageModuleNotInReadyState = "not all modules are in ready state"
ConditionMessageModuleCatalogIsSynced = "module templates are synchronized"
ConditionMessageModuleCatalogIsOutOfSync = "module templates are out of sync and need to be resynchronized"
ConditionMessageSKRWebhookIsSynced = "skrwebhook is synchronized"
ConditionMessageSKRWebhookIsOutOfSync = "skrwebhook is out of sync and needs to be resynchronized"
ConditionMessageModuleStateUnknown = "modules state is unknown"
ConditionMessageModuleCatalogStateUnknown = "module templates synchronization state is unknown"
ConditionMessageModuleInReadyState = "all modules are in ready state"
ConditionMessageModuleNotInReadyState = "not all modules are in ready state"
ConditionMessageModuleCatalogIsSynced = "module templates are synchronized"
ConditionMessageModuleCatalogIsOutOfSync = "module templates are out of sync and need to be resynchronized"
ConditionMessageModuleStateUnknown = "modules state is unknown"
ConditionMessageModuleCatalogStateUnknown = "module templates synchronization state is unknown"
ConditionMessageSKRWebhookIsSynced = "skrwebhook is synchronized"
ConditionMessageSKRWebhookIsOutOfSync = "skrwebhook is out of sync and needs to be resynchronized"
ConditionMessageSKRImagePullSecretSynced = "skr image pull secret is synchronized"
ConditionMessageSKRImagePullSecretOutOfSync = "skr image pull secret is out of sync and needs to be resynchronized"
)

func GenerateMessage(conditionType KymaConditionType, status apimetav1.ConditionStatus) string {
Expand Down Expand Up @@ -57,19 +61,31 @@ func GenerateMessage(conditionType KymaConditionType, status apimetav1.Condition
}

return ConditionMessageSKRWebhookIsOutOfSync
case ConditionTypeSKRImagePullSecretSync:
switch status {
case apimetav1.ConditionTrue:
return ConditionMessageSKRImagePullSecretSynced
case apimetav1.ConditionUnknown:
case apimetav1.ConditionFalse:
}

return ConditionMessageSKRImagePullSecretOutOfSync
case DeprecatedConditionTypeReady:
}

return "no detailed message available as condition or status is unknown to API"
}

// GetRequiredConditionTypes returns all required ConditionTypes for a KymaCR.
func GetRequiredConditionTypes(watcherEnabled bool) []KymaConditionType {
func GetRequiredConditionTypes(watcherEnabled, skrImagePullSecretSyncEnabled bool) []KymaConditionType {
requiredConditions := []KymaConditionType{ConditionTypeModules}
requiredConditions = append(requiredConditions, ConditionTypeModuleCatalog)

if watcherEnabled {
requiredConditions = append(requiredConditions, ConditionTypeSKRWebhook)
}
if skrImagePullSecretSyncEnabled {
requiredConditions = append(requiredConditions, ConditionTypeSKRImagePullSecretSync)
}
return requiredConditions
}
22 changes: 17 additions & 5 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ import (
"github.com/kyma-project/lifecycle-manager/internal/service/manifest/orphan"
"github.com/kyma-project/lifecycle-manager/internal/service/skrclient"
skrclientcache "github.com/kyma-project/lifecycle-manager/internal/service/skrclient/cache"
"github.com/kyma-project/lifecycle-manager/internal/service/skrsync"
"github.com/kyma-project/lifecycle-manager/internal/setup"
"github.com/kyma-project/lifecycle-manager/pkg/log"
"github.com/kyma-project/lifecycle-manager/pkg/matcher"
Expand Down Expand Up @@ -429,12 +430,25 @@ func setupKymaReconciler(mgr ctrl.Manager, descriptorProvider *provider.CachedDe
moduleStatusGen := generator.NewModuleStatusGenerator(fromerror.GenerateModuleStatusFromError)
modulesStatusHandler := modules.NewStatusHandler(moduleStatusGen, kcpClient, kymaMetrics.RemoveModuleStateMetrics)

kymaReconcilerConfig := kyma.ReconcilerConfig{
RemoteSyncNamespace: flagVar.RemoteSyncNamespace,
IsManagedKyma: flagVar.IsKymaManaged,
OCIRegistryHost: ociRegistryHost,
SkrImagePullSecretName: flagVar.SkrImagePullSecret,
}
syncCrdsUseCase := remote.NewSyncCrdsUseCase(kcpClient, skrContextFactory, nil)
skrSyncService := skrsync.NewService(
skrContextFactory,
secretrepository.NewRepository(kcpClient, shared.DefaultControlPlaneNamespace),
&syncCrdsUseCase,
flagVar.SkrImagePullSecret)

if err := (&kyma.Reconciler{
Client: kcpClient,
SkrContextFactory: skrContextFactory,
Event: event,
DescriptorProvider: descriptorProvider,
SyncRemoteCrds: remote.NewSyncCrdsUseCase(kcpClient, skrContextFactory, nil),
SkrSyncService: skrSyncService,
ModulesStatusHandler: modulesStatusHandler,
SKRWebhookManager: skrWebhookManager,
RequeueIntervals: queue.RequeueIntervals{
Expand All @@ -443,14 +457,12 @@ func setupKymaReconciler(mgr ctrl.Manager, descriptorProvider *provider.CachedDe
Error: flagVar.KymaRequeueErrInterval,
Warning: flagVar.KymaRequeueWarningInterval,
},
RemoteSyncNamespace: flagVar.RemoteSyncNamespace,
IsManagedKyma: flagVar.IsKymaManaged,
Metrics: kymaMetrics,
Metrics: kymaMetrics,
RemoteCatalog: remote.NewRemoteCatalogFromKyma(kcpClient, skrContextFactory,
flagVar.RemoteSyncNamespace),
TemplateLookup: templatelookup.NewTemplateLookup(kcpClient, descriptorProvider,
moduleTemplateInfoLookupStrategies),
OCIRegistryHost: ociRegistryHost,
Config: kymaReconcilerConfig,
}).SetupWithManager(
mgr, options, kyma.SetupOptions{
ListenerAddr: flagVar.KymaListenerAddr,
Expand Down
51 changes: 40 additions & 11 deletions internal/controller/kyma/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8slabels "k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
Expand Down Expand Up @@ -71,22 +72,35 @@ type ModuleStatusHandler interface {
UpdateModuleStatuses(ctx context.Context, kyma *v1beta2.Kyma, modules modulecommon.Modules) error
}

type SkrSyncService interface {
SyncCrds(ctx context.Context, kyma *v1beta2.Kyma) (bool, error)
SyncImagePullSecret(ctx context.Context, kyma types.NamespacedName) error
}

// ReconcilerConfig holds configuration values for the Kyma Reconciler.
// Usually read from flags or environment variables.
type ReconcilerConfig struct {
RemoteSyncNamespace string
IsManagedKyma bool
OCIRegistryHost string
SkrImagePullSecretName string
}

type Reconciler struct {
client.Client
event.Event
queue.RequeueIntervals

Config ReconcilerConfig
SkrContextFactory remote.SkrContextProvider
DescriptorProvider *provider.CachedDescriptorProvider
SyncRemoteCrds remote.SyncCrdsUseCase
SkrSyncService SkrSyncService
ModulesStatusHandler ModuleStatusHandler
SKRWebhookManager SKRWebhookManager
RemoteSyncNamespace string
IsManagedKyma bool
Metrics *metrics.KymaMetrics
RemoteCatalog *remote.RemoteCatalog
TemplateLookup *templatelookup.TemplateLookup
OCIRegistryHost string

Metrics *metrics.KymaMetrics
RemoteCatalog *remote.RemoteCatalog
TemplateLookup *templatelookup.TemplateLookup
}

func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
Expand All @@ -111,7 +125,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
return ctrl.Result{}, fmt.Errorf("KymaController: %w", err)
}

status.InitConditions(kyma, r.WatcherEnabled())
status.InitConditions(kyma, r.WatcherEnabled(), r.SkrImagePullSecretSyncEnabled())

if kyma.SkipReconciliation() {
logger.V(log.DebugLevel).Info("skipping reconciliation for Kyma: " + kyma.Name)
Expand Down Expand Up @@ -189,7 +203,11 @@ func (r *Reconciler) WatcherEnabled() bool {
}

func (r *Reconciler) IsKymaManaged() bool {
return r.IsManagedKyma
return r.Config.IsManagedKyma
}

func (r *Reconciler) SkrImagePullSecretSyncEnabled() bool {
return r.Config.SkrImagePullSecretName != ""
}

func (r *Reconciler) GetModuleTemplateList(ctx context.Context) (*v1beta2.ModuleTemplateList, error) {
Expand Down Expand Up @@ -239,6 +257,7 @@ func (r *Reconciler) handleDeletedSkr(ctx context.Context, kyma *v1beta2.Kyma) (
return ctrl.Result{Requeue: true}, nil
}

//nolint:funlen // disable for kyma controller until split is done into provisioning and deprovisioning controllers
func (r *Reconciler) reconcile(ctx context.Context, kyma *v1beta2.Kyma) (ctrl.Result, error) {
if !kyma.DeletionTimestamp.IsZero() && kyma.Status.State != shared.StateDeleting {
if err := r.deleteRemoteKyma(ctx, kyma); err != nil {
Expand All @@ -263,7 +282,7 @@ func (r *Reconciler) reconcile(ctx context.Context, kyma *v1beta2.Kyma) (ctrl.Re
return ctrl.Result{Requeue: true}, nil
}

updateRequired, err := r.SyncRemoteCrds.Execute(ctx, kyma)
updateRequired, err := r.SkrSyncService.SyncCrds(ctx, kyma)
if err != nil {
r.Metrics.RecordRequeueReason(metrics.CrdsSync, queue.UnexpectedRequeue)
return r.requeueWithError(ctx, kyma, fmt.Errorf("could not sync CRDs: %w", err))
Expand All @@ -276,6 +295,16 @@ func (r *Reconciler) reconcile(ctx context.Context, kyma *v1beta2.Kyma) (ctrl.Re
r.Metrics.RecordRequeueReason(metrics.CrdAnnotationsUpdate, queue.IntendedRequeue)
return ctrl.Result{Requeue: true}, nil
}

if r.SkrImagePullSecretSyncEnabled() {
if err := r.SkrSyncService.SyncImagePullSecret(ctx, kyma.GetNamespacedName()); err != nil {
r.Metrics.RecordRequeueReason(metrics.ImagePullSecretSync, queue.UnexpectedRequeue)
kyma.UpdateCondition(v1beta2.ConditionTypeSKRImagePullSecretSync, apimetav1.ConditionFalse)
return r.requeueWithError(ctx, kyma, fmt.Errorf("could not sync image pull secret: %w", err))
}
kyma.UpdateCondition(v1beta2.ConditionTypeSKRImagePullSecretSync, apimetav1.ConditionTrue)
}

// update the control-plane kyma with the changes to the spec of the remote Kyma
if err = r.replaceSpecFromRemote(ctx, kyma); err != nil {
r.Metrics.RecordRequeueReason(metrics.SpecReplacementFromRemote, queue.UnexpectedRequeue)
Expand Down Expand Up @@ -586,7 +615,7 @@ func (r *Reconciler) updateKyma(ctx context.Context, kyma *v1beta2.Kyma) error {

func (r *Reconciler) reconcileManifests(ctx context.Context, kyma *v1beta2.Kyma) error {
templates := r.TemplateLookup.GetRegularTemplates(ctx, kyma)
prsr := parser.NewParser(r.Client, r.DescriptorProvider, r.RemoteSyncNamespace, r.OCIRegistryHost)
prsr := parser.NewParser(r.Client, r.DescriptorProvider, r.Config.RemoteSyncNamespace, r.Config.OCIRegistryHost)
modules := prsr.GenerateModulesFromTemplates(kyma, templates)

runner := sync.New(r)
Expand Down
4 changes: 4 additions & 0 deletions internal/pkg/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,9 @@ func DefineFlagVar() *FlagVar {
"Allows to configure name of the Secret containing the OCI registry credential")
flag.StringVar(&flagVar.OciRegistryHost, "oci-registry-host", "",
"Allows to configure hostname of the OCI registry.")
flag.StringVar(&flagVar.SkrImagePullSecret, "skr-image-pull-secret", "",
"Allows to reference a secret for the SKR clusters to pull images from private registries.")

return flagVar
}

Expand Down Expand Up @@ -362,6 +365,7 @@ type FlagVar struct {
MinMaintenanceWindowSize time.Duration
OciRegistryCredSecretName string
OciRegistryHost string
SkrImagePullSecret string
}

func (f FlagVar) Validate() error {
Expand Down
3 changes: 2 additions & 1 deletion internal/pkg/metrics/kyma.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,14 @@ type KymaMetrics struct {

type KymaRequeueReason string

//nolint:gosec // requeue reason labels, no confidential content
const (
//nolint:gosec // requeue reason label, no confidential content
KymaUnderDeletionAndAccessSecretNotFound KymaRequeueReason = "kyma_under_deletion_with_no_access_secret"
StatusUpdateToDeleting KymaRequeueReason = "kyma_status_update_to_deleting"
LabelsAndFinalizersUpdate KymaRequeueReason = "labels_and_finalizers_update"
CrdAnnotationsUpdate KymaRequeueReason = "crd_annotations_update"
CrdsSync KymaRequeueReason = "crds_sync"
ImagePullSecretSync KymaRequeueReason = "image_pull_secret_sync"
ReconcileManifests KymaRequeueReason = "manifest_reconciliation"
ModuleCatalogSync KymaRequeueReason = "module_catalog_sync"
SkrWebhookResourcesInstallation KymaRequeueReason = "skr_webhook_installation"
Expand Down
Loading
Loading