From 99f210a8af25505f3c4e182f0546149500633112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20=C5=A0alata?= Date: Tue, 7 Jun 2022 12:57:34 +0200 Subject: [PATCH] Add token requestor flow --- Makefile | 4 +- .../shoot-fleet-agent-shoot/Chart.yaml | 4 + .../templates/custom-infrastructure-rbac.yaml | 35 ++++ .../shoot-fleet-agent-shoot/values.yaml | 2 + go.mod | 2 +- pkg/controller/actuator.go | 139 ++++++++++------ .../managed-resource-handler/handler.go | 117 ++++++++++++++ .../token-requestor-handler/handler.go | 150 ++++++++++++++++++ 8 files changed, 403 insertions(+), 50 deletions(-) create mode 100644 charts/internal/shoot-fleet-agent-shoot/Chart.yaml create mode 100644 charts/internal/shoot-fleet-agent-shoot/templates/custom-infrastructure-rbac.yaml create mode 100644 charts/internal/shoot-fleet-agent-shoot/values.yaml create mode 100644 pkg/controller/managed-resource-handler/handler.go create mode 100644 pkg/controller/token-requestor-handler/handler.go diff --git a/Makefile b/Makefile index 9c1c3e8..15a8f1d 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ EXTENSION_PREFIX := gardener-extension NAME := shoot-fleet-agent -REGISTRY := javamachr +REGISTRY := ysoftglobal.azurecr.io IMAGE_PREFIX := $(REGISTRY)/gardener-extension REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) HACK_DIR := $(REPO_ROOT)/hack @@ -100,7 +100,7 @@ revendor: @GO111MODULE=on go mod vendor @GO111MODULE=on go mod tidy @chmod +x $(REPO_ROOT)/vendor/github.com/gardener/gardener/hack/* - @chmod +x $(REPO_ROOT)/vendor/github. com/gardener/gardener/hack/.ci/* + @chmod +x $(REPO_ROOT)/vendor/github.com/gardener/gardener/hack/.ci/* @$(REPO_ROOT)/hack/update-github-templates.sh .PHONY: clean diff --git a/charts/internal/shoot-fleet-agent-shoot/Chart.yaml b/charts/internal/shoot-fleet-agent-shoot/Chart.yaml new file mode 100644 index 0000000..2a26d4b --- /dev/null +++ b/charts/internal/shoot-fleet-agent-shoot/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A Helm chart for shoot-fleet-agent-shoot +name: shoot-fleet-agent-shoot +version: 0.1.0 diff --git a/charts/internal/shoot-fleet-agent-shoot/templates/custom-infrastructure-rbac.yaml b/charts/internal/shoot-fleet-agent-shoot/templates/custom-infrastructure-rbac.yaml new file mode 100644 index 0000000..8b71fa7 --- /dev/null +++ b/charts/internal/shoot-fleet-agent-shoot/templates/custom-infrastructure-rbac.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: extensions.gardener.cloud:extension-shoot-fleet-agent:shoot + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: + - apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' + - nonResourceURLs: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: extensions.gardener.cloud:extension-shoot-fleet-agent:shoot + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: extensions.gardener.cloud:extension-shoot-fleet-agent:shoot +subjects: + - kind: ServiceAccount + name: {{ .Values.shootAccessServiceAccountName }} + namespace: {{ .Values.shootAccessServiceAccountNamespace }} \ No newline at end of file diff --git a/charts/internal/shoot-fleet-agent-shoot/values.yaml b/charts/internal/shoot-fleet-agent-shoot/values.yaml new file mode 100644 index 0000000..e2d66d5 --- /dev/null +++ b/charts/internal/shoot-fleet-agent-shoot/values.yaml @@ -0,0 +1,2 @@ +shootAccessServiceAccountName: "" +shootAccessServiceAccountNamespace: kube-system diff --git a/go.mod b/go.mod index 665bb22..905f816 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( k8s.io/code-generator v0.24.0 k8s.io/component-base v0.24.0 sigs.k8s.io/controller-runtime v0.11.2 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -130,7 +131,6 @@ require ( sigs.k8s.io/controller-tools v0.8.0 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) replace ( diff --git a/pkg/controller/actuator.go b/pkg/controller/actuator.go index 93c5692..5923b89 100644 --- a/pkg/controller/actuator.go +++ b/pkg/controller/actuator.go @@ -17,6 +17,10 @@ package controller import ( "context" "fmt" + managed_resource_handler "github.com/javamachr/gardener-extension-shoot-fleet-agent/pkg/controller/managed-resource-handler" + token_requestor_handler "github.com/javamachr/gardener-extension-shoot-fleet-agent/pkg/controller/token-requestor-handler" + "github.com/javamachr/gardener-extension-shoot-fleet-agent/pkg/utils" + "k8s.io/apimachinery/pkg/api/errors" "reflect" "strings" @@ -89,6 +93,21 @@ func initializeFleetManagers(config config.Config, logger logr.Logger) map[strin return fleetManagers } +func (a *actuator) ensureDependencies(ctx context.Context, cluster *extensions.Cluster) error { + // Initialize token requestor handler + tokenRequestor := token_requestor_handler.NewTokenRequestorHandler(ctx, a.client, cluster) + + // Initialize the managed resoruce handler + managedResourceHandler := managed_resource_handler.NewManagedResourceHandler(ctx, a.client, cluster) + + dependencyFunctions := []func() error{ + tokenRequestor.EnsureKubeconfig, + managedResourceHandler.EnsureManagedResoruces, + } + + return utils.RunParallelFunctions(dependencyFunctions) +} + // Reconcile the Extension resource. func (a *actuator) Reconcile(ctx context.Context, ex *extensionsv1alpha1.Extension) error { namespace := ex.GetNamespace() @@ -106,7 +125,17 @@ func (a *actuator) Reconcile(ctx context.Context, ex *extensionsv1alpha1.Extensi return fmt.Errorf("failed to decode provider config: %+v", err) } } - a.ReconcileClusterInFleetManager(ctx, namespace, cluster, shootsConfigOverride) + + if err := a.ensureDependencies(ctx, cluster); err != nil { + a.logger.Error(err, "Could not ensure dependencies") + return err + } + + if err = a.ReconcileClusterInFleetManager(ctx, namespace, cluster, shootsConfigOverride); err != nil { + a.logger.Error(err, "Could not reconcile cluster in fleet") + return err + } + return a.updateStatus(ctx, ex) } @@ -163,71 +192,87 @@ func (a *actuator) InjectScheme(scheme *runtime.Scheme) error { } // ReconcileClusterInFleetManager reconciles cluster registration in remote fleet manager -func (a *actuator) ReconcileClusterInFleetManager(ctx context.Context, namespace string, cluster *extensions.Cluster, override *config.Config) { +func (a *actuator) ReconcileClusterInFleetManager(ctx context.Context, namespace string, cluster *extensions.Cluster, override *config.Config) error { a.logger.Info("Starting with already registered check") labels := prepareLabels(cluster, getProjectConfig(cluster, &a.serviceConfig), getProjectConfig(cluster, override)) registered, err := a.getFleetManager(cluster).GetCluster(ctx, cluster.Shoot.Name) - if err != nil { + + // We cannot find the cluster because of an unknown error + if err != nil && !errors.IsNotFound(err) { a.logger.Error(err, "Failed to get cluster registration for Shoot", "shoot", cluster.Shoot.Name) + return err } - if err == nil && registered != nil { - if reflect.DeepEqual(registered.Labels, labels) { - a.logger.Info("Cluster already registered - skipping registration", "clientId", registered.Spec.ClientID) - } else { - a.logger.Info("Updating labels of already registered cluster.", "clientId", registered.Spec.ClientID) - a.updateClusterLabelsInFleet(ctx, registered, cluster, labels) - } - return + + // We cannot find the cluster because we haven't registered it yet + if err != nil && errors.IsNotFound(err) { + a.logger.Info("Creating fleet cluster", "shoot", cluster.Shoot.Name) + return a.registerNewClusterInFleet(ctx, namespace, cluster, labels) } - a.registerNewClusterInFleet(ctx, namespace, cluster, labels) + + // The cluster we have in fleet is already in the correct state + if reflect.DeepEqual(registered.Labels, labels) { + a.logger.Info("Cluster already registered - skipping registration", "clientId", registered.Spec.ClientID) + return nil + } + + a.logger.Info("Updating labels of already registered cluster.", "clientId", registered.Spec.ClientID) + return a.updateClusterLabelsInFleet(ctx, registered, cluster, labels) } -func (a *actuator) updateClusterLabelsInFleet(ctx context.Context, clusterRegistration *fleetv1alpha1.Cluster, cluster *extensions.Cluster, labels map[string]string) { +func (a *actuator) updateClusterLabelsInFleet(ctx context.Context, clusterRegistration *fleetv1alpha1.Cluster, cluster *extensions.Cluster, labels map[string]string) error { clusterRegistration.Labels = labels _, err := a.getFleetManager(cluster).UpdateCluster(ctx, clusterRegistration) if err != nil { a.logger.Error(err, "Failed to update cluster labels in Fleet registration.", "clusterName", clusterRegistration.Name) } + return err } -func (a *actuator) registerNewClusterInFleet(ctx context.Context, namespace string, cluster *extensions.Cluster, labels map[string]string) { +func (a *actuator) registerNewClusterInFleet(ctx context.Context, namespace string, cluster *extensions.Cluster, labels map[string]string) error { a.logger.Info("Looking up Secret with KubeConfig for given Shoot.", "namespace", namespace, "secretName", KubeconfigSecretName) secret := &corev1.Secret{} - if err := a.client.Get(ctx, kutil.Key(namespace, KubeconfigSecretName), secret); err == nil { - secretData := make(map[string][]byte) - secretData["value"] = secret.Data[KubeconfigKey] - a.logger.Info("Loaded kubeconfig from secret", "kubeconfig", secret, "namespace", namespace) - - const fleetRegisterNamespace = "clusters" - kubeconfigSecret := corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubecfg-" + buildCrdName(cluster), - Namespace: fleetRegisterNamespace, - }, - Data: secretData, - } - - clusterRegistration := fleetv1alpha1.Cluster{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: buildCrdName(cluster), - Namespace: fleetRegisterNamespace, - Labels: labels, - }, - Spec: fleetv1alpha1.ClusterSpec{ - KubeConfigSecret: "kubecfg-" + buildCrdName(cluster), - }, - } - if _, err = a.getFleetManager(cluster).CreateKubeconfigSecret(ctx, &kubeconfigSecret); err != nil { - a.logger.Error(err, "Failed to create secret with kubeconfig for Fleet registration") - } - if _, err = a.getFleetManager(cluster).CreateCluster(ctx, &clusterRegistration); err != nil { - a.logger.Error(err, "Failed to create Cluster for Fleet registration") - } - a.logger.Info("Registered shoot cluster in Fleet Manager ", "registration", clusterRegistration) - } else { + if err := a.client.Get(ctx, kutil.Key(namespace, KubeconfigSecretName), secret); err != nil { a.logger.Error(err, "Failed to find Secret with kubeconfig for Fleet registration.") + return err } + + secretData := make(map[string][]byte) + secretData["value"] = secret.Data[KubeconfigKey] + a.logger.Info("Loaded kubeconfig from secret", "kubeconfig", secret, "namespace", namespace) + + const fleetRegisterNamespace = "clusters" + kubeconfigSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubecfg-" + buildCrdName(cluster), + Namespace: fleetRegisterNamespace, + }, + Data: secretData, + } + + clusterRegistration := fleetv1alpha1.Cluster{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: buildCrdName(cluster), + Namespace: fleetRegisterNamespace, + Labels: labels, + }, + Spec: fleetv1alpha1.ClusterSpec{ + KubeConfigSecret: "kubecfg-" + buildCrdName(cluster), + }, + } + + if _, err := a.getFleetManager(cluster).CreateKubeconfigSecret(ctx, &kubeconfigSecret); err != nil { + a.logger.Error(err, "Failed to create secret with kubeconfig for Fleet registration") + return err + } + + if _, err := a.getFleetManager(cluster).CreateCluster(ctx, &clusterRegistration); err != nil { + a.logger.Error(err, "Failed to create Cluster for Fleet registration") + return err + } + + a.logger.Info("Registered shoot cluster in Fleet Manager ", "registration", clusterRegistration) + return nil } func prepareLabels(cluster *extensions.Cluster, serviceConfig projConfig.ProjectConfig, override projConfig.ProjectConfig) map[string]string { diff --git a/pkg/controller/managed-resource-handler/handler.go b/pkg/controller/managed-resource-handler/handler.go new file mode 100644 index 0000000..800c4c4 --- /dev/null +++ b/pkg/controller/managed-resource-handler/handler.go @@ -0,0 +1,117 @@ +package managed_resource_handler + +import ( + "context" + "fmt" + "github.com/gardener/gardener/extensions/pkg/util" + "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/extensions" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/gardener/gardener/pkg/utils/kubernetes/health" + "github.com/gardener/gardener/pkg/utils/managedresources" + "github.com/gardener/gardener/pkg/utils/retry" + "github.com/go-logr/logr" + token_requestor_handler "github.com/javamachr/gardener-extension-shoot-fleet-agent/pkg/controller/token-requestor-handler" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "path/filepath" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "time" +) + +var ( + ChartsPath = filepath.Join("charts", "internal") + ManagedResourcesChartValues = map[string]interface{}{ + "shootAccessServiceAccountName": token_requestor_handler.TargetServiceAccountName, + "shootAccessServiceAccountNamespace": token_requestor_handler.TargetServiceAccountNamespace, + } +) + +const ( + ManagedResourceChartName = "shoot-fleet-agent-shoot" + ManagedResourceName = "extension-shoot-fleet-agent-shoot" +) + +type ManagedResourceHandler struct { + ctx context.Context + client client.Client + cluster *extensions.Cluster + logger logr.Logger +} + +func NewManagedResourceHandler(ctx context.Context, client client.Client, cluster *extensions.Cluster) *ManagedResourceHandler { + return &ManagedResourceHandler{ + ctx: ctx, + client: client, + cluster: cluster, + logger: log.Log.WithValues("logger", "managed-resource-handler", "cluster", cluster.ObjectMeta.Name), + } +} + +// renderChart takes the helm chart at charts/internal/ and renders it with the values +func (h *ManagedResourceHandler) renderChart() ([]byte, error) { + chartPath := filepath.Join(ChartsPath, ManagedResourceChartName) + renderer, err := util.NewChartRendererForShoot(h.cluster.Shoot.Spec.Kubernetes.Version) + if err != nil { + return nil, err + } + chart, err := renderer.Render(chartPath, ManagedResourceChartName, h.cluster.ObjectMeta.Name, ManagedResourcesChartValues) + if err != nil { + return nil, err + } + + return chart.Manifest(), nil +} + +func (h *ManagedResourceHandler) EnsureManagedResoruces() error { + h.logger.Info("Rendering chart with the managed resources") + renderedChart, err := h.renderChart() + if err != nil { + return err + } + data := map[string][]byte{ManagedResourceChartName: renderedChart} + keepObjects := false + forceOverwriteAnnotations := false + + h.logger.Info("Creating the ManagedResource") + if err := managedresources.Create(h.ctx, h.client, h.cluster.ObjectMeta.Name, ManagedResourceName, false, "", data, &keepObjects, map[string]string{}, &forceOverwriteAnnotations); err != nil { + h.logger.Error(err, "ManagedResource could not be created") + return err + } + + h.logger.Info("Waiting until the ManagedResource is healthy") + if err := h.waitUntilManagedResourceHealthy(); err != nil { + h.logger.Error(err, "ManagedResource has been created, but hasn't been applied") + return err + } + + h.logger.Info("ManagedResource created") + return nil +} + +func (h *ManagedResourceHandler) waitUntilManagedResourceHealthy() error { + name := ManagedResourceName + namespace := h.cluster.ObjectMeta.Name + + obj := &v1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + return retry.UntilTimeout(h.ctx, 5*time.Second, 60*time.Second, func(ctx context.Context) (done bool, err error) { + if err := h.client.Get(ctx, kutil.Key(namespace, name), obj); err != nil { + h.logger.Info(fmt.Sprintf("Could not wait for the managed resource to be ready: %+v", err)) + return retry.MinorError(err) + } + + // Check whether ManagedResource has proper status + if err := health.CheckManagedResource(obj); err != nil { + h.logger.Info(fmt.Sprintf("Managed resource %s not ready (yet): %+v", name, err)) + return retry.MinorError(err) + } + + return retry.Ok() + }) +} diff --git a/pkg/controller/token-requestor-handler/handler.go b/pkg/controller/token-requestor-handler/handler.go new file mode 100644 index 0000000..92e4b14 --- /dev/null +++ b/pkg/controller/token-requestor-handler/handler.go @@ -0,0 +1,150 @@ +package token_requestor_handler + +import ( + "context" + "fmt" + extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" + constsv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/extensions" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/gardener/gardener/pkg/utils/retry" + "github.com/go-logr/logr" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" + "time" +) + +const ( + TokenRequestorSecretName = "shoot-access-extension-shoot-fleet-agent" + TokenRequestorSecretKey = constsv1alpha1.DataKeyKubeconfig + TargetServiceAccountName = "extension-shoot-fleet-agent" + TargetServiceAccountNamespace = "kube-system" +) + +type TokenRequestorHandler struct { + ctx context.Context + client client.Client + cluster *extensions.Cluster + logger logr.Logger +} + +func NewTokenRequestorHandler(ctx context.Context, client client.Client, cluster *extensions.Cluster) *TokenRequestorHandler { + return &TokenRequestorHandler{ + ctx: ctx, + client: client, + cluster: cluster, + logger: log.Log.WithValues("logger", "token-requestor-handler", "cluster", cluster.ObjectMeta.Name), + } +} + +func (t *TokenRequestorHandler) getGenericKubeconfigName() string { + return extensionscontroller.GenericTokenKubeconfigSecretNameFromCluster(t.cluster) +} + +func (t *TokenRequestorHandler) getGenericKubeconfig() (*clientapiv1.Config, error) { + kubeconfigSecret := &v1.Secret{} + kubeconfigSecretName := t.getGenericKubeconfigName() + if err := t.client.Get(t.ctx, kutil.Key(t.cluster.ObjectMeta.Name, kubeconfigSecretName), kubeconfigSecret); err != nil { + return nil, err + } + + kubeconfigData, ok := kubeconfigSecret.Data[TokenRequestorSecretKey] + if !ok { + return nil, fmt.Errorf("secret %s doesn't have data key %s", kubeconfigSecretName, TokenRequestorSecretKey) + } + + kubeconfig := &clientapiv1.Config{} + if err := yaml.Unmarshal(kubeconfigData, kubeconfig); err != nil { + return nil, err + } + return kubeconfig, nil +} + +func (t *TokenRequestorHandler) isTokenInserted() bool { + kubeconfigSecret := &v1.Secret{} + if err := t.client.Get(t.ctx, kutil.Key(t.cluster.ObjectMeta.Name, TokenRequestorSecretName), kubeconfigSecret); err != nil { + t.logger.Info(fmt.Sprintf("Couldn't fet the kubeconfig secret %s, %+v", TokenRequestorSecretName, err)) + return false + } + + kubeconfigData, ok := kubeconfigSecret.Data[TokenRequestorSecretKey] + if !ok { + t.logger.Info(fmt.Sprintf("Kubeconfig secret %s doesn't contain data item %s", TokenRequestorSecretName, TokenRequestorSecretKey)) + return false + } + + kubeconfig := &clientapiv1.Config{} + if err := yaml.Unmarshal(kubeconfigData, kubeconfig); err != nil { + t.logger.Info(fmt.Sprintf("Data item %s in kubeconfig secret %s couldn't be parsed: %+v", TokenRequestorSecretKey, TokenRequestorSecretName, err)) + return false + } + + if len(kubeconfig.AuthInfos) == 0 || kubeconfig.AuthInfos[0].AuthInfo.Token == "" { + t.logger.Info(fmt.Sprintf("Kubeconfig in secret %s doesn't contain token (yet?)", TokenRequestorSecretName)) + return false + } + + t.logger.Info(fmt.Sprintf("Kubeconfig in secret %s contains a proper token", TokenRequestorSecretName)) + return true +} + +func (t *TokenRequestorHandler) createKubeconfigSecretFromGeneric() error { + kubeconfig, err := t.getGenericKubeconfig() + if err != nil { + return err + } + + kubeconfigYaml, err := yaml.Marshal(kubeconfig) + if err != nil { + return err + } + + existingSecret := &v1.Secret{} + err = t.client.Get(t.ctx, kutil.Key(t.cluster.ObjectMeta.Name, TokenRequestorSecretName), existingSecret) + if err == nil || !apierrors.IsNotFound(err) { + return err + } + + secretObject := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: t.cluster.ObjectMeta.Name, + Name: TokenRequestorSecretName, + Labels: map[string]string{ + constsv1alpha1.ResourceManagerPurpose: constsv1alpha1.LabelPurposeTokenRequest, + }, + Annotations: map[string]string{ + constsv1alpha1.ServiceAccountName: TargetServiceAccountName, + constsv1alpha1.ServiceAccountNamespace: TargetServiceAccountNamespace, + }, + }, + StringData: map[string]string{ + TokenRequestorSecretKey: string(kubeconfigYaml), + }, + } + return t.client.Create(t.ctx, secretObject) +} + +// EnsureKubeconfig creates the secret with a token-requestor label and waits until gardener fills in +// the token into the kubeconfig +func (t *TokenRequestorHandler) EnsureKubeconfig() error { + t.logger.Info(fmt.Sprintf("Trying to create the token requestor secret")) + if err := t.createKubeconfigSecretFromGeneric(); err != nil { + t.logger.Error(err, "Generic kubeconfig could not be copied") + return err + } + + t.logger.Info("Waiting until the token is propagated into the token requestor secret") + if err := retry.UntilTimeout(t.ctx, 5*time.Second, 60*time.Second, func(ctx context.Context) (bool, error) { + return t.isTokenInserted(), nil + }); err != nil { + t.logger.Error(err, "Kubeconfig could not be created") + return err + } + + return nil +}