gardener-extension-shoot-fleet-agent/pkg/controller/actuator.go

// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller

import (
	"bytes"
	"context"
	"fmt"
	managed_resource_handler "github.com/ysoftdevs/gardener-extension-shoot-fleet-agent/pkg/controller/managed-resource-handler"
	token_requestor_handler "github.com/ysoftdevs/gardener-extension-shoot-fleet-agent/pkg/controller/token-requestor-handler"
	"github.com/ysoftdevs/gardener-extension-shoot-fleet-agent/pkg/utils"
	"k8s.io/apimachinery/pkg/api/errors"
	"reflect"
	"strings"

	projConfig "github.com/ysoftdevs/gardener-extension-shoot-fleet-agent/pkg/apis/config"

	"github.com/gardener/gardener/pkg/apis/core/v1beta1"
	"github.com/gardener/gardener/pkg/extensions"
	"github.com/go-logr/logr"
	fleetv1alpha1 "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/rest"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/log"

	gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper"

	"github.com/gardener/gardener/extensions/pkg/controller"
	"github.com/gardener/gardener/extensions/pkg/controller/extension"
	"github.com/ysoftdevs/gardener-extension-shoot-fleet-agent/pkg/controller/config"

	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
)

// ActuatorName is the name of the Fleet agent actuator.
const ActuatorName = "shoot-fleet-agent-actuator"

// KubeconfigSecretName name of secret that holds kubeconfig for Shoot
const KubeconfigSecretName = token_requestor_handler.TokenRequestorSecretName

// KubeconfigKey key in KubeconfigSecretName secret that holds kubeconfig for Shoot
const KubeconfigKey = token_requestor_handler.TokenRequestorSecretKey

// DefaultConfigKey is the name of default config key.
const DefaultConfigKey = "default"

// FleetClusterSecretDataKey is the key of the data item that holds the kubeconfig for the cluster in fleet
const FleetClusterSecretDataKey = "value"

// NewActuator returns an actuator responsible for Extension resources.
func NewActuator(config config.Config) extension.Actuator {
	logger := log.Log.WithName(ActuatorName)
	fleetManagers := initializeFleetManagers(config, logger)

	return &actuator{
		logger:        logger,
		serviceConfig: config,
		fleetManagers: fleetManagers,
	}
}

type actuator struct {
	client        client.Client
	config        *rest.Config
	decoder       runtime.Decoder
	fleetManagers map[string]*FleetManager

	serviceConfig config.Config

	logger logr.Logger
}

// initializeFleetManagers initializes fleet managers for given config
func initializeFleetManagers(config config.Config, logger logr.Logger) map[string]*FleetManager {
	fleetManagers := make(map[string]*FleetManager)
	fleetManagers[DefaultConfigKey] = createFleetManager(config.DefaultConfiguration, logger)
	for name, projConfig := range config.ProjectConfiguration {
		fleetManagers[name] = createFleetManager(projConfig, logger)
	}
	return fleetManagers
}

func (a *actuator) ensureDependencies(ctx context.Context, cluster *extensions.Cluster) error {
	// Initialize token requestor handler
	tokenRequestor := token_requestor_handler.NewTokenRequestorHandler(ctx, a.client, cluster)

	// Initialize the managed resoruce handler
	managedResourceHandler := managed_resource_handler.NewManagedResourceHandler(ctx, a.client, cluster)

	dependencyFunctions := []func() error{
		tokenRequestor.EnsureKubeconfig,
		managedResourceHandler.EnsureManagedResoruces,
	}

	return utils.RunParallelFunctions(dependencyFunctions)
}

// isShootedSeed looks into the Cluster object to determine whether we are in a Shooted Seed cluster
func isShootHibernated(cluster *extensions.Cluster) bool {
	return cluster.Shoot.Status.IsHibernated
}

// Reconcile the Extension resource.
func (a *actuator) Reconcile(ctx context.Context, ex *extensionsv1alpha1.Extension) error {
	namespace := ex.GetNamespace()
	a.logger.Info("Component is being reconciled", "component", "fleet-agent-management", "namespace", namespace)
	cluster, err := controller.GetCluster(ctx, a.client, namespace)
	if err != nil {
		return err
	}
	if isShootedSeedCluster(cluster) {
		return a.updateStatus(ctx, ex)
	}
	if isShootHibernated(cluster) {
		return a.updateStatus(ctx, ex)
	}

	shootsConfigOverride := &config.Config{}
	if ex.Spec.ProviderConfig != nil { //parse providerConfig defaults override for this Shoot
		if _, _, err := a.decoder.Decode(ex.Spec.ProviderConfig.Raw, nil, shootsConfigOverride); err != nil {
			return fmt.Errorf("failed to decode provider config: %+v", err)
		}
	}

	if err := a.ensureDependencies(ctx, cluster); err != nil {
		a.logger.Error(err, "Could not ensure dependencies")
		return err
	}

	if err = a.ReconcileClusterInFleetManager(ctx, namespace, cluster, shootsConfigOverride); err != nil {
		a.logger.Error(err, "Could not reconcile cluster in fleet")
		return err
	}

	return a.updateStatus(ctx, ex)
}

// Delete the Extension resource.
func (a *actuator) Delete(ctx context.Context, ex *extensionsv1alpha1.Extension) error {
	namespace := ex.GetNamespace()
	cluster, err := controller.GetCluster(ctx, a.client, namespace)
	if err != nil {
		return err
	}
	if isShootedSeedCluster(cluster) {
		return nil
	}
	a.logger.Info("Component is being deleted", "component", "fleet-agent-management", "namespace", namespace, "cluster", buildCrdName(cluster))
	err = a.getFleetManager(cluster).DeleteKubeconfigSecret(ctx, buildCrdName(cluster))
	if err != nil {
		a.logger.Error(err, "Failed to delete kubeconfig secret for Shoot cluster.", "cluster", buildCrdName(cluster))
	}
	err = a.getFleetManager(cluster).DeleteCluster(ctx, buildCrdName(cluster))
	if err != nil {
		a.logger.Error(err, "Failed to delete Cluster registration for Shoot cluster.", "cluster", buildCrdName(cluster))
	}
	return nil
}

// Restore the Extension resource.
func (a *actuator) Restore(ctx context.Context, ex *extensionsv1alpha1.Extension) error {
	//NOOP as there are no resources by this controller in Seed
	return nil
}

// Migrate the Extension resource.
func (a *actuator) Migrate(ctx context.Context, ex *extensionsv1alpha1.Extension) error {
	//NOOP as there are no resources by this controller in Seed
	return nil
}

// InjectConfig injects the rest config to this actuator.
func (a *actuator) InjectConfig(config *rest.Config) error {
	a.config = config
	return nil
}

// InjectClient injects the controller runtime client into the reconciler.
func (a *actuator) InjectClient(client client.Client) error {
	a.client = client
	return nil
}

// InjectScheme injects the given scheme into the reconciler.
func (a *actuator) InjectScheme(scheme *runtime.Scheme) error {
	a.decoder = serializer.NewCodecFactory(scheme).UniversalDecoder()
	return nil
}

// ReconcileClusterInFleetManager reconciles cluster registration in remote fleet manager
func (a *actuator) ReconcileClusterInFleetManager(ctx context.Context, namespace string, cluster *extensions.Cluster, override *config.Config) error {
	labels := prepareLabels(cluster, getProjectConfig(cluster, &a.serviceConfig), getProjectConfig(cluster, override))

	a.logger.Info("Looking up Secret with KubeConfig for given Shoot.", "namespace", namespace, "secretName", KubeconfigSecretName)
	kubeconfigSecret := &corev1.Secret{}
	if err := a.client.Get(ctx, kutil.Key(namespace, KubeconfigSecretName), kubeconfigSecret); err != nil {
		a.logger.Error(err, "Failed to find Secret with kubeconfig for Fleet registration.")
		return err
	}

	a.logger.Info("Checking if the fleet cluster already exists")
	// Check whether we already have an existing cluster
	_, err := a.getFleetManager(cluster).GetCluster(ctx, buildCrdName(cluster))

	// We cannot find the cluster because of an unknown error
	if err != nil && !errors.IsNotFound(err) {
		a.logger.Error(err, "Failed to get cluster registration for Shoot", "shoot", cluster.Shoot.Name)
		return err
	}

	// We cannot find the cluster because we haven't registered it yet
	if errors.IsNotFound(err) {
		a.logger.Info("Creating fleet cluster", "shoot", cluster.Shoot.Name)
		return a.registerNewClusterInFleet(ctx, namespace, cluster, labels, kubeconfigSecret.Data[KubeconfigKey])
	}

	a.logger.Info("Updating existing fleet cluster")
	return a.updateClusterInFleet(ctx, cluster, labels, kubeconfigSecret.Data[KubeconfigKey])
}

func (a *actuator) updateClusterInFleet(ctx context.Context, cluster *extensions.Cluster, labels map[string]string, kubeconfig []byte) error {
	updated := false
	clusterRegistration, err := a.getFleetManager(cluster).GetCluster(ctx, buildCrdName(cluster))
	if err != nil {
		a.logger.Error(err, "Could not fetch fleet cluster")
		return err
	}

	fleetKubeconfigSecret, err := a.getFleetManager(cluster).GetKubeconfigSecret(ctx, buildKubecfgName(cluster))
	if err != nil {
		a.logger.Error(err, "Could not fetch fleet kubeconfig secret")
		return err
	}

	if !reflect.DeepEqual(clusterRegistration.Labels, labels) {
		a.logger.Info("Cluster labels changed, updating")
		clusterRegistration.Labels = labels
		_, err := a.getFleetManager(cluster).UpdateCluster(ctx, clusterRegistration)
		if err != nil {
			a.logger.Error(err, "Failed to update cluster labels in Fleet registration.", "clusterName", clusterRegistration.Name)
			return err
		}
		updated = true
	}

	if bytes.Compare(fleetKubeconfigSecret.Data[FleetClusterSecretDataKey], kubeconfig) != 0 {
		a.logger.Info("Shoot kubeconfig changed, updating")
		fleetKubeconfigSecret.Data[FleetClusterSecretDataKey] = kubeconfig
		_, err := a.getFleetManager(cluster).UpdateKubeconfigSecret(ctx, fleetKubeconfigSecret)
		if err != nil {
			a.logger.Error(err, "Failed to update kuebconfig secret in Fleet registration.", "clusterName", clusterRegistration.Name)
			return err
		}
		updated = true
	}

	if updated {
		a.logger.Info("Cluster successfully updated.")
	} else {
		a.logger.Info("Cluster is already up to date.")
	}
	return nil
}

func (a *actuator) registerNewClusterInFleet(ctx context.Context, namespace string, cluster *extensions.Cluster, labels map[string]string, kubeconfig []byte) error {
	secretData := make(map[string][]byte)
	secretData[FleetClusterSecretDataKey] = kubeconfig

	const fleetRegisterNamespace = "clusters"
	kubeconfigSecret := corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      buildKubecfgName(cluster),
			Namespace: fleetRegisterNamespace,
		},
		Data: secretData,
	}

	clusterRegistration := fleetv1alpha1.Cluster{
		TypeMeta: metav1.TypeMeta{},
		ObjectMeta: metav1.ObjectMeta{
			Name:      buildCrdName(cluster),
			Namespace: fleetRegisterNamespace,
			Labels:    labels,
		},
		Spec: fleetv1alpha1.ClusterSpec{
			KubeConfigSecret: "kubecfg-" + buildCrdName(cluster),
		},
	}

	if _, err := a.getFleetManager(cluster).CreateKubeconfigSecret(ctx, &kubeconfigSecret); err != nil {
		a.logger.Error(err, "Failed to create secret with kubeconfig for Fleet registration")
		return err
	}

	if _, err := a.getFleetManager(cluster).CreateCluster(ctx, &clusterRegistration); err != nil {
		a.logger.Error(err, "Failed to create Cluster for Fleet registration")
		return err
	}

	a.logger.Info("Registered shoot cluster in Fleet Manager ", "registration", clusterRegistration)
	return nil
}

func prepareLabels(cluster *extensions.Cluster, serviceConfig projConfig.ProjectConfig, override projConfig.ProjectConfig) map[string]string {
	labels := make(map[string]string)
	labels["corebundle"] = "true"
	labels["region"] = cluster.Shoot.Spec.Region
	labels["cluster"] = cluster.Shoot.Name
	labels["seed"] = cluster.Seed.Name
	if len(override.Labels) > 0 { //adds labels from Shoot configuration
		for key, value := range override.Labels {
			labels[key] = value
		}
	} else {
		if len(serviceConfig.Labels) > 0 { //adds labels from default configuration
			for key, value := range serviceConfig.Labels {
				labels[key] = value
			}
		}
	}
	return labels
}

func (a *actuator) updateStatus(ctx context.Context, ex *extensionsv1alpha1.Extension) error {
	statusUpdater := controller.NewStatusUpdater(a.logger)
	statusUpdater.InjectClient(a.client)
	operationType := gardencorev1beta1helper.ComputeOperationType(ex.ObjectMeta, ex.Status.LastOperation)
	err := statusUpdater.Success(ctx, ex, operationType, "Successfully reconciled Extension")
	return err
}

func (a *actuator) getFleetManager(cluster *extensions.Cluster) *FleetManager {
	manager, present := a.fleetManagers[getProjectName(cluster)]
	if !present {
		return a.fleetManagers[DefaultConfigKey]
	}
	return manager
}

// getProjectConfig return project specific or default config
func getProjectConfig(cluster *extensions.Cluster, serviceConfig *config.Config) projConfig.ProjectConfig {
	name := getProjectName(cluster)
	projectConfig, present := serviceConfig.ProjectConfiguration[name]
	if !present {
		return serviceConfig.DefaultConfiguration
	}
	return projectConfig
}

// buildCrdName creates a unique name for cluster registration resources in Fleet manager cluster
func buildKubecfgName(cluster *extensions.Cluster) string {
	return fmt.Sprintf("kubecfg-%s", buildCrdName(cluster))
}

// buildCrdName creates a unique name for cluster registration resources in Fleet manager cluster
func buildCrdName(cluster *extensions.Cluster) string {
	return fmt.Sprintf("%s%s", cluster.Seed.Name, cluster.Shoot.Name)
}

// isShootedSeedCluster checks if clusters purpose is Infrastructure
func isShootedSeedCluster(cluster *extensions.Cluster) bool {
	return *cluster.Shoot.Spec.Purpose == v1beta1.ShootPurposeInfrastructure
}

// getProjectName extracts project name from Shoots namespace
func getProjectName(cluster *extensions.Cluster) string {
	return cluster.Shoot.Namespace[strings.LastIndex(cluster.Shoot.Namespace, "-")+1:]
}