diff --git a/.gitignore b/.gitignore index 5ef45fd6..a0f59d66 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ *.swp -bin/ _dist/ .idea golib diff --git a/Dockerfile b/Dockerfile index 1c34967d..4bd0b189 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,4 +2,5 @@ FROM scratch ENTRYPOINT ["/configmapcontroller"] +COPY bin/kubectl /kubectl COPY ./build/configmapcontroller-linux-amd64 /configmapcontroller diff --git a/README.md b/README.md index 9fd0a526..75d37743 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # configmapcontroller -This controller watches for changes to `ConfigMap` objects and performs rolling upgrades on their associated deployments for apps which are not capable of watching the `ConfigMap` and updating dynamically. +This controller watches for changes to `ConfigMap` or `Secret` objects and performs rolling upgrades on their associated deployments,daemonsets or statefulsets for apps which are not capable of watching the `ConfigMap` and updating dynamically. -This is particularly useful if the `ConfigMap` is used to define environment variables - or your app cannot easily and reliably watch the `ConfigMap` and update itself on the fly. +This is particularly useful if the `ConfigMap` or `Secret` is used to define environment variables - or your app cannot easily and reliably watch the `ConfigMap` and update itself on the fly. ## How to use configmapcontroller -For a `Deployment` called `foo` have a `ConfigMap` called `foo`. Then add this annotation to your `Deployment` +For an object(DaemonSet, Deployment, StatefulSet) called `foo` have a `ConfigMap` called `foo`. Then add this annotation to your manifest: ```yaml metadata: @@ -14,10 +14,15 @@ metadata: configmap.fabric8.io/update-on-change: "foo" ``` -Then, providing `configmapcontroller` is running, whenever you edit the `ConfigMap` called `foo` the configmapcontroller will update the `Deployment` by adding the environment variable: +For configmaps use the annotation configmap.frabric8.io/update-on-change. + +For secrets use the annotation secret.fabric8.io/update-on-change. + +Then, providing `configmapcontroller` is running, whenever you edit the `ConfigMap` called `foo` the configmapcontroller will update the `Deployment`, `StatefulSet` or `DaemonSet` by labeling it and hence triggering a rolling update on the object provided that .spec.updateStrategy.type is set to `RollingUpdate`. + +The label would be ``` FABRICB_FOO_REVISION=${configMapRevision} ``` -This then triggers a rolling upgrade of your deployment's pods to use the new configuration. diff --git a/bin/kubectl b/bin/kubectl new file mode 100755 index 00000000..b3f0ffc1 Binary files /dev/null and b/bin/kubectl differ diff --git a/configmapcontroller.go b/configmapcontroller.go index dab0bfb0..607528cf 100644 --- a/configmapcontroller.go +++ b/configmapcontroller.go @@ -10,12 +10,9 @@ import ( "syscall" "time" - "github.com/fabric8io/configmapcontroller/client" "github.com/fabric8io/configmapcontroller/controller" - "github.com/fabric8io/configmapcontroller/util" "github.com/fabric8io/configmapcontroller/version" "github.com/golang/glog" - oclient "github.com/openshift/origin/pkg/client" "github.com/spf13/pflag" "k8s.io/kubernetes/pkg/api" kubectlutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" @@ -55,18 +52,6 @@ func main() { glog.Fatalf("failed to create REST client config: %s", err) } - var oc *oclient.Client - typeOfMaster, err := util.TypeOfMaster(kubeClient) - if err != nil { - glog.Fatalf("failed to create REST client config: %s", err) - } - if typeOfMaster == util.OpenShift { - oc, _, err = client.NewOpenShiftClient(restClientConfig) - if err != nil { - glog.Fatalf("failed to create REST client config: %s", err) - } - } - watchNamespaces := api.NamespaceAll currentNamespace := os.Getenv("KUBERNETES_NAMESPACE") if len(currentNamespace) > 0 { @@ -74,8 +59,7 @@ func main() { } glog.Infof("Watching services in namespaces: `%s`", watchNamespaces) - - c, err := controller.NewController(kubeClient, oc, restClientConfig, factory.JSONEncoder(), *resyncPeriod, watchNamespaces) + c, err := controller.NewController(kubeClient, restClientConfig, factory.JSONEncoder(), *resyncPeriod, watchNamespaces) if err != nil { glog.Fatalf("%s", err) } diff --git a/controller/controller.go b/controller/controller.go index c064cd28..a573f332 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -5,9 +5,7 @@ import ( "strings" "time" - "github.com/fabric8io/configmapcontroller/util" "github.com/golang/glog" - "github.com/pkg/errors" "k8s.io/kubernetes/pkg/api" @@ -19,30 +17,45 @@ import ( "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/watch" - "sort" - - oclient "github.com/openshift/origin/pkg/client" deployapi "github.com/openshift/origin/pkg/deploy/api" deployapiv1 "github.com/openshift/origin/pkg/deploy/api/v1" + + "fmt" + "gopkg.in/v2/yaml" + "io" + "os/exec" ) const ( - updateOnChangeAnnotation = "configmap.fabric8.io/update-on-change" + updateOnChangeAnnotationSuffix = ".fabric8.io/update-on-change" ) +type AnnotationsField map[string]string +type MetadataField struct { + Name string + Annotations AnnotationsField +} +type GenericAnnotatedObject struct { + Kind string + Metadata MetadataField +} +type ObjectList struct { + Items []GenericAnnotatedObject +} type Controller struct { client *client.Client - cmController *framework.Controller - cmLister cache.StoreToServiceLister - recorder record.EventRecorder + cmController *framework.Controller + cmLister cache.StoreToServiceLister + secretController *framework.Controller + secretLister cache.StoreToServiceLister + recorder record.EventRecorder stopCh chan struct{} } func NewController( kubeClient *client.Client, - ocClient *oclient.Client, restClientConfig *restclient.Config, encoder runtime.Encoder, resyncPeriod time.Duration, namespace string) (*Controller, error) { @@ -71,42 +84,45 @@ func NewController( resyncPeriod, framework.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - newCM := obj.(*api.ConfigMap) - typeOfMaster, err := util.TypeOfMaster(kubeClient) - if err != nil { - glog.Fatalf("failed to create REST client config: %s", err) - } - if typeOfMaster == util.OpenShift { - err = rollingUpgradeDeploymentsConfigs(newCM, ocClient) - if err != nil { - glog.Errorf("failed to update DeploymentConfig: %v", err) - } - } - err = rollingUpgradeDeployments(newCM, kubeClient) - if err != nil { - glog.Errorf("failed to update Deployment: %v", err) - } - + cm := obj.(*api.ConfigMap) + go rollingUpgradeObject(cm, "Deployment") + go rollingUpgradeObject(cm, "DaemonSet") + go rollingUpgradeObject(cm, "StatefulSet") }, UpdateFunc: func(oldObj interface{}, newObj interface{}) { oldM := oldObj.(*api.ConfigMap) newCM := newObj.(*api.ConfigMap) if oldM.ResourceVersion != newCM.ResourceVersion { - typeOfMaster, err := util.TypeOfMaster(kubeClient) - if err != nil { - glog.Fatalf("failed to create REST client config: %s", err) - } - if typeOfMaster == util.OpenShift { - err = rollingUpgradeDeploymentsConfigs(newCM, ocClient) - if err != nil { - glog.Errorf("failed to update DeploymentConfig: %v", err) - } - } - err = rollingUpgradeDeployments(newCM, kubeClient) - if err != nil { - glog.Errorf("failed to update Deployment: %v", err) - } + go rollingUpgradeObject(newCM, "Deployment") + go rollingUpgradeObject(newCM, "DaemonSet") + go rollingUpgradeObject(newCM, "StatefulSet") + } + }, + }, + ) + c.secretLister.Store, c.secretController = framework.NewInformer( + &cache.ListWatch{ + ListFunc: secretListFunc(c.client, namespace), + WatchFunc: secretWatchFunc(c.client, namespace), + }, + &api.Secret{}, + resyncPeriod, + framework.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + s := obj.(*api.Secret) + go rollingUpgradeObject(s, "Deployment") + go rollingUpgradeObject(s, "DaemonSet") + go rollingUpgradeObject(s, "StatefulSet") + }, + UpdateFunc: func(oldObj interface{}, newObj interface{}) { + oldSec := oldObj.(*api.Secret) + newSec := newObj.(*api.Secret) + + if oldSec.ResourceVersion != newSec.ResourceVersion { + go rollingUpgradeObject(newSec, "Deployment") + go rollingUpgradeObject(newSec, "DaemonSet") + go rollingUpgradeObject(newSec, "StatefulSet") } }, }, @@ -116,10 +132,10 @@ func NewController( // Run starts the controller. func (c *Controller) Run() { - glog.Infof("starting configmapcontroller") - + glog.Infof("Starting configmapcontroller. Watching configmaps") go c.cmController.Run(c.stopCh) - + glog.Infof("Starting configmapcontroller. Watching secrets") + go c.secretController.Run(c.stopCh) <-c.stopCh } @@ -141,126 +157,82 @@ func configMapWatchFunc(c *client.Client, ns string) func(options api.ListOption } } -func rollingUpgradeDeployments(cm *api.ConfigMap, c *client.Client) error { - ns := cm.Namespace - configMapName := cm.Name - configMapVersion := convertConfigMapToToken(cm) - - deployments, err := c.Deployments(ns).List(api.ListOptions{}) - if err != nil { - return errors.Wrap(err, "failed to list deployments") +func secretListFunc(c *client.Client, ns string) func(api.ListOptions) (runtime.Object, error) { + glog.Infof("Listing secrets") + return func(opts api.ListOptions) (runtime.Object, error) { + return c.Secrets(ns).List(opts) } - for _, d := range deployments.Items { - containers := d.Spec.Template.Spec.Containers - // match deployments with the correct annotation - annotationValue, _ := d.ObjectMeta.Annotations[updateOnChangeAnnotation] - if annotationValue != "" { - values := strings.Split(annotationValue, ",") - matches := false - for _, value := range values { - if value == configMapName { - matches = true - break - } - } - if matches { - updateContainers(containers, annotationValue, configMapVersion) +} - // update the deployment - _, err := c.Deployments(ns).Update(&d) - if err != nil { - return errors.Wrap(err, "update deployment failed") - } - glog.Infof("Updated Deployment %s", d.Name) - } - } +func secretWatchFunc(c *client.Client, ns string) func(options api.ListOptions) (watch.Interface, error) { + glog.Infof("Watching secrets") + return func(options api.ListOptions) (watch.Interface, error) { + return c.Secrets(ns).Watch(options) } - return nil } -func rollingUpgradeDeploymentsConfigs(cm *api.ConfigMap, oc *oclient.Client) error { - ns := cm.Namespace - configMapName := cm.Name - configMapVersion := convertConfigMapToToken(cm) - dcs, err := oc.DeploymentConfigs(ns).List(api.ListOptions{}) +func findObjectsByKind(objectKind string) ObjectList { + kubectlbinary := "/kubectl" + cmd := exec.Command(kubectlbinary, "get", objectKind, "-o", "yaml") + stdout, err := cmd.StdoutPipe() if err != nil { - return errors.Wrap(err, "failed to list deploymentsconfigs") + glog.Errorf("Error retrieving objects by kind. Could not start pipe.") + var emptyObjectList ObjectList + return emptyObjectList } + if err := cmd.Start(); err != nil { + glog.Errorf("Error retrieving objects by kind. Could not query.") + var emptyObjectList ObjectList + return emptyObjectList + } + buf := bytes.NewBuffer(nil) + io.Copy(buf, stdout) + var g ObjectList + yaml.Unmarshal(buf.Bytes(), &g) + return g +} - //glog.Infof("found %v DC items in namespace %s", len(dcs.Items), ns) - for _, d := range dcs.Items { - containers := d.Spec.Template.Spec.Containers - // match deployment configs with the correct annotation - annotationValue, _ := d.ObjectMeta.Annotations[updateOnChangeAnnotation] +func rollingUpgradeObject(objectThatChanged interface{}, objectKind string) { + glog.Infof("Rolling upgrade object %s", objectKind) + nameOfObjectThatChanged, typeOfObjectThatChanged, versionOfObjectThatChanged := getObjectVars(objectThatChanged) + glog.Infof("Object that changed: %s %s %s", nameOfObjectThatChanged, typeOfObjectThatChanged, versionOfObjectThatChanged) + if typeOfObjectThatChanged == "" { + glog.Infof("Type of object that changed is not handled. Type: [%s]", typeOfObjectThatChanged) + return + } + updateOnChangeAnnotation := strings.ToLower(typeOfObjectThatChanged) + updateOnChangeAnnotationSuffix + labelName := "FABRIC8_" + convertToEnvVarName(nameOfObjectThatChanged) + "_" + strings.ToUpper(typeOfObjectThatChanged) + glog.Infof("Label: %s", labelName) + glog.Infof("Annotation: %s", updateOnChangeAnnotation) + objects := findObjectsByKind(objectKind) + for _, o := range objects.Items { + annotationValue := o.Metadata.Annotations[updateOnChangeAnnotation] + glog.Infof("Object: %s, Annotation value: %s", o.Metadata.Name, annotationValue) if annotationValue != "" { values := strings.Split(annotationValue, ",") - matches := false for _, value := range values { - if value == configMapName { - matches = true - break - } - } - if matches { - if updateContainers(containers, annotationValue, configMapVersion) { - // update the deployment - _, err := oc.DeploymentConfigs(ns).Update(&d) - if err != nil { - return errors.Wrap(err, "update deployment failed") - } - glog.Infof("Updated DeploymentConfigs %s", d.Name) + if value == nameOfObjectThatChanged { + go RunKubectlPatch(objectKind, o.Metadata.Name, labelName, versionOfObjectThatChanged) } } } } - return nil } -// lets convert the configmap into a unique token based on the data values -func convertConfigMapToToken(cm *api.ConfigMap) string { - values := []string{} - for k, v := range cm.Data { - values = append(values, k+"="+v) +func getObjectVars(o interface{}) (string, string, string) { + glog.Infof("obtaining object variables") + cm, ok := o.(*api.ConfigMap) + if ok { + glog.Infof("returning variables for configmap") + return cm.Name, "CONFIGMAP", cm.ObjectMeta.ResourceVersion } - sort.Strings(values) - text := strings.Join(values, ";") - // we could zip and base64 encode - // but for now we could leave this easy to read so that its easier to diagnose when & why things changed - return text -} - -func updateContainers(containers []api.Container, annotationValue, configMapVersion string) bool { - // we can have multiple configmaps to update - answer := false - configmaps := strings.Split(annotationValue, ",") - for _, cmNameToUpdate := range configmaps { - configmapEnvar := "FABRIC8_" + convertToEnvVarName(cmNameToUpdate) + "_CONFIGMAP" - - for i := range containers { - envs := containers[i].Env - matched := false - for j := range envs { - if envs[j].Name == configmapEnvar { - matched = true - if envs[j].Value != configMapVersion { - glog.Infof("Updating %s to %s", configmapEnvar, configMapVersion) - envs[j].Value = configMapVersion - answer = true - } - } - } - // if no existing env var exists lets create one - if !matched { - e := api.EnvVar{ - Name: configmapEnvar, - Value: configMapVersion, - } - containers[i].Env = append(containers[i].Env, e) - answer = true - } - } + sec, ok := o.(*api.Secret) + if ok { + glog.Infof("returning variables for secret") + return sec.Name, "SECRET", sec.ObjectMeta.ResourceVersion } - return answer + glog.Errorf("The reported object that changed is not a Secret or a ConfigMap") + return "", "", "" } // convertToEnvVarName converts the given text into a usable env var @@ -283,3 +255,14 @@ func convertToEnvVarName(text string) string { } return buffer.String() } + +func RunKubectlPatch(objectKind string, objectId string, labelName string, labelValue string) { + yamlPatch := fmt.Sprintf("spec:\n template:\n metadata:\n labels:\n %s: '%s'", labelName, labelValue) + glog.Infof("About to run patch: %s %s with %s", objectKind, objectId, yamlPatch) + err := exec.Command("/kubectl", "patch", objectKind, objectId, "--patch", yamlPatch).Run() + if err == nil { + glog.Infof("Successfully sent patch request for %s %s", objectKind, objectId) + return + } + glog.Errorf("Could not execute patch %s on object %s of kind %s", yamlPatch, objectId, objectKind) +}