diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3e28b2..a3ddcb0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,8 @@ repos: # -- Pre commit hook is conflicting with the helm-docs exclude: | (?x)( - ^charts/db-instances/README.md + ^charts/db-instances/README.md| + ^charts/db-operator/README.md ) - id: trailing-whitespace - id: detect-private-key diff --git a/README.md b/README.md index 690280a..5a9bff4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ ## Charts for deploying DB Operator +## Migration to the db-operator v3 + +Version `3.0.0` contains breaking changes for ones using the google cloudsql instances. It doesn't mean that the operator will not work with them anymore, but users will have to use **generic** DbInsances and prepare the infrasturture using other tools. + +Other changes that might require migration: + +- Pod names were changed, now instead of `db-operator-` and `db-operator-webhook`, we have `db-opetator-controller` and `db-operator-webhook` +- RBAC separation: the controller and webhook now use different ClusterRoles. The configuration has been moved to .controller.rbac and .webhook.rbac accordingly. +- DB Operator configuration is now set under `controller.config` + ## Get started with the helm chart Docs: diff --git a/charts/db-operator/Chart.yaml b/charts/db-operator/Chart.yaml index 12a5c9b..1b6bc0b 100644 --- a/charts/db-operator/Chart.yaml +++ b/charts/db-operator/Chart.yaml @@ -1,26 +1,21 @@ +--- apiVersion: v2 type: application name: db-operator -version: 2.6.0 -# --------------------------------------------------------------------------------- -# -- All supported k8s versions are in the test: -# -- https://github.com/db-operator/charts/blob/main/.github/workflows/test.yaml -# --------------------------------------------------------------------------------- -kubeVersion: ">= 1.30-prerelease" -appVersion: "2.22.0" -description: The DB Operator creates databases and make them available in the cluster via Custom Resource. -home: https://github.com/db-operator/db-operator +version: 3.0.0 +description: This operator lets you manage databases in a Kubernetes native way, even if they are not deployed to Kubernetes +# All supported k8s versions are in the test: +# https://github.com/db-operator/charts/blob/main/.github/workflows/integration-test.yaml +kubeVersion: ">= 1.32-prerelease" +appVersion: "1b329bffb849b42b0a3899ff585e44df4d30b999" +home: https://github.com/db-operator maintainers: - name: Nikolai Rodionov email: iam@allanger.xyz url: https://badhouseplants.net sources: - https://github.com/db-operator/db-operator - - https://github.com/db-operator/cloudish-sql - - https://github.com/db-operator/pgdump-gcs - - https://github.com/db-operator/mydump-gcs keywords: - database - - Operator - mysql - postgres diff --git a/charts/db-operator/README.md b/charts/db-operator/README.md index 14f03ec..82457d9 100644 --- a/charts/db-operator/README.md +++ b/charts/db-operator/README.md @@ -1,15 +1,10 @@ # db-operator -![Version: 2.6.0](https://img.shields.io/badge/Version-2.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.22.0](https://img.shields.io/badge/AppVersion-2.22.0-informational?style=flat-square) +![Version: 3.0.0](https://img.shields.io/badge/Version-3.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.22.0](https://img.shields.io/badge/AppVersion-2.22.0-informational?style=flat-square) -The DB Operator creates databases and make them available in the cluster via Custom Resource. +This operator lets you manage databases in a Kubernetes native way, even if they are not deployed to Kubernetes -## Source Code - -* -* -* -* +**Homepage:** ## Maintainers @@ -17,96 +12,44 @@ The DB Operator creates databases and make them available in the cluster via Cus | ---- | ------ | --- | | Nikolai Rodionov | | | -## Requirements - -Kubernetes: `>= 1.30-prerelease` - -## Installing the Chart - -```console -$ helm repo add db-operator https://db-operator.github.io/charts -$ helm repo update -$ helm install db-operator/db-operator -``` - -If you want to manage `CRDs` yourself, you need to set `.Values.crds.install` to `false`. In that case they won't be a part of templated manifests. - -We don't support this way of `CRD` management officially though, so we can't promise to answer your questions if you choose to go this way. If you want to know more about it, keep reading, we'll talk about it later. - -## Upgarding the Chart - -If there are breaking changes, they should be described in this README's latest item and in helm-chart notes. So if you run `helm upgrade db-operator db-operator/db-operator --dry-run`, you will see if there are breaking changes there. - -## Uninstalling the Chart - -To uninstall the operator only, run - -```console -$ helm uninstall db-operator -``` - -If you have CRDs installed by the chart, you will also have to run something like: - -```console -# !!! This command will remove all kinda.rocks CRDs from the cluster, -# so please make sure that you really want to run it -$ for crd in $(kubectl get crds | awk '{print $1}' | grep 'kinda.rocks'); do kubectl delete crd "${crd}"; done -``` - -## Why are we packaging CRDs in templates? - -There are several reasons for that. The main one is that we are using webhooks and they require certificates to be created. If you have a look at the `CRDs` templates, you'll see something like that: - -```mustache - annotations: - controller-gen.kubebuilder.io/version: v0.12.1 - {{- if .Values.webhook.certificate.create }} - cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Values.webhook.certificate.name}} - {{ else }} - cert-manager.io/inject-ca-from-secret: {{ .Release.Namespace }}/{{ .Values.webhook.certificate.secretName}} - {{- end }} - {{- if .Values.crds.keep }} - helm.sh/resource-policy: keep - {{- end }} - {{- with .Values.crds.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -``` - -So, as you can see they require some information about the release, so they can't be static. Since the webhook is an important part of what we're distributing, we don't want to have it disabled by default. - -Also, we do not have `CRs` in the package, so there is no chicken-egg problem here. You don't need to have `CRDs` installed to install the chart. Since `CRDs` are a part of the operator and you can't have one without another, having them packaged and distributed together makes more sense to us. - -But it's still possible for users to manage `CRDs` on their own. You need to set `.Values.crds.install` to false and go to the [db-operator rgit repository](https://github.com/db-operator/db-operator). There, in the `config/crd/bases` directory, you'll find `CRD` manifests. But if you decided to go this way and you have problems with `CRDs`, you're most probably on your own, you still can open issues though. +## Source Code -## Why do we need cert-manager? +* -At the moment, `db-operator` chart requires `cert-manager` to be present in the cluster. It's because webhook related resources are using it to create certificates. It should change one day, since we would like this chart to be self-sufficient, but currently that's what we've got. +## Requirements -## What is `.Values.tests`? +Kubernetes: `>= 1.32-prerelease` -We use helm tests feature for testing the chart in CI. Though we know that this feature should be used by users to test their intsallations, we do not think that it can be very helpful in this particular case. Test resources are creating a powerful `ClusterRoles`, so **please make sure you don't have them enabled**, unless you really know what you're doing and why. +## Values -## Chart Values | Key | Type | Default | Description | |-----|------|---------|-------------| | nameOverride | string | `""` | | -| image.repository | string | `"ghcr.io/db-operator/db-operator"` | | +| clusterDomain | string | `"cluster.local"` | | +| image.registry | string | `"ghcr.io"` | | +| image.repository | string | `"db-operator/db-operator"` | | +| image.tag | string | `""` | | | image.pullPolicy | string | `"Always"` | | -| controller.logLevel | string | `"info"` | | -| controller.extraArgs | list | `[]` | | -| reconcileInterval | string | `"60"` | | -| watchNamespace | string | `""` | | -| checkForChanges | bool | `false` | ---------------------------------------------------------- | -| rbac.create | bool | `true` | | -| serviceAccount.create | bool | `true` | | +| metadata.labels | object | `{}` | | +| metadata.annotations | object | `{}` | | | crds.install | bool | `true` | | | crds.keep | bool | `true` | | | crds.annotations | object | `{}` | | -| webhook.extraArgs | list | `[]` | | +| controller.logLevel | string | `"info"` | | +| controller.extraArgs | list | `[]` | | +| controller.serviceAccount.name | string | `""` | | +| controller.serviceAccount.create | bool | `true` | | +| controller.rbac.create | bool | `true` | | +| controller.config.reconcileInterval | string | `"60"` | | +| controller.config.watchNamespace | string | `""` | | +| controller.config.checkForChanges | bool | `false` | | +| controller.config.backupNamespace | string | `""` | | | webhook.enabled | bool | `true` | | | webhook.logLevel | string | `"info"` | | +| webhook.extraArgs | list | `[]` | | | webhook.podLabels | object | `{}` | | +| webhook.rbac.create | bool | `true` | | +| webhook.serviceAccount.name | string | `""` | | | webhook.serviceAccount.create | bool | `true` | | | webhook.names.mutating | string | `"db-operator-mutating-webhook-configuration"` | | | webhook.names.validating | string | `"db-operator-validating-webhook-configuration"` | | @@ -164,53 +107,4 @@ We use helm tests feature for testing the chart in CI. Though we know that this | service.annotations | object | `{}` | | | service.type | string | `"ClusterIP"` | | | service.port | int | `8080` | | -## Dev Notes - -After changing default `Values`, please execute `make gen_docs` to update the `README.md` file. Readme file is generated by the `helm-docs` tool, so make sure not to edit it manually. - -## Upgrading - -If there is an breaking change, or something that might make the upgrade complicated, it should be described here - -
- To v2.0.0 -From `v2.0.0` onwards, the `v` prefix will not be automatically added to the image tag. If you are overriding the image tag through `.Values.image.tag` and are relying on this behaviour, please add the `v` prefix manually to the Helm value. Otherwise, no action is required from your side! -
- -
- To v1.11.0 -Additional selectors were added to the default templates in an attempt to follow the same labelling scheme everywhere, but since selectors are immutable, the upgrade will require removing of the db-operator deployment. - -```bash -$ kubectl get deploy -NAME READY UP-TO-DATE AVAILABLE AGE -db-operator 1/1 1 1 22s -$ kubectl delete deploy db-operator -deployment.apps "db-operator" deleted -$ helm upgrade db-operator db-operator/db-operator --version 1.11.0 -``` - -
- -
- To v1.10.0 - -CRDs are moved to the `templates` folder, so now they are part of the release. It means that after the upgrade, you will get errors about resource ownerships. Thow errors will contain messages about missing `labels` and `annotations`, and the easiest way to fix it, will be just to add the `metadata` that helm can't find. So you can follow those messages one by one and when all the `CRDs` are patched, you'll be able to install the release. - -For example: - -```BASH -$ helm upgrade my-release . -Error: UPGRADE FAILED: rendered manifests contain a resource that already exists. Unable to continue with update: CustomResourceDefinition "databases.kinda.rocks" in namespace "" exists and cannot be imported into the current release: invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"; annotation validation error: missing key "meta.helm.sh/release-name": must be set to "my-release"; annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "default" -``` -So you should add following metadata: -```YAML -metadata: - labels: - "app.kubernetes.io/managed-by": Helm - annotations: - "meta.helm.sh/release-name": my-release - "meta.helm.sh/release-namespace": default -``` -
diff --git a/charts/db-operator/README.md.gotmpl b/charts/db-operator/README.md.gotmpl deleted file mode 100644 index f0598d9..0000000 --- a/charts/db-operator/README.md.gotmpl +++ /dev/null @@ -1,133 +0,0 @@ -{{ template "chart.header" . }} -{{ template "chart.deprecationWarning" . }} - -{{ template "chart.badgesSection" . }} - -{{ template "chart.description" . }} - -{{ template "chart.sourcesSection" . }} - -{{ template "chart.maintainersSection" . }} - -{{ template "chart.requirementsSection" . }} - - -## Installing the Chart - -```console -$ helm repo add db-operator https://db-operator.github.io/charts -$ helm repo update -$ helm install db-operator/db-operator -``` - -If you want to manage `CRDs` yourself, you need to set `.Values.crds.install` to `false`. In that case they won't be a part of templated manifests. - -We don't support this way of `CRD` management officially though, so we can't promise to answer your questions if you choose to go this way. If you want to know more about it, keep reading, we'll talk about it later. - -## Upgarding the Chart - -If there are breaking changes, they should be described in this README's latest item and in helm-chart notes. So if you run `helm upgrade db-operator db-operator/db-operator --dry-run`, you will see if there are breaking changes there. - -## Uninstalling the Chart - -To uninstall the operator only, run - -```console -$ helm uninstall db-operator -``` - -If you have CRDs installed by the chart, you will also have to run something like: - -```console -# !!! This command will remove all kinda.rocks CRDs from the cluster, -# so please make sure that you really want to run it -$ for crd in $(kubectl get crds | awk '{print $1}' | grep 'kinda.rocks'); do kubectl delete crd "${crd}"; done -``` - -## Why are we packaging CRDs in templates? - -There are several reasons for that. The main one is that we are using webhooks and they require certificates to be created. If you have a look at the `CRDs` templates, you'll see something like that: - -```mustache - annotations: - controller-gen.kubebuilder.io/version: v0.12.1 - {{ `{{- if .Values.webhook.certificate.create }} `}} - cert-manager.io/inject-ca-from: {{ `{{ .Release.Namespace }}/{{ .Values.webhook.certificate.name}}` }} - {{ `{{ else }}` }} - cert-manager.io/inject-ca-from-secret: {{ `{{ .Release.Namespace }}/{{ .Values.webhook.certificate.secretName}}` }} - {{ `{{- end }}` }} - {{ `{{- if .Values.crds.keep }}` }} - helm.sh/resource-policy: keep - {{ `{{- end }}` }} - {{ `{{- with .Values.crds.annotations }}` }} - {{ `{{- toYaml . | nindent 4 }}` }} - {{ `{{- end }}` }} -``` - -So, as you can see they require some information about the release, so they can't be static. Since the webhook is an important part of what we're distributing, we don't want to have it disabled by default. - -Also, we do not have `CRs` in the package, so there is no chicken-egg problem here. You don't need to have `CRDs` installed to install the chart. Since `CRDs` are a part of the operator and you can't have one without another, having them packaged and distributed together makes more sense to us. - -But it's still possible for users to manage `CRDs` on their own. You need to set `.Values.crds.install` to false and go to the [db-operator rgit repository](https://github.com/db-operator/db-operator). There, in the `config/crd/bases` directory, you'll find `CRD` manifests. But if you decided to go this way and you have problems with `CRDs`, you're most probably on your own, you still can open issues though. - -## Why do we need cert-manager? - -At the moment, `db-operator` chart requires `cert-manager` to be present in the cluster. It's because webhook related resources are using it to create certificates. It should change one day, since we would like this chart to be self-sufficient, but currently that's what we've got. - -## What is `.Values.tests`? - -We use helm tests feature for testing the chart in CI. Though we know that this feature should be used by users to test their intsallations, we do not think that it can be very helpful in this particular case. Test resources are creating a powerful `ClusterRoles`, so **please make sure you don't have them enabled**, unless you really know what you're doing and why. - -## Chart Values -{{ template "chart.valuesTable" . }} -## Dev Notes - -After changing default `Values`, please execute `make gen_docs` to update the `README.md` file. Readme file is generated by the `helm-docs` tool, so make sure not to edit it manually. - - -## Upgrading - -If there is an breaking change, or something that might make the upgrade complicated, it should be described here - -
- To v2.0.0 -From `v2.0.0` onwards, the `v` prefix will not be automatically added to the image tag. If you are overriding the image tag through `.Values.image.tag` and are relying on this behaviour, please add the `v` prefix manually to the Helm value. Otherwise, no action is required from your side! -
- -
- To v1.11.0 -Additional selectors were added to the default templates in an attempt to follow the same labelling scheme everywhere, but since selectors are immutable, the upgrade will require removing of the db-operator deployment. - -```bash -$ kubectl get deploy -NAME READY UP-TO-DATE AVAILABLE AGE -db-operator 1/1 1 1 22s -$ kubectl delete deploy db-operator -deployment.apps "db-operator" deleted -$ helm upgrade db-operator db-operator/db-operator --version 1.11.0 -``` - -
- -
- To v1.10.0 - -CRDs are moved to the `templates` folder, so now they are part of the release. It means that after the upgrade, you will get errors about resource ownerships. Thow errors will contain messages about missing `labels` and `annotations`, and the easiest way to fix it, will be just to add the `metadata` that helm can't find. So you can follow those messages one by one and when all the `CRDs` are patched, you'll be able to install the release. - -For example: - -```BASH -$ helm upgrade my-release . -Error: UPGRADE FAILED: rendered manifests contain a resource that already exists. Unable to continue with update: CustomResourceDefinition "databases.kinda.rocks" in namespace "" exists and cannot be imported into the current release: invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"; annotation validation error: missing key "meta.helm.sh/release-name": must be set to "my-release"; annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "default" -``` - -So you should add following metadata: -```YAML -metadata: - labels: - "app.kubernetes.io/managed-by": Helm - annotations: - "meta.helm.sh/release-name": my-release - "meta.helm.sh/release-namespace": default -``` -
diff --git a/charts/db-operator/crd/kinda.rocks_databases.yaml b/charts/db-operator/crd/kinda.rocks_databases.yaml index 2f8c612..755d4da 100644 --- a/charts/db-operator/crd/kinda.rocks_databases.yaml +++ b/charts/db-operator/crd/kinda.rocks_databases.yaml @@ -372,10 +372,14 @@ spec: description: DatabaseBackup defines the desired state of backup and schedule properties: + bucket: + type: string cron: type: string enable: type: boolean + envFromSecret: + type: string required: - cron - enable diff --git a/charts/db-operator/templates/NOTES.txt b/charts/db-operator/templates/NOTES.txt index 5601a14..d7dd66f 100644 --- a/charts/db-operator/templates/NOTES.txt +++ b/charts/db-operator/templates/NOTES.txt @@ -1,21 +1,3 @@ ------------------------------------------------------------------------ - .-"``"-. - /______; \ - {_______}\| - (/ a a \)(_) - (.-.).-.) - _______________________ooo__( ^ )___________________________ - / '-.___.-' \ -| Please read these notes, they might be useful and we will try to | -| make them as short as possible | - \________________________________________ooo_______________________/ - |_ | _| jgs - \___|___/ - {___|___} - |_ | _| - /-'Y'-\ - (__/ \__) - ----------------------------------------------------------------------- Release information The Chart name: {{ .Chart.Name }}. @@ -24,14 +6,13 @@ The Release name {{ .Release.Name }}. ----------------------------------------------------------------------- Breaking changes and migration paths: -From 2.5.0, the .image.logLevel field is no longer used, and instead -users should use .controller.logLevel and .webhook.logLevel +Version `3.0.0` contains breaking changes for ones using the google cloudsql instances. It doesn't mean that the operator will not work with them anymore, but users will have to use **generic** DbInsances and prepare the infrasturture using other tools. + +Other changes that might require migration: -From 2.0.0, the "v" prefix will not be -automatically added to the image tag. If you are overriding the image -tag through ".Values.image.tag" and are relying on this behaviour, -please add the "v" prefix manually to the Helm value. -Otherwise, no action is required from your side! +- Pod names were changed, now instead of `db-operator-` and `db-operator-webhook`, we have `db-opetator-controller` and `db-operator-webhook` +- RBAC separation: the controller and webhook now use different ClusterRoles. The configuration has been moved to .controller.rbac and .webhook.rbac accordingly. +- DB Operator configuration is now set under `controller.config` {{- if not .Values.crds.install }} ----------------------------------------------------------------------- diff --git a/charts/db-operator/templates/_helpers.tpl b/charts/db-operator/templates/_helpers.tpl index 6769ae5..26bca8d 100644 --- a/charts/db-operator/templates/_helpers.tpl +++ b/charts/db-operator/templates/_helpers.tpl @@ -1,4 +1,3 @@ -{{/* vim: set filetype=mustache: */}} {{/* Expand the name of the chart. */}} @@ -24,101 +23,19 @@ If release name contains chart name it will be used as a full name. {{- end -}} {{- end -}} -{{/* -Create the name of the service account to use -*/}} -{{- define "db-operator.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "db-operator.name" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} -{{- end -}} - -{{/* -Arguments builder -*/}} -{{- define "db-operator.args" -}} -{{- $args := list -}} -{{- $args = append $args (printf "--zap-log-level=%s" .Values.controller.logLevel) -}} -{{- if .Values.checkForChanges -}} -{{- $args = append $args "--check-for-changes" -}} -{{- end -}} -{{- /* controller-specific extraArgs */ -}} -{{- range .Values.controller.extraArgs -}} -{{- $args = append $args . -}} -{{- end -}} -{{ join "," $args }} -{{- end -}} - -{{/* -Webhook templates -*/}} - -{{/* -Expand the name of the chart. -*/}} -{{- define "webhook.name" -}} -{{- printf "db-operator-webhook" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "webhook.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create the name of the service account to use -*/}} -{{- define "webhook.serviceAccountName" -}} -{{- if .Values.webhook.serviceAccount.create -}} - {{ default (include "webhook.name" .) .Values.webhook.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.webhook.serviceAccount.name }} -{{- end -}} -{{- end -}} - -{{/* -Webhook extra args -*/}} -{{- define "webhook.args" -}} -{{- $args := list -}} -{{- $args = append $args (printf "--zap-log-level=%s" .Values.webhook.logLevel) -}} -{{- $args = append $args "--webhook" -}} -{{- with .Values.webhook }} -{{- range .extraArgs -}} -{{- $args = append $args . -}} -{{- end -}} -{{- end -}} -{{ join "," $args }} -{{- end -}} - {{/* Create chart name and version as used by the chart label. */}} -{{- define "chart" -}} +{{- define "db-operator.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* -Common labels +DB Operator labels */}} -{{- define "labels" -}} -helm.sh/chart: {{ include "chart" . }} -{{ include "selectorLabels" . }} +{{- define "db-operator.labels" -}} +helm.sh/chart: {{ include "db-operator.chart" . }} +{{ include "db-operator.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -128,14 +45,27 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "selectorLabels" -}} +{{- define "db-operator.selectorLabels" -}} app.kubernetes.io/name: {{ include "db-operator.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} {{/* -Image version definition; +Build an image out of object like that: +image: + registry: ghcr.io + repositoru: db-operator/db-operator + tag: latest +Might be used to make it easier to configure mirroring */}} -{{- define "image_version" -}} -{{ default .Chart.AppVersion .Values.image.tag }} -{{- end -}} +{{- define "db-operator.imageBootsrap" -}} +{{- $image := "" }} +{{- if .image.registry }} +{{- $image = printf "%s/" .image.registry }} +{{- end }} +{{- $tag := printf "%s" .chart.AppVersion }} +{{- if .image.tag }} +{{- $tag = .image.tag }} +{{- end }} +{{- printf "%s%s:%s" $image .image.repository $tag }} +{{- end }} diff --git a/charts/db-operator/templates/controller/_helpers.tpl b/charts/db-operator/templates/controller/_helpers.tpl new file mode 100644 index 0000000..2669c44 --- /dev/null +++ b/charts/db-operator/templates/controller/_helpers.tpl @@ -0,0 +1,61 @@ +{{/* +Controller templates +*/}} + +{{/* +Controller specific labels +*/}} +{{- define "db-operator.controller.specificLabels" -}} +app.kubernetes.io/component: "controller" +{{- end -}} + +{{/* +Controller selector labels +*/}} +{{- define "db-operator.controller.selectorLabels" -}} +{{ include "db-operator.selectorLabels" . }} +{{ include "db-operator.controller.specificLabels" . }} +{{- end -}} + +{{/* +Controller labels +*/}} +{{- define "db-operator.controller.labels" -}} +{{ include "db-operator.labels" . }} +{{ include "db-operator.controller.specificLabels" . }} +{{- end -}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "db-operator.controller.name" -}} +{{- printf "db-operator-controller" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "db-operator.controller.serviceAccountName" -}} +{{- if .Values.controller.serviceAccount.create -}} + {{ default (include "db-operator.controller.name" .) .Values.controller.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.controller.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Controller extra args +*/}} +{{- define "db-operator.controller.args" -}} +{{- $args := list -}} +{{- with .Values.controller }} +{{- $args = append $args (printf "--zap-log-level=%s" .logLevel) -}} +{{- if .config.checkForChanges }} +{{- $args = append $args "--check-for-changes" }} +{{- end }} +{{- range .extraArgs -}} +{{- $args = append $args . -}} +{{- end -}} +{{- end -}} +{{ join "," $args }} +{{- end -}} diff --git a/charts/db-operator/templates/controller/rbac.yaml b/charts/db-operator/templates/controller/cluster_role.yaml similarity index 60% rename from charts/db-operator/templates/controller/rbac.yaml rename to charts/db-operator/templates/controller/cluster_role.yaml index bc13831..7bcae61 100644 --- a/charts/db-operator/templates/controller/rbac.yaml +++ b/charts/db-operator/templates/controller/cluster_role.yaml @@ -1,35 +1,74 @@ -{{- if .Values.rbac.create -}} +{{- if .Values.controller.rbac.create -}} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ template "db-operator.name" . }} + name: {{ template "db-operator.controller.name" . }} labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "controller" + {{- include "db-operator.controller.labels" . | nindent 4 }} rules: - apiGroups: - - kinda.rocks + - "" resources: - - "*" + - secrets + - configmaps + - pods + - services + - serviceaccounts verbs: - - watch - - update - get - list + - watch + - create + - update + - patch + - delete + - apiGroups: - - "" + - kinda.rocks resources: - - pods - - configmaps - - secrets - - services + - databases + - dbinstances + - dbusers + - dbbackups + - dbrestores + verbs: + - "*" + +- apiGroups: + - kinda.rocks + resources: + - databases/finalizers + - dbinstances/finalizers + - dbusers/finalizers + - dbbackups/finalizers + - dbrestores/finalizers + verbs: + - "*" + +- apiGroups: + - kinda.rocks + resources: + - databases/status + - dbinstances/status + - dbusers/status + - dbbackups/status + - dbrestores/status + verbs: + - "*" + +- apiGroups: + - "rbac.authorization.k8s.io" + resources: + - roles + - rolebindings verbs: - create - update - get - list - watch + - apiGroups: - "events.k8s.io" resources: @@ -41,28 +80,32 @@ rules: - get - list - watch + - apiGroups: - - "" - resources: - - services - verbs: - - patch -- apiGroups: - - batch + - apps resources: - - cronjobs + - deployments + - replicasets verbs: + - get - create - update - list - watch + - apiGroups: - - batch + - monitoring.coreos.com resources: - - jobs + - servicemonitors verbs: - create - update + - patch + - get + - list + - watch + +{{/* TODO: Check if really needed */}} - apiGroups: - "apiextensions.k8s.io" resources: @@ -71,49 +114,22 @@ rules: - get - list - watch + +{{/* TODO: Remove once backups are there */}} - apiGroups: - - monitoring.coreos.com + - batch resources: - - servicemonitors + - cronjobs verbs: - create - update - - patch - - get - list - watch - apiGroups: - - extensions - resources: - - deployments - verbs: - - create - - update -- apiGroups: - - apps + - batch resources: - - deployments - - replicasets + - jobs verbs: - - get - create - update - - list - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ template "db-operator.name" . }} - labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "controller" -subjects: -- kind: ServiceAccount - name: {{ template "db-operator.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ template "db-operator.name" . }} - apiGroup: rbac.authorization.k8s.io {{- end -}} diff --git a/charts/db-operator/templates/controller/cluster_role_binding.yaml b/charts/db-operator/templates/controller/cluster_role_binding.yaml new file mode 100644 index 0000000..0f8ab99 --- /dev/null +++ b/charts/db-operator/templates/controller/cluster_role_binding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.controller.rbac.create -}} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "db-operator.controller.name" . }} + labels: + {{- include "db-operator.controller.labels" . | nindent 4 }} +subjects: +- kind: ServiceAccount + name: {{ template "db-operator.controller.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "db-operator.controller.name" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/db-operator/templates/controller/config.yaml b/charts/db-operator/templates/controller/config.yaml index 9423900..1a7ea63 100644 --- a/charts/db-operator/templates/controller/config.yaml +++ b/charts/db-operator/templates/controller/config.yaml @@ -1,42 +1,10 @@ +--- apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "db-operator.name" . }}-config + name: {{ template "db-operator.controller.name" . }}-config labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "controller" + {{- include "db-operator.controller.labels" . | nindent 4 }} data: config.yaml: | - instance: - google: - clientSecretName: "cloudsql-readonly-serviceaccount" - proxy: - nodeSelector: - {{- toYaml .Values.config.instance.google.proxy.nodeSelector | nindent 12 }} - image: {{ .Values.config.instance.google.proxy.image }} - metricsPort: {{ .Values.config.instance.google.proxy.metricsPort }} - generic: {} - percona: - proxy: - image: {{ .Values.config.instance.percona.proxy.image }} - backup: - activeDeadlineSeconds: {{ .Values.config.backup.activeDeadlineSeconds }} - nodeSelector: - {{- toYaml .Values.config.backup.nodeSelector | nindent 8 }} - postgres: - image: {{ .Values.config.backup.postgres.image }} - mysql: - image: {{ .Values.config.backup.mysql.image }} - resources: - {{- toYaml .Values.config.backup.resources | nindent 8 }} - monitoring: - {{- if .Values.config.monitoring.promPushGateway }} - promPushGateway: {{ .Values.config.monitoring.promPushGateway }} - {{- end }} - nodeSelector: - {{- toYaml .Values.config.monitoring.nodeSelector | nindent 8 }} - postgres: - image: {{ .Values.config.monitoring.postgres.image }} - queries: |- - {{- toYaml .Values.config.monitoring.postgres.queries | nindent 10 }} - mysql: {} + {{- .Values.controller.config | toYaml | nindent 4 }} diff --git a/charts/db-operator/templates/controller/deployment.yaml b/charts/db-operator/templates/controller/deployment.yaml index e116fbc..bd9afed 100644 --- a/charts/db-operator/templates/controller/deployment.yaml +++ b/charts/db-operator/templates/controller/deployment.yaml @@ -2,24 +2,18 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ template "db-operator.name" . }} + name: {{ template "db-operator.controller.name" $ }} labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "controller" + {{- include "db-operator.controller.labels" $ | nindent 4 }} spec: replicas: 1 selector: matchLabels: - {{- include "selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "controller" + {{- include "db-operator.controller.selectorLabels" . | nindent 6 }} template: metadata: labels: - {{- include "labels" . | nindent 8 }} - app.kubernetes.io/component: "controller" - {{- if .Values.podLabels }} - {{ toYaml .Values.podLabels | trim | nindent 8 }} - {{- end }} + {{- include "db-operator.controller.labels" $ | nindent 8 }} annotations: checksum/config: {{ include (print $.Template.BasePath "/controller/config.yaml") . | sha256sum }} {{- if .Values.annotations }} @@ -29,9 +23,7 @@ spec: {{- if semverCompare ">=1.33-0" .Capabilities.KubeVersion.Version }} hostUsers: {{ .Values.hostUsers }} {{- end }} - {{- if .Values.serviceAccount.create }} - serviceAccountName: {{ template "db-operator.serviceAccountName" . }} - {{- end }} + serviceAccountName: {{ template "db-operator.controller.serviceAccountName" . }} {{- with .Values.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} @@ -42,15 +34,15 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - image: "{{ .Values.image.repository }}:{{ template "image_version" . }}" + image: {{ include "db-operator.imageBootsrap" (dict "image" $.Values.image "chart" $.Chart) }} ports: - containerPort: 60000 name: metrics command: - db-operator - {{- if (include "db-operator.args" . ) }} + {{- if (include "db-operator.controller.args" . ) }} args: - {{- range include "db-operator.args" $ | split "," }} + {{- range include "db-operator.controller.args" $ | split "," }} - {{ . }} {{- end }} {{- end }} @@ -67,27 +59,17 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.name - {{- if .Values.secrets.gsql }} - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/run/secrets/gcloudsqladmin/credentials.json" - {{- if .Values.secrets.gsql.readonly }} - - name: GCSQL_CLIENT_CREDENTIALS - value: "/run/secrets/gcloudsqlclient/credentials.json" - {{- end }} - {{- end }} + - name: DB_BACKUP_NAMESPACE + {{- if .Values.controller.config.backupNamespace }} + value: {{ .Values.controller.config.backupNamespace }} + {{- else }} + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} - name: CONFIG_PATH value: "/run/config/config.yaml" volumeMounts: - {{- if .Values.secrets.gsql }} - - mountPath: /run/secrets/gcloudsqladmin - name: serviceaccount-cloudsql-admin - readOnly: true - {{- if .Values.secrets.gsql.readonly }} - - mountPath: /run/secrets/gcloudsqlclient - name: serviceaccount-cloudsql-client - readOnly: true - {{- end }} - {{- end }} - mountPath: /run/config/ name: config-volume readOnly: true @@ -107,18 +89,6 @@ spec: {{ toYaml .Values.imagePullSecrets | nindent 8 }} {{- end }} volumes: - {{- if .Values.secrets.gsql }} - - name: serviceaccount-cloudsql-admin - secret: - defaultMode: 420 - secretName: cloudsql-admin-serviceaccount - {{- if .Values.secrets.gsql.readonly }} - - name: serviceaccount-cloudsql-client - secret: - defaultMode: 420 - secretName: cloudsql-readonly-serviceaccount - {{- end }} - {{- end }} - name: config-volume configMap: - name: {{ template "db-operator.name" . }}-config + name: {{ template "db-operator.controller.name" . }}-config diff --git a/charts/db-operator/templates/controller/gsql-secret.yaml b/charts/db-operator/templates/controller/gsql-secret.yaml deleted file mode 100644 index bea9ffc..0000000 --- a/charts/db-operator/templates/controller/gsql-secret.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if .Values.secrets }} -{{- if .Values.secrets.gsql }} -# secret which is required for creating database (used by db-operator) -# required permission (Cloud SQL Admin) -# cloudsql.instances.create -# cloudsql.instances.get -# cloudsql.instances.update -apiVersion: v1 -kind: Secret -metadata: - name: cloudsql-admin-serviceaccount - labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "controller" -type: Opaque -data: - credentials.json: |- - {{ .Values.secrets.gsql.admin | b64enc }} -{{- if .Values.secrets.gsql.readonly }} ---- -# secret which is required for connecting/accessing to database (used by cloudproxy of application) -# required permission (Cloud SQL Client) -# cloudsql.instances.connect -# cloudsql.instances.get -apiVersion: v1 -kind: Secret -metadata: - name: cloudsql-readonly-serviceaccount - labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "controller" -type: Opaque -data: - credentials.json: |- - {{ .Values.secrets.gsql.readonly | b64enc }} -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/db-operator/templates/controller/service_account.yaml b/charts/db-operator/templates/controller/service_account.yaml new file mode 100644 index 0000000..2a67741 --- /dev/null +++ b/charts/db-operator/templates/controller/service_account.yaml @@ -0,0 +1,12 @@ +{{- if .Values.controller.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "db-operator.controller.serviceAccountName" . }} + labels: + {{- include "db-operator.controller.labels" $ | nindent 4 }} +{{- if .Values.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/db-operator/templates/controller/serviceaccount.yaml b/charts/db-operator/templates/controller/serviceaccount.yaml deleted file mode 100644 index 2f7205b..0000000 --- a/charts/db-operator/templates/controller/serviceaccount.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ template "db-operator.serviceAccountName" . }} - labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "controller" -{{- if .Values.imagePullSecrets }} -imagePullSecrets: -{{ toYaml .Values.imagePullSecrets | indent 2 }} -{{- end }} -{{- end -}} diff --git a/charts/db-operator/templates/controller/servicemonitor.yaml b/charts/db-operator/templates/controller/servicemonitor.yaml deleted file mode 100644 index 7df04bc..0000000 --- a/charts/db-operator/templates/controller/servicemonitor.yaml +++ /dev/null @@ -1,68 +0,0 @@ -{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ template "db-operator.name" . }} - labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "controller" - {{- with .Values.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: metrics - protocol: TCP - name: metrics - {{- if and (eq .Values.service.type "NodePort") (.Values.service.nodePort) }} - nodePort: {{ .Values.service.nodePort }} - {{- end }} - selector: - {{- include "selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "controller" ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ template "db-operator.name" . }} - labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "controller" - {{- with .Values.serviceMonitor.selector }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if .Values.serviceMonitor.jobLabel }} - jobLabel: {{ .Values.serviceMonitor.jobLabel }} - {{- end }} - endpoints: - - port: metrics - {{- with .Values.serviceMonitor.interval }} - interval: {{ . }} - {{- end }} - {{- with .Values.serviceMonitor.scrapeTimeout }} - scrapeTimeout: {{ . }} - {{- end }} - {{- with .Values.serviceMonitor.metricRelabelings }} - metricRelabelings: - {{- if kindIs "string" . }} - {{- tpl . $ | nindent 8 }} - {{- else }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- end }} - {{- with .Values.serviceMonitor.relabelings }} - relabelings: - {{- toYaml . | nindent 8 }} - {{- end }} - namespaceSelector: - matchNames: - - {{ .Release.Namespace }} - selector: - matchLabels: - {{- include "selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "controller" -{{- end }} diff --git a/charts/db-operator/templates/crds.yaml b/charts/db-operator/templates/crds.yaml index f2faaf2..aeb5fe1 100644 --- a/charts/db-operator/templates/crds.yaml +++ b/charts/db-operator/templates/crds.yaml @@ -41,7 +41,7 @@ spec: clientConfig: service: namespace: {{ $.Release.Namespace }} - name: {{ include "webhook.name" . }} + name: {{ include "db-operator.webhook.name" . }} path: /convert conversionReviewVersions: - v1alpha1 diff --git a/charts/db-operator/templates/webhook/_helpers.tpl b/charts/db-operator/templates/webhook/_helpers.tpl new file mode 100644 index 0000000..7054963 --- /dev/null +++ b/charts/db-operator/templates/webhook/_helpers.tpl @@ -0,0 +1,59 @@ +{{/* +Webhook templates +*/}} + +{{/* +Webhook specific labels +*/}} +{{- define "db-operator.webhook.specificLabels" -}} +app.kubernetes.io/component: "webhook" +{{- end -}} + +{{/* +Webhook selector labels +*/}} +{{- define "db-operator.webhook.selectorLabels" -}} +{{ include "db-operator.selectorLabels" . }} +{{ include "db-operator.webhook.specificLabels" . }} +{{- end -}} + +{{/* +Webhook labels +*/}} +{{- define "db-operator.webhook.labels" -}} +{{ include "db-operator.labels" . }} +{{ include "db-operator.webhook.specificLabels" . }} +{{- end -}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "db-operator.webhook.name" -}} +{{- printf "db-operator-webhook" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "db-operator.webhook.serviceAccountName" -}} +{{- if .Values.webhook.serviceAccount.create -}} + {{ default (include "db-operator.webhook.name" .) .Values.webhook.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.webhook.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Webhook extra args +*/}} +{{- define "db-operator.webhook.args" -}} +{{- $args := list -}} +{{- $args = append $args (printf "--zap-log-level=%s" .Values.webhook.logLevel) -}} +{{- $args = append $args "--webhook" -}} +{{- with .Values.webhook }} +{{- range .extraArgs -}} +{{- $args = append $args . -}} +{{- end -}} +{{- end -}} +{{ join "," $args }} +{{- end -}} diff --git a/charts/db-operator/templates/webhook/certificates/_helpers.tpl b/charts/db-operator/templates/webhook/certificates/_helpers.tpl new file mode 100644 index 0000000..dab55c0 --- /dev/null +++ b/charts/db-operator/templates/webhook/certificates/_helpers.tpl @@ -0,0 +1,22 @@ +{{/* + A helper for managing certificates +*/}} + +{{- define "db-operator.webhook.certificateName" -}} +{{ printf "%s-certificate" (include "db-operator.webhook.name" $)}} +{{- end }} + +{{- define "db-operator.webhook.shortSvcUrl" -}} +{{- include "db-operator.webhook.name" $ }}.{{ $.Release.Namespace }}.svc +{{- end }} + +{{- define "db-operator.webhook.fullSvcUrl" -}} +{{- include "db-operator.webhook.name" $ }}.{{ $.Release.Namespace }}.svc.{{ $.Values.clusterDomain }} +{{- end }} + +{{- define "db-operator.webhook.selfSignedCert" -}} +{{- $shortSvcUrl := include "db-operator.webhook.shortSvcUrl" $ }} +{{- $fullSvcUrl := include "db-operator.webhook.fullSvcUrl " $ }} +{{- $cert := genSelfSignedCert $shortSvcUrl (list) (list $fullSvcUrl) 365 | toYaml -}} +{{- $cert }} +{{- end }} diff --git a/charts/db-operator/templates/certificate/certificate.yaml b/charts/db-operator/templates/webhook/certificates/certificate.yaml similarity index 68% rename from charts/db-operator/templates/certificate/certificate.yaml rename to charts/db-operator/templates/webhook/certificates/certificate.yaml index 5949c10..db53964 100644 --- a/charts/db-operator/templates/certificate/certificate.yaml +++ b/charts/db-operator/templates/webhook/certificates/certificate.yaml @@ -6,8 +6,8 @@ metadata: name: {{ .Values.webhook.certificate.name }} spec: dnsNames: - - {{ include "webhook.name" . }}.{{ .Release.Namespace }}.svc - - {{ include "webhook.name" . }}.{{ .Release.Namespace }}.svc.cluster.local + - {{ include "db-operator.webhook.name" . }}.{{ .Release.Namespace }}.svc + - {{ include "db-operator.webhook.name" . }}.{{ .Release.Namespace }}.svc.cluster.local issuerRef: kind: {{ .Values.webhook.certificate.issuer.kind }} name: {{ .Values.webhook.certificate.issuer.name }} diff --git a/charts/db-operator/templates/certificate/issuer.yaml b/charts/db-operator/templates/webhook/certificates/issuer.yaml similarity index 100% rename from charts/db-operator/templates/certificate/issuer.yaml rename to charts/db-operator/templates/webhook/certificates/issuer.yaml diff --git a/charts/db-operator/templates/webhook/certificates/self_signed_cert.yaml b/charts/db-operator/templates/webhook/certificates/self_signed_cert.yaml new file mode 100644 index 0000000..be11adf --- /dev/null +++ b/charts/db-operator/templates/webhook/certificates/self_signed_cert.yaml @@ -0,0 +1,17 @@ +{{/* + * A self-signed cert generated using the Helm function. + * It's valid for 365 days and is re-generated on every synchronization + * that makes it incompatible with GitOps tools like ArgoCD and FluxCD. +*/}} +{{- $method :=(($.Values.webhook).certificate).method -}} +{{- if (eq $method "helm") }} {{/* if[0] */}} +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + name: {{ include "db-operator.webhook.certificateName" $ }} +data: + tls.crt: |- + {{ include "db-operator.webhook.selfSignedCert" $}} + tls.key: +{{- end -}} {{/* /if[0] */}} diff --git a/charts/db-operator/templates/webhook/cluster_role.yaml b/charts/db-operator/templates/webhook/cluster_role.yaml new file mode 100644 index 0000000..de40c94 --- /dev/null +++ b/charts/db-operator/templates/webhook/cluster_role.yaml @@ -0,0 +1,26 @@ +{{- if .Values.webhook.rbac.create -}} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "db-operator.webhook.name" . }} + labels: + {{- include "db-operator.webhook.labels" . | nindent 4 }} +rules: +- apiGroups: + - kinda.rocks + resources: + - databases + - dbinstances + - dbusers + - dbbackups + - dbrestores + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +{{- end -}} diff --git a/charts/db-operator/templates/webhook/cluster_role_binding.yaml b/charts/db-operator/templates/webhook/cluster_role_binding.yaml new file mode 100644 index 0000000..d327bc8 --- /dev/null +++ b/charts/db-operator/templates/webhook/cluster_role_binding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.webhook.rbac.create -}} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "db-operator.webhook.name" . }} + labels: + {{- include "db-operator.webhook.labels" . | nindent 4 }} +subjects: +- kind: ServiceAccount + name: {{ template "db-operator.webhook.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "db-operator.webhook.name" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/db-operator/templates/webhook/mutating_webhook.yaml b/charts/db-operator/templates/webhook/configs/mutating_webhook.yaml similarity index 87% rename from charts/db-operator/templates/webhook/mutating_webhook.yaml rename to charts/db-operator/templates/webhook/configs/mutating_webhook.yaml index abd1776..49fa1b0 100644 --- a/charts/db-operator/templates/webhook/mutating_webhook.yaml +++ b/charts/db-operator/templates/webhook/configs/mutating_webhook.yaml @@ -10,15 +10,14 @@ metadata: cert-manager.io/inject-ca-from-secret: {{ .Release.Namespace }}/{{ .Values.webhook.certificate.secretName}} {{- end }} labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "webhook" + {{- include "db-operator.webhook.labels" . | nindent 4 }} name: {{ .Values.webhook.names.mutating }} webhooks: - admissionReviewVersions: - v1 clientConfig: service: - name: {{ include "webhook.name" . }} + name: {{ include "db-operator.webhook.name" . }} namespace: {{ .Release.Namespace }} path: /mutate-kinda-rocks-v1beta1-database failurePolicy: Fail @@ -38,7 +37,7 @@ webhooks: - v1 clientConfig: service: - name: {{ include "webhook.name" . }} + name: {{ include "db-operator.webhook.name" . }} namespace: {{ .Release.Namespace }} path: /mutate-kinda-rocks-v1beta1-dbinstance failurePolicy: Fail diff --git a/charts/db-operator/templates/webhook/validation_webhook.yaml b/charts/db-operator/templates/webhook/configs/validating_webhook.yaml similarity index 87% rename from charts/db-operator/templates/webhook/validation_webhook.yaml rename to charts/db-operator/templates/webhook/configs/validating_webhook.yaml index 004eb9d..8df60be 100644 --- a/charts/db-operator/templates/webhook/validation_webhook.yaml +++ b/charts/db-operator/templates/webhook/configs/validating_webhook.yaml @@ -10,15 +10,14 @@ metadata: cert-manager.io/inject-ca-from-secret: {{ .Release.Namespace }}/{{ .Values.webhook.certificate.secretName}} {{- end }} labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "webhook" + {{- include "db-operator.webhook.labels" . | nindent 4 }} name: {{ .Values.webhook.names.validating }} webhooks: - admissionReviewVersions: - v1 clientConfig: service: - name: {{ include "webhook.name" . }} + name: {{ include "db-operator.webhook.name" . }} namespace: {{ .Release.Namespace }} path: /validate-kinda-rocks-v1beta1-database failurePolicy: Fail @@ -38,7 +37,7 @@ webhooks: - v1 clientConfig: service: - name: {{ include "webhook.name" . }} + name: {{ include "db-operator.webhook.name" . }} namespace: {{ .Release.Namespace }} path: /validate-kinda-rocks-v1beta1-dbinstance failurePolicy: Fail @@ -58,7 +57,7 @@ webhooks: - v1 clientConfig: service: - name: {{ include "webhook.name" . }} + name: {{ include "db-operator.webhook.name" . }} namespace: {{ .Release.Namespace }} path: /validate-kinda-rocks-v1beta1-dbuser failurePolicy: Fail diff --git a/charts/db-operator/templates/webhook/deployment.yaml b/charts/db-operator/templates/webhook/deployment.yaml index 95fa3c2..6c3ef50 100644 --- a/charts/db-operator/templates/webhook/deployment.yaml +++ b/charts/db-operator/templates/webhook/deployment.yaml @@ -2,21 +2,18 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "webhook.name" . }} + name: {{ include "db-operator.webhook.name" . }} labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "webhook" + {{- include "db-operator.webhook.labels" $ | nindent 4 }} spec: replicas: 1 selector: matchLabels: - {{- include "selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: "webhook" + {{- include "db-operator.webhook.selectorLabels" . | nindent 6 }} template: metadata: labels: - {{- include "labels" . | nindent 8 }} - app.kubernetes.io/component: "webhook" + {{- include "db-operator.webhook.labels" . | nindent 8 }} {{- if .Values.webhook.podLabels }} {{ toYaml .Values.webhook.podLabels | trim | nindent 8 }} {{- end }} @@ -29,7 +26,7 @@ spec: hostUsers: {{ .Values.hostUsers }} {{- end }} {{- if .Values.webhook.serviceAccount.create }} - serviceAccountName: {{ template "webhook.serviceAccountName" . }} + serviceAccountName: {{ template "db-operator.webhook.serviceAccountName" . }} {{- end }} {{- with .Values.podSecurityContext }} securityContext: @@ -41,7 +38,7 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - image: "{{ .Values.image.repository }}:{{ template "image_version" . }}" + image: {{ include "db-operator.imageBootsrap" (dict "image" $.Values.image "chart" $.Chart) }} ports: - containerPort: 9443 name: webhook-server @@ -49,7 +46,7 @@ spec: command: - db-operator args: - {{- range include "webhook.args" $ | split "," }} + {{- range include "db-operator.webhook.args" $ | split "," }} - {{ . }} {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} diff --git a/charts/db-operator/templates/webhook/service.yaml b/charts/db-operator/templates/webhook/service.yaml index 4ec773a..2aeeaf8 100644 --- a/charts/db-operator/templates/webhook/service.yaml +++ b/charts/db-operator/templates/webhook/service.yaml @@ -3,16 +3,14 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "webhook.name" . }} + name: {{ include "db-operator.webhook.name" . }} labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "webhook" + {{- include "db-operator.webhook.labels" . | nindent 4 }} spec: ports: - port: 443 protocol: TCP targetPort: 9443 selector: - {{- include "selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: "webhook" + {{- include "db-operator.webhook.selectorLabels" . | nindent 4 }} {{- end }} diff --git a/charts/db-operator/templates/webhook/service_account.yaml b/charts/db-operator/templates/webhook/service_account.yaml new file mode 100644 index 0000000..4af659e --- /dev/null +++ b/charts/db-operator/templates/webhook/service_account.yaml @@ -0,0 +1,14 @@ +{{- if .Values.webhook.enabled }} +{{- if .Values.webhook.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "db-operator.webhook.serviceAccountName" . }} + labels: + {{- include "db-operator.webhook.labels" $ | nindent 4 }} +{{- if .Values.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 2 }} +{{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/db-operator/templates/webhook/serviceaccount.yaml b/charts/db-operator/templates/webhook/serviceaccount.yaml deleted file mode 100644 index 1b5c7b0..0000000 --- a/charts/db-operator/templates/webhook/serviceaccount.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if (and .Values.webhook.enabled .Values.webhook.serviceAccount.create) -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ template "webhook.serviceAccountName" . }} - labels: - {{- include "labels" . | nindent 4 }} - app.kubernetes.io/component: "webhook" -{{- if .Values.imagePullSecrets }} -imagePullSecrets: -{{ toYaml .Values.imagePullSecrets | indent 2 }} -{{- end }} -{{- end -}} diff --git a/charts/db-operator/tests/controller_args.yaml b/charts/db-operator/tests/controller_args.yaml index 67afbe7..a638a8b 100644 --- a/charts/db-operator/tests/controller_args.yaml +++ b/charts/db-operator/tests/controller_args.yaml @@ -33,7 +33,9 @@ tests: - it: logLevel=info, checkForChanges and no extraArgs template: controller/deployment.yaml set: - checkForChanges: true + controller: + config: + checkForChanges: true asserts: - equal: path: spec.template.spec.containers[0].args @@ -69,8 +71,9 @@ tests: - it: logLevel=info, extraArgs and checkForChanges template: controller/deployment.yaml set: - checkForChanges: true controller: + config: + checkForChanges: true extraArgs: - --test - --test2 diff --git a/charts/db-operator/tests/controller_rbac.yaml b/charts/db-operator/tests/controller_rbac.yaml new file mode 100644 index 0000000..d84d542 --- /dev/null +++ b/charts/db-operator/tests/controller_rbac.yaml @@ -0,0 +1,36 @@ +--- +suite: Test RBAC templating for the controller +templates: + - controller/cluster_role.yaml + - controller/cluster_role_binding.yaml + - controller/_helpers.tpl + - _helpers.tpl +tests: + - it: ClusterRole should be rendered by default + template: controller/cluster_role.yaml + asserts: + - hasDocuments: + count: 1 + - it: When disabled, RBAC shouldn't be rendered at all + template: controller/cluster_role.yaml + set: + controller: + rbac: + create: false + asserts: + - hasDocuments: + count: 0 + - it: ClusterRoleBinding should be rendered by default + template: controller/cluster_role_binding.yaml + asserts: + - hasDocuments: + count: 1 + - it: When disabled, RBAC shouldn't be rendered at all + template: controller/cluster_role_binding.yaml + set: + controller: + rbac: + create: false + asserts: + - hasDocuments: + count: 0 diff --git a/charts/db-operator/tests/issuer.yaml b/charts/db-operator/tests/issuer.yaml index c2a0468..eee97ae 100644 --- a/charts/db-operator/tests/issuer.yaml +++ b/charts/db-operator/tests/issuer.yaml @@ -1,7 +1,7 @@ --- suite: Test issuer templating templates: - - certificate/issuer.yaml + - webhook/certificates/issuer.yaml tests: - it: Issuer should be rendered by default asserts: diff --git a/charts/db-operator/tests/rbac.yaml b/charts/db-operator/tests/rbac.yaml deleted file mode 100644 index bbd9ce7..0000000 --- a/charts/db-operator/tests/rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -suite: Test RBAC templating -templates: - - controller/rbac.yaml -tests: - - it: RBAC should be rendered by default - asserts: - - hasDocuments: - count: 2 - - it: When disabled, RBAC shouldn't be rendered at all - set: - rbac: - create: false - asserts: - - hasDocuments: - count: 0 diff --git a/charts/db-operator/tests/webhook.yaml b/charts/db-operator/tests/webhook.yaml index 97a597d..fb6e616 100644 --- a/charts/db-operator/tests/webhook.yaml +++ b/charts/db-operator/tests/webhook.yaml @@ -16,15 +16,15 @@ tests: - hasDocuments: count: 1 filterAware: true - template: webhook/serviceaccount.yaml + template: webhook/service_account.yaml - hasDocuments: count: 1 filterAware: true - template: webhook/mutating_webhook.yaml + template: webhook/configs/mutating_webhook.yaml - hasDocuments: count: 1 filterAware: true - template: webhook/validation_webhook.yaml + template: webhook/configs/validating_webhook.yaml - it: Each template should not be rendered when webhooks are disabled set: webhook: @@ -41,12 +41,12 @@ tests: - hasDocuments: count: 0 filterAware: true - template: webhook/serviceaccount.yaml + template: webhook/service_account.yaml - hasDocuments: count: 0 filterAware: true - template: webhook/mutating_webhook.yaml + template: webhook/configs/mutating_webhook.yaml - hasDocuments: count: 0 filterAware: true - template: webhook/validation_webhook.yaml + template: webhook/configs/validating_webhook.yaml diff --git a/charts/db-operator/tests/webhook_rbac.yaml b/charts/db-operator/tests/webhook_rbac.yaml new file mode 100644 index 0000000..ff7c911 --- /dev/null +++ b/charts/db-operator/tests/webhook_rbac.yaml @@ -0,0 +1,36 @@ +--- +suite: Test RBAC templating for the webhook +templates: + - webhook/cluster_role.yaml + - webhook/cluster_role_binding.yaml + - webhook/_helpers.tpl + - _helpers.tpl +tests: + - it: ClusterRole should be rendered by default + template: webhook/cluster_role.yaml + asserts: + - hasDocuments: + count: 1 + - it: When disabled, RBAC shouldn't be rendered at all + template: webhook/cluster_role.yaml + set: + webhook: + rbac: + create: false + asserts: + - hasDocuments: + count: 0 + - it: ClusterRoleBinding should be rendered by default + template: webhook/cluster_role_binding.yaml + asserts: + - hasDocuments: + count: 1 + - it: When disabled, RBAC shouldn't be rendered at all + template: webhook/cluster_role_binding.yaml + set: + webhook: + rbac: + create: false + asserts: + - hasDocuments: + count: 0 diff --git a/charts/db-operator/values-local.yaml b/charts/db-operator/values-local.yaml deleted file mode 100644 index fec4aeb..0000000 --- a/charts/db-operator/values-local.yaml +++ /dev/null @@ -1,33 +0,0 @@ -reconcileInterval: "10" -image: - pullPolicy: IfNotPresent -secrets: - gsql: - admin: | - { - "type": "service_account", - "project_id": "integration-project", - "private_key_id": "", - "private_key": "", - "client_email": "", - "client_id": "admin", - "auth_uri": "", - "token_uri": "", - "auth_provider_x509_cert_url": "", - "client_x509_cert_url": "" - } - readonly: | - { - "type": "service_account", - "project_id": "integration-project", - "private_key_id": "", - "private_key": "", - "client_email": "", - "client_id": "readonly", - "auth_uri": "", - "token_uri": "", - "auth_provider_x509_cert_url": "", - "client_x509_cert_url": "" - } -serviceMonitor: - enabled: true diff --git a/charts/db-operator/values-stanley.yaml b/charts/db-operator/values-stanley.yaml deleted file mode 100644 index 4596f43..0000000 --- a/charts/db-operator/values-stanley.yaml +++ /dev/null @@ -1,16 +0,0 @@ -image: - repository: registry.kinda.rocks/devops/db-operator - tag: latest - pullPolicy: IfNotPresent -resources: - limits: - cpu: "0.5" - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi -nodeSelector: - env: production -annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "60000" diff --git a/charts/db-operator/values.yaml b/charts/db-operator/values.yaml index b43f717..0b96008 100644 --- a/charts/db-operator/values.yaml +++ b/charts/db-operator/values.yaml @@ -1,43 +1,62 @@ nameOverride: "" +clusterDomain: cluster.local + image: - repository: ghcr.io/db-operator/db-operator + registry: ghcr.io + repository: db-operator/db-operator + tag: "" pullPolicy: Always - -controller: - logLevel: info - extraArgs: [] # imagePullSecrets: # - name: myRegistrySecret +# Global metadata, should be applied to each resource +metadata: + labels: {} + annotations: {} -reconcileInterval: "60" -# watchNamespace value is comma-separated list of namespace names. It's necessary to set "" to watch cluster wide. -watchNamespace: "" - -# ------------------------------------------------------------ -# -- If set to true, db-operator will check if Kubernetes -# -- objects were changed before running database queries. -# -- It might reduce the amount of queries, because otherwise -# -- db-operator will try to update databases and users every -# -- ${RECONCILE_INTERVAL} seconds. -# ------------------------------------------------------------ -# -- NOTE: Currently, it's only working with Database Users -# ------------------------------------------------------------ -checkForChanges: false - -rbac: - create: true -serviceAccount: - create: true crds: install: true keep: true annotations: {} -webhook: + +controller: + logLevel: info extraArgs: [] + serviceAccount: + name: "" + create: true + rbac: + create: true + config: + backup: + activeDeadlineSeconds: 600 # 10m + nodeSelector: {} + postgres: + image: "ghcr.io/db-operator/pgdump-rclone:0.1.0" + mysql: + image: "ghcr.io/db-operator/mydump-rclone:0.1.0" + resources: + requests: + memory: 64Mi + cpu: 0.2 + monitoring: + promPushGateway: "" + args: + reconcileInterval: "60" + # watchNamespace value is comma-separated list of namespace names. It's necessary to set "" to watch cluster wide. + watchNamespace: "" + checkForChanges: false + # Which namespace should be used for backup pods + backupNamespace: "" + +webhook: enabled: true logLevel: info + extraArgs: [] podLabels: {} + rbac: + create: true serviceAccount: + name: "" create: true names: mutating: db-operator-mutating-webhook-configuration @@ -89,65 +108,6 @@ annotations: {} podLabels: {} affinity: {} tolerations: [] -config: - instance: - google: - proxy: - nodeSelector: {} - image: ghcr.io/db-operator/db-auth-gateway:v0.1.10 - metricsPort: 9090 - generic: {} - percona: - proxy: - image: severalnines/proxysql:2.0 - metricsPort: 9090 - backup: - activeDeadlineSeconds: 600 # 10m - nodeSelector: {} - postgres: - image: "kloeckneri/pgdump-gcs:latest" - mysql: - image: "kloeckneri/mydump-gcs:latest" - resources: - requests: - memory: 64Mi - cpu: 0.2 - monitoring: - promPushGateway: "" - nodeSelector: {} - postgres: - image: wrouesnel/postgres_exporter:latest - queries: - pg_stat_statements: - query: "SELECT userid, pgss.dbid, pgdb.datname, queryid, query, calls, total_time, mean_time, rows FROM pg_stat_statements pgss LEFT JOIN (select oid as dbid, datname from pg_database) as pgdb on pgdb.dbid = pgss.dbid WHERE not queryid isnull ORDER BY mean_time desc limit 20" - metrics: - - userid: - usage: "LABEL" - description: "User ID" - - dbid: - usage: "LABEL" - description: "database ID" - - datname: - usage: "LABEL" - description: "database NAME" - - queryid: - usage: "LABEL" - description: "Query unique Hash Code" - - query: - usage: "LABEL" - description: "Query class" - - calls: - usage: "COUNTER" - description: "Number of times executed" - - total_time: - usage: "COUNTER" - description: "Total time spent in the statement, in milliseconds" - - mean_time: - usage: "GAUGE" - description: "Mean time spent in the statement, in milliseconds" - - rows: - usage: "COUNTER" - description: "Total number of rows retrieved or affected by the statement" secrets: gsql: {} serviceMonitor: diff --git a/ci/helmfile.yaml.gotmpl b/ci/helmfile.yaml.gotmpl index 885c427..61ec400 100644 --- a/ci/helmfile.yaml.gotmpl +++ b/ci/helmfile.yaml.gotmpl @@ -3,6 +3,7 @@ environments: db-instances: tests: local-dev: + e2e-tests: --- repositories: - name: jetstack @@ -14,6 +15,8 @@ repositories: - name: cloudpirates url: registry-1.docker.io/cloudpirates oci: true + - name: minio-standalone + url: https://charts.min.io/ releases: - name: cert-manager @@ -54,6 +57,53 @@ releases: serviceMonitorSelector: {} serviceMonitorSelectorNilUsesHelmValues: false + - name: minio + namespace: minio + chart: minio-standalone/minio + version: 5.4.0 + createNamespace: true + installed: {{ eq .Environment.Name "local-dev" }} + values: + - rootUser: 'root' + rootPassword: 123123!! + replicas: 1 + mode: standalone + resources: + requests: + memory: 512Mi + environment: + MINIO_SERVER_URL: "http://minio.minio.svc.cluster.local:9000" + tls: + enabled: false + persistence: + enabled: true + accessMode: ReadWriteOnce + size: 1Gi + service: + type: ClusterIP + clusterIP: ~ + port: '9000' + consoleService: + type: ClusterIP + clusterIP: ~ + port: '9001' + buckets: + - name: backups + policy: download + purge: false + policies: + - name: backups + statements: + - resources: + - 'arn:aws:s3:::backups' + - 'arn:aws:s3:::backups/*' + actions: + - 's3:*' + users: + - accessKey: backups + secretKey: qwertyu9 + policy: backups + - name: db-operator namespace: db-operator wait: true @@ -68,7 +118,8 @@ releases: {{- if (eq .Environment.Name "local-dev") }} values: - image: - repository: localhost/my-db-operator + registry: localhost + repository: my-db-operator tag: v1.0.0-dev pullPolicy: Never {{- end }} @@ -83,6 +134,9 @@ releases: values: - auth: rootPassword: 123123!! + - service: + type: NodePort + nodePort: "30306" - name: mysql-test namespace: mysql-test @@ -103,6 +157,9 @@ releases: - auth: username: db-operator password: 123123!! + - service: + type: NodePort + nodePort: 30432 - name: psql-test namespace: psql-test diff --git a/ci/kustomize/databases/kustomization.yaml b/ci/kustomize/databases/kustomization.yaml index 6de6cef..5c3e557 100644 --- a/ci/kustomize/databases/kustomization.yaml +++ b/ci/kustomize/databases/kustomization.yaml @@ -1,3 +1,4 @@ resources: - ./mysql.yaml - ./postgres.yaml + - ./secret.yaml diff --git a/ci/kustomize/databases/mysql.yaml b/ci/kustomize/databases/mysql.yaml index 7c04ed5..79480df 100644 --- a/ci/kustomize/databases/mysql.yaml +++ b/ci/kustomize/databases/mysql.yaml @@ -4,7 +4,10 @@ metadata: name: mysql spec: backup: - enable: false + enable: true + cron: '*/40 * * * *' + bucket: backups + envFromSecret: s3-creds deletionProtected: false instance: mariadb secretName: mysql-creds diff --git a/ci/kustomize/databases/postgres.yaml b/ci/kustomize/databases/postgres.yaml index ec6dae2..9e4b972 100644 --- a/ci/kustomize/databases/postgres.yaml +++ b/ci/kustomize/databases/postgres.yaml @@ -4,22 +4,12 @@ metadata: name: postgres spec: backup: - enable: false + enable: true + cron: '*/40 * * * *' + bucket: backups + envFromSecret: s3-creds deletionProtected: false instance: postgres postgres: {} secretName: postgres-creds cleanup: true ---- -apiVersion: kinda.rocks/v1beta1 -kind: Database -metadata: - name: postgres-2 -spec: - backup: - enable: false - deletionProtected: false - instance: postgres - postgres: {} - secretName: postgres-2-creds - cleanup: true diff --git a/ci/kustomize/databases/secret.yaml b/ci/kustomize/databases/secret.yaml new file mode 100644 index 0000000..e82cdf3 --- /dev/null +++ b/ci/kustomize/databases/secret.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: s3-creds +stringData: + RCLONE_CONFIG_STORAGE_TYPE: s3 + RCLONE_CONFIG_STORAGE_PROVIDER: Minio + RCLONE_CONFIG_STORAGE_ACCESS_KEY_ID: backups + RCLONE_CONFIG_STORAGE_SECRET_ACCESS_KEY: qwertyu9 + RCLONE_CONFIG_STORAGE_ENDPOINT: http://minio.minio.svc.cluster.local:9000 diff --git a/ci/scripts/mysql b/ci/scripts/mysql new file mode 100755 index 0000000..4377dec --- /dev/null +++ b/ci/scripts/mysql @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +export DBPASS=$(kubectl get secret mysql-creds -o yaml | yq .data.PASSWORD | base64 -d) +export DBNAME=$(kubectl get secret mysql-creds -o yaml | yq .data.DB | base64 -d) +export DBUSER=$(kubectl get secret mysql-creds -o yaml | yq .data.USER | base64 -d) + +mysql --host=localhost --port=3306 --user="${DBUSER}" --password="${DBPASS}" --database="${DBNAME}" "$@" diff --git a/ci/scripts/mysql.sql b/ci/scripts/mysql.sql new file mode 100644 index 0000000..7df4fcf --- /dev/null +++ b/ci/scripts/mysql.sql @@ -0,0 +1,142 @@ +-- ========================================================= +-- MariaDB Random Data Generator for Backup Testing +-- ========================================================= + +SET FOREIGN_KEY_CHECKS = 0; + +DROP TABLE IF EXISTS order_items; +DROP TABLE IF EXISTS orders; +DROP TABLE IF EXISTS products; +DROP TABLE IF EXISTS customers; +DROP TABLE IF EXISTS numbers; + +SET FOREIGN_KEY_CHECKS = 1; + +-- ========================================================= +-- Helper numbers table (FAST and MariaDB-friendly) +-- ========================================================= + +CREATE TABLE numbers ( + n INT PRIMARY KEY +) ENGINE=InnoDB; + +-- Fill numbers table (1..1,000,000 max capacity) +INSERT INTO numbers (n) +SELECT a.N + b.N*10 + c.N*100 + d.N*1000 + e.N*10000 + 1 +FROM + (SELECT 0 N UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 + UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) a, + (SELECT 0 N UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 + UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) b, + (SELECT 0 N UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 + UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) c, + (SELECT 0 N UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 + UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) d, + (SELECT 0 N UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 + UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) e; + +-- ========================================================= +-- Tables +-- ========================================================= + +CREATE TABLE customers ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + email VARCHAR(100) NOT NULL, + created_at DATETIME NOT NULL +) ENGINE=InnoDB; + +CREATE TABLE products ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100) NOT NULL, + price DECIMAL(10,2) NOT NULL, + created_at DATETIME NOT NULL +) ENGINE=InnoDB; + +CREATE TABLE orders ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + customer_id BIGINT NOT NULL, + order_date DATETIME NOT NULL, + status VARCHAR(20) NOT NULL, + FOREIGN KEY (customer_id) REFERENCES customers(id) +) ENGINE=InnoDB; + +CREATE TABLE order_items ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + order_id BIGINT NOT NULL, + product_id BIGINT NOT NULL, + quantity INT NOT NULL, + unit_price DECIMAL(10,2) NOT NULL, + FOREIGN KEY (order_id) REFERENCES orders(id), + FOREIGN KEY (product_id) REFERENCES products(id) +) ENGINE=InnoDB; + +-- ========================================================= +-- Insert customers (100k) +-- ========================================================= + +INSERT INTO customers (first_name, last_name, email, created_at) +SELECT + CONCAT('First', n), + CONCAT('Last', n), + CONCAT('user', n, '@example.com'), + NOW() - INTERVAL FLOOR(RAND()*365) DAY +FROM numbers +WHERE n <= 100000; + +-- ========================================================= +-- Insert products (5k) +-- ========================================================= + +INSERT INTO products (name, price, created_at) +SELECT + CONCAT('Product ', n), + ROUND(RAND()*500 + 5, 2), + NOW() - INTERVAL FLOOR(RAND()*365) DAY +FROM numbers +WHERE n <= 5000; + +-- ========================================================= +-- Insert orders (300k) +-- ========================================================= + +INSERT INTO orders (customer_id, order_date, status) +SELECT + FLOOR(RAND()*100000) + 1, + NOW() - INTERVAL FLOOR(RAND()*365) DAY, + ELT(FLOOR(RAND()*4)+1, 'pending','shipped','delivered','cancelled') +FROM numbers +WHERE n <= 300000; + +-- ========================================================= +-- Insert order items (≈1–5 per order) +-- ========================================================= + +INSERT INTO order_items (order_id, product_id, quantity, unit_price) +SELECT + o.id, + FLOOR(RAND()*5000) + 1, + FLOOR(RAND()*4) + 1, + ROUND(RAND()*500 + 5, 2) +FROM orders o +JOIN numbers r ON r.n <= 5 +WHERE RAND() < 0.6; + +-- ========================================================= +-- Indexes +-- ========================================================= + +CREATE INDEX idx_orders_customer ON orders(customer_id); +CREATE INDEX idx_order_items_order ON order_items(order_id); +CREATE INDEX idx_order_items_product ON order_items(product_id); + +-- ========================================================= +-- Summary +-- ========================================================= + +SELECT + (SELECT COUNT(*) FROM customers) AS customers, + (SELECT COUNT(*) FROM products) AS products, + (SELECT COUNT(*) FROM orders) AS orders, + (SELECT COUNT(*) FROM order_items) AS order_items; diff --git a/ci/scripts/postgres.sql b/ci/scripts/postgres.sql new file mode 100644 index 0000000..d71ccee --- /dev/null +++ b/ci/scripts/postgres.sql @@ -0,0 +1,123 @@ +-- ========================================================= +-- Random Test Data Generator for PostgreSQL Backup Testing +-- ========================================================= + +-- Optional: speed up bulk inserts +SET synchronous_commit = OFF; +SET maintenance_work_mem = '512MB'; + +-- ========================================================= +-- Drop existing tables (safe reset) +-- ========================================================= + +DROP TABLE IF EXISTS order_items; +DROP TABLE IF EXISTS orders; +DROP TABLE IF EXISTS products; +DROP TABLE IF EXISTS customers; + +-- ========================================================= +-- Create tables +-- ========================================================= + +CREATE TABLE customers ( + id BIGSERIAL PRIMARY KEY, + first_name TEXT NOT NULL, + last_name TEXT NOT NULL, + email TEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT now() +); + +CREATE TABLE products ( + id BIGSERIAL PRIMARY KEY, + name TEXT NOT NULL, + price NUMERIC(10,2) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT now() +); + +CREATE TABLE orders ( + id BIGSERIAL PRIMARY KEY, + customer_id BIGINT NOT NULL REFERENCES customers(id), + order_date TIMESTAMP NOT NULL, + status TEXT NOT NULL +); + +CREATE TABLE order_items ( + id BIGSERIAL PRIMARY KEY, + order_id BIGINT NOT NULL REFERENCES orders(id), + product_id BIGINT NOT NULL REFERENCES products(id), + quantity INT NOT NULL, + unit_price NUMERIC(10,2) NOT NULL +); + +-- ========================================================= +-- Insert customers +-- Adjust the number in generate_series for scale +-- ========================================================= + +INSERT INTO customers (first_name, last_name, email, created_at) +SELECT + 'First' || gs, + 'Last' || gs, + 'user' || gs || '@example.com', + NOW() - (random() * interval '365 days') +FROM generate_series(1, 100000) AS gs; + +-- ========================================================= +-- Insert products +-- ========================================================= + +INSERT INTO products (name, price, created_at) +SELECT + 'Product ' || gs, + ROUND((random() * 500 + 5)::numeric, 2), + NOW() - (random() * interval '365 days') +FROM generate_series(1, 5000) AS gs; + +-- ========================================================= +-- Insert orders +-- ========================================================= + +INSERT INTO orders (customer_id, order_date, status) +SELECT + (random() * 99999 + 1)::BIGINT, + NOW() - (random() * interval '365 days'), + (ARRAY['pending','shipped','delivered','cancelled'])[floor(random()*4)+1] +FROM generate_series(1, 300000); + +-- ========================================================= +-- Insert order items +-- Each order gets 1–5 items +-- ========================================================= + +INSERT INTO order_items (order_id, product_id, quantity, unit_price) +SELECT + o.id, + (random() * 4999 + 1)::BIGINT, + (random() * 4 + 1)::INT, + ROUND((random() * 500 + 5)::numeric, 2) +FROM orders o +CROSS JOIN LATERAL generate_series(1, (random()*4 + 1)::INT); + +-- ========================================================= +-- Indexes (important for realistic backup size) +-- ========================================================= + +CREATE INDEX idx_orders_customer ON orders(customer_id); +CREATE INDEX idx_order_items_order ON order_items(order_id); +CREATE INDEX idx_order_items_product ON order_items(product_id); + +-- ========================================================= +-- Analyze for realistic planner stats +-- ========================================================= + +ANALYZE; + +-- ========================================================= +-- Summary +-- ========================================================= + +SELECT + (SELECT count(*) FROM customers) AS customers, + (SELECT count(*) FROM products) AS products, + (SELECT count(*) FROM orders) AS orders, + (SELECT count(*) FROM order_items) AS order_items; diff --git a/ci/scripts/psql b/ci/scripts/psql new file mode 100755 index 0000000..5ea6c73 --- /dev/null +++ b/ci/scripts/psql @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +export PGPASSWORD=$(kubectl get secret postgres-creds -o yaml | yq .data.POSTGRES_PASSWORD | base64 -d) +export PGDATABASE=$(kubectl get secret postgres-creds -o yaml | yq .data.POSTGRES_DB | base64 -d) +export PGUSER=$(kubectl get secret postgres-creds -o yaml | yq .data.POSTGRES_USER | base64 -d) +export PGHOST=localhost + +psql "$@"