From c67bbfc0902eecf2873e4ca2271a37e9b4b205b0 Mon Sep 17 00:00:00 2001 From: Shay Goldstein Date: Thu, 22 Jan 2026 18:46:36 +0200 Subject: [PATCH] [SeaweedFS] Migrate from MinIO to SeaweedFS as the S3-compatible storage backend # Conflicts: # charts/mlrun-ce/Chart.yaml # charts/mlrun-ce/requirements.lock # charts/mlrun-ce/values.yaml --- README.md | 8 +- charts/mlrun-ce/Chart.yaml | 2 +- charts/mlrun-ce/README.md | 51 +++-- .../mlrun-ce/admin_installation_values.yaml | 2 +- ..._admin_cluster_ip_installation_values.yaml | 13 +- .../non_admin_installation_values.yaml | 13 +- charts/mlrun-ce/requirements.lock | 6 +- charts/mlrun-ce/requirements.yaml | 8 +- charts/mlrun-ce/templates/NOTES.txt | 32 +-- charts/mlrun-ce/templates/_helpers.tpl | 66 +++++- .../config/jupyter-env-configmap.yaml | 6 +- .../templates/config/mlrun-env-configmap.yaml | 6 +- .../configmaps/pipeline-install-config.yaml | 6 +- .../workflow-controller-configmap.yaml | 4 +- .../pipelines/deployments/ml-pipeline-ui.yaml | 4 +- .../pipelines/deployments/ml-pipeline.yaml | 7 +- .../secrets/mlpipeline-minio-artifact.yaml | 4 +- .../seaweedfs/seaweedfs-s3-config.yaml | 33 +++ .../seaweedfs/seaweedfs-ui-ingress.yaml | 42 ++++ .../seaweedfs/seaweedfs-ui-services.yaml | 35 ++++ charts/mlrun-ce/values.yaml | 194 +++++++++++++++--- 21 files changed, 415 insertions(+), 127 deletions(-) create mode 100644 charts/mlrun-ce/templates/seaweedfs/seaweedfs-s3-config.yaml create mode 100644 charts/mlrun-ce/templates/seaweedfs/seaweedfs-ui-ingress.yaml create mode 100644 charts/mlrun-ce/templates/seaweedfs/seaweedfs-ui-services.yaml diff --git a/README.md b/README.md index 27430b3b..88267190 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,17 @@ This repo contains the Helm charts for the MLRun Community Edition (CE) - a full open source MLOps stack. -The Open source MLRun ce chart includes the following stack: +The Open source MLRun CE chart includes the following stack: * Nuclio - https://github.com/nuclio/nuclio * MLRun - https://github.com/mlrun/mlrun * Jupyter - https://github.com/jupyter/notebook (+MLRun integrated) * MPI Operator - https://github.com/kubeflow/mpi-operator - * Minio - https://github.com/minio/minio/tree/master/helm/minio + * SeaweedFS - https://github.com/seaweedfs/seaweedfs (S3-compatible storage) * Spark Operator - https://github.com/GoogleCloudPlatform/spark-on-k8s-operator * Pipelines - https://github.com/kubeflow/pipelines * Prometheus stack - https://github.com/prometheus-community/helm-charts +## Installation - ## Installation - Refer to the installation instructions in the [README](charts/mlrun-ce/README.md) of the `mlrun-ce` chart. +Refer to the installation instructions in the [README](charts/mlrun-ce/README.md) of the `mlrun-ce` chart. diff --git a/charts/mlrun-ce/Chart.yaml b/charts/mlrun-ce/Chart.yaml index bb311cba..bebcd290 100644 --- a/charts/mlrun-ce/Chart.yaml +++ b/charts/mlrun-ce/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: mlrun-ce -version: 0.11.0-rc4 +version: 0.11.0-rc5 description: MLRun Open Source Stack home: https://iguazio.com icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png diff --git a/charts/mlrun-ce/README.md b/charts/mlrun-ce/README.md index 7c53d2cc..5c999b4b 100644 --- a/charts/mlrun-ce/README.md +++ b/charts/mlrun-ce/README.md @@ -10,7 +10,7 @@ The Open source MLRun ce chart includes the following stack: * MLRun - https://github.com/mlrun/mlrun * Jupyter - https://github.com/jupyter/notebook (+MLRun integrated) * MPI Operator - https://github.com/kubeflow/mpi-operator -* Minio - https://github.com/minio/minio/tree/master/helm/minio +* SeaweedFS - https://github.com/seaweedfs/seaweedfs (S3-compatible storage) * Spark Operator - https://github.com/GoogleCloudPlatform/spark-on-k8s-operator * Pipelines - https://github.com/kubeflow/pipelines * Prometheus stack - https://github.com/prometheus-community/helm-charts @@ -52,15 +52,14 @@ kubectl --namespace mlrun create secret docker-registry registry-credentials \ ``` To install the chart with the release name `my-mlrun` use the following command, -note the reference to the pre-created `registry-credentials` secret in `global.registry.secretName`, -and a `global.registry.url` with an appropriate registry URL which can be authenticated by this secret: +note the reference to the pre-created `registry-credentials` secret in `global.containerRegistry.secretName`, and a `global.containerRegistry.url` with an appropriate registry URL which can be authenticated by this secret: ```bash helm --namespace mlrun \ install my-mlrun \ --wait \ - --set global.registry.url= \ - --set global.registry.secretName=registry-credentials \ + --set global.containerRegistry.url= \ + --set global.containerRegistry.secretName=registry-credentials \ mlrun/mlrun-ce ``` @@ -161,10 +160,10 @@ helm --namespace mlrun \ install my-mlrun \ --wait \ ... other overrides ... \ - --set global.registry.url=${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com \ + --set global.containerRegistry.url=${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com \ + --set global.containerRegistry.secretName=ecr-registry-credentials \ --set nuclio.dashboard.kaniko.registryProviderSecretName=aws-credentials \ --set mlrun.defaultDockerRegistrySecretName=aws-credentials \ - --set global.registry.secretName=ecr-registry-credentials \ mlrun/mlrun-ce ``` @@ -173,19 +172,35 @@ helm --namespace mlrun \ ## Usage Your applications are now available in your local browser: -- jupyter-notebook - http://nodeipaddress:30040 -- nuclio - http://nodeipaddress:30050 -- mlrun UI - http://nodeipaddress:30060 -- mlrun API (external) - http://nodeipaddress:30070 -- minio API - http://nodeipaddress:30080 -- minio UI - http://nodeipaddress:30090 -- pipeline UI - http://nodeipaddress:30100 -- grafana UI - http://nodeipaddress:30110 +- Jupyter Notebook - http://nodeipaddress:30040 +- Nuclio - http://nodeipaddress:30050 +- MLRun UI - http://nodeipaddress:30060 +- MLRun API (external) - http://nodeipaddress:30070 +- SeaweedFS Master UI (cluster status) - http://nodeipaddress:30091 +- SeaweedFS Filer UI (file browser) - http://nodeipaddress:30092 +- SeaweedFS Admin UI (user/policy management) - http://nodeipaddress:30093 +- Pipeline UI - http://nodeipaddress:30100 +- Grafana UI - http://nodeipaddress:30010 +- Prometheus UI - http://nodeipaddress:30020 + +**S3 API Access** (programmatic, for tools like aws-cli): +```bash +kubectl port-forward svc/seaweedfs-s3 8333:8333 -n mlrun +aws --endpoint-url=http://localhost:8333 s3 ls s3://mlrun/ +``` + +**With Ingress enabled**, the UIs are available at: +- `https://seaweedfs-master...lab.iguazeng.com` +- `https://seaweedfs-filer...lab.iguazeng.com` +- `https://seaweedfs-admin...lab.iguazeng.com` + > **Note:** > The above links assume your Kubernetes cluster is exposed on localhost. > If that's not the case, the different components will be available on `externalHostAddress` +> +> For production deployments, consider enabling ingress for each service instead of using NodePorts. ## Start Working @@ -278,6 +293,6 @@ Refer to the [**Kubeflow documentation**](https://www.kubeflow.org/docs/started/ This table shows the versions of the main components in the MLRun CE chart: -| MLRun CE | MLRun | Nuclio | Jupyter | MPI Operator | Minio | Spark Operator | Pipelines | Kube-Prometheus-Stack | Prometheus | Grafana | -|-----------|-------|--------|---------|--------------|------------------------------|----------------|-----------|-----------------------|------------|---------| -| **0.6.3** | 1.6.4 | 1.13.5 | 6.4.0 | 0.2.3 | RELEASE.2022-05-08T23-50-31Z | v1beta2-1.1.25 | 1.8.1 | 17.0.0 | 2.39.1 | 6.40.4 | +| MLRun CE | MLRun | Nuclio | Jupyter | MPI Operator | SeaweedFS | Spark Operator | Pipelines | Kube-Prometheus-Stack | +|-----------|-------|--------|---------|--------------|-----------|----------------|-----------|-----------------------| +| **0.6.3** | 1.6.4 | 1.13.5 | 6.4.0 | 0.2.3 | 4.0.407 | 2.1.0 | 2.14.3 | 72.1.1 | diff --git a/charts/mlrun-ce/admin_installation_values.yaml b/charts/mlrun-ce/admin_installation_values.yaml index 0ef34729..c9b2bf23 100644 --- a/charts/mlrun-ce/admin_installation_values.yaml +++ b/charts/mlrun-ce/admin_installation_values.yaml @@ -36,7 +36,7 @@ mpi-operator: deployment: create: false -minio: +seaweedfs: enabled: false spark-operator: diff --git a/charts/mlrun-ce/non_admin_cluster_ip_installation_values.yaml b/charts/mlrun-ce/non_admin_cluster_ip_installation_values.yaml index e407db6b..8cce57a6 100644 --- a/charts/mlrun-ce/non_admin_cluster_ip_installation_values.yaml +++ b/charts/mlrun-ce/non_admin_cluster_ip_installation_values.yaml @@ -35,17 +35,8 @@ mpi-operator: clusterResources: create: false -minio: - service: - type: ClusterIP - port: 9000 - nodePort: "" - consoleService: - type: ClusterIP - port: 9001 - nodePort: "" - mode: standalone - replicas: 1 +seaweedfs: + enabled: true spark-operator: enabled: false diff --git a/charts/mlrun-ce/non_admin_installation_values.yaml b/charts/mlrun-ce/non_admin_installation_values.yaml index 460b1013..d84f02ee 100644 --- a/charts/mlrun-ce/non_admin_installation_values.yaml +++ b/charts/mlrun-ce/non_admin_installation_values.yaml @@ -34,17 +34,8 @@ mpi-operator: clusterResources: create: false -minio: - service: - type: NodePort - port: 9000 - nodePort: 30080 - consoleService: - type: NodePort - port: 9001 - nodePort: 30090 - mode: standalone - replicas: 1 +seaweedfs: + enabled: true spark-operator: enabled: false diff --git a/charts/mlrun-ce/requirements.lock b/charts/mlrun-ce/requirements.lock index 929d4e4b..b83f638f 100644 --- a/charts/mlrun-ce/requirements.lock +++ b/charts/mlrun-ce/requirements.lock @@ -8,9 +8,9 @@ dependencies: - name: mpi-operator repository: https://v3io.github.io/helm-charts/stable version: 0.6.0 -- name: minio - repository: https://charts.min.io/ - version: 5.2.0 +- name: seaweedfs + repository: https://seaweedfs.github.io/seaweedfs/helm + version: 4.0.407 - name: spark-operator repository: https://kubeflow.github.io/spark-operator version: 2.1.0 diff --git a/charts/mlrun-ce/requirements.yaml b/charts/mlrun-ce/requirements.yaml index 94669693..900754a0 100644 --- a/charts/mlrun-ce/requirements.yaml +++ b/charts/mlrun-ce/requirements.yaml @@ -9,10 +9,10 @@ dependencies: - name: mpi-operator version: "0.6.0" repository: "https://v3io.github.io/helm-charts/stable" - - name: minio - repository: "https://charts.min.io/" - version: "5.2.0" - condition: minio.enabled + - name: seaweedfs + repository: "https://seaweedfs.github.io/seaweedfs/helm" + version: "4.0.407" + condition: seaweedfs.enabled - name: spark-operator repository: "https://kubeflow.github.io/spark-operator" version: "2.1.0" diff --git a/charts/mlrun-ce/templates/NOTES.txt b/charts/mlrun-ce/templates/NOTES.txt index 540fa389..5e2f7be6 100644 --- a/charts/mlrun-ce/templates/NOTES.txt +++ b/charts/mlrun-ce/templates/NOTES.txt @@ -1,4 +1,4 @@ -{{- if or (.Values.jupyterNotebook.enabled) (.Values.mlrun.enabled) (.Values.minio.enabled) (.Values.pipelines.enabled) (index .Values "kube-prometheus-stack" "enabled") }} +{{- if or (.Values.jupyterNotebook.enabled) (.Values.mlrun.enabled) (.Values.seaweedfs.enabled) (.Values.pipelines.enabled) (index .Values "kube-prometheus-stack" "enabled") }} You're up and running! {{- if .Values.jupyterNotebook.enabled }} @@ -57,41 +57,29 @@ mlrun-api.{{ .Values.global.externalHostAddress }} {{- end }} {{- end }} -{{- if .Values.minio.enabled }} -{{- if ne .Values.global.infrastructure.kind "aws" }} -{{- "\n" }} -Minio UI is available at: -{{- if .Values.minio.ingress.enabled }} -{{ (index .Values.minio.consoleIngress.hosts 0) }} -{{- else }} -{{ .Values.global.externalHostAddress }}:{{ .Values.minio.consoleService.nodePort }} -{{- end }} -- username: {{ .Values.minio.rootUser }} -- password: {{ .Values.minio.rootPassword }} -{{- end }} -{{- end }} -{{- if .Values.minio.enabled }} +{{- if .Values.seaweedfs.enabled }} {{- if ne .Values.global.infrastructure.kind "aws" }} +{{- if and .Values.seaweedfs.admin .Values.seaweedfs.admin.enabled }} {{- "\n" }} -Minio API is available at: -{{- if .Values.minio.ingress.enabled }} -{{ (index .Values.minio.ingress.hosts 0) }} +SeaweedFS Admin UI is available at: +{{- if and .Values.seaweedfs.adminService .Values.seaweedfs.adminService.ingress .Values.seaweedfs.adminService.ingress.enabled }} +{{ .Values.seaweedfs.adminService.ingress.host }} {{- else }} -{{ .Values.global.externalHostAddress }}:{{ .Values.minio.service.nodePort }} +{{ .Values.global.externalHostAddress }}:{{ .Values.seaweedfs.adminService.nodePort | default 30093 }} +{{- end }} +- S3 credentials: minio / minio123 {{- end }} {{- end }} {{- end }} -{{- if and .Values.pipelines.enabled .Values.pipelines.ui.enabled -}} }} +{{- if and .Values.pipelines.enabled .Values.pipelines.ui.enabled }} {{- if ne .Values.global.infrastructure.kind "aws" }} -{{- if not .Values.minio.ingress.enabled }} {{- "\n" }} Pipelines UI is available at: {{ .Values.global.externalHostAddress }}:{{ .Values.pipelines.service.nodePort }} {{- end }} {{- end }} -{{- end }} {{- if index .Values "kube-prometheus-stack" "enabled" }} {{- "\n" }} diff --git a/charts/mlrun-ce/templates/_helpers.tpl b/charts/mlrun-ce/templates/_helpers.tpl index 94e0d5a9..67f38831 100644 --- a/charts/mlrun-ce/templates/_helpers.tpl +++ b/charts/mlrun-ce/templates/_helpers.tpl @@ -123,25 +123,83 @@ Create chart name and version as used by the chart label. {{- end -}} {{/* -Minio Service URL +============================================================================= +S3 Storage Backend Helpers +Supports both MinIO and SeaweedFS as S3-compatible storage backends +============================================================================= +*/}} + +{{/* +Validate S3 storage backend configuration +Fails if seaweedfs is not enabled +*/}} +{{- define "mlrun-ce.s3.validateConfig" -}} +{{- if not .Values.seaweedfs.enabled -}} +{{- fail "seaweedfs.enabled must be set to true for S3 storage backend." -}} +{{- end -}} +{{- end -}} + +{{/* +S3 Service URL - returns the endpoint URL for SeaweedFS +*/}} +{{- define "mlrun-ce.s3.service.url" -}} +{{- include "mlrun-ce.s3.validateConfig" . -}} +http://seaweedfs-s3.{{.Release.Namespace}}.svc.cluster.local:{{ .Values.seaweedfs.s3.port }} +{{- end -}} + +{{/* +S3 Service Host - returns just the hostname for pipeline config +*/}} +{{- define "mlrun-ce.s3.service.host" -}} +{{- include "mlrun-ce.s3.validateConfig" . -}} +seaweedfs-s3.{{.Release.Namespace}}.svc.cluster.local +{{- end -}} + +{{/* +S3 Service Port - returns the port for pipeline config +*/}} +{{- define "mlrun-ce.s3.service.port" -}} +{{- include "mlrun-ce.s3.validateConfig" . -}} +{{- .Values.seaweedfs.s3.port | toString -}} +{{- end -}} + +{{/* +S3 Access Key +*/}} +{{- define "mlrun-ce.s3.accessKey" -}} +{{- include "mlrun-ce.s3.validateConfig" . -}} +{{- .Values.pipelines.s3.accessKey -}} +{{- end -}} + +{{/* +S3 Secret Key +*/}} +{{- define "mlrun-ce.s3.secretKey" -}} +{{- include "mlrun-ce.s3.validateConfig" . -}} +{{- .Values.pipelines.s3.secretKey -}} +{{- end -}} + +{{/* +Legacy Minio Service URL - kept for backward compatibility */}} {{- define "mlrun-ce.minio.service.url" -}} -http://minio.{{.Release.Namespace}}.svc.cluster.local:{{ .Values.minio.service.port }} +{{ include "mlrun-ce.s3.service.url" . }} {{- end -}} {{- define "mlrun-ce.minio-pipeline.service.url" -}} -minio.{{.Release.Namespace}}.svc.cluster.local +{{ include "mlrun-ce.s3.service.host" . }} {{- end -}} {{/* MLRun storage auto mount params Global toggle is for fast toggling between on-prem/standalone and s3 cases Can be overriden if params are explicitly specified +Uses SeaweedFS as the storage backend */}} {{- define "mlrun.storage.auto.mount.params" -}} {{- if hasKey .Values.mlrun "storageAutoMountParams" -}} {{ .Values.mlrun.storageAutoMountParams }} {{- else if not .Values.global.infrastructure.aws.s3NonAnonymous -}} - "aws_access_key={{ .Values.minio.rootUser }},aws_secret_key={{ .Values.minio.rootPassword }},endpoint_url={{ include "mlrun-ce.minio.service.url" . }}" + "aws_access_key={{ include "mlrun-ce.s3.accessKey" . }},aws_secret_key={{ include "mlrun-ce.s3.secretKey" . }},endpoint_url={{ include "mlrun-ce.s3.service.url" . }}" {{- else -}} "non_anonymous=True" {{- end -}} diff --git a/charts/mlrun-ce/templates/config/jupyter-env-configmap.yaml b/charts/mlrun-ce/templates/config/jupyter-env-configmap.yaml index f26b38a6..06ae60cc 100644 --- a/charts/mlrun-ce/templates/config/jupyter-env-configmap.yaml +++ b/charts/mlrun-ce/templates/config/jupyter-env-configmap.yaml @@ -6,9 +6,9 @@ metadata: name: jupyter-common-env data: {{- if not .Values.global.infrastructure.aws.s3NonAnonymous }} - AWS_ENDPOINT_URL_S3: {{ include "mlrun-ce.minio.service.url" . }} - AWS_SECRET_ACCESS_KEY: {{ .Values.minio.rootPassword }} - AWS_ACCESS_KEY_ID: {{ .Values.minio.rootUser }} + AWS_ENDPOINT_URL_S3: {{ include "mlrun-ce.s3.service.url" . }} + AWS_SECRET_ACCESS_KEY: {{ include "mlrun-ce.s3.secretKey" . }} + AWS_ACCESS_KEY_ID: {{ include "mlrun-ce.s3.accessKey" . }} {{- end }} MLRUN_STORAGE__AUTO_MOUNT_TYPE: {{ default "" .Values.mlrun.storageAutoMountType }} S3_NON_ANONYMOUS: {{ .Values.global.infrastructure.aws.s3NonAnonymous | toString | title | quote | default "\"True\"" }} diff --git a/charts/mlrun-ce/templates/config/mlrun-env-configmap.yaml b/charts/mlrun-ce/templates/config/mlrun-env-configmap.yaml index 2381993e..4ec44658 100644 --- a/charts/mlrun-ce/templates/config/mlrun-env-configmap.yaml +++ b/charts/mlrun-ce/templates/config/mlrun-env-configmap.yaml @@ -12,9 +12,9 @@ data: MLRUN_HTTPDB__PROJECTS__LEADER: mlrun MLRUN_HTTPDB__PROJECTS__FOLLOWERS: nuclio {{- if not .Values.global.infrastructure.aws.s3NonAnonymous }} - AWS_SECRET_ACCESS_KEY: {{ .Values.minio.rootPassword }} - AWS_ACCESS_KEY_ID: {{ .Values.minio.rootUser }} - AWS_ENDPOINT_URL_S3: {{ include "mlrun-ce.minio.service.url" . }} + AWS_SECRET_ACCESS_KEY: {{ include "mlrun-ce.s3.secretKey" . }} + AWS_ACCESS_KEY_ID: {{ include "mlrun-ce.s3.accessKey" . }} + AWS_ENDPOINT_URL_S3: {{ include "mlrun-ce.s3.service.url" . }} {{- end }} MLRUN_FUNCTION__SPEC__SERVICE_ACCOUNT__DEFAULT: {{ .Values.mlrun.api.functionSpecServiceAccountDefault | default "" | quote }} MLRUN_HTTPDB__REAL_PATH: s3:// diff --git a/charts/mlrun-ce/templates/pipelines/configmaps/pipeline-install-config.yaml b/charts/mlrun-ce/templates/pipelines/configmaps/pipeline-install-config.yaml index 872d2736..d1ec9aa8 100644 --- a/charts/mlrun-ce/templates/pipelines/configmaps/pipeline-install-config.yaml +++ b/charts/mlrun-ce/templates/pipelines/configmaps/pipeline-install-config.yaml @@ -1,13 +1,13 @@ {{- if .Values.pipelines.enabled -}} apiVersion: v1 data: - minioServiceHost: {{ include "mlrun-ce.minio-pipeline.service.url" . }} - minioServicePort: "{{ .Values.pipelines.minio.endpointPort }}" + objectStoreServiceHost: {{ include "mlrun-ce.s3.service.host" . }} + objectStoreServicePort: {{ include "mlrun-ce.s3.service.port" . | quote }} ConMaxLifeTime: 120s appName: pipeline appVersion: {{ .Values.pipelines.images.apiServer.tag }} autoUpdatePipelineDefaultVersion: "false" - bucketName: "{{ .Values.pipelines.minio.bucket }}" + bucketName: "{{ .Values.pipelines.s3.bucket }}" cacheDb: cachedb cacheImage: {{ .Values.pipelines.images.cacheImage.repository }}:{{ .Values.pipelines.images.cacheImage.tag }} cacheNodeRestrictions: "false" diff --git a/charts/mlrun-ce/templates/pipelines/configmaps/workflow-controller-configmap.yaml b/charts/mlrun-ce/templates/pipelines/configmaps/workflow-controller-configmap.yaml index 63cca9dd..34ef8277 100644 --- a/charts/mlrun-ce/templates/pipelines/configmaps/workflow-controller-configmap.yaml +++ b/charts/mlrun-ce/templates/pipelines/configmaps/workflow-controller-configmap.yaml @@ -9,8 +9,8 @@ data: artifactRepository: | archiveLogs: true s3: - endpoint: "{{ include "mlrun-ce.minio-pipeline.service.url" . }}:{{ .Values.pipelines.minio.endpointPort }}" - bucket: "{{ .Values.pipelines.minio.bucket }}" + endpoint: "{{ include "mlrun-ce.s3.service.host" . }}:{{ include "mlrun-ce.s3.service.port" . }}" + bucket: "{{ .Values.pipelines.s3.bucket }}" insecure: true accessKeySecret: name: mlpipeline-minio-artifact diff --git a/charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline-ui.yaml b/charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline-ui.yaml index 2218bc88..3d008149 100644 --- a/charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline-ui.yaml +++ b/charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline-ui.yaml @@ -33,9 +33,9 @@ spec: - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH value: /etc/config/viewer-pod-template.json - name: MINIO_HOST - value: {{ include "mlrun-ce.minio-pipeline.service.url" . }} + value: {{ include "mlrun-ce.s3.service.host" . }} - name: MINIO_PORT - value: "{{ .Values.pipelines.minio.endpointPort }}" + value: {{ include "mlrun-ce.s3.service.port" . | quote }} - name: MINIO_NAMESPACE # This is required because otherwise the namespace is appended to the MinIO hostname used to fetch artifacts, causing the fetch to fail value: "" diff --git a/charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline.yaml b/charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline.yaml index ddaa48c9..fb124aa8 100644 --- a/charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline.yaml +++ b/charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline.yaml @@ -61,9 +61,12 @@ spec: valueFrom: configMapKeyRef: name: pipeline-install-config - key: minioServiceHost + key: objectStoreServiceHost - name: OBJECTSTORECONFIG_PORT - value: "9000" + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: objectStoreServicePort - name: OBJECTSTORECONFIG_SECURE value: "false" - name: OBJECTSTORECONFIG_BUCKETNAME diff --git a/charts/mlrun-ce/templates/pipelines/secrets/mlpipeline-minio-artifact.yaml b/charts/mlrun-ce/templates/pipelines/secrets/mlpipeline-minio-artifact.yaml index 0fc8a151..669c9232 100644 --- a/charts/mlrun-ce/templates/pipelines/secrets/mlpipeline-minio-artifact.yaml +++ b/charts/mlrun-ce/templates/pipelines/secrets/mlpipeline-minio-artifact.yaml @@ -1,8 +1,8 @@ {{- if .Values.pipelines.enabled -}} apiVersion: v1 data: - accesskey: {{ .Values.pipelines.minio.accessKey | b64enc | quote }} - secretkey: {{ .Values.pipelines.minio.secretKey | b64enc | quote }} + accesskey: {{ include "mlrun-ce.s3.accessKey" . | b64enc | quote }} + secretkey: {{ include "mlrun-ce.s3.secretKey" . | b64enc | quote }} kind: Secret metadata: annotations: diff --git a/charts/mlrun-ce/templates/seaweedfs/seaweedfs-s3-config.yaml b/charts/mlrun-ce/templates/seaweedfs/seaweedfs-s3-config.yaml new file mode 100644 index 00000000..3015df54 --- /dev/null +++ b/charts/mlrun-ce/templates/seaweedfs/seaweedfs-s3-config.yaml @@ -0,0 +1,33 @@ +{{- if .Values.seaweedfs.enabled }} +{{- if .Values.seaweedfs.s3.enableAuth }} +apiVersion: v1 +kind: Secret +metadata: + name: seaweedfs-s3-config + labels: + {{- include "mlrun-ce.common.labels" . | nindent 4 }} +type: Opaque +stringData: + seaweedfs_s3_config: | + { + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": "{{ include "mlrun-ce.s3.accessKey" . }}", + "secretKey": "{{ include "mlrun-ce.s3.secretKey" . }}" + } + ], + "actions": [ + "Admin", + "Read", + "List", + "Tagging", + "Write" + ] + } + ] + } +{{- end }} +{{- end }} diff --git a/charts/mlrun-ce/templates/seaweedfs/seaweedfs-ui-ingress.yaml b/charts/mlrun-ce/templates/seaweedfs/seaweedfs-ui-ingress.yaml new file mode 100644 index 00000000..aff9d418 --- /dev/null +++ b/charts/mlrun-ce/templates/seaweedfs/seaweedfs-ui-ingress.yaml @@ -0,0 +1,42 @@ +{{- if .Values.seaweedfs.enabled }} +# ============================================================================= +# SeaweedFS Admin UI Ingress +# This ingress exposes the Admin UI for external access via ingress controller +# ============================================================================= + +{{- if and .Values.seaweedfs.admin .Values.seaweedfs.admin.enabled .Values.seaweedfs.adminService .Values.seaweedfs.adminService.ingress .Values.seaweedfs.adminService.ingress.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-seaweedfs-admin + namespace: {{ .Release.Namespace }} + labels: + {{- include "mlrun-ce.common.labels" . | nindent 4 }} + app.kubernetes.io/component: seaweedfs-admin-ui + {{- with .Values.seaweedfs.adminService.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.seaweedfs.adminService.ingress.className }} + ingressClassName: {{ .Values.seaweedfs.adminService.ingress.className }} + {{- end }} + rules: + - host: {{ .Values.seaweedfs.adminService.ingress.host }} + http: + paths: + - path: {{ .Values.seaweedfs.adminService.ingress.path | default "/" }} + pathType: {{ .Values.seaweedfs.adminService.ingress.pathType | default "Prefix" }} + backend: + service: + name: seaweedfs-admin-ui + port: + number: {{ .Values.seaweedfs.adminService.port | default 23646 }} + {{- if .Values.seaweedfs.adminService.ingress.tls }} + tls: + {{- toYaml .Values.seaweedfs.adminService.ingress.tls | nindent 4 }} + {{- end }} +{{- end }} + +{{- end }} diff --git a/charts/mlrun-ce/templates/seaweedfs/seaweedfs-ui-services.yaml b/charts/mlrun-ce/templates/seaweedfs/seaweedfs-ui-services.yaml new file mode 100644 index 00000000..c47c0b9b --- /dev/null +++ b/charts/mlrun-ce/templates/seaweedfs/seaweedfs-ui-services.yaml @@ -0,0 +1,35 @@ +{{- if .Values.seaweedfs.enabled }} +# ============================================================================= +# SeaweedFS Admin UI Service +# This service exposes the SeaweedFS Admin UI for external access, providing +# user & policy management functionality similar to MinIO Console IAM +# ============================================================================= + +{{- if and .Values.seaweedfs.admin .Values.seaweedfs.admin.enabled .Values.seaweedfs.adminService }} +--- +# Admin UI Service - User & policy management +# Similar to MinIO's Identity/Access Management (IAM) +apiVersion: v1 +kind: Service +metadata: + name: seaweedfs-admin-ui + namespace: {{ .Release.Namespace }} + labels: + {{- include "mlrun-ce.common.labels" . | nindent 4 }} + app.kubernetes.io/component: seaweedfs-admin-ui +spec: + type: {{ .Values.seaweedfs.adminService.type | default "NodePort" }} + ports: + - name: admin-ui + port: {{ .Values.seaweedfs.adminService.port | default 23646 }} + targetPort: {{ .Values.seaweedfs.admin.port | default 23646 }} + {{- if .Values.seaweedfs.adminService.nodePort }} + nodePort: {{ .Values.seaweedfs.adminService.nodePort }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/name: seaweedfs + app.kubernetes.io/component: admin +{{- end }} + +{{- end }} diff --git a/charts/mlrun-ce/values.yaml b/charts/mlrun-ce/values.yaml index 35d0c3b3..3b2c0b2a 100644 --- a/charts/mlrun-ce/values.yaml +++ b/charts/mlrun-ce/values.yaml @@ -2,7 +2,7 @@ global: # External host/ip to reach the k8s node. This might take various values if k8s is run in a VM or a cloud env externalHostAddress: localhost - registry: + registry: &userRegistry url: mustprovide secretName: nuclio: @@ -21,6 +21,8 @@ global: domainNameCertificate: ~ nuclio: + global: + registry: *userRegistry # coupled with mlrun.nuclio.dashboardName template in mlrun chart fullnameOverride: nuclio controller: @@ -71,6 +73,8 @@ nuclio: mlrun: # set the type of filesystem to use: filesystem, s3 enabled: true + global: + registry: *userRegistry defaultFunctionPodResources: limits: cpu: "2" @@ -270,33 +274,165 @@ mpi-operator: deployment: create: true -minio: +seaweedfs: enabled: true - rootUser: minio - rootPassword: minio123 - ingress: - enabled: false - mode: standalone - replicas: 1 - resources: - requests: - memory: 0.5Gi - service: - type: NodePort - port: 9000 - nodePort: 30080 - consoleService: - type: NodePort - port: 9001 - nodePort: 30090 - persistence: + global: + createClusterRole: true + imageName: chrislusf/seaweedfs + imagePullPolicy: IfNotPresent + imagePullSecrets: "" + restartPolicy: Always + loggingLevel: 1 + enableSecurity: false + serviceAccountName: "seaweedfs" + automountServiceAccountToken: true + monitoring: + enabled: false + enableReplication: false + replicationPlacement: "000" + extraEnvironmentVars: + WEED_CLUSTER_DEFAULT: "sw" + + # Master server - metadata management + master: + enabled: true + replicas: 1 + port: 9333 + grpcPort: 19333 + metricsPort: 9327 + ipBind: "0.0.0.0" + volumeSizeLimitMB: 1000 + defaultReplication: "000" + disableHttp: false + # Storage configuration - use PVC for persistence + data: + type: "persistentVolumeClaim" + size: "1Gi" + storageClass: "" + logs: + type: "emptyDir" + # Node selector - adjust based on your cluster architecture + # For AMD64: kubernetes.io/arch: amd64 + # For ARM64 (Apple Silicon): kubernetes.io/arch: arm64 + nodeSelector: "" + resources: {} + tolerations: "" + + # Volume server - actual data storage + volume: + enabled: true + replicas: 1 + port: 8080 + grpcPort: 18080 + metricsPort: 9327 + ipBind: "0.0.0.0" + minFreeSpacePercent: 1 + # Storage configuration + dataDirs: + - name: data + type: "persistentVolumeClaim" + size: "10Gi" + storageClass: "" + maxVolumes: 0 + idx: {} + logs: + type: "emptyDir" + nodeSelector: "" + resources: {} + tolerations: "" + + # Filer server - file system interface + filer: enabled: true - size: 1Gi - fullnameOverride: minio - buckets: - - name: mlrun - policy: none - purge: false + replicas: 1 + port: 8888 + grpcPort: 18888 + metricsPort: 9327 + ipBind: "0.0.0.0" + defaultReplicaPlacement: "000" + disableDirListing: false + dirListLimit: 100000 + disableHttp: false + encryptVolumeData: false + # Storage configuration + data: + type: "persistentVolumeClaim" + size: "1Gi" + storageClass: "" + logs: + type: "emptyDir" + # S3 configuration on filer + s3: + enabled: true + port: 8333 + domainName: "" + allowEmptyFolder: false + nodeSelector: "" + resources: {} + tolerations: "" + + # S3 API gateway - MLRun connects to this endpoint + s3: + enabled: true + replicas: 1 + bindAddress: "0.0.0.0" + port: 8333 + httpsPort: 0 + metricsPort: 9327 + # Enable S3 authentication + enableAuth: true + # S3 credentials configuration + # If enableAuth is true, create a secret named 'seaweedfs-s3-config' + # with the S3 authentication JSON configuration + existingConfigSecret: "seaweedfs-s3-config" + domainName: "" + auditLogConfig: {} + nodeSelector: "" + resources: {} + tolerations: "" + + # Admin server - user and policy management UI (similar to MinIO Console IAM) + admin: + enabled: true + replicas: 1 + port: 23646 + grpcPort: 33646 + # Admin authentication + secret: + existingSecret: "" + userKey: "" + pwKey: "" + adminUser: "admin" + adminPassword: "admin123" # Set password to enable authentication + # Data directory for admin configuration + dataDir: "/data" + data: + type: "persistentVolumeClaim" + size: "1Gi" + storageClass: "" + logs: + type: "emptyDir" + nodeSelector: "" + resources: {} + tolerations: "" + + # ============================================================================= + # SeaweedFS Admin UI Service (NodePort) - Expose Admin UI for external access + # ============================================================================= + # Admin UI provides user & policy management (similar to MinIO Console IAM) + # ============================================================================= + adminService: + type: NodePort + port: 23646 + nodePort: 30093 + ingress: + enabled: false + className: "" + host: "" + path: / + pathType: Prefix + annotations: {} + tls: [] spark-operator: enabled: true @@ -347,7 +483,7 @@ pipelines: runAsGroup: 1001 fsGroup: 1001 fsGroupChangePolicy: OnRootMismatch - minio: + s3: enabled: true accessKey: "minio" secretKey: "minio123" @@ -403,9 +539,6 @@ pipelines: cacheImage: repository: gcr.io/google-containers/busybox tag: latest - minio: - repository: minio/minio - tag: "RELEASE.2025-10-15T17-29-55Z" kube-prometheus-stack: fullnameOverride: monitoring @@ -455,7 +588,6 @@ kube-prometheus-stack: nodePort: 30020 kube-state-metrics: fullnameOverride: state-metrics - prometheus-node-exporter: fullnameOverride: node-exporter hostNetwork: false