diff --git a/examples/terraform/proxmox/files/cilium-install.yaml b/examples/terraform/proxmox/files/cilium-install.yaml new file mode 100644 index 0000000..d305c59 --- /dev/null +++ b/examples/terraform/proxmox/files/cilium-install.yaml @@ -0,0 +1,84 @@ +cluster: + inlineManifests: + - name: cilium-install + contents: | + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cilium-install + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: cilium-install + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cilium-install + namespace: kube-system + --- + apiVersion: batch/v1 + kind: Job + metadata: + name: cilium-install + namespace: kube-system + spec: + backoffLimit: 10 + template: + metadata: + labels: + app: cilium-install + spec: + restartPolicy: OnFailure + tolerations: + - operator: Exists + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - effect: PreferNoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoExecute + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: PreferNoSchedule + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccount: cilium-install + serviceAccountName: cilium-install + hostNetwork: true + containers: + - name: cilium-install + image: quay.io/cilium/cilium-cli-ci:latest + env: + - name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: KUBERNETES_SERVICE_PORT + value: "6443" + command: + - cilium + - install + - --helm-set=ipam.mode=kubernetes + - --helm-set=kubeProxyReplacement=disabled + - --helm-set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID} + - --helm-set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE} + - --helm-set=cgroup.autoMount.enabled=false + - --helm-set=cgroup.hostRoot=/sys/fs/cgroup diff --git a/examples/terraform/proxmox/files/no-cni.yaml b/examples/terraform/proxmox/files/no-cni.yaml new file mode 100644 index 0000000..60aaf0e --- /dev/null +++ b/examples/terraform/proxmox/files/no-cni.yaml @@ -0,0 +1,4 @@ +cluster: + network: + cni: + name: none diff --git a/examples/terraform/proxmox/main.tf b/examples/terraform/proxmox/main.tf new file mode 100644 index 0000000..040da66 --- /dev/null +++ b/examples/terraform/proxmox/main.tf @@ -0,0 +1,215 @@ +locals { + envs = { for tuple in regexall("(.*)=(.*)", file("${path.module}/../.env")) : tuple[0] => sensitive(tuple[1]) } +} + +resource "random_id" "cluster_id" { + byte_length = 32 +} + +resource "talos_machine_secrets" "this" {} + +resource "proxmox_vm_qemu" "talos_control_plane_node" { + count = var.control_plane_nodes_count + + name = "${local.envs["CLUSTER"]}-controlplane-${count.index + 1}" + vmid = var.pve_vmid_start + count.index + clone = var.pve_talos_template_name + full_clone = true + target_node = var.pve_node + tags = var.pve_tags + + agent = 1 + agent_timeout = var.pve_agent_timeout + + bios = "ovmf" + + memory = floor(var.total_control_plane_memory / var.control_plane_nodes_count) + cores = var.control_plane_cores + sockets = var.control_plane_sockets + scsihw = "virtio-scsi-single" + onboot = true + + efidisk { + efitype = "4m" + storage = var.pve_boot_disk_storage + } + + disks { + ide { + ide0 { + cdrom { + iso = var.pve_talos_iso + } + } + } + + scsi { + scsi0 { + disk { + size = var.pve_boot_disk_size + storage = var.pve_boot_disk_storage + emulatessd = true + discard = true + } + } + } + } + + network { + bridge = var.pve_bridge + tag = var.pve_vlan_tag + model = "virtio" + firewall = false + } +} + +resource "proxmox_vm_qemu" "talos_worker_node" { + count = var.worker_nodes_count + + name = "${local.envs["CLUSTER"]}-workplane-${count.index + 1}" + vmid = var.pve_vmid_start + var.control_plane_nodes_count + count.index + clone = var.pve_talos_template_name + full_clone = true + target_node = var.pve_node + tags = var.pve_tags + + agent = 1 + agent_timeout = var.pve_agent_timeout + + bios = "ovmf" + boot = "order=ide0;scsi0;net0" + + memory = floor(var.total_work_plane_memory / var.worker_nodes_count) + cores = var.worker_cores + sockets = var.worker_sockets + scsihw = "virtio-scsi-single" + onboot = true + + efidisk { + efitype = "4m" + storage = var.pve_boot_disk_storage + } + + disks { + ide { + ide0 { + cdrom { + iso = var.pve_talos_iso + } + } + } + + scsi { + scsi0 { + disk { + size = var.pve_boot_disk_size + storage = var.pve_boot_disk_storage + emulatessd = true + discard = true + } + } + + scsi1 { + passthrough { + file = tolist(var.pve_passthrough_disks)[count.index] + } + } + } + } + + network { + bridge = var.pve_bridge + tag = var.pve_vlan_tag + model = "virtio" + firewall = false + } +} + +data "talos_client_configuration" "this" { + cluster_name = local.envs["CLUSTER"] + client_configuration = talos_machine_secrets.this.client_configuration + endpoints = [for i,v in proxmox_vm_qemu.talos_control_plane_node: v.default_ipv4_address] + nodes = [for i,v in proxmox_vm_qemu.talos_control_plane_node: v.default_ipv4_address] +} + +data "talos_machine_configuration" "controlplane" { + cluster_name = local.envs["CLUSTER"] + cluster_endpoint = "https://${proxmox_vm_qemu.talos_control_plane_node[0].default_ipv4_address}:6443" + machine_type = "controlplane" + machine_secrets = talos_machine_secrets.this.machine_secrets + config_patches = [ + templatefile("${path.module}/templates/installer.yaml.tmpl", { + install_image = var.talos_install_image + }), + file("${path.module}/files/no-cni.yaml"), + ] +} + +resource "talos_machine_configuration_apply" "controlplane" { + for_each = {for i,v in proxmox_vm_qemu.talos_control_plane_node: i => v} + + client_configuration = talos_machine_secrets.this.client_configuration + machine_configuration_input = data.talos_machine_configuration.controlplane.machine_configuration + node = each.value.default_ipv4_address + + config_patches = [ + yamlencode({ + machine = { + install = { + disk = "/dev/sda" + } + } + }), + file("${path.module}/files/cilium-install.yaml"), + ] +} + +resource "talos_machine_bootstrap" "controlplane" { + depends_on = [ + talos_machine_configuration_apply.controlplane + ] + node = proxmox_vm_qemu.talos_control_plane_node[0].default_ipv4_address + client_configuration = talos_machine_secrets.this.client_configuration +} + +data "talos_machine_configuration" "worker" { + cluster_name = local.envs["CLUSTER"] + cluster_endpoint = "https://${proxmox_vm_qemu.talos_control_plane_node[0].default_ipv4_address}:6443" + machine_type = "worker" + machine_secrets = talos_machine_secrets.this.machine_secrets + depends_on = [proxmox_vm_qemu.talos_control_plane_node[0]] + config_patches = [ + templatefile("${path.module}/templates/installer.yaml.tmpl", { + install_image = var.talos_install_image + }), + + file("${path.module}/files/no-cni.yaml"), + ] +} + +resource "talos_machine_configuration_apply" "worker" { + for_each = {for i,v in proxmox_vm_qemu.talos_worker_node: i => v} + + client_configuration = talos_machine_secrets.this.client_configuration + machine_configuration_input = data.talos_machine_configuration.worker.machine_configuration + node = each.value.default_ipv4_address + + config_patches = [ + + yamlencode({ + machine = { + install = { + disk = "/dev/sda" + }, + } + }), + + ] +} + +data "talos_cluster_kubeconfig" "this" { + depends_on = [talos_machine_bootstrap.controlplane] + client_configuration = talos_machine_secrets.this.client_configuration + node = proxmox_vm_qemu.talos_control_plane_node[0].default_ipv4_address +} + diff --git a/examples/terraform/proxmox/output.tf b/examples/terraform/proxmox/output.tf new file mode 100644 index 0000000..c06cf74 --- /dev/null +++ b/examples/terraform/proxmox/output.tf @@ -0,0 +1,20 @@ +output "talosconfig" { + value = data.talos_client_configuration.this.talos_config + sensitive = true +} + +output "kubeconfig" { + value = data.talos_cluster_kubeconfig.this.kubeconfig_raw + sensitive = true +} + +output "controlplaneconfig" { + value = yamlencode(data.talos_machine_configuration.controlplane) + sensitive = true +} + +output "workerconfig" { + value = yamlencode(data.talos_machine_configuration.worker) + sensitive = true +} + diff --git a/examples/terraform/proxmox/providers.tf b/examples/terraform/proxmox/providers.tf new file mode 100644 index 0000000..9781758 --- /dev/null +++ b/examples/terraform/proxmox/providers.tf @@ -0,0 +1,4 @@ +provider "proxmox" {} +provider "random" {} +provider "tls" {} +provider "talos" {} diff --git a/examples/terraform/proxmox/templates/installer.yaml.tmpl b/examples/terraform/proxmox/templates/installer.yaml.tmpl new file mode 100644 index 0000000..0fb506a --- /dev/null +++ b/examples/terraform/proxmox/templates/installer.yaml.tmpl @@ -0,0 +1,3 @@ +machine: + install: + image: ${install_image} diff --git a/examples/terraform/proxmox/variables.tf b/examples/terraform/proxmox/variables.tf new file mode 100644 index 0000000..3007bc5 --- /dev/null +++ b/examples/terraform/proxmox/variables.tf @@ -0,0 +1,120 @@ +variable "pve_node" { + description = "The name of the PVE node to put Talos nodes on" + type = string +} + +variable "control_plane_nodes_count" { + description = "Number of control plane nodes" + type = number + default = 3 +} + +variable "total_control_plane_memory" { + description = "Total memory to be split up amongst control plane nodes" + type = number + default = 12 * 1024 +} + +variable "control_plane_cores" { + description = "How many cores to be given to each control plane node" + type = number + default = 2 +} + +variable "control_plane_sockets" { + description = "How many sockets to be given to each control plane node" + type = number + default = 2 +} + +variable "total_work_plane_memory" { + description = "Total memory to be split up amongst control plane nodes" + type = number + default = 12 * 1024 +} + +variable "worker_nodes_count" { + description = "Number of worker nodes" + type = number + default = 3 +} + +variable "worker_cores" { + description = "How many cores to be given to each work plane node" + type = number + default = 2 +} + +variable "worker_sockets" { + description = "How many sockets to be given to each work plane node" + type = number + default = 2 +} + +variable "pve_talos_template_name" { + description = "The name of the template to clone" + type = string +} + +variable "pve_bridge" { + description = "The Linux virtual bridge on which nodes' NICs will be assigned" + type = string + default = "vmbr0" +} + +variable "pve_vlan_tag" { + description = "The VLAN tag to assign to the nodes' NICs" + type = number + default = -1 +} + +variable "pve_tags" { + description = "Any tags to add to the nodes" + type = string + default = "" +} + +variable "pve_boot_disk_storage" { + description = "Where to store nodes' boot disks" + type = string +} + +variable "pve_boot_disk_size" { + description = "Storage size of nodes' boot disks" + type = string + default = "12G" +} + +variable "pve_talos_iso" { + description = "The ISO PVE should boot nodes from (this must be downloaded to PVE already)" + type = string +} + +variable "pve_passthrough_disks" { + description = "A list of physical disks to pass through to worker nodes" + type = set(string) + default = [] +} + +variable "pve_vmid_start" { + description = "The ID for VMs to start from" + type = number + default = 100 +} + +variable "pve_agent_timeout" { + description = "How many seconds to wait for VMs' guest agents" + type = number + default = 60 +} + +variable "talos_cluster_name" { + description = "The name of your Talos cluster" + type = string + default = "pve-talos" +} + +variable "talos_install_image" { + description = "The Talos image to install from (get this from factory.talos.dev)" + type = string +} diff --git a/examples/terraform/proxmox/vars/.gitignore b/examples/terraform/proxmox/vars/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/examples/terraform/proxmox/vars/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/examples/terraform/proxmox/versions.tf b/examples/terraform/proxmox/versions.tf new file mode 100644 index 0000000..df4bd68 --- /dev/null +++ b/examples/terraform/proxmox/versions.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + proxmox = { + source = "registry.opentofu.org/telmate/proxmox" + version = "3.0.1-rc3" + } + random = { + source = "registry.opentofu.org/hashicorp/random" + version = "3.5.1" + } + tls = { + source = "registry.opentofu.org/hashicorp/tls" + version = "4.0.4" + } + talos = { + source = "registry.opentofu.org/siderolabs/talos" + version = "0.5.0" + } + } +}