From 5a07d3c13d404d50723f6cba8b8734fca8715ac8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 11 Mar 2022 10:32:30 +0100 Subject: [PATCH 1/9] The || && construct never worked without kubectl --- kafka/10broker-config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 2635aae3..0d023617 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -14,9 +14,9 @@ data: LABELS="kafka-broker-id=$KAFKA_BROKER_ID" ANNOTATIONS="" - hash kubectl 2>/dev/null || { + if ! command -v kubectl >/dev/null; then SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/") - } && { + else ZONE=$(kubectl get node "$NODE_NAME" -o=go-template='{{index .metadata.labels "topology.kubernetes.io/zone"}}') if [ "x$ZONE" == "x" ]; then SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/") @@ -37,7 +37,7 @@ data: if [ ! -z "$ANNOTATIONS" ]; then kubectl -n $POD_NAMESPACE annotate pod $POD_NAME $ANNOTATIONS || echo "Failed to annotate $POD_NAMESPACE.$POD_NAME - RBAC issue?" fi - } + fi printf '%s\n' "${SEDS[@]}" | sed -f - /etc/kafka-configmap/server.properties > /etc/kafka/server.properties.tmp [ $? -eq 0 ] && mv /etc/kafka/server.properties.tmp /etc/kafka/server.properties ln -s /etc/kafka/server.properties /etc/kafka/server.properties.$POD_NAME From 1758ba037da161a797f8a7f920910ed673c714b7 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 11 Mar 2022 11:17:29 +0100 Subject: [PATCH 2/9] Advertisement can probably never be omitted when using kraft and we can move it out of the if because init-config without kubectl nerver worked anyway --- kafka/10broker-config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 0d023617..3e8c00ef 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -14,6 +14,12 @@ data: LABELS="kafka-broker-id=$KAFKA_BROKER_ID" ANNOTATIONS="" + [ -z "$ADVERTISE_ADDR" ] && echo "ADVERTISE_ADDR is empty, will advertise detected DNS name" + OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') + OUTSIDE_PORT=$((32400 + ${KAFKA_BROKER_ID})) + SEDS+=("s|#init#advertised.listeners=PLAINTEXT://#init#|advertised.listeners=PLAINTEXT://${ADVERTISE_ADDR}:9092,CONTROLLER://${ADVERTISE_ADDR}:9093,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|") + ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" + if ! command -v kubectl >/dev/null; then SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/") else @@ -25,12 +31,6 @@ data: LABELS="$LABELS kafka-broker-rack=$ZONE" fi - [ -z "$ADVERTISE_ADDR" ] && echo "ADVERTISE_ADDR is empty, will advertise detected DNS name" - OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') - OUTSIDE_PORT=$((32400 + ${KAFKA_BROKER_ID})) - SEDS+=("s|#init#advertised.listeners=PLAINTEXT://#init#|advertised.listeners=PLAINTEXT://${ADVERTISE_ADDR}:9092,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|") - ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" - if [ ! -z "$LABELS" ]; then kubectl -n $POD_NAMESPACE label pod $POD_NAME $LABELS || echo "Failed to label $POD_NAMESPACE.$POD_NAME - RBAC issue?" fi @@ -73,7 +73,7 @@ data: ############################# Socket Server Settings ############################# - # The address the socket server listens on. It will get the value returned from + # The address the socket server listens on. It will get the value returned from # java.net.InetAddress.getCanonicalHostName() if not configured. # FORMAT: # listeners = listener_name://host_name:port @@ -82,7 +82,7 @@ data: #listeners=PLAINTEXT://:9092 listeners=PLAINTEXT://:9092,OUTSIDE://:9094 - # Hostname and port the broker will advertise to producers and consumers. If not set, + # Hostname and port the broker will advertise to producers and consumers. If not set, # it uses the value for "listeners" if configured. Otherwise, it will use the value # returned from java.net.InetAddress.getCanonicalHostName(). #advertised.listeners=PLAINTEXT://your.host.name:9092 From b8f876f185939ef4f8250ab701b1f8b8574ce276 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 11 Mar 2022 11:22:24 +0100 Subject: [PATCH 3/9] Clarifies the ADVERTISE_ADDR idea --- kafka/10broker-config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 3e8c00ef..4f212f70 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -14,7 +14,9 @@ data: LABELS="kafka-broker-id=$KAFKA_BROKER_ID" ANNOTATIONS="" - [ -z "$ADVERTISE_ADDR" ] && echo "ADVERTISE_ADDR is empty, will advertise detected DNS name" + # A custom ADVERTISE_ADDR can be set as env on the init container using for example value: $(POD_NAME).kafka.$(POD_NAMESPACE).custom.cluster.dns + [ -n "$ADVERTISE_ADDR" ] && echo "ADVERTISE_ADDR=$ADVERTISE_ADDR" || echo "ADVERTISE_ADDR is empty, Kafka will detect a hostname to advertise" + OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') OUTSIDE_PORT=$((32400 + ${KAFKA_BROKER_ID})) SEDS+=("s|#init#advertised.listeners=PLAINTEXT://#init#|advertised.listeners=PLAINTEXT://${ADVERTISE_ADDR}:9092,CONTROLLER://${ADVERTISE_ADDR}:9093,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|") From 95cbf1a9406eb0aaf91bb7129a33cdf790636e03 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 11 Mar 2022 12:19:48 +0100 Subject: [PATCH 4/9] Makes outside host opt-in and provides configurability for #187 --- kafka/10broker-config.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 4f212f70..061193c3 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -17,8 +17,13 @@ data: # A custom ADVERTISE_ADDR can be set as env on the init container using for example value: $(POD_NAME).kafka.$(POD_NAMESPACE).custom.cluster.dns [ -n "$ADVERTISE_ADDR" ] && echo "ADVERTISE_ADDR=$ADVERTISE_ADDR" || echo "ADVERTISE_ADDR is empty, Kafka will detect a hostname to advertise" - OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') - OUTSIDE_PORT=$((32400 + ${KAFKA_BROKER_ID})) + ADVERTISED_LISTENERS="PLAINTEXT://${ADVERTISE_ADDR}:9092" + + # OUTSIDE_HOST and OUTSIDE_PORT can be set as envs, or set OUTSIDE_NODE_JSONPATH to get the value from the node, for example: '{.status.addresses[?(@.type=="InternalIP")].address}' + [ -z "$OUTSIDE_HOST_JSONPATH" ] || OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath="$OUTSIDE_HOST_JSONPATH") + [ -n "$OUTSIDE_PORT" ] || OUTSIDE_PORT=$((32400 + ${KAFKA_BROKER_ID})) + [ -z "$OUTSITE_HOST" ] || ADVERTISED_LISTENERS="$ADVERTISED_LISTENERS,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}" + SEDS+=("s|#init#advertised.listeners=PLAINTEXT://#init#|advertised.listeners=PLAINTEXT://${ADVERTISE_ADDR}:9092,CONTROLLER://${ADVERTISE_ADDR}:9093,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|") ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" From dc2efad40c3543795c6d4590a293b7ecc30f9db9 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 11 Mar 2022 12:21:00 +0100 Subject: [PATCH 5/9] A valid image is plainly a vanilla Kafka distribution in workdir, and from now on it should always be nonroot --- kafka/50kafka.yml | 7 +++-- nonroot/entrypoint-from-image.yaml | 2 -- nonroot/fsgroup-65534.yaml | 4 --- nonroot/kustomization.yaml | 44 ---------------------------- nonroot/nonroot-image-kafka.yaml | 15 ---------- nonroot/nonroot-image-zookeeper.yaml | 31 -------------------- 6 files changed, 5 insertions(+), 98 deletions(-) delete mode 100644 nonroot/entrypoint-from-image.yaml delete mode 100644 nonroot/fsgroup-65534.yaml delete mode 100644 nonroot/kustomization.yaml delete mode 100644 nonroot/nonroot-image-kafka.yaml delete mode 100644 nonroot/nonroot-image-zookeeper.yaml diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index a877f709..b55af7a1 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -18,9 +18,11 @@ spec: annotations: spec: terminationGracePeriodSeconds: 30 + securityContext: + fsGroup: 65534 initContainers: - name: init-config - image: solsson/kafka:initutils@sha256:0cd27e24dea0b27e17e72cd05259a4de394d527473c6707e89f26b87e356a04d + image: yolean/kafka:latest env: - name: NODE_NAME valueFrom: @@ -44,7 +46,7 @@ spec: mountPath: /opt/kafka/libs/extensions containers: - name: broker - image: solsson/kafka:2.8.0@sha256:69274245ad16fba37e5b277335502a02e41c0efeae3ccbc3374f6cf624b58347 + image: yolean/kafka:latest env: - name: POD_NAME valueFrom: @@ -70,6 +72,7 @@ spec: command: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties.$(POD_NAME) + args: [] lifecycle: preStop: exec: diff --git a/nonroot/entrypoint-from-image.yaml b/nonroot/entrypoint-from-image.yaml deleted file mode 100644 index fe110c35..00000000 --- a/nonroot/entrypoint-from-image.yaml +++ /dev/null @@ -1,2 +0,0 @@ -- op: remove - path: /spec/template/spec/containers/0/command diff --git a/nonroot/fsgroup-65534.yaml b/nonroot/fsgroup-65534.yaml deleted file mode 100644 index 5ebd6cba..00000000 --- a/nonroot/fsgroup-65534.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- op: add - path: /spec/template/spec/securityContext - value: - fsGroup: 65534 diff --git a/nonroot/kustomization.yaml b/nonroot/kustomization.yaml deleted file mode 100644 index a3526dfb..00000000 --- a/nonroot/kustomization.yaml +++ /dev/null @@ -1,44 +0,0 @@ -bases: -- ../rbac-namespace-default -- ../kafka -- ../zookeeper -patchesStrategicMerge: -- nonroot-image-kafka.yaml -- nonroot-image-zookeeper.yaml -patchesJson6902: -- target: - group: apps - version: v1 - kind: StatefulSet - name: kafka - path: fsgroup-65534.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: pzoo - path: fsgroup-65534.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: zoo - path: fsgroup-65534.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: kafka - path: entrypoint-from-image.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: pzoo - path: entrypoint-from-image.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: zoo - path: entrypoint-from-image.yaml diff --git a/nonroot/nonroot-image-kafka.yaml b/nonroot/nonroot-image-kafka.yaml deleted file mode 100644 index 292b21fa..00000000 --- a/nonroot/nonroot-image-kafka.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: kafka -spec: - template: - spec: - initContainers: - - name: init-config - image: solsson/kafka:initutils-nonroot@sha256:c239d8c5c767cfe9ffd81cce96af9a61e487400d167f58a7e89478c958ca73b9 - containers: - - name: broker - image: solsson/kafka:2.8.0-kafka-server-start@sha256:825a7e70563a786a43c327f06257c3c502f8bc191d4fb91cf53fdc07aa69703f - args: - - /etc/kafka/server.properties.$(POD_NAME) diff --git a/nonroot/nonroot-image-zookeeper.yaml b/nonroot/nonroot-image-zookeeper.yaml deleted file mode 100644 index eceb7353..00000000 --- a/nonroot/nonroot-image-zookeeper.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: pzoo -spec: - template: - spec: - initContainers: - - name: init-config - image: solsson/kafka:initutils-nonroot@sha256:c239d8c5c767cfe9ffd81cce96af9a61e487400d167f58a7e89478c958ca73b9 - containers: - - name: zookeeper - image: solsson/kafka:2.8.0-zookeeper-server-start@sha256:88da70e37b878f5a6edfb3de206f7155b430c605cf956735741d5a011a10b7e9 - args: - - /etc/kafka/zookeeper.properties.scale-$(REPLICAS).$(POD_NAME) ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: zoo -spec: - template: - spec: - initContainers: - - name: init-config - image: solsson/kafka:initutils-nonroot@sha256:c239d8c5c767cfe9ffd81cce96af9a61e487400d167f58a7e89478c958ca73b9 - containers: - - name: zookeeper - image: solsson/kafka:2.8.0-zookeeper-server-start@sha256:88da70e37b878f5a6edfb3de206f7155b430c605cf956735741d5a011a10b7e9 - args: - - /etc/kafka/zookeeper.properties.scale-$(REPLICAS).$(POD_NAME) From 19e04484e80c3b0bf841e2bad6b742f2620f3a65 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 11 Mar 2022 09:43:35 +0100 Subject: [PATCH 6/9] Essential config for KRaft --- kafka/10broker-config.yml | 62 +++++++++++++------- kafka/50kafka.yml | 14 +++++ variants/scale-1/kafka-scale1-overrides.json | 2 - variants/scale-1/kustomization.yaml | 3 +- 4 files changed, 56 insertions(+), 25 deletions(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 061193c3..66197081 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -10,7 +10,6 @@ data: cp /etc/kafka-configmap/log4j.properties /etc/kafka/ KAFKA_BROKER_ID=${HOSTNAME##*-} - SEDS=("s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/") LABELS="kafka-broker-id=$KAFKA_BROKER_ID" ANNOTATIONS="" @@ -24,7 +23,19 @@ data: [ -n "$OUTSIDE_PORT" ] || OUTSIDE_PORT=$((32400 + ${KAFKA_BROKER_ID})) [ -z "$OUTSITE_HOST" ] || ADVERTISED_LISTENERS="$ADVERTISED_LISTENERS,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}" - SEDS+=("s|#init#advertised.listeners=PLAINTEXT://#init#|advertised.listeners=PLAINTEXT://${ADVERTISE_ADDR}:9092,CONTROLLER://${ADVERTISE_ADDR}:9093,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|") + if grep 'node.id' /etc/kafka-configmap/server.properties; then + echo "KRaft config detected" + SEDS=("s/#init#node.id=#init#/node.id=$KAFKA_BROKER_ID/") + SEDS+=("s/#init#controller.quorum.voters=#init#/controller.quorum.voters=$KAFKA_BROKER_ID@${ADVERTISE_ADDR}:9093/") + # We don't need to, actually should not, advertise controllers + # ADVERTISED_LISTENERS="$ADVERTISED_LISTENERS,CONTROLLER://${ADVERTISE_ADDR}:9093" + else + SEDS=("s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/") + fi + + [ -n "$OUTSIDE_HOST" ] || SEDS+=("s|,OUTSIDE://:9094||") + + SEDS+=("s|#init#advertised.listeners=PLAINTEXT://#init#|advertised.listeners=$ADVERTISED_LISTENERS|") ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" if ! command -v kubectl >/dev/null; then @@ -49,12 +60,26 @@ data: [ $? -eq 0 ] && mv /etc/kafka/server.properties.tmp /etc/kafka/server.properties ln -s /etc/kafka/server.properties /etc/kafka/server.properties.$POD_NAME + init-kraft.sh: |- + #!/bin/bash + set -e + set -x + if [ -f /var/lib/kafka/data/kraft-combined/meta.properties ]; then + echo "Storage dir appears initialized already" + elif [ ! -f ./bin/kafka-storage.sh ]; then + echo "./bin/kafka-storage.sh not found, unable to set up KRaft storage directories" + else + echo "See https://github.com/apache/kafka/tree/trunk/config/kraft#format-storage-directories" + UUID=$(./bin/kafka-storage.sh random-uuid) + ./bin/kafka-storage.sh format -t $UUID -c /etc/kafka/server.properties + fi + server.properties: |- ############################# Log Basics ############################# # A comma seperated list of directories under which to store log files # Overrides log.dir - log.dirs=/var/lib/kafka/data/topics + log.dirs=/var/lib/kafka/data/kraft-combined # The default number of log partitions per topic. More partitions allow greater # parallelism for consumption, but this will also result in more files across @@ -73,8 +98,12 @@ data: ############################# Server Basics ############################# + # The role of this server. Setting this puts us in KRaft mode + process.roles=broker,controller + # The id of the broker. This must be set to a unique integer for each broker. - #init#broker.id=#init# + #init#node.id=#init# + #init#controller.quorum.voters=#init# #init#broker.rack=#init# @@ -87,17 +116,21 @@ data: # EXAMPLE: # listeners = PLAINTEXT://your.host.name:9092 #listeners=PLAINTEXT://:9092 - listeners=PLAINTEXT://:9092,OUTSIDE://:9094 + listeners=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094 # Hostname and port the broker will advertise to producers and consumers. If not set, # it uses the value for "listeners" if configured. Otherwise, it will use the value # returned from java.net.InetAddress.getCanonicalHostName(). - #advertised.listeners=PLAINTEXT://your.host.name:9092 + #advertised.listeners=PLAINTEXT://your.host.name:9092,CONTROLLER://:9093 #init#advertised.listeners=PLAINTEXT://#init# + # Listener, host name, and port for the controller to advertise to the brokers. If + # this server is a controller, this listener must be configured. + controller.listener.names=CONTROLLER + # Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details - #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL - listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL,OUTSIDE:PLAINTEXT + #listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL,OUTSIDE:PLAINTEXT inter.broker.listener.name=PLAINTEXT # The number of threads that the server uses for receiving requests from the network and sending responses to the network @@ -163,19 +196,6 @@ data: # to the retention policies #log.retention.check.interval.ms=300000 - ############################# Zookeeper ############################# - - # Zookeeper connection string (see zookeeper docs for details). - # This is a comma separated host:port pairs, each corresponding to a zk - # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". - # You can also append an optional chroot string to the urls to specify the - # root directory for all kafka znodes. - zookeeper.connect=zookeeper:2181 - - # Timeout in ms for connecting to zookeeper - #zookeeper.connection.timeout.ms=18000 - - ############################# Group Coordinator Settings ############################# # The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index b55af7a1..e66b085c 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -44,6 +44,18 @@ spec: mountPath: /etc/kafka - name: extensions mountPath: /opt/kafka/libs/extensions + - name: init-kraft + image: yolean/kafka:latest + command: ['/bin/bash', '/etc/kafka-configmap/init-kraft.sh'] + volumeMounts: + - name: configmap + mountPath: /etc/kafka-configmap + - name: config + mountPath: /etc/kafka + - name: data + mountPath: /var/lib/kafka/data + - name: extensions + mountPath: /opt/kafka/libs/extensions containers: - name: broker image: yolean/kafka:latest @@ -65,6 +77,8 @@ spec: ports: - name: inside containerPort: 9092 + - name: controller + containerPort: 9093 - name: outside containerPort: 9094 - name: jmx diff --git a/variants/scale-1/kafka-scale1-overrides.json b/variants/scale-1/kafka-scale1-overrides.json index b4dcceca..13187671 100644 --- a/variants/scale-1/kafka-scale1-overrides.json +++ b/variants/scale-1/kafka-scale1-overrides.json @@ -1,6 +1,4 @@ [ - {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "zookeeper.connect=zoo-0.zoo.$(POD_NAMESPACE).svc.cluster.local:2181" }, {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "default.replication.factor=1"}, {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, diff --git a/variants/scale-1/kustomization.yaml b/variants/scale-1/kustomization.yaml index 7ea89636..33182f71 100644 --- a/variants/scale-1/kustomization.yaml +++ b/variants/scale-1/kustomization.yaml @@ -1,8 +1,7 @@ bases: -- ../../native +- ../../kafka patchesStrategicMerge: - kafka.yaml -- zookeeper.yaml patchesJson6902: - target: group: apps From a3bf432368050afbd9aaacfb50590c311b658aac Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 11 Mar 2022 15:30:58 +0100 Subject: [PATCH 7/9] Outside host given rbac-namespace-default; dev-small args overlap --- kafka/10broker-config.yml | 7 ++++--- kafka/50kafka.yml | 8 +++++--- variants/dev-small/listener-localhost.json | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 66197081..fa1b5746 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -21,7 +21,7 @@ data: # OUTSIDE_HOST and OUTSIDE_PORT can be set as envs, or set OUTSIDE_NODE_JSONPATH to get the value from the node, for example: '{.status.addresses[?(@.type=="InternalIP")].address}' [ -z "$OUTSIDE_HOST_JSONPATH" ] || OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath="$OUTSIDE_HOST_JSONPATH") [ -n "$OUTSIDE_PORT" ] || OUTSIDE_PORT=$((32400 + ${KAFKA_BROKER_ID})) - [ -z "$OUTSITE_HOST" ] || ADVERTISED_LISTENERS="$ADVERTISED_LISTENERS,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}" + [ -z "$OUTSIDE_HOST" ] || ADVERTISED_LISTENERS="$ADVERTISED_LISTENERS,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}" if grep 'node.id' /etc/kafka-configmap/server.properties; then echo "KRaft config detected" @@ -33,7 +33,8 @@ data: SEDS=("s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/") fi - [ -n "$OUTSIDE_HOST" ] || SEDS+=("s|,OUTSIDE://:9094||") + # Note how the outside-N services have a fixed target port + [ -z "$OUTSIDE_HOST" ] || SEDS+=("s|^listeners=\(.*\)|listeners=\1,OUTSIDE://:9094|") SEDS+=("s|#init#advertised.listeners=PLAINTEXT://#init#|advertised.listeners=$ADVERTISED_LISTENERS|") ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" @@ -116,7 +117,7 @@ data: # EXAMPLE: # listeners = PLAINTEXT://your.host.name:9092 #listeners=PLAINTEXT://:9092 - listeners=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094 + listeners=PLAINTEXT://:9092,CONTROLLER://:9093 # Hostname and port the broker will advertise to producers and consumers. If not set, # it uses the value for "listeners" if configured. Otherwise, it will use the value diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index e66b085c..0e510bc5 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -22,7 +22,7 @@ spec: fsGroup: 65534 initContainers: - name: init-config - image: yolean/kafka:latest + image: docker.io/yolean/toil:ab798035e825381728daffe35ade7c9c836021aa@sha256:eb1dd353e46270d7cdc69a118198bfafa9c2861520680276421c34f1d2b5c1a2 env: - name: NODE_NAME valueFrom: @@ -36,6 +36,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OUTSIDE_HOST_JSONPATH + value: '{.status.addresses[?(@.type=="InternalIP")].address}' command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] volumeMounts: - name: configmap @@ -45,7 +47,7 @@ spec: - name: extensions mountPath: /opt/kafka/libs/extensions - name: init-kraft - image: yolean/kafka:latest + image: yolean/kafka:3.1.0-java11 command: ['/bin/bash', '/etc/kafka-configmap/init-kraft.sh'] volumeMounts: - name: configmap @@ -58,7 +60,7 @@ spec: mountPath: /opt/kafka/libs/extensions containers: - name: broker - image: yolean/kafka:latest + image: yolean/kafka:3.1.0-java11 env: - name: POD_NAME valueFrom: diff --git a/variants/dev-small/listener-localhost.json b/variants/dev-small/listener-localhost.json index 496b1664..ab8eb946 100644 --- a/variants/dev-small/listener-localhost.json +++ b/variants/dev-small/listener-localhost.json @@ -1,4 +1,4 @@ [ - {"op": "add", "path": "/spec/template/spec/containers/0/args/1", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/args/2", "value": "advertised.listeners=PLAINTEXT://:9092,OUTSIDE://localhost:9094"} + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "advertised.listeners=PLAINTEXT://:9092,OUTSIDE://localhost:9094"} ] From 0e45918d46e67f7f3e0a8463ab833e7540d0020d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 11 Mar 2022 15:58:52 +0100 Subject: [PATCH 8/9] Multi-arch kminion image While waiting for an official build from cloudhut/kminion#139 --- consumers-prometheus/kminion.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consumers-prometheus/kminion.yaml b/consumers-prometheus/kminion.yaml index 96c8bb28..4693d1f8 100644 --- a/consumers-prometheus/kminion.yaml +++ b/consumers-prometheus/kminion.yaml @@ -18,7 +18,7 @@ spec: spec: containers: - name: kminion - image: quay.io/cloudhut/kminion:v2.1.0@sha256:99e3217ec4a6a61a0eb68da86d3f2c16fa9847a6db8498fea74044935096353b + image: solsson/kafka-consumers-prometheus@sha256:3d902ed8fa9e2cee7f44ff8a5567a5562a21d6bbfb11fe12faea49e85bd65059 env: - name: TELEMETRY_HOST value: 0.0.0.0 From d9662e4ae24ad0715071bcea3e38d64ccf9dad1c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 11 Mar 2022 16:03:12 +0100 Subject: [PATCH 9/9] Multi-arch image for kafka-topics.sh use cases - solsson/kafka-cli uses GraalVM that can't cross-compile - JRE based ./bin/kafka-topics.sh is too resource heavy to be practical for automation --- events-kube/topic-create.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/events-kube/topic-create.yaml b/events-kube/topic-create.yaml index cffbfb33..4bf15141 100644 --- a/events-kube/topic-create.yaml +++ b/events-kube/topic-create.yaml @@ -7,15 +7,14 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:native-cli@sha256:fbf29c59182fb87921c5199783d2d5796856ecbfe34a9c03eca658b3cf50f3c4 + image: vectorized/redpanda:v21.11.10@sha256:0841e5a06ee5ee60385121aa2e766ceee5964c359a4ab42e73475f6d25fdd057 command: - - ./bin/kafka-topics.sh - - --zookeeper - - zookeeper.kafka.svc.cluster.local:2181 - - --create - - --if-not-exists - - --topic - - ops.kube-events.stream.json + - rpk + - topic + - create + - --brokers + - bootstrap:9092 + - ops.kube-events.stream.json resources: limits: cpu: 100m