diff --git a/jobs/pingcap/ticdc/release-8.5/aa_folder.groovy b/jobs/pingcap/ticdc/release-8.5/aa_folder.groovy new file mode 100644 index 0000000000..e5e705d58a --- /dev/null +++ b/jobs/pingcap/ticdc/release-8.5/aa_folder.groovy @@ -0,0 +1,3 @@ +folder('pingcap/ticdc/release-8.5') { + description("Folder for pipelines of pingcap/ticdc repo for v8.5") +} diff --git a/jobs/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_heavy.groovy b/jobs/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_heavy.groovy new file mode 100644 index 0000000000..643bc20fef --- /dev/null +++ b/jobs/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_heavy.groovy @@ -0,0 +1,38 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +// For trunk and latest release branches. +pipelineJob('pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_heavy') { + logRotator { + daysToKeep(30) + } + parameters { + // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC") + } + properties { + githubProjectUrl("https://github.com/pingcap/ticdc") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_heavy.groovy") + scm { + git{ + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} diff --git a/jobs/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_light.groovy b/jobs/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_light.groovy new file mode 100644 index 0000000000..0670f0da32 --- /dev/null +++ b/jobs/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_light.groovy @@ -0,0 +1,38 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +// For trunk and latest release branches. +pipelineJob('pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_light') { + logRotator { + daysToKeep(30) + } + parameters { + // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC") + } + properties { + githubProjectUrl("https://github.com/pingcap/ticdc") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_light.groovy") + scm { + git{ + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} diff --git a/jobs/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_heavy.groovy b/jobs/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_heavy.groovy new file mode 100644 index 0000000000..4f02eecd21 --- /dev/null +++ b/jobs/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_heavy.groovy @@ -0,0 +1,37 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +pipelineJob('pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_heavy') { + logRotator { + daysToKeep(30) + } + parameters { + // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC") + } + properties { + githubProjectUrl("https://github.com/pingcap/ticdc") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_heavy.groovy") + scm { + git{ + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} diff --git a/jobs/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_light.groovy b/jobs/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_light.groovy new file mode 100644 index 0000000000..2adb94fab3 --- /dev/null +++ b/jobs/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_light.groovy @@ -0,0 +1,37 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +pipelineJob('pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_light') { + logRotator { + daysToKeep(30) + } + parameters { + // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC") + } + properties { + githubProjectUrl("https://github.com/pingcap/ticdc") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_light.groovy") + scm { + git{ + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} diff --git a/jobs/pingcap/ticdc/release-8.5/pull_cdc_pulsar_integration_light.groovy b/jobs/pingcap/ticdc/release-8.5/pull_cdc_pulsar_integration_light.groovy new file mode 100644 index 0000000000..414af4a808 --- /dev/null +++ b/jobs/pingcap/ticdc/release-8.5/pull_cdc_pulsar_integration_light.groovy @@ -0,0 +1,38 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +// For trunk and latest release branches. +pipelineJob('pingcap/ticdc/release-8.5/pull_cdc_pulsar_integration_light') { + logRotator { + daysToKeep(30) + } + parameters { + // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC") + } + properties { + githubProjectUrl("https://github.com/pingcap/ticdc") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/pingcap/ticdc/release-8.5/pull_cdc_pulsar_integration_light.groovy") + scm { + git{ + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} diff --git a/jobs/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_heavy.groovy b/jobs/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_heavy.groovy new file mode 100644 index 0000000000..9ae6109d1f --- /dev/null +++ b/jobs/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_heavy.groovy @@ -0,0 +1,38 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +// For trunk and latest release branches. +pipelineJob('pingcap/ticdc/release-8.5/pull_cdc_storage_integration_heavy') { + logRotator { + daysToKeep(30) + } + parameters { + // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC") + } + properties { + githubProjectUrl("https://github.com/pingcap/ticdc") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_heavy.groovy") + scm { + git{ + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} diff --git a/jobs/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_light.groovy b/jobs/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_light.groovy new file mode 100644 index 0000000000..bfcc0f3871 --- /dev/null +++ b/jobs/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_light.groovy @@ -0,0 +1,38 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +// For trunk and latest release branches. +pipelineJob('pingcap/ticdc/release-8.5/pull_cdc_storage_integration_light') { + logRotator { + daysToKeep(30) + } + parameters { + // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC") + } + properties { + githubProjectUrl("https://github.com/pingcap/ticdc") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_light.groovy") + scm { + git{ + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} diff --git a/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_integration_build.yaml b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_integration_build.yaml new file mode 100644 index 0000000000..66f0454642 --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_integration_build.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - name: golang + image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" + tty: true + resources: + limits: + memory: 16Gi + cpu: "12" + - name: net-tool + image: hub.pingcap.net/jenkins/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 100m + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_kafka_integration_heavy.yaml b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_kafka_integration_heavy.yaml new file mode 100644 index 0000000000..568cc54c00 --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_kafka_integration_heavy.yaml @@ -0,0 +1,159 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - image: wurstmeister/zookeeper + imagePullPolicy: IfNotPresent + name: zookeeper + resources: + requests: + cpu: 1000m + memory: 4Gi + limits: + cpu: 1000m + memory: 4Gi + tty: true + volumeMounts: + - mountPath: /tmp + name: volume-0 + - args: + - cat + image: hub.pingcap.net/jenkins/rocky8_golang-1.23:tini + imagePullPolicy: Always + name: golang + resources: + limits: + cpu: "12" + memory: 16Gi + tty: true + volumeMounts: + - mountPath: /tmp + name: volume-0 + - env: + - name: KAFKA_CREATE_TOPICS + value: big-message-test:1:1 + - name: KAFKA_BROKER_ID + value: "1" + - name: KAFKA_SSL_KEYSTORE_PASSWORD + value: test1234 + - name: KAFKA_ZOOKEEPER_CONNECT + value: localhost:2181 + - name: KAFKA_MESSAGE_MAX_BYTES + value: "11534336" + - name: KAFKA_REPLICA_FETCH_MAX_BYTES + value: "11534336" + - name: KAFKA_ADVERTISED_LISTENERS + value: SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092 + - name: ZK + value: zk + - name: KAFKA_SSL_KEYSTORE_LOCATION + value: /tmp/kafka.server.keystore.jks + - name: KAFKA_SSL_KEY_PASSWORD + value: test1234 + - name: KAFKA_SSL_TRUSTSTORE_PASSWORD + value: test1234 + - name: KAFKA_LISTENERS + value: SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092 + - name: KAFKA_SSL_TRUSTSTORE_LOCATION + value: /tmp/kafka.server.truststore.jks + - name: RACK_COMMAND + value: curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks -o /tmp/kafka.server.truststore.jks + image: wurstmeister/kafka:2.12-2.4.1 + imagePullPolicy: IfNotPresent + name: kafka + resources: + requests: + cpu: 2000m + memory: 6Gi + limits: + cpu: 2000m + memory: 6Gi + tty: true + volumeMounts: + - mountPath: /tmp + name: volume-0 + - env: + - name: KAFKA_SERVER + value: 127.0.0.1:9092 + - name: ZOOKEEPER_SERVER + value: 127.0.0.1:2181 + - name: DOWNSTREAM_DB_HOST + value: 127.0.0.1 + - name: USE_FLAT_MESSAGE + value: "true" + - name: DOWNSTREAM_DB_PORT + value: "3306" + - name: DB_NAME + value: test + image: rustinliu/ticdc-canal-json-adapter:latest + imagePullPolicy: IfNotPresent + name: canal-adapter + # TODO: add resource limit + # can't add resource limit now, because canal-adapter will OOM + # issue: https://github.com/PingCAP-QE/ci/issues/1893 + resources: + requests: + cpu: 2000m + memory: 4Gi + tty: true + volumeMounts: + - mountPath: /tmp + name: volume-0 + - name: net-tool + image: hub.pingcap.net/jenkins/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 600m + - name: mysql + image: quay.io/debezium/example-mysql:2.4 + imagePullPolicy: IfNotPresent + env: + - name: MYSQL_ROOT_PASSWORD + value: "" + - name: MYSQL_USER + value: mysqluser + - name: MYSQL_PASSWORD + value: mysqlpw + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "yes" + - name: MYSQL_TCP_PORT + value: "3310" + resources: + requests: + cpu: 2000m + memory: 4Gi + - name: connect + image: quay.io/debezium/connect:2.4 + env: + - name: BOOTSTRAP_SERVERS + value: "127.0.0.1:9092" + - name: GROUP_ID + value: "1" + - name: CONFIG_STORAGE_TOPIC + value: "my_connect_configs" + - name: OFFSET_STORAGE_TOPIC + value: "my_connect_offsets" + - name: STATUS_STORAGE_TOPIC + value: "my_connect_statuses" + - name: LANG + value: "C.UTF-8" + resources: + requests: + cpu: 2000m + memory: 4Gi + volumes: + - emptyDir: {} + name: volume-0 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_kafka_integration_light.yaml b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_kafka_integration_light.yaml new file mode 100644 index 0000000000..3535f227df --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_kafka_integration_light.yaml @@ -0,0 +1,159 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - image: wurstmeister/zookeeper + imagePullPolicy: IfNotPresent + name: zookeeper + resources: + requests: + cpu: 1000m + memory: 4Gi + limits: + cpu: 1000m + memory: 4Gi + tty: true + volumeMounts: + - mountPath: /tmp + name: volume-0 + - args: + - cat + image: hub.pingcap.net/jenkins/rocky8_golang-1.23:tini + imagePullPolicy: Always + name: golang + resources: + limits: + cpu: "6" + memory: 16Gi + tty: true + volumeMounts: + - mountPath: /tmp + name: volume-0 + - env: + - name: KAFKA_CREATE_TOPICS + value: big-message-test:1:1 + - name: KAFKA_BROKER_ID + value: "1" + - name: KAFKA_SSL_KEYSTORE_PASSWORD + value: test1234 + - name: KAFKA_ZOOKEEPER_CONNECT + value: localhost:2181 + - name: KAFKA_MESSAGE_MAX_BYTES + value: "11534336" + - name: KAFKA_REPLICA_FETCH_MAX_BYTES + value: "11534336" + - name: KAFKA_ADVERTISED_LISTENERS + value: SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092 + - name: ZK + value: zk + - name: KAFKA_SSL_KEYSTORE_LOCATION + value: /tmp/kafka.server.keystore.jks + - name: KAFKA_SSL_KEY_PASSWORD + value: test1234 + - name: KAFKA_SSL_TRUSTSTORE_PASSWORD + value: test1234 + - name: KAFKA_LISTENERS + value: SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092 + - name: KAFKA_SSL_TRUSTSTORE_LOCATION + value: /tmp/kafka.server.truststore.jks + - name: RACK_COMMAND + value: curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks -o /tmp/kafka.server.truststore.jks + image: wurstmeister/kafka:2.12-2.4.1 + imagePullPolicy: IfNotPresent + name: kafka + resources: + requests: + cpu: 2000m + memory: 6Gi + limits: + cpu: 2000m + memory: 6Gi + tty: true + volumeMounts: + - mountPath: /tmp + name: volume-0 + - env: + - name: KAFKA_SERVER + value: 127.0.0.1:9092 + - name: ZOOKEEPER_SERVER + value: 127.0.0.1:2181 + - name: DOWNSTREAM_DB_HOST + value: 127.0.0.1 + - name: USE_FLAT_MESSAGE + value: "true" + - name: DOWNSTREAM_DB_PORT + value: "3306" + - name: DB_NAME + value: test + image: rustinliu/ticdc-canal-json-adapter:latest + imagePullPolicy: IfNotPresent + name: canal-adapter + # TODO: add resource limit + # can't add resource limit now, because canal-adapter will OOM + # issue: https://github.com/PingCAP-QE/ci/issues/1893 + resources: + requests: + cpu: 2000m + memory: 4Gi + tty: true + volumeMounts: + - mountPath: /tmp + name: volume-0 + - name: net-tool + image: hub.pingcap.net/jenkins/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 600m + - name: mysql + image: quay.io/debezium/example-mysql:2.4 + imagePullPolicy: IfNotPresent + env: + - name: MYSQL_ROOT_PASSWORD + value: "" + - name: MYSQL_USER + value: mysqluser + - name: MYSQL_PASSWORD + value: mysqlpw + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "yes" + - name: MYSQL_TCP_PORT + value: "3310" + resources: + requests: + cpu: 2000m + memory: 4Gi + - name: connect + image: quay.io/debezium/connect:2.4 + env: + - name: BOOTSTRAP_SERVERS + value: "127.0.0.1:9092" + - name: GROUP_ID + value: "1" + - name: CONFIG_STORAGE_TOPIC + value: "my_connect_configs" + - name: OFFSET_STORAGE_TOPIC + value: "my_connect_offsets" + - name: STATUS_STORAGE_TOPIC + value: "my_connect_statuses" + - name: LANG + value: "C.UTF-8" + resources: + requests: + cpu: 2000m + memory: 4Gi + volumes: + - emptyDir: {} + name: volume-0 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_mysql_integration_heavy.yaml b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_mysql_integration_heavy.yaml new file mode 100644 index 0000000000..66f0454642 --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_mysql_integration_heavy.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - name: golang + image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" + tty: true + resources: + limits: + memory: 16Gi + cpu: "12" + - name: net-tool + image: hub.pingcap.net/jenkins/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 100m + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_mysql_integration_light.yaml b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_mysql_integration_light.yaml new file mode 100644 index 0000000000..114a884b14 --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_mysql_integration_light.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - name: golang + image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" + tty: true + resources: + limits: + memory: 16Gi + cpu: "4" + - name: net-tool + image: hub.pingcap.net/jenkins/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 100m + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_pulsar_integration_light.yaml b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_pulsar_integration_light.yaml new file mode 100644 index 0000000000..8a0cead34c --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_pulsar_integration_light.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - name: golang + image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" + tty: true + resources: + limits: + memory: 32Gi + cpu: "6" + - name: net-tool + image: hub.pingcap.net/jenkins/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 600m + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_storage_integration_heavy.yaml b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_storage_integration_heavy.yaml new file mode 100644 index 0000000000..5703efbd29 --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_storage_integration_heavy.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - name: golang + image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" + tty: true + resources: + limits: + memory: 16Gi + cpu: "12" + - name: net-tool + image: hub.pingcap.net/jenkins/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 600m + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_storage_integration_light.yaml b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_storage_integration_light.yaml new file mode 100644 index 0000000000..1d56bdbcdd --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_storage_integration_light.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - name: golang + image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" + tty: true + resources: + limits: + memory: 16Gi + cpu: "6" + - name: net-tool + image: hub.pingcap.net/jenkins/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 600m + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_heavy.groovy b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_heavy.groovy new file mode 100644 index 0000000000..c77c028dd2 --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_heavy.groovy @@ -0,0 +1,166 @@ +// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline +// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984 +// should triggerd for master branches +@Library('tipipeline') _ + +final K8S_NAMESPACE = "jenkins-tiflow" +final GIT_FULL_REPO_NAME = 'pingcap/ticdc' +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final POD_TEMPLATE_FILE = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_kafka_integration_heavy.yaml' +final POD_TEMPLATE_FILE_BUILD = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_integration_build.yaml' +final REFS = readJSON(text: params.JOB_SPEC).refs + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE_BUILD + defaultContainer 'golang' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 65, unit: 'MINUTES') + parallelsAlwaysFailFast() + } + stages { + stage('Debug info') { + steps { + sh label: 'Debug info', script: """ + printenv + echo "-------------------------" + go env + echo "-------------------------" + echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash" + """ + container(name: 'net-tool') { + sh 'dig github.com' + script { + prow.setPRDescription(REFS) + } + } + } + } + stage('Checkout') { + options { timeout(time: 10, unit: 'MINUTES') } + steps { + dir("ticdc") { + cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + retry(2) { + script { + prow.checkoutRefs(REFS) + } + } + } + } + } + } + stage("prepare") { + options { timeout(time: 20, unit: 'MINUTES') } + steps { + dir("third_party_download") { + script { + def tidbBranch = component.computeBranchFromPR('tidb', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def pdBranch = component.computeBranchFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tikvBranch = component.computeBranchFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tiflashBranch = component.computeBranchFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + retry(2) { + sh label: "download third_party", script: """ + export TIDB_BRANCH=${tidbBranch} + export PD_BRANCH=${pdBranch} + export TIKV_BRANCH=${tikvBranch} + export TIFLASH_BRANCH=${tiflashBranch} + cd ../ticdc && ./tests/scripts/download-integration-test-binaries.sh ${REFS.base_ref} && ls -alh ./bin + make check_third_party_binary + cd - && mkdir -p bin && mv ../ticdc/bin/* ./bin/ + ls -alh ./bin + ./bin/tidb-server -V + ./bin/pd-server -V + ./bin/tikv-server -V + ./bin/tiflash --version + """ + } + } + } + dir("ticdc") { + cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'cdc-integration-test')) { + // build cdc, kafka_consumer, storage_consumer, cdc.test for integration test + // only build binarys if not exist, use the cached binarys if exist + sh label: "prepare", script: """ + ls -alh ./bin + [ -f ./bin/cdc ] || make cdc + [ -f ./bin/cdc_kafka_consumer ] || make kafka_consumer + [ -f ./bin/cdc_storage_consumer ] || make storage_consumer + [ -f ./bin/cdc.test ] || make integration_test_build + ls -alh ./bin + ./bin/cdc version + """ + } + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "prepare", script: """ + cp -r ../third_party_download/bin/* ./bin/ + ls -alh ./bin + """ + } + } + } + } + + stage('Tests') { + matrix { + axes { + axis { + name 'TEST_GROUP' + values 'G00', 'G01', 'G02', 'G03', 'G04', 'G05', 'G06', 'G07', 'G08', + 'G09', 'G10', 'G11', 'G12', 'G13', 'G14', 'G15' + } + } + agent{ + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + stages { + stage("Test") { + options { timeout(time: 45, unit: 'MINUTES') } + steps { + dir('ticdc') { + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + container("kafka") { + timeout(time: 6, unit: 'MINUTES') { + sh label: "Waiting for kafka ready", script: """ + echo "Waiting for zookeeper to be ready..." + while ! nc -z localhost 2181; do sleep 10; done + echo "Waiting for kafka to be ready..." + while ! nc -z localhost 9092; do sleep 10; done + echo "Waiting for kafka-broker to be ready..." + while ! echo dump | nc localhost 2181 | grep brokers | awk '{\$1=\$1;print}' | grep -F -w "/brokers/ids/1"; do sleep 10; done + """ + } + } + sh label: "${TEST_GROUP}", script: """ + ./tests/integration_tests/run_heavy_it_in_ci.sh kafka ${TEST_GROUP} + """ + } + } + } + post { + failure { + sh label: "collect logs", script: """ + ls /tmp/tidb_cdc_test/ + tar --warning=no-file-changed -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/tidb_cdc_test/ -type f -name "*.log") + ls -alh log-${TEST_GROUP}.tar.gz + """ + archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true + } + } + } + } + } + } + } +} diff --git a/pipelines/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_light.groovy b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_light.groovy new file mode 100644 index 0000000000..7a37aa148e --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_light.groovy @@ -0,0 +1,166 @@ +// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline +// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984 +// should triggerd for master branches +@Library('tipipeline') _ + +final K8S_NAMESPACE = "jenkins-tiflow" +final GIT_FULL_REPO_NAME = 'pingcap/ticdc' +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final POD_TEMPLATE_FILE = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_kafka_integration_light.yaml' +final POD_TEMPLATE_FILE_BUILD = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_integration_build.yaml' +final REFS = readJSON(text: params.JOB_SPEC).refs + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE_BUILD + defaultContainer 'golang' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 65, unit: 'MINUTES') + parallelsAlwaysFailFast() + } + stages { + stage('Debug info') { + steps { + sh label: 'Debug info', script: """ + printenv + echo "-------------------------" + go env + echo "-------------------------" + echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash" + """ + container(name: 'net-tool') { + sh 'dig github.com' + script { + prow.setPRDescription(REFS) + } + } + } + } + stage('Checkout') { + options { timeout(time: 10, unit: 'MINUTES') } + steps { + dir("ticdc") { + cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + retry(2) { + script { + prow.checkoutRefs(REFS) + } + } + } + } + } + } + stage("prepare") { + options { timeout(time: 20, unit: 'MINUTES') } + steps { + dir("third_party_download") { + script { + def tidbBranch = component.computeBranchFromPR('tidb', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def pdBranch = component.computeBranchFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tikvBranch = component.computeBranchFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tiflashBranch = component.computeBranchFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + retry(2) { + sh label: "download third_party", script: """ + export TIDB_BRANCH=${tidbBranch} + export PD_BRANCH=${pdBranch} + export TIKV_BRANCH=${tikvBranch} + export TIFLASH_BRANCH=${tiflashBranch} + cd ../ticdc && ./tests/scripts/download-integration-test-binaries.sh ${REFS.base_ref} && ls -alh ./bin + make check_third_party_binary + cd - && mkdir -p bin && mv ../ticdc/bin/* ./bin/ + ls -alh ./bin + ./bin/tidb-server -V + ./bin/pd-server -V + ./bin/tikv-server -V + ./bin/tiflash --version + """ + } + } + } + dir("ticdc") { + cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'cdc-integration-test')) { + // build cdc, kafka_consumer, storage_consumer, cdc.test for integration test + // only build binarys if not exist, use the cached binarys if exist + sh label: "prepare", script: """ + ls -alh ./bin + [ -f ./bin/cdc ] || make cdc + [ -f ./bin/cdc_kafka_consumer ] || make kafka_consumer + [ -f ./bin/cdc_storage_consumer ] || make storage_consumer + [ -f ./bin/cdc.test ] || make integration_test_build + ls -alh ./bin + ./bin/cdc version + """ + } + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "prepare", script: """ + cp -r ../third_party_download/bin/* ./bin/ + ls -alh ./bin + """ + } + } + } + } + + stage('Tests') { + matrix { + axes { + axis { + name 'TEST_GROUP' + values 'G00', 'G01', 'G02', 'G03', 'G04', 'G05', 'G06', 'G07', 'G08', + 'G09', 'G10', 'G11', 'G12', 'G13', 'G14', 'G15' + } + } + agent{ + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + stages { + stage("Test") { + options { timeout(time: 45, unit: 'MINUTES') } + steps { + dir('ticdc') { + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + container("kafka") { + timeout(time: 6, unit: 'MINUTES') { + sh label: "Waiting for kafka ready", script: """ + echo "Waiting for zookeeper to be ready..." + while ! nc -z localhost 2181; do sleep 10; done + echo "Waiting for kafka to be ready..." + while ! nc -z localhost 9092; do sleep 10; done + echo "Waiting for kafka-broker to be ready..." + while ! echo dump | nc localhost 2181 | grep brokers | awk '{\$1=\$1;print}' | grep -F -w "/brokers/ids/1"; do sleep 10; done + """ + } + } + sh label: "${TEST_GROUP}", script: """ + ./tests/integration_tests/run_light_it_in_ci.sh kafka ${TEST_GROUP} + """ + } + } + } + post { + failure { + sh label: "collect logs", script: """ + ls /tmp/tidb_cdc_test/ + tar --warning=no-file-changed -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/tidb_cdc_test/ -type f -name "*.log") + ls -alh log-${TEST_GROUP}.tar.gz + """ + archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true + } + } + } + } + } + } + } +} diff --git a/pipelines/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_heavy.groovy b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_heavy.groovy new file mode 100644 index 0000000000..e9f854b3ef --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_heavy.groovy @@ -0,0 +1,151 @@ +// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline +// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984 +// should triggerd for master branches +@Library('tipipeline') _ + +final K8S_NAMESPACE = "jenkins-tiflow" +final GIT_FULL_REPO_NAME = 'pingcap/ticdc' +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final POD_TEMPLATE_FILE = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_mysql_integration_heavy.yaml' +final REFS = readJSON(text: params.JOB_SPEC).refs +def skipRemainingStages = false + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 60, unit: 'MINUTES') + parallelsAlwaysFailFast() + } + stages { + stage('Debug info') { + steps { + sh label: 'Debug info', script: """ + printenv + echo "-------------------------" + go env + echo "-------------------------" + echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash" + """ + container(name: 'net-tool') { + sh 'dig github.com' + script { + prow.setPRDescription(REFS) + } + } + } + } + stage('Checkout') { + options { timeout(time: 10, unit: 'MINUTES') } + steps { + dir("ticdc") { + cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + retry(2) { + script { + prow.checkoutRefs(REFS) + } + } + } + } + } + } + stage("prepare") { + options { timeout(time: 20, unit: 'MINUTES') } + steps { + dir("third_party_download") { + script { + def tidbBranch = component.computeBranchFromPR('tidb', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def pdBranch = component.computeBranchFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tikvBranch = component.computeBranchFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tiflashBranch = component.computeBranchFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + retry(2) { + sh label: "download third_party", script: """ + export TIDB_BRANCH=${tidbBranch} + export PD_BRANCH=${pdBranch} + export TIKV_BRANCH=${tikvBranch} + export TIFLASH_BRANCH=${tiflashBranch} + cd ../ticdc && ./tests/scripts/download-integration-test-binaries.sh ${REFS.base_ref} && ls -alh ./bin + make check_third_party_binary + cd - && mkdir -p bin && mv ../ticdc/bin/* ./bin/ + ls -alh ./bin + ./bin/tidb-server -V + ./bin/pd-server -V + ./bin/tikv-server -V + ./bin/tiflash --version + """ + } + } + } + dir("ticdc") { + cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'cdc-mysql-integration')) { + // build cdc, kafka_consumer, storage_consumer, cdc.test for integration test + // only build binarys if not exist, use the cached binarys if exist + sh label: "prepare", script: """ + ls -alh ./bin + [ -f ./bin/cdc ] || make cdc + [ -f ./bin/cdc.test ] || make integration_test_build + ls -alh ./bin + ./bin/cdc version + """ + } + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "prepare", script: """ + cp -r ../third_party_download/bin/* ./bin/ + ls -alh ./bin + """ + } + } + } + } + + stage('Tests') { + matrix { + axes { + axis { + name 'TEST_GROUP' + values 'G00', 'G01', 'G02', 'G03', 'G04', 'G05', 'G06', 'G07', 'G08', 'G09', 'G10', 'G11', 'G12', 'G13', 'G14', 'G15' + } + } + agent{ + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + stages { + stage("Test") { + options { timeout(time: 40, unit: 'MINUTES') } + steps { + dir('ticdc') { + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "${TEST_GROUP}", script: """ + ./tests/integration_tests/run_heavy_it_in_ci.sh mysql ${TEST_GROUP} + """ + } + } + } + post { + failure { + sh label: "collect logs", script: """ + ls /tmp/tidb_cdc_test/ + tar --warning=no-file-changed -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/tidb_cdc_test/ -type f -name "*.log") + ls -alh log-${TEST_GROUP}.tar.gz + """ + archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true + } + } + } + } + } + } + } +} diff --git a/pipelines/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_light.groovy b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_light.groovy new file mode 100644 index 0000000000..523cf1a2ad --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_light.groovy @@ -0,0 +1,152 @@ +// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline +// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984 +// should triggerd for master branches +@Library('tipipeline') _ + +final K8S_NAMESPACE = "jenkins-tiflow" +final GIT_FULL_REPO_NAME = 'pingcap/ticdc' +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final POD_TEMPLATE_FILE = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_mysql_integration_light.yaml' +final POD_TEMPLATE_FILE_BUILD = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_integration_build.yaml' +final REFS = readJSON(text: params.JOB_SPEC).refs +def skipRemainingStages = false + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE_BUILD + defaultContainer 'golang' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 60, unit: 'MINUTES') + parallelsAlwaysFailFast() + } + stages { + stage('Debug info') { + steps { + sh label: 'Debug info', script: """ + printenv + echo "-------------------------" + go env + echo "-------------------------" + echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash" + """ + container(name: 'net-tool') { + sh 'dig github.com' + script { + prow.setPRDescription(REFS) + } + } + } + } + stage('Checkout') { + options { timeout(time: 10, unit: 'MINUTES') } + steps { + dir("ticdc") { + cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + retry(2) { + script { + prow.checkoutRefs(REFS) + } + } + } + } + } + } + stage("prepare") { + options { timeout(time: 20, unit: 'MINUTES') } + steps { + dir("third_party_download") { + script { + def tidbBranch = component.computeBranchFromPR('tidb', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def pdBranch = component.computeBranchFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tikvBranch = component.computeBranchFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tiflashBranch = component.computeBranchFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + retry(2) { + sh label: "download third_party", script: """ + export TIDB_BRANCH=${tidbBranch} + export PD_BRANCH=${pdBranch} + export TIKV_BRANCH=${tikvBranch} + export TIFLASH_BRANCH=${tiflashBranch} + cd ../ticdc && ./tests/scripts/download-integration-test-binaries.sh ${REFS.base_ref} && ls -alh ./bin + make check_third_party_binary + cd - && mkdir -p bin && mv ../ticdc/bin/* ./bin/ + ls -alh ./bin + ./bin/tidb-server -V + ./bin/pd-server -V + ./bin/tikv-server -V + ./bin/tiflash --version + """ + } + } + } + dir("ticdc") { + cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'cdc-mysql-integration')) { + // build cdc, kafka_consumer, storage_consumer, cdc.test for integration test + // only build binarys if not exist, use the cached binarys if exist + sh label: "prepare", script: """ + ls -alh ./bin + [ -f ./bin/cdc ] || make cdc + [ -f ./bin/cdc.test ] || make integration_test_build + ls -alh ./bin + ./bin/cdc version + """ + } + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "prepare", script: """ + cp -r ../third_party_download/bin/* ./bin/ + ls -alh ./bin + """ + } + } + } + } + + stage('Tests') { + matrix { + axes { + axis { + name 'TEST_GROUP' + values 'G00', 'G01', 'G02', 'G03', 'G04', 'G05', 'G06', 'G07', 'G08', 'G09', 'G10', 'G11', 'G12', 'G13', 'G14', 'G15' + } + } + agent{ + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + stages { + stage("Test") { + options { timeout(time: 40, unit: 'MINUTES') } + steps { + dir('ticdc') { + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "${TEST_GROUP}", script: """ + ./tests/integration_tests/run_light_it_in_ci.sh mysql ${TEST_GROUP} + """ + } + } + } + post { + failure { + sh label: "collect logs", script: """ + ls /tmp/tidb_cdc_test/ + tar --warning=no-file-changed -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/tidb_cdc_test/ -type f -name "*.log") + ls -alh log-${TEST_GROUP}.tar.gz + """ + archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true + } + } + } + } + } + } + } +} diff --git a/pipelines/pingcap/ticdc/release-8.5/pull_cdc_pulsar_integration_light.groovy b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_pulsar_integration_light.groovy new file mode 100644 index 0000000000..9902c6ee96 --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_pulsar_integration_light.groovy @@ -0,0 +1,153 @@ +// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline +// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984 +// should triggerd for master branches +@Library('tipipeline') _ + +final K8S_NAMESPACE = "jenkins-tiflow" +final GIT_FULL_REPO_NAME = 'pingcap/ticdc' +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final POD_TEMPLATE_FILE = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_pulsar_integration_light.yaml' +final POD_TEMPLATE_FILE_BUILD = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_integration_build.yaml' +final REFS = readJSON(text: params.JOB_SPEC).refs + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE_BUILD + defaultContainer 'golang' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 60, unit: 'MINUTES') + parallelsAlwaysFailFast() + } + stages { + stage('Debug info') { + steps { + sh label: 'Debug info', script: """ + printenv + echo "-------------------------" + go env + echo "-------------------------" + echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash" + """ + container(name: 'net-tool') { + sh 'dig github.com' + script { + prow.setPRDescription(REFS) + } + } + } + } + stage('Checkout') { + options { timeout(time: 10, unit: 'MINUTES') } + steps { + dir("ticdc") { + cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + retry(2) { + script { + prow.checkoutRefs(REFS) + } + } + } + } + } + } + stage("prepare") { + options { timeout(time: 20, unit: 'MINUTES') } + steps { + dir("third_party_download") { + script { + def tidbBranch = component.computeBranchFromPR('tidb', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def pdBranch = component.computeBranchFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tikvBranch = component.computeBranchFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tiflashBranch = component.computeBranchFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + retry(2) { + sh label: "download third_party", script: """ + export TIDB_BRANCH=${tidbBranch} + export PD_BRANCH=${pdBranch} + export TIKV_BRANCH=${tikvBranch} + export TIFLASH_BRANCH=${tiflashBranch} + cd ../ticdc && ./tests/scripts/download-integration-test-binaries.sh ${REFS.base_ref} && ls -alh ./bin + make check_third_party_binary + cd - && mkdir -p bin && mv ../ticdc/bin/* ./bin/ + ls -alh ./bin + ./bin/tidb-server -V + ./bin/pd-server -V + ./bin/tikv-server -V + ./bin/tiflash --version + """ + } + } + } + dir("ticdc") { + cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'cdc-pulsar-integration')) { + // build cdc, pulsar_consumer, cdc.test for integration test + // only build binarys if not exist, use the cached binarys if exist + sh label: "prepare", script: """ + ls -alh ./bin + [ -f ./bin/cdc ] || make cdc + [ -f ./bin/cdc_pulsar_consumer ] || make pulsar_consumer + [ -f ./bin/cdc.test ] || make integration_test_build + ls -alh ./bin + ./bin/cdc version + """ + } + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "prepare", script: """ + cp -r ../third_party_download/bin/* ./bin/ + ls -alh ./bin + """ + } + } + } + } + + stage('Tests') { + matrix { + axes { + axis { + name 'TEST_GROUP' + values 'G00', 'G01', 'G02', 'G03', 'G04', 'G05', 'G06', 'G07', 'G08', 'G09', + 'G10', 'G11', 'G12', 'G13', 'G14', 'G15' + } + } + agent{ + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + stages { + stage("Test") { + options { timeout(time: 40, unit: 'MINUTES') } + steps { + dir('ticdc') { + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "${TEST_GROUP}", script: """ + ./tests/integration_tests/run_light_it_in_ci.sh pulsar ${TEST_GROUP} + """ + } + } + } + post { + failure { + sh label: "collect logs", script: """ + ls /tmp/tidb_cdc_test/ + tar --warning=no-file-changed -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/tidb_cdc_test/ -type f -name "*.log") + ls -alh log-${TEST_GROUP}.tar.gz + """ + archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true + } + } + } + } + } + } + } +} diff --git a/pipelines/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_heavy.groovy b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_heavy.groovy new file mode 100644 index 0000000000..8bc3a002ad --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_heavy.groovy @@ -0,0 +1,154 @@ +// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline +// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984 +// should triggerd for master branches +@Library('tipipeline') _ + +final K8S_NAMESPACE = "jenkins-tiflow" +final GIT_FULL_REPO_NAME = 'pingcap/ticdc' +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final POD_TEMPLATE_FILE = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_storage_integration_heavy.yaml' +final POD_TEMPLATE_FILE_BUILD = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_integration_build.yaml' +final REFS = readJSON(text: params.JOB_SPEC).refs + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE_BUILD + defaultContainer 'golang' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 60, unit: 'MINUTES') + parallelsAlwaysFailFast() + } + stages { + stage('Debug info') { + steps { + sh label: 'Debug info', script: """ + printenv + echo "-------------------------" + go env + echo "-------------------------" + echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash" + """ + container(name: 'net-tool') { + sh 'dig github.com' + script { + prow.setPRDescription(REFS) + } + } + } + } + stage('Checkout') { + options { timeout(time: 10, unit: 'MINUTES') } + steps { + dir("ticdc") { + cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + retry(2) { + script { + prow.checkoutRefs(REFS) + } + } + } + } + } + } + stage("prepare") { + options { timeout(time: 20, unit: 'MINUTES') } + steps { + dir("third_party_download") { + script { + def tidbBranch = component.computeBranchFromPR('tidb', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def pdBranch = component.computeBranchFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tikvBranch = component.computeBranchFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tiflashBranch = component.computeBranchFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + retry(2) { + sh label: "download third_party", script: """ + export TIDB_BRANCH=${tidbBranch} + export PD_BRANCH=${pdBranch} + export TIKV_BRANCH=${tikvBranch} + export TIFLASH_BRANCH=${tiflashBranch} + cd ../ticdc && ./tests/scripts/download-integration-test-binaries.sh ${REFS.base_ref} && ls -alh ./bin + make check_third_party_binary + cd - && mkdir -p bin && mv ../ticdc/bin/* ./bin/ + ls -alh ./bin + ./bin/tidb-server -V + ./bin/pd-server -V + ./bin/tikv-server -V + ./bin/tiflash --version + """ + } + } + } + dir("ticdc") { + cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'cdc-storage-integration')) { + // build cdc, kafka_consumer, storage_consumer, cdc.test for integration test + // only build binarys if not exist, use the cached binarys if exist + sh label: "prepare", script: """ + ls -alh ./bin + [ -f ./bin/cdc ] || make cdc + [ -f ./bin/cdc_kafka_consumer ] || make kafka_consumer + [ -f ./bin/cdc_storage_consumer ] || make storage_consumer + [ -f ./bin/cdc.test ] || make integration_test_build + ls -alh ./bin + ./bin/cdc version + """ + } + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "prepare", script: """ + cp -r ../third_party_download/bin/* ./bin/ + ls -alh ./bin + """ + } + } + } + } + + stage('Tests') { + matrix { + axes { + axis { + name 'TEST_GROUP' + values 'G00', 'G01', 'G02', 'G03', 'G04', 'G05', 'G06', 'G07', 'G08', 'G09', + 'G10', 'G11', 'G12', 'G13', 'G14', 'G15' + } + } + agent{ + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + stages { + stage("Test") { + options { timeout(time: 40, unit: 'MINUTES') } + steps { + dir('ticdc') { + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "${TEST_GROUP}", script: """ + ./tests/integration_tests/run_heavy_it_in_ci.sh storage ${TEST_GROUP} + """ + } + } + } + post { + failure { + sh label: "collect logs", script: """ + ls /tmp/tidb_cdc_test/ + tar --warning=no-file-changed -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/tidb_cdc_test/ -type f -name "*.log") + ls -alh log-${TEST_GROUP}.tar.gz + """ + archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true + } + } + } + } + } + } + } +} diff --git a/pipelines/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_light.groovy b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_light.groovy new file mode 100644 index 0000000000..10efdf3514 --- /dev/null +++ b/pipelines/pingcap/ticdc/release-8.5/pull_cdc_storage_integration_light.groovy @@ -0,0 +1,154 @@ +// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline +// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984 +// should triggerd for master branches +@Library('tipipeline') _ + +final K8S_NAMESPACE = "jenkins-tiflow" +final GIT_FULL_REPO_NAME = 'pingcap/ticdc' +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final POD_TEMPLATE_FILE = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_storage_integration_light.yaml' +final POD_TEMPLATE_FILE_BUILD = 'pipelines/pingcap/ticdc/release-8.5/pod-pull_cdc_integration_build.yaml' +final REFS = readJSON(text: params.JOB_SPEC).refs + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE_BUILD + defaultContainer 'golang' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 60, unit: 'MINUTES') + parallelsAlwaysFailFast() + } + stages { + stage('Debug info') { + steps { + sh label: 'Debug info', script: """ + printenv + echo "-------------------------" + go env + echo "-------------------------" + echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash" + """ + container(name: 'net-tool') { + sh 'dig github.com' + script { + prow.setPRDescription(REFS) + } + } + } + } + stage('Checkout') { + options { timeout(time: 10, unit: 'MINUTES') } + steps { + dir("ticdc") { + cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + retry(2) { + script { + prow.checkoutRefs(REFS) + } + } + } + } + } + } + stage("prepare") { + options { timeout(time: 20, unit: 'MINUTES') } + steps { + dir("third_party_download") { + script { + def tidbBranch = component.computeBranchFromPR('tidb', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def pdBranch = component.computeBranchFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tikvBranch = component.computeBranchFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + def tiflashBranch = component.computeBranchFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref) + retry(2) { + sh label: "download third_party", script: """ + export TIDB_BRANCH=${tidbBranch} + export PD_BRANCH=${pdBranch} + export TIKV_BRANCH=${tikvBranch} + export TIFLASH_BRANCH=${tiflashBranch} + cd ../ticdc && ./tests/scripts/download-integration-test-binaries.sh ${REFS.base_ref} && ls -alh ./bin + make check_third_party_binary + cd - && mkdir -p bin && mv ../ticdc/bin/* ./bin/ + ls -alh ./bin + ./bin/tidb-server -V + ./bin/pd-server -V + ./bin/tikv-server -V + ./bin/tiflash --version + """ + } + } + } + dir("ticdc") { + cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'cdc-storage-integration')) { + // build cdc, kafka_consumer, storage_consumer, cdc.test for integration test + // only build binarys if not exist, use the cached binarys if exist + sh label: "prepare", script: """ + ls -alh ./bin + [ -f ./bin/cdc ] || make cdc + [ -f ./bin/cdc_kafka_consumer ] || make kafka_consumer + [ -f ./bin/cdc_storage_consumer ] || make storage_consumer + [ -f ./bin/cdc.test ] || make integration_test_build + ls -alh ./bin + ./bin/cdc version + """ + } + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "prepare", script: """ + cp -r ../third_party_download/bin/* ./bin/ + ls -alh ./bin + """ + } + } + } + } + + stage('Tests') { + matrix { + axes { + axis { + name 'TEST_GROUP' + values 'G00', 'G01', 'G02', 'G03', 'G04', 'G05', 'G06', 'G07', 'G08', 'G09', + 'G10', 'G11', 'G12', 'G13', 'G14', 'G15' + } + } + agent{ + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + stages { + stage("Test") { + options { timeout(time: 40, unit: 'MINUTES') } + steps { + dir('ticdc') { + cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/ticdc") { + sh label: "${TEST_GROUP}", script: """ + ./tests/integration_tests/run_light_it_in_ci.sh storage ${TEST_GROUP} + """ + } + } + } + post { + failure { + sh label: "collect logs", script: """ + ls /tmp/tidb_cdc_test/ + tar --warning=no-file-changed -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/tidb_cdc_test/ -type f -name "*.log") + ls -alh log-${TEST_GROUP}.tar.gz + """ + archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true + } + } + } + } + } + } + } +} diff --git a/prow-jobs/kustomization.yaml b/prow-jobs/kustomization.yaml index b3a5d8e47b..263fbe7268 100644 --- a/prow-jobs/kustomization.yaml +++ b/prow-jobs/kustomization.yaml @@ -32,6 +32,7 @@ configMapGenerator: - pingcap_ng-monitoring_postsubmits.yaml=pingcap/ng-monitoring/postsubmits.yaml - pingcap_ticdc_latest-presubmits-next-gen.yaml=pingcap/ticdc/latest-presubmits-next-gen.yaml - pingcap_ticdc_latest-presubmits.yaml=pingcap/ticdc/latest-presubmits.yaml + - pingcap_ticdc_release-8.5-presubmits.yaml=pingcap/ticdc/release-8.5-presubmits.yaml - pingcap_ticdc_release-9.0-beta-presubmits.yaml=pingcap/ticdc/release-9.0-beta-presubmits.yaml - pingcap_tidb-binlog_latest-presubmits.yaml=pingcap/tidb-binlog/latest-presubmits.yaml - pingcap_tidb-engine-ext_latest-presubmits.yaml=pingcap/tidb-engine-ext/latest-presubmits.yaml diff --git a/prow-jobs/pingcap/ticdc/release-8.5-presubmits.yaml b/prow-jobs/pingcap/ticdc/release-8.5-presubmits.yaml new file mode 100644 index 0000000000..2b95b20bee --- /dev/null +++ b/prow-jobs/pingcap/ticdc/release-8.5-presubmits.yaml @@ -0,0 +1,173 @@ +global_definitions: + branches: &branches + - ^release-8\.5(\.\d)?$ + skip_if_only_changed: &skip_if_only_changed "(\\.(md|png|jpeg|jpg|gif|svg|pdf)|Dockerfile|OWNERS|OWNERS_ALIASES)$" + +# struct ref: https://pkg.go.dev/sigs.k8s.io/prow/pkg/config#Presubmit +presubmits: + pingcap/ticdc: + - name: pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_light + agent: jenkins + # skip_if_only_changed: *skip_if_only_changed + run_before_merge: true + context: pull-cdc-mysql-integration-light + trigger: "(?m)^/test (?:.*? )?(pull-cdc-mysql-integration-light|all)(?: .*?)?$" + rerun_command: "/test pull-cdc-mysql-integration-light" + branches: *branches + + - name: pingcap/ticdc/release-8.5/pull_cdc_mysql_integration_heavy + agent: jenkins + decorate: false # need add this. + # skip_if_only_changed: *skip_if_only_changed + run_before_merge: true + context: pull-cdc-mysql-integration-heavy + trigger: "(?m)^/test (?:.*? )?(pull-cdc-mysql-integration-heavy|all)(?: .*?)?$" + rerun_command: "/test pull-cdc-mysql-integration-heavy" + branches: *branches + + - name: pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_light + agent: jenkins + decorate: false # need add this. + # skip_if_only_changed: *skip_if_only_changed + run_before_merge: true + context: pull-cdc-kafka-integration-light + trigger: "(?m)^/test (?:.*? )?(pull-cdc-kafka-integration-light|all)(?: .*?)?$" + rerun_command: "/test pull-cdc-kafka-integration-light" + branches: *branches + + - name: pingcap/ticdc/release-8.5/pull_cdc_kafka_integration_heavy + agent: jenkins + decorate: false # need add this. + # skip_if_only_changed: *skip_if_only_changed + run_before_merge: true + context: pull-cdc-kafka-integration-heavy + trigger: "(?m)^/test (?:.*? )?(pull-cdc-kafka-integration-heavy|all)(?: .*?)?$" + rerun_command: "/test pull-cdc-kafka-integration-heavy" + branches: *branches + + - name: pingcap/ticdc/release-8.5/pull_cdc_pulsar_integration_light + agent: jenkins + decorate: false # need add this. + # skip_if_only_changed: *skip_if_only_changed + run_before_merge: true + context: pull-cdc-pulsar-integration-light + optional: true # TODO: change this after the job is ready + trigger: "(?m)^/test (?:.*? )?(pull-cdc-pulsar-integration-light|all)(?: .*?)?$" + rerun_command: "/test pull-cdc-pulsar-integration-light" + branches: *branches + + - name: pingcap/ticdc/release-8.5/pull_cdc_storage_integration_light + agent: jenkins + decorate: false # need add this. + # skip_if_only_changed: *skip_if_only_changed + run_before_merge: true + context: pull-cdc-storage-integration-light + trigger: "(?m)^/test (?:.*? )?(pull-cdc-storage-integration-light|all)(?: .*?)?$" + rerun_command: "/test pull-cdc-storage-integration-light" + branches: *branches + + - name: pingcap/ticdc/release-8.5/pull_cdc_storage_integration_heavy + agent: jenkins + decorate: false # need add this. + # skip_if_only_changed: *skip_if_only_changed + run_before_merge: true + context: pull-cdc-storage-integration-heavy + trigger: "(?m)^/test (?:.*? )?(pull-cdc-storage-integration-heavy|all)(?: .*?)?$" + rerun_command: "/test pull-cdc-storage-integration-heavy" + branches: *branches + + - name: pull-unit-test + decorate: true # need add this. + skip_if_only_changed: *skip_if_only_changed + branches: + - ^master$ + spec: + containers: + - name: check + image: &image ghcr.io/pingcap-qe/ci/base:v2024.10.8-32-ge807718-go1.23 + command: [bash, -ce] + args: + - | + make unit_test_in_verify_ci + resources: + requests: + memory: 24Gi + cpu: "12" + limits: + memory: 24Gi + cpu: "12" + + - name: pull-build + decorate: true # need add this. + skip_if_only_changed: *skip_if_only_changed + branches: + - ^master$ + spec: + containers: + - name: build + image: *image + command: [bash, -ce] + args: + - | + make cdc + resources: &resources + requests: + memory: 12Gi + cpu: "6" + limits: + memory: 24Gi + cpu: "6" + + - name: pull-check + decorate: true # need add this. + skip_if_only_changed: *skip_if_only_changed + branches: + - ^master$ + spec: + containers: + - name: check + image: *image + command: [bash, -ce] + args: + - | + make check + resources: *resources + + - name: pull-error-log-review + branches: + - ^master$ + decorate: true + decoration_config: + timeout: 20m + skip_cloning: true + trigger: "(?m)^/test (?:.*? )?pull-error-log-review(?: .*?)?$" + rerun_command: "/test pull-error-log-review" + skip_if_only_changed: *skip_if_only_changed + spec: + containers: + - name: check + image: ghcr.io/pingcap-qe/ci/base:v2024.10.8-32-ge807718-go1.23 + command: [/bin/bash, -ce] + args: + - | + if [[ "${JOB_TYPE:-}" == "batch" ]]; then + echo "JOB_TYPE is batch, skipping error log review check" + exit 0 + fi + wget -O /tmp/config.yaml https://raw.githubusercontent.com/pingcap-qe/ci/main/configs/error-log-review/config.yaml + go run github.com/PingCAP-QE/ci/tools/error-log-review@main \ + -config /tmp/config.yaml \ + -pr "${REPO_OWNER}/${REPO_NAME}#${PULL_NUMBER}" + env: + - name: GITHUB_API_TOKEN + valueFrom: + secretKeyRef: + key: token + name: github-token + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "1" + memory: 2Gi