diff --git a/.github/workflows/util/install-spark-resources.sh b/.github/workflows/util/install-spark-resources.sh index 31defd69b845..a00c684a03a8 100755 --- a/.github/workflows/util/install-spark-resources.sh +++ b/.github/workflows/util/install-spark-resources.sh @@ -124,7 +124,7 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then # Spark-4.x, scala 2.12 // using 2.12 as a hack as 4.0 does not have 2.13 suffix cd ${INSTALL_DIR} && \ install_spark "4.1.1" "3" "2.12" - mv /opt/shims/spark40/spark_home/assembly/target/scala-2.12 /opt/shims/spark40/spark_home/assembly/target/scala-2.13 + mv /opt/shims/spark41/spark_home/assembly/target/scala-2.12 /opt/shims/spark41/spark_home/assembly/target/scala-2.13 ;; *) echo "Spark version is expected to be specified." diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 75b700ad9be2..be84662e1a50 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -1363,11 +1363,10 @@ jobs: pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.5.5 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Prepare Spark Resources for Spark 4.1.0 #TODO remove after image update + - name: Prepare Spark Resources for Spark 4.1.1 #TODO remove after image update run: | rm -rf /opt/shims/spark41 bash .github/workflows/util/install-spark-resources.sh 4.1 - mv /opt/shims/spark41/spark_home/assembly/target/scala-2.12 /opt/shims/spark41/spark_home/assembly/target/scala-2.13 - name: Build and Run unit test for Spark 4.1.0 with scala-2.13 (other tests) run: | cd $GITHUB_WORKSPACE/ @@ -1413,11 +1412,10 @@ jobs: with: name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - - name: Prepare Spark Resources for Spark 4.1.0 #TODO remove after image update + - name: Prepare Spark Resources for Spark 4.1.1 #TODO remove after image update run: | rm -rf /opt/shims/spark41 bash .github/workflows/util/install-spark-resources.sh 4.1 - mv /opt/shims/spark41/spark_home/assembly/target/scala-2.12 /opt/shims/spark41/spark_home/assembly/target/scala-2.13 - name: Build and Run unit test for Spark 4.0 (slow tests) run: | cd $GITHUB_WORKSPACE/