Revert "ci: install upstream ddtrace wheel during PR-triggered unit tests"

ci/input_files/build.yaml.tpl

@@ -7,6 +7,13 @@ stages:
  - publish
  - e2e
 
+.python-before-script: &python-before-script
+  - pip install virtualenv
+  - virtualenv venv
+  - source venv/bin/activate
+  - pip install .[dev]
+  - pip install poetry
+
 default:
   retry:
     max: 1
@@ -66,8 +73,7 @@ lint python:
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
   cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-  before_script:
-    - PYTHON_VERSION={{ $runtime.python_version }} ./scripts/setup_python_env.sh
+  before_script: *python-before-script
   script:
     - source venv/bin/activate
     - ./scripts/check_format.sh
@@ -77,8 +83,7 @@ unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
   cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-  before_script:
-    - PYTHON_VERSION={{ $runtime.python_version }} ./scripts/setup_python_env.sh
+  before_script: *python-before-script
   script:
     - source venv/bin/activate
     - pytest -vv
@@ -190,8 +195,7 @@ publish-pypi-package:
   stage: publish
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10-py3
-  before_script:
-    - ./scripts/setup_python_env.sh
+  before_script: *python-before-script
   cache: []
   rules:
     - if: '$CI_COMMIT_TAG =~ /^v.*/'

scripts/build_layers.sh

@@ -88,16 +88,68 @@ cleanup() {
 }
 trap cleanup EXIT
 
-# Source the shared ddtrace-dep specification logic. spec_ddtrace_dep reads
-# DD_TRACE_COMMIT / DD_TRACE_COMMIT_BRANCH / DD_TRACE_WHEEL / UPSTREAM_PIPELINE_ID
-# (PYTHON_VERSION + ARCH for the S3 path) and rewrites the ddtrace dep block
-# in pyproject.toml.
-source "$(dirname "$0")/_spec_ddtrace_dep.sh"
+# Helper: replace the multi-line ddtrace dependency in pyproject.toml.
+# Uses perl instead of sed -z for macOS/Linux portability.
+replace_ddtrace_dep() {
+    echo "Replacing dep with $1"
+    perl -i -0777 -pe "s|ddtrace = \[[^\]]*\]|$1|gs" pyproject.toml
+}
 
 function make_path_absolute {
     echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
 }
 
+function search_wheel {
+    # Args: [wheel base name] [index]
+
+    WHEEL_BASENAME=$1
+    INDEX=$2
+
+    SEARCH_PATTERN="${WHEEL_BASENAME}-[^\"]*${PY_TAG}[^\"]*${PLATFORM}[^\"]*\.whl"
+    INDEX_URL="${S3_BASE}/index-${INDEX}.html" 
+    echo "Searching for wheel ${SEARCH_PATTERN}"
+    export WHEEL_FILE=$(curl -sSfL ${INDEX_URL} | grep -o "$SEARCH_PATTERN" | head -n 1)
+    if [ ! -z "${WHEEL_FILE}" ]; then
+        curl -sSfL "${S3_BASE}/${WHEEL_FILE}" -o "${WHEEL_FILE}"
+        echo "Using S3 wheel: ${WHEEL_FILE}"
+        replace_ddtrace_dep "${WHEEL_BASENAME} = { file = \"${WHEEL_FILE}\" }"
+    fi
+}
+
+function find_and_spec_wheel {
+    # Args: [python version] [wheel base name] [index]
+
+    arch=$2
+    wheel_basename=$3
+    index=$4
+
+    # Restore pyproject.toml to a clean state for each build iteration
+    cp pyproject.toml.bak pyproject.toml
+
+    # Replace ddtrace source if necessary
+    if [ -n "$DD_TRACE_COMMIT" ]; then
+        replace_ddtrace_dep "${wheel_basename} = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"$DD_TRACE_COMMIT\" }"
+    elif [ -n "$DD_TRACE_COMMIT_BRANCH" ]; then
+        replace_ddtrace_dep "${wheel_basename} = { git = \"https://github.com/DataDog/dd-trace-py.git\", branch = \"$DD_TRACE_COMMIT_BRANCH\" }"
+    elif [ -n "$DD_TRACE_WHEEL" ]; then
+        wheel_basename=$(sed 's/^.*\///' <<< ${DD_TRACE_WHEEL%%-*})
+        replace_ddtrace_dep "${wheel_basename} = { file = \"$DD_TRACE_WHEEL\" }"
+    elif [ -n "$UPSTREAM_PIPELINE_ID" ]; then
+        S3_BASE="https://dd-trace-py-builds.s3.amazonaws.com/${UPSTREAM_PIPELINE_ID}"
+        if [ "${arch}" = "amd64" ]; then
+            PLATFORM="manylinux2014_x86_64"
+        else
+            PLATFORM="manylinux2014_aarch64"
+        fi
+        PY_TAG="cp$(echo "$1" | tr -d '.')"
+        search_wheel ${wheel_basename} ${index}
+        if [ -z "${WHEEL_FILE}" ]; then
+            echo "No S3 wheel found for ${PY_TAG} ${PLATFORM}, using default pyproject.toml version"
+            return 1
+        fi
+    fi
+}
+
 function docker_build_zip {
     # Args: [python version] [zip destination]
 
@@ -128,10 +180,14 @@ do
     for architecture in "${ARCHS[@]}"
     do
         echo "Building layer for Python ${python_version} arch=${architecture}"
-        # Restore pyproject.toml to a clean state before each iteration so the
-        # rewrite is deterministic regardless of what the previous loop did.
-        cp pyproject.toml.bak pyproject.toml
-        PYTHON_VERSION="${python_version}" ARCH="${architecture}" spec_ddtrace_dep
+        set +e
+        find_and_spec_wheel ${python_version} ${architecture} "ddtrace_serverless" "serverless"
+        FAILURE=$?
+        if [ $FAILURE != 0 ]; then
+            echo "Attempting layer build again with package ddtrace"
+            find_and_spec_wheel ${python_version} ${architecture} "ddtrace" "manylinux2014"
+        fi
+        set -e
         docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-${architecture}-${python_version}.zip ${architecture}
     done
 done