debug

.github/workflows/conda-python-build.yaml

@@ -27,6 +27,10 @@ on:
       upload_to_conda:
         type: boolean
         required: false
+      sccache_local_only:
+        description: "Use sccache on local disk only (no S3); use with Actions cache of ~/.cache/sccache to verify hits."
+        type: boolean
+        default: true
 defaults:
   run:
     shell: bash
@@ -90,6 +94,7 @@ jobs:
       RAPIDS_ARTIFACTS_DIR: ${{ github.workspace }}/artifacts
       RAPIDS_CONDA_BLD_OUTPUT_DIR: /tmp/conda-bld-output
       RAPIDS_CONDA_BLD_ROOT_DIR: /tmp/conda-bld-workspace
+      NUMBAST_SCCACHE_LOCAL_ONLY: ${{ inputs.sccache_local_only && '1' || '0' }}
     container:
       image: condaforge/miniforge3:26.1.1-3
       env:
@@ -105,10 +110,21 @@ jobs:
           repository: ${{ inputs.repo }}
           ref: ${{ inputs.sha }}
           fetch-depth: 0
+      - name: Cache apt archives
+        uses: actions/cache@v5
+        with:
+          path: /var/cache/apt/archives
+          key: apt-archives-${{ matrix.ARCH }}-${{ hashFiles('.github/workflows/conda-python-build.yaml') }}
+          restore-keys: |
+            apt-archives-${{ matrix.ARCH }}-
       - name: Install required packages
+        env:
+          DEBIAN_FRONTEND: noninteractive
         run: |
-          apt update
-          apt install -y jq gh unzip curl gettext
+          set -euo pipefail
+          rm -f /etc/apt/apt.conf.d/docker-clean
+          apt-get update
+          apt-get install -y jq gh unzip curl gettext
       - name: Add Rapids tools
         run: |
           mkdir gha-tools
@@ -126,6 +142,17 @@ jobs:
           SCCACHE_VERSION: 0.14.0-rapids.1
           GH_TOKEN: ${{ github.token }}
         run: rapids-install-sccache
+      # - name: Cache conda caches
+      #   id: cache-conda
+      #   uses: actions/cache@v5
+      #   with:
+      #     path: /opt/conda/pkgs
+      #     key: conda-cache-${{ matrix.ARCH }}-${{ matrix.PY_VER }}
+      # - name: Check conda cache
+      #   if: steps.cache-conda.outputs.cache-hit == 'true'
+      #   run: |
+      #     ls -lh /opt/conda/pkgs
+      #     du -sh /opt/conda/pkgs
       - name: Update conda and install conda build
         run: |
           set -x
@@ -162,6 +189,18 @@ jobs:
           echo "GIT_DESCRIBE_TAG: ${GIT_DESCRIBE_TAG}"
           echo "git describe: $(git describe)"
           echo "git count commits since tag: $(git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count)"
+      - name: Cache sccache
+        id: cache-sccache
+        uses: actions/cache@v5
+        with:
+          path: /github/home/.cache/sccache
+          key: 2-sccache-${{ matrix.ARCH }}-${{ matrix.PY_VER }}
+      - name: Check sccache
+        if: steps.cache-sccache.outputs.cache-hit == 'true'
+        run: |
+          set -x
+          ls -lh /github/home/.cache/sccache
+          du -sh /github/home/.cache/sccache
       - name: Build Conda Package
         run: ${{ inputs.script }}
         env:

.github/workflows/trace.md

@@ -0,0 +1,19 @@
+Comparing main branch to debug branch
+Initialize container went down from   38s (before merge) down to 9s
+Install required packages:            8s     -> 1s (hit) + 6s + 0s
+                                             -> 0s (miss) + 8s + ?
+Update conda and install conda build: 32s    -> 0s (miss) + 37s + 21s
+                                             -> 1m 11s (hit) + 42s +
+Install python:                       6s     -> 7s
+Build conda package:                  8m 32s -> 1s + 9m 28s + 0s (hit)
+
+Cache not found for input keys: conda-cache-amd64-3.10
+
+Cache Size: ~1 MB (539303 B)
+Cache restored from key: sccache-amd64-3.10
+
+Cache hit occurred on the primary key sccache-amd64-3.10, not saving cache.
+
+Cache saved with key: conda-cache-amd64-3.10
+
+echo

ci/build_conda_python.sh

@@ -4,8 +4,17 @@
 
 set -euo pipefail
 
+# Maximum sccache verbosity (includes per-request / storage detail). Remove after debugging.
+export SCCACHE_LOG=trace
+
 source rapids-configure-sccache
 
+# Disk-only cache (no S3). Set NUMBAST_SCCACHE_LOCAL_ONLY=1 from CI or locally.
+if [[ "${NUMBAST_SCCACHE_LOCAL_ONLY:-0}" == "1" ]]; then
+  unset SCCACHE_BUCKET SCCACHE_REGION SCCACHE_S3_KEY_PREFIX SCCACHE_S3_USE_SSL SCCACHE_S3_NO_CREDENTIALS
+  export SCCACHE_DIR="${SCCACHE_DIR:-${HOME}/.cache/sccache}"
+fi
+
 source rapids-date-string
 
 export CMAKE_GENERATOR=Ninja

conda/recipes/ast_canopy/meta.yaml

@@ -5,6 +5,7 @@
 {% set project_data = data.get("project") %}
 {% set project_urls = project_data["urls"] %}
 {% set py_version = environ['CONDA_PY'] %}
+{% set sccache_use_s3 = environ.get('NUMBAST_SCCACHE_LOCAL_ONLY', '0') != '1' %}
 
 package:
   name: ast_canopy
@@ -22,13 +23,18 @@ build:
     - CMAKE_CXX_COMPILER_LAUNCHER
     - CMAKE_GENERATOR
     - PARALLEL_LEVEL
+{% if sccache_use_s3 %}
     - SCCACHE_BUCKET
     - SCCACHE_IDLE_TIMEOUT
     - SCCACHE_REGION
     - SCCACHE_S3_KEY_PREFIX=numbast-aarch64 # [aarch64]
     - SCCACHE_S3_KEY_PREFIX=numbast-linux64 # [linux64]
     - SCCACHE_S3_USE_SSL
     - SCCACHE_S3_NO_CREDENTIALS
+{% else %}
+    - SCCACHE_IDLE_TIMEOUT
+    - SCCACHE_DIR
+{% endif %}
 
 source:
   path: ../../..

conda/recipes/numbast/meta.yaml

@@ -5,6 +5,7 @@
 {% set project_data = data.get("project") %}
 {% set project_urls = project_data["urls"] %}
 {% set py_version = environ['CONDA_PY'] %}
+{% set sccache_use_s3 = environ.get('NUMBAST_SCCACHE_LOCAL_ONLY', '0') != '1' %}
 
 package:
   name: numbast
@@ -25,13 +26,18 @@ build:
     - CMAKE_CXX_COMPILER_LAUNCHER
     - CMAKE_GENERATOR
     - PARALLEL_LEVEL
+{% if sccache_use_s3 %}
     - SCCACHE_BUCKET
     - SCCACHE_IDLE_TIMEOUT
     - SCCACHE_REGION
     - SCCACHE_S3_KEY_PREFIX=numbast-aarch64 # [aarch64]
     - SCCACHE_S3_KEY_PREFIX=numbast-linux64 # [linux64]
     - SCCACHE_S3_USE_SSL
     - SCCACHE_S3_NO_CREDENTIALS
+{% else %}
+    - SCCACHE_IDLE_TIMEOUT
+    - SCCACHE_DIR
+{% endif %}
   ignore_run_exports_from:
     - libnvjitlink-dev
 

conda/recipes/numbast_extensions/meta.yaml

@@ -5,6 +5,7 @@
 {% set project_data = data.get("project") %}
 {% set project_urls = project_data["urls"] %}
 {% set py_version = environ['CONDA_PY'] %}
+{% set sccache_use_s3 = environ.get('NUMBAST_SCCACHE_LOCAL_ONLY', '0') != '1' %}
 
 package:
   name: numbast-extensions
@@ -25,13 +26,18 @@ build:
     - CMAKE_CXX_COMPILER_LAUNCHER
     - CMAKE_GENERATOR
     - PARALLEL_LEVEL
+{% if sccache_use_s3 %}
     - SCCACHE_BUCKET
     - SCCACHE_IDLE_TIMEOUT
     - SCCACHE_REGION
     - SCCACHE_S3_KEY_PREFIX=numbast-aarch64 # [aarch64]
     - SCCACHE_S3_KEY_PREFIX=numbast-linux64 # [linux64]
     - SCCACHE_S3_USE_SSL
     - SCCACHE_S3_NO_CREDENTIALS
+{% else %}
+    - SCCACHE_IDLE_TIMEOUT
+    - SCCACHE_DIR
+{% endif %}
 
 requirements:
   build: