mac signing: install python 3.14.6 via uv and rebuild venvs when the pin moves

data/common.yaml

@@ -198,6 +198,9 @@ scriptworker_config:
         verify_chain_of_trust: true
         verify_cot_signature: true
         scriptworker_scripts_revision: "1980cf7349ba5604943cdba88e3fdd79a3aaedec"
+        # Pins both the uv-managed interpreter and the venvs built from it. A bump
+        # here rebuilds every scriptworker venv on the next puppet run.
+        python_version: "3.14.6"
 
     ff-prod:
         <<: *defaults

modules/packages/manifests/uvpython.pp

@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Installs a uv-managed CPython (a python-build-standalone build) into
+# $install_dir and points `python`, `python3` and `python3.<minor>` in $bin_dir
+# at it.
+#
+# uv itself is NOT managed here: the caller installs uv and orders this class
+# after it.
+class packages::uvpython (
+  Pattern[/^\d+\.\d+\.\d+$/] $version,
+  String $uv_bin      = '/usr/local/bin/uv',
+  String $install_dir = '/opt/tools/python',
+  String $bin_dir     = '/usr/local/bin',
+  String $platform    = 'macos-aarch64-none',
+) {
+  $short_version = split($version, '[.]')[0, 2].join('.')
+  $interpreter = "${install_dir}/cpython-${version}-${platform}/bin/python${short_version}"
+
+  # 0755 so the scriptworker users can run the interpreter root installs here.
+  file { $install_dir:
+    ensure => 'directory',
+    owner  => 'root',
+    group  => 'wheel',
+    mode   => '0755',
+  }
+
+  # --no-bin because the links are managed below instead. uv's own `--default`
+  # links resolve through a `cpython-<minor>-<platform>` symlink that it only
+  # ever advances to newer patches: pointing at that would leave whatever patch
+  # was installed last in charge rather than $version, and pinning back to an
+  # older patch would never converge.
+  exec { "install uv-managed python ${version}":
+    command     => "${uv_bin} python install --no-bin ${version}",
+    environment => ["UV_PYTHON_INSTALL_DIR=${install_dir}"],
+    creates     => $interpreter,
+    path        => ['/usr/bin', '/bin'],
+    timeout     => 600,
+    require     => File[$install_dir],
+  }
+
+  # force, so these can take over the links a python.org framework install
+  # leaves in $bin_dir.
+  ['python', 'python3', "python${short_version}"].each |String $link| {
+    file { "${bin_dir}/${link}":
+      ensure  => 'link',
+      target  => $interpreter,
+      force   => true,
+      require => Exec["install uv-managed python ${version}"],
+    }
+  }
+}

modules/reprovision_runner/manifests/init.pp

@@ -152,6 +152,7 @@
     }
 
     # ---- python 3.11 (framework build, same source as scriptworker_prereqs) ----
+    # TODO: python install moved to UV - investigate if this needs to be updated
     class { 'packages::python3':
       version => $python_version,
     }

modules/roles_profiles/manifests/profiles/mac_signing.pp

@@ -14,8 +14,6 @@
 
       include dirs::tools
 
-      class { 'scriptworker_prereqs': }
-
       $widevine_user = lookup('widevine_config.user')
       $widevine_key = lookup('widevine_config.key')
 
@@ -33,6 +31,11 @@
       $worker_common = lookup("scriptworker_config.${role}", Hash, undef, {})
       $worker_secrets = lookup("scriptworker_secrets.${role}", Hash, undef, {})
       $worker_config = deep_merge($worker_common, $worker_secrets)
+      $python_version = $worker_config['python_version']
+
+      class { 'scriptworker_prereqs':
+        python_version => $python_version,
+      }
 
       $role_common = lookup("signingworker_roles.${role}", Hash, undef, {})
       $role_secrets = lookup("signing_secrets.${role}", Hash, undef, {})
@@ -90,7 +93,12 @@
           keychain_filename   => $user_data['keychain_filename'],
           worker_config       => $worker_config,
           role_config         => $role_config,
+          python_version      => $python_version,
           ed_key_filename     => $user_data['ed_key_filename'],
+          # If the uv-managed interpreter fails to install, skip the worker
+          # outright. Without this edge the worker's stop exec still runs and
+          # takes the signer down for a venv rebuild that cannot succeed.
+          require             => Class['scriptworker_prereqs'],
         }
       }
 

modules/scriptworker_prereqs/manifests/init.pp

@@ -1,104 +1,81 @@
-class scriptworker_prereqs {
+class scriptworker_prereqs (
+  String $python_version,
+) {
 
   # Determine macOS version
   $mac_version = $facts['os']['release']['major']
 
-  # Select Python version based on OS
-  $python_version = $mac_version ? {
-    '18'    => '3.8.3',
-    '19'    => '3.8.3',
-    '21'    => '3.11.0',
-    '23'    => '3.11.0',
-    '24'    => '3.11.0',
-    default => fail("Unsupported macOS version: ${mac_version}"),
+  # Everything below is aarch64-only: both the uv release asset and the
+  # uv-managed python build are pinned to that architecture. Facter spells it
+  # arm64 on Darwin.
+  $arch = $facts['os']['architecture']
+  unless $arch in ['aarch64', 'arm64'] {
+    fail("Unsupported CPU architecture: ${arch} - scriptworker_prereqs requires aarch64")
   }
 
-  # Install appropriate Python version
-  class { 'packages::python3':
-    version => $python_version,
-  }
-
-  # Install virtualenv and setup tools symlink for macOS 14+ (mac_version 21 or 23)
-  if $mac_version in ['21', '23', '24'] {
-    exec { 'install python3 virtualenv':
-      command => '/Library/Frameworks/Python.framework/Versions/3.11/bin/python3 -m pip install virtualenv',
-      unless  => '/Library/Frameworks/Python.framework/Versions/3.11/bin/python3 -m virtualenv --version',
-      path    => ['/Library/Frameworks/Python.framework/Versions/3.11/bin', '/usr/bin', '/bin'],
-      user    => 'root',
-      require => Class['packages::python3'],
-    }
-
-    file { '/usr/local/tools':
-      ensure => 'directory',
-      owner  => 'root',
-      group  => 'wheel',
-      mode   => '0755',
-    }
-
-    file { '/usr/local/tools/python3':
-      ensure  => 'link',
-      target  => '/usr/local/bin/python3',
-      require => [Class['packages::python3'], File['/usr/local/tools']],
-    }
-
-    $uv_version = '0.8.2'
-    $uv_checksum = '954d24634d5f37fa26c7af75eb79893d11623fc81b4de4b82d60d1ade4bfca22'
-    # Note: This is hard-coded to aarch64 architecture - we currently don't have plans on supporting old signers
-
-    file { "/opt/tools/uv-${uv_version}":
-      ensure => 'directory',
-      owner  => 'root',
-      group  => 'wheel',
-      mode   => '0755',
-    }
-
-    archive { 'install UV':
-      path            => '/tmp/uv.tar.gz',
-      source          => "https://github.com/astral-sh/uv/releases/download/${uv_version}/uv-aarch64-apple-darwin.tar.gz",
-      checksum        => $uv_checksum,
-      checksum_type   => 'sha256',
-      extract_command => 'tar xfz %s --strip-components=1',
-      extract         => true,
-      extract_path    => "/opt/tools/uv-${uv_version}",
-      creates         => "/opt/tools/uv-${uv_version}/uv",
-      require         => File["/opt/tools/uv-${uv_version}"],
+  case $mac_version {
+    # macOS 14+ (aarch64). uv owns both the venv tooling and the interpreter.
+    '21', '23', '24': {
+      $uv_version = '0.12.4'
+      $uv_checksum = '99a913b606194867b43086404412c1afe079547fee72ecfb6af7e7b0dd54b0c6'
+
+      file { "/opt/tools/uv-${uv_version}":
+        ensure => 'directory',
+        owner  => 'root',
+        group  => 'wheel',
+        mode   => '0755',
+      }
+
+      archive { 'install UV':
+        path            => '/tmp/uv.tar.gz',
+        source          => "https://github.com/astral-sh/uv/releases/download/${uv_version}/uv-aarch64-apple-darwin.tar.gz",
+        checksum        => $uv_checksum,
+        checksum_type   => 'sha256',
+        extract_command => 'tar xfz %s --strip-components=1',
+        extract         => true,
+        extract_path    => "/opt/tools/uv-${uv_version}",
+        creates         => "/opt/tools/uv-${uv_version}/uv",
+        require         => File["/opt/tools/uv-${uv_version}"],
+      }
+
+      file { '/usr/local/bin/uv':
+        ensure  => 'link',
+        target  => "/opt/tools/uv-${uv_version}/uv",
+        require => Archive['install UV'],
+      }
+
+      # Scoped to uv-* so it leaves the uv-managed pythons (/opt/tools/python) alone.
+      exec { 'cleanup other UV installs':
+        command     => "find /opt/tools/ -mindepth 1 -maxdepth 1 -type d -name 'uv-*' ! -name \"uv-${uv_version}\" -exec rm -rf {} +",
+        onlyif      => 'ls /opt/tools/uv-*',
+        path        => ['/usr/bin', '/bin'],
+        subscribe   => Archive['install UV'],
+        refreshonly => true,
+      }
+
+      # The interpreter the scriptworker venvs are built from. `uv venv` picks it
+      # up through the python3 link this drops in /usr/local/bin.
+      class { 'packages::uvpython':
+        version => $python_version,
+        require => File['/usr/local/bin/uv'],
+      }
+
+      file { '/usr/local/tools':
+        ensure => 'directory',
+        owner  => 'root',
+        group  => 'wheel',
+        mode   => '0755',
+      }
+
+      file { '/usr/local/tools/python3':
+        ensure  => 'link',
+        target  => '/usr/local/bin/python3',
+        require => [Class['packages::uvpython'], File['/usr/local/tools']],
+      }
     }
-
-    file { '/usr/local/bin/uv':
-      ensure  => 'link',
-      target  => "/opt/tools/uv-${uv_version}/uv",
-      require => Archive['install UV'],
-    }
-
-    exec { 'cleanup other UV installs':
-      command     => "find /opt/tools/ -mindepth 1 -maxdepth 1 -type d ! -name \"uv-${uv_version}\" -exec rm -rf {} +",
-      onlyif      => 'ls /opt/tools/uv-*',
-      path        => ['/usr/bin', '/bin'],
-      subscribe   => Archive['install UV'],
-      refreshonly => true,
-    }
-  }
-
-  # Legacy dirs and symlink for older macOS (macOS 10.14 and 10.15)
-  if $mac_version in ['18', '19'] {
-    file { '/tools':
-      ensure => 'directory',
-      owner  => 'root',
-      group  => 'wheel',
-      mode   => '0755',
-    }
-
-    file { '/tools/python3':
-      ensure  => 'link',
-      target  => '/usr/local/bin/python3',
-      require => [Class['packages::python3'], File['/tools']],
-    }
-
-    file { '/builds':
-      ensure => 'directory',
-      owner  => 'root',
-      group  => 'wheel',
-      mode   => '0755',
+    # Older versions of macos are not supported.
+    default: {
+      fail("Unsupported macOS version: ${mac_version}")
     }
   }