diff --git a/ci/az-relogin.ps1 b/ci/az-relogin.ps1 new file mode 100644 index 00000000..c973336d --- /dev/null +++ b/ci/az-relogin.ps1 @@ -0,0 +1,61 @@ +<# +.SYNOPSIS + Re-authenticate the Azure CLI mid-job using the GitHub Actions OIDC token. + +.DESCRIPTION + The workflow authenticates once via azure/login (OIDC). That yields an Azure access + token valid for ~1h that CANNOT be refreshed. The WIM bake step runs ~1h40m+, so by + the time the completion marker appears the runner's token is dead and every + `az ... --auth-mode login` call fails silently — the poll never detects completion + (240-min timeout) and the always() teardown can't delete the VM (quota leak). + + This re-runs `az login` with a FRESH GitHub OIDC token. The job has + `permissions: id-token: write`, so any step can mint a new OIDC JWT on demand; a token + minted in the same job carries the same subject claim azure/login already succeeded + with, so the federated login succeeds again and yields a fresh ~1h Azure token. + + Client/tenant/subscription are recovered from the CACHED az profile when not passed: + `az account show` reads ~/.azure locally and works even after the access token has + expired (SP login -> user.name is the app/client id). So callers need thread NO + secrets through the workflow YAML — this fix stays entirely in the checked-out ci/ + scripts on the pipeline branch. + +.NOTES + Requires the GH OIDC request vars (present whenever the job has id-token: write): + ACTIONS_ID_TOKEN_REQUEST_URL / ACTIONS_ID_TOKEN_REQUEST_TOKEN. +#> +[CmdletBinding()] +param( + [string] $ClientId, + [string] $TenantId, + [string] $SubscriptionId +) +$ErrorActionPreference = 'Stop' + +if (-not $env:ACTIONS_ID_TOKEN_REQUEST_URL -or -not $env:ACTIONS_ID_TOKEN_REQUEST_TOKEN) { + throw 'GH OIDC token endpoint unavailable (need permissions: id-token: write on the job).' +} + +# Recover any missing identity fields from the cached az profile (survives token expiry). +if (-not ($ClientId -and $TenantId -and $SubscriptionId)) { + $acct = az account show -o json 2>$null | ConvertFrom-Json + if (-not $acct) { throw 'Cannot recover identity for OIDC re-login: no cached az profile (az account show returned nothing).' } + if (-not $ClientId) { $ClientId = $acct.user.name } # SP login: user.name = app/client id + if (-not $TenantId) { $TenantId = $acct.tenantId } + if (-not $SubscriptionId) { $SubscriptionId = $acct.id } +} +if (-not ($ClientId -and $TenantId -and $SubscriptionId)) { + throw "Incomplete identity for OIDC re-login (client='$ClientId' tenant='$TenantId' sub='$SubscriptionId')." +} + +# Mint a fresh GitHub OIDC token scoped to the Azure token-exchange audience. +$uri = "$($env:ACTIONS_ID_TOKEN_REQUEST_URL)&audience=api://AzureADTokenExchange" +$jwt = (Invoke-RestMethod -Uri $uri -Method GET -Headers @{ Authorization = "Bearer $($env:ACTIONS_ID_TOKEN_REQUEST_TOKEN)" }).value +if (-not $jwt) { throw 'GH OIDC token request returned an empty token.' } + +az login --service-principal -u $ClientId -t $TenantId --federated-token $jwt --output none +if ($LASTEXITCODE -ne 0) { throw "az login (federated OIDC) failed with exit $LASTEXITCODE." } +az account set --subscription $SubscriptionId --output none +if ($LASTEXITCODE -ne 0) { throw "az account set --subscription '$SubscriptionId' failed with exit $LASTEXITCODE." } + +Write-Host " [az-relogin] re-authenticated to Azure via GH OIDC (subscription $SubscriptionId)" diff --git a/ci/kickoff-win-hw-wim-build.ps1 b/ci/kickoff-win-hw-wim-build.ps1 new file mode 100644 index 00000000..f03b5309 --- /dev/null +++ b/ci/kickoff-win-hw-wim-build.ps1 @@ -0,0 +1,264 @@ +<# +.SYNOPSIS + Run a Windows HW WIM build on the ephemeral Azure build VM and WAIT for it. Runs on + the GitHub Actions runner (pwsh + az, already authenticated by azure/login). + +.DESCRIPTION + The build needs nested Hyper-V (can't run on the GH-hosted runner) and exceeds the + ~90-min az vm run-command limit, so this: + 1. has the VM check out worker-images at PIPELINE_REF and launch the build as a + detached scheduled task (provisioners/.../scripts/run-build-task.ps1, which + writes C:\win-hw-wim-build\build.done with the exit code when finished), then + 2. polls that marker until the build finishes, streaming the tail of the log, + 3. exits with the build's result so the workflow job passes/fails accordingly. + + The workflow creates the VM before this step and destroys it after (if: always()). + + Env (set by the workflow): + IMAGE, PIPELINE_REF, BUILD_ID (optional), VM_NAME, RESOURCE_GROUP +#> +[CmdletBinding()] +param() +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$image = $env:IMAGE; if (-not $image) { throw 'IMAGE not set' } +$ref = $env:PIPELINE_REF; if (-not $ref) { throw 'PIPELINE_REF not set' } +$buildId = $env:BUILD_ID +$stages = $env:STAGES # blank = default prep,build,publish bake; e.g. 'iso' for the ISO stage +$vm = if ($env:VM_NAME) { $env:VM_NAME } else { 'win-hw-wim-builder' } +$rg = if ($env:RESOURCE_GROUP) { $env:RESOURCE_GROUP } else { 'rg-central-us-hardware-imaging' } + +# Inputs flow into a remote script — allow only safe characters. +foreach ($v in @($image, $ref)) { if ($v -notmatch '^[A-Za-z0-9._/-]+$') { throw "Illegal input: '$v'" } } +if ($buildId -and $buildId -notmatch '^[A-Za-z0-9._-]+$') { throw "Illegal BUILD_ID: '$buildId'" } +if ($stages -and $stages -notmatch '^[A-Za-z,]+$') { throw "Illegal STAGES: '$stages'" } + +# NOTE: no quotes around the values below. This becomes a scheduled-task -Argument +# string parsed by powershell.exe -File via CommandLineToArgvW, which strips double +# quotes but keeps single quotes LITERAL (they'd end up in $Image). Inputs are already +# validated to [A-Za-z0-9._/-]+ (no spaces), so they need no quoting. +$buildArg = if ($buildId) { "-BuildId $buildId" } else { '' } +if ($stages) { $buildArg = "$buildArg -Stages $stages".Trim() } + +# The build VM has only a USER-assigned managed identity, so New-WinHwWim must log in +# with `az login --identity --username `. Resolve that client id here (this +# runner is az-authenticated via OIDC) and pass it through to the build. +$idName = if ($env:BUILDER_IDENTITY_NAME) { $env:BUILDER_IDENTITY_NAME } else { 'id-central-us-hardware-imaging-builder' } +$idClientId = (az identity show -g $rg -n $idName --query clientId -o tsv 2>$null) +if ($idClientId) { $buildArg = "$buildArg -IdentityClientId $idClientId".Trim() } +else { Write-Warning "Could not resolve client id for identity '$idName' in '$rg'; build may fail to az login." } + +# Forward a GitHub token (build-scoped) so puppet's tooltool download in the bake can +# authenticate. Set it as a MACHINE env var so the SYSTEM scheduled task inherits it and +# New-WinHwWim's -GithubPat default ($env:GITHUB_TOKEN) picks it up. Empty is fine — +# tooltool.py is public and downloads without a token. +$ghToken = ($env:GITHUB_TOKEN, $env:PACKER_GITHUB_API_TOKEN, '' | Where-Object { $_ } | Select-Object -First 1) +$ghTokenLine = if ($ghToken) { + "[Environment]::SetEnvironmentVariable('GITHUB_TOKEN', '$ghToken', 'Machine'); `$env:GITHUB_TOKEN = '$ghToken'" +} +else { "# no GitHub token provided; tooltool downloads unauthenticated (public)" } + +# --- Storage-backed completion signal ----------------------------------------- +# The bake (run-build-task.ps1) uploads a _status/.done marker (+ .log) to blob; +# we poll THAT with the runner's own az instead of polling build.done over +# `az vm run-command`, which wedges under the bake's nested-virt load and left finished +# builds undetected (job hung ~2h). Best-effort delete any stale marker from a prior run - +# but do NOT rely on it: the runner SP may lack blob-delete, so the delete can silently +# no-op (it did - a yesterday marker survived and made a run FALSE-pass in ~1 min). The +# real guard is the freshness gate below: only a marker written AFTER this kickoff counts. +$stAccount = 'hardwareimaging' +$stContainer = 'captured' +foreach ($n in @("_status/$image.done", "_status/$image.log", "_status/$image.live.log")) { + az storage blob delete --account-name $stAccount --container-name $stContainer --name $n --auth-mode login --only-show-errors 2>$null +} + +# --- OIDC re-auth setup ------------------------------------------------------- +# azure/login's Azure token lasts ~1h and can't be refreshed, but this step waits ~2h +# for the bake. Capture our identity NOW (token still fresh) so we can re-login with a +# fresh GitHub OIDC token during the poll (ci/az-relogin.ps1) and keep the blob poll +# authenticated. Not doing this is what hid the finished build for 4h (expired-token +# downloads failed silently -> the .done marker was never read -> 240-min timeout). +$reloginScript = Join-Path $PSScriptRoot 'az-relogin.ps1' +$acctNow = az account show -o json 2>$null | ConvertFrom-Json +$azClientId = if ($acctNow) { $acctNow.user.name } else { $null } +$azTenantId = if ($acctNow) { $acctNow.tenantId } else { $null } +$azSubId = if ($acctNow) { $acctNow.id } else { $null } +function Invoke-AzRelogin { + try { & $reloginScript -ClientId $azClientId -TenantId $azTenantId -SubscriptionId $azSubId } + catch { Write-Warning "az OIDC re-login failed (will retry next interval): $_" } +} + +# --- Start the build (checkout repo + register/start the scheduled task) ------- +$start = @" +`$ErrorActionPreference = 'Stop' +$ghTokenLine +# Refresh PATH from the machine env — tools installed by choco after the guest agent +# started (no reboot since) aren't on this run-command's inherited PATH otherwise. +`$env:Path = [Environment]::GetEnvironmentVariable('Path','Machine') + ';' + [Environment]::GetEnvironmentVariable('Path','User') +`$repo = 'C:\worker-images' +if (Test-Path `$repo) { + git -C `$repo fetch --all --prune + git -C `$repo checkout '$ref' + git -C `$repo reset --hard 'origin/$ref' +} else { + git clone --branch '$ref' https://github.com/mozilla-platform-ops/worker-images.git `$repo +} +if (-not (Test-Path (Join-Path `$repo 'provisioners\windows\win-hw-wim\scripts\run-build-task.ps1'))) { throw 'repo checkout missing run-build-task.ps1' } +`$task = Join-Path `$repo 'provisioners\windows\win-hw-wim\scripts\run-build-task.ps1' +`$arg = "-NoProfile -ExecutionPolicy Bypass -File ""`$task"" -Image $image $buildArg" +`$act = New-ScheduledTaskAction -Execute 'powershell.exe' -Argument `$arg +Register-ScheduledTask -TaskName 'win-hw-wim-build' -Action `$act -RunLevel Highest -User 'SYSTEM' -Force | Out-Null +Start-ScheduledTask -TaskName 'win-hw-wim-build' +Write-Output 'KICKOFF_OK' +"@ + +# Freshness gate: any .done marker older than this instant is stale (a prior run's) and +# must be ignored. Captured just before we start the build, so only THIS build's marker +# (written ~1h40m later) counts as completion. +$kickoffUtc = (Get-Date).ToUniversalTime() +Write-Host "== Starting build '$image' (ref '$ref') on $vm (kickoff $($kickoffUtc.ToString('o'))) ==" +$out = az vm run-command invoke -g $rg -n $vm --command-id RunPowerShellScript --scripts "$start" --query "join('`n', value[].message)" -o tsv +Write-Host $out +# az vm run-command returns 0 even if the inner script threw — assert the sentinel so a +# failed checkout/register fails fast instead of polling a build that never started. +if ("$out" -notmatch 'KICKOFF_OK') { throw "Failed to start the build on $vm (no KICKOFF_OK):`n$out" } + +# --- Poll the completion marker (from blob, NOT run-command) ------------------- +# run-build-task.ps1 uploads _status/.done (content = exit code) when the build +# finishes. We poll that blob with the runner's own az - completely independent of the +# VM's run-command extension, which wedged under bake load and never surfaced build.done. +$intervalSec = 60 +$maxMinutes = 240 # hard cap; a real bake is ~90-150 min (windows_update=true images + # sit at the top of that range, and their two windows-restart + # provisioners may each burn up to 60m). The poll stays + # authenticated (OIDC re-login below), so hitting this means a + # genuinely stuck build - not a monitoring blind spot as before. +$reloginEverySec = 1200 # re-auth every ~20 min so the ~1h azure/login token never lapses +$rc = $null +$doneName = "_status/$image.done" +$doneTmp = Join-Path ([IO.Path]::GetTempPath()) "bake-$image.done" + +# --- Live log streaming ------------------------------------------------------- +# Two append-only logs are pushed to blob during the build and tailed here, so the GH job +# follows the bake in real time instead of getting a 200-line tail two hours later - and +# gets nothing at all when a hang runs the job into its timeout and the VM is destroyed: +# .live.log the whole build.log (packer, puppet, capture, publish) +# .watchdog.log the host-side boot watchdog, incl. its PowerShell Direct capture +# of the GUEST's event log when a step stalls. PS Direct goes over +# the Hyper-V VMBus, so it still works when the guest's network and +# WinRM are dead - which is exactly when packer goes blind and we +# have historically had no idea what the guest was doing. +# Each stream prints only lines not yet printed; a per-blob counter tracks how far we got. +$streams = [ordered]@{ + "_status/$image.live.log" = '' + "_status/$image.watchdog.log" = ' [guest] ' +} +$streamPrinted = @{} +foreach ($k in $streams.Keys) { $streamPrinted[$k] = 0 } + +function Show-LogDelta { + param([string] $BlobName, [string] $Prefix) + + $tmp = Join-Path ([IO.Path]::GetTempPath()) ("bake-" + ($BlobName -replace '[\\/:]', '-')) + Remove-Item $tmp -Force -ErrorAction SilentlyContinue + # Freshness-gate exactly like the .done marker: a leftover log from a previous run + # would otherwise replay a stale build into this job's output. + $mod = az storage blob show --account-name $stAccount --container-name $stContainer --name $BlobName --auth-mode login --query "properties.lastModified" -o tsv 2>$null + if (($LASTEXITCODE -ne 0) -or (-not $mod)) { return } + if (([datetimeoffset]("$mod".Trim())).UtcDateTime -le $kickoffUtc) { return } + az storage blob download --account-name $stAccount --container-name $stContainer --name $BlobName --file $tmp --auth-mode login --only-show-errors -o none 2>$null + if (-not (Test-Path $tmp)) { return } + $lines = @(Get-Content $tmp -ErrorAction SilentlyContinue) + $seen = $script:streamPrinted[$BlobName] + if ($lines.Count -le $seen) { return } + $lines[$seen..($lines.Count - 1)] | ForEach-Object { Write-Host ($Prefix + $_) } + $script:streamPrinted[$BlobName] = $lines.Count +} + +function Show-AllLogDeltas { + foreach ($k in $streams.Keys) { Show-LogDelta -BlobName $k -Prefix $streams[$k] } +} + +for ($elapsed = 0; $elapsed -lt ($maxMinutes * 60); $elapsed += $intervalSec) { + Start-Sleep -Seconds $intervalSec + # Refresh the Azure token before it can expire (azure/login's lasts ~1h; bake ~2h). + if (($elapsed % $reloginEverySec) -eq 0) { Invoke-AzRelogin } + # Check the marker's lastModified FIRST (not just existence) - a leftover marker from a + # prior run would otherwise false-pass instantly. Capture stderr instead of blackholing + # it (2>$null hid the expired-token failure for 4h): BlobNotFound is the normal + # not-done-yet case, but anything else (auth/network) must be SURFACED. + $modOut = az storage blob show --account-name $stAccount --container-name $stContainer --name $doneName --auth-mode login --query "properties.lastModified" -o tsv 2>&1 + $showRc = $LASTEXITCODE + if (($showRc -eq 0) -and $modOut) { + $modUtc = ([datetimeoffset]("$modOut".Trim())).UtcDateTime + if ($modUtc -gt $kickoffUtc) { + # Fresh marker from THIS build - read the exit code it carries. + Remove-Item $doneTmp -Force -ErrorAction SilentlyContinue + az storage blob download --account-name $stAccount --container-name $stContainer --name $doneName --file $doneTmp --auth-mode login --only-show-errors -o none 2>$null + if (Test-Path $doneTmp) { $rc = [int]((Get-Content $doneTmp -Raw).Trim()); break } + } else { + Write-Host " (ignoring stale marker from $modUtc; predates kickoff $kickoffUtc)" + } + } elseif (($showRc -ne 0) -and ("$modOut" -notmatch '(?i)not\s*found|does not exist|BlobNotFound|ResourceNotFound')) { + Write-Warning ("blob poll error (exit {0}): {1}" -f $showRc, (("$modOut" -replace '\s+', ' ').Trim())) + Invoke-AzRelogin # most likely the token lapsed between refreshes; re-auth now + } + Write-Host ("... building ({0} min elapsed)" -f [int]($elapsed / 60)) + Show-AllLogDeltas +} + +# --- Closing output ----------------------------------------------------------- +Invoke-AzRelogin # the build may have outlived the last refresh; ensure auth for the log pull +# Final delta first: run-build-task pushes the complete log just before the .done marker, +# so this picks up everything the last streaming cycle missed - including the failure. +Show-AllLogDeltas +# The 200-line tail is now a FALLBACK: if live streaming worked there is no point +# reprinting lines already above, but if it produced nothing (blob unreachable, old build +# VM image without the uploader) the tail is still the only visibility there is. +if ($streamPrinted["_status/$image.live.log"] -eq 0) { + Write-Host "== Build log (tail) ==" + $logTmp = Join-Path ([IO.Path]::GetTempPath()) "bake-$image.log" + Remove-Item $logTmp -Force -ErrorAction SilentlyContinue + az storage blob download --account-name $stAccount --container-name $stContainer --name "_status/$image.log" --file $logTmp --auth-mode login --only-show-errors -o none 2>$null + if (Test-Path $logTmp) { Get-Content $logTmp | ForEach-Object { Write-Host $_ } } +} +else { Write-Host ("== Build log streamed live above ({0} lines) ==" -f $streamPrinted["_status/$image.live.log"]) } + +if ($null -eq $rc) { throw "Build did not finish within $maxMinutes minutes (timed out; VM will be torn down)." } +if ($rc -ne 0) { throw "Build FAILED (exit $rc). See log above." } + +# --- Release notes / SBOM ----------------------------------------------------- +# The bake generates the same -.md the Azure gallery images do (see +# win-hw-wim.pkr.hcl) and publishes it to _status/sbom/. Pull it into the workspace and +# hand the filename to the workflow via GITHUB_ENV - exactly how sig-*.yml passes +# sharedimageversion - so the upload-artifact step can feed +# .github/workflows/upload-release-notes.yml, which commits it to sboms/ on main. +# Listed by PREFIX rather than an exact name because the build id is generated by +# New-WinHwWim, so the runner never knows it up front. +# Best-effort: a good WIM must not fail the job over its release notes. +try { + $sbomPrefix = "_status/sbom/$image-" + $sbomJson = az storage blob list --account-name $stAccount --container-name $stContainer --prefix $sbomPrefix --auth-mode login --query "[].{name:name, mod:properties.lastModified}" -o json 2>$null + $sbomList = @() + if ($sbomJson) { $sbomList = @($sbomJson | ConvertFrom-Json) } + # Freshness-gated like every other marker here: only THIS build's notes count. + $sbomPick = $sbomList | + Where-Object { ([datetimeoffset]$_.mod).UtcDateTime -gt $kickoffUtc } | + Sort-Object { ([datetimeoffset]$_.mod).UtcDateTime } -Descending | + Select-Object -First 1 + if ($sbomPick) { + $sbomFile = Split-Path $sbomPick.name -Leaf + az storage blob download --account-name $stAccount --container-name $stContainer --name $sbomPick.name --file $sbomFile --auth-mode login --only-show-errors -o none 2>$null + if (Test-Path $sbomFile) { + Write-Host "== Release notes: $sbomFile ==" + if ($env:GITHUB_ENV) { "sbom_file=$sbomFile" | Add-Content -Path $env:GITHUB_ENV } + } + else { Write-Warning "release notes blob '$($sbomPick.name)' could not be downloaded; nothing to commit to sboms/" } + } + else { Write-Warning "no release notes found under $sbomPrefix newer than this kickoff; nothing to commit to sboms/" } +} +catch { + Write-Warning "release-notes fetch failed (build itself succeeded): $_" +} +Write-Host "== Build succeeded: $image -> captured/$image/ ==" diff --git a/ci/win-hw-wim-vm.ps1 b/ci/win-hw-wim-vm.ps1 new file mode 100644 index 00000000..6fe04b7b --- /dev/null +++ b/ci/win-hw-wim-vm.ps1 @@ -0,0 +1,130 @@ +<# +.SYNOPSIS + Create or destroy the EPHEMERAL Azure build VM for a Windows HW WIM build. Runs on + the GitHub Actions runner (pwsh + az, already authenticated by azure/login). + +.DESCRIPTION + The workflow spins this VM up per run and tears it down afterward (in an + if: always() step), so there is no idle cost and no long-lived build host. + + -Action create : nested-virt VM (no public IP / no NSG — driven only via + az vm run-command), system-assigned managed identity granted Storage Blob Data + Contributor on the storage account, Premium data disk, then bootstrap Hyper-V + + tooling (provisioners/windows/win-hw-wim/scripts/bootstrap-build-host.ps1). + -Action destroy : remove the VM + its OS disk + NIC + the MI role assignment. + Idempotent and best-effort so teardown never leaves the job stuck; safe to run + even if create only partially succeeded. + + The VM name is run-scoped (e.g. win-hw-wim-build-) so parallel runs don't + collide and teardown targets exactly this run's VM. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)][ValidateSet('create', 'destroy')] [string] $Action, + [Parameter(Mandatory)] [string] $VmName, + [string] $ResourceGroup = 'rg-central-us-hardware-imaging', + [string] $VnetName = 'vn-central-us-hardware-imaging', + [string] $SubnetName = 'sn-central-us-hardware-imaging-packer', + [string] $Size = 'Standard_D64ads_v5', # AMD, 64 vCPU/256 GiB, nested-virt capable (fits DADSv5 64-core quota) + # Plain Windows Server 2025 gen2 (NOT azure-edition — azure-edition pushes Trusted + # Launch, which is incompatible with nested virtualization). + [string] $Image = 'MicrosoftWindowsServer:WindowsServer:2025-datacenter-g2:latest', + [int] $DataDiskGB = 512, + # Pre-provisioned user-assigned identity (Terraform) attached to the VM for blob + # access — so no per-run role assignment (and no role-assignment rights) is needed. + [string] $BuilderIdentityName = 'id-central-us-hardware-imaging-builder' +) +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +# Resolve the real az executable — PowerShell is case-insensitive, so a function +# named "Az" would otherwise shadow "az" and recurse infinitely. +$azExe = (Get-Command az -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source +function Az { $o = & $azExe @args 2>&1; if ($LASTEXITCODE) { throw "az $($args -join ' ') failed:`n$o" }; return $o } +function AzTry { & $azExe @args 2>&1 | Out-Null } # best-effort (teardown) + +if ($Action -eq 'create') { + $subnetId = (Az network vnet subnet show -g $ResourceGroup --vnet-name $VnetName -n $SubnetName --query id -o tsv) + + # Random admin password (never used — no RDP; az just requires one). Portable: + # System.Web isn't available in pwsh 7 on Linux. Guid hex + 'Aa1!' meets Azure complexity. + $pw = 'Aa1!' + [guid]::NewGuid().ToString('N') + [guid]::NewGuid().ToString('N').Substring(0, 12) + + $uamiId = (Az identity show -g $ResourceGroup -n $BuilderIdentityName --query id -o tsv) + # Windows computer name is capped at 15 chars, so derive a short one from the + # (longer) run-scoped VM resource name. It's just an ephemeral standalone host. + $computerName = ($VmName -replace '[^a-zA-Z0-9]', '') + if ($computerName.Length -gt 15) { $computerName = $computerName.Substring(0, 15) } + Write-Host "== Creating ephemeral VM $VmName ($Size, no public IP; computer $computerName; identity $BuilderIdentityName) ==" + Az vm create -g $ResourceGroup -n $VmName ` + --image $Image --size $Size --computer-name $computerName ` + --security-type Standard ` + --admin-username nucadmin --admin-password $pw ` + --subnet $subnetId --public-ip-address "" --nsg "" ` + --assign-identity $uamiId ` + --os-disk-size-gb 128 --storage-sku Premium_LRS ` + --data-disk-sizes-gb $DataDiskGB --output none + + # Forward slashes so the path resolves on the Linux runner too. + $boot = "$PSScriptRoot/../provisioners/windows/win-hw-wim/scripts/bootstrap-build-host.ps1" + if (-not (Test-Path $boot)) { throw "bootstrap script not found: $boot" } + $bootBody = Get-Content -Raw $boot + + # Run a bootstrap phase and ASSERT its success sentinel. `az vm run-command` returns + # exit 0 even if the inner script throws, so we can't rely on the az exit code. + # Inline the phase assignment + script content as ONE --scripts value (mixing a + # literal line with @file, or --parameters -> params, proved unreliable). + function Invoke-Phase([string]$Ph) { + $script = "`$Phase = '$Ph'`n" + $bootBody + # Capture ALL message streams (value[0]=stdout, value[1]=stderr) — a thrown + # error lands in stderr, so querying only value[0] hid the real failure. + $msg = Az vm run-command invoke -g $ResourceGroup -n $VmName --command-id RunPowerShellScript ` + --scripts $script --query "join('`n', value[].message)" -o tsv + if ("$msg" -notmatch 'BOOTSTRAP_PHASE_OK') { throw "bootstrap phase '$Ph' did not succeed:`n$msg" } + Write-Host " phase $Ph OK" + } + + Write-Host '== Bootstrap phase 1: Hyper-V ==' + Invoke-Phase 'Hyperv' + Write-Host '== Reboot for Hyper-V ==' + Az vm restart -g $ResourceGroup -n $VmName --output none + Start-Sleep -Seconds 30 + Write-Host '== Bootstrap phase 2: tooling (retry for guest-agent readiness) ==' + $ok = $false + for ($i = 1; $i -le 5; $i++) { + try { Invoke-Phase 'Tooling'; $ok = $true; break } + catch { Write-Warning "phase 2 attempt $i failed; retry in 30s. $_"; Start-Sleep 30 } + } + if (-not $ok) { throw 'Bootstrap phase 2 (tooling) failed after retries.' } + Write-Host "== Ephemeral build VM $VmName ready ==" +} +else { + Write-Host "== Destroying ephemeral VM $VmName (best-effort) ==" + # This if: always() step runs AFTER the ~2h build, by which point azure/login's ~1h + # token is dead - so `az vm delete` silently no-ops (auth error) and the VM leaks, + # holding the single-bake 64-core quota. Re-auth with a fresh GitHub OIDC token first. + # Best-effort: never let a re-login failure abort teardown. + try { & (Join-Path $PSScriptRoot 'az-relogin.ps1') } + catch { Write-Warning "teardown OIDC re-login failed (continuing best-effort): $_" } + + # Capture child resource ids before deleting the VM (az vm delete doesn't cascade). + # Use $azExe directly (best-effort; the VM may not exist) — not the throwing Az wrapper. + $diskId = (& $azExe vm show -g $ResourceGroup -n $VmName --query "storageProfile.osDisk.managedDisk.id" -o tsv 2>$null) + $nicIds = (& $azExe vm show -g $ResourceGroup -n $VmName --query "networkProfile.networkInterfaces[].id" -o tsv 2>$null) + $dataDisks = (& $azExe vm show -g $ResourceGroup -n $VmName --query "storageProfile.dataDisks[].managedDisk.id" -o tsv 2>$null) + # The UAMI is persistent (Terraform-managed) and just attached — nothing to detach/remove here. + + AzTry vm delete -g $ResourceGroup -n $VmName --yes + foreach ($nic in ($nicIds -split "`n" | Where-Object { $_ })) { Write-Host " deleting nic $nic"; AzTry network nic delete --ids $nic } + foreach ($d in (@($diskId) + ($dataDisks -split "`n") | Where-Object { $_ })) { Write-Host " deleting disk $d"; AzTry disk delete --ids $d --yes } + + # Sweep by name prefix — catches resources az created before a FAILED 'vm create' + # (the VM never existed, so 'az vm show' above found nothing). az's default NIC is + # VMNic; disks are _*. + foreach ($n in ((& $azExe network nic list -g $ResourceGroup --query "[?starts_with(name,'$VmName')].name" -o tsv 2>$null) -split "`n" | Where-Object { $_ })) { + Write-Host " deleting leaked nic $n"; AzTry network nic delete -g $ResourceGroup -n $n + } + foreach ($d in ((& $azExe disk list -g $ResourceGroup --query "[?starts_with(name,'$VmName')].name" -o tsv 2>$null) -split "`n" | Where-Object { $_ })) { + Write-Host " deleting leaked disk $d"; AzTry disk delete -g $ResourceGroup -n $d --yes + } + Write-Host "== Teardown complete for $VmName ==" +} diff --git a/provisioners/windows/MDC1Windows/Get-Bootstrap.ps1 b/provisioners/windows/MDC1Windows/Get-Bootstrap.ps1 index 73cb693b..561a1ae1 100644 --- a/provisioners/windows/MDC1Windows/Get-Bootstrap.ps1 +++ b/provisioners/windows/MDC1Windows/Get-Bootstrap.ps1 @@ -358,22 +358,30 @@ Test-ConnectionUntilOnline # Enable SSH and import keys Set-SSH -# Enable WinRM (non-fatal, retry twice) -$winrmOk = Set-WinRM -Retries 2 -DelaySeconds 10 -if (-not $winrmOk) { - Write-Log -message 'get-bootstrap :: WinRM setup did not succeed after retries; continuing without it.' -severity 'WARN' -} +# WinRM disabled for the pre-baked NUC deploy. +# RELOPS-2487: Set-WinRM is NOT called here - Enable-PSRemoting can't publish a listener on the +# NUC's Public/workgroup network (it failed every deploy) and nothing in the bootstrap path needs +# WinRM; SSH (baked) is the access path. The Set-WinRM function is kept for future use / if Puppet +# ever needs it. Mirrors the same skip in bootstrap.ps1. +# $winrmOk = Set-WinRM -Retries 2 -DelaySeconds 10 +# if (-not $winrmOk) { +# Write-Log -message 'get-bootstrap :: WinRM setup did not succeed after retries; continuing without it.' -severity 'WARN' +# } # Install Chocolatey Install-Choco # Fetch bootstrap.ps1 +# TEMPORARY (RELOPS-2487 canary): pull bootstrap.ps1 from the `nuc-wim-pipeline` feature +# branch instead of `main` so we can iterate on a prebake-aware bootstrap (consume baked +# Git/openvox/puppet/ronin/registry instead of re-doing them). *** REVERT THIS URL TO +# `main` BEFORE MERGING TO main *** - production must always take bootstrap.ps1 from main. $local_bootstrap = "C:\bootstrap\bootstrap.ps1" if (-Not (Test-Path "D:\Secrets\pat.txt")) { - $splat = @{ Url = "https://raw.githubusercontent.com/mozilla-platform-ops/worker-images/main/provisioners/windows/MDC1Windows/bootstrap.ps1"; Path = $local_bootstrap } + $splat = @{ Url = "https://raw.githubusercontent.com/mozilla-platform-ops/worker-images/nuc-wim-pipeline/provisioners/windows/MDC1Windows/bootstrap.ps1"; Path = $local_bootstrap } Invoke-DownloadWithRetry @splat } else { - $splat = @{ Url = "https://raw.githubusercontent.com/mozilla-platform-ops/worker-images/main/provisioners/windows/MDC1Windows/bootstrap.ps1"; Path = $local_bootstrap; PAT = Get-Content "D:\Secrets\pat.txt" } + $splat = @{ Url = "https://raw.githubusercontent.com/mozilla-platform-ops/worker-images/nuc-wim-pipeline/provisioners/windows/MDC1Windows/bootstrap.ps1"; Path = $local_bootstrap; PAT = Get-Content "D:\Secrets\pat.txt" } Invoke-DownloadWithRetryGithub @splat } diff --git a/provisioners/windows/MDC1Windows/OS-deploy.ps1 b/provisioners/windows/MDC1Windows/OS-deploy.ps1 index 910be1ff..5ebf34ad 100644 --- a/provisioners/windows/MDC1Windows/OS-deploy.ps1 +++ b/provisioners/windows/MDC1Windows/OS-deploy.ps1 @@ -54,6 +54,27 @@ function Deploy-OS-Dev { powershell $deploy_script -deployuser "deployment" -deploymentaccess "$Password" -devlopment_script -branch "$branch" } +function Get-DeploySendoff { + param( + ) + ## Sign-off line printed just before we hand the node over to Setup / reboot into the + ## deployed OS. Cosmetic only - nothing parses this. + $lines = @( + 'This is probably fine in every timeline.' + 'Please keep all limbs inside the deployment.' + 'Here be undocumented behavior.' + 'The wizard responsible has been notified.' + 'Success is now statistically possible.' + 'Do not feed the production environment.' + 'We have angered the dependency gods.' + 'Something ancient just returned exit code 1.' + 'The deployment must flow.' + 'Good luck. The machines are watching.' + 'The machine spirit is willing.' + ) + return (Get-Random -InputObject $lines) +} + function Mount-ZDrive { param( ) @@ -488,7 +509,16 @@ $DomainSuffix = $ResolvedName -replace '^[^.]*\.', '' Write-Host "Host name set to be $ResolvedName" ## Get data -## Assumes files is in the same dir +## Assumes files is in the same dir. +## In dev mode the initial (default-branch) run staged pools.yml, but Deploy-OS-Dev only +## re-downloads OS-deploy.ps1 - so refresh pools.yml from the dev branch here too. That +## lets the WHOLE canary config (image / src_Branch / hash, not just the scripts) live on +## the feature branch; the default branch only needs the `dev:` trigger on the pool. +if ($devlopment_script) { + $poolsUrl = "https://raw.githubusercontent.com/mozilla-platform-ops/worker-images/$branch/provisioners/windows/MDC1Windows/pools.yml" + Write-Host "DEV: refreshing pools.yml from branch '$branch'" + Invoke-WebRequest -Uri $poolsUrl -OutFile "pools.yml" +} $YAML = Convertfrom-Yaml (Get-Content "pools.yml" -raw) foreach ($pool in $YAML.pools) { @@ -658,7 +688,15 @@ $source_app = $source_dir + "applications" $local_app = $local_install + "applications" -if (!(Test-Path $setup)) { +# Resync the local deploy files from the share only when they're actually missing. The +# sentinel for setup-media deploys is setup.exe; for baked-WIM deploys it's the WIM itself. +# D: PERSISTS across (re)deploys when partitioning is skipped, so keying only on setup.exe made +# the WIM path ALWAYS wipe D:\* and recopy the ~6 GB WIM every single deploy. Also require the +# needed WIM to be absent, so a same-image redeploy reuses the cached WIM. (Get-Bootstrap + +# pools.yml are refreshed separately from GitHub, so skipping the resync doesn't stale those; +# on an image change the new WIM name is absent -> resync runs and wipes the old one.) +$needWim = Join-Path $OS_files "$neededImage.wim" +if ((!(Test-Path $setup)) -and (!(Test-Path $needWim))) { Write-Host "Install files wrong or missing." Write-Host "Will resync files." if ((Get-ChildItem -Path $local_install -Force).Count -gt 0) { @@ -749,6 +787,170 @@ Copy-Item -Path pools.yml $local_yaml -Force Set-Location -Path $OS_files Write-Host "Initializing OS installation." -Write-Host Running: Start-Process -FilePath $setup -ArgumentList "/unattend:$unattend" -Write-Host "Have a nice day! :)" -Start-Process -FilePath $setup -ArgumentList "/unattend:$unattend" + +if (Test-Path $setup) { + ## Standard path: Windows Setup applies sources\install.wim per the unattend. + Write-Host Running: Start-Process -FilePath $setup -ArgumentList "/unattend:$unattend" + Write-Host (Get-DeploySendoff) + Start-Process -FilePath $setup -ArgumentList "/unattend:$unattend" +} +else { + ## RELOPS-2487 baked-WIM path (DISM /Apply-Image). The image folder holds no + ## setup.exe - just a bare, already-sysprep/generalize'd .wim. Apply it + ## directly, make the disk bootable with bcdboot, and drop the (already edited) + ## unattend where a generalized image processes it on first boot + ## (\Windows\Panther\unattend.xml -> specialize + oobeSystem -> FirstLogonCommands + ## -> D:\scripts\Get-Bootstrap.ps1), i.e. the same first-boot chain as the setup path. + $wim = Join-Path $OS_files "$neededImage.wim" + if (-not (Test-Path $wim)) { + throw "No setup.exe and no baked WIM at '$wim' - nothing to deploy for image '$neededImage'." + } + + $winVol = "C:" # Windows target (primary NTFS; diskpart 'assign letter=C') + + # Clean the Windows volume before applying. DISM /Apply-Image writes into the target AS-IS + # (it does NOT format), and on a redeploy partitioning is skipped (C:/D: already labeled), so + # C: would otherwise still hold the PREVIOUS OS and we'd layer the new image over stale files. + # Quick-format just C: in place - keeps its drive letter (so the skip-partitioning check still + # passes) and leaves the ESP and the persistent D: (cached WIM) untouched - for a clean apply + # every deploy. Done AFTER the WIM existence check above so we never wipe C: then find no WIM. + Write-Host "== Quick-formatting $winVol before apply (clean DISM target) ==" + Format-Volume -DriveLetter C -FileSystem NTFS -Force -Confirm:$false -ErrorAction Stop | Out-Null + + Write-Host "== DISM /Apply-Image '$wim' (index 1) -> $winVol\ ==" + dism.exe /Apply-Image /ImageFile:"$wim" /Index:1 /ApplyDir:"$winVol\" + if ($LASTEXITCODE -ne 0) { throw "DISM /Apply-Image failed rc=$LASTEXITCODE" } + + ## --- Deterministically set the node name in the OFFLINE image registry --- + ## Do it HERE in WinPE (before the OS ever boots) so the very first boot already comes up + ## as $shortname - i.e. BEFORE the baked nxlog service starts shipping logs, so every log + ## line reports the node name from the start. On the DISM-applied generalized image the + ## specialize-pass from the unattend was NOT taking effect, so the baked + ## 'nuc-bake' name persisted and all logs shipped as nuc-bake (verified in SolarWinds). + ## A post-boot Rename-Computer would need a reboot to go active and would leak nuc-bake- + ## labelled logs until then; the offline edit avoids that. $shortname = node reverse-DNS + ## short name (e.g. nuc13-160), the same value substituted into the unattend ComputerName. + if ($shortname) { + $sysHive = "$winVol\Windows\System32\config\SYSTEM" + Write-Host "== Offline-setting ComputerName -> $shortname in $sysHive ==" + reg load "HKLM\OFFSYS" "$sysHive" | Out-Null + try { + reg add "HKLM\OFFSYS\ControlSet001\Control\ComputerName\ComputerName" /v ComputerName /t REG_SZ /d $shortname /f | Out-Null + reg add "HKLM\OFFSYS\ControlSet001\Control\ComputerName\ActiveComputerName" /v ComputerName /t REG_SZ /d $shortname /f | Out-Null + reg add "HKLM\OFFSYS\ControlSet001\Services\Tcpip\Parameters" /v Hostname /t REG_SZ /d $shortname /f | Out-Null + reg add "HKLM\OFFSYS\ControlSet001\Services\Tcpip\Parameters" /v "NV Hostname" /t REG_SZ /d $shortname /f | Out-Null + } + finally { + [gc]::Collect(); Start-Sleep -Seconds 1 + reg unload "HKLM\OFFSYS" | Out-Null + } + Write-Host "== Offline ComputerName set to $shortname ==" + } + else { + Write-Warning "shortname is empty - skipping offline rename; node would keep the baked name." + } + + ## bcdboot writes the UEFI boot files (\EFI\Microsoft\Boot + BCD) to the EFI System + ## Partition, and /s can only address it by drive letter. diskpart does 'assign + ## letter=S' at partition time, but that letter does NOT reliably persist on a GPT + ## system partition (observed: list volume shows the ESP with no letter -> bcdboot + ## /s S: fails rc=87). So locate the ESP on the SAME disk as C: and give it a letter + ## right here. Targeting C:'s disk explicitly means a leftover/stale ESP on another + ## disk can't be picked. (TODO/disk-clutter: diskpart may not be fully cleaning the + ## disk - a ~644 MB leftover partition was seen on nuc13-160; see WORKLOG follow-up.) + $espGuid = '{c12a7328-f81f-11d2-ba4b-00a0c93ec93b}' + $cDisk = (Get-Partition -DriveLetter C).DiskNumber + $esp = Get-Partition -DiskNumber $cDisk | + Where-Object { $_.GptType -eq $espGuid } | Select-Object -First 1 + if (-not $esp) { throw "No EFI System Partition on disk $cDisk - cannot run bcdboot." } + if ($esp.DriveLetter) { + $efiVol = "$($esp.DriveLetter):" + } + else { + $espDp = "select disk $cDisk`r`nselect partition $($esp.PartitionNumber)`r`nassign letter=S`r`nexit" + $espDp | Out-File -FilePath "$env:TEMP\assign_esp.txt" -Encoding ASCII + Start-Process "diskpart.exe" -ArgumentList "/s $env:TEMP\assign_esp.txt" -Wait + $efiVol = "S:" + } + Write-Host "== ESP = disk $cDisk / partition $($esp.PartitionNumber) -> $efiVol ==" + + Write-Host "== bcdboot $winVol\Windows /s $efiVol /f UEFI ==" + bcdboot.exe "$winVol\Windows" /s $efiVol /f UEFI + if ($LASTEXITCODE -ne 0) { throw "bcdboot failed rc=$LASTEXITCODE" } + + ## Reuse the unattend the resync block already fetched + edited (ComputerName, + ## admin password). A generalized image runs specialize + oobeSystem from + ## \Windows\Panther\unattend.xml on first boot; the windowsPE/ImageInstall pass in + ## it is simply ignored (the image is already applied). + $panther = Join-Path "$winVol\" "Windows\Panther" + New-Item -ItemType Directory -Path $panther -Force | Out-Null + Copy-Item -Path $unattend -Destination (Join-Path $panther "unattend.xml") -Force + Write-Host "== Placed unattend at $panther\unattend.xml ==" + + ## --- Re-assert the node name AFTER specialize (RELOPS-2487) --- + ## The offline rename above is necessary but NOT sufficient: it runs in WinPE, and the + ## first-boot SPECIALIZE pass runs AFTER it and regenerates a random WIN-xxxxxxxx into + ## ActiveComputerName (the unattend's does not take effect on this image). + ## Observed 2026-08-20 on nuc13-158: ComputerName=NUC13-158 but ActiveComputerName= + ## WIN-D81J5HC82S0, with Tcpip Hostname/NV Hostname still correct. That mismatch is not + ## cosmetic - maintainsystem-hw looked the node up under the WIN- name, missed, and + ## Set-PXE'd into an unbreakable re-image loop, and generic-worker's workerId reads the + ## same value. + ## + ## SetupComplete.cmd is the first hook that runs AFTER specialize/oobeSystem and before + ## any logon, so it is the earliest point where the name can be made authoritative. + ## Deliberately NOT a Rename-Computer: that cmdlet compares against the PERSISTENT name, + ## which is already correct, so it refuses with "the new name is the same as the current + ## name". Writing ActiveComputerName directly is the only thing that works (verified on + ## all five canary nodes, 2026-08-20). + $setupScripts = Join-Path "$winVol\" "Windows\Setup\Scripts" + New-Item -ItemType Directory -Path $setupScripts -Force | Out-Null + + $nameFixPs1 = @' +# Set-ActiveComputerName.ps1 - re-assert the node name after the specialize pass. +# Reads nothing from the deploy; the authoritative value is the persistent ComputerName +# that OS-deploy.ps1 set offline, so this is safe to run unconditionally on first boot. +$log = 'C:\Windows\Temp\setupcomplete-name.log' +function W([string]$m) { "$([DateTime]::UtcNow.ToString('o')) $m" | Out-File -FilePath $log -Append -Encoding utf8 } + +$cnKey = 'HKLM:\SYSTEM\CurrentControlSet\Control\ComputerName\ComputerName' +$acnKey = 'HKLM:\SYSTEM\CurrentControlSet\Control\ComputerName\ActiveComputerName' +$tcpip = 'HKLM:\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters' + +try { + $persistent = "$((Get-ItemProperty -Path $cnKey -ErrorAction Stop).ComputerName)".Trim() + $active = "$((Get-ItemProperty -Path $acnKey -ErrorAction SilentlyContinue).ComputerName)".Trim() + W "persistent=$persistent active=$active" + + if (-not $persistent) { W 'persistent ComputerName empty - nothing to assert'; exit 0 } + if ($active -eq $persistent) { W 'already in sync - no action'; exit 0 } + + New-ItemProperty -Path $acnKey -Name ComputerName -Value $persistent -PropertyType String -Force | Out-Null + New-ItemProperty -Path $tcpip -Name Hostname -Value $persistent -PropertyType String -Force | Out-Null + New-ItemProperty -Path $tcpip -Name 'NV Hostname' -Value $persistent -PropertyType String -Force | Out-Null + W "wrote ActiveComputerName/Hostname/NV Hostname = $persistent; restarting" + Restart-Computer -Force +} +catch { + W "FAILED: $($_.Exception.Message)" + exit 1 +} +'@ + + $setupCompleteCmd = @' +@echo off +REM RELOPS-2487: re-assert the node name after specialize. See Set-ActiveComputerName.ps1. +powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "%~dp0Set-ActiveComputerName.ps1" +exit /b 0 +'@ + + $utf8NoBom = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::WriteAllText((Join-Path $setupScripts 'Set-ActiveComputerName.ps1'), $nameFixPs1, $utf8NoBom) + # SetupComplete.cmd must be ANSI/ASCII - cmd.exe will not parse a UTF-8 BOM. + [System.IO.File]::WriteAllText((Join-Path $setupScripts 'SetupComplete.cmd'), $setupCompleteCmd, [System.Text.Encoding]::ASCII) + Write-Host "== Placed SetupComplete.cmd + Set-ActiveComputerName.ps1 in $setupScripts ==" + + Write-Host "Baked WIM applied. Rebooting into the deployed OS. $(Get-DeploySendoff)" + Start-Sleep -Seconds 5 + wpeutil reboot +} diff --git a/provisioners/windows/MDC1Windows/bootstrap.ps1 b/provisioners/windows/MDC1Windows/bootstrap.ps1 index 51c0dc23..316cf81f 100644 --- a/provisioners/windows/MDC1Windows/bootstrap.ps1 +++ b/provisioners/windows/MDC1Windows/bootstrap.ps1 @@ -400,6 +400,13 @@ function Set-Logging { Write-Host ('{0} :: begin - {1:o}' -f $($MyInvocation.MyCommand.Name), (Get-Date).ToUniversalTime()) } process { + # --- nxlog: skip when it is already present (consume the prebake) --- + # Version doesn't matter here - if nxlog is installed at all (service present or the binary + # is on disk), move on without reinstalling. Its conf + papertrail cert are baked alongside it. + if ((Get-Service -Name nxlog -ErrorAction SilentlyContinue) -or (Test-Path "$nxlog_dir\nxlog.exe")) { + Write-Host ('{0} :: nxlog already installed; skipping install' -f $($MyInvocation.MyCommand.Name)) + return + } $null = New-Item -ItemType Directory -Force -Path $local_dir -ErrorAction SilentlyContinue Invoke-DownloadWithRetry $ext_src/$nxlog_msi -Path $local_dir\$nxlog_msi #Invoke-WebRequest $ext_src/$nxlog_msi -outfile $local_dir\$nxlog_msi -UseBasicParsing @@ -432,44 +439,27 @@ function Get-PSModules { ## https://github.com/PowerShell/PowerShellGallery/issues/328 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - $maxAttempts = 10 - $attemptDelay = 60 - - $nugetProvider = $null - for ($i = 1; $i -le $maxAttempts; $i++) { - Write-Log -message ('{0} :: Checking for NuGet provider (attempt {1}/{2})' -f $MyInvocation.MyCommand.Name, $i, $maxAttempts) -severity 'DEBUG' - $nugetProvider = Get-PackageProvider -Name NuGet -ListAvailable -ForceBootstrap -ErrorAction SilentlyContinue - - if ($null -ne $nugetProvider) { - Write-Log -message ('{0} :: NuGet provider is present.' -f $MyInvocation.MyCommand.Name) -severity 'DEBUG' - break - } - - if ($i -lt $maxAttempts) { - Write-Log -message ('{0} :: NuGet provider not found. Sleeping {1}s before retry.' -f $MyInvocation.MyCommand.Name, $attemptDelay) -severity 'DEBUG' - Start-Sleep -Seconds $attemptDelay - } - } - + # NuGet provider: check ONCE, then install directly if missing. The old loop polled + # 10x with 60s sleeps (~9 min) for a provider that never appears on its own, THEN + # installed it anyway - pure dead time. On the baked WIM the provider is pre-installed + # (bake-bootstrap), so this check passes immediately. + $nugetProvider = Get-PackageProvider -Name NuGet -ListAvailable -ForceBootstrap -ErrorAction SilentlyContinue if ($null -eq $nugetProvider) { - Write-Log -message ('{0} :: Installing NuGet Package Provider after {1} failed checks' -f $MyInvocation.MyCommand.Name, $maxAttempts) -severity 'DEBUG' + Write-Log -message ('{0} :: NuGet provider absent; installing.' -f $MyInvocation.MyCommand.Name) -severity 'DEBUG' try { Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.208 -Force -Confirm:$false -ForceBootstrap -ErrorAction Stop } catch { Write-Log -message ('{0} :: Failed to install NuGet Package Provider: {1}' -f $MyInvocation.MyCommand.Name, $_.Exception.Message) -severity 'ERROR' } - - # Verify installation $nugetProvider = Get-PackageProvider -Name NuGet -ListAvailable -ForceBootstrap -ErrorAction SilentlyContinue if ($null -eq $nugetProvider) { Write-Log -message ('{0} :: NuGet provider still not available after install attempt; exiting 3' -f $MyInvocation.MyCommand.Name) -severity 'ERROR' Write-Host exit 3 return - } else { - Write-Log -message ('{0} :: NuGet provider installed successfully.' -f $MyInvocation.MyCommand.Name) -severity 'DEBUG' } } + Write-Log -message ('{0} :: NuGet provider present.' -f $MyInvocation.MyCommand.Name) -severity 'DEBUG' foreach ($module in $modules) { $hit = Get-Module -Name $module @@ -509,6 +499,47 @@ function Get-PSModules { } } +# Return the highest installed version (as [version]) matching any of the given +# Uninstall-registry DisplayName patterns, or $null if not installed. +function Get-InstalledVersion { + param ( + [Parameter(Mandatory)] + [string[]] $NameLike + ) + $uninstallKeys = @( + 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*', + 'HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\*' + ) + $found = $null + $entries = Get-ItemProperty $uninstallKeys -ErrorAction SilentlyContinue + foreach ($pattern in $NameLike) { + foreach ($e in ($entries | Where-Object { $_.DisplayName -like $pattern -and $_.DisplayVersion })) { + # Normalise e.g. "2.54.0.windows.1" / "8.19.2" -> a comparable [version]. + $m = [regex]::Match([string]$e.DisplayVersion, '\d+(\.\d+){1,3}') + if ($m.Success) { + try { + $v = [version]$m.Value + if ($null -eq $found -or $v -gt $found) { $found = $v } + } + catch { } + } + } + } + return $found +} + +# True when $Installed is present and >= the $Minimum version string. +function Test-VersionAtLeast { + param ( + [version] $Installed, + [string] $Minimum + ) + if ($null -eq $Installed) { return $false } + $m = [regex]::Match([string]$Minimum, '\d+(\.\d+){1,3}') + if (-not $m.Success) { return $false } + try { return ($Installed -ge [version]$m.Value) } catch { return $false } +} + function Get-PreRequ { param ( [string] @@ -524,11 +555,21 @@ function Get-PreRequ { Write-Log -message ('{0} :: begin - {1:o}' -f $($MyInvocation.MyCommand.Name), (Get-Date).ToUniversalTime()) -severity 'DEBUG' } process { + # The versions passed in (from pools.yml) are treated as a MINIMUM. With the + # pre-baked install.wim these tools are already present at the win defaults, so + # we detect the installed version and only download/install when the box is + # missing the tool or is below the pools.yml minimum. if ($openvox_version) { - $puppet = "openvox-agent-$openvox_version-x64.msi" + $puppet = "openvox-agent-$openvox_version-x64.msi" + $agentMin = $openvox_version + $agentNames = @('OpenVox Agent*', 'Openvox*', '*openvox-agent*') + $agentLabel = 'OpenVox agent' } else { - $puppet = ("puppet-agent-{0}-x64.msi") -f $puppet_version + $puppet = ("puppet-agent-{0}-x64.msi") -f $puppet_version + $agentMin = $puppet_version + $agentNames = @('Puppet Agent*', '*puppet-agent*') + $agentLabel = 'Puppet agent' } switch ($env:PROCESSOR_ARCHITECTURE) { @@ -544,45 +585,135 @@ function Get-PreRequ { } $git_url = "https://github.com/git-for-windows/git/releases/download/v$($git_version).windows.1/$($git)" - if (-Not (Test-Path "$env:systemdrive\$puppet")) { - Write-Log -Message ('{0} :: Downloading Puppet' -f $($MyInvocation.MyCommand.Name)) -severity 'DEBUG' - Invoke-DownloadWithRetry "$ext_src/$puppet" -Path "$env:systemdrive\$puppet" + # --- Agent (Puppet/OpenVox): skip when baked WIM already meets the minimum --- + $agentInstalled = Get-InstalledVersion -NameLike $agentNames + if (Test-VersionAtLeast -Installed $agentInstalled -Minimum $agentMin) { + Write-Log -Message ('{0} :: {1} {2} already present (>= min {3}); skipping install' -f $($MyInvocation.MyCommand.Name), $agentLabel, $agentInstalled, $agentMin) -severity 'DEBUG' + Write-Host ('{0} :: {1} {2} satisfies minimum {3}; skipping install' -f $($MyInvocation.MyCommand.Name), $agentLabel, $agentInstalled, $agentMin) + } + else { + Write-Log -Message ('{0} :: {1} install needed (installed={2}, min={3})' -f $($MyInvocation.MyCommand.Name), $agentLabel, $agentInstalled, $agentMin) -severity 'DEBUG' if (-Not (Test-Path "$env:systemdrive\$puppet")) { - Write-Log -Message ('{0} :: Puppet failed to download' -f $($MyInvocation.MyCommand.Name)) -severity 'DEBUG' + Invoke-DownloadWithRetry "$ext_src/$puppet" -Path "$env:systemdrive\$puppet" + } + if (-Not (Test-Path "$env:systemdrive\$puppet")) { + Write-Log -Message ('{0} :: {1} failed to download' -f $($MyInvocation.MyCommand.Name), $agentLabel) -severity 'ERROR' + exit 1 + } + Start-Process msiexec -ArgumentList @("/qn", "/norestart", "/i", "$env:systemdrive\$puppet") -Wait + $agentInstalled = Get-InstalledVersion -NameLike $agentNames + # Fall back to the bin-dir check (original behavior) so a registry + # DisplayName miss can't fail a deploy where the agent installed fine. + $agentOk = (Test-VersionAtLeast -Installed $agentInstalled -Minimum $agentMin) -or + (Test-Path 'C:\Program Files\Puppet Labs\Puppet\bin') -or + (Test-Path 'C:\Program Files\OpenVox\Puppet\bin') + if (-Not $agentOk) { + Write-Host ('Did not install {0} to minimum {1} (got {2})' -f $agentLabel, $agentMin, $agentInstalled) + Write-Log -message ('{0} :: {1} did not meet minimum {2} (got {3})' -f $($MyInvocation.MyCommand.Name), $agentLabel, $agentMin, $agentInstalled) -severity 'ERROR' + exit 1 } + Write-Log -message ('{0} :: {1} installed :: {2}' -f $($MyInvocation.MyCommand.Name), $agentLabel, $agentInstalled) -severity 'DEBUG' + Write-Host ('{0} :: {1} installed :: {2}' -f $($MyInvocation.MyCommand.Name), $agentLabel, $agentInstalled) } - if (-Not (Test-Path "$env:systemdrive\$git")) { - Write-Log -Message ('{0} :: Downloading Git from {1}' -f $($MyInvocation.MyCommand.Name), $git_url) -severity 'DEBUG' - Invoke-DownloadWithRetryGithub -Url $git_url -Path "$env:systemdrive\$git" -PAT (Get-Content "D:\Secrets\pat.txt") + # --- Git: skip when baked WIM already meets the minimum --- + $gitInstalled = Get-InstalledVersion -NameLike @('Git version*', 'Git') + if (Test-VersionAtLeast -Installed $gitInstalled -Minimum $git_version) { + Write-Log -Message ('{0} :: Git {1} already present (>= min {2}); skipping install' -f $($MyInvocation.MyCommand.Name), $gitInstalled, $git_version) -severity 'DEBUG' + Write-Host ('{0} :: Git {1} satisfies minimum {2}; skipping install' -f $($MyInvocation.MyCommand.Name), $gitInstalled, $git_version) + } + else { + Write-Log -Message ('{0} :: Git install needed (installed={1}, min={2}) from {3}' -f $($MyInvocation.MyCommand.Name), $gitInstalled, $git_version, $git_url) -severity 'DEBUG' if (-Not (Test-Path "$env:systemdrive\$git")) { - Write-Log -Message ('{0} :: Git failed to download' -f $($MyInvocation.MyCommand.Name)) -severity 'DEBUG' + Invoke-DownloadWithRetryGithub -Url $git_url -Path "$env:systemdrive\$git" -PAT (Get-Content "D:\Secrets\pat.txt") + } + if (-Not (Test-Path "$env:systemdrive\$git")) { + Write-Log -Message ('{0} :: Git failed to download' -f $($MyInvocation.MyCommand.Name)) -severity 'ERROR' + exit 1 + } + Start-Process "$env:systemdrive\$git" -ArgumentList "/verysilent" -Wait -NoNewWindow + $gitInstalled = Get-InstalledVersion -NameLike @('Git version*', 'Git') + # Bin-dir fallback (original behavior) guards against a registry detection miss. + $gitOk = (Test-VersionAtLeast -Installed $gitInstalled -Minimum $git_version) -or (Test-Path 'C:\Program Files\Git\bin') + if (-Not $gitOk) { + Write-Host "Git not installed to minimum $git_version (got $gitInstalled)" + Write-Log -message ('{0} :: Git did not meet minimum {1} (got {2})' -f $($MyInvocation.MyCommand.Name), $git_version, $gitInstalled) -severity 'ERROR' + exit 1 } + Write-Log -message ('{0} :: Git installed :: {1}' -f $($MyInvocation.MyCommand.Name), $gitInstalled) -severity 'DEBUG' + Write-Host ('{0} :: Git installed :: {1}' -f $($MyInvocation.MyCommand.Name), $gitInstalled) } - Start-Process "$env:systemdrive\$git" -ArgumentList "/verysilent" -Wait -NoNewWindow - if (-Not (Test-Path "C:\Program Files\Git\bin")) { - Write-Host "Git not installed" - Write-Log -message ('{0} :: Git not installed' -f $($MyInvocation.MyCommand.Name)) -severity 'DEBUG' - exit 1 + # Ensure tool bin dirs are on PATH whether we just installed them or inherited + # them from the baked WIM. + foreach ($bin in @( + 'C:\Program Files\Git\bin', + 'C:\Program Files\Git\cmd', + 'C:\Program Files\Puppet Labs\Puppet\bin', + 'C:\Program Files\OpenVox\Puppet\bin')) { + if ((Test-Path $bin) -and ($env:PATH -notlike "*$bin*")) { $env:PATH += ";$bin" } } - Write-Log -message ('{0} :: Git installed :: {1}' -f $($MyInvocation.MyCommand.Name), $git) -severity 'DEBUG' - $env:PATH += ";C:\Program Files\git\bin" - Write-Host ('{0} :: Git installed :: {1}' -f $($MyInvocation.MyCommand.Name), $git) - - if (-Not (Test-Path "C:\Program Files\Puppet Labs\Puppet\bin")) { - Write-Log -Message ('{0} :: Installing puppet' -f $($MyInvocation.MyCommand.Name)) -severity 'DEBUG' - Start-Process msiexec -ArgumentList @("/qn", "/norestart", "/i", "$env:systemdrive\$puppet") -Wait - Write-Log -message ('{0} :: Puppet installed :: {1}' -f $($MyInvocation.MyCommand.Name), $puppet) -severity 'DEBUG' - Write-Host ('{0} :: Puppet installed :: {1}' -f $($MyInvocation.MyCommand.Name), $puppet) - if (-Not (Test-Path "C:\Program Files\Puppet Labs\Puppet\bin")) { - Write-Host "Did not install puppet" - write-host exit 1 + [Environment]::SetEnvironmentVariable("PATH", $env:PATH, [System.EnvironmentVariableTarget]::Machine) + } + end { + Write-Log -message ('{0} :: end - {1:o}' -f $($MyInvocation.MyCommand.Name), (Get-Date).ToUniversalTime()) -severity 'DEBUG' + } +} +function Install-Drivers { + <# + .SYNOPSIS + On-host driver install for the pre-baked (DISM /Apply-Image) deploy path (RELOPS-2487). + .DESCRIPTION + The pre-baked WIM may not carry every NUC hardware driver (e.g. the Intel GPU), which + leaves the node on the Microsoft Basic Display Adapter. Windows Update is disabled so + nothing fetches the missing driver on its own. This actively installs a driver pack from + the RelOps blob mirror on the host (pnputil /add-driver /install + /scan-devices): + - installs from local files, no WU needed (proven: baked NIC drivers bind with WU off); + - iterates without a full ~2h WIM re-bake (just refresh the pack on the mirror). + Idempotent: skips when a real (non-generic) display adapter is already in use, so it is a + no-op on normally-imaged nodes and on re-runs once the GPU driver is bound. + #> + param( + [string] $ext_src = "https://roninpuppetassets.blob.core.windows.net/binaries/drivers/nuc13", + [string[]] $packs = @("nuc13-24h2-nuc_driver.zip"), + [string] $work = "$env:systemdrive\drivers" + ) + begin { + Write-Log -message ('{0} :: begin - {1:o}' -f $($MyInvocation.MyCommand.Name), (Get-Date).ToUniversalTime()) -severity 'DEBUG' + } + process { + try { + # Idempotent guard: real GPU already bound -> nothing to do (consume the prebake). + $vc = @(Get-CimInstance Win32_VideoController | Select-Object -ExpandProperty Name) + $real = $vc | Where-Object { $_ -notmatch 'Basic Display Adapter|Basic Render|Hyper-V Video' } + if ($real) { + Write-Log -message ('{0} :: real display driver already present ({1}); skipping driver install' -f $($MyInvocation.MyCommand.Name), ($real -join ', ')) -severity 'DEBUG' + return } - $env:PATH += ";C:\Program Files\Puppet Labs\Puppet\bin" - [Environment]::SetEnvironmentVariable("PATH", $env:PATH, [System.EnvironmentVariableTarget]::Machine) + if (Test-Path $work) { Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue } + New-Item -ItemType Directory -Path $work -Force | Out-Null + foreach ($p in $packs) { + $url = "$ext_src/$p" + $dst = Join-Path $work $p + $sub = Join-Path $work ([System.IO.Path]::GetFileNameWithoutExtension($p)) + New-Item -ItemType Directory -Path $sub -Force | Out-Null + Write-Log -message ('{0} :: downloading driver pack {1}' -f $($MyInvocation.MyCommand.Name), $url) -severity 'DEBUG' + Invoke-DownloadWithRetry -Url $url -Path $dst + $ext = [System.IO.Path]::GetExtension($p).ToLowerInvariant() + if ($ext -eq '.zip') { Expand-Archive -LiteralPath $dst -DestinationPath $sub -Force } + elseif ($ext -eq '.cab') { & expand.exe -F:* "$dst" "$sub" | Out-Null } + else { Write-Log -message ('{0} :: unsupported pack type {1}, skipping' -f $($MyInvocation.MyCommand.Name), $ext) -severity 'WARN'; continue } + Write-Log -message ('{0} :: pnputil /add-driver {1}\*.inf /subdirs /install' -f $($MyInvocation.MyCommand.Name), $sub) -severity 'DEBUG' + & pnputil.exe /add-driver "$sub\*.inf" /subdirs /install | Out-Null + } + & pnputil.exe /scan-devices | Out-Null + $vc2 = @(Get-CimInstance Win32_VideoController | Select-Object -ExpandProperty Name) + Write-Log -message ('{0} :: display adapters after install: {1}' -f $($MyInvocation.MyCommand.Name), ($vc2 -join ', ')) -severity 'DEBUG' + } + catch { + # Best-effort: a missing/incomplete pack must not brick the deploy. + Write-Log -message ('{0} :: driver install failed (continuing): {1}' -f $($MyInvocation.MyCommand.Name), $_.Exception.Message) -severity 'WARN' } - [Environment]::SetEnvironmentVariable("PATH", $env:PATH, [System.EnvironmentVariableTarget]::Machine) } end { Write-Log -message ('{0} :: end - {1:o}' -f $($MyInvocation.MyCommand.Name), (Get-Date).ToUniversalTime()) -severity 'DEBUG' @@ -595,15 +726,23 @@ function Set-Ronin-Registry { Write-Log -message ('{0} :: begin - {1:o}' -f $($MyInvocation.MyCommand.Name), (Get-Date).ToUniversalTime()) -severity 'DEBUG' } process { + # Prefer the params passed by Get-Bootstrap; only fall back to a value already in the + # registry when the corresponding param is empty (a genuine no-param resume). This lets + # us CONSUME the prebaked image's leftover HKLM\...\ronin_puppet key WITHOUT an empty/ + # stale value there clobbering the correct deploy params. The stock read-everything logic + # did clobber them on the baked WIM (empty Organisation/Repository/GITHASH) -> git clone + # https://github.com// -> invalid C:\ronin -> "cannot find nodes.pp" -> puppet exit 1 -> + # Set-PXE re-image loop. (RELOPS-2487 prebake canary.) If ((test-path "HKLM:\SOFTWARE\Mozilla\ronin_puppet")) { - $worker_pool_id = (Get-ItemProperty -path "HKLM:\SOFTWARE\Mozilla\ronin_puppet").worker_pool_id - $role = (Get-ItemProperty -path "HKLM:\SOFTWARE\Mozilla\ronin_puppet").role - $src_Organisation = (Get-ItemProperty -path "HKLM:\SOFTWARE\Mozilla\ronin_puppet").Organisation - $src_Repository = (Get-ItemProperty -path "HKLM:\SOFTWARE\Mozilla\ronin_puppet").Repository - $src_Branch = (Get-ItemProperty -path "HKLM:\SOFTWARE\Mozilla\ronin_puppet").Branch - $image_provisioner = (Get-ItemProperty -path "HKLM:\SOFTWARE\Mozilla\ronin_puppet").image_provisioner - $secret_date = (Get-ItemProperty -path "HKLM:\SOFTWARE\Mozilla\ronin_puppet").secret_date - $hash = (Get-ItemProperty -path "HKLM:\SOFTWARE\Mozilla\ronin_puppet").GITHASH + $r = Get-ItemProperty -path "HKLM:\SOFTWARE\Mozilla\ronin_puppet" + if ([string]::IsNullOrWhiteSpace($worker_pool_id)) { $worker_pool_id = $r.worker_pool_id } + if ([string]::IsNullOrWhiteSpace($role)) { $role = $r.role } + if ([string]::IsNullOrWhiteSpace($src_Organisation)) { $src_Organisation = $r.Organisation } + if ([string]::IsNullOrWhiteSpace($src_Repository)) { $src_Repository = $r.Repository } + if ([string]::IsNullOrWhiteSpace($src_Branch)) { $src_Branch = $r.Branch } + if ([string]::IsNullOrWhiteSpace($image_provisioner)) { $image_provisioner = $r.image_provisioner } + if ([string]::IsNullOrWhiteSpace($secret_date)) { $secret_date = $r.secret_date } + if ([string]::IsNullOrWhiteSpace($hash)) { $hash = $r.GITHASH } } Write-Log -Message ('{0} :: Creating HKLM:\SOFTWARE\Mozilla\ronin_puppet' -f $($MyInvocation.MyCommand.Name)) -severity 'DEBUG' New-Item -Path HKLM:\SOFTWARE -Name Mozilla -force @@ -965,10 +1104,12 @@ Set-ExecutionPolicy Unrestricted -Force -ErrorAction SilentlyContinue powercfg.exe -x -standby-timeout-ac 0 powercfg.exe -x -monitor-timeout-ac 0 -## Enable OpenSSH and WinRM -## Installation through Puppet is intermittent. -## It works here, but ultimately should be done through Puppet. -Set-WinRM +## Enable OpenSSH (WinRM disabled for the pre-baked NUC deploy). +## RELOPS-2487: Set-WinRM is skipped here - Enable-PSRemoting can't publish a listener on the +## NUC's Public/workgroup network (failed every deploy: "Listener=False ... continuing anyway") +## and nothing in the bootstrap path needs WinRM; SSH (baked) is the access path. If WinRM is +## ever actually required, let Puppet configure it. +# Set-WinRM Set-SSH ## This is not being set yet, so it won't find the ronin_puppet registry entry @@ -1027,6 +1168,10 @@ If ($stage -ne 'complete') { git_version = $git_version } Get-PreRequ @preReq + ## RELOPS-2487: temporarily disabled while we test a PXE boot with the KVM display + ## disconnected (isolating whether the Raritan CIM EDID drives the display result). + ## Re-enable once we resume on-host driver installs. Install-Drivers is idempotent/best-effort. + # Install-Drivers Set-Ronin-Registry Get-Ronin Run-Ronin-Run diff --git a/provisioners/windows/MDC1Windows/pools.yml b/provisioners/windows/MDC1Windows/pools.yml index bec7bdcd..b1ca45af 100644 --- a/provisioners/windows/MDC1Windows/pools.yml +++ b/provisioners/windows/MDC1Windows/pools.yml @@ -28,14 +28,20 @@ pools: - name: "win11-64-24h2-hw-ref-alpha" puppet_version: "8.10.0" openvox_version: "8.19.2" - git_version: "2.50.1" + # Matched to perf-debug (was 2.50.1) so the two canary pools are identical on every + # deployment field. Inert either way: the WIM bakes Git 2.54.0 and the deploy-time + # install is version-guarded, so neither value ever triggers an install. + git_version: "2.50.0" #dev: RELOPS-1470 Description: "NUC12 Hardware Reference Staging/Testing Pool" - image: "win11-24H2-NUC-01-16-2025" + # RELOPS-2487 canary: same dev trigger perf-debug carries, so both canary + # pools run identically off the feature branches. + dev: "nuc-wim-pipeline" + image: "win11-24h2-hw-20260908-172915" src_Organisation: "mozilla-platform-ops" src_Repository: "ronin_puppet" - src_Branch: "RELOPS-2195-thermal" - hash: "e47164b" + src_Branch: "wim-bake-role" + hash: "459102e1" secret_date: "02-24-2026" domain_suffix: "wintest2.releng.mdc1.mozilla.com" nodes: @@ -48,17 +54,23 @@ pools: openvox_version: "8.19.2" git_version: "2.50.0" Description: "NUC12 Hardware Performance Staging/Testing Pool" - image: "win11-24H2-NUC-01-16-2025" + # RELOPS-2487 canary: pre-baked install.wim + DISM /Apply-Image deploy path. + # dev -> pull the MDC1Windows deploy scripts (OS-deploy.ps1 etc.) from our + # worker-images feature branch; src_* -> our ronin branch; image -> the baked + # WIM staged on the deployment share at /.wim. + dev: "nuc-wim-pipeline" + image: "win11-24h2-hw-test-20260805-193514" src_Organisation: "mozilla-platform-ops" src_Repository: "ronin_puppet" - src_Branch: "master" - hash: "edef633" + src_Branch: "wim-bake-role" + hash: "5230f32" secret_date: "02-24-2026" domain_suffix: "wintest2.releng.mdc1.mozilla.com" nodes: - # Moved from alpha. - - nuc13-159 - - nuc13-160 + # Known dead PSUs - present before the RELOPS-2514 PSU swap-out validation began. + # Originally moved here from alpha. These are not awaiting a swap like the nodes in + # win11-64-24h2-hw-alpha; their PSUs are known dead. They appear in none of the swap + # tickets (IO-3797, IO-3844, IO-3884). - nuc13-035 - nuc13-060 - nuc13-068 @@ -67,25 +79,61 @@ pools: - nuc13-150 - nuc13-155 - nuc13-157 - # First 20 nodes to swap out PSUs and be tested: 10 down/assumed-PSU-failure nodes from relops1213 and 10 poorly-performing nodes from alpha. + # No KVM - cannot be re-imaged or inspected remotely until console access is fixed. + # Two cross-wired pairs are in here: nuc13-037's KVM port is swapped with + # nuc13-038's and nuc13-053's with nuc13-054's, so connecting to one reaches the + # other and KVM-derived status for any of the four is unreliable. + - nuc13-011 + - nuc13-037 + - nuc13-053 + - nuc13-078 + - nuc13-132 + - nuc13-143 + # Down / no SSH - unresponsive on every SSH attempt. nuc13-119 PXE-booted once and + # never returned to the network. + - nuc13-022 + - nuc13-023 + - nuc13-028 + - nuc13-069 + - nuc13-119 + # Other - individual cases, see the note above each node. + # nuc13-077: PXE does not take. It accepts the bootsequence change and reboots but comes + # back on its original 05-01 image, confirmed across three checks over three + # days. Needs a recovery route other than a repeat PXE. + - nuc13-077 + # nuc13-111: Unreachable with its display flashing. Did not come back after being + # PXE-booted, so it could not be scored. + - nuc13-111 - name: "win11-64-24h2-hw-perf-debug" puppet_version: "8.10.0" openvox_version: "8.19.2" git_version: "2.50.0" Description: "NUC12 Hardware Performance Staging/Testing Pool" - image: "win11-24H2-NUC-01-16-2025" + # RELOPS-2487 canary: pre-baked install.wim + DISM /Apply-Image deploy path, on a pool with + # WORKING Taskcluster worker credentials (the relops1213 client is under-scoped: missing + # assume:worker-pool + assume:worker-id). Points at our worker-images + ronin feature branches + # and the drivers-baked WIM to validate the prebake worker end-to-end. + # *** REVERT before master: image win11-24H2-NUC-01-16-2025, dev (remove), + # src_Branch RELOPS-2467-xperf-dynamic-trace, hash a22e7ac. *** + dev: "nuc-wim-pipeline" + image: "win11-24h2-hw-20260908-172915" src_Organisation: "mozilla-platform-ops" src_Repository: "ronin_puppet" - src_Branch: "RELOPS-2467-xperf-dynamic-trace" - hash: "a22e7ac" + src_Branch: "wim-bake-role" + hash: "459102e1" secret_date: "02-06-2026" domain_suffix: "wintest2.releng.mdc1.mozilla.com" nodes: - - nuc13-024 - - nuc13-059 - - nuc13-119 - # Questionable nodes — CPU suppression confirmed or notable floor events detected in stress testing. - # Not cleared for high-priority CPU-intensive work. Pending PSU inspection/replacement. + # nuc13-158: never successfully validated. It was reachable but with an empty + # bootstrap_stage before this re-image, and produced no Speedometer results in any + # of the three 2026-08-17 runs (32073991787, 32074035219, 32074056736) while all + # 22 of its pool peers scored. Under investigation. Tracked in RELOPS-2514. + - nuc13-158 + # Development work - two nodes pulled from win11-64-24h2-hw at random to have a + # working pair available in this pool. Both have swapped PSUs and passed + # validation, so they are not here for a fault. + - nuc13-074 + - nuc13-115 - name: "win11-64-24h2-hw-alpha" puppet_version: "8.10.0" openvox_version: "8.19.2" @@ -100,179 +148,171 @@ pools: secret_date: "02-06-2026" domain_suffix: "wintest2.releng.mdc1.mozilla.com" nodes: - # Known-good nodes moved from win11-64-24h2-hw for fleetbench testing. + # Original PSUs - these 39 nodes were never included in a PSU swap-out. They are + # not covered by IO-3797 (20 nodes, 2026-05-29), IO-3844 (nuc13-159 and nuc13-160, + # 2026-07-08) or IO-3884 (87 PSUs plus nuc13-101 and nuc13-046, 2026-08-07), and + # so have not been through the RELOPS-2514 validation either. Moved from + # win11-64-24h2-hw. + - nuc13-004 + - nuc13-005 + - nuc13-007 + - nuc13-008 + - nuc13-009 + - nuc13-010 + - nuc13-015 + - nuc13-016 + - nuc13-017 + - nuc13-018 + - nuc13-033 + - nuc13-040 + - nuc13-041 + - nuc13-048 + - nuc13-049 + - nuc13-056 + - nuc13-057 + - nuc13-064 + - nuc13-066 + - nuc13-081 + - nuc13-083 + - nuc13-086 + - nuc13-088 + - nuc13-089 + - nuc13-090 + - nuc13-097 + - nuc13-104 + - nuc13-108 + - nuc13-109 + - nuc13-110 + - nuc13-113 + - nuc13-118 + - nuc13-121 + - nuc13-123 + - nuc13-127 + - nuc13-128 + - nuc13-135 + - nuc13-145 + - nuc13-153 + - name: "win11-64-24h2-hw-perf-sheriff" + openvox_version: "8.19.2" + puppet_version: "8.10.0" + git_version: "2.50.0" + Description: "Tracking metric variability with performance sheriffing. Tracking in RELOPS-1288" + image: "win11-24H2-NUC-01-16-2025" + src_Organisation: "mozilla-platform-ops" + src_Repository: "ronin_puppet" + src_Branch: "master" + hash: "7511c8e" + secret_date: "02-24-2026" + domain_suffix: "wintest2.releng.mdc1.mozilla.com" + nodes: + - nuc13-021 + - name: "win11-64-24h2-hw" + openvox_version: "8.19.2" + puppet_version: "8.10.0" + git_version: "2.50.0" + Description: "NUC12 Hardware Performance Staging/Testing Pool" + image: "win11-24H2-NUC-01-16-2025" + src_Organisation: "mozilla-platform-ops" + src_Repository: "ronin_puppet" + src_Branch: "master" + hash: "edef633" + secret_date: "02-24-2026" + domain_suffix: "wintest2.releng.mdc1.mozilla.com" + nodes: - nuc13-001 - - nuc13-003 - - nuc13-027 - - nuc13-038 - - nuc13-082 - - nuc13-126 - - nuc13-132 - nuc13-002 + - nuc13-003 + - nuc13-006 + - nuc13-012 - nuc13-013 - nuc13-014 - nuc13-019 - nuc13-020 - - nuc13-022 - - nuc13-023 + - nuc13-024 - nuc13-025 - nuc13-026 - - nuc13-028 + - nuc13-027 + - nuc13-029 - nuc13-030 - nuc13-031 + - nuc13-032 - nuc13-034 - - nuc13-037 + - nuc13-036 + - nuc13-038 - nuc13-039 - nuc13-042 - nuc13-043 + - nuc13-044 + - nuc13-045 + - nuc13-046 - nuc13-047 - nuc13-050 - nuc13-051 - nuc13-052 - - nuc13-053 - nuc13-054 - nuc13-055 - nuc13-058 + - nuc13-059 + - nuc13-061 + - nuc13-062 - nuc13-063 - nuc13-065 - nuc13-067 - - nuc13-069 + - nuc13-070 - nuc13-071 - nuc13-072 - nuc13-073 - nuc13-076 - - nuc13-077 - - nuc13-078 - nuc13-079 - nuc13-080 + - nuc13-082 - nuc13-084 + - nuc13-085 - nuc13-087 + - nuc13-091 - nuc13-092 + - nuc13-093 - nuc13-094 - nuc13-095 + - nuc13-096 - nuc13-098 - nuc13-099 - nuc13-100 + - nuc13-101 - nuc13-102 - nuc13-103 + - nuc13-105 - nuc13-106 - - nuc13-111 - nuc13-114 - - nuc13-115 - nuc13-116 - nuc13-117 - nuc13-120 - nuc13-122 - nuc13-125 + - nuc13-126 - nuc13-129 + - nuc13-130 - nuc13-131 - nuc13-133 - nuc13-134 - nuc13-136 + - nuc13-137 - nuc13-138 - nuc13-139 - nuc13-140 - nuc13-141 - nuc13-142 - - nuc13-143 + - nuc13-144 - nuc13-146 - nuc13-147 - nuc13-148 - - nuc13-152 - - nuc13-158 - # Belongs in alpha; was previously misplaced. - - nuc13-045 - - nuc13-006 - - nuc13-093 - - name: "win11-64-24h2-hw-perf-sheriff" - openvox_version: "8.19.2" - puppet_version: "8.10.0" - git_version: "2.50.0" - Description: "Tracking metric variability with performance sheriffing. Tracking in RELOPS-1288" - image: "win11-24H2-NUC-01-16-2025" - src_Organisation: "mozilla-platform-ops" - src_Repository: "ronin_puppet" - src_Branch: "master" - hash: "7511c8e" - secret_date: "02-24-2026" - domain_suffix: "wintest2.releng.mdc1.mozilla.com" - nodes: - - nuc13-021 - - name: "win11-64-24h2-hw" - openvox_version: "8.19.2" - puppet_version: "8.10.0" - git_version: "2.50.0" - Description: "NUC12 Hardware Performance Staging/Testing Pool" - image: "win11-24H2-NUC-01-16-2025" - src_Organisation: "mozilla-platform-ops" - src_Repository: "ronin_puppet" - src_Branch: "master" - hash: "edef633" - secret_date: "02-24-2026" - domain_suffix: "wintest2.releng.mdc1.mozilla.com" - nodes: - - nuc13-004 - - nuc13-005 - - nuc13-007 - - nuc13-008 - - nuc13-009 - - nuc13-010 - - nuc13-011 - - nuc13-012 - - nuc13-015 - - nuc13-016 - - nuc13-017 - - nuc13-018 - - nuc13-029 - - nuc13-032 - - nuc13-033 - - nuc13-036 - - nuc13-040 - - nuc13-041 - - nuc13-044 - - nuc13-046 - - nuc13-048 - - nuc13-049 - - nuc13-056 - - nuc13-057 - - nuc13-061 - - nuc13-062 - - nuc13-064 - - nuc13-066 - - nuc13-070 - - nuc13-074 - - nuc13-081 - - nuc13-083 - - nuc13-085 - - nuc13-086 - - nuc13-088 - - nuc13-089 - - nuc13-090 - - nuc13-091 - - nuc13-096 - - nuc13-097 - - nuc13-101 - - nuc13-104 - - nuc13-105 - - nuc13-108 - - nuc13-109 - - nuc13-110 - - nuc13-113 - - nuc13-118 - - nuc13-121 - - nuc13-123 - - nuc13-127 - - nuc13-128 - - nuc13-130 - - nuc13-135 - - nuc13-137 - - nuc13-144 - - nuc13-145 - nuc13-149 - nuc13-151 - - nuc13-153 + - nuc13-152 - nuc13-154 - nuc13-156 + - nuc13-159 + - nuc13-160 defaults: NV_domain: "mdc1.mozilla.com" Validate: diff --git a/provisioners/windows/win-hw-wim/.gitignore b/provisioners/windows/win-hw-wim/.gitignore new file mode 100644 index 00000000..30819c27 --- /dev/null +++ b/provisioners/windows/win-hw-wim/.gitignore @@ -0,0 +1,10 @@ +# Build artifacts — never commit images +output/ +work/ +*.vhdx +*.wim +*.iso +packer_cache/ +*.pkrvars.hcl +!example.pkrvars.hcl +crash.log diff --git a/provisioners/windows/win-hw-wim/DEPLOY-INTEGRATION.md b/provisioners/windows/win-hw-wim/DEPLOY-INTEGRATION.md new file mode 100644 index 00000000..7b74b39d --- /dev/null +++ b/provisioners/windows/win-hw-wim/DEPLOY-INTEGRATION.md @@ -0,0 +1,98 @@ +# Deploy integration + first-boot personalization + +How the baked `install.wim` plugs into the existing NUC deploy flow, and what +still happens at first boot. All `worker-images` / `ronin_puppet` edits below go +via **feature branch + PR — never a push to main**. + +## Where it slots in (existing MDC1Windows flow) + +Today `provisioners/windows/MDC1Windows/OS-deploy.ps1` (in WinPE): +1. partitions the disk, mounts the MDT share as `Z:`, +2. copies `Z:\Images\` → `D:\` (extracted Win11 media), +3. templates `autounattend.xml` (disk/partition, hostname, admin pw), +4. runs `setup.exe /unattend:autounattend.xml`. Setup applies + `sources\install.wim` **index 3**. + +`` comes from `pools.yml` (`image:` field). + +### Option A — drop-in (recommended, least change) +1. `publish-wim.ps1` clones the media folder and swaps in the baked WIM at + `Images\\sources\install.wim`. +2. In `worker-images` (PR): set the hw pool's `image:` in + `provisioners/windows/MDC1Windows/pools.yml` to ``. +3. In `worker-images` (PR): set the install image index/name in + `base-autounattend.xml` to match the captured image. A custom captured WIM is + index **1** (not 3): + ```xml + + /IMAGE/INDEX1 + + ``` + (or switch to an `/IMAGE/NAME` key = `win11-24h2-ci-baked`). + Everything else in the autounattend (hostname/disk/password templating, + FirstLogonCommands → `Get-Bootstrap.ps1`) stays as-is. + +### Option B — full DISM apply (DEFERRED follow-up) + +> **Status: deferred.** Decision (2026-07-21): ship Option A first and get the +> current bake→publish→deploy pipeline working end-to-end; take on the DISM +> `/Apply-Image` conversion afterward. Rationale: the imaging phase is only ~13 of +> the ~63 min, so the DISM win is modest vs. the ~30-min AppX bake — its value is +> determinism + dropping the ~5 GB media copy and the `setup.exe` variable, not +> raw minutes. When we do it, fold in offline driver injection at the same time. + +Replace `OS-deploy.ps1:750-754` (`Start-Process setup.exe /unattend`) with: +```powershell +dism /Apply-Image /ImageFile:D:\\sources\install.wim /Index:1 /ApplyDir:W:\ /CheckIntegrity /Verify +W:\Windows\System32\bcdboot W:\Windows /s S: /f UEFI +``` +Keep the partition/format block (`:151-261`) and the secret copy (`:681-684`). +Drive specialize/OOBE from an offline-injected unattend instead of the windowsPE +pass. More work; only pursue if you want to drop `setup.exe` entirely. + +### Driver injection (both options) +The bake VM lacks NUC hardware, so inject NUC drivers into the applied offline +image at deploy: +```powershell +dism /Image:W:\ /Add-Driver /Driver:Z:\Drivers\NUC13 /Recurse +``` + +## First-boot personalization (unchanged, but one guard) + +First boot still runs the existing chain: `autounattend` FirstLogonCommands → +`Get-Bootstrap.ps1` → `bootstrap.ps1`. Because the stable catalog is already +baked, this run is short and does only the machine-specific work: + +- **Seed identity** into `HKLM:\SOFTWARE\Mozilla\ronin_puppet` + (`role`, `workerType`, `worker_pool_id`, `GITHASH`, `secret_date`) — from the + pool, via `Set-Ronin-Registry`. +- **Copy the dated secret** `D:\secrets\-.yaml` → ronin + `data\secrets\vault.yaml` (`Get-Ronin`). +- **Run the FULL `win116424h2hw` role** `puppet apply` — now fast/idempotent + because AppX/updates/packages are baked; it applies the four deploy-time + profiles (`windows_worker_runner`, `microsoft_kms`, `nuc_bios`, + `nuc_management`) and registers worker-runner. + +### Required guard (ronin PR, feature branch) +`bootstrap.ps1`'s `Get-Ronin` currently **deletes and re-clones** `C:\ronin` +every run (`~:645-647`). To reuse the baked checkout (and keep first boot fast), +guard that so a deploy-time run reuses the pre-baked repo when the pinned hash +already matches: +```powershell +if ((Test-Path $ronin_repo) -and ((git -C $ronin_repo rev-parse HEAD) -like "$hash*")) { + # reuse baked checkout +} else { + Remove-Item $ronin_repo -Recurse -Force; git clone ... +} +``` +(If you prefer zero bootstrap changes, leave it — the re-clone costs ~1 min; the +big win is AppX, not the clone.) + +## Verify on a canary NUC + +1. `publish-wim.ps1` → baked WIM on the share; pools.yml/autounattend pointed at it. +2. PXE-dance `nuc13-004` (see `PXE_DANCE_RUNBOOK.md` / `AuditAndPXE.ps1`). +3. Re-run the SolarWinds timing analysis (filter `nuc13-004`) and compare to the + ~63 min / ~30-min-AppX baseline. Target ≈ 20–25 min, worker-runner healthy. +4. SSH-check: `bootstrap_stage=complete`, `last_run_exit` ∈ {0,2}, AppX already + absent (bake worked), worker claims a task. diff --git a/provisioners/windows/win-hw-wim/New-WinHwWimBuildVm.ps1 b/provisioners/windows/win-hw-wim/New-WinHwWimBuildVm.ps1 new file mode 100644 index 00000000..d3288e45 --- /dev/null +++ b/provisioners/windows/win-hw-wim/New-WinHwWimBuildVm.ps1 @@ -0,0 +1,122 @@ +<# +.SYNOPSIS + Provision the Azure VM that builds Windows HW install.wim images with Packer (nested + Hyper-V). The "scripting in worker-images" entry point for standing up the + build host; the WIM build itself stays Packer (bin/WinHwWim/New-WinHwWim.ps1). + +.DESCRIPTION + Creates a nested-virtualization-capable Windows Server VM in the existing + win-hw-wim network (rg-central-us-hardware-imaging / sn-central-us-hardware-imaging-packer, from the + storage Terraform), attaches a Premium data disk for build artifacts, gives it a + system-assigned managed identity, grants that identity Storage Blob Data + Contributor on hardwareimaging (so azcopy --auth-mode login works with no secrets), + and bootstraps Hyper-V + Packer + ADK + azcopy + git. + + Nested virtualization requires a supported SKU (Dv3/Dv4/Dv5, Ev3+, Fsv2, ...); + default Standard_D8s_v5. + + After it finishes: RDP in (or `az vm run-command`) and run the build: + cd C:\worker-images\provisioners\windows\win-hw-wim # (clone the repo there) + az login --identity + .\bin\WinHwWim\New-WinHwWim.ps1 -Image win11-24h2-hw + +.PARAMETER AllowRdpFrom + CIDR(s) allowed to RDP (NSG). Default: Mozilla VPN netblocks. Pass your real + egress if VPN is split-tunnel. Use -NoPublicIp to skip inbound entirely. + +.EXAMPLE + ./New-WinHwWimBuildVm.ps1 -AllowRdpFrom 63.245.208.132/32 +#> +[CmdletBinding()] +param( + [string] $VmName = 'win-hw-wim-builder', + [string] $ResourceGroup = 'rg-central-us-hardware-imaging', + [string] $Location = 'centralus', + [string] $VnetName = 'vn-central-us-hardware-imaging', + [string] $SubnetName = 'sn-central-us-hardware-imaging-packer', + [string] $Size = 'Standard_D64ads_v5', # AMD, 64 vCPU/256 GiB, nested-virt capable (fits DADSv5 64-core quota) + # Plain WS2025 gen2 (azure-edition forces Trusted Launch, which breaks nested virt). + [string] $Image = 'MicrosoftWindowsServer:WindowsServer:2025-datacenter-g2:latest', + [int] $DataDiskGB = 512, + [string] $StorageAccount = 'hardwareimaging', + [string] $AdminUsername = 'nucadmin', + [string] $AdminPassword, # generated + printed if omitted + [string[]] $AllowRdpFrom = @('63.245.208.132/32', '63.245.208.133/32', '63.245.210.132/32', '63.245.210.133/32', '185.155.182.210/32'), + [switch] $NoPublicIp +) +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +# Resolve the real az executable — a function named "Az" would otherwise shadow +# "az" (PowerShell is case-insensitive) and recurse infinitely. +$azExe = (Get-Command az -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source +function Az { $o = & $azExe @args 2>&1; if ($LASTEXITCODE) { throw "az $($args -join ' ') failed:`n$o" }; return $o } + +$sub = (Az account show --query id -o tsv) +Write-Host "== Subscription: $sub ==" + +$subnetId = (Az network vnet subnet show -g $ResourceGroup --vnet-name $VnetName -n $SubnetName --query id -o tsv) +$storageId = (Az storage account show -g $ResourceGroup -n $StorageAccount --query id -o tsv) + +if (-not $AdminPassword) { + # Portable random password (System.Web is Windows-only / absent in pwsh 7). + $AdminPassword = 'Aa1!' + [guid]::NewGuid().ToString('N') + [guid]::NewGuid().ToString('N').Substring(0, 12) + Write-Host "== Generated admin password (save it now): $AdminPassword ==" +} + +Write-Host "== Creating VM $VmName ($Size, nested-virt) in $SubnetName ==" +$pip = if ($NoPublicIp) { '""' } else { "$VmName-pip" } +Az vm create ` + -g $ResourceGroup -n $VmName -l $Location ` + --image $Image --size $Size --security-type Standard ` + --admin-username $AdminUsername --admin-password $AdminPassword ` + --subnet $subnetId --public-ip-address $pip --nsg "$VmName-nsg" ` + --assign-identity '[system]' ` + --os-disk-size-gb 128 --storage-sku Premium_LRS ` + --data-disk-sizes-gb $DataDiskGB ` + --output none + +# Lock RDP down to the allow-list (default-deny otherwise). +if (-not $NoPublicIp) { + Write-Host "== NSG: allow RDP only from $($AllowRdpFrom -join ', ') ==" + Az network nsg rule create -g $ResourceGroup --nsg-name "$VmName-nsg" -n allow-rdp ` + --priority 300 --access Allow --protocol Tcp --direction Inbound ` + --destination-port-ranges 3389 --source-address-prefixes @AllowRdpFrom --output none +} + +# Grant the VM's managed identity blob data access (Entra-only storage). +$principalId = (Az vm show -g $ResourceGroup -n $VmName --query identity.principalId -o tsv) +Write-Host "== Granting Storage Blob Data Contributor to VM identity $principalId ==" +Az role assignment create --assignee-object-id $principalId --assignee-principal-type ServicePrincipal ` + --role 'Storage Blob Data Contributor' --scope $storageId --output none + +# Bootstrap: Hyper-V (phase 1) -> reboot -> tooling (phase 2). +$boot = Join-Path $PSScriptRoot 'scripts\bootstrap-build-host.ps1' +Write-Host '== Bootstrap phase 1: enable Hyper-V ==' +Az vm run-command invoke -g $ResourceGroup -n $VmName --command-id RunPowerShellScript ` + --scripts "@$boot" --parameters 'Phase=Hyperv' --output none +Write-Host '== Rebooting for Hyper-V ==' +Az vm restart -g $ResourceGroup -n $VmName --output none +Start-Sleep -Seconds 30 # let the guest agent come back before run-command +Write-Host '== Bootstrap phase 2: install Packer/ADK/azcopy/git/az + powershell-yaml ==' +$phase2 = $false +for ($i = 1; $i -le 5; $i++) { + try { + Az vm run-command invoke -g $ResourceGroup -n $VmName --command-id RunPowerShellScript ` + --scripts "@$boot" --parameters 'Phase=Tooling' --output none + $phase2 = $true; break + } + catch { + Write-Warning "phase 2 attempt $i failed (guest agent may still be starting); retrying in 30s" + Start-Sleep -Seconds 30 + } +} +if (-not $phase2) { throw 'Bootstrap phase 2 (tooling) failed after retries.' } + +$ip = if ($NoPublicIp) { '(no public IP — use Bastion/jumpbox)' } else { (Az vm show -d -g $ResourceGroup -n $VmName --query publicIps -o tsv) } +Write-Host "" +Write-Host "== Build host ready ==" +Write-Host " VM : $VmName ($Size) RDP: $ip user: $AdminUsername" +Write-Host " Identity: $principalId (Storage Blob Data Contributor on $StorageAccount)" +Write-Host " Next : RDP in, clone worker-images, then:" +Write-Host " az login --identity" +Write-Host " .\provisioners\windows\win-hw-wim\bin\WinHwWim\New-WinHwWim.ps1 -Image win11-24h2-hw" diff --git a/provisioners/windows/win-hw-wim/README.md b/provisioners/windows/win-hw-wim/README.md new file mode 100644 index 00000000..924dc8e6 --- /dev/null +++ b/provisioners/windows/win-hw-wim/README.md @@ -0,0 +1,208 @@ +# win-hw-wim — Baked `install.wim` pipeline for Windows HW CI workers + +A Packer (**nested Hyper-V**) workflow that starts from **your own base +`install.wim`** (BYO, from the `hardwareimaging` `resources/` blob), runs the ronin Puppet +*bake* (stable, expensive config), Sysprep generalizes, and captures a new golden +`install.wim`. No Azure marketplace or compute-gallery dependency. + +Goal: move the ~30-min deploy-time AppX removal (and most of the ~36-min Puppet +run) into a pre-baked image, cutting NUC deploy time from ~63 min toward ~20–25 min. + +The build runs on an **Azure VM** (nested-virtualization SKU) stood up by +`New-WinHwWimBuildVm.ps1`; the WIM build itself is Packer (`bin/WinHwWim/New-WinHwWim.ps1`). +Lives in worker-images at `provisioners/windows/win-hw-wim/`. + +> Constraint: nothing here pushes to `main`/`master`. The one ronin change (the +> `win116424h2hwbake` role) is authored on a feature branch in the `ronin_puppet` +> checkout. + +## Pipeline + +``` +your base install.wim + 1. prepare-base-vhdx.ps1 apply WIM -> bootable VHDX (DISM /Apply-Image + bcdboot) + 2. packer build win-hw-wim Hyper-V boots VHDX + 3. bake-bootstrap.ps1 install puppet/git, clone ronin, AppX (provisioned) removal, + WU/choco, puppet apply of the BAKE role + 4. sysprep-generalize scrub machine state + Sysprep /generalize /shutdown + 5. capture-wim.ps1 mount generalized VHDX -> DISM /Capture-Image -> install.wim + SHA256 + 6. publish copy install.wim to the MDT/WDS deployment share + 7. deploy existing PXE dance -> first-boot personalization + worker registration +``` + +## Layout + +| Path | Purpose | +| --- | --- | +| `config/win-hw-wim-defaults.yaml` | Shared defaults (storage, ronin, versions, VM size). Fields set to `"default"` in an image config resolve here. | +| `config/.yaml` | **One file per WIM.** Base WIM, edition, optional driver injection, bake role, branch, versions. Adding a WIM = adding a file. | +| `New-WinHwWimBuildVm.ps1` | Provision the Azure nested-virt build host (managed identity + Hyper-V + tooling). | +| `scripts/bootstrap-build-host.ps1` | On-VM bootstrap (Hyper-V + Packer/ADK/azcopy/git), run by the provisioner. | +| `bin/WinHwWim/New-WinHwWim.ps1` | **Orchestrator.** `-Image ` runs prep → build → publish with per-image namespacing. | +| `win-hw-wim.pkr.hcl` | Packer Hyper-V template (build + provision + capture) | +| `variables.pkr.hcl` | Input variable declarations | +| `example.pkrvars.hcl` | Reference only — the orchestrator generates the real var-file per build | +| `scripts/prepare-base-vhdx.ps1` | BYO WIM → bootable VHDX (Windows host, admin) | +| `scripts/bake-bootstrap.ps1` | Build-time bake (runs inside the VM via Packer) | +| `scripts/sysprep-generalize.ps1` | Scrub + Sysprep (runs inside the VM) | +| `scripts/capture-wim.ps1` | Capture WIM from generalized VHDX (Windows host, admin) | +| `scripts/download-wim.ps1` / `upload-wim.ps1` | Move WIMs to/from the private store (Entra auth) | +| `scripts/extract-wim-from-iso.ps1` | Extract `sources\install.wim` from a base ISO (prep's `base.iso` fallback when no base WIM exists) | +| `scripts/publish-wim.ps1` | Copy WIM to MDT share (Windows host) | +| `work//` | Per-image build artifacts (base WIM, VHDX, build dir, golden WIM) — gitignored | + +Config style and tooling mirror **worker-images** (`config/*.yaml` + a `bin/` driver, +with the `"default"` → defaults-file resolution). + +## Prerequisites (Windows host) + +- Hyper-V enabled; `packer`, `az`, and `azcopy` on PATH; Windows ADK (DISM) available. +- Admin PowerShell; `powershell-yaml` module (auto-installed by the orchestrator); ~60–80 GB free disk. +- A base WIM uploaded to the `resources` container (e.g. `win11-24h2-base-install.wim`). +- Entra identity with Storage Blob Data access (a Relops member, or the uploader/downloader SP). +- For publish/deploy: write access to the MDT share + a canary NUC. + +## Storage + +Everything lives in the **`hardwareimaging`** Azure Blob account, **Entra-only** +(open network, no account keys, no anonymous — access is gated purely by Storage Blob +Data RBAC). Layout (folders are blob prefixes within each container): + +``` +resources/ SOURCES + WIMs/ BYO base WIMs (-base-install.wim) + ISOs/ source Win11 ISOs + drivers/ offline driver packs (injected at bake) + tools/ adksetup.exe + cached oscdimg (iso stage) +captured/ OUTPUTS + WIMs/ golden WIMs (/-.wim) + ISOs/ nocheck ISOs (/-.iso) +legacy-images/ old, previously-built images (archive) +``` + +See `STORAGE-DESIGN.md`; provisioned by Terraform (PR #313). + +## Azure build host (one-time) + +The build runs on an Azure VM with **nested virtualization** (for Hyper-V). Stand it +up from any machine with `az`: + +```powershell +./New-WinHwWimBuildVm.ps1 -AllowRdpFrom +``` + +That creates `win-hw-wim-builder` (Standard_D8s_v5) in `rg-central-us-hardware-imaging` on the +existing `sn-central-us-hardware-imaging-packer` subnet, attaches a Premium data disk, gives it +a **system-assigned managed identity** granted *Storage Blob Data Contributor* on +`hardwareimaging` (no secrets on the box), and bootstraps Hyper-V + Packer + ADK + azcopy + +git (`scripts/bootstrap-build-host.ps1`, 2-phase around the Hyper-V reboot). SKU must +support nested virt (Dv3/Dv4/Dv5, Ev3+, Fsv2, …). + +## Build (on the Azure build host) + +RDP in (or `az vm run-command`), clone worker-images, then one command per WIM. The VM +uses its managed identity, so just: + +```powershell +az login --identity # storage is Entra-only +.\bin\WinHwWim\New-WinHwWim.ps1 -Image win11-24h2-hw +``` + +That runs, per `config/win11-24h2-hw.yaml`: +`prep` (download base WIM → `prepare-base-vhdx` → `register-base-vm`) → +`build` (`packer build`: WU → bake role → sysprep → capture) → +`publish` (upload golden WIM to `captured/WIMs//`). Run a subset with `-Stages`, keep +the VM/VHDX with `-KeepArtifacts`, or re-publish a prior build with `-Stages publish -BuildId `. + +### Win11 ISO builder (`iso` stage — separate from the ronin base WIMs) + +A DISTINCT function from the WIM bakes: build a Win11 install ISO from a base ISO, running a +config-selected set of **inject-library scripts** against the media before repackaging. The config +mirrors the WIM shape — a `base:` source plus a provisioning source — but the provisioning source is +`scripts:` (inject-library) instead of `ronin:` (clone+apply ronin_puppet); a config uses EITHER. + +`config/win11-25h2-iso.yaml`: +```yaml +base: + iso: "Win11_25H2_English_x64_v2.iso" # base Win11 ISO you uploaded to resources/ISOs/ +scripts: + - nocheck # scripts/inject/nocheck.ps1 — TPM/SecureBoot/RAM/CPU/storage bypass +iso: + enabled: true # build the ISO (iso stage) instead of the WIM bake + label: "WIN11_25H2_NOCHK" # volume label; output -> captured/ISOs//-.iso +``` +```powershell +.\bin\WinHwWim\New-WinHwWim.ps1 -Image win11-25h2-iso # iso.enabled selects the iso stage +``` + +Flow: azcopy-download `resources/ISOs/` (the source) → for each name in `scripts:` run +`scripts/inject/.ps1 -MediaDir ` (e.g. `nocheck` writes an `autounattend.xml` +with the `HKLM\SYSTEM\Setup\LabConfig` bypass keys + `MoSetup` for windowsPE) → repackage a bootable ISO +with `oscdimg` (ADK) → upload the built ISO (+ `.sha256`) to +**`captured/ISOs//-.iso`** (a captured OUTPUT, same naming as the golden WIMs). Add a +new behavior by dropping a script into `scripts/inject/` (contract: `-MediaDir`) and listing its name. +Ref: https://woshub.com/upgrade-to-windows-11-unsupported-pc/ + +`oscdimg` isn't native to Windows, so the iso stage runs `ensure-oscdimg.ps1` first: it restores the +ADK Deployment Tools' `oscdimg` from **our blob** (`resources/tools/oscdimg/`); on the first-ever build it +installs Deployment Tools from `resources/tools/adksetup.exe` (hosted by us) and caches `oscdimg` back to +`resources/tools/oscdimg/`, so later builds never touch the Microsoft ADK CDN. + +Publishing to the MDT share for a canary deploy is still a deliberate step: + +```powershell +.\scripts\publish-wim.ps1 -Wim .\work\win11-24h2-hw\win11-24h2-hw-.wim -MediaTemplate -ImageName +# On-site MDC1 pulls privately (Entra SP): .\scripts\download-wim.ps1 -Blob captured/win11-24h2-hw/ -Dest \\mdt2022...\staging\install.wim +``` + +## Adding another WIM (scalable — OS version *or* specialized pool) + +1. Upload a base WIM to `base/` (convention: `-base-install.wim`), if it's a new OS. +2. Add `config/.yaml` (copy `win11-24h2-hw.yaml`). The image id is the filename. + - **New OS version:** point `base.wim` at the new base, e.g. `win11-25h2-hw.yaml`. + - **Specialized pool:** reuse the same `base.wim`, change `bake_role` / `worker_pool_id`, + e.g. `win11-24h2-perf.yaml`. +3. `.\bin\WinHwWim\New-WinHwWim.ps1 -Image `. Names (VHDX, VM, output, blob) are + namespaced by image id, so builds never collide. + +### `base.edition` and `drivers` (per-image config) + +- **`base.iso` (fallback source)** — optional. If `base/` is not present, prep + extracts `sources\install.wim` from `base/`, saves it as `base.wim` (naming + convention `-base-install.wim`), and caches it back to `resources/WIMs/` so later bakes reuse + it — so a WIM bake can start from **just an uploaded ISO** (no manual WIM upload). If the + media ships `install.esd`, every edition is exported to a WIM (`extract-wim-from-iso.ps1`). +- **`base.edition`** — the edition NAME inside the WIM (a multi-edition `install.wim` + has several indexes). `prepare-base-vhdx.ps1` resolves it to the index via + `Get-WindowsImage`; if the name isn't found it fails and lists what's available. Run + `dism /Get-WimInfo /WimFile:` to see the names. +- **`drivers`** — optional offline driver injection, **OFF by default**. `cabs` is a + list, so any number of driver packs can be injected; each entry may be a **`.cab` or + a `.zip`** (sniffed by extension — zip a driver directory with `Compress-Archive`): + ```yaml + drivers: + inject: true + cabs: + - https://.../nuc13-24h2-nuc_driver.zip # .cab or .zip; must expand to an .inf tree + - https://.../extra-pack.cab # add more as needed + ``` + When `inject: true`, `prepare-base-vhdx.ps1` downloads each pack, expands them (cab via + `expand.exe`, zip via `Expand-Archive`) into separate subdirs, and runs a single + recursive `DISM /Add-Driver /Recurse` into the applied image **before capture**, so the + drivers land in the golden WIM. (A single `cab_url: ` string is still accepted for + back-compat.) + +Everything is parameterized; nothing is hardcoded to a machine. Review each script +before running — these touch disks and Sysprep. + +## Follow-ups (deferred until the current pipeline is working) + +- **Deploy via `DISM /Apply-Image` (Option B in `DEPLOY-INTEGRATION.md`).** Replace + `setup.exe /unattend` with a direct WIM apply + `bcdboot` + offline + Panther unattend + offline driver injection. Deferred 2026-07-21: get Option A + (drop baked WIM into the media folder, keep `setup.exe`) working first; the DISM + path is a determinism/IO improvement, not the main time win. +- Fast-path the ronin AppX removal script itself (`Remove-AppxProvisionedPackage` + only, drop the `Wait-AppxIdle`/`Invoke-WithTimeout` loop) — also speeds the + deploy-time reconciliation run. +- Optionally bake Puppet/Git into the *base* WIM to shave the ~12-min early bootstrap. diff --git a/provisioners/windows/win-hw-wim/STORAGE-DESIGN.md b/provisioners/windows/win-hw-wim/STORAGE-DESIGN.md new file mode 100644 index 00000000..ea50eb4e --- /dev/null +++ b/provisioners/windows/win-hw-wim/STORAGE-DESIGN.md @@ -0,0 +1,58 @@ +# WIM storage design (Azure Blob, Entra-only) + +Store both the base and captured WIMs in an **Azure Blob** account +(`hardwareimaging`, Central US). Access is **Entra-only**: the account has a public +endpoint open to all networks, but no anonymous access and **no shared account +keys** — every caller must present an Entra identity holding a Storage Blob Data +RBAC role. This avoids building an Azure↔MDC1 VPN (which does not exist today). + +> History: this started as a Tier-1 IP-firewalled account, but split-tunnel VPN +> made per-workstation IP allow-listing unworkable, so it moved to Entra-only +> (RBAC gates access regardless of source network). + +## Why not reuse `roninpuppetassets` +It's fully public (`container_access_type = "blob"`, anonymous read). WIMs must +not be anonymously downloadable, so we use a **separate, Entra-gated** account. + +## What is provisioned (Terraform) +`relops_infra_as_code/terraform/azure_fxci/nuc-wim-storage.tf` (branch +`nuc-wim-storage`, **PR #313**) — applied to the FXCI DevTest subscription: +- RG `rg-central-us-hardware-imaging`, VNet `vn-central-us-hardware-imaging` + subnet + `sn-central-us-hardware-imaging-packer` (retained; not required for access now). +- Storage account **`hardwareimaging`** (StorageV2, LRS): `network_rules.default_action + = Allow` (no IP firewall), `shared_access_key_enabled = false` (no key/SAS), + `allow_nested_items_to_be_public = false`, TLS1.2, HTTPS-only. Managed via an + aliased `azurerm` provider with `storage_use_azuread = true` (keys disabled). +- Containers: `resources` (sources: WIMs/, ISOs/, drivers/, tools/), `captured` (outputs: WIMs/, ISOs/), and `legacy-images` (old previously-built images) — all private. +- RBAC: Packer/`worker_images` SP = **Blob Data Contributor**; MDC1 downloader SP + = **Blob Data Reader** on `captured` only; **Relops group** = Blob Data + Owner + Contributor (+ Queue/File Data roles so Terraform can read service + properties via AAD). + +`relops_infra_as_code/terraform/azure_ad/sp_nuc_wim_downloader.tf`: +- Entra app/SP `sp-relops-nuc-wim-downloader` for the on-site MDC1 server + a + client secret. **Not** stored in Key Vault (MDC1 isn't Entra-joined) — kept as a + local file on the box; lives only in Terraform state. + +## Access paths (all Entra `--auth-mode login`, any network) +- **Azure build VM**: authenticates with its **system-assigned managed identity** + (granted Blob Data Contributor by `New-WinHwWimBuildVm.ps1`) — reads `resources`, + writes `captured`. No secret on the box. +- **On-site MDC1 server**: `az login --service-principal` with the downloader SP + (needs outbound reach to `login.microsoftonline.com`), reads `captured`. +- **Operators**: their own Entra identity via the Relops group. + +## Pipeline wiring +- Build host, before build: `download-wim.ps1` pulls the base WIM from `resources/WIMs/` → + `prepare-base-vhdx.ps1`. +- Build host, after capture: `upload-wim.ps1` pushes the WIM (+ `.sha256`) to + `captured/WIMs//`. +- Deploy: MDC1 server `download-wim.ps1` pulls from `captured/WIMs/` to the MDT share, + then the existing PXE dance applies it. + +## Notes +- Verified: anonymous → `PublicAccessNotPermitted`; account-key → `KeyBasedAuthenticationNotPermitted`; + Entra identity + Blob Data role → works from any network. +- Resources rebranded `nuc-wim` → `hardware-imaging` (RG/VNet/subnet/UAMI/account) for + prod prep; the pipeline tooling uses the generic `win-hw-wim` naming. The downloader SP + `sp-relops-nuc-wim-downloader` keeps its name (renaming would rotate its secret). diff --git a/provisioners/windows/win-hw-wim/bin/WinHwWim/New-WinHwWim.ps1 b/provisioners/windows/win-hw-wim/bin/WinHwWim/New-WinHwWim.ps1 new file mode 100644 index 00000000..2dc78b20 --- /dev/null +++ b/provisioners/windows/win-hw-wim/bin/WinHwWim/New-WinHwWim.ps1 @@ -0,0 +1,635 @@ +<# +.SYNOPSIS + Build a golden Windows HW install.wim from a per-image YAML config. The scalable entry + point for the wim-packer pipeline (one config = one WIM). Mirrors the + worker-images bin/WorkerImages driver + config/*.yaml convention. + +.DESCRIPTION + Given -Image , reads config/.yaml (falling back to + config/win-hw-wim-defaults.yaml for any field set to the string "default"), + derives per-image namespaced names so many WIMs coexist, and runs the pipeline: + + prep : get base WIM (download, or extract from base.iso if the WIM is + absent) -> prepare-base-vhdx -> register-base-vm + build : packer build (WU -> bake role -> sysprep -> capture) -> .wim + publish : upload captured WIM (+ .sha256) to captured// + + Derived, per-image (from -Image and -BuildId): + work dir work// + base WIM work// (from resources/WIMs/) + VHDX work//base.vhdx + VM name wim-bake- + build dir work//build (packer output_directory) + golden WIM work//-.wim + blob captured/WIMs//-.wim + + Auth: if AZ_CLIENT_ID / AZ_CLIENT_SECRET / AZ_TENANT are set, logs in as that + SP; otherwise assumes an existing `az login` (a Relops member). Storage is + Entra-only (no keys). + +.PARAMETER Image + Config basename under config/ (e.g. win11-24h2-hw). + +.PARAMETER Stages + Subset of prep,build,publish to run (default: all three, in order). + +.PARAMETER WinRMPassword + Password for the build-only WinRM account injected into the base VHDX. + Auto-generated if not supplied (build-scoped; scrubbed before capture). + +.PARAMETER BuildId + Build identifier used in output names. Default: yyyyMMdd-HHmmss. + +.PARAMETER KeepArtifacts + Keep the per-image VM / VHDX / build dir after a successful run (default: clean up). + +.EXAMPLE + # Full build of the Win11 24H2 hw image: + .\bin\WinHwWim\New-WinHwWim.ps1 -Image win11-24h2-hw + +.EXAMPLE + # Just re-publish an already-captured WIM: + .\bin\WinHwWim\New-WinHwWim.ps1 -Image win11-24h2-hw -Stages publish -BuildId 20260723-101500 +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $Image, + [ValidateSet('prep', 'build', 'publish', 'iso')] [string[]] $Stages = @('prep', 'build', 'publish'), + [string] $WinRMPassword, + [string] $BuildId, + # Client ID of the user-assigned managed identity to log in with on the build VM. + # The VM is attached a USER-assigned identity (no system-assigned), so bare + # `az login --identity` fails ("Please run az login") — it must be told which one. + [string] $IdentityClientId, + # Build-scoped GitHub token for puppet's tooltool download in the bake. Defaults to + # $env:GITHUB_TOKEN / $env:PACKER_GITHUB_API_TOKEN (CI sets these). Empty is OK — + # tooltool.py is public and downloads without a token. Not baked into the WIM. + [string] $GithubPat = ($env:GITHUB_TOKEN, $env:PACKER_GITHUB_API_TOKEN, '' | Where-Object { $_ } | Select-Object -First 1), + [switch] $KeepArtifacts +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +# --- Paths ------------------------------------------------------------------- +$Root = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path # wim-packer/ +$ConfigDir = Join-Path $Root 'config' +$ScriptDir = Join-Path $Root 'scripts' +$imgCfg = Join-Path $ConfigDir "$Image.yaml" +$defCfg = Join-Path $ConfigDir 'win-hw-wim-defaults.yaml' +foreach ($p in @($imgCfg, $defCfg)) { if (-not (Test-Path $p)) { throw "Config not found: $p" } } + +# --- YAML --------------------------------------------------------------------- +if (-not (Get-Module -ListAvailable -Name powershell-yaml)) { + Write-Host '== Installing powershell-yaml module ==' + # Avoid the non-interactive NuGet-provider / PSGallery-trust prompt hang. + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force | Out-Null + Set-PSRepository -Name PSGallery -InstallationPolicy Trusted + Install-Module powershell-yaml -Scope CurrentUser -Force -Confirm:$false +} +Import-Module powershell-yaml +$cfg = ConvertFrom-Yaml (Get-Content -Raw $imgCfg) +$def = ConvertFrom-Yaml (Get-Content -Raw $defCfg) + +# Resolve a config value, falling back to the defaults file when it is "default" +# or absent. $Section/$Key index into both maps. +function Get-Val { + param([string]$Section, [string]$Key) + $v = $null + if ($cfg.ContainsKey($Section) -and $cfg[$Section] -and $cfg[$Section].ContainsKey($Key)) { $v = $cfg[$Section][$Key] } + if ($null -eq $v -or "$v" -eq 'default') { + if ($def.ContainsKey($Section) -and $def[$Section] -and $def[$Section].ContainsKey($Key)) { $v = $def[$Section][$Key] } + } + return $v +} + +if (-not $BuildId) { $BuildId = Get-Date -Format 'yyyyMMdd-HHmmss' } + +# --- Resolved settings -------------------------------------------------------- +$account = $def['storage']['account'] +$baseCont = $def['storage']['base_container'] +$capCont = $def['storage']['captured_container'] + +# Null-safe section access: an iso-only config (config/*-iso.yaml) has no base:/ronin: sections. +$baseCfg = if ($cfg.ContainsKey('base') -and $cfg['base']) { $cfg['base'] } else { @{} } +$baseWim = $baseCfg['wim'] # source WIM blob (WIM bakes) +$edition = $baseCfg['edition'] +$baseIso = $baseCfg['iso'] # source Win11 ISO blob (iso builds) + +# Robust bool: handles real YAML booleans AND quoted strings ("true"/"false"). +$drvInject = ("$(Get-Val 'drivers' 'inject')".Trim() -match '^(true|1|yes)$') +# Scalable driver injection: 'drivers.cabs' is a YAML list of cab URLs. Normalize to +# a trimmed, non-empty string array. Back-compat: also accept a single 'drivers.cab_url'. +$drvCabUrls = @(Get-Val 'drivers' 'cabs' | ForEach-Object { "$_".Trim() } | Where-Object { $_ }) +# 'extras.files' is a YAML list of payloads copied verbatim into the image at C:\extras\ +# for the DEPLOY-time puppet apply to run (e.g. the Intel graphics installer, which +# supplies IntelGraphicsSoftwareService). Not drivers: never expanded, never DISM-injected. +# Staged offline because neither the guest nor a deployed NUC has an Azure identity and the +# payloads live in the Entra-only hardwareimaging account. These DO ship in the golden WIM. +$extraUrls = @(Get-Val 'extras' 'files' | ForEach-Object { "$_".Trim() } | Where-Object { $_ }) +if ($drvCabUrls.Count -eq 0) { + $legacyCab = "$(Get-Val 'drivers' 'cab_url')".Trim() + if ($legacyCab) { $drvCabUrls = @($legacyCab) } +} + +# ISO builder: source media is base.iso; the requirement bypass etc. come from scripts:. +# The built ISO is a captured OUTPUT (see $capIsoBlob) named like the golden WIMs, so the +# only per-image setting is the volume label. +$isoLabel = "$(Get-Val 'iso' 'label')".Trim() +if (-not $isoLabel) { $isoLabel = 'WIN11_NOCHK' } + +# Provisioning option 'scripts': inject-library scripts (scripts/inject/.ps1) run against the +# media/image before capture (the alternative to 'ronin'). e.g. scripts: [nocheck]. +$scripts = @() +if ($cfg.ContainsKey('scripts') -and $cfg['scripts']) { $scripts = @($cfg['scripts'] | ForEach-Object { "$_".Trim() } | Where-Object { $_ }) } + +# Config-driven stage selection: iso.enabled=true means this config builds a requirement-bypass +# ISO instead of the WIM bake, so just selecting the image is enough. An explicit -Stages overrides. +$isoEnabled = ("$(Get-Val 'iso' 'enabled')".Trim() -match '^(true|1|yes)$') +if (-not $PSBoundParameters.ContainsKey('Stages')) { + $Stages = if ($isoEnabled) { @('iso') } else { @('prep', 'build', 'publish') } +} + +$roninOrg = Get-Val 'ronin' 'org' +$roninRepo = Get-Val 'ronin' 'repo' +# Null-safe: an iso-only config has no ronin: section. +$roninCfg = if ($cfg.ContainsKey('ronin') -and $cfg['ronin']) { $cfg['ronin'] } else { @{} } +$roninBr = $roninCfg['branch'] +$roninHash = if ($roninCfg.ContainsKey('hash')) { [string]$roninCfg['hash'] } else { '' } +$bakeRole = $roninCfg['bake_role'] +$extSrc = $def['ronin']['ext_src'] + +$puppetV = Get-Val 'vm' 'puppet_version' +$gitV = Get-Val 'vm' 'git_version' +$openvoxV = Get-Val 'vm' 'openvox_version' +$cpus = [int](Get-Val 'vm' 'cpus') +$memMb = [int](Get-Val 'vm' 'memory_mb') +$switch = Get-Val 'vm' 'switch_name' +# Robust bool (real YAML bool OR quoted string). Shared default is false. +$winUpdate = ("$(Get-Val 'vm' 'windows_update')".Trim() -match '^(true|1|yes)$') + +# --- Validate required inputs (fail fast, before touching disks/Azure) --------- +# The WIM-bake inputs (base.wim/edition/bake_role/drivers) are only needed for prep/build; +# the 'iso' stage is standalone and validates its own inputs. +$wimStages = ($Stages -contains 'prep') -or ($Stages -contains 'build') +if ($wimStages -and -not $baseWim) { throw "config/$Image.yaml: base.wim is required." } +if ($wimStages -and -not $edition) { throw "config/$Image.yaml: base.edition is required (the WIM edition name; empty would silently default to index 1)." } +if (($Stages -contains 'build') -and -not $bakeRole) { throw "config/$Image.yaml: ronin.bake_role is required." } +if ($drvInject -and $drvCabUrls.Count -eq 0) { throw "config/$Image.yaml: drivers.inject is true but drivers.cabs is empty." } +if (($Stages -contains 'iso') -and -not $baseIso) { throw "config/$Image.yaml: base.iso is required for the iso stage." } +# Provisioning is EITHER ronin (bake_role) OR scripts, not both. +if ($scripts.Count -gt 0 -and $bakeRole) { throw "config/$Image.yaml: use EITHER ronin (bake_role) OR scripts, not both." } + +# --- Derived, per-image names ------------------------------------------------- +# Large artifacts (base WIM ~5.6 GB, base VHDX, packer's clone/export, captured WIM) +# go on the big data disk (F:, 512 GB) when present — the C: OS disk (128 GB) is far +# too small to hold them all. bootstrap-build-host.ps1 formats F: as the data disk. +$workRoot = if (Test-Path 'F:\') { 'F:\wim-work' } else { Join-Path $Root 'work' } +$work = Join-Path $workRoot $Image +$localBase = Join-Path $work $baseWim +# Blob layout (folders are prefixes): SOURCES under resources/WIMs & resources/ISOs, plus +# resources/drivers & resources/tools; OUTPUTS under captured/WIMs & captured/ISOs. +# (Containers: 'resources' = $baseCont, 'captured' = $capCont.) +$baseWimBlob = "WIMs/$baseWim" # source base WIM -> /WIMs/ +$baseIsoBlob = "ISOs/$baseIso" # source Win11 ISO -> /ISOs/ +$vhdx = Join-Path $work 'base.vhdx' +$vmName = "wim-bake-$Image" +$buildDir = Join-Path $work 'build' +$goldenWim = Join-Path $work "$Image-$BuildId.wim" +$capBlob = "WIMs/$Image/$Image-$BuildId.wim" +# Release notes / SBOM: Packer downloads the guest's markdown here (same filename the +# Azure gallery images use, -.md, so it drops straight into sboms/). +# Published twice: next to the WIM as provenance, and under _status/sbom/ where the GH +# runner can find it by prefix without needing to know the build id. +$sbomMd = Join-Path $work "$Image-$BuildId.md" +$sbomBlob = "WIMs/$Image/$Image-$BuildId.md" +$sbomRunBlob = "_status/sbom/$Image-$BuildId.md" +# iso stage output — a captured OUTPUT artifact, so it mirrors the golden-WIM naming +# (captured/ISOs//-.iso) instead of a fixed name in resources/. +$goldenIso = Join-Path $work "$Image-$BuildId.iso" +$capIsoBlob = "ISOs/$Image/$Image-$BuildId.iso" + +New-Item -ItemType Directory -Path $work -Force | Out-Null + +Write-Host "===================================================================" +Write-Host " Image : $Image (build $BuildId)" +Write-Host " Base WIM : $baseCont/$baseWimBlob (edition '$edition')" +Write-Host " Bake role : $bakeRole ronin $roninOrg/$roninRepo@$roninBr" +Write-Host " Versions : puppet $puppetV / git $gitV / openvox $openvoxV" +Write-Host " WindowsUpd : $(if ($winUpdate) { 'ON (full online patch pass)' } else { 'OFF (skipped)' })" +Write-Host " Output : $(if ($isoEnabled) { "$goldenIso -> $capCont/$capIsoBlob" } else { "$goldenWim -> $capCont/$capBlob" })" +Write-Host " Stages : $($Stages -join ', ')" +Write-Host "===================================================================" + +# --- Auth: SP if creds present; else managed identity if nothing logged in ---- +# azcopy reuses the az CLI identity (scripts set AZCOPY_AUTO_LOGIN_TYPE=AZCLI), so +# az must be logged in. On the build VM (headless) fall back to its managed identity. +# +# IMPORTANT: `az` writes routine diagnostics (incl. the "Please run 'az login'" notice) +# to stderr. Under this script's $ErrorActionPreference='Stop', WinPS 5.1 turns any +# native-command stderr into a TERMINATING NativeCommandError - even when redirected with +# 2>$null - so a harmless "am I logged in?" probe was aborting the whole build. Probe via +# exit code with the preference relaxed, and only hard-fail on a genuine login failure. +function Test-AzLoggedIn { + $prev = $ErrorActionPreference + $ErrorActionPreference = 'SilentlyContinue' + try { + az account show 1>$null 2>$null + return ($LASTEXITCODE -eq 0) + } + finally { $ErrorActionPreference = $prev } +} + +# Does a blob exist? Same stderr-under-Stop caveat as Test-AzLoggedIn (az writes +# diagnostics to stderr, which WinPS 5.1 turns into a terminating NativeCommandError +# under $ErrorActionPreference='Stop'), so probe with the preference relaxed. +function Test-BlobExists { + param([string]$Acct, [string]$Container, [string]$Name) + $prev = $ErrorActionPreference + $ErrorActionPreference = 'SilentlyContinue' + try { + $r = az storage blob exists --account-name $Acct --container-name $Container --name $Name --auth-mode login --query exists -o tsv 2>$null + return ("$r".Trim() -eq 'true') + } + finally { $ErrorActionPreference = $prev } +} + +if ($env:AZ_CLIENT_ID -and $env:AZ_CLIENT_SECRET -and $env:AZ_TENANT) { + Write-Host '== az login (service principal) ==' + $ErrorActionPreference = 'Continue' + az login --service-principal -u $env:AZ_CLIENT_ID -p $env:AZ_CLIENT_SECRET --tenant $env:AZ_TENANT --only-show-errors | Out-Null + $ErrorActionPreference = 'Stop' + if (-not (Test-AzLoggedIn)) { throw 'az login (service principal) failed - no active account.' } +} +elseif (-not (Test-AzLoggedIn)) { + # The build VM has a USER-assigned identity (no system-assigned), so bare + # `az login --identity` fails - the UAMI client id must be passed explicitly + # (az CLI >= 2.88 uses --client-id, not --username). Retry: on a freshly created + # VM the identity/IMDS token endpoint can lag a bit behind first boot, and errors + # are surfaced (not hidden) so a real failure is diagnosable in the build log. + if (-not $IdentityClientId) { + throw 'No active az session and no -IdentityClientId supplied; cannot authenticate on the build VM.' + } + Write-Host "== az login (user-assigned managed identity $IdentityClientId) ==" + $loggedIn = $false + for ($attempt = 1; $attempt -le 6; $attempt++) { + $ErrorActionPreference = 'Continue' + $out = az login --identity --client-id $IdentityClientId 2>&1 + $rc = $LASTEXITCODE + $ErrorActionPreference = 'Stop' + if ($rc -eq 0 -and (Test-AzLoggedIn)) { $loggedIn = $true; break } + Write-Warning "az login --identity attempt $attempt/6 failed (rc=$rc): $($out -join ' ')" + Start-Sleep -Seconds 10 + } + if (-not $loggedIn) { throw "az login (managed identity $IdentityClientId) failed after 6 attempts." } + Write-Host '== az login OK ==' +} + +$ps = { param($f, $a) & powershell -NoProfile -ExecutionPolicy Bypass -File (Join-Path $ScriptDir $f) @a; if ($LASTEXITCODE) { throw "$f failed rc=$LASTEXITCODE" } } + +# --- Stage: prep -------------------------------------------------------------- +if ($Stages -contains 'prep') { + Write-Host "`n### prep ########################################################" + # Base WIM: download it if present. Otherwise, if the config names a fallback base.iso, + # extract sources\install.wim from that ISO, save it as the base WIM (naming convention + # -base-install.wim), and cache it back to resources/WIMs/ so later bakes reuse it — so a WIM + # bake can start from just an uploaded ISO. + if (Test-BlobExists $account $baseCont $baseWimBlob) { + & $ps 'download-wim.ps1' @('-Blob', "$baseCont/$baseWimBlob", '-Dest', $localBase, '-Account', $account) + } + elseif ($baseIso) { + Write-Host " $baseCont/$baseWimBlob not present -> extracting it from $baseCont/$baseIsoBlob" + $localSrcIso = Join-Path $work $baseIso + & $ps 'download-wim.ps1' @('-Blob', "$baseCont/$baseIsoBlob", '-Dest', $localSrcIso, '-Account', $account) + & $ps 'extract-wim-from-iso.ps1' @('-SourceIso', $localSrcIso, '-OutWim', $localBase) + Write-Host " caching extracted base WIM back to $baseCont/$baseWimBlob" + & $ps 'upload-wim.ps1' @('-Wim', $localBase, '-Container', $baseCont, '-Account', $account, '-BlobName', $baseWimBlob) + Remove-Item $localSrcIso -Force -ErrorAction SilentlyContinue + } + else { + throw "$baseCont/$baseWimBlob not found and config/$Image.yaml has no base.iso fallback to extract it from." + } + + if (-not $WinRMPassword) { + # Portable random password (avoid the Windows-only System.Web assembly). + $WinRMPassword = 'Aa1!' + [guid]::NewGuid().ToString('N') + [guid]::NewGuid().ToString('N').Substring(0, 12) + Write-Host ' (generated a build-only WinRM password)' + } + if (Test-Path $vhdx) { Remove-Item $vhdx -Force } + $prepArgs = @('-SourceWim', $localBase, '-OutVhdx', $vhdx, '-Edition', $edition, '-WinRMPassword', $WinRMPassword, '-ComputerName', 'nuc-bake') + if ($drvInject) { + Write-Host " driver injection ON -> $($drvCabUrls.Count) cab(s):" + $drvCabUrls | ForEach-Object { Write-Host " $_" } + # Join with '|' into ONE arg: PowerShell -File can't bind a real array param + # (extra space-separated values spill onto positional params). prepare-base-vhdx splits it. + $prepArgs += @('-InjectDrivers', '-DriverCabUrls', ($drvCabUrls -join '|')) + } + if ($extraUrls.Count -gt 0) { + Write-Host " staging $($extraUrls.Count) extra(s) -> C:\extras:" + $extraUrls | ForEach-Object { Write-Host " $_" } + $prepArgs += @('-ExtrasUrls', ($extraUrls -join '|')) + } + & $ps 'prepare-base-vhdx.ps1' $prepArgs + + if (Get-VM -Name $vmName -ErrorAction SilentlyContinue) { Remove-VM -Name $vmName -Force } + & $ps 'register-base-vm.ps1' @('-VmName', $vmName, '-Vhdx', $vhdx, '-SwitchName', $switch, '-Cpus', $cpus, '-MemoryStartupMB', $memMb) +} + +# --- Stage: build ------------------------------------------------------------- +if ($Stages -contains 'build') { + Write-Host "`n### build #######################################################" + if (-not $WinRMPassword) { throw 'build stage needs -WinRMPassword (the one used in prep).' } + if (Test-Path $buildDir) { Remove-Item $buildDir -Recurse -Force } + if (Test-Path $goldenWim) { Remove-Item $goldenWim -Force } + # Packer builds the working VM (+ its RAM-sized memory file) under temp_path. + $pkrTmp = Join-Path $work 'pkrtmp' + New-Item -ItemType Directory -Path $pkrTmp -Force | Out-Null + + # Per-image var-file (gitignored under work/). winrm_password is sensitive. + $varFile = Join-Path $work 'build.pkrvars.hcl' + @" +source_vm_name = "$vmName" +switch_name = "$switch" +winrm_username = "packer" +winrm_password = "$WinRMPassword" +cpus = $cpus +memory_mb = $memMb +ronin_org = "$roninOrg" +ronin_repo = "$roninRepo" +ronin_branch = "$roninBr" +ronin_hash = "$roninHash" +bake_role = "$bakeRole" +puppet_version = "$puppetV" +git_version = "$gitV" +openvox_version = "$openvoxV" +ronin_ext_src = "$extSrc" +github_pat = "$GithubPat" +windows_update = $($winUpdate.ToString().ToLower()) +output_directory = "$($buildDir -replace '\\','/')" +temp_path = "$($pkrTmp -replace '\\','/')" +output_wim = "$($goldenWim -replace '\\','/')" +capture_name = "$Image-$BuildId" +image_name = "$Image" +build_id = "$BuildId" +sbom_path = "$($sbomMd -replace '\\','/')" +"@ | Set-Content -Path $varFile -Encoding utf8 + + # --- Boot watchdog -------------------------------------------------------- + # EVERY guest-initiated reboot on this Gen2 clone lands as a full power-OFF, not a + # soft reboot (nested-virt reboot behavior, observed 2026-07-28 on both the + # post-specialize reboot AND the mid-bake windows-restart provisioners). Packer + # never restarts a VM that powered itself off, so it hangs at "Waiting for WinRM" + # / "Waiting for machine to restart". Run a background watchdog that (re)starts the + # clone whenever it is Off, for the WHOLE build, UNTIL the sysprep provisioner + # announces its intended /shutdown (the 'WIM-WATCHDOG-STOP' marker emitted at the + # top of sysprep-generalize.ps1, streamed by Packer into $pkrLog). After that the + # power-off is expected and Packer captures the VHDX, so the watchdog must NOT + # restart it. + # The watchdog logs every state transition + restart it performs to $wdLog. It runs + # in a background job, so that file is the ONLY record of what it did; on a build + # failure we tail it, which tells us whether a "Timeout waiting for machine to + # restart" was a guest that never came back up (watchdog restarted it, WinRM stayed + # dead) or one the watchdog never touched (VM was Running the whole time). + $cloneVm = 'packer-nuc' # hyperv builder default vm_name = packer- + $pkrLog = Join-Path $work 'packer-build.log' + # Written into run-build-task's dir when it exists (C:\win-hw-wim-build) so that script's + # live uploader can stream it to blob and the GH job can tail it AS THE BUILD RUNS; + # falls back to the work dir for standalone/dev runs where that dir doesn't exist. + $wdDir = 'C:\win-hw-wim-build' + $wdLog = if (Test-Path $wdDir) { Join-Path $wdDir 'boot-watchdog.log' } else { Join-Path $work 'boot-watchdog.log' } + Remove-Item $pkrLog, $wdLog -Force -ErrorAction SilentlyContinue + $watchdog = Start-Job -Name 'wim-boot-watchdog' -ScriptBlock { + param($vm, $log, $wdLog, $guestUser, $guestPass) + function Write-Wd($msg) { + "$([DateTime]::UtcNow.ToString('yyyy-MM-ddTHH:mm:ssZ')) $msg" | + Add-Content -Path $wdLog -Encoding utf8 -ErrorAction SilentlyContinue + } + + # --- Stall capture over PowerShell Direct --------------------------------- + # When Packer stops making progress there is normally NO way to see what the guest + # is doing: its network/WinRM are usually exactly what died (run 31428853582 sat 30 + # min in "Waiting for machine to restart" and we never learned whether Windows was + # applying updates or was simply wedged). PowerShell Direct talks over the Hyper-V + # VMBus, so it needs neither network nor WinRM - it works precisely when the normal + # channels are gone. Dump the guest's recent event log plus a few boot/servicing + # signals into $wdLog, which is streamed to the GH job. + function Get-GuestSnapshot { + # PSAvoidUsingConvertToSecureStringWithPlainText is unavoidable here: PSCredential + # needs a SecureString and this is the build-scoped packer password, which is + # already plaintext in the generated pkrvars file (same pattern as OS-deploy.ps1). + # It never leaves the build VM and dies with it. + $cred = New-Object System.Management.Automation.PSCredential( + $guestUser, (ConvertTo-SecureString $guestPass -AsPlainText -Force)) + Invoke-Command -VMName $vm -Credential $cred -ErrorAction Stop -ScriptBlock { + $os = Get-CimInstance Win32_OperatingSystem + "uptime: booted $($os.LastBootUpTime) (now $(Get-Date))" + $pending = @( + 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Component Based Servicing\RebootPending', + 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update\RebootRequired' + ) | Where-Object { Test-Path $_ } + if ($pending) { "reboot pending: $($pending -join ', ')" } else { 'reboot pending: no' } + 'recent events:' + Get-WinEvent -MaxEvents 25 -ErrorAction SilentlyContinue -FilterHashtable @{ + LogName = 'System', 'Application'; StartTime = (Get-Date).AddMinutes(-20) + } | Sort-Object TimeCreated | ForEach-Object { + ' {0:HH:mm:ss} {1,-11} {2}: {3}' -f $_.TimeCreated, $_.LevelDisplayName, $_.ProviderName, + (($_.Message -split "`r?`n") | Select-Object -First 1) + } + } + } + # PHASE 1 - stay HANDS-OFF until Packer has cloned+configured the VM and started it + # once. StepCloneVM sets the CPU count + secure boot on the freshly-cloned (Off) VM; + # if the watchdog Start-VMs it during that window those Set-VMProcessor/Set-VMFirmware + # calls fail with "cannot be performed while the object is in its current state" and + # the build dies at StepCloneVM (this was the intermittent ~50% clone flake). Wait + # for Packer's own "Starting the virtual machine" (logged at StepRun, AFTER config). + Write-Wd "watchdog started (vm=$vm); phase 1: waiting for Packer to start the VM" + while ($true) { + if (Test-Path $log) { + if (Select-String -Path $log -Pattern 'WIM-WATCHDOG-STOP' -SimpleMatch -Quiet) { + Write-Wd 'sysprep marker seen during phase 1; exiting' + return + } + if (Select-String -Path $log -Pattern 'Starting the virtual machine' -SimpleMatch -Quiet) { break } + } + Start-Sleep -Seconds 3 + } + # PHASE 2 - Packer has started the VM; now (re)start it on any guest-initiated + # power-off until the sysprep provisioner announces its intended /shutdown. + Write-Wd 'phase 2: restarting the VM on any guest-initiated power-off' + $lastState = '' + # Packer logs 'Starting the virtual machine' BEFORE its Start-VM completes, so phase 1 + # can break while the VM is still Off. Without this gate the very first phase-2 poll + # would race Packer's own start: whichever Start-VM lands second dies with "failed to + # change state / the operation cannot be performed while the object is in its current + # state" and the build errors in ~3 min at StepStartVM. Only ever restart a VM we have + # positively seen Running - that is what "guest-initiated power-off" means, and it makes + # the watchdog incapable of fighting Packer's initial start. + $hasRun = $false + # Stall = Packer's log has not grown for this long. 10 min is well clear of the + # normal quiet stretches (a WU pass or an AppX sweep logs nothing for minutes) while + # still firing four times inside a 60m restart_timeout. + $stallSec = 600 + $maxSnapshots = 6 # bounded so a truly dead build can't fill the log + $lastSize = -1 + $lastGrowth = [DateTime]::UtcNow + $snapshots = 0 + while ($true) { + if ((Test-Path $log) -and (Select-String -Path $log -Pattern 'WIM-WATCHDOG-STOP' -SimpleMatch -Quiet)) { + Write-Wd 'sysprep marker seen; hands off from here (the /shutdown is expected)' + break + } + $v = Get-VM -Name $vm -ErrorAction SilentlyContinue + $state = if ($v) { [string]$v.State } else { '' } + if ($state -ne $lastState) { Write-Wd "VM state -> $state"; $lastState = $state } + if ($v -and $v.State -eq 'Running') { $hasRun = $true } + if ($v -and $v.State -eq 'Off') { + if (-not $hasRun) { + Write-Wd 'VM is Off but has never been seen Running - leaving it to Packer (not racing its initial start)' + } + else { + Write-Wd 'VM is Off; issuing Start-VM' + Start-VM -Name $vm -ErrorAction SilentlyContinue + } + } + + $size = if (Test-Path $log) { (Get-Item $log -ErrorAction SilentlyContinue).Length } else { 0 } + if ($size -ne $lastSize) { $lastSize = $size; $lastGrowth = [DateTime]::UtcNow } + elseif ((([DateTime]::UtcNow - $lastGrowth).TotalSeconds -ge $stallSec) -and $v -and $v.State -eq 'Running') { + $lastGrowth = [DateTime]::UtcNow # re-arm regardless of the outcome below + if ($snapshots -ge $maxSnapshots) { + Write-Wd "stalled again; snapshot cap ($maxSnapshots) reached, not capturing further" + } + else { + $snapshots++ + # ${...} around the trailing var: "$maxSnapshots:" parses the ':' as a + # scope/drive separator and fails at parse time. + Write-Wd "no Packer output for $([int]($stallSec / 60)) min and the VM is Running - PowerShell Direct snapshot $snapshots/${maxSnapshots}:" + try { Get-GuestSnapshot | ForEach-Object { Write-Wd " $_" } } + catch { + # Itself diagnostic: 'credential invalid' means the guest is mid-boot + # or the account is gone, not that PS Direct is broken. + Write-Wd " PowerShell Direct snapshot failed: $($_.Exception.Message)" + } + } + } + Start-Sleep -Seconds 6 + } + } -ArgumentList $cloneVm, $pkrLog, $wdLog, 'packer', $WinRMPassword + + Push-Location $Root + try { + # Pass the DIRECTORY (.), not a single file: `packer build foo.pkr.hcl` loads + # only that file and ignores variables.pkr.hcl, so var.* declarations go missing. + & packer init .; if ($LASTEXITCODE) { throw "packer init rc=$LASTEXITCODE" } + # Tee Packer's output to $pkrLog so the watchdog can see the sysprep marker. + # (Tee-Object is a cmdlet, so $LASTEXITCODE still reflects packer's exit code.) + # -on-error=abort leaves the failed VM + dirs in place so the bake (esp. the + # puppet apply) can be inspected / iterated via PowerShell Direct instead of a + # ~40-min rebuild. Successful runs still clean up normally. + # OPTION-1 EXPERIMENT for the StepCloneVM "Set-VMFirmware/Set-VMProcessor ... + # cannot be performed while the object is in its current state" flake. A plain + # in-place retry (re-clone only) was proven futile - every clone on a given host + # failed identically. This retry instead REBUILDS THE SOURCE VM before re-cloning, + # to test whether stale SOURCE state (vs. the host itself) is the cause: if a fresh + # source clears it, great; if register-base-vm's OWN Set-VMFirmware also throws + # "current state", that proves the host is bad and we abandon it (re-dispatch). + $maxTries = 2 + for ($try = 1; $try -le $maxTries; $try++) { + & packer build -on-error=abort -var-file="$varFile" . 2>&1 | Tee-Object -FilePath $pkrLog + if ($LASTEXITCODE -eq 0) { break } + # Match the step that actually FAILED, not any mention of StepCloneVM: Packer + # prints `aborted: skipping cleanup of step "StepCloneVM"` on EVERY failure, + # so a bare 'StepCloneVM' match classified unrelated failures as clone flakes + # and burned a second full build on them (run 31428853582: a post-bake + # windows-restart timeout was retried as a "clone flake", and the retry then + # died on the real host flake). StepEnableIntegrationService is part of the + # same host clone-flake family and IS retryable. + $flakeSteps = 'StepCloneVM', 'StepEnableIntegrationService' + $cloneFlake = $flakeSteps | Where-Object { + Select-String -Path $pkrLog -Pattern "Step `"$_`" failed" -SimpleMatch -Quiet -ErrorAction SilentlyContinue + } | Select-Object -First 1 + if (-not $cloneFlake -or $try -eq $maxTries) { + if (Test-Path $wdLog) { + Write-Host "`n-- boot watchdog log ($wdLog) --" + # 200, not 40: a single PowerShell Direct stall capture is ~30 lines and + # truncating it would defeat the point of taking it. + Get-Content $wdLog -Tail 200 + Write-Host "-- end boot watchdog log --`n" + } + throw "packer build rc=$LASTEXITCODE" + } + Write-Warning "$cloneFlake flake (attempt $try): rebuilding the SOURCE VM, then re-cloning once." + Get-VM -Name $cloneVm -ErrorAction SilentlyContinue | ForEach-Object { + Stop-VM $_ -TurnOff -Force -ErrorAction SilentlyContinue + Remove-VM $_ -Force -ErrorAction SilentlyContinue + } + if (Test-Path $buildDir) { Remove-Item $buildDir -Recurse -Force -ErrorAction SilentlyContinue } + Get-ChildItem $pkrTmp -Force -ErrorAction SilentlyContinue | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue + if (Get-VM -Name $vmName -ErrorAction SilentlyContinue) { Remove-VM -Name $vmName -Force } + & $ps 'register-base-vm.ps1' @('-VmName', $vmName, '-Vhdx', $vhdx, '-SwitchName', $switch, '-Cpus', $cpus, '-MemoryStartupMB', $memMb) + Start-Sleep -Seconds 10 + } + } + finally { + Pop-Location + if ($watchdog) { Stop-Job $watchdog -ErrorAction SilentlyContinue; Remove-Job $watchdog -Force -ErrorAction SilentlyContinue } + } + if (-not (Test-Path $goldenWim)) { throw "build finished but golden WIM missing: $goldenWim" } +} + +# --- Stage: publish ----------------------------------------------------------- +if ($Stages -contains 'publish') { + Write-Host "`n### publish #####################################################" + if (-not (Test-Path $goldenWim)) { throw "no captured WIM to publish at $goldenWim (run build first, or pass the matching -BuildId)." } + & $ps 'upload-wim.ps1' @('-Wim', $goldenWim, '-Container', $capCont, '-Account', $account, '-BlobName', $capBlob) + Write-Host "== Published $capCont/$capBlob ==" + + # Release notes / SBOM alongside the WIM. az (not azcopy) because this is a ~10 KB + # file with no .sha256 sidecar, and az is already logged in as the UAMI - same call + # shape run-build-task.ps1 uses for its _status blobs. + # Best-effort: a missing or unpublishable SBOM must not fail a build that produced a + # good WIM, but it IS reported loudly so it can't rot unnoticed. + if (Test-Path $sbomMd) { + foreach ($dest in @($sbomBlob, $sbomRunBlob)) { + az storage blob upload --account-name $account --container-name $capCont --name $dest --file $sbomMd --overwrite --auth-mode login --only-show-errors -o none + if ($LASTEXITCODE -eq 0) { Write-Host "== Published $capCont/$dest ==" } + else { Write-Warning "release notes upload FAILED (rc=$LASTEXITCODE): $capCont/$dest" } + } + } + else { Write-Warning "no release notes at $sbomMd - the SBOM will be missing for this build" } +} + +# --- Stage: iso (requirement-bypass Win11 ISO) -------------------------------- +# Standalone from prep/build/publish (run with -Stages iso). Downloads the base Win11 +# ISO from resources/ISOs/ (the SOURCE), injects the LabConfig/MoSetup requirement-bypass +# autounattend, and uploads the repackaged bootable ISO (+ .sha256) to +# captured/ISOs//-.iso (same naming as the golden WIMs). NOT a ronin base image. +if ($Stages -contains 'iso') { + Write-Host "`n### iso #########################################################" + $localSrcIso = Join-Path $work $baseIso + & $ps 'download-wim.ps1' @('-Blob', "$baseCont/$baseIsoBlob", '-Dest', $localSrcIso, '-Account', $account) + # oscdimg (ADK Deployment Tools) is needed to repackage a bootable ISO and isn't native; + # pull it from our blob (resources/tools) instead of the MS CDN at build time. + & $ps 'ensure-oscdimg.ps1' @('-Account', $account) + # create-iso runs the config's scripts: (inject-library names) against the media before oscdimg, + # and (if drivers.inject) DISM-injects the config's drivers.cabs into boot.wim + install.wim. + $isoArgs = @('-SourceIso', $localSrcIso, '-OutIso', $goldenIso, '-Label', $isoLabel, '-InjectScripts', ($scripts -join ','), '-Account', $account) + if ($drvInject) { + Write-Host " iso driver injection ON -> $($drvCabUrls.Count) pack(s)" + $isoArgs += @('-DriverZips', ($drvCabUrls -join '|')) + } + & $ps 'create-iso.ps1' $isoArgs + & $ps 'upload-wim.ps1' @('-Wim', $goldenIso, '-Container', $capCont, '-Account', $account, '-BlobName', $capIsoBlob) + Write-Host "== Published $capCont/$capIsoBlob (Win11 ISO; injected: $($scripts -join ', ')) ==" +} + +# --- Cleanup ------------------------------------------------------------------ +if (-not $KeepArtifacts -and ($Stages -contains 'build')) { + Write-Host "`n== Cleanup (VM + VHDX + build dir; pass -KeepArtifacts to retain) ==" + if (Get-VM -Name $vmName -ErrorAction SilentlyContinue) { Remove-VM -Name $vmName -Force } + foreach ($p in @($vhdx, $buildDir)) { if (Test-Path $p) { Remove-Item $p -Recurse -Force -ErrorAction SilentlyContinue } } +} + +Write-Host "`n== DONE: $Image ($BuildId) ==" diff --git a/provisioners/windows/win-hw-wim/config/win-hw-wim-defaults.yaml b/provisioners/windows/win-hw-wim/config/win-hw-wim-defaults.yaml new file mode 100644 index 00000000..ebf0efc8 --- /dev/null +++ b/provisioners/windows/win-hw-wim/config/win-hw-wim-defaults.yaml @@ -0,0 +1,70 @@ +--- +# Shared defaults for the Windows HW baked-WIM pipeline. Any field set to the literal +# string "default" in a per-image config (config/.yaml) resolves to the +# value here. Mirrors worker-images config/windows_production_defaults.yaml. +# +# Source of truth for tool versions is worker-images +# config/windows_production_defaults.yaml — keep these in sync. + +storage: + # Blob layout (folders are prefixes within each container): + # resources/ WIMs/ (BYO base WIMs) ISOs/ (source Win11 ISOs) drivers/ tools/ + # captured/ WIMs/ (/-.wim) ISOs/ (/-.iso) + # legacy-images/ (old, previously-built images) + account: hardwareimaging + base_container: resources # SOURCES: base WIMs (WIMs/), ISOs (ISOs/), drivers/, tools/ + captured_container: captured # OUTPUTS: golden WIMs (WIMs/) + nocheck ISOs (ISOs/) + +ronin: + org: mozilla-platform-ops + repo: ronin_puppet + # Public assets blob with the pinned prerequisite installers (openvox/puppet/git). + ext_src: https://roninpuppetassets.blob.core.windows.net/binaries/prerequisites + +vm: + puppet_version: "8.10.0" + git_version: "2.54.0" + openvox_version: "8.24.2" + # The build host is a dedicated single-purpose nested-virt VM (Standard_D64ads_v5: + # 64 vCPU / 256 GiB), so give the bake guest as much as we can. NOTE: the Packer + # hyperv builder hard-caps memory at 32768 MB (32 GiB) — that's the ceiling here, + # not the host. CPUs aren't capped by the plugin, so take 56 (leave 8 for root). + cpus: 56 + memory_mb: 32768 + # Hyper-V switch on the build host. The bootstrap creates an internal NAT switch + # 'wim-nat' (Windows Server has no client "Default Switch"); the bake guest gets a + # matching static IP via the injected unattend. Override per host if needed. + switch_name: "wim-nat" + # Run a full online Windows Update pass during the bake. Default OFF (fast bakes / + # pipeline iteration); production images opt IN per config (windows_update: true). + # A full pass pulls the latest non-Preview cumulative/SSU/.NET/Defender KBs but is + # slow (large download over the NAT link) and a single pass (see win-hw-wim.pkr.hcl). + windows_update: false + +# Offline driver injection defaults (per-image config may override). OFF by default. +# 'cabs' is a list of driver .cab URLs; per-image configs add entries as needed. +drivers: + inject: false + cabs: [] + +# Payloads copied verbatim into the image at C:\extras\ for the DEPLOY-time puppet apply +# to run. Not drivers: never expanded, never DISM-injected. Unlike C:\bake (which sysprep +# deletes before capture) these SHIP in the golden WIM, so each entry costs its own size in +# every WIM. Off by default; set per image. +extras: + files: [] + +# Windows 11 ISO builder (SEPARATE from the ronin base WIMs). Enabled per-config via iso.enabled. +# Source media is base.iso; provisioning is the config's scripts: list (inject-library scripts run +# against the media before repackaging with oscdimg). e.g. scripts: [nocheck] writes the +# HKLM\SYSTEM\Setup\LabConfig + MoSetup requirement bypass. See scripts/create-iso.ps1 and +# scripts/inject/. The built ISO is a captured OUTPUT: uploaded to +# captured//-.iso (+ .sha256), same naming as the golden WIMs. +iso: + enabled: false # set true in a config to build the ISO instead of the WIM bake + label: "WIN11_NOCHK" # volume label of the built ISO + +tags: + sourceOrganization: mozilla-platform-ops + sourceRepository: ronin_puppet + managed_by: wim-packer diff --git a/provisioners/windows/win-hw-wim/config/win11-24h2-hw-test.yaml b/provisioners/windows/win-hw-wim/config/win11-24h2-hw-test.yaml new file mode 100644 index 00000000..9295a52f --- /dev/null +++ b/provisioners/windows/win-hw-wim/config/win11-24h2-hw-test.yaml @@ -0,0 +1,67 @@ +--- +# TEMP / TESTING variant of win11-24h2-hw — a throwaway image for validating the +# pipeline and bake changes WITHOUT touching the production win11-24h2-hw output. +# +# Build: .\bin\WinHwWim\New-WinHwWim.ps1 -Image win11-24h2-hw-test +# +# Same base WIM + edition + bake role as win11-24h2-hw; differs only in identity +# so it captures to its own namespace (captured/win11-24h2-hw-test/…) and tracks +# the bake-role branch HEAD for fast iteration. Delete this file when done testing. + +base: + wim: win11-24h2-base-install.wim + edition: "Windows 11 Enterprise" + +# NUC13 (Arena Canyon / Raptor Lake) driver pack injected offline into the base +# image before capture, so the golden WIM ships with real Intel drivers (iGPU +# A7A0, Smart Sound audio, serial-IO/SMBus, DPTF, ISH/MEI, Bluetooth, NIC/GNA). +# Without this the DISM-applied node falls back to the Microsoft Basic Display +# Adapter (RELOPS-2487). Public blob mirror (anonymous read, same host as the +# prereqs) so the bake's Invoke-WebRequest needs no auth. Each pack (.cab or .zip) +# must expand to an .inf tree. Source: MDT Out-of-Box Drivers for NUC13. +drivers: + inject: true + # One or more driver-pack URLs, each a .cab OR a .zip (scalable — add more list + # entries to inject extra packs; every pack is downloaded, expanded, and applied in + # a single recursive DISM /Add-Driver before capture). + cabs: + # Network drivers (Intel PROSet/NIC) - public roninpuppetassets mirror (Invoke-WebRequest). + - "https://roninpuppetassets.blob.core.windows.net/binaries/drivers/nuc13/nuc13-24h2-nuc_driver.zip" + # Intel graphics (Raptor Lake-P Iris Xe DEV_A7A0), DCH 32.0.101.7085 from the MS Update Catalog. + # Hosted in hardwareimaging (Entra-secured, no anon) so the bake pulls it via azcopy AAD. Proven on 160: + # flips MS Basic Display Adapter (RR=1) -> Intel(R) Iris(R) Xe Graphics (RR=60). + - "https://hardwareimaging.blob.core.windows.net/resources/drivers/nuc13-24h2-intel-gfx-32.0.101.7085.cab" + # Platform/chipset packs - kept in step with win11-24h2-hw.yaml so a test bake has the same + # driver coverage. See that file for the full rationale (exported off the MDT reference nodes + # nuc13-006 / t-nuc12-005, DCH graphics family pruned so it cannot fight the 7085 cab above). + - "https://hardwareimaging.blob.core.windows.net/resources/drivers/nuc13-24h2-platform-drivers-20260827.zip" + - "https://hardwareimaging.blob.core.windows.net/resources/drivers/nuc12-24h2-platform-drivers-20260827.zip" + +ronin: + org: default + repo: default + branch: wim-bake-role # bake role branch + hash: "" # testing: track branch HEAD (no pin) so iterations don't need a bump + bake_role: win116424h2hwbake + +vm: + puppet_version: default + git_version: default + openvox_version: default + cpus: default + memory_mb: default + switch_name: default + # Testing: skip the slow full Windows Update pass for fast iteration (also the + # shared default). Flip to true to validate the production WU path. + windows_update: false + +# Deploy-side metadata — test pool label so this never looks like production. +sharedimage: + worker_pool_id: win11-64-24h2-hw-test + +tags: + base_image: win11-24h2-base-install.wim + sourceOrganization: default + sourceRepository: default + sourceBranch: wim-bake-role + temp: "true" diff --git a/provisioners/windows/win-hw-wim/config/win11-24h2-hw.yaml b/provisioners/windows/win-hw-wim/config/win11-24h2-hw.yaml new file mode 100644 index 00000000..038fe0fc --- /dev/null +++ b/provisioners/windows/win-hw-wim/config/win11-24h2-hw.yaml @@ -0,0 +1,145 @@ +--- +# Baked Windows HW install.wim: Windows 11 24H2, generic hardware (hw) bake. +# +# Build: .\bin\WinHwWim\New-WinHwWim.ps1 -Image win11-24h2-hw +# +# Naming is scalable across two dimensions — OS version AND specialized worker +# pool. The image id is this file's basename (win11-24h2-hw). Add a new WIM by +# dropping in another config/.yaml, e.g.: +# - win11-25h2-hw.yaml (new OS version; new base_wim) +# - win11-24h2-perf.yaml (same base_wim, specialized bake_role/pool) +# - win2022-hw.yaml (new OS) +# Several variants can share one base_wim but differ in bake_role/worker_pool_id. + +base: + # Blob in resources/WIMs/. Convention: -base-install.wim + wim: win11-24h2-base-install.wim + # Edition NAME inside the WIM; prepare-base-vhdx resolves it to the image index + # via Get-WindowsImage (a multi-edition install.wim has several indexes). + # Verify the exact name with: dism /Get-WimInfo /WimFile: + edition: "Windows 11 Enterprise" + +# Offline driver injection into the applied base image (DISM /Add-Driver before +# capture, so drivers land in the golden WIM). NUC13 (Arena Canyon / Raptor Lake) +# pack: without it the DISM-applied node has no Intel drivers (iGPU A7A0, Smart +# Sound audio, serial-IO GPIO/I2C/SMBus, MEI, Bluetooth, NIC/GNA) and falls back +# to the Microsoft Basic Display Adapter (RELOPS-2487). Each URL must be publicly +# reachable (the bake downloads it with an unauthenticated Invoke-WebRequest) and be +# a .cab OR .zip that expands to an .inf tree. Source: MDT Out-of-Box Drivers. +drivers: + inject: true + # One or more driver-pack URLs, each a .cab OR a .zip (scalable — add more list + # entries to inject extra packs; every pack is downloaded, expanded, and applied in + # a single recursive DISM /Add-Driver before capture). + cabs: + # Network drivers (Intel PROSet/NIC) - public roninpuppetassets mirror (Invoke-WebRequest). + - "https://roninpuppetassets.blob.core.windows.net/binaries/drivers/nuc13/nuc13-24h2-nuc_driver.zip" + # Intel graphics (Raptor Lake-P Iris Xe DEV_A7A0), DCH 32.0.101.7085 from the MS Update Catalog. + # Hosted in hardwareimaging (Entra-secured, no anon) so the bake pulls it via azcopy AAD. Proven on 160: + # flips MS Basic Display Adapter (RR=1) -> Intel(R) Iris(R) Xe Graphics (RR=60). + - "https://hardwareimaging.blob.core.windows.net/resources/drivers/nuc13-24h2-intel-gfx-32.0.101.7085.cab" + # Platform/chipset packs (added 2026-08-27, RELOPS-2487). Exported with + # `Export-WindowsDriver -Online` off the MDT reference nodes - nuc13-006 (NUC13 / Raptor + # Lake-P) and t-nuc12-005 (NUC12 / Alder Lake-P) - i.e. the exact driver set production runs. + # + # Measured gap they close: a baked node has 10 devices in error 28 (CM_PROB_FAILED_INSTALL) + # that the MDT image binds cleanly - Serial IO GPIO INTC1055 + I2C 51E8/51E9 + # (iaLPSS2_*_ADL), SMBus 51A3 + SPI 51A4 (AlderLakePCH-PSystem), Smart Sound audio + # 51CA/51C8 (IntcAudioBus/intcsst - a baked node's ONLY sound device is the Virtual Audio + # Cable), MEI 51E0 (heci), GNA A74F/464F, Wi-Fi 51F1/51F0 (Netwtw6e), Bluetooth PID_0033 + # (ibtusb). NOTE: INTC1055 is Serial IO GPIO, NOT DPTF - Intel Dynamic Tuning is absent from + # the MDT image too, so it is not a baked-vs-production difference. + # + # PRUNED before upload: the DCH graphics family (iigd_dch, iigd_ext, hdbusext, cui_dch, + # igcc_dch, mshdadac). The cab above already supplies a COMPLETE and NEWER matched + # 32.0.101.7085 set (Display + Extension + MEDIA + 2 SoftwareComponent), while the reference + # nodes carry 32.0.101.7079 (NUC13) / 31.0.101.3729 (NUC12); mixing DCH graphics component + # versions against a 7085 base buys nothing and is what Intel warns against. + # + # BOTH packs are injected because ONE WIM serves BOTH canary pools (perf-debug = NUC13, + # ref-alpha = NUC12). Overlap is safe: DISM stores every version and PnP binds the best + # match per device (they share the Alder Lake-P PCH driver family, so most INFs cover both). + - "https://hardwareimaging.blob.core.windows.net/resources/drivers/nuc13-24h2-platform-drivers-20260827.zip" + - "https://hardwareimaging.blob.core.windows.net/resources/drivers/nuc12-24h2-platform-drivers-20260827.zip" + +# Payloads copied VERBATIM into the image at C:\extras\ - not expanded, not DISM-injected, +# not run here. They SHIP in the golden WIM (~+740 MB for the entry below) and the +# DEPLOY-time puppet apply is what runs them. +extras: + files: + # Intel's full graphics installer. Supplies IntelGraphicsSoftwareService, which the + # MDT production image has and a baked node did not: the service ships inside the + # AppUp.IntelArcSoftware MSIX at Resources/Extras/IntelGraphicsSoftware_26.18.2353.2_Release.exe, + # and under DCH that MSIX arrives via Windows Update as a driver companion app - which + # never happens here because WU is disabled by design, so the driver cab above (INF + # only) cannot supply it. ronin's win_intel_graphics_software runs this WITHOUT + # --noExtras, the flag that would skip exactly that folder. + # + # It is staged rather than installed at bake because installing at bake CANNOT work: + # the installer returns rc=1008 in the GPU-less Hyper-V build guest (rc=1001 + service + # Running on real NUC13 hardware), and the MSIX it lays down is per-user, which + # sysprep /generalize strips from the WIM. So the WIM carries the installer and the + # node installs it on first puppet run. + - "https://hardwareimaging.blob.core.windows.net/resources/drivers/gfx_win_101.7088.exe" + +ronin: + org: default + repo: default + branch: wim-bake-role # feature branch carrying the bake role (until merged) + hash: "a312287f" # ronin_puppet#1297 HEAD; bake role excludes hardware_observability (no marlin_pw lookup). Bump when the branch moves. + # a312287f SUPERSEDES 4c2e223d: the bake role no longer includes + # intel_graphics_software (rc=1008 in the GPU-less build guest, and the MSIX is a + # per-user install sysprep would strip anyway). The installer is staged to C:\extras + # by extras.files below and run at DEPLOY time instead. Pin BOTH this and the + # pools.yml hash to a312287f or the deployed node looks for the old C:\bake\extras. + # 4c2e223d SUPERSEDED 7dac7f0a. Two reasons: (a) wim-bake-role was rebased onto + # master on 2026-08-26 and 7dac7f0a no longer exists on the branch - Get-Ronin does + # a --single-branch clone, which carries no unreachable objects, so the checkout + # would fail outright; (b) it predates the two things this bake is FOR - VBS/HVCI + # (win_device_guard, so baked nodes match the MDT image's Credential Guard + HVCI) + # and Intel Graphics Software (win_intel_graphics_software, run at DEPLOY time from + # the installer that extras.files stages into the WIM at C:\extras). + # HISTORY: 7dac7f0a (RELOPS-2487) SUPERSEDED b64eb0b5. b64eb0b5 stopped the bake disabling + # AppXSvc so the image shipped it Manual; that fixed the codecs but changed sysprep - + # the WIM then got a full generalize, and the first-boot specialize pass regenerated a + # random WIN-xxxxxxxx into ActiveComputerName, which sent maintainsystem-hw into a + # Set-PXE re-image loop. The bake never needed changing: the media extensions are + # PROVISIONED at bake time (DISM-level, no AppXSvc), and only the per-user registration + # at first task_* logon needs the service. So 7dac7f0a bakes the disable again and + # re-enables AppXSvc at DEPLOY time for ref/ref-alpha instead. Stage at bake, register + # at deploy. + bake_role: win116424h2hwbake # trimmed hw role that performs the one-time AppX/catalog bake + +vm: + puppet_version: default + git_version: default + openvox_version: default + cpus: default + memory_mb: default + switch_name: default + # OFF deliberately (was true; flipped 2026-08-11 after bake run 31428853582). + # Patching at bake time fights the bake role: win116424h2hwbake includes + # roles_profiles::profiles::disable_services -> win_disable_services::disable_windows_update, + # which disables wuauserv. So the WU-on order was: install updates -> reboot -> + # puppet disables the update stack -> reboot, and that last reboot never came back + # ("A system shutdown is in progress.(1115)" -> "Timeout waiting for machine to + # restart"), losing a 1h41m build. hw-test bakes the SAME role from the SAME branch + # with WU off and has gone green repeatedly - windows_update was the only difference. + # Consequence, accepted: the WIM ships at the patch level of base_wim, and the + # deployed workers never patch themselves (that same puppet class keeps WU disabled + # in production), so refreshing patches means uploading newer base media. Also saves + # ~40 min per bake and makes the image content deterministic instead of build-date + # dependent. If bake-time patching is ever wanted back, do it OFFLINE - a pinned + # SSU+LCU .msu via DISM /Add-Package in prepare-base-vhdx.ps1 (which already does + # offline driver injection) - not with the online WU provisioner. + windows_update: false + +# Deploy-side metadata (informational; used for output tagging / provenance). +sharedimage: + worker_pool_id: win11-64-24h2-hw + +tags: + base_image: win11-24h2-base-install.wim + sourceOrganization: default + sourceRepository: default + sourceBranch: wim-bake-role diff --git a/provisioners/windows/win-hw-wim/config/win11-25h2-hw.yaml b/provisioners/windows/win-hw-wim/config/win11-25h2-hw.yaml new file mode 100644 index 00000000..70ddb4ae --- /dev/null +++ b/provisioners/windows/win-hw-wim/config/win11-25h2-hw.yaml @@ -0,0 +1,63 @@ +--- +# Baked Windows HW install.wim: Windows 11 25H2, generic hardware (hw) bake. +# +# Build: .\bin\WinHwWim\New-WinHwWim.ps1 -Image win11-25h2-hw +# +# POC note: this reuses the NUC13 driver packs and the 24h2 bake role — the hardware +# (NUC13 / Raptor Lake) is unchanged, only the OS media moves to 25H2. Split the role +# out to a 25h2-specific one if/when the bake diverges. + +base: + # Blob in resources/WIMs/. Convention: -base-install.wim + wim: win11-25h2-base-install.wim + # Fallback SOURCE: if base.wim is not present in resources/WIMs/, prep extracts + # sources\install.wim from this ISO, saves it as base.wim (above), and caches it back + # to base/ so later bakes reuse it. Lets a WIM bake start from just an uploaded ISO. + iso: "Win11_25H2_English_x64_v2.iso" + # Edition NAME inside the WIM; prepare-base-vhdx resolves it to the image index + # via Get-WindowsImage (a multi-edition install.wim has several indexes). + # Verify the exact name with: dism /Get-WimInfo /WimFile: + edition: "Windows 11 Enterprise" + +# Offline driver injection into the applied base image (DISM /Add-Driver before +# capture, so drivers land in the golden WIM). NUC13 (Arena Canyon / Raptor Lake) +# pack: without it the DISM-applied node has no Intel drivers (iGPU A7A0, Smart +# Sound audio, serial-IO/SMBus, DPTF, ISH/MEI, Bluetooth, NIC/GNA) and falls back +# to the Microsoft Basic Display Adapter (RELOPS-2487). Each URL must be publicly +# reachable (the bake downloads it with an unauthenticated Invoke-WebRequest) and be +# a .cab OR .zip that expands to an .inf tree. Source: MDT Out-of-Box Drivers. +drivers: + inject: true + cabs: + # Network drivers (Intel PROSet/NIC) - public roninpuppetassets mirror (Invoke-WebRequest). + - "https://roninpuppetassets.blob.core.windows.net/binaries/drivers/nuc13/nuc13-24h2-nuc_driver.zip" + # Intel graphics (Raptor Lake-P Iris Xe DEV_A7A0), DCH 32.0.101.7085 from the MS Update Catalog. + # Hosted in hardwareimaging (Entra-secured, no anon) so the bake pulls it via azcopy AAD. + - "https://hardwareimaging.blob.core.windows.net/resources/drivers/nuc13-24h2-intel-gfx-32.0.101.7085.cab" + +ronin: + org: default + repo: default + branch: wim-bake-role # feature branch carrying the bake role (until merged) + hash: "4463ab7" # short SHA (matches deploymentId convention); ronin_puppet#1297 HEAD. Bump when the branch moves. + bake_role: win116424h2hwbake # trimmed hw role that performs the one-time AppX/catalog bake + +vm: + puppet_version: default + git_version: default + openvox_version: default + cpus: default + memory_mb: default + switch_name: default + # Production image: fully patch at bake time (shared default is OFF). + windows_update: true + +# Deploy-side metadata (informational; used for output tagging / provenance). +sharedimage: + worker_pool_id: win11-64-25h2-hw + +tags: + base_image: win11-25h2-base-install.wim + sourceOrganization: default + sourceRepository: default + sourceBranch: wim-bake-role diff --git a/provisioners/windows/win-hw-wim/config/win11-25h2-iso.yaml b/provisioners/windows/win-hw-wim/config/win11-25h2-iso.yaml new file mode 100644 index 00000000..c37e3cef --- /dev/null +++ b/provisioners/windows/win-hw-wim/config/win11-25h2-iso.yaml @@ -0,0 +1,29 @@ +--- +# Windows 11 25H2 requirement-bypass ISO — a DISTINCT function from the ronin base WIM +# bakes (config/win11-*-hw*.yaml). Build: New-WinHwWim -Image win11-25h2-iso +# +# Config shape mirrors the WIM configs (base: + a provisioning source), but the provisioning +# source is `scripts:` (inject-library scripts) instead of `ronin:` (clone+apply ronin_puppet). +# A config uses EITHER ronin OR scripts, not both. + +base: + # Source media: the base Win11 25H2 ISO uploaded to resources/ISOs/ (yaml value). + # This is the latest retail media (Win11_25H2_English_x64_v2) — see WORKLOG. + iso: "Win11_25H2_English_x64_v2.iso" + +# Provisioning = inject-library scripts (scripts/inject/.ps1) run against the extracted +# media before repackaging. Each name is a reusable script; add more by dropping one in + naming it. +scripts: + - nocheck # bypass TPM/SecureBoot/RAM/CPU/storage checks (LabConfig + MoSetup autounattend) + +# Offline driver injection into the ISO's boot.wim (Windows Setup sees storage/NIC during +# install) + install.wim (every edition). HPE ProLiant DL360 Gen10 Windows driver pack, +# extracted from the HPE SPP Smart Components (101 INF drivers) and hosted in our blob. +drivers: + inject: true + cabs: + - "https://hardwareimaging.blob.core.windows.net/resources/drivers/hpe-proliant-dl360-gen10-win-drivers.zip" + +iso: + enabled: true # build the ISO (iso stage) instead of the WIM bake + label: "WIN11_25H2_NOCHK" # volume label; output -> captured//-.iso diff --git a/provisioners/windows/win-hw-wim/example.pkrvars.hcl b/provisioners/windows/win-hw-wim/example.pkrvars.hcl new file mode 100644 index 00000000..757cd47d --- /dev/null +++ b/provisioners/windows/win-hw-wim/example.pkrvars.hcl @@ -0,0 +1,29 @@ +# REFERENCE ONLY for a manual `packer build -var-file=...` run. +# Normal builds go through bin/WinHwWim/New-WinHwWim.ps1, which reads config/.yaml +# and generates work//build.pkrvars.hcl for you — you don't edit this file. +# Do NOT commit real values (.gitignore ignores *.pkrvars.hcl except this example). + +source_vm_name = "win-hw-wim-base" # created by register-base-vm.ps1 +switch_name = "Default Switch" # or your external switch with internet + +winrm_username = "packer" +winrm_password = "CHANGE-ME-build-only" # build-scoped; scrubbed before capture + +cpus = 4 +memory_mb = 8192 + +# ronin bake source — use the FEATURE branch carrying win116424h2hwbake (not main) +ronin_org = "mozilla-platform-ops" +ronin_repo = "ronin_puppet" +ronin_branch = "wim-bake-role" +ronin_hash = "" # optional pinned commit +bake_role = "win116424h2hwbake" + +# Pinned to worker-images config/windows_production_defaults.yaml (source of truth). +# openvox_version is set, so Get-PreRequ installs openvox-agent--x64.msi and +# ignores puppet_version for the installer (kept for reference/parity). +puppet_version = "8.10.0" +git_version = "2.54.0" +openvox_version = "8.24.2" + +output_directory = "./output/build" diff --git a/provisioners/windows/win-hw-wim/scripts/bake-bootstrap.ps1 b/provisioners/windows/win-hw-wim/scripts/bake-bootstrap.ps1 new file mode 100644 index 00000000..a750ecc5 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/bake-bootstrap.ps1 @@ -0,0 +1,348 @@ +<# +.SYNOPSIS + Step 3 (runs INSIDE the build VM via Packer): perform the ronin "bake". + +.DESCRIPTION + Reproduces the STABLE half of ronin's bootstrap.ps1: disable Windows Update / + Store auto-update, install Git + Puppet/OpenVox, clone ronin at the pinned + branch/hash, seed a BAKE registry identity, write a placeholder bake vault.yaml + (only secrets referenced by BAKED profiles; no worker-registration secrets), + generate nodes.pp for the bake role, and run `puppet apply`. + + The puppet run applies the bake role, which (via disable_services) performs the + AppX removal ONCE here at bake time instead of on every NUC deploy. + + Inputs come from environment variables set by win-hw-wim.pkr.hcl: + RONIN_ORG RONIN_REPO RONIN_BRANCH RONIN_HASH BAKE_ROLE + PUPPET_VERSION GIT_VERSION OPENVOX_VERSION + + Exit code 2 from `puppet apply` (changes applied) is success; the Packer + provisioner is configured with valid_exit_codes = [0, 2]. +#> +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$org = $env:RONIN_ORG +$repo = $env:RONIN_REPO +$branch = $env:RONIN_BRANCH +$hash = $env:RONIN_HASH +$role = if ($env:BAKE_ROLE) { $env:BAKE_ROLE } else { 'win116424h2hwbake' } +$log = 'C:\bake\logs' +$roninDir = 'C:\ronin' +New-Item -ItemType Directory -Path $log -Force | Out-Null +Start-Transcript -Path (Join-Path $log 'bake-bootstrap.log') -Append | Out-Null + +function Step($m) { Write-Host "== $m ==" } + +# --- 1. Stop Windows Update / Store fighting the AppX removal (root cause fix) --- +Step 'Disabling Windows Update + Store auto-update for the bake' +$wu = 'HKLM:\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate\AU' +New-Item -Path $wu -Force | Out-Null +Set-ItemProperty -Path $wu -Name NoAutoUpdate -Value 1 -Type DWord +$store = 'HKLM:\SOFTWARE\Policies\Microsoft\WindowsStore' +New-Item -Path $store -Force | Out-Null +Set-ItemProperty -Path $store -Name AutoDownload -Value 2 -Type DWord +foreach ($svc in 'wuauserv','UsoSvc') { + Stop-Service $svc -Force -ErrorAction SilentlyContinue + Set-Service $svc -StartupType Disabled -ErrorAction SilentlyContinue +} + +# --- 2. Install Git + Puppet/OpenVox (versions pinned to the hw pool) --- +# Prerequisite installers come from ronin's public assets blob under +# /binaries/prerequisites — the SAME source worker-images MDC1Windows/bootstrap.ps1 +# uses (Get-PreRequ). Versions are pinned in the pkrvars to match +# worker-images config/windows_production_defaults.yaml. +$extSrc = if ($env:RONIN_EXT_SRC) { $env:RONIN_EXT_SRC } else { 'https://roninpuppetassets.blob.core.windows.net/binaries/prerequisites' } +$dlDir = 'C:\bake\prereq' +New-Item -ItemType Directory -Path $dlDir -Force | Out-Null + +function Get-PrereqFile { + param([string[]]$Urls, [string]$OutFile) + foreach ($u in $Urls) { + try { + Write-Host " downloading $u" + Invoke-WebRequest -Uri $u -OutFile $OutFile -UseBasicParsing + if (Test-Path $OutFile) { return } + } catch { Write-Warning " failed $u : $($_.Exception.Message)" } + } + throw "could not download any of: $($Urls -join ', ')" +} + +# Puppet vs OpenVox: mirror Get-PreRequ — if OPENVOX_VERSION is set it wins. +if ($env:OPENVOX_VERSION) { + $agentMsi = "openvox-agent-$($env:OPENVOX_VERSION)-x64.msi" +} else { + $agentMsi = "puppet-agent-$($env:PUPPET_VERSION)-x64.msi" +} +$gitExe = "Git-$($env:GIT_VERSION)-64-bit.exe" + +Step "Installing agent ($agentMsi) and Git ($gitExe) from $extSrc" + +# Puppet/OpenVox agent (MSI) — assets blob only. +$agentPath = Join-Path $dlDir $agentMsi +Get-PrereqFile -Urls @("$extSrc/$agentMsi") -OutFile $agentPath +$p = Start-Process msiexec.exe -ArgumentList "/i `"$agentPath`" /qn /norestart" -Wait -PassThru +if ($p.ExitCode -ne 0 -and $p.ExitCode -ne 3010) { throw "agent MSI install failed rc=$($p.ExitCode)" } + +# Git — prefer the assets-blob mirror, fall back to the git-for-windows upstream +# (upstream needs no PAT for a public release asset). +$gitPath = Join-Path $dlDir $gitExe +$gitUpstream = "https://github.com/git-for-windows/git/releases/download/v$($env:GIT_VERSION).windows.1/$gitExe" +Get-PrereqFile -Urls @("$extSrc/$gitExe", $gitUpstream) -OutFile $gitPath +$p = Start-Process $gitPath -ArgumentList '/VERYSILENT /NORESTART /SUPPRESSMSGBOXES /NOCANCEL' -Wait -PassThru +if ($p.ExitCode -ne 0) { throw "Git install failed rc=$($p.ExitCode)" } + +# Refresh PATH in this session so `puppet` / `git` resolve for the steps below. +$env:Path = [Environment]::GetEnvironmentVariable('Path','Machine') + ';' + [Environment]::GetEnvironmentVariable('Path','User') +foreach ($extra in 'C:\Program Files\Puppet Labs\Puppet\bin','C:\Program Files\OpenVox\Puppet\bin','C:\Program Files\Git\cmd') { + if ((Test-Path $extra) -and ($env:Path -notlike "*$extra*")) { $env:Path += ";$extra" } +} +if (-not (Get-Command puppet -ErrorAction SilentlyContinue)) { throw 'puppet not on PATH after install' } +if (-not (Get-Command git -ErrorAction SilentlyContinue)) { throw 'git not on PATH after install' } + +# --- 3. Clone ronin at the pinned branch/hash --- +Step "Cloning $org/$repo@$branch" +if (Test-Path $roninDir) { Remove-Item $roninDir -Recurse -Force } +& git clone --single-branch --branch $branch "https://github.com/$org/$repo.git" $roninDir +if ($LASTEXITCODE -ne 0) { throw "git clone failed rc=$LASTEXITCODE" } +if ($hash) { + Push-Location $roninDir; & git checkout $hash; if ($LASTEXITCODE -ne 0) { Pop-Location; throw "checkout $hash failed" }; Pop-Location +} +& git config --global --add safe.directory $roninDir + +# --- 4. Seed BAKE registry identity (generic, no pool/worker secrets) --- +Step 'Seeding bake registry identity' +$ron = 'HKLM:\SOFTWARE\Mozilla\ronin_puppet' +New-Item -Path $ron -Force | Out-Null +Set-ItemProperty -Path $ron -Name role -Value $role -Type String +Set-ItemProperty -Path $ron -Name workerType -Value $role -Type String # drives win_hiera lookup +Set-ItemProperty -Path $ron -Name worker_pool_id -Value 'bake' -Type String +Set-ItemProperty -Path $ron -Name image_provisioner -Value 'wim-packer' -Type String +Set-ItemProperty -Path $ron -Name bootstrap_stage -Value 'inprogress' -Type String + +# --- 5. Placeholder bake vault.yaml (secret-free) --- +# The bake role references NO Vault secrets (windows_worker_runner, +# hardware_observability, and windows_datacenter_administrator are all excluded — see +# ronin roles/win116424h2hwbake.pp), so this is an empty placeholder that just satisfies +# hiera's secrets/vault.yaml level. It contains no secrets and is scrubbed by +# sysprep-generalize.ps1 before capture. MUST be written WITHOUT a UTF-8 BOM — WinPS 5.1 +# `Set-Content -Encoding utf8` emits a BOM, which the YAML parser rejects. +Step 'Writing placeholder bake vault.yaml' +$secretsDir = Join-Path $roninDir 'data\secrets' +New-Item -ItemType Directory -Path $secretsDir -Force | Out-Null +$utf8NoBom = New-Object System.Text.UTF8Encoding($false) +[System.IO.File]::WriteAllText((Join-Path $secretsDir 'vault.yaml'), + "---`n# bake placeholder - no secrets; scrubbed before capture`n", $utf8NoBom) + +# --- 6. Generate nodes.pp for the bake role --- +# WITHOUT a BOM: puppet's parser rejects a leading UTF-8 BOM ("Illegal UTF-8 Byte Order +# mark"), which is exactly what `Set-Content -Encoding utf8` produces on WinPS 5.1. +Step "Generating nodes.pp -> roles::$role" +$manifestDir = Join-Path $roninDir 'manifests\nodes' +New-Item -ItemType Directory -Path $manifestDir -Force | Out-Null +[System.IO.File]::WriteAllText((Join-Path $roninDir 'manifests\nodes.pp'), + "node default {`n include roles_profiles::roles::$role`n}`n", $utf8NoBom) + +# --- 7. puppet apply AS SYSTEM (this is where the AppX removal + stable catalog bake) --- +# Puppet MUST run as NT AUTHORITY\SYSTEM. The bake role disables protected services +# (e.g. NgcCtnrSvc / Microsoft Passport Container) whose SCM handle an ordinary admin +# cannot open for write ("Access is denied", sc.exe rc=5) — only SYSTEM/TrustedInstaller +# can. Packer's WinRM session runs as the local 'packer' admin, so we relaunch the apply +# under a SYSTEM scheduled task, mirroring ronin .kitchen/provision_windows.ps1 +# (Invoke-AsSystem) and production maintainsystem.ps1 (which runs puppet as SYSTEM). +# +# Use hiera.yaml (the role-aware config: has the roles/%{facts.custom_win_role}.yaml +# level that loads data/roles/.yaml). win_hiera.yaml has NO roles/ level, so the +# role's win-worker.* data would not resolve. +Step 'Running puppet apply (bake catalog) as SYSTEM' +$puppetLog = Join-Path $log 'bake-puppet.log' +$sysScript = 'C:\bake\run-puppet-system.ps1' +$exitFile = 'C:\bake\bake-puppet.exitcode' +Remove-Item $puppetLog, $exitFile -ErrorAction SilentlyContinue + +# Child script executed by the SYSTEM task. Written BOM-less. A fresh SYSTEM process does +# not inherit this WinRM session's env, so it re-resolves PATH from the machine env, +# forwards the build-scoped GitHub token to the catalog (tooltool), and sets the role fact +# explicitly. Puppet writes to console; Tee-Object captures it to the shared log the parent +# tails (so output still streams into the packer/build log even if the VM is cleaned up). +$child = @" +`$ErrorActionPreference = 'Continue' +`$env:Path = [Environment]::GetEnvironmentVariable('Path','Machine') +foreach (`$e in 'C:\Program Files\Puppet Labs\Puppet\bin','C:\Program Files\OpenVox\Puppet\bin','C:\Program Files\Git\cmd') { if (Test-Path `$e) { `$env:Path = `$e + ';' + `$env:Path } } +`$env:custom_win_github_pat = '$($env:custom_win_github_pat)' +`$env:FACTER_custom_win_role = '$role' +Set-Location '$roninDir' +# Write the puppet log as UTF-8, NOT via Tee-Object. Windows PowerShell 5.1's Tee-Object +# has no -Encoding and writes UTF-16LE, while the parent's Write-NewLog reads mid-stream at +# a byte offset (so there is no BOM to detect) and decodes as UTF-8. The result was every +# puppet line arriving space-interleaved - "N o t i c e : / S t a g e [ m a i n ]" - which +# is unreadable and, worse, ungreppable, so class-level output like +# intel_graphics_software's "provisioned packages matching: N" could not be found in the +# bake log at all. AutoFlush keeps it streaming live rather than landing in one lump. +# --color=false drops the ANSI escapes that were also littering the log. +`$sw = New-Object System.IO.StreamWriter('$puppetLog', `$false, (New-Object System.Text.UTF8Encoding(`$false))) +`$sw.AutoFlush = `$true +try { + & puppet apply manifests\nodes.pp --onetime --verbose --detailed-exitcodes --color=false --modulepath="modules;r10k_modules" --hiera_config=hiera.yaml --logdest console *>&1 | ForEach-Object { `$sw.WriteLine(`$_.ToString()) } + `$rc = `$LASTEXITCODE +} finally { `$sw.Dispose() } +Set-Content -Path '$exitFile' -Value `$rc +"@ +[System.IO.File]::WriteAllText($sysScript, $child, $utf8NoBom) + +# Stream new bytes of a growing log file to this console (so puppet output reaches the +# packer/build log live). Shared-read so it doesn't block the SYSTEM writer. +function Write-NewLog { + param([string]$Path, [ref]$Offset) + if (-not (Test-Path $Path)) { return } + $fs = [System.IO.File]::Open($Path, 'Open', 'Read', 'ReadWrite') + try { + if ($fs.Length -lt $Offset.Value) { $Offset.Value = 0 } + if ($fs.Length -eq $Offset.Value) { return } + $fs.Seek($Offset.Value, 'Begin') | Out-Null + # Decode explicitly as UTF-8 (the child writes UTF-8 via StreamWriter). Do NOT rely on + # BOM detection: every read after the first seeks to a byte offset mid-file, where + # there is no BOM, so the default would silently guess. + $sr = New-Object System.IO.StreamReader($fs, (New-Object System.Text.UTF8Encoding($false))) + try { + $content = $sr.ReadToEnd() + if ($content) { $content.TrimEnd("`r", "`n").Split(@("`r`n", "`n"), [System.StringSplitOptions]::None) | ForEach-Object { if ($_) { Write-Host $_ } } } + } finally { $Offset.Value = $fs.Position; $sr.Dispose() } + } finally { $fs.Dispose() } +} + +$taskName = 'BakePuppetAsSystem' +Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue +$action = New-ScheduledTaskAction -Execute 'powershell.exe' -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$sysScript`"" +$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest +$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Hours 2) +Register-ScheduledTask -TaskName $taskName -Action $action -Principal $principal -Settings $settings -Force | Out-Null +Start-ScheduledTask -TaskName $taskName + +# Wait for the SYSTEM task to record its exit code, tailing the puppet log meanwhile. +$deadline = (Get-Date).AddHours(2) +$offset = [long]0 +while (-not (Test-Path $exitFile)) { + if ((Get-Date) -gt $deadline) { throw 'Timed out waiting for the SYSTEM puppet-apply task.' } + Write-NewLog -Path $puppetLog -Offset ([ref]$offset) + Start-Sleep -Seconds 5 +} +Write-NewLog -Path $puppetLog -Offset ([ref]$offset) # flush the final chunk +$rc = [int]((Get-Content $exitFile -Raw).Trim()) +Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue +Stop-Transcript | Out-Null + +# detailed-exitcodes: 0 = no changes, 2 = changes applied (both OK), 4/6 = failures. +# rc=1 = puppet failed to run/compile the catalog (not a resource-level failure). +if ($rc -eq 0 -or $rc -eq 2) { + Write-Host "Bake puppet apply OK (rc=$rc)" + + # --- Remove the baked ronin clone so the DEPLOY re-clones fresh --- + # Don't ship C:\ronin in the golden WIM. The deploy-time bootstrap (Get-Ronin) removes and + # re-clones ronin anyway, so a baked copy is just stale weight pinned to this bake's hash; + # dropping it forces a clean clone at the pool's current branch/hash on first boot (and keeps + # the WIM smaller). The leftover HKLM\...\ronin_puppet registry values are a separate concern. + if (Test-Path $roninDir) { + Step "Removing baked ronin clone ($roninDir) so deploy re-clones fresh" + Remove-Item $roninDir -Recurse -Force -ErrorAction SilentlyContinue + } + + # --- 8. Bake OpenSSH server (mirrors Get-Bootstrap.ps1 Set-SSH) --- + # Bake sshd + the audit key so SSH is up at FIRST BOOT, independent of the deploy-time + # bootstrap. Makes the golden image self-sufficient and gives operator access even if + # first-boot bootstrap stalls. Assets come from the same source Get-Bootstrap uses; + # sysprep-generalize.ps1 removes ssh_host_* so host keys regenerate per node. + Step 'Baking OpenSSH server + audit key' + $sshAssets = 'https://raw.githubusercontent.com/mozilla-platform-ops/worker-images/main/provisioners/windows/MDC1Windows/ssh' + $sshMsi = Join-Path $dlDir 'OpenSSH-Win64.msi' + Get-PrereqFile -Urls @('https://github.com/PowerShell/Win32-OpenSSH/releases/download/v9.8.3.0p2-Preview/OpenSSH-Win64-v9.8.3.0.msi') -OutFile $sshMsi + $s = Start-Process msiexec.exe -ArgumentList "/i `"$sshMsi`" /quiet /norestart ADDLOCAL=Server" -Wait -PassThru + if ($s.ExitCode -ne 0 -and $s.ExitCode -ne 3010) { throw "OpenSSH MSI install failed rc=$($s.ExitCode)" } + New-Item -ItemType Directory -Path 'C:\ProgramData\ssh' -Force | Out-Null + Get-PrereqFile -Urls @("$sshAssets/sshd_config") -OutFile 'C:\ProgramData\ssh\sshd_config' + $adminSsh = 'C:\Users\Administrator\.ssh' + New-Item -ItemType Directory -Path $adminSsh -Force | Out-Null + Get-PrereqFile -Urls @("$sshAssets/authorized_keys") -OutFile (Join-Path $adminSsh 'authorized_keys') + + # Also bake the audit key as an ADMIN-GROUP key. Win32-OpenSSH treats members of the + # local Administrators group specially: with the "Match Group administrators" block + # below it authenticates them ONLY against %ProgramData%\ssh\administrators_authorized_keys + # (NOT the per-user .ssh\authorized_keys). Dropping the key there means ANY enabled admin + # can SSH in with the key from first boot - notably the built-in Administrator, which + # sysprep-generalize.ps1 enables so the deploy-time autologon works. That gives operator + # access to diagnose a node even if the first-boot bootstrap stalls. (The build-only + # 'packer' admin is disabled at the end of the bake, so it is not a usable path.) + $adminKeys = 'C:\ProgramData\ssh\administrators_authorized_keys' + Copy-Item (Join-Path $adminSsh 'authorized_keys') $adminKeys -Force + # StrictModes: sshd IGNORES administrators_authorized_keys unless it is owned by an admin + # and writable ONLY by Administrators/SYSTEM. Reset inheritance and grant those two alone. + icacls $adminKeys /inheritance:r /grant 'Administrators:F' /grant 'SYSTEM:F' | Out-Null + # Ensure the admin-group Match block is present (append once; idempotent across re-bakes). + $sshdCfg = 'C:\ProgramData\ssh\sshd_config' + if ((Get-Content $sshdCfg -Raw) -notmatch '(?im)^\s*Match\s+Group\s+administrators') { + Add-Content -Path $sshdCfg -Value "`r`nMatch Group administrators`r`n AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys`r`n" + } + + if (-not (Get-NetFirewallRule -Name 'AllowSSH' -ErrorAction SilentlyContinue)) { + New-NetFirewallRule -Name 'AllowSSH' -DisplayName 'Allow SSH' -Profile Any -Direction Inbound -Action Allow -Protocol TCP -LocalPort 22 | Out-Null + } + Set-Service -Name sshd -StartupType Automatic + $mp = [Environment]::GetEnvironmentVariable('Path', 'Machine') + if ($mp -notlike '*OpenSSH*') { [Environment]::SetEnvironmentVariable('Path', "$mp;$env:ProgramFiles\OpenSSH", 'Machine') } + + # --- 9. Bake nxlog log shipping (mirrors Get-Bootstrap.ps1 / bootstrap.ps1 Set-Logging) --- + # Install nxlog CE and drop the papertrail config + CA cert so the deployed worker ships + # logs to papertrail from FIRST BOOT, instead of waiting for the deploy-time bootstrap to + # do it. Same assets, same source blob ($extSrc = .../binaries/prerequisites) the upstream + # Set-Logging uses, so behaviour matches a normally-bootstrapped node. + Step 'Baking nxlog log shipping' + $nxMsi = 'nxlog-ce-2.10.2150.msi' + $nxConf = 'nxlog.conf' + $nxPem = 'papertrail-bundle.pem' + $nxDir = "$env:SystemDrive\Program Files (x86)\nxlog" + $nxMsiPath = Join-Path $dlDir $nxMsi + Get-PrereqFile -Urls @("$extSrc/$nxMsi") -OutFile $nxMsiPath + $nx = Start-Process msiexec.exe -ArgumentList "/i `"$nxMsiPath`" /passive /norestart" -Wait -PassThru + if ($nx.ExitCode -ne 0 -and $nx.ExitCode -ne 3010) { throw "nxlog MSI install failed rc=$($nx.ExitCode)" } + # msiexec /passive returns before the service dir is fully laid down; wait for conf\. + for ($i = 0; $i -lt 30 -and -not (Test-Path "$nxDir\conf\"); $i++) { Start-Sleep 10 } + if (-not (Test-Path "$nxDir\conf\")) { throw "nxlog conf dir never appeared at $nxDir\conf" } + New-Item -ItemType Directory -Path "$nxDir\cert" -Force | Out-Null + Get-PrereqFile -Urls @("$extSrc/$nxConf") -OutFile "$nxDir\conf\$nxConf" + Get-PrereqFile -Urls @("$extSrc/$nxPem") -OutFile "$nxDir\cert\$nxPem" + # Leave nxlog Automatic so it starts on the deployed node; it will pick up the config + # above on that boot. (No point starting it now - the bake VM's logs aren't wanted, and + # sysprep/capture follow immediately.) + Set-Service -Name nxlog -StartupType Automatic -ErrorAction SilentlyContinue + + # --- 10. Bake PowerShell prereqs (NuGet provider + modules) so DEPLOY skips them --- + # The deploy bootstrap's Get-PSModules otherwise installs the NuGet provider + the ugit and + # Powershell-Yaml modules from PSGallery on first boot (previously a ~9-min poll+install). + # Bake them now so Get-PSModules finds them present and skips. Pre-install the NuGet provider + # and mark PSGallery Trusted FIRST so Install-Module can't block on the hidden trust prompt. + Step 'Baking NuGet provider + PS modules (ugit, Powershell-Yaml)' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.208 -Force -Confirm:$false -ForceBootstrap -Scope AllUsers -ErrorAction SilentlyContinue | Out-Null + try { Set-PSRepository -Name PSGallery -InstallationPolicy Trusted -ErrorAction SilentlyContinue } catch { } + foreach ($m in @('ugit', 'Powershell-Yaml')) { + if (-not (Get-Module -Name $m -ListAvailable)) { + Write-Host " installing module $m (AllUsers)" + Install-Module -Name $m -Scope AllUsers -AllowClobber -Force -Confirm:$false -ErrorAction SilentlyContinue + } + if (Get-Module -Name $m -ListAvailable) { Write-Host " module $m present" } else { Write-Warning " module $m NOT installed (deploy will install it)" } + } + + # NOTE: the first-boot bootstrap runner (which launches Get-Bootstrap) is NOT baked + # here. It is registered as a SYSTEM startup scheduled task at the very END of + # sysprep-generalize.ps1 - after all bake work, immediately before Sysprep /shutdown - + # so it can ONLY fire on the DEPLOYED node's first boot, never during the bake (the + # bake VM has no further boots before capture). SetupComplete.cmd was tried here first + # but did not run on the DISM/generalized boot; the startup task is the reliable path. + exit $rc +} +Write-Host "----- bake-puppet.log (tail 120) -----" +Get-Content (Join-Path $log 'bake-puppet.log') -Tail 120 -ErrorAction SilentlyContinue +throw "Bake puppet apply FAILED rc=$rc" diff --git a/provisioners/windows/win-hw-wim/scripts/bootstrap-build-host.ps1 b/provisioners/windows/win-hw-wim/scripts/bootstrap-build-host.ps1 new file mode 100644 index 00000000..b6615956 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/bootstrap-build-host.ps1 @@ -0,0 +1,110 @@ +<# +.SYNOPSIS + Prepare an Azure VM to be a win-hw-wim build host: nested Hyper-V + tooling. + Runs ON the VM via `az vm run-command`. + +.DESCRIPTION + Driven by a script-scope $Phase variable (NOT a param) because + `az vm run-command --parameters` does not reliably map to script parameters — + the caller prepends `$Phase = 'Hyperv'` (or 'Tooling') as a separate --scripts line. + + Hyperv : enable the Hyper-V role (caller reboots afterward). + Tooling : install Packer, azcopy, git, az CLI (DISM is native), powershell-yaml. + + On success each phase prints the sentinel BOOTSTRAP_PHASE_OK — the caller asserts it, + because `az vm run-command` returns exit 0 even when the inner script throws. + Needs a nested-virtualization-capable SKU (Dv3/Dv4/Dv5, Ev3+, Fsv2, ...). +#> +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +if (-not $Phase) { throw "Set `$Phase to 'Hyperv' or 'Tooling' before running this script." } +if ($Phase -notin @('Hyperv', 'Tooling')) { throw "Invalid `$Phase '$Phase' (expected Hyperv or Tooling)." } +$DataDriveLetter = 'F' + +if ($Phase -eq 'Hyperv') { + Write-Host '== Enabling Hyper-V role (reboot required afterwards) ==' + $r = Install-WindowsFeature -Name Hyper-V -IncludeManagementTools + Write-Host " Success=$($r.Success) RestartNeeded=$($r.RestartNeeded)" + if (-not $r.Success) { throw "Install-WindowsFeature Hyper-V failed: $($r | Out-String)" } + Write-Output 'BOOTSTRAP_PHASE_OK' + return +} + +# ---- Phase: Tooling ---------------------------------------------------------- +if (-not (Get-WindowsFeature -Name Hyper-V).Installed) { + throw 'Hyper-V not installed yet. Run Phase Hyperv and reboot first.' +} + +# Nested-VM network: an INTERNAL switch + NAT. Windows Server has no client +# "Default Switch", and an Azure host can't bridge a nested VM onto the vnet, so +# the bake VM gets outbound internet (ronin/git/choco/Windows Update) and a +# host-reachable IP for Packer WinRM through NAT. The guest gets a matching STATIC +# IP via the injected unattend (scripts/unattend/unattend.xml.template) because a +# NAT switch has no DHCP. Keep these in sync with that template. +$SwitchName = 'wim-nat' +$HostNatIp = '192.168.234.1' +$NatPrefix = '192.168.234.0/24' +if (-not (Get-VMSwitch -Name $SwitchName -ErrorAction SilentlyContinue)) { + Write-Host "== Creating internal NAT switch $SwitchName ==" + New-VMSwitch -Name $SwitchName -SwitchType Internal | Out-Null +} +$ifAlias = "vEthernet ($SwitchName)" +if (-not (Get-NetIPAddress -InterfaceAlias $ifAlias -IPAddress $HostNatIp -ErrorAction SilentlyContinue)) { + New-NetIPAddress -InterfaceAlias $ifAlias -IPAddress $HostNatIp -PrefixLength 24 | Out-Null +} +if (-not (Get-NetNat -Name $SwitchName -ErrorAction SilentlyContinue)) { + New-NetNat -Name $SwitchName -InternalIPInterfaceAddressPrefix $NatPrefix | Out-Null +} +Write-Host "== NAT switch $SwitchName ready ($NatPrefix via $HostNatIp) ==" + +# Initialize + mount the data disk (if a raw disk is attached) for build artifacts. +$raw = Get-Disk | Where-Object PartitionStyle -eq 'RAW' | Select-Object -First 1 +if ($raw) { + Write-Host "== Initializing data disk #$($raw.Number) as ${DataDriveLetter}: ==" + $raw | Initialize-Disk -PartitionStyle GPT -PassThru | + New-Partition -DriveLetter $DataDriveLetter -UseMaximumSize | + Format-Volume -FileSystem NTFS -NewFileSystemLabel 'build' -Confirm:$false | Out-Null +} + +# Chocolatey. +if (-not (Get-Command choco -ErrorAction SilentlyContinue)) { + Write-Host '== Installing Chocolatey ==' + Set-ExecutionPolicy Bypass -Scope Process -Force + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-Expression ((New-Object Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + $env:Path += ";$env:ProgramData\chocolatey\bin" + if (-not (Get-Command choco -ErrorAction SilentlyContinue)) { throw 'Chocolatey install failed.' } +} + +# DISM is built into Windows (System32) — no ADK package needed for the WIM +# apply/capture cmdlets (Expand-WindowsImage / New-WindowsImage) the pipeline uses. +foreach ($pkg in 'packer', 'azcopy10', 'git', 'azure-cli') { + Write-Host "== choco install $pkg ==" + & choco install $pkg -y --no-progress --limit-output + if ($LASTEXITCODE -notin 0, 3010) { throw "choco install $pkg failed rc=$LASTEXITCODE" } +} + +Write-Host '== Installing powershell-yaml module ==' +if (-not (Get-Module -ListAvailable powershell-yaml)) { + # In a non-interactive run-command session, Install-Module HANGS forever + # prompting to bootstrap the NuGet provider / trust the PSGallery repo. + # Pre-install the provider and mark PSGallery trusted so nothing prompts. + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force | Out-Null + Set-PSRepository -Name PSGallery -InstallationPolicy Trusted + Install-Module powershell-yaml -Scope AllUsers -Force -Confirm:$false +} + +# Refresh PATH from the machine env so the just-installed tools resolve now. +$env:Path = [Environment]::GetEnvironmentVariable('Path', 'Machine') + ';' + [Environment]::GetEnvironmentVariable('Path', 'User') + +# Verify the toolchain actually installed (fail loudly — run-command hides inner errors). +$missing = @() +foreach ($t in 'packer', 'git', 'azcopy', 'az', 'dism') { + if (-not (Get-Command $t -ErrorAction SilentlyContinue)) { $missing += $t } +} +if ($missing) { throw "Tooling missing after install: $($missing -join ', ')" } + +Write-Host '== Build host ready (packer/git/azcopy/az/dism present). ==' +Write-Output 'BOOTSTRAP_PHASE_OK' diff --git a/provisioners/windows/win-hw-wim/scripts/capture-wim.ps1 b/provisioners/windows/win-hw-wim/scripts/capture-wim.ps1 new file mode 100644 index 00000000..b420eecf --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/capture-wim.ps1 @@ -0,0 +1,87 @@ +<# +.SYNOPSIS + Step 5 (runs on the Windows HOST, elevated, after the Packer build): capture the + generalized VHDX into a golden install.wim. + +.DESCRIPTION + Finds the generalized VHDX in the Packer output directory, mounts it, locates + the Windows volume, and runs DISM /Capture-Image. Records a SHA-256. Always + dismounts the VHDX. + +.PARAMETER BuildDir + Packer output_directory (contains the cloned VM + its Virtual Hard Disks). + +.PARAMETER OutWim + Destination install.wim path. + +.PARAMETER Name + Image /Name metadata. + +.EXAMPLE + .\capture-wim.ps1 -BuildDir .\output\build -OutWim .\output\install.wim +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $BuildDir, + [Parameter(Mandatory)] [string] $OutWim, + [string] $Name = 'win11-24h2-ci-baked' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$id = [Security.Principal.WindowsIdentity]::GetCurrent() +if (-not (New-Object Security.Principal.WindowsPrincipal($id)).IsInRole( + [Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw 'Must run elevated (Administrator).' +} + +$vhdx = Get-ChildItem -Path $BuildDir -Recurse -Filter *.vhdx -ErrorAction Stop | + Sort-Object Length -Descending | Select-Object -First 1 +if (-not $vhdx) { throw "No .vhdx found under $BuildDir" } +Write-Host "== Capturing from $($vhdx.FullName) ==" + +$outDir = Split-Path -Parent $OutWim +if ($outDir -and -not (Test-Path $outDir)) { New-Item -ItemType Directory -Path $outDir -Force | Out-Null } +if (Test-Path $OutWim) { throw "OutWim already exists (refusing to overwrite): $OutWim" } + +$mounted = $null +try { + $before = (Get-Volume).DriveLetter + Mount-VHD -Path $vhdx.FullName + $mounted = $vhdx.FullName + Start-Sleep -Seconds 2 + # The Windows volume = the new NTFS volume that appeared with a \Windows dir. + $win = Get-Volume | Where-Object { + $_.DriveLetter -and ($_.DriveLetter -notin $before) -and + (Test-Path ("{0}:\Windows\System32\ntoskrnl.exe" -f $_.DriveLetter)) + } | Select-Object -First 1 + if (-not $win) { throw 'Could not locate the Windows volume on the mounted VHDX.' } + $root = "$($win.DriveLetter):\" + Write-Host "== Windows volume: $root ==" + + Write-Host "== DISM /Capture-Image -> $OutWim ==" + New-WindowsImage -ImagePath $OutWim -CapturePath $root -Name $Name ` + -Description "Baked Windows HW CI image (ronin bake role)" -CompressionType Max -Verify | Out-Null +} +finally { + if ($mounted) { Dismount-VHD -Path $mounted -ErrorAction SilentlyContinue } +} + +$sha = (Get-FileHash -Algorithm SHA256 -Path $OutWim).Hash +$sizeGB = [math]::Round((Get-Item $OutWim).Length / 1GB, 2) +"$sha $(Split-Path -Leaf $OutWim)" | Set-Content -Path "$OutWim.sha256" +Write-Host "== Captured $OutWim ($sizeGB GB) ==" +Write-Host " SHA256: $sha" +# Best-effort image-info dump. Normalize to backslashes: DISM /Capture-Image tolerates the +# forward-slash path Packer passes, but the Get-WindowsImage cmdlet rejects it with "The +# parameter is incorrect" (0x80070057). Never fail the capture on a verification hiccup - +# the WIM and its .sha256 already exist at this point. +try { + $wimPath = ([string]$OutWim).Replace('/', '\') + Get-WindowsImage -ImagePath $wimPath | Format-List ImageName, ImageIndex, ImageSize +} +catch { + $vmsg = $_.Exception.Message + Write-Warning (" (image-info verification skipped: " + $vmsg + ")") +} diff --git a/provisioners/windows/win-hw-wim/scripts/create-iso.ps1 b/provisioners/windows/win-hw-wim/scripts/create-iso.ps1 new file mode 100644 index 00000000..7a180048 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/create-iso.ps1 @@ -0,0 +1,173 @@ +<# +.SYNOPSIS + Build a Windows 11 install ISO with the hardware-requirement checks bypassed + (TPM 2.0 / Secure Boot / RAM / CPU / storage) so it can clean-install on + unsupported hardware. SEPARATE from the ronin base WIMs - this only patches + Windows Setup's compatibility gate; it does not bake any ronin/worker content. + +.DESCRIPTION + Mounts a base Win11 ISO, copies its contents to a writable working dir, injects an + autounattend.xml at the media root whose windowsPE pass writes the + HKLM\SYSTEM\Setup\LabConfig bypass keys (and MoSetup AllowUpgradesWithUnsupportedTPMOrCPU) + BEFORE Setup's compatibility check, then repackages a UEFI+BIOS bootable ISO with + oscdimg (Windows ADK Deployment Tools). Only the windowsPE pass is set, so after the + bypass Setup continues as a normal (interactive) Win11 install - i.e. a stock ISO that + also works on unsupported hardware. See https://woshub.com/upgrade-to-windows-11-unsupported-pc/. + + Runs on the Windows build host: needs the ADK (oscdimg) and elevation (Mount-DiskImage). + +.PARAMETER SourceIso + Path to the base Win11 ISO to patch. + +.PARAMETER OutIso + Path to write the requirement-bypass ISO. A ".sha256" sidecar is written too + (so upload-wim.ps1 can publish both). + +.PARAMETER Label + Volume label for the output ISO. Default 'WIN11_NOCHK'. + +.EXAMPLE + .\create-iso.ps1 -SourceIso F:\iso\win11-24h2-base.iso -OutIso F:\iso\win11-24h2-nocheck.iso +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $SourceIso, + [Parameter(Mandatory)] [string] $OutIso, + [string] $Label = 'WIN11_NOCHK', + # Comma-delimited names of inject-library scripts (scripts/inject/.ps1) to run against + # the extracted media before repackaging (e.g. 'nocheck'). New-WinHwWim passes the config's + # scripts: list. Each is invoked as .ps1 -MediaDir . + [string] $InjectScripts, + [string] $InjectDir, + # Optional offline driver injection into the ISO's boot.wim (Windows Setup, so it sees + # storage/NIC during install) AND install.wim (every edition, so the installed OS has them). + # '|'-delimited list of driver-pack URLs (.zip/.cab that expands to an .inf tree); e.g. the + # HPE ProLiant DL360 Gen10 Windows driver pack. New-WinHwWim passes the config's drivers.cabs. + [string] $DriverZips, + # Storage account for azcopy AAD downloads of *.blob.core.windows.net driver packs. + [string] $Account = 'hardwareimaging' +) + +$ErrorActionPreference = 'Stop' +if (-not (Test-Path -LiteralPath $SourceIso)) { throw "SourceIso not found: $SourceIso" } + +# --- Locate oscdimg (ADK Deployment Tools) -------------------------------------------- +$oscExe = $null +$adkOsc = 'C:\Program Files (x86)\Windows Kits\10\Assessment and Deployment Kit\Deployment Tools\amd64\Oscdimg\oscdimg.exe' +if (Test-Path -LiteralPath $adkOsc) { + $oscExe = $adkOsc +} +else { + $c = Get-Command oscdimg.exe -ErrorAction SilentlyContinue + if ($c) { $oscExe = $c.Source } +} +if (-not $oscExe) { throw 'oscdimg.exe not found - install the Windows ADK Deployment Tools.' } + +# --- Working dir for the extracted media ---------------------------------------------- +$outDir = [System.IO.Path]::GetDirectoryName($OutIso) +if (-not $outDir) { $outDir = (Get-Location).Path } +$work = Join-Path $outDir ('isobuild-' + [System.IO.Path]::GetFileNameWithoutExtension($OutIso)) +if (Test-Path $work) { Remove-Item $work -Recurse -Force } +New-Item -ItemType Directory -Path $work -Force | Out-Null + +# --- Mount the source ISO and copy all contents out (ISO is read-only) ----------------- +Write-Host "== Mounting $SourceIso ==" +$mount = Mount-DiskImage -ImagePath $SourceIso -PassThru +try { + $vol = ($mount | Get-Volume).DriveLetter + if (-not $vol) { Start-Sleep -Seconds 2; $vol = ($mount | Get-Volume).DriveLetter } + if (-not $vol) { throw 'Could not determine the mounted ISO drive letter.' } + $src = "${vol}:\" + Write-Host "== Copying media $src -> $work ==" + Copy-Item -Path (Join-Path $src '*') -Destination $work -Recurse -Force +} +finally { + Dismount-DiskImage -ImagePath $SourceIso | Out-Null +} + +# --- Run the configured inject-library scripts against the extracted media ------------- +# Each name maps to scripts/inject/.ps1 and is invoked as `.ps1 -MediaDir ` +# to modify the media in place (e.g. 'nocheck' writes the requirement-bypass autounattend). +if (-not $InjectDir) { $InjectDir = Join-Path $PSScriptRoot 'inject' } +$names = @($InjectScripts -split ',' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) +if ($names.Count -eq 0) { Write-Warning 'No inject scripts specified (-InjectScripts); media left unmodified.' } +foreach ($n in $names) { + if ($n -notmatch '^[A-Za-z0-9._-]+$') { throw "Illegal inject script name: '$n'" } + $s = Join-Path $InjectDir "$n.ps1" + if (-not (Test-Path -LiteralPath $s)) { throw "inject script not found: $s" } + Write-Host "== inject: $n ($s) ==" + & $s -MediaDir $work +} + +# --- Offline driver injection into the media WIMs (optional) --------------------------- +# Adds drivers so the installer sees storage/NIC (boot.wim = Windows Setup) and the +# installed OS has them (install.wim, every edition). DISM /Add-Driver /Recurse. +$drvUrls = @("$DriverZips" -split '\|' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) +if ($drvUrls.Count -gt 0) { + $drvRoot = Join-Path $work '_drivers' + New-Item -ItemType Directory -Path $drvRoot -Force | Out-Null + foreach ($u in $drvUrls) { + $fn = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetFileName($u)) + Write-Host "== driver pack: $u ==" + if ($u -match '\.blob\.core\.windows\.net/') { + if (-not $env:AZCOPY_AUTO_LOGIN_TYPE) { $env:AZCOPY_AUTO_LOGIN_TYPE = 'AZCLI' } + & azcopy copy "$u" "$fn" --overwrite=true + if ($LASTEXITCODE -ne 0) { throw "azcopy driver download failed rc=$LASTEXITCODE ($u)" } + } + else { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -Uri $u -OutFile $fn -UseBasicParsing + } + $dest = Join-Path $drvRoot ([System.IO.Path]::GetFileNameWithoutExtension($u)) + if ($fn -match '\.cab$') { New-Item -ItemType Directory -Path $dest -Force | Out-Null; & expand.exe $fn -F:* $dest | Out-Null } + else { Expand-Archive -LiteralPath $fn -DestinationPath $dest -Force } + } + $mnt = Join-Path $work '_mnt' + New-Item -ItemType Directory -Path $mnt -Force | Out-Null + function Add-DriversToWim { param($Wim, $Index) + (Get-Item -LiteralPath $Wim).IsReadOnly = $false # media copied from read-only ISO + Write-Host "== DISM /Add-Driver -> $(Split-Path -Leaf $Wim) index $Index ==" + & dism /Mount-Image /ImageFile:"$Wim" /Index:$Index /MountDir:"$mnt" | Out-Null + if ($LASTEXITCODE -ne 0) { throw "DISM mount failed ($Wim idx $Index) rc=$LASTEXITCODE" } + & dism /Image:"$mnt" /Add-Driver /Driver:"$drvRoot" /Recurse + $addrc = $LASTEXITCODE + & dism /Unmount-Image /MountDir:"$mnt" /Commit | Out-Null + if ($LASTEXITCODE -ne 0) { & dism /Unmount-Image /MountDir:"$mnt" /Discard | Out-Null; throw "DISM commit failed ($Wim idx $Index)" } + if ($addrc -ne 0) { Write-Warning "Add-Driver rc=$addrc on $(Split-Path -Leaf $Wim) idx $Index (some INFs may not apply)" } + } + # boot.wim index 2 = Windows Setup (needs storage/NIC to see the disk during install). + $bootWim = Join-Path $work 'sources\boot.wim' + if (Test-Path -LiteralPath $bootWim) { Add-DriversToWim $bootWim 2 } + # install.wim = the OS image; inject into every edition index. + $installWim = Join-Path $work 'sources\install.wim' + if (Test-Path -LiteralPath $installWim) { + foreach ($img in (Get-WindowsImage -ImagePath $installWim)) { Add-DriversToWim $installWim $img.ImageIndex } + } + else { Write-Warning "sources\install.wim not found (install.esd media?) - OS-image drivers not injected." } + Remove-Item $drvRoot, $mnt -Recurse -Force -ErrorAction SilentlyContinue + Write-Host "== driver injection done ==" +} + +# --- Repackage a UEFI(+BIOS) bootable ISO with oscdimg -------------------------------- +$etfs = Join-Path $work 'boot\etfsboot.com' +$efisys = Join-Path $work 'efi\microsoft\boot\efisys.bin' +if (-not (Test-Path -LiteralPath $efisys)) { throw "efisys.bin not found ($efisys) - source is not a valid Windows ISO?" } +if (Test-Path -LiteralPath $etfs) { + $bootdata = "-bootdata:2#p0,e,b$etfs#pEF,e,b$efisys" # BIOS (etfsboot) + UEFI (efisys) +} +else { + $bootdata = "-bootdata:1#pEF,e,b$efisys" # UEFI-only (no etfsboot on this media) +} +if (Test-Path -LiteralPath $OutIso) { Remove-Item -LiteralPath $OutIso -Force } +Write-Host "== oscdimg -> $OutIso ==" +& $oscExe '-m' '-o' '-u2' '-udfver102' "-l$Label" $bootdata "$work" "$OutIso" +if ($LASTEXITCODE -ne 0) { throw "oscdimg failed rc=$LASTEXITCODE" } + +# --- sha256 sidecar (upload-wim.ps1 publishes .sha256 alongside) ------------------ +$hash = (Get-FileHash -LiteralPath $OutIso -Algorithm SHA256).Hash.ToLower() +[System.IO.File]::WriteAllText("$OutIso.sha256", "$hash $(Split-Path -Leaf $OutIso)", [System.Text.ASCIIEncoding]::new()) +$sizeGb = [math]::Round((Get-Item -LiteralPath $OutIso).Length / 1GB, 2) +Write-Host "== Done: $OutIso ($sizeGb GB) sha256=$hash ==" + +# --- Clean up the extracted media (large) --------------------------------------------- +Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue diff --git a/provisioners/windows/win-hw-wim/scripts/download-wim.ps1 b/provisioners/windows/win-hw-wim/scripts/download-wim.ps1 new file mode 100644 index 00000000..b75d7b1c --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/download-wim.ps1 @@ -0,0 +1,52 @@ +<# +.SYNOPSIS + Download a WIM from the private Windows HW WIM storage account (base or captured). + Used on the Packer host (fetch base) and on the on-site MDC1 server (fetch + captured -> MDT share). + +.DESCRIPTION + Two auth modes: + -AuthMode login : Entra SP/managed identity (run `az login --service-principal` + first). Preferred. + -AuthMode sas : append a read-only SAS token (from Key Vault) via -Sas. + Storage is Entra-only (no IP firewall, no keys): the caller needs an Entra + identity with a Storage Blob Data role (managed identity, SP, or a Relops member). + +.EXAMPLE + # Packer host, Entra: + .\download-wim.ps1 -Blob resources/WIMs/win11-24h2-base-install.wim -Dest D:\images\install.wim + + # MDC1 server, SAS: + .\download-wim.ps1 -Blob captured/WIMs/win11-24h2-hw/win11-24h2-hw.wim -Dest \\mdt2022\deployments\staging\install.wim -AuthMode sas -Sas $sas +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $Blob, # e.g. captured/WIMs// + [Parameter(Mandatory)] [string] $Dest, + [string] $Account = 'hardwareimaging', + [ValidateSet('login','sas')] [string] $AuthMode = 'login', + [string] $Sas +) +$ErrorActionPreference = 'Stop' +if (-not (Get-Command azcopy -ErrorAction SilentlyContinue)) { throw 'azcopy not on PATH.' } + +$url = "https://$Account.blob.core.windows.net/$Blob" +$destDir = Split-Path -Parent $Dest +if ($destDir -and -not (Test-Path $destDir)) { New-Item -ItemType Directory -Path $destDir -Force | Out-Null } + +if ($AuthMode -eq 'sas') { + if (-not $Sas) { throw '-Sas required when -AuthMode sas' } + $sep = if ($Sas.StartsWith('?')) { '' } else { '?' } + & azcopy copy "$url$sep$Sas" "$Dest" --overwrite=ifSourceNewer +} else { + # azcopy has its own credential store — it does NOT inherit `az login`. Tell it + # to reuse the az CLI identity (the VM's managed identity, an SP, or a user). + # (azcopy 10.32 dropped --auth-mode on copy; AZCOPY_AUTO_LOGIN_TYPE drives OAuth.) + if (-not $env:AZCOPY_AUTO_LOGIN_TYPE) { $env:AZCOPY_AUTO_LOGIN_TYPE = 'AZCLI' } + & azcopy copy "$url" "$Dest" --overwrite=ifSourceNewer +} +if ($LASTEXITCODE -ne 0) { throw "azcopy download failed rc=$LASTEXITCODE" } + +# Verify SHA-256 if a sidecar is present next to the source. +Write-Host "== Downloaded $Blob -> $Dest ==" +Write-Host " (verify against captured/.sha256 if present)" diff --git a/provisioners/windows/win-hw-wim/scripts/ensure-oscdimg.ps1 b/provisioners/windows/win-hw-wim/scripts/ensure-oscdimg.ps1 new file mode 100644 index 00000000..3852e427 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/ensure-oscdimg.ps1 @@ -0,0 +1,68 @@ +<# +.SYNOPSIS + Ensure oscdimg.exe (ADK Deployment Tools) is available for the iso stage, served from + OUR blob (resources/tools) rather than depending on the Microsoft ADK CDN at build time. + +.DESCRIPTION + create-iso.ps1 needs oscdimg to repackage a bootable ISO; DISM (native) can't. This + guarantees oscdimg is present, in order of preference: + 1. Already installed (ADK default path or on PATH) -> nothing to do. + 2. Restore the cached Oscdimg folder from resources/tools/oscdimg/ (fast, fully self-contained). + 3. First-ever seed: pull resources/tools/adksetup.exe from our blob, install just + OptionId.DeploymentTools, then CACHE the resulting Oscdimg folder back to + resources/tools/oscdimg/ so every later build is served entirely from our blob. + Only the one-time seed touches the Microsoft CDN (for the Deployment Tools payload); + after that oscdimg lives in our blob. Runs on the build host (elevated; azcopy reuses + the VM's az / managed-identity session set up by New-WinHwWim's `az login`). + +.PARAMETER Account + Storage account holding the tools (default hardwareimaging). +#> +[CmdletBinding()] +param( + [string] $Account = 'hardwareimaging', + [string] $Container = 'resources', + [string] $ToolsPrefix = 'tools' +) +$ErrorActionPreference = 'Stop' + +# ADK installs oscdimg to this fixed location (Windows Kits\10 root is version-independent); +# it is exactly where create-iso.ps1 looks first. +$adkOscDir = 'C:\Program Files (x86)\Windows Kits\10\Assessment and Deployment Kit\Deployment Tools\amd64\Oscdimg' +$oscExe = Join-Path $adkOscDir 'oscdimg.exe' + +if ((Test-Path -LiteralPath $oscExe) -or (Get-Command oscdimg.exe -ErrorAction SilentlyContinue)) { + Write-Host '== oscdimg already present ==' + return +} +if (-not (Get-Command azcopy -ErrorAction SilentlyContinue)) { throw 'azcopy not on PATH.' } +# azcopy has its own credential store; point it at the az CLI identity (the VM's managed identity). +if (-not $env:AZCOPY_AUTO_LOGIN_TYPE) { $env:AZCOPY_AUTO_LOGIN_TYPE = 'AZCLI' } +$baseUrl = "https://$Account.blob.core.windows.net/$Container/$ToolsPrefix" +$cacheUrl = "$baseUrl/oscdimg" + +# 1) Try the cached Oscdimg folder in our blob. +$stage = Join-Path $env:TEMP 'oscdimg-cache' +Remove-Item $stage -Recurse -Force -ErrorAction SilentlyContinue +& azcopy copy "$cacheUrl/*" "$stage" --recursive 2>&1 | Out-Null # no-op/err if the cache doesn't exist yet +if (Test-Path -LiteralPath (Join-Path $stage 'oscdimg.exe')) { + Write-Host "== oscdimg: restoring from $Container/$ToolsPrefix/oscdimg (our blob) ==" + New-Item -ItemType Directory -Force -Path $adkOscDir | Out-Null + Copy-Item (Join-Path $stage '*') $adkOscDir -Recurse -Force + if (-not (Test-Path -LiteralPath $oscExe)) { throw "restore failed: $oscExe missing." } + Write-Host "== oscdimg ready ($oscExe) ==" + return +} + +# 2) Seed: install ADK Deployment Tools from our hosted adksetup.exe, then cache oscdimg back. +Write-Host "== oscdimg: seeding via $Container/$ToolsPrefix/adksetup.exe (one-time; payload from MS CDN) ==" +$adkSetup = Join-Path $env:TEMP 'adksetup.exe' +& azcopy copy "$baseUrl/adksetup.exe" "$adkSetup" --overwrite=true +if ($LASTEXITCODE -ne 0) { throw "azcopy adksetup download failed rc=$LASTEXITCODE" } +$p = Start-Process -FilePath $adkSetup -ArgumentList '/quiet', '/features', 'OptionId.DeploymentTools', '/norestart', '/ceip', 'off' -Wait -PassThru +if ($p.ExitCode -notin 0, 3010) { throw "adksetup install failed rc=$($p.ExitCode)" } +if (-not (Test-Path -LiteralPath $oscExe)) { throw "oscdimg not found after ADK install ($oscExe)." } +Write-Host "== oscdimg: caching Oscdimg folder to $Container/$ToolsPrefix/oscdimg for future builds ==" +& azcopy copy "$adkOscDir\*" "$cacheUrl" --recursive +if ($LASTEXITCODE -ne 0) { Write-Warning "azcopy cache upload failed rc=$LASTEXITCODE (non-fatal; oscdimg is installed locally for this build)." } +Write-Host "== oscdimg ready ($oscExe) ==" diff --git a/provisioners/windows/win-hw-wim/scripts/extract-wim-from-iso.ps1 b/provisioners/windows/win-hw-wim/scripts/extract-wim-from-iso.ps1 new file mode 100644 index 00000000..d68e5fa8 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/extract-wim-from-iso.ps1 @@ -0,0 +1,74 @@ +<# +.SYNOPSIS + Extract the base install.wim from a Windows install ISO (the media's + sources\install.wim), so a WIM bake can start from just an uploaded ISO when no + base WIM exists yet. + +.DESCRIPTION + Mounts the ISO, copies out sources\install.wim (the full multi-edition OS image, so + edition-name resolution in prepare-base-vhdx still works). If the media ships a + compressed sources\install.esd instead (consumer media), every edition is exported + into a WIM with Export-WindowsImage so the result is edition-equivalent. Writes an + .sha256 sidecar (so upload-wim.ps1 can publish both). + + Runs on the Windows build host: needs Mount-DiskImage (elevation) and DISM + (Get-WindowsImage / Export-WindowsImage) for the .esd path. NOT a ronin operation — + this only lifts the OS image out of the media; no worker content is baked here. + +.PARAMETER SourceIso + Path to the Windows install ISO to extract from. + +.PARAMETER OutWim + Path to write the extracted base WIM (convention: -base-install.wim). + +.EXAMPLE + .\extract-wim-from-iso.ps1 -SourceIso F:\wim-work\Win11_25H2_English_x64_v2.iso ` + -OutWim F:\wim-work\win11-25h2-hw\win11-25h2-base-install.wim +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $SourceIso, + [Parameter(Mandatory)] [string] $OutWim +) +$ErrorActionPreference = 'Stop' +if (-not (Test-Path -LiteralPath $SourceIso)) { throw "SourceIso not found: $SourceIso" } + +$outDir = Split-Path -Parent $OutWim +if ($outDir -and -not (Test-Path $outDir)) { New-Item -ItemType Directory -Path $outDir -Force | Out-Null } +if (Test-Path -LiteralPath $OutWim) { Remove-Item -LiteralPath $OutWim -Force } + +Write-Host "== Mounting $SourceIso ==" +$mount = Mount-DiskImage -ImagePath $SourceIso -PassThru +try { + $vol = ($mount | Get-Volume).DriveLetter + if (-not $vol) { Start-Sleep -Seconds 2; $vol = ($mount | Get-Volume).DriveLetter } + if (-not $vol) { throw 'Could not determine the mounted ISO drive letter.' } + $srcWim = "${vol}:\sources\install.wim" + $srcEsd = "${vol}:\sources\install.esd" + if (Test-Path -LiteralPath $srcWim) { + Write-Host "== Copying $srcWim -> $OutWim ==" + Copy-Item -LiteralPath $srcWim -Destination $OutWim -Force + } + elseif (Test-Path -LiteralPath $srcEsd) { + # Consumer media ships a compressed install.esd; export every edition into a WIM + # so edition-name resolution (prepare-base-vhdx) still works. + Write-Host "== No install.wim; exporting editions from $srcEsd -> $OutWim ==" + foreach ($img in (Get-WindowsImage -ImagePath $srcEsd)) { + Write-Host " export index $($img.ImageIndex): $($img.ImageName)" + Export-WindowsImage -SourceImagePath $srcEsd -SourceIndex $img.ImageIndex ` + -DestinationImagePath $OutWim -CompressionType Max | Out-Null + } + } + else { + throw "Neither sources\install.wim nor sources\install.esd found on $SourceIso - not a Windows install ISO?" + } +} +finally { + Dismount-DiskImage -ImagePath $SourceIso | Out-Null +} + +# sha256 sidecar (upload-wim.ps1 publishes .sha256 alongside). +$hash = (Get-FileHash -LiteralPath $OutWim -Algorithm SHA256).Hash.ToLower() +[System.IO.File]::WriteAllText("$OutWim.sha256", "$hash $(Split-Path -Leaf $OutWim)", [System.Text.ASCIIEncoding]::new()) +$sizeGb = [math]::Round((Get-Item -LiteralPath $OutWim).Length / 1GB, 2) +Write-Host "== Extracted base WIM: $OutWim ($sizeGb GB) sha256=$hash ==" diff --git a/provisioners/windows/win-hw-wim/scripts/inject/nocheck.ps1 b/provisioners/windows/win-hw-wim/scripts/inject/nocheck.ps1 new file mode 100644 index 00000000..39703a86 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/inject/nocheck.ps1 @@ -0,0 +1,42 @@ +<# +.SYNOPSIS + Inject-library script 'nocheck': make Windows 11 install media bypass the hardware requirement + checks (TPM 2.0 / Secure Boot / RAM / CPU / storage) so it clean-installs on unsupported hardware. + +.DESCRIPTION + One of the reusable scripts under scripts/inject/, referenced by name from a config's `scripts:` + list and run by create-iso.ps1 against the extracted install media before repackaging. Writes an + autounattend.xml at the media root whose windowsPE pass sets HKLM\SYSTEM\Setup\LabConfig bypass + keys + MoSetup AllowUpgradesWithUnsupportedTPMOrCPU BEFORE Setup's compat check; Setup then + continues normally. Ref: https://woshub.com/upgrade-to-windows-11-unsupported-pc/ + + Inject-script contract: takes -MediaDir (the extracted media root) and modifies it in place. + +.PARAMETER MediaDir + Path to the extracted install media (ISO contents) to modify. +#> +[CmdletBinding()] +param([Parameter(Mandatory)] [string] $MediaDir) + +$ErrorActionPreference = 'Stop' +if (-not (Test-Path -LiteralPath $MediaDir)) { throw "nocheck: MediaDir not found: $MediaDir" } + +$autounattend = @' + + + + + + 1reg add HKLM\System\Setup\LabConfig /v BypassTPMCheck /t REG_DWORD /d 1 /f + 2reg add HKLM\System\Setup\LabConfig /v BypassSecureBootCheck /t REG_DWORD /d 1 /f + 3reg add HKLM\System\Setup\LabConfig /v BypassRAMCheck /t REG_DWORD /d 1 /f + 4reg add HKLM\System\Setup\LabConfig /v BypassCPUCheck /t REG_DWORD /d 1 /f + 5reg add HKLM\System\Setup\LabConfig /v BypassStorageCheck /t REG_DWORD /d 1 /f + 6reg add HKLM\System\Setup\MoSetup /v AllowUpgradesWithUnsupportedTPMOrCPU /t REG_DWORD /d 1 /f + + + + +'@ +[System.IO.File]::WriteAllText((Join-Path $MediaDir 'autounattend.xml'), $autounattend, (New-Object System.Text.UTF8Encoding($false))) +Write-Host " nocheck: wrote requirement-bypass autounattend.xml (LabConfig + MoSetup) to $MediaDir" diff --git a/provisioners/windows/win-hw-wim/scripts/prepare-base-vhdx.ps1 b/provisioners/windows/win-hw-wim/scripts/prepare-base-vhdx.ps1 new file mode 100644 index 00000000..63b9afdc --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/prepare-base-vhdx.ps1 @@ -0,0 +1,269 @@ +<# +.SYNOPSIS + Step 1 of the wim-packer pipeline: apply a bring-your-own base install.wim into + a bootable UEFI/GPT VHDX that Packer's Hyper-V builder can boot. + +.DESCRIPTION + Creates a dynamic VHDX, partitions it GPT (EFI + MSR + Windows), applies the + chosen image index from the source WIM with DISM, and writes UEFI boot files + with bcdboot. The result is a generation-2 (UEFI) bootable disk. + + Run on the Windows host in an ELEVATED PowerShell. This touches disks via + diskpart-equivalent cmdlets scoped to the new VHDX only (never a physical disk). + +.PARAMETER SourceWim + Path to your base install.wim (Win11 24H2). + +.PARAMETER OutVhdx + Path of the VHDX to create. + +.PARAMETER Index + Image index inside the WIM to apply (default 1). Use Get-WindowsImage to list. + +.PARAMETER SizeGB + Maximum (dynamic) size of the VHDX. Default 80. + +.EXAMPLE + .\prepare-base-vhdx.ps1 -SourceWim D:\images\install.wim -OutVhdx .\output\base.vhdx -Index 1 +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $SourceWim, + [Parameter(Mandatory)] [string] $OutVhdx, + # Pick the image inside the WIM by edition NAME (resolved to an index via + # Get-WindowsImage). Falls back to -Index when -Edition is not given. + [string] $Edition, + [int] $Index = 0, + [int] $SizeGB = 80, + # Optional offline driver injection (DISM /Add-Driver) into the applied image. + # One or more driver-pack URLs (scalable: pass as many as needed). Each may be a + # .cab OR a .zip (sniffed by extension); all are downloaded, expanded, and injected + # in a single recursive /Add-Driver. + [switch] $InjectDrivers, + # '|'-delimited list of driver-pack URLs. PowerShell's -File invocation can't bind a real + # array parameter (extra space-separated values spill onto the next positional param), so + # New-WinHwWim joins the list with '|' and we split it here. + [string] $DriverCabUrls, + # Optional payloads copied VERBATIM into the image at C:\extras\ (no expansion, + # no execution here). Same '|'-delimited convention as -DriverCabUrls. + # + # This exists because a deployed NUC cannot reach our storage: hardwareimaging is + # Entra-only (anonymous GET -> 409) and the nodes have no Azure identity. Only the + # build host has one, so anything a NUC needs to run has to be staged into the + # image from here. + # + # NOTE the destination is C:\extras, NOT C:\bake\extras: these payloads must SHIP in + # the golden WIM, and sysprep-generalize.ps1 deletes C:\bake wholesale before capture. + # ronin's win_intel_graphics_software runs the Intel installer from here at DEPLOY + # time, on real hardware - it cannot run at bake (rc=1008 in the GPU-less Hyper-V + # guest, and the MSIX it installs is per-user, so sysprep would strip it anyway). + [string] $ExtrasUrls, + # Build-only WinRM account injected via unattend so Packer can connect. + # Scrubbed by sysprep-generalize.ps1 before capture — never ships in the WIM. + [string] $WinRMUser = 'packer', + [Parameter(Mandatory)] [string] $WinRMPassword, + [string] $ComputerName = 'nuc-bake' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +function Assert-Admin { + $id = [Security.Principal.WindowsIdentity]::GetCurrent() + $p = New-Object Security.Principal.WindowsPrincipal($id) + if (-not $p.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw 'Must run elevated (Administrator).' + } +} + +# Fetch a bake asset, picking the transport from the URL. +# +# Authenticated Azure blob (e.g. hardwareimaging - no anonymous access, an unauthenticated +# GET returns 409): azcopy with the build host's AAD login. azcopy has its own credential +# store and does NOT inherit `az login`, so AZCOPY_AUTO_LOGIN_TYPE drives OAuth (matches +# download-wim.ps1; azcopy 10.32 dropped --auth-mode on copy). Anything else is treated as +# a public URL (e.g. the roninpuppetassets prereq mirror). +function Get-BakeAsset { + param( + [Parameter(Mandatory)] [string] $Url, + [Parameter(Mandatory)] [string] $Destination + ) + + if ($Url -match '\.blob\.core\.windows\.net/') { + if (-not $env:AZCOPY_AUTO_LOGIN_TYPE) { $env:AZCOPY_AUTO_LOGIN_TYPE = 'AZCLI' } + & azcopy copy "$Url" "$Destination" --overwrite=true + if ($LASTEXITCODE -ne 0) { throw "azcopy failed rc=$LASTEXITCODE for $Url" } + } + else { + Invoke-WebRequest -Uri $Url -OutFile $Destination -UseBasicParsing + } +} + +Assert-Admin +if (-not (Test-Path -LiteralPath $SourceWim)) { throw "SourceWim not found: $SourceWim" } +$SourceWim = (Resolve-Path -LiteralPath $SourceWim).Path + +# Resolve output path (may not exist yet) and ensure parent dir. +$outDir = Split-Path -Parent $OutVhdx +if ($outDir -and -not (Test-Path -LiteralPath $outDir)) { New-Item -ItemType Directory -Path $outDir -Force | Out-Null } +# Make relative paths absolute against the CWD, but leave an already-rooted path +# alone — Join-Path'ing a rooted path onto the CWD yields 'C:\cwd\C:\...' which +# GetFullPath rejects ("path's format is not supported"). +if (-not [System.IO.Path]::IsPathRooted($OutVhdx)) { $OutVhdx = Join-Path (Get-Location) $OutVhdx } +$OutVhdx = [System.IO.Path]::GetFullPath($OutVhdx) +if (Test-Path -LiteralPath $OutVhdx) { throw "OutVhdx already exists (refusing to overwrite): $OutVhdx" } + +# Resolve the image index from the edition name when provided. +if ($Edition) { + $all = Get-WindowsImage -ImagePath $SourceWim + $match = $all | Where-Object { $_.ImageName -eq $Edition } + if (-not $match) { + $list = ($all | ForEach-Object { '[{0}] {1}' -f $_.ImageIndex, $_.ImageName }) -join '; ' + throw "Edition '$Edition' not found in $SourceWim. Available: $list" + } + $Index = [int]($match | Select-Object -First 1).ImageIndex + Write-Host "== Resolved edition '$Edition' -> index $Index ==" +} +elseif ($Index -le 0) { + $Index = 1 +} + +Write-Host "== Validating source WIM index $Index ==" +$img = Get-WindowsImage -ImagePath $SourceWim -Index $Index +Write-Host (" {0} ({1})" -f $img.ImageName, $img.Architecture) + +$vhd = $null +try { + Write-Host "== Creating VHDX ($SizeGB GB dynamic): $OutVhdx ==" + New-VHD -Path $OutVhdx -SizeBytes ($SizeGB * 1GB) -Dynamic | Out-Null + + Write-Host "== Mounting and partitioning (GPT: EFI + MSR + Windows) ==" + $vhd = Mount-VHD -Path $OutVhdx -Passthru | Get-Disk + Initialize-Disk -Number $vhd.Number -PartitionStyle GPT -Confirm:$false | Out-Null + + # EFI System Partition (260 MB, FAT32) + $efi = New-Partition -DiskNumber $vhd.Number -Size 260MB -GptType '{c12a7328-f81f-11d2-ba4b-00a0c93ec93b}' + Format-Volume -Partition $efi -FileSystem FAT32 -NewFileSystemLabel 'System' -Confirm:$false | Out-Null + $efi | Set-Partition -NewDriveLetter 'S' + + # Microsoft Reserved Partition (16 MB) + New-Partition -DiskNumber $vhd.Number -Size 16MB -GptType '{e3c9e316-0b5c-4db8-817d-f92df00215ae}' | Out-Null + + # Windows partition (rest of disk, NTFS) + $win = New-Partition -DiskNumber $vhd.Number -UseMaximumSize -GptType '{ebd0a0a2-b9e5-4433-87c0-68b6b72699c7}' + Format-Volume -Partition $win -FileSystem NTFS -NewFileSystemLabel 'Windows' -Confirm:$false | Out-Null + $win | Set-Partition -NewDriveLetter 'W' + + Write-Host "== Applying image (DISM /Apply-Image index $Index) to W:\ ==" + Expand-WindowsImage -ImagePath $SourceWim -Index $Index -ApplyPath 'W:\' | Out-Null + + # --- Optional: offline driver injection (default OFF) --- + # Scalable: any number of driver packs, each a .cab OR a .zip (sniffed by extension). + # Each pack is downloaded and expanded into its OWN subdir under a common root (so + # files from different packs never collide), then a SINGLE recursive /Add-Driver over + # the root injects them all at once. + if ($InjectDrivers) { + $cabList = @($DriverCabUrls -split '\|' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + if ($cabList.Count -eq 0) { throw 'InjectDrivers set but -DriverCabUrls is empty.' } + $drvRoot = Join-Path $env:TEMP ("winhwdrv-" + [System.IO.Path]::GetRandomFileName()) + New-Item -ItemType Directory -Path $drvRoot -Force | Out-Null + $n = 0 + foreach ($url in $cabList) { + $n++ + $sub = Join-Path $drvRoot ("pkg{0:D2}" -f $n) + New-Item -ItemType Directory -Path $sub -Force | Out-Null + # Archive type from the URL path (ignore any ?query); name the temp file with + # the right extension so Expand-Archive accepts it. + $ext = [System.IO.Path]::GetExtension((($url -split '\?')[0])).ToLowerInvariant() + $arc = Join-Path $sub ("pack" + $ext) + Write-Host "== [$n/$($cabList.Count)] Downloading driver pack ($ext): $url ==" + Get-BakeAsset -Url $url -Destination $arc + switch ($ext) { + '.cab' { + Write-Host "== Expanding cab -> $sub ==" + & expand.exe -F:* "$arc" "$sub" | Out-Null + if ($LASTEXITCODE -ne 0) { throw "expand.exe failed rc=$LASTEXITCODE for $url" } + } + '.zip' { + Write-Host "== Extracting zip -> $sub ==" + Expand-Archive -LiteralPath $arc -DestinationPath $sub -Force + } + default { throw "Unsupported driver pack extension '$ext' for $url (expected .cab or .zip)." } + } + Remove-Item $arc -Force + } + Write-Host "== DISM /Add-Driver (recurse) -> W:\ from $($cabList.Count) pack(s) ==" + & dism.exe /Image:W:\ /Add-Driver /Driver:"$drvRoot" /Recurse + if ($LASTEXITCODE -ne 0) { throw "DISM /Add-Driver failed rc=$LASTEXITCODE" } + } + + # --- Optional: stage payloads to C:\extras (default: none) --- + # Copied verbatim - NOT expanded and NOT executed here. The DEPLOY-time puppet apply is + # what runs them (ronin win_intel_graphics_software globs C:\extras\gfx_win_*.exe). + # + # These have to be staged offline because neither the packer guest nor a deployed NUC + # can fetch them: the payloads live in hardwareimaging, which is Entra-only, and only + # this build host has an Azure identity. + # + # C:\extras SHIPS in the golden WIM by design - that is the point. Do not move it under + # C:\bake, which sysprep-generalize.ps1 deletes before capture. + if ($ExtrasUrls) { + $extraList = @($ExtrasUrls -split '\|' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + if ($extraList.Count -gt 0) { + $extrasDir = 'W:\extras' + New-Item -ItemType Directory -Path $extrasDir -Force | Out-Null + $m = 0 + foreach ($url in $extraList) { + $m++ + # Strip any ?query so the staged file keeps its real name - the ronin class + # globs on gfx_win_*.exe. + $leaf = [System.IO.Path]::GetFileName((($url -split '\?')[0])) + if (-not $leaf) { throw "Could not derive a file name from extras URL: $url" } + $dest = Join-Path $extrasDir $leaf + Write-Host "== [$m/$($extraList.Count)] Staging extra -> C:\extras\$leaf ==" + Get-BakeAsset -Url $url -Destination $dest + $sz = (Get-Item -LiteralPath $dest).Length + Write-Host ("== Staged {0} ({1:n0} bytes) ==" -f $leaf, $sz) + } + } + } + + Write-Host "== Injecting build-only unattend (WinRM + admin) into Panther ==" + $tmpl = Join-Path $PSScriptRoot 'unattend\unattend.xml.template' + if (-not (Test-Path -LiteralPath $tmpl)) { throw "unattend template not found: $tmpl" } + $xml = Get-Content -LiteralPath $tmpl -Raw + $xml = $xml.Replace('@@WINRM_USER@@', $WinRMUser) + $xml = $xml.Replace('@@WINRM_PASSWORD@@', $WinRMPassword) + $xml = $xml.Replace('@@COMPUTERNAME@@', $ComputerName) + $panther = 'W:\Windows\Panther' + New-Item -ItemType Directory -Path $panther -Force | Out-Null + # Write UTF-8 without BOM (Windows Setup is picky about the unattend encoding). + [System.IO.File]::WriteAllText((Join-Path $panther 'unattend.xml'), $xml, (New-Object System.Text.UTF8Encoding($false))) + + # Drop the first-logon network bring-up helper. The unattend's FirstLogonCommands + # invokes this to assign the static NAT IP + WinRM (see set-bake-network.ps1 for the + # why — specialize runs too early, before the NIC is Up). Build-only; sysprep scrubs it. + Write-Host "== Injecting first-logon network helper (Setup\Scripts) ==" + $netHelperSrc = Join-Path $PSScriptRoot 'unattend\set-bake-network.ps1' + if (-not (Test-Path -LiteralPath $netHelperSrc)) { throw "network helper not found: $netHelperSrc" } + $setupScripts = 'W:\Windows\Setup\Scripts' + New-Item -ItemType Directory -Path $setupScripts -Force | Out-Null + Copy-Item -LiteralPath $netHelperSrc -Destination (Join-Path $setupScripts 'set-bake-network.ps1') -Force + + Write-Host "== Writing UEFI boot files (bcdboot) ==" + $bcd = & "$env:SystemRoot\System32\bcdboot.exe" 'W:\Windows' '/s' 'S:' '/f' 'UEFI' + if ($LASTEXITCODE -ne 0) { throw "bcdboot failed rc=$LASTEXITCODE`n$bcd" } + Write-Host " $bcd" + + Write-Host "== Success. Bootable base VHDX ready: $OutVhdx ==" +} +catch { + Write-Warning "prepare-base-vhdx failed: $($_.Exception.Message)" + throw +} +finally { + # Always dismount so the VHDX is not left attached. + if ($vhd) { + try { Dismount-VHD -Path $OutVhdx -ErrorAction SilentlyContinue } catch {} + } +} diff --git a/provisioners/windows/win-hw-wim/scripts/publish-wim.ps1 b/provisioners/windows/win-hw-wim/scripts/publish-wim.ps1 new file mode 100644 index 00000000..9accfcd5 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/publish-wim.ps1 @@ -0,0 +1,52 @@ +<# +.SYNOPSIS + Step 6 (Windows host): publish the baked install.wim into the MDT/WDS deployment + share so the existing OS-deploy.ps1 + PXE dance applies it. + +.DESCRIPTION + Copies the baked install.wim into an image media folder on the share, replacing + sources\install.wim inside a copy of the standard Win11 media. OS-deploy.ps1 + copies "Images\" to the node and runs setup.exe /unattend, so the + baked WIM must live at Images\\sources\install.wim. + + Verifies the SHA-256 before copying. Does NOT edit pools.yml (that is a commit + to worker-images main; see DEPLOY-INTEGRATION.md — do it via PR, not here). + +.EXAMPLE + .\publish-wim.ps1 -Wim .\output\install.wim ` + -MediaTemplate "\\mdt2022.ad.mozilla.com\deployments\Images\win11-24H2-NUC-01-16-2025" ` + -ImageName "win11-24H2-NUC-baked-2026-07-17" +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $Wim, + [Parameter(Mandatory)] [string] $MediaTemplate, # existing extracted-ISO media folder to clone + [Parameter(Mandatory)] [string] $ImageName, # new folder name under Images\ + [switch] $WhatIf +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +if (-not (Test-Path -LiteralPath $Wim)) { throw "WIM not found: $Wim" } +if (Test-Path "$Wim.sha256") { + $expected = (Get-Content "$Wim.sha256").Split(' ')[0] + $actual = (Get-FileHash -Algorithm SHA256 -Path $Wim).Hash + if ($expected -ne $actual) { throw "SHA-256 mismatch for $Wim (expected $expected, got $actual)" } + Write-Host "SHA-256 verified: $actual" +} + +$dest = Join-Path (Split-Path -Parent $MediaTemplate) $ImageName +Write-Host "== Will clone media template -> $dest and swap in baked install.wim ==" +if ($WhatIf) { Write-Host '(WhatIf) no changes made'; return } + +if (Test-Path -LiteralPath $dest) { throw "Destination already exists: $dest" } +Copy-Item -LiteralPath $MediaTemplate -Destination $dest -Recurse -Force + +$target = Join-Path $dest 'sources\install.wim' +$esd = Join-Path $dest 'sources\install.esd' +if (Test-Path $esd) { Remove-Item $esd -Force } # setup prefers install.wim if present +Copy-Item -LiteralPath $Wim -Destination $target -Force + +Write-Host "== Published. Point pools.yml image: -> $ImageName (via worker-images PR) ==" +Write-Host " Then update base-autounattend.xml image index to the captured index (see DEPLOY-INTEGRATION.md)." diff --git a/provisioners/windows/win-hw-wim/scripts/register-base-vm.ps1 b/provisioners/windows/win-hw-wim/scripts/register-base-vm.ps1 new file mode 100644 index 00000000..006eac17 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/register-base-vm.ps1 @@ -0,0 +1,46 @@ +<# +.SYNOPSIS + Wrap the prepared base VHDX in a pristine Gen2 Hyper-V VM that Packer clones + from (source.hyperv-vmcx.clone_from_vm_name). Cloning leaves this VM untouched, + so you can rebuild repeatedly from a clean base. + +.DESCRIPTION + Run on the Windows host, elevated, after prepare-base-vhdx.ps1. Creates a + Generation 2 VM pointing at the VHDX, sets firmware to boot from the disk, and + configures Secure Boot to match win-hw-wim.pkr.hcl (MicrosoftWindows template). + The VM is left OFF; Packer clones and boots a copy. + +.EXAMPLE + .\register-base-vm.ps1 -VmName win-hw-wim-base -Vhdx .\output\base.vhdx -SwitchName "Default Switch" +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $VmName, + [Parameter(Mandatory)] [string] $Vhdx, + [Parameter(Mandatory)] [string] $SwitchName, + [int] $MemoryStartupMB = 8192, + [int] $Cpus = 4 +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +if (-not (Test-Path -LiteralPath $Vhdx)) { throw "VHDX not found: $Vhdx" } +$Vhdx = (Resolve-Path -LiteralPath $Vhdx).Path + +if (Get-VM -Name $VmName -ErrorAction SilentlyContinue) { + throw "VM '$VmName' already exists. Remove it first (Remove-VM) or choose another name." +} + +Write-Host "== Creating Gen2 VM '$VmName' from $Vhdx ==" +$vm = New-VM -Name $VmName -Generation 2 -MemoryStartupBytes ($MemoryStartupMB * 1MB) ` + -VHDPath $Vhdx -SwitchName $SwitchName +Set-VM -Name $VmName -ProcessorCount $Cpus -AutomaticCheckpointsEnabled $false + +# Boot from the OS disk; Secure Boot template must match the Packer source. +$drive = Get-VMHardDiskDrive -VMName $VmName +Set-VMFirmware -VMName $VmName -FirstBootDevice $drive ` + -EnableSecureBoot On -SecureBootTemplate MicrosoftWindows + +Write-Host "== Done. Set source_vm_name = '$VmName' in your *.auto.pkrvars.hcl ==" +Write-Host " Packer will clone this VM; leave it powered off." diff --git a/provisioners/windows/win-hw-wim/scripts/run-build-task.ps1 b/provisioners/windows/win-hw-wim/scripts/run-build-task.ps1 new file mode 100644 index 00000000..efdc923a --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/run-build-task.ps1 @@ -0,0 +1,143 @@ +<# +.SYNOPSIS + On-VM build wrapper run as a scheduled task. Runs New-WinHwWim.ps1 to completion, + tees output to a log, and writes a completion marker with the exit code so the + workflow (on the GH runner) can poll for done/success without the ~90-min + run-command limit applying to the build itself. + +.DESCRIPTION + Writes: + C:\win-hw-wim-build\build.log full build output (append-tee) + C:\win-hw-wim-build\build.done the exit code (0 = success) — created only when done +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $Image, + [string] $BuildId, + # Optional New-WinHwWim -Stages (e.g. 'iso'). Blank = the default prep,build,publish bake. + [string] $Stages, + [string] $IdentityClientId, + # Where to drop the completion marker the GH runner polls (see below). Defaults match + # the pipeline's storage account / captured container. + [string] $StatusAccount = 'hardwareimaging', + [string] $StatusContainer = 'captured' +) +$base = 'C:\win-hw-wim-build' +New-Item -ItemType Directory -Path $base -Force | Out-Null +$log = Join-Path $base 'build.log' +$done = Join-Path $base 'build.done' +Remove-Item $log, $done -ErrorAction SilentlyContinue + +# Refresh PATH from the machine env so packer/az/azcopy/git resolve under the +# scheduled task (choco updated the machine PATH after the guest agent started). +$env:Path = [Environment]::GetEnvironmentVariable('Path', 'Machine') + ';' + [Environment]::GetEnvironmentVariable('Path', 'User') + +# --- Live log streaming ------------------------------------------------------- +# Without this the runner sees only "... building (N min elapsed)" for ~2h and then a +# 200-line tail at the END - and on a hang it sees NOTHING at all, because the job times +# out and the VM (with build.log on it) is destroyed. That blind spot is what made the +# post-bake windows-restart hang in run 31428853582 so expensive to diagnose. So push the +# WHOLE log to blob every minute; the runner prints each new chunk as it lands +# (ci/kickoff-win-hw-wim-build.ps1). Best-effort throughout: a failed upload must never +# affect the build, so every error here is swallowed and retried next cycle. +$liveBlob = "_status/$Image.live.log" +# The boot watchdog (New-WinHwWim) writes here, including its PowerShell Direct capture of +# the GUEST's event log when a step stalls. Streamed as its own blob so the GH job can see +# what the guest was doing WHILE a hang is happening, rather than after Packer gives up. +$wdLocal = Join-Path $base 'boot-watchdog.log' +$wdBlob = "_status/$Image.watchdog.log" +$liveJob = Start-Job -Name 'wim-live-log' -ScriptBlock { + param($log, $account, $container, $blob, $clientId, $pathEnv, $wdLocal, $wdBlob) + $env:Path = $pathEnv + # Tee-Object holds build.log open for writing, so read it with FileShare::ReadWrite + # and upload a snapshot - a plain Copy-Item/az on the live file hits a sharing violation. + $snap = "$log.live" + while ($true) { + Start-Sleep -Seconds 60 + try { + if (-not (Test-Path $log)) { continue } + $in = [IO.File]::Open($log, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite) + try { + $out = [IO.File]::Create($snap) + try { $in.CopyTo($out) } finally { $out.Dispose() } + } + finally { $in.Dispose() } + # az may not be logged in yet on the first cycles (New-WinHwWim logs in itself). + az account show 1>$null 2>$null + if (($LASTEXITCODE -ne 0) -and $clientId) { az login --identity --client-id $clientId 1>$null 2>$null } + az storage blob upload --account-name $account --container-name $container --name $blob --file $snap --overwrite --auth-mode login --only-show-errors 2>$null + if (Test-Path $wdLocal) { + $wdSnap = "$wdLocal.live" + $win = [IO.File]::Open($wdLocal, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite) + try { + $wout = [IO.File]::Create($wdSnap) + try { $win.CopyTo($wout) } finally { $wout.Dispose() } + } + finally { $win.Dispose() } + az storage blob upload --account-name $account --container-name $container --name $wdBlob --file $wdSnap --overwrite --auth-mode login --only-show-errors 2>$null + } + } + catch { + # Never let a streaming hiccup touch the build: log it into the snapshot's + # sidecar (visible on the VM) and back off a cycle before trying again. + "$([DateTime]::UtcNow.ToString('o')) live-log upload failed: $_" | + Add-Content -Path "$snap.err" -ErrorAction SilentlyContinue + Start-Sleep -Seconds 30 + } + } +} -ArgumentList $log, $StatusAccount, $StatusContainer, $liveBlob, $IdentityClientId, $env:Path, $wdLocal, $wdBlob + +$nuc = 'C:\worker-images\provisioners\windows\win-hw-wim\bin\WinHwWim\New-WinHwWim.ps1' +$rc = 0 +try { + $a = @('-Image', $Image) + if ($BuildId) { $a += @('-BuildId', $BuildId) } + if ($Stages) { $a += @('-Stages'); $a += ($Stages -split ',' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) } + if ($IdentityClientId) { $a += @('-IdentityClientId', $IdentityClientId) } + # Run in a child process so we get a reliable exit code (New-WinHwWim uses -EA Stop). + & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $nuc @a *>&1 | Tee-Object -FilePath $log + $rc = if ($null -eq $LASTEXITCODE) { 0 } else { $LASTEXITCODE } +} +catch { + $rc = 1 + $_ | Out-String | Add-Content -Path $log +} +if ($liveJob) { Stop-Job $liveJob -ErrorAction SilentlyContinue; Remove-Job $liveJob -Force -ErrorAction SilentlyContinue } +Set-Content -Path $done -Value $rc + +# --- Signal completion to the GH runner via BLOB storage ---------------------- +# The runner used to detect completion by polling this build.done over +# `az vm run-command`, but that extension WEDGES under the bake's heavy nested-virt +# load and left finished builds undetected (the job hung ~2h). Uploading a marker the +# runner reads with its OWN az (ci/kickoff-win-hw-wim-build.ps1) is immune to that. +# az is normally already logged in as the UAMI (New-WinHwWim), but log in defensively so +# we can ALWAYS signal - including on an early New-WinHwWim failure before its own login. +try { + if ($IdentityClientId) { + $prevEap = $ErrorActionPreference + $ErrorActionPreference = 'SilentlyContinue' + az account show 1>$null 2>$null + if ($LASTEXITCODE -ne 0) { az login --identity --client-id $IdentityClientId 1>$null 2>$null } + $ErrorActionPreference = $prevEap + } + # FINAL live-log push before the marker: the last streaming cycle can be up to a minute + # behind, and the interesting part of a failure is always the last few lines. Uploaded + # before .done so the runner's closing delta is complete the moment it sees the marker. + az storage blob upload --account-name $StatusAccount --container-name $StatusContainer --name $liveBlob --file $log --overwrite --auth-mode login --only-show-errors 2>$null + if (Test-Path $wdLocal) { + az storage blob upload --account-name $StatusAccount --container-name $StatusContainer --name $wdBlob --file $wdLocal --overwrite --auth-mode login --only-show-errors 2>$null + } + # A tail of the build log for visibility (uploaded first, so it's present when 'done' appears). + $statusLog = Join-Path $base 'status.log' + Get-Content $log -Tail 200 -ErrorAction SilentlyContinue | Set-Content -Path $statusLog -Encoding utf8 + az storage blob upload --account-name $StatusAccount --container-name $StatusContainer --name "_status/$Image.log" --file $statusLog --overwrite --auth-mode login --only-show-errors 2>$null + # Authoritative marker LAST, with a small retry (its content is the exit code). + for ($u = 1; $u -le 3; $u++) { + az storage blob upload --account-name $StatusAccount --container-name $StatusContainer --name "_status/$Image.done" --file $done --overwrite --auth-mode login --only-show-errors 2>$null + if ($LASTEXITCODE -eq 0) { break } + Start-Sleep -Seconds 10 + } +} +catch { + $_ | Out-String | Add-Content -Path $log +} diff --git a/provisioners/windows/win-hw-wim/scripts/sysprep-generalize.ps1 b/provisioners/windows/win-hw-wim/scripts/sysprep-generalize.ps1 new file mode 100644 index 00000000..f4840f9d --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/sysprep-generalize.ps1 @@ -0,0 +1,180 @@ +<# +.SYNOPSIS + Step 4 (runs INSIDE the build VM via Packer): scrub machine-specific state, + then Sysprep /generalize /oobe /shutdown so the disk can be captured clean. + +.DESCRIPTION + Removes everything that must NOT ship in a generalized golden image: + - the placeholder bake vault.yaml (secret hygiene) + - the bake registry identity (role/workerType/worker_pool_id) so first boot + re-seeds real values; leaves bootstrap_stage = 'setup' so deploy bootstraps + - SSH host keys and any generic-worker keys (none baked, defensive) + - build-only autologon + Then runs Sysprep. The classic Win11 failure here is per-user AppX left behind, + so we assert none remain (the bake removed provisioned packages) and surface + Panther logs on failure. + + IMPORTANT: capture must happen AFTER this scrub (it does — capture is a + post-build step in win-hw-wim.pkr.hcl). +#> +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +function Step($m) { Write-Host "== $m ==" } + +# Signal the host-side boot watchdog (New-WinHwWim.ps1) to STOP restarting the guest: +# from here we deliberately Sysprep /shutdown, and that power-off is what Packer waits +# for to capture the VHDX. Write-Output (not Write-Host) so it reaches Packer's captured +# stdout stream and lands in packer-build.log where the watchdog greps for it. +Write-Output 'WIM-WATCHDOG-STOP: sysprep starting; the guest power-off from here is expected (capture).' + +# --- Secret + identity scrub --- +Step 'Scrubbing bake secrets and identity' +Remove-Item 'C:\ronin\data\secrets\vault.yaml' -Force -ErrorAction SilentlyContinue +# Remove the whole bake work dir. It holds the SYSTEM puppet-apply helper +# (C:\bake\run-puppet-system.ps1) which EMBEDS the build-scoped GitHub token +# (custom_win_github_pat) when one is supplied, plus prereq installers and puppet logs. +# None of it belongs in the golden WIM — drop it before capture. +Remove-Item 'C:\bake' -Recurse -Force -ErrorAction SilentlyContinue +$ron = 'HKLM:\SOFTWARE\Mozilla\ronin_puppet' +if (Test-Path $ron) { + foreach ($v in 'role','workerType','worker_pool_id','GITHASH','secret_date') { + Remove-ItemProperty -Path $ron -Name $v -ErrorAction SilentlyContinue + } + # Leave a clean state so the deploy-time bootstrap runs. + Set-ItemProperty -Path $ron -Name bootstrap_stage -Value 'setup' -Type String + Set-ItemProperty -Path $ron -Name hand_off_ready -Value 'no' -Type String +} + +# --- SSH host keys (regenerated at first boot) --- +Step 'Removing SSH host keys' +Remove-Item 'C:\ProgramData\ssh\ssh_host_*' -Force -ErrorAction SilentlyContinue + +# --- Disable the build-only autologon --- +Step 'Disabling autologon' +$wl = 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon' +Set-ItemProperty -Path $wl -Name AutoAdminLogon -Value '0' -ErrorAction SilentlyContinue +Remove-ItemProperty -Path $wl -Name DefaultPassword -ErrorAction SilentlyContinue + +# --- Enable the built-in Administrator so the DEPLOY-time autologon works --- +# The whole first-boot kickoff on the deployed node runs through the production +# base-autounattend.xml oobeSystem pass: logs in Administrator ONCE +# (LogonCount=1) and - which run only inside that interactive +# logon - seed C:\bootstrap + vault.yaml and launch Get-Bootstrap.ps1 (whose final +# `psexec -i` also needs that interactive session). OS-deploy.ps1 already stages that +# unattend to \Windows\Panther and substitutes the real Administrator password +# (win_adminpw) into it. BUT the built-in Administrator is DISABLED by default and this +# bake ran as the 'packer' account, so it was never enabled - the deploy autologon then +# fails and the node sits at the login screen (nuc13-160's "packer login"), FirstLogonCommands +# never fire, and nothing bootstraps. Enable it here so the deploy autologon can take. +# (No password is set in the image; oobeSystem sets it to win_adminpw before any logon.) +Step 'Enabling built-in Administrator for deploy-time autologon' +& net.exe user Administrator /active:yes +if ($LASTEXITCODE -ne 0) { Write-Warning "net user Administrator /active:yes returned $LASTEXITCODE" } + +# --- Remove the build-only first-logon network helper --- +# set-bake-network.ps1 (dropped by prepare-base-vhdx.ps1 and invoked by the build +# unattend's FirstLogonCommands) is build-only. It is inert in a deployed image (a +# bare .ps1 in Setup\Scripts is not auto-run; only SetupComplete.cmd is), but it must +# not ship in the golden WIM. Also drop its log. +Step 'Removing build-only network helper' +Remove-Item 'C:\Windows\Setup\Scripts\set-bake-network.ps1' -Force -ErrorAction SilentlyContinue +Remove-Item 'C:\Windows\Temp\bake-network.log' -Force -ErrorAction SilentlyContinue +# The helper self-registers a SYSTEM 'BakeNetwork' startup task (re-asserts network/WinRM +# on every boot so mid-bake restarts don't strand Packer). Build-only - unregister it so +# it never ships in the golden WIM. +Unregister-ScheduledTask -TaskName 'BakeNetwork' -Confirm:$false -ErrorAction SilentlyContinue + +# NOTE: there is deliberately no pre-Sysprep leftover-AppX enumeration here. The bake +# disables AppXSvc (win_disable_services::disable_appxsvc), and Get-AppxPackage needs that +# service, so any such check post-bake can only ever fail/no-op. If a per-user AppX +# Sysprep blocker is ever suspected, check it inside bake-bootstrap.ps1 (right after +# puppet apply) where AppXSvc is still running. + +# --- Bake the first-boot bootstrap runner (SYSTEM startup task) --- +# Registered HERE, as the last thing before Sysprep, so it CANNOT run during the bake: +# the bake VM only ever shuts down from this point (no more boots before capture). On the +# DEPLOYED node's first boot it reproduces the shape the unattend FirstLogonCommands would +# have (seed C:\bootstrap + vault.yaml, disable sleep) and launches Get-Bootstrap.ps1 +# (which OS-deploy.ps1 stages on D:\ with the pool params already substituted). One-shot: +# it flags + unregisters itself once it has launched Get-Bootstrap, so it does not re-run +# on later ronin reboots (matches the production one-shot FirstLogonCommands launcher). +Step 'Baking first-boot bootstrap runner (RunDeployBootstrap startup task)' +$deployDir = 'C:\deploy' +New-Item -ItemType Directory -Path $deployDir -Force | Out-Null +$runner = @' +$ErrorActionPreference = 'Continue' +$log = 'C:\deploy\run-bootstrap.log' +function L($m) { ('{0} {1}' -f (Get-Date -Format o), $m) | Tee-Object -FilePath $log -Append | Out-Null } +$flag = 'C:\deploy\.bootstrap-launched' +if (Test-Path $flag) { + L 'bootstrap already launched on a prior boot; unregistering task and exiting' + Unregister-ScheduledTask -TaskName 'RunDeployBootstrap' -Confirm:$false -ErrorAction SilentlyContinue + return +} +L 'run-bootstrap: start' +# OS-deploy.ps1 stages Get-Bootstrap.ps1 (templated with the pool params) to D:\scripts in +# WinPE before this boot; wait for D: to mount and the script to appear. +$gb = 'D:\scripts\Get-Bootstrap.ps1' +for ($i = 0; $i -lt 60 -and -not (Test-Path $gb); $i++) { Start-Sleep -Seconds 10 } +if (-not (Test-Path $gb)) { L "ERROR: $gb not found; leaving task registered to retry next boot"; return } +# FirstLogonCommands-equivalent prerequisites (base-autounattend.xml oobeSystem pass): +if (-not (Test-Path 'C:\bootstrap')) { New-Item -ItemType Directory -Path 'C:\bootstrap' -Force | Out-Null } +if (Test-Path 'D:\secrets\vault.yaml') { Copy-Item 'D:\secrets\vault.yaml' 'C:\bootstrap\' -Force } +powercfg -x -standby-timeout-ac 0 2>$null +powercfg -x -monitor-timeout-ac 0 2>$null +L 'run-bootstrap: launching Get-Bootstrap.ps1' +& powershell.exe -NoProfile -ExecutionPolicy Bypass -File $gb *>&1 | Tee-Object -FilePath $log -Append +L ('run-bootstrap: Get-Bootstrap returned rc=' + $LASTEXITCODE) +# Mark launched + one-shot self-remove; bootstrap.ps1 continues via ronin's own task. +New-Item -Path $flag -ItemType File -Force | Out-Null +Unregister-ScheduledTask -TaskName 'RunDeployBootstrap' -Confirm:$false -ErrorAction SilentlyContinue +'@ +$utf8NoBomRunner = New-Object System.Text.UTF8Encoding($false) +[System.IO.File]::WriteAllText((Join-Path $deployDir 'run-bootstrap.ps1'), $runner, $utf8NoBomRunner) +$rdbAction = New-ScheduledTaskAction -Execute 'powershell.exe' -Argument '-NoProfile -ExecutionPolicy Bypass -File C:\deploy\run-bootstrap.ps1' +$rdbTrigger = New-ScheduledTaskTrigger -AtStartup +try { $rdbTrigger.Delay = 'PT2M' } catch { } # best-effort boot delay; wrapper also waits for D:/network +$rdbPrincipal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest +$rdbSettings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable -ExecutionTimeLimit (New-TimeSpan -Hours 3) +Register-ScheduledTask -TaskName 'RunDeployBootstrap' -Action $rdbAction -Trigger $rdbTrigger -Principal $rdbPrincipal -Settings $rdbSettings -Force | Out-Null +Write-Host ' RunDeployBootstrap SYSTEM startup task registered (fires only on the deployed node).' + +# --- Bake hygiene: neutralize the build-only 'packer' admin account ([[bake-hygiene-todo]]) --- +# The build unattend created an Administrators-group account (@@WINRM_USER@@, i.e. 'packer') +# for Packer/WinRM. It must not remain a usable admin in the golden WIM. We DISABLE it rather +# than delete it: this script runs AS that account (over WinRM), and you cannot delete the +# account you are currently logged in as. Disabling fully neutralizes it - a disabled account +# cannot log in locally, over WinRM, or via SSH - which is the security goal. (Deletion, if +# ever wanted, must happen as SYSTEM on the deployed node where packer isn't logged in.) +# This is the LAST WinRM-affecting step before Sysprep /shutdown; no provisioner runs after it. +Step 'Disabling build-only packer account' +$buildAcct = $env:USERNAME # the account this provisioner runs under IS the build account +& net.exe user $buildAcct /active:no +if ($LASTEXITCODE -ne 0) { Write-Warning "net user $buildAcct /active:no returned $LASTEXITCODE" } + +# Clear any autologon the bake configured. The build 'packer' account auto-logs in during the bake, +# leaving AutoAdminLogon/DefaultUserName/DefaultPassword/AutoLogonSID in Winlogon. Left in the golden +# image these collide with the deploy-time Administrator autologon and generic-worker's task-user +# autologon (observed on nuc13-160/024: AutoLogonSID still pointed at the baked packer account, and the +# task user never logged in). Scrub them so the deployed node starts with no baked autologon. +Step 'Clearing baked autologon (Winlogon) keys' +$wl = 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon' +Set-ItemProperty -Path $wl -Name 'AutoAdminLogon' -Value '0' -Force +foreach ($v in 'DefaultUserName', 'DefaultPassword', 'DefaultDomainName', 'AutoLogonSID', 'AutoLogonCount') { + Remove-ItemProperty -Path $wl -Name $v -ErrorAction SilentlyContinue +} + +# --- Sysprep generalize + shutdown --- +Step 'Running Sysprep /generalize /oobe /shutdown' +$sp = "$env:SystemRoot\System32\Sysprep" +Remove-Item (Join-Path $sp 'unattend.xml') -Force -ErrorAction SilentlyContinue +& (Join-Path $sp 'Sysprep.exe') /generalize /oobe /shutdown /quiet +if ($LASTEXITCODE -ne 0) { + Write-Warning "Sysprep returned rc=$LASTEXITCODE - dumping Panther errors:" + Get-Content (Join-Path $sp 'Panther\setuperr.log') -ErrorAction SilentlyContinue | Select-Object -Last 40 + throw "Sysprep failed rc=$LASTEXITCODE" +} +# On success the VM powers off; Packer finalizes the artifact. diff --git a/provisioners/windows/win-hw-wim/scripts/unattend/set-bake-network.ps1 b/provisioners/windows/win-hw-wim/scripts/unattend/set-bake-network.ps1 new file mode 100644 index 00000000..74cfe327 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/unattend/set-bake-network.ps1 @@ -0,0 +1,104 @@ +# set-bake-network.ps1 — build-only network bring-up for the bake VM. +# +# Dropped into the base image at C:\Windows\Setup\Scripts\ by prepare-base-vhdx.ps1 +# and invoked from the unattend's FirstLogonCommands (oobeSystem). It assigns the +# static IP the NAT switch expects (no DHCP on wim-nat) and re-asserts the WinRM +# HTTP listener + firewall so Packer can connect. +# +# Why here and not (only) in the specialize pass: during specialize the NIC is +# frequently not yet enumerated/"Up", so New-NetIPAddress silently no-ops and the +# guest is left on an APIPA 169.254.x.x address that the host can't reach — Packer +# then hangs forever at "Waiting for WinRM". Running at first logon (network stack +# fully initialized) with a retry loop makes the assignment deterministic. +# +# All of this is build-only and scrubbed by sysprep-generalize.ps1 before capture, +# so it never ships in the golden WIM. + +$ip = '192.168.234.10' +$pfx = 24 +$gw = '192.168.234.1' +$log = 'C:\Windows\Temp\bake-network.log' + +function Write-BakeLog([string] $m) { + "{0} {1}" -f (Get-Date -Format o), $m | Out-File -FilePath $log -Append -Encoding utf8 +} + +Write-BakeLog 'set-bake-network: start' + +# Re-assert this bring-up on EVERY boot, not just first logon. The bake's windows-restart +# provisioners reboot the guest mid-build; the NAT NIC then re-classifies as 'Public' and +# NTLM WinRM stops answering, so Packer times out at "waiting for machine to restart". +# A SYSTEM startup task re-runs this script each boot so WinRM is back before Packer +# reconnects. Idempotent (self-registers on first logon); removed by sysprep-generalize.ps1. +$self = if ($PSCommandPath) { $PSCommandPath } else { 'C:\Windows\Setup\Scripts\set-bake-network.ps1' } +try { + $act = New-ScheduledTaskAction -Execute 'powershell.exe' -Argument ('-NoProfile -ExecutionPolicy Bypass -File "{0}"' -f $self) + $trg = New-ScheduledTaskTrigger -AtStartup + $pri = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest + $st = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable + Register-ScheduledTask -TaskName 'BakeNetwork' -Action $act -Trigger $trg -Principal $pri -Settings $st -Force | Out-Null + Write-BakeLog 'registered BakeNetwork startup task (re-asserts network/WinRM on every boot)' +} +catch { Write-BakeLog ('WARN: could not register BakeNetwork startup task: ' + $_.Exception.Message) } + +$assigned = $false +for ($i = 0; $i -lt 60; $i++) { + $a = Get-NetAdapter -Physical -ErrorAction SilentlyContinue | Where-Object { $_.Status -eq 'Up' } | Select-Object -First 1 + if (-not $a) { $a = Get-NetAdapter -ErrorAction SilentlyContinue | Where-Object { $_.Status -eq 'Up' } | Select-Object -First 1 } + if ($a) { + Write-BakeLog ("adapter={0} ifIndex={1} status={2}" -f $a.Name, $a.ifIndex, $a.Status) + if (-not (Get-NetIPAddress -InterfaceIndex $a.ifIndex -IPAddress $ip -ErrorAction SilentlyContinue)) { + # drop any self-assigned APIPA lease first so the static add is clean + Get-NetIPAddress -InterfaceIndex $a.ifIndex -AddressFamily IPv4 -ErrorAction SilentlyContinue | + Where-Object { $_.IPAddress -like '169.254.*' } | + Remove-NetIPAddress -Confirm:$false -ErrorAction SilentlyContinue + New-NetIPAddress -InterfaceIndex $a.ifIndex -IPAddress $ip -PrefixLength $pfx -DefaultGateway $gw -ErrorAction SilentlyContinue | Out-Null + Set-DnsClientServerAddress -InterfaceIndex $a.ifIndex -ServerAddresses '8.8.8.8', '1.1.1.1' -ErrorAction SilentlyContinue + } + if (Get-NetIPAddress -InterfaceIndex $a.ifIndex -IPAddress $ip -ErrorAction SilentlyContinue) { + Write-BakeLog ("static IP {0}/{1} gw {2} assigned" -f $ip, $pfx, $gw) + $assigned = $true + break + } + } + Start-Sleep -Seconds 2 +} + +if (-not $assigned) { Write-BakeLog 'WARNING: never assigned static IP (no Up adapter?)' } + +# Classify the NAT link as Private. This is THE critical WinRM enabler: the NAT link +# comes up 'Public' (no gateway/domain for NLA to identify), and on a Public network +# NTLM auth to a local account fails with 0x8009030d ("A specified logon session does +# not exist") — which is exactly what makes Packer hang at "Waiting for WinRM". With the +# profile Private (+ LocalAccountTokenFilterPolicy set in the specialize pass), NTLM to +# the local build account works. Set-NetConnectionProfile only takes once NLA has +# actually categorized the adapter, which lags first logon, so RETRY until it sticks. +for ($j = 0; $j -lt 30; $j++) { + Get-NetConnectionProfile -ErrorAction SilentlyContinue | ForEach-Object { + Set-NetConnectionProfile -InterfaceIndex $_.InterfaceIndex -NetworkCategory Private -ErrorAction SilentlyContinue + } + $cats = @(Get-NetConnectionProfile -ErrorAction SilentlyContinue | Select-Object -Expand NetworkCategory) + if ($cats.Count -gt 0 -and -not ($cats | Where-Object { $_ -ne 'Private' })) { break } + Start-Sleep -Seconds 3 +} +Write-BakeLog ("network profile(s): " + ((Get-NetConnectionProfile -ErrorAction SilentlyContinue | ForEach-Object { $_.Name + '=' + $_.NetworkCategory }) -join ', ')) + +# Belt-and-suspenders: WinRM service policy keys enable Basic + unencrypted regardless +# of the network profile (they bypass the interactive 'network is Public' guard), so a +# Basic-auth fallback also works if NTLM/Private ever fails. Harmless with NTLM. +$winrmPol = 'HKLM:\SOFTWARE\Policies\Microsoft\Windows\WinRM\Service' +New-Item -Path $winrmPol -Force -ErrorAction SilentlyContinue | Out-Null +Set-ItemProperty -Path $winrmPol -Name AllowBasic -Value 1 -Type DWord -Force -ErrorAction SilentlyContinue +Set-ItemProperty -Path $winrmPol -Name AllowUnencryptedTraffic -Value 1 -Type DWord -Force -ErrorAction SilentlyContinue + +# Bring up the WinRM HTTP listener. Explicit `winrm create Listener` (unlike +# Enable-PSRemoting / winrm quickconfig) does NOT check the network-connection profile, +# so it works regardless. Idempotent. +cmd.exe /c 'sc config WinRM start= auto' | Out-Null +cmd.exe /c 'net start WinRM' 2>$null | Out-Null +cmd.exe /c 'winrm create winrm/config/Listener?Address=*+Transport=HTTP' 2>$null | Out-Null +netsh advfirewall firewall add rule name="WinRM-HTTP-In-5985" dir=in action=allow protocol=TCP localport=5985 | Out-Null + +$ok = $false +try { $ok = [bool](Get-NetFirewallRule -DisplayName 'WinRM-HTTP-In-5985' -ErrorAction SilentlyContinue) } catch {} +Write-BakeLog ("set-bake-network: done (ip_assigned={0}, fw_rule={1})" -f $assigned, $ok) diff --git a/provisioners/windows/win-hw-wim/scripts/unattend/unattend.xml.template b/provisioners/windows/win-hw-wim/scripts/unattend/unattend.xml.template new file mode 100644 index 00000000..858f2c72 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/unattend/unattend.xml.template @@ -0,0 +1,102 @@ + + + + + + @@COMPUTERNAME@@ + + + + + + + 1 + reg.exe add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\OOBE" /v BypassNRO /t REG_DWORD /d 1 /f + + + + 2 + reg.exe add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System" /v LocalAccountTokenFilterPolicy /t REG_DWORD /d 1 /f + + + + + + + + + 0409:00000409 + en-US + en-US + en-US + + + + + + @@WINRM_USER@@ + Administrators + + @@WINRM_PASSWORD@@ + true</PlainText> + </Password> + </LocalAccount> + </LocalAccounts> + </UserAccounts> + <AutoLogon> + <Enabled>true</Enabled> + <Username>@@WINRM_USER@@</Username> + <Password> + <Value>@@WINRM_PASSWORD@@</Value> + <PlainText>true</PlainText> + </Password> + <LogonCount>2</LogonCount> + </AutoLogon> + <OOBE> + <ProtectYourPC>3</ProtectYourPC> + <HideEULAPage>true</HideEULAPage> + <HideWirelessSetupInOOBE>true</HideWirelessSetupInOOBE> + </OOBE> + <!-- All build-only network bring-up happens here at first logon, where the network + stack is fully up and there is no unattend length limit: set-bake-network.ps1 + (dropped into Setup\Scripts by prepare-base-vhdx.ps1) assigns the static NAT IP + (retry loop — the NIC isn't reliably Up during specialize) and creates the WinRM + HTTP listener via explicit `winrm create Listener` (profile-independent, unlike + Enable-PSRemoting). Autologon fires this; the packer account below is what Packer + authenticates as. Specialize is kept minimal (reg.exe only) so it always parses. --> + <FirstLogonCommands> + <SynchronousCommand wcm:action="add"> + <Order>1</Order> + <CommandLine>powershell.exe -NoProfile -ExecutionPolicy Bypass -File "C:\Windows\Setup\Scripts\set-bake-network.ps1"</CommandLine> + <Description>Assign static NAT IP and ensure WinRM (build-only)</Description> + </SynchronousCommand> + </FirstLogonCommands> + </component> + </settings> +</unattend> diff --git a/provisioners/windows/win-hw-wim/scripts/upload-wim.ps1 b/provisioners/windows/win-hw-wim/scripts/upload-wim.ps1 new file mode 100644 index 00000000..17628c43 --- /dev/null +++ b/provisioners/windows/win-hw-wim/scripts/upload-wim.ps1 @@ -0,0 +1,50 @@ +<# +.SYNOPSIS + Upload a WIM (+ its .sha256) to the private Windows HW WIM storage account. + +.DESCRIPTION + Uses azcopy with Entra auth (--auth-mode login). Storage is Entra-only (no IP + firewall, no keys): the caller needs an Entra identity with a Storage Blob Data + Contributor role — the build VM's managed identity, the uploader SP, or a Relops + member (az login / az login --identity first). + +.PARAMETER Wim + Local WIM path (e.g. .\output\install.wim). + +.PARAMETER Container + Target container: 'captured' (default) or 'base'. + +.PARAMETER Account + Storage account name (default hardwareimaging — the Terraform output). + +.EXAMPLE + az login --service-principal -u $env:AZ_CLIENT_ID -p $env:AZ_CLIENT_SECRET --tenant $env:AZ_TENANT + .\upload-wim.ps1 -Wim .\output\install.wim -Container captured +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $Wim, + [ValidateSet('captured','resources')] [string] $Container = 'captured', + [string] $Account = 'hardwareimaging', + # Blob name within the container. Default = the file's leaf name. Set this to + # namespace the output, e.g. 'WIMs/win11-24h2-hw/win11-24h2-hw-20260723.wim'. + [string] $BlobName +) +$ErrorActionPreference = 'Stop' +if (-not (Test-Path -LiteralPath $Wim)) { throw "WIM not found: $Wim" } +if (-not (Get-Command azcopy -ErrorAction SilentlyContinue)) { throw 'azcopy not on PATH.' } + +if (-not $BlobName) { $BlobName = Split-Path -Leaf $Wim } +# azcopy has its own credential store — it does NOT inherit `az login`. Tell it to +# reuse the az CLI identity (the build VM's managed identity, an SP, or a user). +if (-not $env:AZCOPY_AUTO_LOGIN_TYPE) { $env:AZCOPY_AUTO_LOGIN_TYPE = 'AZCLI' } +$base = "https://$Account.blob.core.windows.net/$Container" +# Upload the WIM and its .sha256 sidecar under the same blob name. +foreach ($pair in @(@{ Src = $Wim; Dest = $BlobName }, @{ Src = "$Wim.sha256"; Dest = "$BlobName.sha256" })) { + if (Test-Path -LiteralPath $pair.Src) { + Write-Host "== Uploading $($pair.Src) -> $base/$($pair.Dest) ==" + & azcopy copy "$($pair.Src)" "$base/$($pair.Dest)" --overwrite=ifSourceNewer + if ($LASTEXITCODE -ne 0) { throw "azcopy upload failed rc=$LASTEXITCODE ($($pair.Dest))" } + } +} +Write-Host "== Done. ==" diff --git a/provisioners/windows/win-hw-wim/variables.pkr.hcl b/provisioners/windows/win-hw-wim/variables.pkr.hcl new file mode 100644 index 00000000..3f960e4c --- /dev/null +++ b/provisioners/windows/win-hw-wim/variables.pkr.hcl @@ -0,0 +1,143 @@ +# Input variables for the Windows HW baked-WIM Packer build. +# Copy example.auto.pkrvars.hcl to <name>.auto.pkrvars.hcl and set values. + +variable "source_vm_name" { + type = string + description = "Name of the pristine Gen2 Hyper-V VM built around the base VHDX (created by register-base-vm.ps1). Packer clones from this VM so it stays untouched." +} + +variable "switch_name" { + type = string + description = "Hyper-V virtual switch the build VM attaches to (must reach the internet for Puppet/choco)." +} + +variable "winrm_username" { + type = string + default = "packer" + description = "Local admin account created by the injected unattend for Packer WinRM." +} + +variable "winrm_password" { + type = string + sensitive = true + description = "Password for the build-only WinRM account (build-scoped; not baked into the final image)." +} + +variable "github_pat" { + type = string + default = "" + sensitive = true + description = "Build-scoped GitHub token for puppet's tooltool download (exposed to the bake as env custom_win_github_pat). Empty is OK — tooltool.py is public. Not baked into the WIM." +} + +variable "windows_update" { + type = bool + default = false + description = "Run a full online Windows Update pass during the bake. Default OFF (fast iteration); production images set this true via config. See the windows-update provisioner in win-hw-wim.pkr.hcl." +} + +variable "cpus" { + type = number + default = 4 +} + +variable "memory_mb" { + type = number + default = 8192 +} + +# --- ronin bake inputs (passed to bake-bootstrap.ps1) --- + +variable "ronin_org" { + type = string + default = "mozilla-platform-ops" +} + +variable "ronin_repo" { + type = string + default = "ronin_puppet" +} + +variable "ronin_branch" { + type = string + description = "Branch carrying the win116424h2hwbake role (feature branch; do not use main until merged)." +} + +variable "ronin_hash" { + type = string + default = "" + description = "Optional pinned commit to checkout after clone. Empty = branch HEAD." +} + +variable "bake_role" { + type = string + default = "win116424h2hwbake" +} + +variable "puppet_version" { + type = string +} + +variable "git_version" { + type = string +} + +variable "openvox_version" { + type = string + default = "" +} + +# Ronin's public assets blob that hosts the pinned prerequisite installers under +# /binaries/prerequisites (e.g. openvox-agent-<ver>-x64.msi, puppet-agent-<ver>-x64.msi). +# Same source Get-PreRequ uses in worker-images MDC1Windows/bootstrap.ps1. +variable "ronin_ext_src" { + type = string + default = "https://roninpuppetassets.blob.core.windows.net/binaries/prerequisites" +} + +variable "output_directory" { + type = string + default = "./output/build" + description = "Where Packer writes the cloned VM + generalized VHDX." +} + +variable "temp_path" { + type = string + default = "" + description = "Where the hyperv builder creates the working VM (clone + its RAM-sized memory-state file). Must be on a disk large enough for the VHDX clone + the VM's memory file (= memory_mb). Set to the big data disk; empty = system temp (C:, too small)." +} + +variable "output_wim" { + type = string + default = "./output/install.wim" + description = "Path the capture post-processor writes the golden WIM to. Set per-image by the orchestrator (e.g. work/<image>/<image>-<buildid>.wim) so parallel/repeat builds never collide." +} + +variable "capture_name" { + type = string + default = "nuc-ci-baked" + description = "DISM /Name metadata written into the captured image." +} + +# --- Release notes / SBOM ------------------------------------------------------ +# Mirrors what azure.pkr.hcl does for the Azure gallery images: Set-ReleaseNotes (from +# the BootStrap module) writes C:\<image_name>-<build_id>.md in the guest and Packer +# downloads it. Kept as two variables rather than reusing capture_name because +# Set-ReleaseNotes takes -Config and -Version separately and derives the filename itself. +variable "image_name" { + type = string + default = "" + description = "Image/config id (e.g. win11-24h2-hw) - the -Config half of the release-notes filename." +} + +variable "build_id" { + type = string + default = "" + description = "Build stamp (e.g. 20260811-164648) - the -Version half of the release-notes filename." +} + +variable "sbom_path" { + type = string + default = "" + description = "Host path Packer downloads the guest's release-notes markdown to. Empty = skip generation (set per-image by the orchestrator)." +} diff --git a/provisioners/windows/win-hw-wim/win-hw-wim.pkr.hcl b/provisioners/windows/win-hw-wim/win-hw-wim.pkr.hcl new file mode 100644 index 00000000..0d409045 --- /dev/null +++ b/provisioners/windows/win-hw-wim/win-hw-wim.pkr.hcl @@ -0,0 +1,179 @@ +# ============================================================================= +# win-hw-wim.pkr.hcl — Standalone Hyper-V bake for a golden Windows HW install.wim +# +# Flow: clone a pristine VM (built from your BYO base VHDX) -> WinRM in -> +# run the ronin BAKE role -> Sysprep /generalize /shutdown -> +# (post-build) capture the generalized VHDX to install.wim. +# +# NOT related to worker-images/azure.pkr.hcl. No azure-arm source, no gallery. +# Modeled on the provisioner ORDER of azure.pkr.hcl (bootstrap -> puppet -> +# restart -> sysprep) but with a Hyper-V source and a WIM capture output. +# ============================================================================= + +packer { + required_plugins { + hyperv = { + source = "github.com/hashicorp/hyperv" + version = ">= 1.1.3" + } + windows-update = { + source = "github.com/rgl/windows-update" + version = ">= 0.16.0" + } + } +} + +source "hyperv-vmcx" "nuc" { + # Clone from the pristine base VM (register-base-vm.ps1 wraps the prepared VHDX). + clone_from_vm_name = var.source_vm_name + + generation = 2 + cpus = var.cpus + memory = var.memory_mb + switch_name = var.switch_name + output_directory = var.output_directory + # Build the working VM (clone + its RAM-sized memory-state file) on the big data + # disk; the default (C: system temp) is too small for a 32 GB memory file. + temp_path = var.temp_path + + # Gen2 UEFI. Secure Boot template must match how the base VHDX was prepared. + enable_secure_boot = true + secure_boot_template = "MicrosoftWindows" + + # WinRM — the HTTP listener + static NAT IP are set up at first logon by + # scripts/unattend/set-bake-network.ps1 (dropped in by prepare-base-vhdx.ps1). + # Use NTLM (message-encrypted), NOT Basic-over-HTTP: the NAT link is classified + # a 'Public' network, and WinRM refuses to enable AllowUnencrypted there (the + # firewall-exception guard blocks it), so Basic/plaintext auth can't be turned + # on. NTLM needs no AllowUnencrypted and works with the local build account. + communicator = "winrm" + winrm_username = var.winrm_username + winrm_password = var.winrm_password + winrm_use_ntlm = true + winrm_timeout = "60m" + + # Sysprep in the last provisioner shuts the VM down; let Packer treat that as done. + shutdown_timeout = "30m" +} + +build { + name = "win-hw-wim-bake" + sources = ["source.hyperv-vmcx.nuc"] + + # ---- 1. Windows updates (bake them in, so deploy doesn't fight WU) ---- + # Controlled by var.windows_update (per-image config; default OFF for fast + # iteration, ON for production). Packer HCL can't conditionally include a + # provisioner, so when disabled we search already-installed updates and exclude + # everything -> the provisioner finds nothing to install and returns in seconds. + provisioner "windows-update" { + # Enabled: latest applicable, not-yet-installed, non-Preview KBs (single pass). + # Disabled: a no-op search that installs nothing. + search_criteria = var.windows_update ? "IsInstalled=0" : "IsInstalled=1" + filters = var.windows_update ? [ + "exclude:$_.Title -like '*Preview*'", + "include:$true", + ] : ["exclude:$true"] + } + # 60m, not the 30m default-ish value we started with: on a windows_update=true + # image this reboot runs the "Working on updates" apply pass through BOTH shutdown + # and boot, and on this nested-virt guest the reboot also lands as a full power-off + # that the host watchdog has to restart (see New-WinHwWim.ps1). 30m was not enough. + provisioner "windows-restart" { + restart_timeout = "60m" + } + + # ---- 2. Upload bake scripts ---- + provisioner "file" { + source = "${path.root}/scripts/" + destination = "C:/wim-bake/" + } + + # ---- 3. Bake: install puppet/git, clone ronin, AppX (provisioned) removal, puppet apply of the BAKE role ---- + provisioner "powershell" { + elevated_user = var.winrm_username + elevated_password = var.winrm_password + environment_vars = [ + "RONIN_ORG=${var.ronin_org}", + "RONIN_REPO=${var.ronin_repo}", + "RONIN_BRANCH=${var.ronin_branch}", + "RONIN_HASH=${var.ronin_hash}", + "BAKE_ROLE=${var.bake_role}", + "PUPPET_VERSION=${var.puppet_version}", + "GIT_VERSION=${var.git_version}", + "OPENVOX_VERSION=${var.openvox_version}", + "RONIN_EXT_SRC=${var.ronin_ext_src}", + # Build-scoped GitHub token for puppet's tooltool download (ronin fact + # custom_win_github_pat falls back to this env var when there is no D: secrets + # drive). Empty is fine — tooltool.py is public and the download works without a + # token. Never written to disk / not captured into the WIM. + "custom_win_github_pat=${var.github_pat}", + ] + scripts = ["${path.root}/scripts/bake-bootstrap.ps1"] + # puppet apply returns 2 when it applied changes — that is success here. + valid_exit_codes = [0, 2] + } + + # 60m for the same reason as the post-WU restart above: run 31428853582 + # (win11-24h2-hw, the first windows_update=true bake) timed out here at 30m with + # "A system shutdown is in progress.(1115)" -> "Timeout waiting for machine to + # restart" AFTER a clean puppet apply, losing the whole 1h41m build. + provisioner "windows-restart" { + restart_timeout = "60m" + } + + # ---- 3b. Release notes / SBOM (same mechanism as the Azure gallery images) ---- + # azure.pkr.hcl drops the BootStrap module into the guest's module path, calls + # Set-ReleaseNotes, and downloads the markdown it writes; the workflow then commits it + # to sboms/ on main via .github/workflows/upload-release-notes.yml. Same three steps + # here, so a baked WIM gets an inventory the same way a gallery image does. + # Runs AFTER puppet (so the catalog's software is installed) and BEFORE sysprep. + # NOTE the coverage difference vs a gallery image: the bake role excludes + # windows_worker_runner, so generic-worker/worker-runner are installed at DEPLOY time + # and cannot appear here. This inventories what the WIM actually ships. + provisioner "file" { + source = "${path.root}/../../../scripts/windows/CustomFunctions/Bootstrap" + destination = "C:/Windows/System32/WindowsPowerShell/v1.0/Modules/" + max_retries = 3 + } + + provisioner "powershell" { + elevated_user = var.winrm_username + elevated_password = var.winrm_password + environment_vars = [ + "sbom_config=${var.image_name}", + "sbom_version=${var.build_id}", + "src_organisation=${var.ronin_org}", + "src_Repository=${var.ronin_repo}", + "src_Branch=${var.ronin_branch}", + # The gallery images pass a deploymentId here; the ronin commit is our equivalent - + # it is what pins the puppet content this image was baked from. + "deploymentId=${var.ronin_hash}", + ] + inline = [ + "Import-Module BootStrap -Force;", + "Set-ReleaseNotes -Config $ENV:sbom_config -Version $ENV:sbom_version -Organization $ENV:src_organisation -Branch $ENV:src_Branch -Repository $ENV:src_Repository -DeploymentId $ENV:deploymentId" + ] + } + + provisioner "file" { + source = "C:/${var.image_name}-${var.build_id}.md" + destination = var.sbom_path + direction = "download" + max_retries = 3 + } + + # ---- 4. Scrub machine-specific state + Sysprep /generalize /shutdown ---- + # (this powers the VM off; Packer then finalizes the artifact) + provisioner "powershell" { + elevated_user = var.winrm_username + elevated_password = var.winrm_password + scripts = ["${path.root}/scripts/sysprep-generalize.ps1"] + } + + # ---- 5. Capture the generalized VHDX -> golden WIM (runs on the host) ---- + post-processor "shell-local" { + inline = [ + "powershell -NoProfile -ExecutionPolicy Bypass -File \"${path.root}/scripts/capture-wim.ps1\" -BuildDir \"${var.output_directory}\" -OutWim \"${var.output_wim}\" -Name \"${var.capture_name}\"" + ] + } +} diff --git a/scripts/windows/CustomFunctions/Bootstrap/Public/Get-GenericWorkerVersion.ps1 b/scripts/windows/CustomFunctions/Bootstrap/Public/Get-GenericWorkerVersion.ps1 index 85af6da4..789124b4 100644 --- a/scripts/windows/CustomFunctions/Bootstrap/Public/Get-GenericWorkerVersion.ps1 +++ b/scripts/windows/CustomFunctions/Bootstrap/Public/Get-GenericWorkerVersion.ps1 @@ -8,6 +8,16 @@ function Get-GenericWorkerVersion { $StandardOutput = "C:\gwversion.txt" ) + # Not every image ships the Taskcluster binaries: the win-hw-wim bake role excludes + # windows_worker_runner (generic-worker + worker-runner are installed at DEPLOY time), + # and Start-Process on a missing path throws a TERMINATING error, which Set-ReleaseNotes' + # trap rethrows - killing the build right before Sysprep. Report nothing instead, so the + # release notes just omit the row. No change where the binary exists. + if (-not (Test-Path $FilePath)) { + Write-Verbose ('{0}: {1} not present; omitting from the release notes' -f $MyInvocation.MyCommand.Name, $FilePath) + return + } + ## Generic Worker Start-Process -FilePath $FilePath -ArgumentList "--short-version" -RedirectStandardOutput $StandardOutput -Wait -NoNewWindow [PSCustomObject]@{ diff --git a/scripts/windows/CustomFunctions/Bootstrap/Public/Get-LivelogVersion.ps1 b/scripts/windows/CustomFunctions/Bootstrap/Public/Get-LivelogVersion.ps1 index 12521f88..dab0f51b 100644 --- a/scripts/windows/CustomFunctions/Bootstrap/Public/Get-LivelogVersion.ps1 +++ b/scripts/windows/CustomFunctions/Bootstrap/Public/Get-LivelogVersion.ps1 @@ -8,6 +8,15 @@ function Get-LiveLogVersion { $StandardOutput = "C:\livelogversion.txt" ) + # Not every image ships the Taskcluster binaries: the win-hw-wim bake role excludes + # windows_worker_runner (generic-worker + worker-runner are installed at DEPLOY time), + # and Start-Process on a missing path throws a TERMINATING error, which Set-ReleaseNotes' + # trap rethrows - killing the build right before Sysprep. Report nothing instead, so the + # release notes just omit the row. No change where the binary exists. + if (-not (Test-Path $FilePath)) { + Write-Verbose ('{0}: {1} not present; omitting from the release notes' -f $MyInvocation.MyCommand.Name, $FilePath) + return + } Start-Process -FilePath $FilePath -ArgumentList "--short-version" -RedirectStandardOutput $StandardOutput -Wait -NoNewWindow [PSCustomObject]@{ Name = "LiveLog" diff --git a/scripts/windows/CustomFunctions/Bootstrap/Public/Get-ProxyVersion.ps1 b/scripts/windows/CustomFunctions/Bootstrap/Public/Get-ProxyVersion.ps1 index c0e6d74a..c2459b00 100644 --- a/scripts/windows/CustomFunctions/Bootstrap/Public/Get-ProxyVersion.ps1 +++ b/scripts/windows/CustomFunctions/Bootstrap/Public/Get-ProxyVersion.ps1 @@ -8,6 +8,15 @@ function Get-ProxyVersion { $StandardOutput = "C:\proxyversion.txt" ) + # Not every image ships the Taskcluster binaries: the win-hw-wim bake role excludes + # windows_worker_runner (generic-worker + worker-runner are installed at DEPLOY time), + # and Start-Process on a missing path throws a TERMINATING error, which Set-ReleaseNotes' + # trap rethrows - killing the build right before Sysprep. Report nothing instead, so the + # release notes just omit the row. No change where the binary exists. + if (-not (Test-Path $FilePath)) { + Write-Verbose ('{0}: {1} not present; omitting from the release notes' -f $MyInvocation.MyCommand.Name, $FilePath) + return + } Start-Process -FilePath $FilePath -ArgumentList "--short-version" -RedirectStandardOutput $StandardOutput -Wait -NoNewWindow [PSCustomObject]@{ Name = "Proxy" diff --git a/scripts/windows/CustomFunctions/Bootstrap/Public/Get-WorkerRunnerVersion.ps1 b/scripts/windows/CustomFunctions/Bootstrap/Public/Get-WorkerRunnerVersion.ps1 index 71893369..a20d770d 100644 --- a/scripts/windows/CustomFunctions/Bootstrap/Public/Get-WorkerRunnerVersion.ps1 +++ b/scripts/windows/CustomFunctions/Bootstrap/Public/Get-WorkerRunnerVersion.ps1 @@ -8,6 +8,15 @@ function Get-WorkerRunnerVersion { $StandardOutput = "C:\gwversion.txt" ) + # Not every image ships the Taskcluster binaries: the win-hw-wim bake role excludes + # windows_worker_runner (generic-worker + worker-runner are installed at DEPLOY time), + # and Start-Process on a missing path throws a TERMINATING error, which Set-ReleaseNotes' + # trap rethrows - killing the build right before Sysprep. Report nothing instead, so the + # release notes just omit the row. No change where the binary exists. + if (-not (Test-Path $FilePath)) { + Write-Verbose ('{0}: {1} not present; omitting from the release notes' -f $MyInvocation.MyCommand.Name, $FilePath) + return + } Start-Process -FilePath $FilePath -ArgumentList "--short-version" -RedirectStandardOutput $StandardOutput -Wait -NoNewWindow [Hashtable]@{ Name = "StartWorker" diff --git a/scripts/windows/CustomFunctions/Bootstrap/Public/Show-TaskclusterBinaries.ps1 b/scripts/windows/CustomFunctions/Bootstrap/Public/Show-TaskclusterBinaries.ps1 index 56bf1fe6..5242f4e3 100644 --- a/scripts/windows/CustomFunctions/Bootstrap/Public/Show-TaskclusterBinaries.ps1 +++ b/scripts/windows/CustomFunctions/Bootstrap/Public/Show-TaskclusterBinaries.ps1 @@ -12,5 +12,7 @@ function Show-TaskclusterBinaries { Name = $PSItem.Name Version = $PSItem.Version } - } + # A binary that isn't installed yields an empty array here, whose .Name is $null - + # drop those rather than emit a blank row into the markdown table. + } | Where-Object { $PSItem.Name } } \ No newline at end of file