diff --git a/modules/roles_profiles/manifests/profiles/disable_services.pp b/modules/roles_profiles/manifests/profiles/disable_services.pp index 5143308be..9310e6dc0 100644 --- a/modules/roles_profiles/manifests/profiles/disable_services.pp +++ b/modules/roles_profiles/manifests/profiles/disable_services.pp @@ -49,6 +49,23 @@ case $facts['custom_win_location'] { 'datacenter': { include win_disable_services::disable_optional_services + # Disable Windows Defender real-time/on-access scanning on the + # datacenter hardware fleet explicitly. Do NOT rely on the + # custom_win_release_id in ['2004','2009'] gate above: Win11 freezes + # ReleaseId at 2009 (DisplayVersion carries 24H2), so that branch + # only fires by coincidence and would silently stop disabling + # Defender if the reported ReleaseId ever changed. Including the + # class is idempotent, so this is harmless where the gate also fires. + # Tamper Protection is enforced on this fleet (TamperProtectionSource=5), + # so Set-MpPreference / policy DisableRealtimeMonitoring do not stick; + # the schtask renames the WdFilter/WdBoot/WdNisDrv drivers at boot, + # which works below Tamper Protection. State is asserted by the + # win_nsclient check_defender check. + include win_disable_services::disable_windows_defender_schtask + # Declare the policy/passive/real-time-off intent. Ignored while Tamper + # Protection is on (the schtask rename is what works there), but correct + # and immediate on any Tamper-off image. + include win_disable_services::disable_windows_defender } 'azure': { $apx_uninstall = 'uninstall.ps1' diff --git a/modules/win_disable_services/manifests/disable_windows_defender.pp b/modules/win_disable_services/manifests/disable_windows_defender.pp index 01e97a47c..9817c186c 100644 --- a/modules/win_disable_services/manifests/disable_windows_defender.pp +++ b/modules/win_disable_services/manifests/disable_windows_defender.pp @@ -2,27 +2,94 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Policy-based Windows Defender / real-time protection disable. +# +# IMPORTANT: on Windows 11 with Tamper Protection ENABLED (the state on the +# datacenter hardware fleet, TamperProtectionSource=5, which cannot be turned off +# in-OS), Defender IGNORES these policy values - Tamper Protection guards them. +# They are honored only when Tamper Protection is OFF (e.g. disabled in the image +# before first boot). They declare intent and make the disable correct + immediate +# for any Tamper-off image, but they are NOT sufficient on their own while Tamper +# is on. +# +# The mechanism that actually disables on-access scanning while Tamper is on is the +# driver rename performed by win_disable_services::disable_windows_defender_schtask +# (renames WdFilter/WdBoot/WdNisDrv at boot, below Tamper's reach), re-asserted at +# boot by the maintain-system script (Invoke-DefenderRealtimeGuard), and monitored +# by the win_nsclient check_defender check. class win_disable_services::disable_windows_defender { if $facts['os']['name'] == 'Windows' { - ## Taken from https://github.com/mozilla-platform-ops/worker-images/blob/main/scripts/windows/CustomFunctions/Bootstrap/Public/Disable-AntiVirus.ps1 - #exec { 'disable_windows_defender': - # command => file('win_disable_services/windows_defender/set.ps1'), - # onlyif => file('win_disable_services/windows_defender/validate.ps1'), - # provider => powershell, - # timeout => 300, - #} + registry_key { 'HKLM\SOFTWARE\Policies\Microsoft\Windows Defender': + ensure => present, + } + registry_value { 'HKLM\SOFTWARE\Policies\Microsoft\Windows Defender\DisableAntiSpyware': + ensure => present, + type => dword, + data => '1', + } - ## Taken from https://github.com/mozilla-platform-ops/worker-images/blob/main/scripts/windows/CustomFunctions/Bootstrap/Public/Disable-AntiVirus.ps1 - registry_value { 'HKLM\\SOFTWARE\\Policies\\Microsoft\\Windows Advanced Threat Protection': + registry_key { 'HKLM\SOFTWARE\Policies\Microsoft\Windows Defender\Real-Time Protection': + ensure => present, + } + registry_value { 'HKLM\SOFTWARE\Policies\Microsoft\Windows Defender\Real-Time Protection\DisableRealtimeMonitoring': + ensure => present, + type => dword, + data => '1', + } + registry_value { 'HKLM\SOFTWARE\Policies\Microsoft\Windows Defender\Real-Time Protection\DisableBehaviorMonitoring': + ensure => present, + type => dword, + data => '1', + } + registry_value { 'HKLM\SOFTWARE\Policies\Microsoft\Windows Defender\Real-Time Protection\DisableOnAccessProtection': ensure => present, + type => dword, + data => '1', + } + registry_value { 'HKLM\SOFTWARE\Policies\Microsoft\Windows Defender\Real-Time Protection\DisableIOAVProtection': + ensure => present, + type => dword, + data => '1', } - ## Taken from https://github.com/mozilla-platform-ops/worker-images/blob/main/scripts/windows/CustomFunctions/Bootstrap/Public/Disable-AntiVirus.ps1 - registry_value { 'HKLM\\SOFTWARE\\Policies\\Microsoft\\Windows Advanced Threat Protection\\ForceDefenderPassiveMode': + # Force Defender into passive mode (honored when Tamper is off / another AV is + # registered). Harmless otherwise. + registry_key { 'HKLM\SOFTWARE\Policies\Microsoft\Windows Advanced Threat Protection': + ensure => present, + } + registry_value { 'HKLM\SOFTWARE\Policies\Microsoft\Windows Advanced Threat Protection\ForceDefenderPassiveMode': ensure => present, type => dword, data => '1', } + + # The following DO take effect even while Tamper Protection is on (verified on the + # 24h2 hw fleet) - they are not guarded by Tamper Protection the way the AV + # engine/services/drivers are: + + # Disable the Defender for Endpoint (EDR) sensor service. This is the ONE Defender + # service whose Start value is writable under Tamper (WinDefend/WdFilter/WdNisSvc are + # not). 4 = disabled. + registry_value { 'HKLM\SYSTEM\CurrentControlSet\Services\Sense\Start': + ensure => present, + type => dword, + data => '4', + } + + # Disable Defender's built-in scheduled tasks (scans/cleanup/cache/verification) so + # they cannot fire during CI tasks. Tamper Protection does not protect these. + exec { 'disable_defender_scheduled_tasks': + command => 'Get-ScheduledTask -TaskPath "\\Microsoft\\Windows\\Windows Defender\\" -ErrorAction SilentlyContinue | Disable-ScheduledTask -ErrorAction SilentlyContinue | Out-Null', + onlyif => 'if (Get-ScheduledTask -TaskPath "\\Microsoft\\Windows\\Windows Defender\\" -ErrorAction SilentlyContinue | Where-Object { $_.State -ne "Disabled" }) { exit 0 } else { exit 1 }', + provider => powershell, + } + + # Blanket on-access path exclusions for the CI volumes (GP-managed; writable and + # honored under Tamper). These minimise scan overhead during the window where + # WdFilter is still loaded (e.g. right after a Defender platform update, before the + # maintain-system guard reboots). They are (re)asserted every boot by + # Invoke-DefenderRealtimeGuard in maintainsystem-hw.ps1 (value names contain a + # trailing backslash, which is set there rather than via registry_value titles). } } # Bug List diff --git a/modules/win_nsclient/files/check_defender.ps1 b/modules/win_nsclient/files/check_defender.ps1 new file mode 100644 index 000000000..7ae2680b4 --- /dev/null +++ b/modules/win_nsclient/files/check_defender.ps1 @@ -0,0 +1,78 @@ +# scripts\check_defender.ps1 +# NSClient++ external check that asserts Windows Defender real-time / on-access +# scanning is effectively DISABLED on the datacenter hardware fleet. +# +# Why not Get-MpComputerStatus? On this fleet the Defender management provider +# fails to load ("Provider load failure") because the disable mechanism renames +# the Defender driver binaries (see win_disable_services::disable_windows_defender_schtask). +# So we assert the ground-truth signals instead: +# - WdFilter (the file-system minifilter that performs on-access scanning) is +# NOT running, AND its driver binary is renamed to WdFilter.sys.bak. +# Real-time scanning cannot occur without WdFilter loaded, regardless of whether +# the WinDefend service or MsMpEng process happen to be alive. +# +# The key risk this guards against: a Defender platform/signature update can +# restore WdFilter.sys and re-arm on-access scanning until the next boot (when +# the disable schtask re-renames it). That silently injects scan overhead into +# CI perf runs, so we surface it as CRITICAL. +# +# Nagios contract: print one line "STATE - text | perfdata" and exit 0/1/2/3. + +$ErrorActionPreference = 'Stop' + +function Exit-With([int]$code, [string]$msg) { + Write-Output $msg + exit $code +} + +$driver = "$env:SystemRoot\System32\drivers\WdFilter.sys" +$driverBak = "$env:SystemRoot\System32\drivers\WdFilter.sys.bak" + +# 1) Is the on-access minifilter currently running? +$wdfState = 'unknown' +try { + $q = (& "$env:SystemRoot\System32\sc.exe" query WdFilter 2>$null | Select-String 'STATE') + if ($q -match 'RUNNING') { $wdfState = 'running' } + elseif ($q -match 'STOPPED') { $wdfState = 'stopped' } + elseif (-not $q) { $wdfState = 'absent' } +} +catch { $wdfState = 'unknown' } + +# 2) Is the driver binary renamed (disabled) or restored (re-armed)? +$sysPresent = Test-Path -LiteralPath $driver +$bakPresent = Test-Path -LiteralPath $driverBak + +# 3) WinDefend service state (informational only — not the deciding factor) +$winDefend = 'unknown' +try { + $svc = Get-Service WinDefend -ErrorAction SilentlyContinue + if ($svc) { $winDefend = "$($svc.Status)" } else { $winDefend = 'absent' } +} +catch { } + +# Numeric health code for Grafana/InfluxDB: 0 disabled(OK) / 2 active(CRIT) / 3 unknown +$rtRunning = if ($wdfState -eq 'running' -or $sysPresent) { 1 } else { 0 } +$health = switch ($wdfState) { + 'stopped' { if ($sysPresent) { 2 } else { 0 } } + 'absent' { if ($sysPresent) { 2 } else { 0 } } + 'running' { 2 } + default { 3 } +} + +$perf = "defender_rt_on=$rtRunning;1;1;0;1 health=$health;1;2;0;3" +$summary = "WdFilter=$wdfState WdFilter.sys=$(if($sysPresent){'PRESENT'}else{'renamed'}) bak=$(if($bakPresent){'yes'}else{'no'}) WinDefend=$winDefend" + +if ($wdfState -eq 'unknown') { + Exit-With 3 "UNKNOWN - cannot determine WdFilter state - $summary | $perf" +} + +# Re-armed: the minifilter is running, or its binary was restored by an update. +if ($wdfState -eq 'running') { + Exit-With 2 "CRITICAL - Defender on-access scanning is ACTIVE (WdFilter running) - $summary | $perf" +} +if ($sysPresent) { + Exit-With 2 "CRITICAL - WdFilter.sys restored (likely Defender platform update); will re-arm at next boot - $summary | $perf" +} + +# WdFilter stopped/absent and binary renamed => real-time effectively off. +Exit-With 0 "OK - Defender real-time scanning disabled (WdFilter not loaded) - $summary | $perf" diff --git a/modules/win_nsclient/manifests/init.pp b/modules/win_nsclient/manifests/init.pp index 18f4f2287..b0ee711f2 100644 --- a/modules/win_nsclient/manifests/init.pp +++ b/modules/win_nsclient/manifests/init.pp @@ -64,6 +64,14 @@ notify => Service['nscp'], } + # Asserts Windows Defender real-time/on-access scanning is disabled (WdFilter + # minifilter not loaded). Catches a Defender platform update re-arming it. + file { "${scripts_dir}\\check_defender.ps1": + content => file('win_nsclient/check_defender.ps1'), + require => File[$scripts_dir], + notify => Service['nscp'], + } + service { 'nscp': ensure => running, enable => true, diff --git a/modules/win_nsclient/templates/nsclient.ini.epp b/modules/win_nsclient/templates/nsclient.ini.epp index a6a9ecbe2..f05036227 100644 --- a/modules/win_nsclient/templates/nsclient.ini.epp +++ b/modules/win_nsclient/templates/nsclient.ini.epp @@ -38,6 +38,7 @@ worker_bootstrap_stage = powershell.exe -ExecutionPolicy RemoteSigned -File scri thermal = powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File scripts\check_thermal.ps1 thermal_hp = powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File scripts\check_thermal_hp.ps1 worker_pool_id = powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File scripts\worker_pool_id.ps1 +defender = powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File scripts\check_defender.ps1 [/settings/external scripts/alias/cpu_5s] alias = cpu_5s diff --git a/modules/win_scheduled_tasks/files/maintainsystem-hw.ps1 b/modules/win_scheduled_tasks/files/maintainsystem-hw.ps1 index 044f13800..ea49978df 100644 --- a/modules/win_scheduled_tasks/files/maintainsystem-hw.ps1 +++ b/modules/win_scheduled_tasks/files/maintainsystem-hw.ps1 @@ -473,6 +473,178 @@ function Wait-ForUserInitReady { return $false } +function Write-DefenderStatus { + # Persist the latest Defender real-time guard state to a small status file that + # NSClient++ surfaces to Marlin (read by scripts\check_defender.ps1). Best-effort. + param ( + [Parameter(Mandatory)] [string] $Path, + [Parameter(Mandatory)] $Status + ) + try { + $Status | ConvertTo-Json -Compress | Out-File -FilePath $Path -Encoding utf8 + } + catch { + Write-Log -message ('Write-DefenderStatus :: failed: {0}' -f $_.Exception.Message) -severity 'WARN' + } +} + +function Invoke-DefenderRealtimeGuard { + # Ensure Windows Defender real-time / on-access scanning is OFF before worker-runner. + # + # On this fleet Tamper Protection is ENFORCED (cannot be disabled in-OS), so the + # supported toggles (Set-MpPreference, sc config WdFilter, fltmc unload, the policy + # registry values) are all blocked or ignored. The only lever that works is renaming + # the Defender driver binaries (WdFilter/WdBoot/WdNisDrv) - done below Tamper's reach + # via takeown - which takes effect on the NEXT boot. WdFilter is a BOOT-START + # minifilter, so a boot that follows a Defender platform update (which restores + # WdFilter.sys) comes up with on-access scanning ACTIVE for the whole session. + # + # This guard runs at boot, before worker-runner. It: (1) re-renames any restored + # driver so the next boot is clean, (2) tries the supported in-session unload (works + # only if Tamper happens to be off, e.g. a Tamper-off image), and (3) if the filter is + # still running under Tamper, reboots ONCE (boot-loop guarded) so the node comes up + # clean before any CI task runs. Writes defender_status.json for Marlin/Grafana. + param ( + [string] $StatusDir = 'C:\fleetbench\results', # dir NSClient++ already reads + [int] $MaxReboots = 1, # max reboots per restore event + [int] $CooldownMin = 60 # suppress re-reboot within this window + ) + begin { + Write-Log -message ('{0} :: begin - {1:o}' -f $($MyInvocation.MyCommand.Name), (Get-Date).ToUniversalTime()) -severity 'DEBUG' + } + process { + try { + $drvDir = Join-Path $env:SystemRoot 'System32\drivers' + $drivers = @('WdFilter', 'WdBoot', 'WdNisDrv') + if (-not (Test-Path $StatusDir)) { New-Item -ItemType Directory -Path $StatusDir -Force | Out-Null } + $statusFile = Join-Path $StatusDir 'defender_status.json' + $markerFile = Join-Path $StatusDir 'defender_guard.json' + + # (0) Blanket on-access path exclusions for the CI volumes. Unlike the AV + # engine/services, GP-managed exclusions ARE writable and honored under Tamper + # Protection, so even on a boot where WdFilter is still loaded (right after a + # Defender platform update, before the reboot below) on-access scanning of the + # work volumes is skipped. Re-asserted every boot in case an update clears them. + try { + $exRoot = 'HKLM:\SOFTWARE\Policies\Microsoft\Windows Defender\Exclusions' + $exPaths = Join-Path $exRoot 'Paths' + foreach ($k in @($exRoot, $exPaths)) { if (-not (Test-Path $k)) { New-Item -Path $k -Force | Out-Null } } + Set-ItemProperty -Path $exRoot -Name 'Exclusions_Paths' -Value 1 -Type DWord + foreach ($p in @('C:\', 'D:\')) { + if (Test-Path -LiteralPath $p) { + New-ItemProperty -Path $exPaths -Name $p -Value 0 -PropertyType DWord -Force | Out-Null + } + } + } + catch { + Write-Log -message ('{0} :: exclusion set partial: {1}' -f $($MyInvocation.MyCommand.Name), $_.Exception.Message) -severity 'WARN' + } + + # (1) Re-disable persistently: rename any restored driver binaries so the NEXT + # boot comes up clean. No-op when already renamed (the normal steady state). + $renamed = @() + foreach ($d in $drivers) { + $sys = Join-Path $drvDir ($d + '.sys') + if (Test-Path -LiteralPath $sys) { + try { + & "$env:SystemRoot\System32\takeown.exe" /f $sys /a | Out-Null + & "$env:SystemRoot\System32\icacls.exe" $sys /grant 'Administrators:F' | Out-Null + $bak = $sys + '.bak' + if (Test-Path -LiteralPath $bak) { Remove-Item -LiteralPath $bak -Force -ErrorAction SilentlyContinue } + Rename-Item -LiteralPath $sys -NewName ($d + '.sys.bak') -Force + $renamed += $d + } + catch { + Write-Log -message ('{0} :: could not rename {1}.sys: {2}' -f $($MyInvocation.MyCommand.Name), $d, $_.Exception.Message) -severity 'WARN' + } + } + } + if ($renamed.Count) { + Write-Log -message ('{0} :: renamed restored Defender driver(s): {1} (likely a Defender platform update)' -f $($MyInvocation.MyCommand.Name), ($renamed -join ',')) -severity 'WARN' + } + + # Is the on-access minifilter running right now? + $wdfRunning = $false + try { $wdfRunning = (((& "$env:SystemRoot\System32\sc.exe" query WdFilter 2>$null) | Select-String 'STATE') -match 'RUNNING') } catch { } + $tamper = (Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows Defender\Features' -ErrorAction SilentlyContinue).TamperProtection + + $action = if ($renamed.Count) { 'renamed_only' } else { 'none' } + + if ($wdfRunning) { + # Try the supported in-session unload first (succeeds only if Tamper is OFF). + try { + $u = & "$env:SystemRoot\System32\fltMC.exe" unload WdFilter 2>&1 + if ($LASTEXITCODE -eq 0) { + Start-Sleep -Seconds 2 + $wdfRunning = (((& "$env:SystemRoot\System32\sc.exe" query WdFilter 2>$null) | Select-String 'STATE') -match 'RUNNING') + if (-not $wdfRunning) { + $action = 'unloaded' + Write-Log -message ('{0} :: WdFilter unloaded in-session (Tamper Protection off).' -f $($MyInvocation.MyCommand.Name)) -severity 'WARN' + } + } + else { + Write-Log -message ('{0} :: fltmc unload blocked (Tamper Protection on): {1}' -f $($MyInvocation.MyCommand.Name), ($u -join ' ')) -severity 'INFO' + } + } + catch { } + } + + if ($wdfRunning) { + # Cannot unload under Tamper. The .sys is now renamed, so a reboot yields a + # clean boot. Reboot once, boot-loop guarded by a marker file. + $count = 0; $lastUtc = $null + if (Test-Path $markerFile) { + try { + $m = Get-Content -LiteralPath $markerFile -Raw | ConvertFrom-Json + $count = [int]$m.reboot_count + $lastUtc = [datetime]$m.last_reboot_utc + } + catch { } + } + $now = (Get-Date).ToUniversalTime() + $withinCooldown = $false + if ($lastUtc) { $withinCooldown = ((($now) - $lastUtc.ToUniversalTime()).TotalMinutes -lt $CooldownMin) } + + if ($count -ge $MaxReboots -and $withinCooldown) { + # Already rebooted recently and it is STILL active -> stop, do not loop. + $action = 'degraded' + Write-Log -message ('{0} :: WdFilter STILL ACTIVE after {1} reboot(s) within {2}m; proceeding WITHOUT another reboot to avoid a boot loop. On-access scanning may affect this session. Driver renamed for next clean boot.' -f $($MyInvocation.MyCommand.Name), $count, $CooldownMin) -severity 'ERROR' + } + else { + $newCount = if ($withinCooldown) { $count + 1 } else { 1 } + ([pscustomobject]@{ reboot_count = $newCount; last_reboot_utc = $now.ToString('o') } | ConvertTo-Json -Compress) | Out-File -FilePath $markerFile -Encoding utf8 + $action = 'renamed_reboot' + Write-DefenderStatus -Path $statusFile -Status ([pscustomobject]@{ + timestamp_utc = $now.ToString('yyyy-MM-ddTHH:mm:ssZ'); host = $env:COMPUTERNAME + wdfilter_running = $true; sys_renamed = (-not (Test-Path (Join-Path $drvDir 'WdFilter.sys'))) + tamper_protection = $tamper; action = $action; reboot_count = $newCount + }) + Write-Log -message ('{0} :: WdFilter active under Tamper; renamed driver(s) and rebooting (#{1}) to clear on-access scanning before worker-runner.' -f $($MyInvocation.MyCommand.Name), $newCount) -severity 'WARN' + Restart-Computer -Force + exit + } + } + else { + # Clean: not running. Clear the reboot marker so a future event starts fresh. + if (Test-Path $markerFile) { Remove-Item -LiteralPath $markerFile -Force -ErrorAction SilentlyContinue } + } + + Write-DefenderStatus -Path $statusFile -Status ([pscustomobject]@{ + timestamp_utc = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ'); host = $env:COMPUTERNAME + wdfilter_running = $wdfRunning; sys_renamed = (-not (Test-Path (Join-Path $drvDir 'WdFilter.sys'))) + tamper_protection = $tamper; action = $action; reboot_count = 0 + }) + } + catch { + # Must never block worker-runner from starting. + Write-Log -message ('{0} :: error: {1}' -f $($MyInvocation.MyCommand.Name), $_.Exception.Message) -severity 'WARN' + } + } + end { + Write-Log -message ('{0} :: end - {1:o}' -f $($MyInvocation.MyCommand.Name), (Get-Date).ToUniversalTime()) -severity 'DEBUG' + } +} + Write-Log -message ('{0} :: maintained system started' -f $($MyInvocation.MyCommand.Name)) -severity 'DEBUG' if (-not (Get-Process explorer -ErrorAction SilentlyContinue)) { Write-Log -message ('{0} :: No user logged in (no explorer.exe); sleeping 60s' -f $($MyInvocation.MyCommand.Name)) -severity 'DEBUG' @@ -509,6 +681,12 @@ If ($bootstrap_stage -eq 'complete') { ## Let's make sure the machine is online before checking the internet Test-ConnectionUntilOnline + # Ensure Windows Defender real-time / on-access scanning is OFF before worker-runner. + # Re-disables (renames) a driver restored by a Defender platform update and reboots + # once to clear the running minifilter under Tamper Protection. Runs before the long + # UI-init wait and the fleetbench benchmark so any needed reboot happens early. + Invoke-DefenderRealtimeGuard + ## Let's check for the latest install of google chrome using chocolatey before starting worker runner ## Instead of querying chocolatey each time this runs, let's query chrome json endoint and check locally installed version Get-LatestGoogleChrome