fix(rack-controller): recover power-blocked firmware updates

crates/api-core/src/tests/rack_state_controller/handler.rs

@@ -271,6 +271,30 @@ async fn set_machine_host_reprovision_state(
     Ok(())
 }
 
+async fn set_machine_power_states(
+    pool: &sqlx::PgPool,
+    machine_id: &MachineId,
+    desired_power_state: model::power_manager::PowerState,
+    actual_power_state: model::power_manager::PowerState,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let mut txn = pool.begin().await?;
+    let power_options = db::power_options::get_by_ids(&[*machine_id], txn.as_mut())
+        .await?
+        .pop()
+        .expect("machine should have power options");
+    let mut power_options = db::power_options::update_desired_state(
+        machine_id,
+        desired_power_state,
+        &power_options.desired_power_state_version,
+        txn.as_mut(),
+    )
+    .await?;
+    power_options.last_fetched_power_state = actual_power_state;
+    db::power_options::persist(&power_options, txn.as_mut()).await?;
+    txn.commit().await?;
+    Ok(())
+}
+
 fn waiting_for_rack_firmware_upgrade_state() -> model::machine::HostReprovisionState {
     model::machine::HostReprovisionState::WaitingForRackFirmwareUpgrade
 }
@@ -1644,6 +1668,116 @@ async fn test_ingestion_transitions_to_firmware_upgrade_and_submits_rack_profile
     Ok(())
 }
 
+#[crate::sqlx_test]
+async fn test_firmware_upgrade_start_rejects_desired_off_machine_before_rms_submission(
+    pool: sqlx::PgPool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let env = create_test_env_with_overrides(
+        pool.clone(),
+        TestEnvOverrides {
+            config: Some(config_with_rack_profiles()),
+            ..Default::default()
+        },
+    )
+    .await;
+    let (rack_id, host) = create_single_compute_rack(&env, &pool).await?;
+    set_machine_power_states(
+        &pool,
+        &host.host_snapshot.id,
+        model::power_manager::PowerState::Off,
+        model::power_manager::PowerState::On,
+    )
+    .await?;
+
+    let config = RackConfig {
+        maintenance_requested: Some(MaintenanceScope {
+            machine_ids: vec![host.host_snapshot.id],
+            activities: vec![MaintenanceActivity::FirmwareUpgrade {
+                firmware_version: Some(r#"{"Id":"fw-json"}"#.to_string()),
+                components: vec!["BMC".to_string()],
+                force_update: false,
+            }],
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    let mut txn = pool.acquire().await?;
+    db_rack::update(txn.as_mut(), &rack_id, &config).await?;
+    drop(txn);
+    env.api
+        .credential_manager
+        .set_credentials(
+            &CredentialKey::RackMaintenanceAccessToken {
+                rack_id: rack_id.clone(),
+            },
+            &Credentials::UsernamePassword {
+                username: "access_token".to_string(),
+                password: "token".to_string(),
+            },
+        )
+        .await
+        .map_err(|error| eyre::eyre!("failed to set maintenance access token: {}", error))?;
+
+    let mut rack = get_db_rack(env.db_reader().as_mut(), &rack_id).await;
+    let handler = RackStateHandler::default();
+    let mut services = env.rack_state_handler_services();
+    let mut metrics = RackMetrics::default();
+    let mut db_writes = DbWriteBatch::default();
+    let mut ctx = StateHandlerContext::<RackStateHandlerContextObjects> {
+        services: &mut services,
+        metrics: &mut metrics,
+        pending_db_writes: &mut db_writes,
+    };
+    let fw_state = RackState::Maintenance {
+        maintenance_state: RackMaintenanceState::FirmwareUpgrade {
+            rack_firmware_upgrade: FirmwareUpgradeState::Start,
+        },
+    };
+
+    let mut outcome = handler
+        .handle_object_state(&rack_id, &mut rack, &fw_state, &mut ctx)
+        .await?;
+    if let Some(txn) = outcome.take_transaction() {
+        txn.commit().await?;
+    }
+
+    let StateHandlerOutcome::Transition {
+        next_state: RackState::Error { cause },
+        ..
+    } = outcome
+    else {
+        panic!("desired-Off target should fail rack firmware start");
+    };
+    assert!(cause.contains(&host.host_snapshot.id.to_string()));
+    assert!(cause.contains("desired power state is Off"));
+    assert!(
+        env.rms_sim
+            .submitted_apply_firmware_object_requests()
+            .await
+            .is_empty()
+    );
+
+    let rack = get_db_rack(env.db_reader().as_mut(), &rack_id).await;
+    assert!(rack.config.maintenance_requested.is_none());
+    let machine = db::machine::find_one(
+        &pool,
+        &host.host_snapshot.id,
+        model::machine::machine_search_config::MachineSearchConfig::default(),
+    )
+    .await?
+    .expect("machine should exist");
+    assert!(machine.host_reprovision_requested.is_none());
+    let token = env
+        .test_credential_manager
+        .get_credentials(&CredentialKey::RackMaintenanceAccessToken {
+            rack_id: rack_id.clone(),
+        })
+        .await?;
+    assert!(token.is_none());
+
+    Ok(())
+}
+
 #[crate::sqlx_test]
 async fn test_firmware_upgrade_start_submits_json_and_deletes_access_token(
     pool: sqlx::PgPool,
@@ -1883,6 +2017,151 @@ async fn test_firmware_upgrade_start_missing_profile_deletes_access_token(
     Ok(())
 }
 
+/// A machine that cannot consume its rack reprovision request because desired
+/// power is Off fails the rack job without clearing requests already owned by
+/// active device reprovisioning state machines.
+#[crate::sqlx_test]
+async fn test_firmware_upgrade_wait_for_complete_recovers_power_blocked_machine(
+    pool: sqlx::PgPool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let env = create_test_env_with_overrides(
+        pool.clone(),
+        TestEnvOverrides {
+            config: Some(config_with_rack_profiles()),
+            ..Default::default()
+        },
+    )
+    .await;
+    let (rack_id, blocked_host, active_host) = create_two_compute_rack(&env, &pool).await?;
+    set_machine_power_states(
+        &pool,
+        &blocked_host.host_snapshot.id,
+        model::power_manager::PowerState::Off,
+        model::power_manager::PowerState::Off,
+    )
+    .await?;
+
+    let scope = MaintenanceScope {
+        machine_ids: vec![blocked_host.host_snapshot.id, active_host.host_snapshot.id],
+        activities: vec![MaintenanceActivity::FirmwareUpgrade {
+            firmware_version: Some(r#"{"Id":"fw-json"}"#.to_string()),
+            components: vec!["BMC".to_string()],
+            force_update: false,
+        }],
+        ..Default::default()
+    };
+    let job = FirmwareUpgradeJob {
+        job_id: Some("parent-job".to_string()),
+        status: Some("in_progress".to_string()),
+        started_at: Some(chrono::Utc::now()),
+        ..Default::default()
+    };
+    let initiator = format!("rack-{rack_id}");
+    let mut txn = pool.begin().await?;
+    let config = RackConfig {
+        maintenance_requested: Some(scope),
+        ..Default::default()
+    };
+    db_rack::update(txn.as_mut(), &rack_id, &config).await?;
+    db_rack::update_firmware_upgrade_job(txn.as_mut(), &rack_id, Some(&job)).await?;
+    db::host_machine_update::trigger_host_reprovisioning_request(
+        txn.as_mut(),
+        &initiator,
+        &blocked_host.host_snapshot.id,
+    )
+    .await?;
+    db::host_machine_update::trigger_host_reprovisioning_request(
+        txn.as_mut(),
+        &initiator,
+        &active_host.host_snapshot.id,
+    )
+    .await?;
+    txn.commit().await?;
+    env.api
+        .credential_manager
+        .set_credentials(
+            &CredentialKey::RackMaintenanceAccessToken {
+                rack_id: rack_id.clone(),
+            },
+            &Credentials::UsernamePassword {
+                username: "access_token".to_string(),
+                password: "token".to_string(),
+            },
+        )
+        .await
+        .map_err(|error| eyre::eyre!("failed to set maintenance access token: {}", error))?;
+    set_machine_host_reprovision_state(
+        &pool,
+        &active_host.host_snapshot.id,
+        waiting_for_rack_firmware_upgrade_state(),
+    )
+    .await?;
+
+    let mut rack = get_db_rack(env.db_reader().as_mut(), &rack_id).await;
+    let handler = RackStateHandler::default();
+    let mut services = env.rack_state_handler_services();
+    let mut metrics = RackMetrics::default();
+    let mut db_writes = DbWriteBatch::default();
+    let mut ctx = StateHandlerContext::<RackStateHandlerContextObjects> {
+        services: &mut services,
+        metrics: &mut metrics,
+        pending_db_writes: &mut db_writes,
+    };
+    let fw_state = RackState::Maintenance {
+        maintenance_state: RackMaintenanceState::FirmwareUpgrade {
+            rack_firmware_upgrade: FirmwareUpgradeState::WaitForComplete,
+        },
+    };
+
+    let outcome = handler
+        .handle_object_state(&rack_id, &mut rack, &fw_state, &mut ctx)
+        .await?;
+
+    let StateHandlerOutcome::Transition {
+        next_state: RackState::Error { cause },
+        ..
+    } = outcome
+    else {
+        panic!("power-blocked rack firmware job should transition to Error");
+    };
+    assert!(cause.contains(&blocked_host.host_snapshot.id.to_string()));
+    assert!(!cause.contains(&active_host.host_snapshot.id.to_string()));
+
+    let rack = get_db_rack(env.db_reader().as_mut(), &rack_id).await;
+    assert!(rack.config.maintenance_requested.is_none());
+    let job = rack
+        .firmware_upgrade_job
+        .expect("failed firmware job should be retained");
+    assert_eq!(job.status.as_deref(), Some("failed"));
+    assert!(job.completed_at.is_some());
+
+    let blocked_machine = db::machine::find_one(
+        &pool,
+        &blocked_host.host_snapshot.id,
+        model::machine::machine_search_config::MachineSearchConfig::default(),
+    )
+    .await?
+    .expect("blocked machine should exist");
+    assert!(blocked_machine.host_reprovision_requested.is_none());
+    let active_machine = db::machine::find_one(
+        &pool,
+        &active_host.host_snapshot.id,
+        model::machine::machine_search_config::MachineSearchConfig::default(),
+    )
+    .await?
+    .expect("active machine should exist");
+    assert!(active_machine.host_reprovision_requested.is_some());
+    let token = env
+        .test_credential_manager
+        .get_credentials(&CredentialKey::RackMaintenanceAccessToken {
+            rack_id: rack_id.clone(),
+        })
+        .await?;
+    assert!(token.is_none());
+
+    Ok(())
+}
+
 /// test_firmware_upgrade_wait_for_complete_waits_while_jobs_running verifies
 /// that WaitForComplete remains in a wait state while machines are still in
 /// WaitingForRackFirmwareUpgrade and writes in-progress rack firmware status
@@ -1957,10 +2236,11 @@ async fn test_firmware_upgrade_wait_for_complete_waits_while_jobs_running(
         txn.commit().await?;
     }
 
-    assert!(
-        matches!(outcome, StateHandlerOutcome::Wait { .. }),
-        "Expected Wait while machine controller is still WaitingForRackFirmwareUpgrade"
-    );
+    let StateHandlerOutcome::Wait { reason, .. } = outcome else {
+        panic!("Expected Wait while machine controller is still WaitingForRackFirmwareUpgrade");
+    };
+    assert!(reason.contains(&host.host_snapshot.id.to_string()));
+    assert!(reason.contains("pending=1"));
 
     let machine = db::machine::find_one(
         &pool,

crates/api-db/src/host_machine_update.rs

@@ -123,6 +123,32 @@ pub async fn clear_host_reprovisioning_request(
     Ok(())
 }
 
+/// Clears a rack-owned host reprovisioning request if the machine has not left `Ready`.
+///
+/// Returns whether the request was cleared. A concurrent transition out of `Ready`, or a
+/// request from another initiator, leaves the request untouched so its owning controller can
+/// unwind it.
+pub async fn clear_ready_host_reprovisioning_request(
+    txn: &mut PgConnection,
+    machine_id: &MachineId,
+    initiator: &str,
+) -> Result<bool, DatabaseError> {
+    let query = r#"UPDATE machines
+        SET host_reprovisioning_requested = NULL
+        WHERE id = $1
+          AND controller_state->>'state' = 'ready'
+          AND host_reprovisioning_requested->>'initiator' = $2
+        RETURNING id"#;
+    let cleared = sqlx::query_as::<_, MachineId>(query)
+        .bind(machine_id)
+        .bind(initiator)
+        .fetch_optional(txn)
+        .await
+        .map_err(|e| DatabaseError::query(query, e))?;
+
+    Ok(cleared.is_some())
+}
+
 pub async fn reset_host_reprovisioning_request(
     txn: &mut PgConnection,
     machine_id: &MachineId,