Add remote execution for package-backed managed tests

src/src/Commands/ExtensionSetTrait.php

@@ -20,8 +20,8 @@
  *
  * For local commands, the set name is expanded into the extensions it contains
  * and merged into `--plugin`, which env:up then resolves like any other plugin
- * slug. Woo API/E2E commands use this trait to switch extension-set runs back
- * to the Manager-hosted flow instead.
+ * slug. Managed package commands also use this trait to opt into the
+ * Manager-hosted flow explicitly or when an extension set requires it.
  *
  * This trait is meant to be used in the context of Command classes.
  *
@@ -51,6 +51,18 @@ protected function configure_extension_set_option(): void {
 		);
 	}
 
+	/**
+	 * Register the explicit Manager-hosted execution option.
+	 */
+	protected function configure_remote_option(): void {
+		$this->addOption(
+			'remote',
+			null,
+			InputOption::VALUE_NONE,
+			'Run the managed test on QIT servers instead of locally'
+		);
+	}
+
 	/**
 	 * Resolve `--extension_set` into `--plugin` options.
 	 *
@@ -86,26 +98,34 @@ protected function resolve_extension_set( QITInput $input, OutputInterface $outp
 		return null;
 	}
 
-	protected function run_remote_extension_set_if_provided(
+	protected function run_remote_if_requested(
 		QITInput $input,
 		OutputInterface $output,
 		string $remote_test_type,
 		?string $profile_test_type = null
 	): ?int {
 		$extension_set_name = $input->getOption( 'extension_set' );
-		if ( empty( $extension_set_name ) ) {
+		if ( ! $input->getOption( 'remote' ) && empty( $extension_set_name ) ) {
 			return null;
 		}
 
-		$extension_set_error = $this->validate_extension_set_exists( (string) $extension_set_name, $output );
-		if ( $extension_set_error !== null ) {
-			return $extension_set_error;
+		if ( getenv( 'QIT_TEST_RUN_ID' ) !== false ) {
+			$output->writeln( '<error>Cannot start a remote test while QIT_TEST_RUN_ID is set.</error>' );
+
+			return Command::INVALID;
+		}
+
+		if ( ! empty( $extension_set_name ) ) {
+			$extension_set_error = $this->validate_extension_set_exists( (string) $extension_set_name, $output );
+			if ( $extension_set_error !== null ) {
+				return $extension_set_error;
+			}
 		}
 
 		$runner          = App::make( RemoteTestRunner::class );
 		$options_to_send = $runner->get_options_to_send_for_schema( $remote_test_type );
 
-		$unsupported_options_error = $this->validate_only_remote_options_for_remote_extension_set( $input, $output, array_keys( $options_to_send ) );
+		$unsupported_options_error = $this->validate_only_remote_options( $input, $output, array_keys( $options_to_send ) );
 		if ( $unsupported_options_error !== null ) {
 			return $unsupported_options_error;
 		}
@@ -147,7 +167,7 @@ private function validate_extension_set_exists( string $extension_set_name, Outp
 	 * @param OutputInterface $output         The command output.
 	 * @param array<string>   $schema_options Options the Manager schema accepts.
 	 */
-	private function validate_only_remote_options_for_remote_extension_set( QITInput $input, OutputInterface $output, array $schema_options ): ?int {
+	private function validate_only_remote_options( QITInput $input, OutputInterface $output, array $schema_options ): ?int {
 		$allowed_options = array_unique( array_merge(
 			$schema_options,
 			[
@@ -164,6 +184,7 @@ private function validate_only_remote_options_for_remote_extension_set( QITInput
 				'print-report-url',
 				'profile',
 				'quiet',
+				'remote',
 				'timeout',
 				'verbose',
 				'version',
@@ -193,7 +214,7 @@ private function validate_only_remote_options_for_remote_extension_set( QITInput
 		}
 
 		$output->writeln( sprintf(
-			'<error>--extension_set runs managed compatibility-set tests on QIT servers and cannot be combined with local-only option(s): %s.</error>',
+			'<error>Remote managed tests cannot be combined with local-only option(s): %s.</error>',
 			implode( ', ', array_unique( $provided ) )
 		) );
 

src/src/Commands/RunActivationTestCommand.php

@@ -29,6 +29,7 @@ protected function configure(): void {
 		parent::configure();
 		$this->setDescription( 'Run activation tests' );
 		$this->configure_extension_set_option();
+		$this->configure_remote_option();
 	}
 
 	/******************************************************************
@@ -37,7 +38,7 @@ protected function configure(): void {
 	protected function doExecute( QITInput $input, OutputInterface $output ): int {
 		/** @var \QIT_CLI\QITInput $input */
 
-		$remote_result = $this->run_remote_extension_set_if_provided( $input, $output, 'activation' );
+		$remote_result = $this->run_remote_if_requested( $input, $output, 'activation' );
 		if ( $remote_result !== null ) {
 			return $remote_result;
 		}

src/src/Commands/RunE2ECommand.php

@@ -164,7 +164,7 @@ protected function configureMainOptions(): void {
 			->addOption( 'notify', null, InputOption::VALUE_NONE, 'Notify on failures' )
 			->addOption( 'group', 'g', InputOption::VALUE_NEGATABLE, 'Register into a group', false )
 			->addOption( 'print-report-url', null, InputOption::VALUE_NONE, 'Print the test report URL (contains sensitive data - use cautiously in public logs)' )
-			->addOption( 'timeout', null, InputOption::VALUE_OPTIONAL, 'Wait timeout in seconds for remote extension-set runs', null )
+			->addOption( 'timeout', null, InputOption::VALUE_OPTIONAL, 'Wait timeout in seconds for remote runs', null )
 			->addOption( 'ui', null, InputOption::VALUE_NONE, 'Run tests in Playwright UI mode' )
 			->addOption( 'keep-env', null, InputOption::VALUE_NONE, 'Keep the environment running after tests complete (for debugging with Playwright MCP)' );
 	}
@@ -1463,8 +1463,9 @@ protected function runTestPackages( \QIT_CLI\Environment\Environments\E2E\E2EEnv
 
 					// Run the command using package phase runner
 					$temp_manifest_data = [
-						'package' => $pkg_id,
-						'test'    => [
+						'package'   => $pkg_id,
+						'test_type' => $manifest->get_test_type(),
+						'test'      => [
 							'phases' => [
 								'globalSetup' => [ $command ],
 							],
@@ -1818,8 +1819,9 @@ protected function runTestPackages( \QIT_CLI\Environment\Environments\E2E\E2EEnv
 					try {
 						// Run the command using package phase runner
 						$temp_manifest_data = [
-							'package' => $pkg_id,
-							'test'    => [
+							'package'   => $pkg_id,
+							'test_type' => $manifest->get_test_type(),
+							'test'      => [
 								'phases' => [
 									'globalTeardown' => [ $command ],
 								],

src/src/Commands/RunWooApiTestCommand.php

@@ -16,6 +16,7 @@ protected function configure(): void {
 		parent::configure();
 		$this->setDescription( 'Run WooCommerce Core API tests' );
 		$this->configure_extension_set_option();
+		$this->configure_remote_option();
 		$this->addOption(
 			'optional_features',
 			null,
@@ -26,7 +27,7 @@ protected function configure(): void {
 	}
 
 	protected function doExecute( QITInput $input, OutputInterface $output ): int {
-		$remote_result = $this->run_remote_extension_set_if_provided( $input, $output, 'woo-api' );
+		$remote_result = $this->run_remote_if_requested( $input, $output, 'woo-api' );
 		if ( $remote_result !== null ) {
 			return $remote_result;
 		}

src/src/Commands/RunWooE2ETestCommand.php

@@ -15,10 +15,11 @@ protected function configure(): void {
 		parent::configure();
 		$this->setDescription( 'Run WooCommerce Core E2E tests' );
 		$this->configure_extension_set_option();
+		$this->configure_remote_option();
 	}
 
 	protected function doExecute( QITInput $input, OutputInterface $output ): int {
-		$remote_result = $this->run_remote_extension_set_if_provided( $input, $output, 'woo-e2e', 'woo-e2e' );
+		$remote_result = $this->run_remote_if_requested( $input, $output, 'woo-e2e', 'woo-e2e' );
 		if ( $remote_result !== null ) {
 			return $remote_result;
 		}

src/src/PreCommand/Extensions/ExtensionResolver.php

@@ -128,8 +128,10 @@ public function resolve( array $extensions, string $cache_dir ): ResolvedExtensi
 				// Step 2: Check if extension is already cached (avoid metadata API call if possible)
 				$is_cached = $this->cache_manager->is_cached( $extension, $cache_dir );
 
-				// Step 3: Fetch metadata only if not cached (version, download URL, etc.)
-				if ( ! $is_cached ) {
+				// Local metadata comes from the source itself, even when it is already cached.
+				if ( in_array( $extension->from, [ 'local', 'build' ], true ) ) {
+					$this->metadata_fetcher->fetch_metadata( [ $extension ] );
+				} elseif ( ! $is_cached ) {
 					try {
 						if ( in_array( $extension->from, [ 'wporg', 'wccom' ], true ) ) {
 							debug_log( '  Extension not cached, fetching metadata for remote extension' );

src/src/RemoteTestRunner.php

@@ -175,30 +175,30 @@ private function build_options(
 	): array {
 		$profile_name      = $input->get_profile_name();
 		$profile_test_type = $profile_test_type ?? $test_type;
-		$profile           = $this->get_profile_with_fallback( $command, $profile_test_type, $profile_name );
-		$options           = [];
+		$profile           = EnvironmentConfigResolver::normalize_aliases(
+			$this->get_profile_with_fallback( $command, $profile_test_type, $profile_name )
+		);
+		$environment       = [];
 
 		$environment_name = null;
 		$cli_environment  = $input->hasOption( 'environment' ) ? $input->getOption( 'environment' ) : null;
-		if ( is_string( $cli_environment ) ) {
+		if ( $input->hasOption( 'environment' ) && is_string( $cli_environment ) ) {
 			$environment_name = $cli_environment;
 		} elseif ( isset( $profile['environment'] ) && is_string( $profile['environment'] ) ) {
 			$environment_name = $profile['environment'];
 		}
 
 		if ( $environment_name !== null ) {
-			$options = EnvironmentConfigResolver::normalize_aliases( $command->get_environment_config( $environment_name ) );
+			$environment = EnvironmentConfigResolver::normalize_aliases( $command->get_environment_config( $environment_name ) );
 		}
 
-		foreach ( $profile as $key => $value ) {
-			if ( $key === 'environment' ) {
-				continue;
-			}
+		$effective_config = array_replace( $environment, $profile );
+		$payload_keys     = array_values( array_diff( array_keys( $options_to_send ), [ 'environment', 'remote', 'sut', 'zip' ] ) );
+		$this->validate_effective_remote_config( $effective_config, $payload_keys );
 
-			$options[ $key ] = $value;
-		}
+		$options = array_intersect_key( $effective_config, array_flip( $payload_keys ) );
 
-		foreach ( array_keys( $options_to_send ) as $opt_name ) {
+		foreach ( $payload_keys as $opt_name ) {
 			if ( ! $input->hasOption( $opt_name ) ) {
 				continue;
 			}
@@ -212,10 +212,15 @@ private function build_options(
 			}
 		}
 
-		$sut_arg = $input->getArgument( 'sut' ) ?: ( $options['sut']['slug'] ?? '' );
+		$positional_sut = $input->getArgument( 'sut' );
+		$effective_sut  = isset( $effective_config['sut'] ) && is_array( $effective_config['sut'] )
+			? $effective_config['sut']
+			: [];
+		$sut_arg        = $positional_sut ?: ( $effective_sut['slug'] ?? '' );
 		if ( empty( $sut_arg ) ) {
 			throw new \InvalidArgumentException( 'No System-Under-Test specified (argument or profile).' );
 		}
+		$this->validate_remote_sut_source( $input, $positional_sut, $effective_sut );
 
 		$options['woo_id'] = $this->slug_or_id_to_id( (string) $sut_arg );
 		$this->process_zip_option( $input, $output, (string) $sut_arg, $options );
@@ -227,6 +232,59 @@ private function build_options(
 		return $options;
 	}
 
+	/**
+	 * @param array<string,mixed> $effective_config Selected profile and environment configuration.
+	 * @param array<string>       $payload_keys     Manager-supported payload keys.
+	 */
+	private function validate_effective_remote_config( array $effective_config, array $payload_keys ): void {
+		$allowed     = array_merge( $payload_keys, [ 'environment', 'sut' ] );
+		$unsupported = [];
+
+		foreach ( $effective_config as $key => $value ) {
+			if ( in_array( $key, $allowed, true ) || $this->is_empty_config_value( $value ) ) {
+				continue;
+			}
+
+			$unsupported[] = $key;
+		}
+
+		if ( empty( $unsupported ) ) {
+			return;
+		}
+
+		sort( $unsupported );
+		throw new \InvalidArgumentException( sprintf(
+			'Remote execution does not support effective qit.json option(s): %s.',
+			implode( ', ', $unsupported )
+		) );
+	}
+
+	/**
+	 * @param mixed $value
+	 */
+	private function is_empty_config_value( $value ): bool {
+		return $value === null || $value === false || $value === '' || $value === [];
+	}
+
+	/**
+	 * @param QITInput            $input          The command input.
+	 * @param string|null         $positional_sut Explicit positional SUT override.
+	 * @param array<string,mixed> $effective_sut
+	 */
+	private function validate_remote_sut_source( QITInput $input, ?string $positional_sut, array $effective_sut ): void {
+		if ( ! empty( $positional_sut ) || $input->hasOption( 'zip' ) ) {
+			return;
+		}
+
+		$source      = $effective_sut['source'] ?? [];
+		$source_type = is_array( $source ) ? ( $source['type'] ?? null ) : null;
+		if ( in_array( $source_type, [ 'build', 'local', 'url' ], true ) ) {
+			throw new \InvalidArgumentException(
+				'Remote execution does not infer development SUT sources from qit.json. Pass the source explicitly with --zip.'
+			);
+		}
+	}
+
 	/**
 	 * @return array<string,mixed>
 	 */

src/tests/unit/ExtensionResolverWooVersionTest.php

@@ -89,12 +89,31 @@ public function test_resolve_source_maps_woo_nightly_to_github_url(): void {
 	}
 
 	public function test_resolve_source_leaves_woo_dev_url_untouched_when_already_a_url(): void {
-		$ext          = $this->woo( '11.0.0-dev', 'url' );
-		$ext->source  = 'https://example.com/custom-woocommerce.zip';
+		$ext         = $this->woo( '11.0.0-dev', 'url' );
+		$ext->source = 'https://example.com/custom-woocommerce.zip';
 		$this->invoke( $this->resolver(), 'resolve_extension_source', $ext );
 
 		// An explicit url source is already resolved and must not be overridden.
 		$this->assertSame( 'url', $ext->from );
 		$this->assertSame( 'https://example.com/custom-woocommerce.zip', $ext->source );
 	}
+
+	public function test_resolve_extracts_version_from_cached_local_theme(): void {
+		$directory = sys_get_temp_dir() . '/qit-local-theme-' . uniqid();
+		mkdir( $directory );
+		file_put_contents( $directory . '/style.css', "/*\nTheme Name: Local Theme\nVersion: 1.2.3\n*/" );
+
+		$extension            = new Extension( 'local-theme', 'theme' );
+		$extension->from      = 'local';
+		$extension->source    = $directory;
+		$extension->directory = $directory;
+
+		try {
+			$this->resolver()->resolve( [ $extension ], sys_get_temp_dir() );
+			$this->assertSame( '1.2.3', $extension->version );
+		} finally {
+			unlink( $directory . '/style.css' );
+			rmdir( $directory );
+		}
+	}
 }