diff --git a/docs/PUSH-TERMINOLOGY.md b/docs/PUSH-TERMINOLOGY.md index 367eae1f..df2251e3 100644 --- a/docs/PUSH-TERMINOLOGY.md +++ b/docs/PUSH-TERMINOLOGY.md @@ -66,6 +66,9 @@ local relative path to a document-root-relative path. patch result. To copy only source changes, the source snapshots from the last sync and now are the patch base and result. The planner owns the index diff and the active deletion roots file. +- A **file sync patch processor** builds a fresh local index, then runs a file + sync patch planner in the direction selected by its start method. Its cursor + owns both phases, so a caller can store it unchanged. - The **pull index WAL** records completed pull mutations awaiting application to the remote and local indexes. - A **pull plan** lists remote absolute paths still scheduled for download or @@ -306,18 +309,19 @@ The local index contains the fresh filesystem-root scan the sender saved after a target-confirmed files-push commit, advanced path by path by later completed files-pull mutations. Files-pull does not scan unrelated paths or accept their pending local changes. -PushPlan diffs its fresh local index against the local index its caller -supplies. Its FileSyncPatchPlanner owns the FileIndexDiffProcessor and the -active deletion roots file. That file remembers directory deletions which -cover index paths the planner has not processed yet. +PushPlan starts FileSyncPatchProcessor with the supplied local index as the +patch base and the fresh local index as the patch result. The processor owns +FreshLocalIndexProcessor and FileSyncPatchPlanner. The planner owns +FileIndexDiffProcessor and the active deletion roots file. That file remembers +directory deletions which cover index paths the planner has not processed yet. The PushPlan cursor is stored in `sender.json`. It contains `plan_directory`, -`local_index_file`, and the current -planning position. During `indexing`, that position contains the complete -FreshLocalIndexProcessor cursor. During `diffing`, it contains the output -offsets and the complete FileSyncPatchPlanner cursor. PushPlan stores either -nested cursor without unpacking or rebuilding it. The active -deletion roots file is append-only; each entry links to the preceding active +`local_index_file`, and the current planning position. While processing, that +position contains the complete FileSyncPatchProcessor cursor and both output +offsets. PushPlan stores the processor cursor without unpacking or rebuilding +it. The processor cursor contains the complete FreshLocalIndexProcessor cursor +while scanning and the complete FileSyncPatchPlanner cursor while planning. +The active deletion roots file is append-only; each entry links to the preceding active directory. The exclusions have a maximum of 100 paths. The `sender.json` phases are `creating`, `finishing_previous_commit`, `starting_plan`, `planning`, `pushing_paths`, `pushing_deletes`, `committing`, @@ -523,15 +527,9 @@ Use these names verbatim inside `PushPlan`: | Index entry and shape | `$index_entry`, `$local_index_entry`, `$local_index_entry_shape`, `index_entry_shape()` | | Cursor | `$cursor`, `get_cursor()` | | Plan-owned excluded paths | `$excluded_paths_file` | -| Fresh local index processor | `FreshLocalIndexProcessor`, `$fresh_local_index_processor` | -| Fresh local indexing cursor | `fresh_local_index_cursor`, `$fresh_local_index_cursor` | -| Fresh local index byte offset | `$fresh_local_index_byte_offset` | -| Open fresh local index | `$fresh_local_index_handle` | -| Combined index bytes | `$index_bytes_total` | -| File-sync planner cursor | `file_sync_planner_cursor`, `$file_sync_planner_cursor` | +| File sync patch processor | `FileSyncPatchProcessor`, `$file_sync_patch_processor` | +| File sync patch processor cursor | `file_sync_patch_processor_cursor`, `$file_sync_patch_processor_cursor` | | Plan progress | `get_progress()` | -| Start index diff | `start_index_diff()` | -| Seek an index file | `seek_index_file_to_byte_offset()`, `$index_file_handle` | | Open push-plan output file | `open_push_plan_output_file_at_byte_offset()`, `$push_plan_output_file_handle` | ## Protocol names diff --git a/packages/reprint-client/src/lib/index/class-file-sync-patch-planner.php b/packages/reprint-client/src/lib/index/class-file-sync-patch-planner.php index 6e18c35b..13756efd 100644 --- a/packages/reprint-client/src/lib/index/class-file-sync-patch-planner.php +++ b/packages/reprint-client/src/lib/index/class-file-sync-patch-planner.php @@ -86,7 +86,7 @@ * interruption. resume() ignores those bytes. * * @phpstan-type IndexDiffCursor array{old_index_byte_offset:int,new_index_byte_offset:int,preceding_new_index_entry_path_b64:string|null} - * @phpstan-type Cursor array{patch_base_index_file:string,patch_result_index_file:string,active_deletion_roots_file:string,included_index_path_roots:list,excluded_index_path_roots:list,index_diff_cursor:IndexDiffCursor,active_deletion_root_byte_offset:int|null} + * @phpstan-type Cursor array{patch_base_index_file_b64:string,patch_result_index_file_b64:string,active_deletion_roots_file_b64:string,included_index_path_roots_b64:list,excluded_index_path_roots_b64:list,index_diff_cursor:IndexDiffCursor,active_deletion_root_byte_offset:int|null} * @phpstan-type ActiveDeletionRoot array{path:string,previous_byte_offset:int|null} * @phpstan-type ExpectedSource array{type:string,size:int,ctime:int} * @phpstan-type DeleteOperation array{action:'delete',path:string} @@ -152,11 +152,23 @@ public static function create( } return self::resume( [ - "patch_base_index_file" => $patch_base_index_file, - "patch_result_index_file" => $patch_result_index_file, - "active_deletion_roots_file" => $active_deletion_roots_file, - "included_index_path_roots" => $included_index_path_roots, - "excluded_index_path_roots" => $excluded_index_path_roots, + "patch_base_index_file_b64" => base64_encode( + $patch_base_index_file + ), + "patch_result_index_file_b64" => base64_encode( + $patch_result_index_file + ), + "active_deletion_roots_file_b64" => base64_encode( + $active_deletion_roots_file + ), + "included_index_path_roots_b64" => array_map( + "base64_encode", + $included_index_path_roots + ), + "excluded_index_path_roots_b64" => array_map( + "base64_encode", + $excluded_index_path_roots + ), "index_diff_cursor" => [ "old_index_byte_offset" => 0, "new_index_byte_offset" => 0, @@ -176,11 +188,11 @@ public static function create( * @param array $cursor { * Cursor returned by get_cursor(). * - * @type string $patch_base_index_file Tree state before the patch. - * @type string $patch_result_index_file Tree state described by the patch. - * @type string $active_deletion_roots_file State for active directory deletions. - * @type list $included_index_path_roots Roots within which changes may be planned. - * @type list $excluded_index_path_roots Roots which changes must not affect. + * @type string $patch_base_index_file_b64 Base64-encoded path to the tree state before the patch. + * @type string $patch_result_index_file_b64 Base64-encoded path to the tree state described by the patch. + * @type string $active_deletion_roots_file_b64 Base64-encoded path to the active directory-deletion state. + * @type list $included_index_path_roots_b64 Base64-encoded roots within which changes may be planned. + * @type list $excluded_index_path_roots_b64 Base64-encoded roots which changes must not affect. * @type array $index_diff_cursor File-index diff cursor. * @type int|null $active_deletion_root_byte_offset Active deletion-root offset. * } @@ -191,24 +203,40 @@ public static function resume(array $cursor): self { $planner = new self(); $planner->cursor = $cursor; - $planner->included_index_path_roots = - $cursor["included_index_path_roots"]; - $planner->excluded_index_path_roots = - $cursor["excluded_index_path_roots"]; + $patch_base_index_file = self::decode_cursor_path( + $cursor["patch_base_index_file_b64"], + "patch base index file" + ); + $patch_result_index_file = self::decode_cursor_path( + $cursor["patch_result_index_file_b64"], + "patch result index file" + ); + $active_deletion_roots_file = self::decode_cursor_path( + $cursor["active_deletion_roots_file_b64"], + "active deletion roots file" + ); + $planner->included_index_path_roots = array_map( + [self::class, "decode_index_path_root"], + $cursor["included_index_path_roots_b64"] + ); + $planner->excluded_index_path_roots = array_map( + [self::class, "decode_index_path_root"], + $cursor["excluded_index_path_roots_b64"] + ); $planner->index_diff = FileIndexDiffProcessor::resume( - $cursor["patch_base_index_file"], - $cursor["patch_result_index_file"], + $patch_base_index_file, + $patch_result_index_file, $cursor["index_diff_cursor"] ); $planner->active_deletion_roots_handle = fopen( - $cursor["active_deletion_roots_file"], + $active_deletion_roots_file, "a+b" ); if (!is_resource($planner->active_deletion_roots_handle)) { $planner->index_diff->close(); throw new RuntimeException( "Failed to open the active deletion roots file: " - . $cursor["active_deletion_roots_file"] + . $active_deletion_roots_file ); } $planner->active_deletion_root = @@ -314,9 +342,13 @@ public function next_path(): bool ); // Find the highest patch-base directory without a patch-result - // entry below it. Only the adjacent result entries can neighbor - // each parent in byte order, so no index rescan is needed. - $candidate_path_to_delete = $index_path; + // entry below it which path selection allows us to delete. A + // higher directory may sit outside an included root or contain an + // excluded root, so keep looking below it. Only the adjacent + // result entries can neighbor each parent in byte order, so no + // index rescan is needed. + $candidate_path_to_delete = + $this->path_may_change($index_path) ? $index_path : null; $index_path_components = wp_unix_path_segments($index_path); $candidate_path_components = []; for ( @@ -335,6 +367,7 @@ public function next_path(): bool $this->index_diff->get_preceding_path_in_new_index(), $this->index_diff->get_following_path_in_new_index() ) + && $this->path_may_change($candidate_path) ) { $candidate_path_to_delete = $candidate_path; break; @@ -342,7 +375,7 @@ public function next_path(): bool } if ( !$patch_base_empty_directory_is_implied_by_patch_result_descendant - && $this->path_may_change($candidate_path_to_delete) + && $candidate_path_to_delete !== null && !$this->active_deletion_root_covers_path($index_path) ) { $path_to_delete = $candidate_path_to_delete; @@ -460,11 +493,11 @@ public function get_operation(): ?array * @return array { * Cursor for resume(). * - * @type string $patch_base_index_file Tree state before the patch. - * @type string $patch_result_index_file Tree state described by the patch. - * @type string $active_deletion_roots_file State for active directory deletions. - * @type list $included_index_path_roots Roots within which changes may be planned. - * @type list $excluded_index_path_roots Roots which changes must not affect. + * @type string $patch_base_index_file_b64 Base64-encoded path to the tree state before the patch. + * @type string $patch_result_index_file_b64 Base64-encoded path to the tree state described by the patch. + * @type string $active_deletion_roots_file_b64 Base64-encoded path to the active directory-deletion state. + * @type list $included_index_path_roots_b64 Base64-encoded roots within which changes may be planned. + * @type list $excluded_index_path_roots_b64 Base64-encoded roots which changes must not affect. * @type array $index_diff_cursor File-index diff cursor. * @type int|null $active_deletion_root_byte_offset Active deletion-root offset. * } @@ -668,6 +701,33 @@ private function path_may_change(string $index_path): bool return true; } + /** Decodes one arbitrary-byte path root stored in a JSON cursor. */ + private static function decode_index_path_root( + string $index_path_root_b64 + ): string { + $index_path_root = base64_decode($index_path_root_b64, true); + if ($index_path_root === false) { + throw new InvalidArgumentException( + "File sync patch planner cursor contains an invalid base64 path root." + ); + } + return $index_path_root; + } + + /** Decodes one arbitrary-byte file path stored in the JSON cursor. */ + private static function decode_cursor_path( + string $encoded_path, + string $field_name + ): string { + $path = base64_decode($encoded_path, true); + if ($path === false) { + throw new InvalidArgumentException( + "File sync patch planner cursor contains an invalid base64 {$field_name}." + ); + } + return $path; + } + /** Rejects calls after close(). */ private function assert_open(): void { diff --git a/packages/reprint-client/src/lib/index/class-file-sync-patch-processor.php b/packages/reprint-client/src/lib/index/class-file-sync-patch-processor.php new file mode 100644 index 00000000..a299030e --- /dev/null +++ b/packages/reprint-client/src/lib/index/class-file-sync-patch-processor.php @@ -0,0 +1,479 @@ +next_step(); + * $operation = $processor->get_operation(); + * if ($operation !== null) { + * append_operation($operation); + * } + * $processor->flush_pending_outputs(); + * save_cursor($processor->get_cursor()); + * } while ($has_next_step); + * $processor->close(); + * + * @phpstan-type FileIndexCursor array{stack:list} + * @phpstan-type FreshIndexPosition array{phase:'indexing',file_index_cursor:FileIndexCursor,fresh_local_index_byte_offset:int}|array{phase:'sorting'}|array{phase:'complete'} + * @phpstan-type FreshIndexCursor array{fresh_local_index_file_b64:string,filesystem_root_b64:string,storage_path_b64:string,include_caches:bool,position:FreshIndexPosition} + * @phpstan-type PlannerIndexDiffCursor array{old_index_byte_offset:int,new_index_byte_offset:int,preceding_new_index_entry_path_b64:string|null} + * @phpstan-type PlannerCursor array{patch_base_index_file_b64:string,patch_result_index_file_b64:string,active_deletion_roots_file_b64:string,included_index_path_roots_b64:list,excluded_index_path_roots_b64:list,index_diff_cursor:PlannerIndexDiffCursor,active_deletion_root_byte_offset:int|null} + * @phpstan-type FreshTreePosition array{phase:'indexing'|'sorting'|'starting_patch',patch_base_index_file_b64:string,patch_result_index_file_b64:string,included_index_path_roots_b64:list,excluded_index_path_roots_b64:list,fresh_local_index_cursor:FreshIndexCursor} + * @phpstan-type Position FreshTreePosition|array{phase:'planning',file_sync_patch_planner_cursor:PlannerCursor}|array{phase:'complete'} + * @phpstan-type Cursor array{fresh_local_index_file_b64:string,position:Position} + * @phpstan-type SyncOperation array{action:'copy'|'delete'|'replace',path:string,expected_source?:array{type:string,size:int,ctime:int}} + */ +final class FileSyncPatchProcessor { + /** @var Cursor */ + private array $cursor; + + /** Fresh local index path decoded from the cursor. */ + private string $fresh_local_index_file; + + private FreshLocalIndexProcessor $fresh_local_index_processor; + + private FileSyncPatchPlanner $patch_planner; + + /** @var SyncOperation|null */ + private ?array $operation = null; + + private bool $closed = false; + + /** + * Plans the patch which changes a saved index into the current local tree. + * + * @param string $work_directory Existing directory for the fresh index and planner state. + * @param string $filesystem_root Filesystem root scanned for the fresh index. + * @param string $patch_base_index_file Saved tree before the patch, or a missing file for an empty tree. + * @param string $storage_path Reprint storage path omitted from the fresh index. + * @param list $included_index_path_roots Roots within which changes may be planned. + * @param list $excluded_index_path_roots Roots which changes must not affect. + * @param bool $include_caches Whether the local scan includes cache directories. + */ + public static function start_to_fresh_local_tree( + string $work_directory, + string $filesystem_root, + string $patch_base_index_file, + string $storage_path, + array $included_index_path_roots = [""], + array $excluded_index_path_roots = [], + bool $include_caches = false + ): self { + $work_directory = trim_right_slash($work_directory); + return self::create( + $work_directory, + $filesystem_root, + $patch_base_index_file, + wp_join_unix_paths($work_directory, "fresh_local_index.jsonl"), + $storage_path, + $included_index_path_roots, + $excluded_index_path_roots, + $include_caches + ); + } + + /** + * Plans the patch which changes the current local tree into a saved index. + * + * Copy and replace operations read their expected source state from the + * supplied patch-result index. + * + * @param string $work_directory Existing directory for the fresh index and planner state. + * @param string $filesystem_root Filesystem root scanned for the fresh index. + * @param string $patch_result_index_file Tree which the patch must produce. + * @param string $storage_path Reprint storage path omitted from the fresh index. + * @param list $included_index_path_roots Roots within which changes may be planned. + * @param list $excluded_index_path_roots Roots which changes must not affect. + * @param bool $include_caches Whether the local scan includes cache directories. + */ + public static function start_from_fresh_local_tree( + string $work_directory, + string $filesystem_root, + string $patch_result_index_file, + string $storage_path, + array $included_index_path_roots = [""], + array $excluded_index_path_roots = [], + bool $include_caches = false + ): self { + $work_directory = trim_right_slash($work_directory); + return self::create( + $work_directory, + $filesystem_root, + wp_join_unix_paths($work_directory, "fresh_local_index.jsonl"), + $patch_result_index_file, + $storage_path, + $included_index_path_roots, + $excluded_index_path_roots, + $include_caches + ); + } + + /** @phpstan-param Cursor $cursor Cursor returned by get_cursor(). */ + public static function resume(array $cursor): self + { + $processor = new self(); + $processor->cursor = $cursor; + $processor->fresh_local_index_file = self::decode_cursor_path( + $cursor["fresh_local_index_file_b64"], + "fresh local index file" + ); + $position = $cursor["position"]; + if ( + $position["phase"] === "indexing" + || $position["phase"] === "sorting" + || $position["phase"] === "starting_patch" + ) { + $processor->fresh_local_index_processor = + FreshLocalIndexProcessor::resume( + $position["fresh_local_index_cursor"] + ); + } elseif ($position["phase"] === "planning") { + $processor->patch_planner = FileSyncPatchPlanner::resume( + $position["file_sync_patch_planner_cursor"] + ); + } + return $processor; + } + + /** + * Performs one local-index or patch-planning step. + * + * get_operation() returns the operation selected by a planning step and + * null for every other step. The last planning step returns false and may + * carry the last operation. Read get_operation() before using the boolean + * to end the loop. Later calls remain false and return no operation. + */ + public function next_step(): bool + { + $position = $this->cursor["position"]; + if ($position["phase"] === "complete") { + $this->operation = null; + return false; + } + if ($this->closed) { + throw new LogicException( + "Cannot take a file sync patch step after close()." + ); + } + $this->operation = null; + + if ( + $position["phase"] === "indexing" + || $position["phase"] === "sorting" + ) { + $this->fresh_local_index_processor->next_step(); + $fresh_local_index_cursor = + $this->fresh_local_index_processor->get_cursor(); + $fresh_local_index_phase = + $this->fresh_local_index_processor->get_phase(); + $position["phase"] = $fresh_local_index_phase === "complete" + ? "starting_patch" + : $fresh_local_index_phase; + $position["fresh_local_index_cursor"] = + $fresh_local_index_cursor; + $this->cursor["position"] = $position; + return true; + } + + if ($position["phase"] === "starting_patch") { + $this->fresh_local_index_processor->close(); + $included_index_path_roots = []; + foreach ( + $position["included_index_path_roots_b64"] + as $included_index_path_root_b64 + ) { + $included_index_path_root = base64_decode( + $included_index_path_root_b64, + true + ); + if ($included_index_path_root === false) { + throw new InvalidArgumentException( + "File sync patch processor cursor contains an invalid base64 included path root." + ); + } + $included_index_path_roots[] = $included_index_path_root; + } + $excluded_index_path_roots = []; + foreach ( + $position["excluded_index_path_roots_b64"] + as $excluded_index_path_root_b64 + ) { + $excluded_index_path_root = base64_decode( + $excluded_index_path_root_b64, + true + ); + if ($excluded_index_path_root === false) { + throw new InvalidArgumentException( + "File sync patch processor cursor contains an invalid base64 excluded path root." + ); + } + $excluded_index_path_roots[] = $excluded_index_path_root; + } + $this->patch_planner = FileSyncPatchPlanner::create( + self::decode_cursor_path( + $position["patch_base_index_file_b64"], + "patch base index file" + ), + self::decode_cursor_path( + $position["patch_result_index_file_b64"], + "patch result index file" + ), + wp_join_unix_paths( + dirname($this->fresh_local_index_file), + "deleted_directories_stack.jsonl" + ), + $included_index_path_roots, + $excluded_index_path_roots + ); + $this->cursor["position"] = [ + "phase" => "planning", + "file_sync_patch_planner_cursor" => + $this->patch_planner->get_cursor(), + ]; + return true; + } + + if (!$this->patch_planner->next_path()) { + $this->cursor["position"] = ["phase" => "complete"]; + return false; + } + $this->operation = $this->patch_planner->get_operation(); + if ($this->patch_planner->is_complete()) { + $this->cursor["position"] = ["phase" => "complete"]; + return false; + } + $this->cursor["position"] = [ + "phase" => "planning", + "file_sync_patch_planner_cursor" => + $this->patch_planner->get_cursor(), + ]; + return true; + } + + /** + * Returns the operation selected by the latest planning step. + * + * Delete operations contain only `action` and `path`. Copy and replace + * operations also contain the result index entry which must be copied. + * Non-planning steps and processed paths which need no change return null. + * + * @return array|null { + * @type string $action `copy`, `delete`, or `replace`. + * @type string $path Local relative path selected by the patch. + * @type array $expected_source { + * Result index entry required by `copy` and `replace`. + * + * @type string $type Expected `file`, `link`, or `dir` type. + * @type int $size Expected size. + * @type int $ctime Expected inode change time. + * } + * } + * @phpstan-return SyncOperation|null + */ + public function get_operation(): ?array + { + return $this->operation; + } + + /** @phpstan-return Cursor Cursor after the latest completed step. */ + public function get_cursor(): array + { + return $this->cursor; + } + + /** Returns `indexing`, `sorting`, `starting_patch`, `planning`, or `complete`. */ + public function get_phase(): string + { + return $this->cursor["position"]["phase"]; + } + + /** Returns the processor-owned fresh local index path. */ + public function get_fresh_local_index_path(): string + { + return $this->fresh_local_index_file; + } + + /** + * Returns index bytes consumed while planning the patch. + * + * @return array { + * @type string $phase Current processor phase. + * @type int $index_bytes_done Index bytes consumed. Present while planning. + * @type int $index_bytes_total Combined index size. Present while planning. + * } + * @phpstan-return array{phase:string,index_bytes_done?:int,index_bytes_total?:int} + */ + public function get_progress(): array + { + $position = $this->cursor["position"]; + $progress = ["phase" => $position["phase"]]; + if ($position["phase"] !== "planning") { + return $progress; + } + $file_sync_patch_planner_cursor = + $position["file_sync_patch_planner_cursor"]; + $patch_base_index_file = self::decode_cursor_path( + $file_sync_patch_planner_cursor["patch_base_index_file_b64"], + "patch base index file" + ); + $patch_result_index_file = self::decode_cursor_path( + $file_sync_patch_planner_cursor["patch_result_index_file_b64"], + "patch result index file" + ); + $patch_base_index_bytes = is_file( + $patch_base_index_file + ) + ? filesize($patch_base_index_file) + : 0; + $patch_result_index_bytes = is_file( + $patch_result_index_file + ) + ? filesize($patch_result_index_file) + : 0; + if ( + !is_int($patch_base_index_bytes) + || !is_int($patch_result_index_bytes) + ) { + return $progress; + } + $index_bytes_total = $patch_base_index_bytes + + $patch_result_index_bytes; + $index_diff_cursor = + $file_sync_patch_planner_cursor["index_diff_cursor"]; + $index_bytes_done = $index_diff_cursor["new_index_byte_offset"] + + $index_diff_cursor["old_index_byte_offset"]; + $progress["index_bytes_done"] = min( + $index_bytes_done, + $index_bytes_total + ); + $progress["index_bytes_total"] = $index_bytes_total; + return $progress; + } + + /** Flushes nested output before the caller stores get_cursor(). */ + public function flush_pending_outputs(): void + { + if (isset($this->fresh_local_index_processor)) { + $this->fresh_local_index_processor->flush_pending_output(); + } + if (isset($this->patch_planner)) { + $this->patch_planner->flush_pending_outputs(); + } + } + + /** Closes retained handles. Repeated calls do nothing. */ + public function close(): void + { + if ($this->closed) { + return; + } + if (isset($this->fresh_local_index_processor)) { + $this->fresh_local_index_processor->close(); + } + if (isset($this->patch_planner)) { + $this->patch_planner->close(); + } + $this->closed = true; + } + + private static function create( + string $work_directory, + string $filesystem_root, + string $patch_base_index_file, + string $patch_result_index_file, + string $storage_path, + array $included_index_path_roots, + array $excluded_index_path_roots, + bool $include_caches + ): self { + if (!is_dir($work_directory)) { + throw new LogicException( + "Cannot start file sync patch processing without its work directory: {$work_directory}" + ); + } + $processor = new self(); + $fresh_local_index_file = wp_join_unix_paths( + $work_directory, + "fresh_local_index.jsonl" + ); + $processor->fresh_local_index_file = $fresh_local_index_file; + $processor->fresh_local_index_processor = + FreshLocalIndexProcessor::start( + $fresh_local_index_file, + $filesystem_root, + $storage_path, + $include_caches + ); + $processor->cursor = [ + "fresh_local_index_file_b64" => base64_encode( + $fresh_local_index_file + ), + "position" => [ + "phase" => "indexing", + "patch_base_index_file_b64" => base64_encode( + $patch_base_index_file + ), + "patch_result_index_file_b64" => base64_encode( + $patch_result_index_file + ), + "included_index_path_roots_b64" => array_map( + "base64_encode", + $included_index_path_roots + ), + "excluded_index_path_roots_b64" => array_map( + "base64_encode", + $excluded_index_path_roots + ), + "fresh_local_index_cursor" => + $processor->fresh_local_index_processor->get_cursor(), + ], + ]; + return $processor; + } + + /** Decodes one arbitrary-byte path stored in the JSON cursor. */ + private static function decode_cursor_path( + string $encoded_path, + string $field_name + ): string { + $path = base64_decode($encoded_path, true); + if ($path === false) { + throw new InvalidArgumentException( + "File sync patch processor cursor contains an invalid base64 {$field_name}." + ); + } + return $path; + } +} diff --git a/packages/reprint-client/src/lib/index/class-fresh-local-index-processor.php b/packages/reprint-client/src/lib/index/class-fresh-local-index-processor.php index 98d9a6bb..e1f4774c 100644 --- a/packages/reprint-client/src/lib/index/class-fresh-local-index-processor.php +++ b/packages/reprint-client/src/lib/index/class-fresh-local-index-processor.php @@ -47,13 +47,19 @@ * @phpstan-type SortingPosition array{phase:'sorting'} * @phpstan-type CompletePosition array{phase:'complete'} * @phpstan-type Position IndexingPosition|SortingPosition|CompletePosition - * @phpstan-type Cursor array{fresh_local_index_file:string,filesystem_root:string,storage_path:string,position:Position} + * @phpstan-type Cursor array{fresh_local_index_file_b64:string,filesystem_root_b64:string,storage_path_b64:string,include_caches:bool,position:Position} */ final class FreshLocalIndexProcessor { /** @var Cursor Current cursor returned to the caller. */ private array $cursor; + /** Fresh local index path decoded from the cursor. */ + private string $fresh_local_index_file; + + /** Filesystem root decoded from the cursor. */ + private string $filesystem_root; + /** Filesystem traversal retained during indexing. */ private FileIndexProcessor $file_index_processor; @@ -72,14 +78,18 @@ final class FreshLocalIndexProcessor * @param string $fresh_local_index_file Output JSONL file. * @param string $filesystem_root Filesystem root represented by the index. * @param string $storage_path Reprint storage path omitted by FileIndexProcessor. + * @param bool $include_caches Whether generated caches and development files are indexed. */ public static function start( string $fresh_local_index_file, string $filesystem_root, - string $storage_path + string $storage_path, + bool $include_caches = false ): self { $processor = new self(); $filesystem_root = $processor->resolve_filesystem_root($filesystem_root); + $processor->fresh_local_index_file = $fresh_local_index_file; + $processor->filesystem_root = $filesystem_root; $processor->fresh_local_index_handle = fopen( $fresh_local_index_file, "w+b" @@ -93,13 +103,16 @@ public static function start( [$filesystem_root], $filesystem_root, false, - false, + $include_caches, $storage_path ); $processor->cursor = [ - "fresh_local_index_file" => $fresh_local_index_file, - "filesystem_root" => $filesystem_root, - "storage_path" => $storage_path, + "fresh_local_index_file_b64" => base64_encode( + $fresh_local_index_file + ), + "filesystem_root_b64" => base64_encode($filesystem_root), + "storage_path_b64" => base64_encode($storage_path), + "include_caches" => $include_caches, "position" => [ "phase" => "indexing", "file_index_cursor" => @@ -124,27 +137,39 @@ public static function resume(array $cursor): self { $processor = new self(); $processor->cursor = $cursor; + $processor->fresh_local_index_file = self::decode_cursor_path( + $cursor["fresh_local_index_file_b64"], + "fresh local index file" + ); + $processor->filesystem_root = self::decode_cursor_path( + $cursor["filesystem_root_b64"], + "filesystem root" + ); + $storage_path = self::decode_cursor_path( + $cursor["storage_path_b64"], + "storage path" + ); $position = $cursor["position"]; if ($position["phase"] !== "indexing") { return $processor; } $filesystem_root = $processor->resolve_filesystem_root( - $cursor["filesystem_root"] + $processor->filesystem_root ); - if ($filesystem_root !== $cursor["filesystem_root"]) { + if ($filesystem_root !== $processor->filesystem_root) { throw new RuntimeException( "The fresh local index filesystem root no longer resolves to its saved path." ); } $processor->fresh_local_index_handle = fopen( - $cursor["fresh_local_index_file"], + $processor->fresh_local_index_file, "r+b" ); if (!is_resource($processor->fresh_local_index_handle)) { throw new RuntimeException( "Failed to reopen the fresh local index: " - . $cursor["fresh_local_index_file"] + . $processor->fresh_local_index_file ); } if ( @@ -172,8 +197,8 @@ public static function resume(array $cursor): self JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR ), false, - false, - $cursor["storage_path"] + $cursor["include_caches"], + $storage_path ); return $processor; } @@ -197,10 +222,10 @@ public function next_step(): bool } if ($position["phase"] === "sorting") { - if (!sort_index_file($this->cursor["fresh_local_index_file"])) { + if (!sort_index_file($this->fresh_local_index_file)) { throw new RuntimeException( "Failed to sort the fresh local index: " - . $this->cursor["fresh_local_index_file"] + . $this->fresh_local_index_file ); } $this->cursor["position"] = ["phase" => "complete"]; @@ -252,7 +277,7 @@ public function next_step(): bool $local_relative_path = relative_path_under( $file_index_processor_entry["path"], - $this->cursor["filesystem_root"] + $this->filesystem_root ); if ($local_relative_path === null) { throw new LogicException( @@ -363,6 +388,20 @@ private function resolve_filesystem_root(string $filesystem_root): string return trim_right_slash($resolved_filesystem_root); } + /** Decodes one arbitrary-byte path stored in the JSON cursor. */ + private static function decode_cursor_path( + string $encoded_path, + string $field_name + ): string { + $path = base64_decode($encoded_path, true); + if ($path === false) { + throw new InvalidArgumentException( + "Fresh local index cursor contains an invalid base64 {$field_name}." + ); + } + return $path; + } + /** Closes the fresh local index retained during filesystem traversal. */ private function close_fresh_local_index_handle(): void { diff --git a/packages/reprint-client/src/lib/push/class-push-files-sender.php b/packages/reprint-client/src/lib/push/class-push-files-sender.php index fa9ad370..43cba99f 100644 --- a/packages/reprint-client/src/lib/push/class-push-files-sender.php +++ b/packages/reprint-client/src/lib/push/class-push-files-sender.php @@ -515,7 +515,16 @@ public function get_progress(): array $progress = ['phase' => $phase]; if ($phase === 'planning') { $plan_progress = $this->plan->get_progress(); - $progress['planning_phase'] = $plan_progress['phase']; + if ($plan_progress['phase'] === 'indexing') { + $progress['planning_phase'] = 'indexing'; + } elseif ( + $plan_progress['phase'] === 'sorting' + || $plan_progress['phase'] === 'starting_patch' + ) { + $progress['planning_phase'] = 'starting_diff'; + } else { + $progress['planning_phase'] = 'diffing'; + } if (isset($plan_progress['index_bytes_done'], $plan_progress['index_bytes_total'])) { $progress['index_bytes_done'] = $plan_progress['index_bytes_done']; $progress['index_bytes_total'] = $plan_progress['index_bytes_total']; @@ -755,20 +764,19 @@ private function next_plan_step(): void // output before storing the cursor. This avoids one flush and // state write per path while preserving each phase boundary. $has_next_step = true; - $plan_cursor = $this->plan->get_cursor(); - $plan_phase = $plan_cursor['position']['phase']; + $plan_phase = $this->plan->get_phase(); for ( $steps_processed = 0; $has_next_step && $steps_processed < self::MAXIMUM_PUSH_PLAN_STEPS_PER_SENDER_STEP; ++$steps_processed ) { $has_next_step = $this->plan->next_step(); - $plan_cursor = $this->plan->get_cursor(); - if ($plan_cursor['position']['phase'] !== $plan_phase) { + if ($this->plan->get_phase() !== $plan_phase) { break; } } $this->plan->flush_pending_outputs(); + $plan_cursor = $this->plan->get_cursor(); } catch (RuntimeException $exception) { $this->fail('local_io_error', $exception->getMessage()); return; diff --git a/packages/reprint-client/src/lib/push/class-push-plan.php b/packages/reprint-client/src/lib/push/class-push-plan.php index bbff65f3..424441f5 100644 --- a/packages/reprint-client/src/lib/push/class-push-plan.php +++ b/packages/reprint-client/src/lib/push/class-push-plan.php @@ -4,72 +4,38 @@ use function WordPress\Reprint\Exporter\relative_path_under; use function WordPress\Reprint\Exporter\trim_right_slash; -require_once __DIR__ . '/../index/class-file-sync-patch-planner.php'; -require_once __DIR__ . '/../index/class-fresh-local-index-processor.php'; +require_once __DIR__ . '/../index/class-file-sync-patch-processor.php'; // phpcs:disable WordPress.Security.EscapeOutput.ExceptionNotEscaped -- Journal failures are CLI/API values, never HTML output. /** - * Internal bounded local-index and change planner. + * Writes the path lists used by a files push. * - * PushPlan builds a path-sorted fresh local index, then diffs it against the - * local index supplied by its caller. It writes durable lists of - * local paths to push and local paths to delete without accumulating an index - * or path list in memory. + * FileSyncPatchProcessor scans the current local tree and compares its fresh + * local index with the saved local index. PushPlan writes each resulting + * operation to one of two files: JSONL paths to push and NUL-delimited paths to + * delete. It also counts the files and file bytes selected for push. * - * PushFilesSender or the files-diff command owns the caller-visible lifecycle, - * lock, top-level phase, result, and terminal behavior. PushPlan owns - * FreshLocalIndexProcessor, FileSyncPatchPlanner, the meaning of its cursor, - * and the two completed path lists. A caller which resumes across processes - * stores the cursor returned by get_cursor(). + * PushFilesSender owns the public lifecycle and stores the PushPlan cursor. + * PushPlan stores the FileSyncPatchProcessor cursor unchanged beside the + * durable offsets of its two output files. resume() truncates both outputs to + * those offsets before it appends more operations. + * PushFilesSender also owns the plan directory after the plan reaches a + * terminal result. PushPlan closes its handles but does not remove artifacts + * which the sender still needs to upload or save. * - * ## Durable boundary + * File and symlink changes use type, ctime, and size. Directory changes use the + * indexer's empty-directory entry; descendants represent non-empty + * directories. With no saved local index, the plan pushes every file, symlink, + * and empty directory and finds no deletion. Target exclusions remain in the + * fresh local index but produce no push or delete operation. * - * The PushPlan cursor contains one of four internal phases: `indexing`, - * `starting_diff`, `diffing`, or `complete`. A false next_step() result means - * both indexes reached EOF; the caller stores the returned cursor and closes - * the plan before changing its phase. The completed files remain in the - * caller-owned plan directory until the caller no longer needs them. + * Each next_step() call advances the processor once and writes at most one + * operation. Neither class loads an index or path list in memory. * - * ## Change detection - * - * ctime is machine-local, so the local index must describe the same filesystem - * root on the same local machine. The caller supplies the local index for its - * remote Reprint API URL. File and symlink changes are determined by type, - * ctime, and size. Directory changes use the indexer's empty-directory marker; - * non-empty directories are represented by their descendants. - * - * With no local index, every file, symlink, and empty directory is - * selected, and no deletion can be detected. Excluded paths are omitted from - * both path lists but remain in the fresh local index. - * - * The index reader trusts the entry values produced by the indexer. It retains - * failure handling for reading lines, decoding JSON, and decoding base64 paths. - * - * ## Durability and memory - * - * Each indexing step advances FreshLocalIndexProcessor once and stores its - * cursor unchanged. A separate step starts the index diff. Each diff step - * compares at most one path represented by either index and updates its next cursor. - * The owner flushes pending output before storing a cursor. `resume()` discards - * bytes beyond saved offsets, so an interrupted step cannot leave duplicate - * durable entries. - * - * FileSyncPatchPlanner retains the next entry from each index and writes the - * active directory-deletion roots needed to suppress redundant descendant - * deletions. PushPlan stores its cursor without unpacking it. Neither class - * loads an index, path list, or the active deletion roots file in full. - * - * @phpstan-type FreshLocalIndexFileIndexCursor array{stack:list} - * @phpstan-type FreshLocalIndexPosition array{phase:'indexing',file_index_cursor:FreshLocalIndexFileIndexCursor,fresh_local_index_byte_offset:int}|array{phase:'sorting'}|array{phase:'complete'} - * @phpstan-type FreshLocalIndexCursor array{fresh_local_index_file:string,filesystem_root:string,storage_path:string,position:FreshLocalIndexPosition} - * @phpstan-type IndexingCursor array{phase:'indexing',fresh_local_index_cursor:FreshLocalIndexCursor} - * @phpstan-type StartingDiffCursor array{phase:'starting_diff',fresh_local_index_cursor:FreshLocalIndexCursor} - * @phpstan-type FileSyncPlannerIndexDiffCursor array{old_index_byte_offset:int,new_index_byte_offset:int,preceding_new_index_entry_path_b64:string|null} - * @phpstan-type FileSyncPlannerCursor array{patch_base_index_file:string,patch_result_index_file:string,active_deletion_roots_file:string,included_index_path_roots:list,excluded_index_path_roots:list,index_diff_cursor:FileSyncPlannerIndexDiffCursor,active_deletion_root_byte_offset:int|null} - * @phpstan-type IndexDiffCursor array{phase:'diffing',file_sync_planner_cursor:FileSyncPlannerCursor,byte_offset_in_local_paths_to_push:int,byte_offset_in_local_paths_to_delete:int,local_paths_to_push_count:int|null,local_file_bytes_to_push:int|null} + * @phpstan-type ProcessingCursor array{phase:'processing',file_sync_patch_processor_cursor:array,byte_offset_in_local_paths_to_push:int,byte_offset_in_local_paths_to_delete:int,local_paths_to_push_count:int|null,local_file_bytes_to_push:int|null} * @phpstan-type CompleteCursor array{phase:'complete',local_paths_to_push_count:int|null,local_file_bytes_to_push:int|null} - * @phpstan-type PushPlanPosition IndexingCursor|StartingDiffCursor|IndexDiffCursor|CompleteCursor + * @phpstan-type PushPlanPosition ProcessingCursor|CompleteCursor * @phpstan-type PushPlanCursor array{plan_directory:string,local_index_file:string,document_root_local_relative_path:string,position:PushPlanPosition} */ class PushPlan @@ -89,36 +55,22 @@ class PushPlan /** @var string Raw NUL-delimited local paths to delete. */ private string $local_paths_to_delete; - /** @var string Plan-owned fresh local index file. */ - private string $fresh_local_index_file; - /** @var string Plan path containing receiver-owned exclusions for the active push. */ private string $excluded_paths_file; - /** @var string State for directory deletions which cover paths not processed yet. */ - private string $active_deletion_roots_file; - - /** @var list Receiver-owned paths that the plan must not push or delete. */ - private array $excluded_paths = []; - /** @var PushPlanCursor Current cursor returned to the caller. */ private array $cursor; /** @var bool Whether close() has closed this plan's file handles. */ private bool $closed = false; - /** Fresh local index retained during indexing. */ - private FreshLocalIndexProcessor $fresh_local_index_processor; - - /** File-sync patch planner retained during the diff phase. */ - private FileSyncPatchPlanner $patch_planner; + /** Local scan and patch planner retained while the plan is open. */ + private FileSyncPatchProcessor $file_sync_patch_processor; /** @var resource|null */ private $local_paths_to_push_handle = null; /** @var resource|null */ private $local_paths_to_delete_handle = null; - /** @var int|null Combined size of the two retained indexes during the index diff. */ - private ?int $index_bytes_total = null; /** * Starts a push plan by opening a fresh local index traversal. @@ -149,20 +101,64 @@ public static function start( if (!@copy($excluded_paths_path, $plan->excluded_paths_file)) { throw new RuntimeException("Failed to copy excluded paths into the push plan: {$excluded_paths_path}"); } - $plan->excluded_paths = $plan->load_excluded_paths(); - $plan->fresh_local_index_processor = FreshLocalIndexProcessor::start( - $plan->fresh_local_index_file, - $filesystem_root, - $plan->plan_directory - ); + $contents = file_get_contents($plan->excluded_paths_file); + if (!is_string($contents)) { + throw new RuntimeException( + "Failed to read excluded paths: {$plan->excluded_paths_file}" + ); + } + try { + $excluded_paths_b64 = json_decode( + $contents, + true, + 512, + JSON_THROW_ON_ERROR + ); + } catch (JsonException $exception) { + throw new RuntimeException( + "Failed to decode excluded paths: {$plan->excluded_paths_file}", + 0, + $exception + ); + } + /** @var list $excluded_paths_b64 */ + $excluded_index_path_roots = []; + foreach ($excluded_paths_b64 as $excluded_path_b64) { + $excluded_path = base64_decode($excluded_path_b64, true); + if ($excluded_path === false) { + throw new RuntimeException( + "Failed to decode an excluded path: {$plan->excluded_paths_file}" + ); + } + $excluded_index_path_roots[] = + $plan->document_root_local_relative_path === "" + ? $excluded_path + : wp_join_unix_paths( + $plan->document_root_local_relative_path, + $excluded_path + ); + } + $plan->file_sync_patch_processor = + FileSyncPatchProcessor::start_to_fresh_local_tree( + $plan->plan_directory, + $filesystem_root, + $plan->local_index_file, + $plan->plan_directory, + [$plan->document_root_local_relative_path], + $excluded_index_path_roots + ); $plan->cursor = [ "plan_directory" => $plan->plan_directory, "local_index_file" => $plan->local_index_file, "document_root_local_relative_path" => $plan->document_root_local_relative_path, "position" => [ - "phase" => "indexing", - "fresh_local_index_cursor" => - $plan->fresh_local_index_processor->get_cursor(), + "phase" => "processing", + "file_sync_patch_processor_cursor" => + $plan->file_sync_patch_processor->get_cursor(), + "byte_offset_in_local_paths_to_push" => 0, + "byte_offset_in_local_paths_to_delete" => 0, + "local_paths_to_push_count" => 0, + "local_file_bytes_to_push" => 0, ], ]; return $plan; @@ -179,28 +175,6 @@ public static function start( */ public static function resume(array $cursor): self { - if (!array_key_exists("document_root_local_relative_path", $cursor)) { - // Older cursors had no document-root mapping and used local relative paths unchanged. - $cursor["document_root_local_relative_path"] = ""; - } - $position = $cursor["position"]; - if ( - ($position["phase"] === "diffing" || $position["phase"] === "complete") - && !array_key_exists("local_paths_to_push_count", $position) - ) { - // Keep the plan single-pass when an older cursor has no path count. - $position["local_paths_to_push_count"] = null; - $cursor["position"] = $position; - } - if ( - ( $position["phase"] === "diffing" || $position["phase"] === "complete" ) - && !array_key_exists("local_file_bytes_to_push", $position) - ) { - // Keep the plan single-pass when an older cursor has no file byte total. - $position["local_file_bytes_to_push"] = null; - $cursor["position"] = $position; - } - $plan = new self( $cursor["plan_directory"], $cursor["local_index_file"], @@ -209,27 +183,16 @@ public static function resume(array $cursor): self $plan->cursor = $cursor; $position = $plan->cursor["position"]; if ($position["phase"] !== "complete") { - $plan->excluded_paths = $plan->load_excluded_paths(); - } - if ( - $position["phase"] === "indexing" - || $position["phase"] === "starting_diff" - ) { - $plan->fresh_local_index_processor = - FreshLocalIndexProcessor::resume( - $position["fresh_local_index_cursor"] + $plan->file_sync_patch_processor = + FileSyncPatchProcessor::resume( + $position["file_sync_patch_processor_cursor"] ); - } elseif ($position["phase"] === "diffing") { - $plan->open_plan_output_files( - $position["byte_offset_in_local_paths_to_push"], - $position["byte_offset_in_local_paths_to_delete"] - ); - $file_sync_planner_cursor = - $position["file_sync_planner_cursor"]; - $plan->patch_planner = FileSyncPatchPlanner::resume( - $file_sync_planner_cursor - ); - $plan->set_index_bytes_total(); + if ($plan->file_sync_patch_processor->get_phase() === "planning") { + $plan->open_plan_output_files( + $position["byte_offset_in_local_paths_to_push"], + $position["byte_offset_in_local_paths_to_delete"] + ); + } } return $plan; } @@ -269,21 +232,33 @@ public function get_local_paths_to_push_path(): string */ public function get_progress(): array { - $position = $this->cursor["position"]; - $progress = ["phase" => $position["phase"]]; - if ($position["phase"] !== "diffing" || $this->index_bytes_total === null) { - return $progress; + if ($this->cursor["position"]["phase"] === "complete") { + return ["phase" => "complete"]; + } + $processor_progress = + $this->file_sync_patch_processor->get_progress(); + $progress = ["phase" => $this->get_phase()]; + if (isset( + $processor_progress["index_bytes_done"], + $processor_progress["index_bytes_total"] + )) { + $progress["index_bytes_done"] = + $processor_progress["index_bytes_done"]; + $progress["index_bytes_total"] = + $processor_progress["index_bytes_total"]; } - - $index_diff_cursor = - $position["file_sync_planner_cursor"]["index_diff_cursor"]; - $index_bytes_done = $index_diff_cursor["new_index_byte_offset"] - + $index_diff_cursor["old_index_byte_offset"]; - $progress["index_bytes_done"] = min($index_bytes_done, $this->index_bytes_total); - $progress["index_bytes_total"] = $this->index_bytes_total; return $progress; } + /** Returns the current file sync patch processor phase. */ + public function get_phase(): string + { + if ($this->cursor["position"]["phase"] === "complete") { + return "complete"; + } + return $this->file_sync_patch_processor->get_phase(); + } + /** * Returns the raw NUL-delimited path list produced for local deletions. */ @@ -297,7 +272,10 @@ public function get_local_paths_to_delete_path(): string */ public function get_fresh_local_index_path(): string { - return $this->fresh_local_index_file; + return wp_join_unix_paths( + $this->plan_directory, + "fresh_local_index.jsonl" + ); } /** @@ -307,8 +285,8 @@ public function get_fresh_local_index_path(): string */ public function flush_pending_outputs(): void { - if (isset($this->fresh_local_index_processor)) { - $this->fresh_local_index_processor->flush_pending_output(); + if (isset($this->file_sync_patch_processor)) { + $this->file_sync_patch_processor->flush_pending_outputs(); } if ( ( is_resource($this->local_paths_to_push_handle) && !fflush($this->local_paths_to_push_handle) ) @@ -316,9 +294,6 @@ public function flush_pending_outputs(): void ) { throw new RuntimeException("Failed to flush a push-plan output."); } - if (isset($this->patch_planner)) { - $this->patch_planner->flush_pending_outputs(); - } } /** @@ -343,9 +318,7 @@ private function __construct( rtrim($document_root_local_relative_path, "/"); $this->local_paths_to_push = wp_join_unix_paths($plan_directory, "local_paths_to_push.jsonl"); $this->local_paths_to_delete = wp_join_unix_paths($plan_directory, "local_paths_to_delete"); - $this->fresh_local_index_file = wp_join_unix_paths($plan_directory, "fresh_local_index.jsonl"); $this->excluded_paths_file = wp_join_unix_paths($plan_directory, "excluded_paths.json"); - $this->active_deletion_roots_file = wp_join_unix_paths($plan_directory, "deleted_directories_stack.jsonl"); } /** @@ -366,141 +339,22 @@ public function next_step(): bool throw new LogicException("Cannot take a push plan step after close()."); } - switch ($position["phase"]) { - case "indexing": - $this->fresh_local_index_processor->next_step(); - $fresh_local_index_cursor = - $this->fresh_local_index_processor->get_cursor(); - $this->cursor["position"] = - $this->fresh_local_index_processor->get_phase() - === "sorting" - ? [ - "phase" => "starting_diff", - "fresh_local_index_cursor" => - $fresh_local_index_cursor, - ] - : [ - "phase" => "indexing", - "fresh_local_index_cursor" => - $fresh_local_index_cursor, - ]; - return true; - case "starting_diff": - $this->start_index_diff(); - return true; - case "diffing": - return $this->next_index_diff_step(); - } - } - - /** - * Sorts the fresh local index by raw path, then starts the index diff. - */ - private function start_index_diff(): void - { - if ($this->fresh_local_index_processor->next_step()) { - throw new LogicException( - "Fresh local index sorting did not complete in one step." - ); - } - $this->fresh_local_index_processor->close(); - $this->open_plan_output_files(0, 0); - $this->patch_planner = FileSyncPatchPlanner::create( - $this->local_index_file, - $this->fresh_local_index_file, - $this->active_deletion_roots_file, - [$this->document_root_local_relative_path], - $this->get_excluded_index_path_roots() - ); - $this->cursor["position"] = [ - "phase" => "diffing", - "file_sync_planner_cursor" => $this->patch_planner->get_cursor(), - "byte_offset_in_local_paths_to_push" => 0, - "byte_offset_in_local_paths_to_delete" => 0, - "local_paths_to_push_count" => 0, - "local_file_bytes_to_push" => 0, - ]; - $this->set_index_bytes_total(); - } - - /** Opens both patch-plan outputs at their durable byte offsets. */ - private function open_plan_output_files( - int $byte_offset_in_local_paths_to_push, - int $byte_offset_in_local_paths_to_delete - ): void { - $this->local_paths_to_push_handle = - $this->open_push_plan_output_file_at_byte_offset( - $this->local_paths_to_push, - $byte_offset_in_local_paths_to_push - ); - $this->local_paths_to_delete_handle = - $this->open_push_plan_output_file_at_byte_offset( - $this->local_paths_to_delete, - $byte_offset_in_local_paths_to_delete + $has_next_step = $this->file_sync_patch_processor->next_step(); + if ( + $this->file_sync_patch_processor->get_phase() === "planning" + && !is_resource($this->local_paths_to_push_handle) + ) { + $this->open_plan_output_files( + $position["byte_offset_in_local_paths_to_push"], + $position["byte_offset_in_local_paths_to_delete"] ); - } - - /** Returns target exclusions in local-index coordinates. */ - private function get_excluded_index_path_roots(): array - { - $excluded_index_path_roots = []; - foreach ($this->excluded_paths as $excluded_path) { - $excluded_index_path_roots[] = - $this->document_root_local_relative_path === "" - ? $excluded_path - : wp_join_unix_paths( - $this->document_root_local_relative_path, - $excluded_path - ); } - return $excluded_index_path_roots; - } - /** Stores the byte total used to report index-diff progress. */ - private function set_index_bytes_total(): void - { - $fresh_local_index_bytes = filesize($this->fresh_local_index_file); - $local_index_bytes = is_file($this->local_index_file) - ? filesize($this->local_index_file) - : 0; - if (is_int($fresh_local_index_bytes) && is_int($local_index_bytes)) { - $this->index_bytes_total = $fresh_local_index_bytes - + $local_index_bytes; - } - } - - /** - * Compares at most one path and updates the resulting push plan cursor. - * - * Exclusions suppress planned changes, not entries in the retained fresh - * local index. - * - * @return bool Whether another index diff step may be performed. - */ - private function next_index_diff_step(): bool - { - /** @var IndexDiffCursor $cursor */ - $cursor = $this->cursor["position"]; - $local_paths_to_push_count = $cursor["local_paths_to_push_count"]; - $local_file_bytes_to_push = $cursor["local_file_bytes_to_push"]; - - if (!$this->patch_planner->next_path()) { - if ( - !fflush($this->local_paths_to_push_handle) - || !fflush($this->local_paths_to_delete_handle) - ) { - throw new RuntimeException("Failed to flush a push-plan output."); - } - $this->patch_planner->flush_pending_outputs(); - $this->cursor["position"] = [ - "phase" => "complete", - "local_paths_to_push_count" => $local_paths_to_push_count, - "local_file_bytes_to_push" => $local_file_bytes_to_push, - ]; - return false; - } - - $operation = $this->patch_planner->get_operation(); + $local_paths_to_push_count = + $position["local_paths_to_push_count"]; + $local_file_bytes_to_push = + $position["local_file_bytes_to_push"]; + $operation = $this->file_sync_patch_processor->get_operation(); if ($operation !== null) { if ($operation["action"] !== "copy") { $this->append_local_path_to_delete($operation["path"]); @@ -520,34 +374,63 @@ private function next_index_diff_step(): bool } } - $complete = $this->patch_planner->is_complete(); - if ($complete) { - if ( - !fflush($this->local_paths_to_push_handle) - || !fflush($this->local_paths_to_delete_handle) - ) { - throw new RuntimeException("Failed to flush a push-plan output."); - } - $this->patch_planner->flush_pending_outputs(); - } - $this->cursor["position"] = $complete - ? [ + if (!$has_next_step) { + $this->flush_pending_outputs(); + $this->cursor["position"] = [ "phase" => "complete", "local_paths_to_push_count" => $local_paths_to_push_count, "local_file_bytes_to_push" => $local_file_bytes_to_push, - ] - : [ - "phase" => "diffing", - "file_sync_planner_cursor" => - $this->patch_planner->get_cursor(), - "byte_offset_in_local_paths_to_push" => - ftell($this->local_paths_to_push_handle), - "byte_offset_in_local_paths_to_delete" => - ftell($this->local_paths_to_delete_handle), - "local_paths_to_push_count" => $local_paths_to_push_count, - "local_file_bytes_to_push" => $local_file_bytes_to_push, ]; - return !$complete; + return false; + } + + $local_paths_to_push_byte_offset = is_resource( + $this->local_paths_to_push_handle + ) + ? ftell($this->local_paths_to_push_handle) + : $position["byte_offset_in_local_paths_to_push"]; + $local_paths_to_delete_byte_offset = is_resource( + $this->local_paths_to_delete_handle + ) + ? ftell($this->local_paths_to_delete_handle) + : $position["byte_offset_in_local_paths_to_delete"]; + if ( + !is_int($local_paths_to_push_byte_offset) + || !is_int($local_paths_to_delete_byte_offset) + ) { + throw new RuntimeException( + "Failed to determine a push-plan output byte offset." + ); + } + $this->cursor["position"] = [ + "phase" => "processing", + "file_sync_patch_processor_cursor" => + $this->file_sync_patch_processor->get_cursor(), + "byte_offset_in_local_paths_to_push" => + $local_paths_to_push_byte_offset, + "byte_offset_in_local_paths_to_delete" => + $local_paths_to_delete_byte_offset, + "local_paths_to_push_count" => $local_paths_to_push_count, + "local_file_bytes_to_push" => $local_file_bytes_to_push, + ]; + return true; + } + + /** Opens both patch-plan outputs at their durable byte offsets. */ + private function open_plan_output_files( + int $byte_offset_in_local_paths_to_push, + int $byte_offset_in_local_paths_to_delete + ): void { + $this->local_paths_to_push_handle = + $this->open_push_plan_output_file_at_byte_offset( + $this->local_paths_to_push, + $byte_offset_in_local_paths_to_push + ); + $this->local_paths_to_delete_handle = + $this->open_push_plan_output_file_at_byte_offset( + $this->local_paths_to_delete, + $byte_offset_in_local_paths_to_delete + ); } /** @@ -559,11 +442,8 @@ private function next_index_diff_step(): bool */ public function close(): void { - if (isset($this->fresh_local_index_processor)) { - $this->fresh_local_index_processor->close(); - } - if (isset($this->patch_planner)) { - $this->patch_planner->close(); + if (isset($this->file_sync_patch_processor)) { + $this->file_sync_patch_processor->close(); } if (is_resource($this->local_paths_to_push_handle)) { fclose($this->local_paths_to_push_handle); @@ -672,32 +552,4 @@ private function append_local_path_to_delete(string $path): void } } - /** - * Loads the caller-owned exclusions used throughout one planning run. - * - * @return list Decoded document-root-relative excluded paths. - */ - private function load_excluded_paths(): array - { - $contents = file_get_contents($this->excluded_paths_file); - if (!is_string($contents)) { - throw new RuntimeException("Failed to read excluded paths: {$this->excluded_paths_file}"); - } - try { - $excluded_paths_b64 = json_decode($contents, true, 512, JSON_THROW_ON_ERROR); - } catch (JsonException $exception) { - throw new RuntimeException("Failed to decode excluded paths: {$this->excluded_paths_file}", 0, $exception); - } - /** @var list $excluded_paths_b64 */ - $excluded_paths = []; - foreach ($excluded_paths_b64 as $excluded_path_b64) { - $excluded_path = base64_decode($excluded_path_b64, true); - if ($excluded_path === false) { - throw new RuntimeException("Failed to decode an excluded path: {$this->excluded_paths_file}"); - } - $excluded_paths[] = $excluded_path; - } - return $excluded_paths; - } - } diff --git a/tests/Import/FileSyncPatchPlannerTest.php b/tests/Import/FileSyncPatchPlannerTest.php index 01ad1bd8..09275112 100644 --- a/tests/Import/FileSyncPatchPlannerTest.php +++ b/tests/Import/FileSyncPatchPlannerTest.php @@ -130,6 +130,56 @@ public function testCollapsesADeletedSubtreeToItsRoot(): void $planner->close(); } + public function testCollapsesADeletedSubtreeAtANestedIncludedRoot(): void + { + $patch_base_index = $this->write_index('base.jsonl', [ + 'outside/selected/child.txt' => + $this->entry('outside/selected/child.txt'), + ]); + $patch_result_index = $this->write_index('result.jsonl', []); + $planner = FileSyncPatchPlanner::create( + $patch_base_index, + $patch_result_index, + $this->active_deletion_roots_file(), + ['outside/selected'] + ); + + $this->assertSame( + [ + $this->delete_operation('outside/selected'), + ], + $this->collect_operations($planner) + ); + $planner->close(); + } + + public function testCollapsesAnAllowedSiblingWithoutDeletingAnExcludedSubtree(): void + { + $patch_base_index = $this->write_index('base.jsonl', [ + 'selected/delete/child.txt' => + $this->entry('selected/delete/child.txt'), + 'selected/keep/child.txt' => + $this->entry('selected/keep/child.txt'), + ]); + $patch_result_index = $this->write_index('result.jsonl', []); + $planner = FileSyncPatchPlanner::create( + $patch_base_index, + $patch_result_index, + $this->active_deletion_roots_file(), + ['selected'], + ['selected/keep'] + ); + + $this->assertSame( + [ + $this->delete_operation('selected/delete'), + null, + ], + $this->collect_operations($planner) + ); + $planner->close(); + } + public function testIncludedAndExcludedRootsLimitBothOperations(): void { $patch_base_index = $this->write_index('base.jsonl', [ @@ -165,6 +215,64 @@ public function testIncludedAndExcludedRootsLimitBothOperations(): void $planner->close(); } + public function testCursorBase64EncodesArbitraryBytePaths(): void + { + $included_root = "selected-\xff"; + $patch_base_index = + $this->temp_dir . "/base-\xfe.jsonl"; + $patch_result_index = $this->write_index('result.jsonl', [ + $included_root . '/file.txt' => + $this->entry($included_root . '/file.txt'), + ]); + $active_deletion_roots_file = + $this->active_deletion_roots_file(); + $planner = FileSyncPatchPlanner::create( + $patch_base_index, + $patch_result_index, + $active_deletion_roots_file, + [$included_root] + ); + $cursor = json_decode( + json_encode($planner->get_cursor(), JSON_THROW_ON_ERROR), + true, + 512, + JSON_THROW_ON_ERROR + ); + $this->assertIsArray($cursor); + $this->assertSame( + [base64_encode($included_root)], + $cursor['included_index_path_roots_b64'] + ); + $this->assertSame( + base64_encode($patch_base_index), + $cursor['patch_base_index_file_b64'] + ); + $this->assertSame( + base64_encode($patch_result_index), + $cursor['patch_result_index_file_b64'] + ); + $this->assertSame( + base64_encode($active_deletion_roots_file), + $cursor['active_deletion_roots_file_b64'] + ); + $planner->close(); + + $resumed = FileSyncPatchPlanner::resume($cursor); + $this->assertSame( + [ + $this->copy_operation( + 'copy', + $included_root . '/file.txt', + 'file', + 1, + 1 + ), + ], + $this->collect_operations($resumed) + ); + $resumed->close(); + } + public function testResumeKeepsAnActiveDeletionRootAcrossASibling(): void { $patch_base_index = $this->write_index('base.jsonl', [ diff --git a/tests/Import/FileSyncPatchProcessorTest.php b/tests/Import/FileSyncPatchProcessorTest.php new file mode 100644 index 00000000..146db365 --- /dev/null +++ b/tests/Import/FileSyncPatchProcessorTest.php @@ -0,0 +1,527 @@ +temporary_directory = sys_get_temp_dir() + . '/file-sync-patch-processor-test-' + . uniqid(); + $this->filesystem_root = + $this->temporary_directory . '/filesystem-root'; + mkdir($this->filesystem_root, 0755, true); + } + + protected function tearDown(): void + { + $this->remove_path($this->temporary_directory); + parent::tearDown(); + } + + public function testStartMethodsPlanOppositePatchDirections(): void + { + file_put_contents($this->filesystem_root . '/current.txt', 'current'); + $saved_index = $this->write_index('saved.jsonl', [ + $this->entry('saved.txt', 5), + ]); + + $to_fresh_work_directory = $this->work_directory('to-fresh'); + $to_fresh = FileSyncPatchProcessor::start_to_fresh_local_tree( + $to_fresh_work_directory, + $this->filesystem_root, + $saved_index, + $to_fresh_work_directory + ); + $this->assertSame( + [ + ['action' => 'copy', 'path' => 'current.txt'], + ['action' => 'delete', 'path' => 'saved.txt'], + ], + $this->operation_names($this->run_to_completion($to_fresh)) + ); + + $from_fresh_work_directory = $this->work_directory('from-fresh'); + $from_fresh = FileSyncPatchProcessor::start_from_fresh_local_tree( + $from_fresh_work_directory, + $this->filesystem_root, + $saved_index, + $from_fresh_work_directory + ); + $this->assertSame( + [ + [ + 'action' => 'delete', + 'path' => 'current.txt', + ], + [ + 'action' => 'copy', + 'path' => 'saved.txt', + 'expected_source' => [ + 'type' => 'file', + 'size' => 5, + 'ctime' => 1, + ], + ], + ], + $this->run_to_completion($from_fresh) + ); + } + + public function testSortingAndPlannerStartupHaveSeparateSteps(): void + { + file_put_contents($this->filesystem_root . '/file.txt', 'file'); + $work_directory = $this->work_directory('phases'); + $processor = FileSyncPatchProcessor::start_to_fresh_local_tree( + $work_directory, + $this->filesystem_root, + $this->temporary_directory . '/missing-index.jsonl', + $work_directory + ); + + $this->assertSame('indexing', $processor->get_phase()); + for ($step = 0; $step < 20; ++$step) { + $this->assertTrue($processor->next_step()); + if ($processor->get_phase() === 'sorting') { + break; + } + } + $this->assertSame('sorting', $processor->get_phase()); + + $this->assertTrue($processor->next_step()); + $this->assertSame('starting_patch', $processor->get_phase()); + $this->assertNull($processor->get_operation()); + + $this->assertTrue($processor->next_step()); + $this->assertSame('planning', $processor->get_phase()); + $this->assertNull($processor->get_operation()); + $this->assertSame( + ['phase', 'file_sync_patch_planner_cursor'], + array_keys($processor->get_cursor()['position']) + ); + $processor->close(); + } + + public function testResumeUsesOnlyTheCursorAndDiscardsUnstoredIndexBytes(): void + { + file_put_contents($this->filesystem_root . '/a.txt', 'a'); + file_put_contents($this->filesystem_root . '/b.txt', 'b'); + $work_directory = $this->work_directory('resume'); + $processor = FileSyncPatchProcessor::start_to_fresh_local_tree( + $work_directory, + $this->filesystem_root, + $this->temporary_directory . '/missing-index.jsonl', + $work_directory + ); + + do { + $this->assertTrue($processor->next_step()); + $processor->flush_pending_outputs(); + $saved_cursor = $processor->get_cursor(); + $saved_offset = $saved_cursor['position'][ + 'fresh_local_index_cursor' + ]['position']['fresh_local_index_byte_offset']; + } while ($saved_offset === 0); + + do { + $this->assertTrue($processor->next_step()); + clearstatcache( + true, + $processor->get_fresh_local_index_path() + ); + $fresh_index_bytes = filesize( + $processor->get_fresh_local_index_path() + ); + $this->assertIsInt($fresh_index_bytes); + } while ($fresh_index_bytes <= $saved_offset); + $processor->close(); + + $resumed = FileSyncPatchProcessor::resume($saved_cursor); + $this->run_to_completion($resumed); + $this->assertSame( + ['a.txt', 'b.txt'], + $this->read_index_paths( + $work_directory . '/fresh_local_index.jsonl' + ) + ); + } + + public function testCompleteAndCloseAreStable(): void + { + $work_directory = $this->work_directory('complete'); + $processor = FileSyncPatchProcessor::start_to_fresh_local_tree( + $work_directory, + $this->filesystem_root, + $this->temporary_directory . '/missing-index.jsonl', + $work_directory + ); + $this->run_to_completion($processor); + $complete_cursor = $processor->get_cursor(); + + $this->assertFalse($processor->next_step()); + $processor->close(); + $processor->close(); + $this->assertFalse($processor->next_step()); + $this->assertSame($complete_cursor, $processor->get_cursor()); + + $resumed = FileSyncPatchProcessor::resume($complete_cursor); + $this->assertFalse($resumed->next_step()); + $resumed->close(); + } + + public function testFinalStepReturnsItsOperationBeforeStableComplete(): void + { + file_put_contents($this->filesystem_root . '/only.txt', 'only'); + $work_directory = $this->work_directory('final-operation'); + $processor = FileSyncPatchProcessor::start_to_fresh_local_tree( + $work_directory, + $this->filesystem_root, + $this->temporary_directory . '/missing-index.jsonl', + $work_directory + ); + for ($step = 0; $step < 20; ++$step) { + if ($processor->get_phase() === 'planning') { + break; + } + $this->assertTrue($processor->next_step()); + } + + $this->assertSame('planning', $processor->get_phase()); + $this->assertFalse($processor->next_step()); + $operation = $processor->get_operation(); + $this->assertIsArray($operation); + $this->assertSame('copy', $operation['action']); + $this->assertSame('only.txt', $operation['path']); + + $this->assertFalse($processor->next_step()); + $this->assertNull($processor->get_operation()); + $processor->close(); + } + + public function testCursorBase64EncodesArbitraryByteSelectionRoots(): void + { + $included_root = "selected-\xff"; + $excluded_root = $included_root . "/excluded-\xfe"; + $saved_index = $this->write_index('arbitrary-byte-saved.jsonl', [ + $this->entry($included_root . '/file.txt', 1), + ]); + $work_directory = $this->work_directory('arbitrary-byte-root'); + $storage_path = $work_directory . "/storage-\xfd"; + $processor = FileSyncPatchProcessor::start_to_fresh_local_tree( + $work_directory, + $this->filesystem_root, + $saved_index, + $storage_path, + [$included_root], + [$excluded_root] + ); + $cursor = $this->serialize_cursor($processor->get_cursor()); + $this->assertSame( + ['fresh_local_index_file_b64', 'position'], + array_keys($cursor) + ); + $this->assertSame( + [ + 'phase', + 'patch_base_index_file_b64', + 'patch_result_index_file_b64', + 'included_index_path_roots_b64', + 'excluded_index_path_roots_b64', + 'fresh_local_index_cursor', + ], + array_keys($cursor['position']) + ); + $this->assertSame( + [base64_encode($included_root)], + $cursor['position']['included_index_path_roots_b64'] + ); + $this->assertSame( + [base64_encode($excluded_root)], + $cursor['position']['excluded_index_path_roots_b64'] + ); + $this->assertSame( + base64_encode($storage_path), + $cursor['position']['fresh_local_index_cursor']['storage_path_b64'] + ); + $processor->close(); + + $resumed = FileSyncPatchProcessor::resume($cursor); + while ($resumed->get_phase() !== 'sorting') { + $this->assertTrue($resumed->next_step()); + } + $sorting_cursor = $this->serialize_cursor($resumed->get_cursor()); + $resumed->close(); + + $resumed = FileSyncPatchProcessor::resume($sorting_cursor); + $this->assertTrue($resumed->next_step()); + $starting_patch_cursor = $this->serialize_cursor( + $resumed->get_cursor() + ); + $this->assertSame('starting_patch', $resumed->get_phase()); + $resumed->close(); + + $resumed = FileSyncPatchProcessor::resume($starting_patch_cursor); + $this->assertTrue($resumed->next_step()); + $planning_cursor = $this->serialize_cursor($resumed->get_cursor()); + $this->assertSame('planning', $resumed->get_phase()); + $resumed->close(); + + $resumed = FileSyncPatchProcessor::resume($planning_cursor); + $this->assertSame( + [ + [ + 'action' => 'delete', + 'path' => $included_root . '/file.txt', + ], + ], + $this->operation_names($this->run_to_completion($resumed)) + ); + $complete_cursor = $this->serialize_cursor($resumed->get_cursor()); + $resumed = FileSyncPatchProcessor::resume($complete_cursor); + $this->assertFalse($resumed->next_step()); + $resumed->close(); + } + + public function testCursorKeepsArbitraryByteWorkAndIndexPathsThroughEveryPhase(): void + { + $filesystem_root = + $this->temporary_directory . "/filesystem-root-\xff"; + if (!@mkdir($filesystem_root)) { + $this->markTestSkipped( + 'This filesystem does not accept non-UTF-8 path components.' + ); + } + $work_directory = $this->work_directory("work-\xfe"); + $storage_path = $this->temporary_directory . "/storage-\xfd"; + mkdir($storage_path); + $current_path = "current-\xfc.txt"; + $saved_path = "saved-\xfb.txt"; + file_put_contents($filesystem_root . '/' . $current_path, 'current'); + $saved_index = $this->write_index("saved-\xfa.jsonl", [ + $this->entry($saved_path, 5), + ]); + $fresh_local_index_file = + $work_directory . '/fresh_local_index.jsonl'; + + $processor = FileSyncPatchProcessor::start_to_fresh_local_tree( + $work_directory, + $filesystem_root, + $saved_index, + $storage_path + ); + $seen_phases = []; + $operations = []; + for ($step = 0; $step < 100; ++$step) { + $processor->flush_pending_outputs(); + $cursor = $this->serialize_cursor($processor->get_cursor()); + $phase = $cursor['position']['phase']; + $seen_phases[$phase] = true; + $this->assertSame( + $fresh_local_index_file, + base64_decode($cursor['fresh_local_index_file_b64'], true) + ); + $processor->close(); + $processor = FileSyncPatchProcessor::resume($cursor); + if ($phase === 'complete') { + $this->assertFalse($processor->next_step()); + $processor->close(); + break; + } + $processor->next_step(); + $operation = $processor->get_operation(); + if ($operation !== null) { + $operations[] = $operation; + } + } + + $this->assertSame( + ['indexing', 'sorting', 'starting_patch', 'planning', 'complete'], + array_keys($seen_phases) + ); + $this->assertSame( + [ + ['action' => 'copy', 'path' => $current_path], + ['action' => 'delete', 'path' => $saved_path], + ], + $this->operation_names($operations) + ); + } + + public function testFactoryPassesIncludeCachesToFreshIndexing(): void + { + mkdir($this->filesystem_root . '/node_modules'); + file_put_contents( + $this->filesystem_root . '/node_modules/package.js', + 'package' + ); + $work_directory = $this->work_directory('include-caches'); + $processor = FileSyncPatchProcessor::start_to_fresh_local_tree( + $work_directory, + $this->filesystem_root, + $this->temporary_directory . '/missing-index.jsonl', + $work_directory, + [''], + [], + true + ); + $this->assertTrue( + $processor->get_cursor()['position'][ + 'fresh_local_index_cursor' + ]['include_caches'] + ); + + $operations = $this->operation_names( + $this->run_to_completion($processor) + ); + $this->assertSame( + [ + [ + 'action' => 'copy', + 'path' => 'node_modules/package.js', + ], + ], + $operations + ); + } + + /** @return list> */ + private function run_to_completion( + FileSyncPatchProcessor $processor + ): array { + $operations = []; + for ($step = 0; $step < 100; ++$step) { + $has_next_step = $processor->next_step(); + $operation = $processor->get_operation(); + if ($operation !== null) { + $operations[] = $operation; + } + $processor->flush_pending_outputs(); + if (!$has_next_step) { + $processor->close(); + return $operations; + } + } + $this->fail('File sync patch processing did not complete in 100 steps.'); + } + + /** + * @param array $cursor + * @return array + */ + private function serialize_cursor(array $cursor): array + { + $stored_cursor = json_decode( + json_encode($cursor, JSON_THROW_ON_ERROR), + true, + 512, + JSON_THROW_ON_ERROR + ); + $this->assertIsArray($stored_cursor); + return $stored_cursor; + } + + /** + * @param list> $operations + * @return list + */ + private function operation_names(array $operations): array + { + return array_map( + static fn (array $operation): array => [ + 'action' => $operation['action'], + 'path' => $operation['path'], + ], + $operations + ); + } + + /** @param list> $entries */ + private function write_index(string $filename, array $entries): string + { + $lines = []; + foreach ($entries as $entry) { + $entry['path'] = base64_encode($entry['path']); + $lines[] = json_encode( + $entry, + JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR + ); + } + $path = $this->temporary_directory . '/' . $filename; + file_put_contents($path, implode("\n", $lines) . "\n"); + return $path; + } + + /** @return array{path:string,ctime:int,size:int,type:string} */ + private function entry(string $path, int $size): array + { + return [ + 'path' => $path, + 'ctime' => 1, + 'size' => $size, + 'type' => 'file', + ]; + } + + private function work_directory(string $name): string + { + $path = $this->temporary_directory . '/' . $name; + mkdir($path, 0755, true); + return $path; + } + + /** @return list */ + private function read_index_paths(string $index_file): array + { + $lines = file( + $index_file, + FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES + ); + $this->assertIsArray($lines); + return array_map( + static function (string $line): string { + $entry = json_decode( + $line, + true, + 512, + JSON_THROW_ON_ERROR + ); + $path = base64_decode($entry['path'], true); + if (!is_string($path)) { + throw new RuntimeException('Failed to decode an index path.'); + } + return $path; + }, + $lines + ); + } + + private function remove_path(string $path): void + { + if (is_link($path) || is_file($path)) { + unlink($path); + return; + } + if (!is_dir($path)) { + return; + } + $children = scandir($path); + if (is_array($children)) { + foreach ($children as $child) { + if ($child !== '.' && $child !== '..') { + $this->remove_path($path . '/' . $child); + } + } + } + rmdir($path); + } +} diff --git a/tests/Import/FreshLocalIndexProcessorTest.php b/tests/Import/FreshLocalIndexProcessorTest.php index 9c65699e..72840a34 100644 --- a/tests/Import/FreshLocalIndexProcessorTest.php +++ b/tests/Import/FreshLocalIndexProcessorTest.php @@ -114,6 +114,95 @@ public function testSortingIsASeparateResumableStep(): void $resumed_processor->close(); } + public function testIncludeCachesIsRetainedByTheCursor(): void + { + mkdir($this->filesystem_root . '/node_modules'); + file_put_contents( + $this->filesystem_root . '/node_modules/package.js', + 'package' + ); + $processor = FreshLocalIndexProcessor::start( + $this->fresh_local_index_file, + $this->filesystem_root, + $this->storage_path, + true + ); + $this->assertTrue($processor->next_step()); + $processor->flush_pending_output(); + $cursor = $processor->get_cursor(); + $processor->close(); + + $resumed_processor = FreshLocalIndexProcessor::resume($cursor); + $this->run_to_completion($resumed_processor); + + $this->assertSame( + ['node_modules/package.js'], + array_column($this->read_index_entries(), 'decoded_path') + ); + } + + public function testCursorSerializesAndResumesEveryPhaseWithArbitraryBytePaths(): void + { + $arbitrary_byte_directory = + $this->temporary_directory . "/paths-\xff"; + if (!@mkdir($arbitrary_byte_directory)) { + $this->markTestSkipped( + 'This filesystem does not accept non-UTF-8 path components.' + ); + } + $this->filesystem_root = + $arbitrary_byte_directory . "/filesystem-root-\xfe"; + $this->storage_path = + $arbitrary_byte_directory . "/storage-\xfd"; + $this->fresh_local_index_file = + $arbitrary_byte_directory . "/fresh-index-\xfc.jsonl"; + mkdir($this->filesystem_root); + mkdir($this->storage_path); + file_put_contents($this->filesystem_root . '/value.txt', 'value'); + + $processor = FreshLocalIndexProcessor::start( + $this->fresh_local_index_file, + $this->filesystem_root, + $this->storage_path + ); + $seen_phases = []; + for ($step = 0; $step < 100; ++$step) { + $processor->flush_pending_output(); + $cursor = $this->serialize_cursor($processor->get_cursor()); + $phase = $cursor['position']['phase']; + $seen_phases[$phase] = true; + $this->assertSame( + $this->fresh_local_index_file, + base64_decode($cursor['fresh_local_index_file_b64'], true) + ); + $this->assertSame( + realpath($this->filesystem_root), + base64_decode($cursor['filesystem_root_b64'], true) + ); + $this->assertSame( + $this->storage_path, + base64_decode($cursor['storage_path_b64'], true) + ); + $processor->close(); + $processor = FreshLocalIndexProcessor::resume($cursor); + if ($phase === 'complete') { + $this->assertFalse($processor->next_step()); + $processor->close(); + break; + } + $processor->next_step(); + } + + $this->assertSame( + ['indexing', 'sorting', 'complete'], + array_keys($seen_phases) + ); + $this->assertSame( + ['value.txt'], + array_column($this->read_index_entries(), 'decoded_path') + ); + } + private function run_to_completion( FreshLocalIndexProcessor $processor ): void { @@ -126,6 +215,22 @@ private function run_to_completion( $this->fail('Fresh local index did not complete within 100 steps.'); } + /** + * @param array $cursor + * @return array + */ + private function serialize_cursor(array $cursor): array + { + $stored_cursor = json_decode( + json_encode($cursor, JSON_THROW_ON_ERROR), + true, + 512, + JSON_THROW_ON_ERROR + ); + $this->assertIsArray($stored_cursor); + return $stored_cursor; + } + /** @return list> */ private function read_index_entries(): array { diff --git a/tests/Import/PushPlanTest.php b/tests/Import/PushPlanTest.php index 4f272302..220e57fd 100644 --- a/tests/Import/PushPlanTest.php +++ b/tests/Import/PushPlanTest.php @@ -302,7 +302,9 @@ public function testActiveDeletionRootsAppendWithoutGrowingTheCursor(): void $first_cursor = $this->planCursor(); $this->assertSame( 0, - $first_cursor['file_sync_planner_cursor'][ + $first_cursor['file_sync_patch_processor_cursor']['position'][ + 'file_sync_patch_planner_cursor' + ][ 'active_deletion_root_byte_offset' ] ); @@ -446,7 +448,7 @@ public function testIndexingFinishesBeforeDiffingAndClearsOldOutput(): void $current = $this->writeIndex($this->manyFileEntries(2)); $plan = $this->startPlan($current); $this->assertCount(2, $this->indexEntries($this->planPath('fresh_local_index.jsonl'))); - $this->assertSame('diffing', $this->planCursor()['phase']); + $this->assertSame('planning', $plan->get_phase()); $this->assertFileDoesNotExist($this->planPath('cursor.json')); $this->assertSame(0, filesize($this->planPath('local_paths_to_push.jsonl'))); $this->assertSame(0, filesize($this->planPath('local_paths_to_delete'))); @@ -483,13 +485,13 @@ public function testReportsDiffProgressAcrossResume(): void $plan = $this->startPlan($this->writeIndex($this->manyFileEntries(3))); $progress = $plan->get_progress(); - $this->assertSame('diffing', $progress['phase']); + $this->assertSame('planning', $progress['phase']); $this->assertSame(0, $progress['index_bytes_done']); $this->assertGreaterThan(0, $progress['index_bytes_total']); $this->assertTrue($this->nextPlanStep($plan)); $progress = $plan->get_progress(); - $this->assertSame('diffing', $progress['phase']); + $this->assertSame('planning', $progress['phase']); $this->assertGreaterThan(0, $progress['index_bytes_done']); $this->assertLessThan($progress['index_bytes_total'], $progress['index_bytes_done']); $plan->close(); @@ -523,8 +525,12 @@ public function testResumeDiscardsACompletedStepWhoseCursorWasNotStored(): void public function testStepRetainsTheFollowingFreshLocalIndexEntryUntilClose(): void { $plan = $this->startPlan($this->writeIndex($this->manyFileEntries(3))); - $patch_planner_property = new ReflectionProperty( + $file_sync_patch_processor_property = new ReflectionProperty( PushPlan::class, + 'file_sync_patch_processor' + ); + $patch_planner_property = new ReflectionProperty( + FileSyncPatchProcessor::class, 'patch_planner' ); $index_diff_property = new ReflectionProperty( @@ -542,14 +548,20 @@ public function testStepRetainsTheFollowingFreshLocalIndexEntryUntilClose(): voi $this->assertTrue($this->nextPlanStep($plan)); $first_plan_cursor = $this->planCursor(); - $patch_planner = $patch_planner_property->getValue($plan); + $file_sync_patch_processor = + $file_sync_patch_processor_property->getValue($plan); + $patch_planner = $patch_planner_property->getValue( + $file_sync_patch_processor + ); $index_diff = $index_diff_property->getValue($patch_planner); $fresh_local_index_handle = $new_index_handle_property->getValue( $index_diff ); $this->assertIsResource($fresh_local_index_handle); $this->assertGreaterThan( - $first_plan_cursor['file_sync_planner_cursor'][ + $first_plan_cursor['file_sync_patch_processor_cursor']['position'][ + 'file_sync_patch_planner_cursor' + ][ 'index_diff_cursor' ]['new_index_byte_offset'], ftell($fresh_local_index_handle) @@ -563,7 +575,9 @@ public function testStepRetainsTheFollowingFreshLocalIndexEntryUntilClose(): voi $this->assertTrue($this->nextPlanStep($plan)); $second_plan_cursor = $this->planCursor(); $this->assertGreaterThan( - $second_plan_cursor['file_sync_planner_cursor'][ + $second_plan_cursor['file_sync_patch_processor_cursor']['position'][ + 'file_sync_patch_planner_cursor' + ][ 'index_diff_cursor' ]['new_index_byte_offset'], ftell($fresh_local_index_handle) @@ -597,21 +611,29 @@ public function testCursorContainsResumePathsOffsetsAndCompletionState(): void $this->assertSame('', $cursor['document_root_local_relative_path']); $this->assertSame([ 'phase', - 'file_sync_planner_cursor', + 'file_sync_patch_processor_cursor', 'byte_offset_in_local_paths_to_push', 'byte_offset_in_local_paths_to_delete', 'local_paths_to_push_count', 'local_file_bytes_to_push', ], array_keys($cursor['position'])); $this->assertSame([ - 'patch_base_index_file', - 'patch_result_index_file', - 'active_deletion_roots_file', - 'included_index_path_roots', - 'excluded_index_path_roots', + 'fresh_local_index_file_b64', + 'position', + ], array_keys($cursor['position']['file_sync_patch_processor_cursor'])); + $this->assertSame([ + 'phase', + 'file_sync_patch_planner_cursor', + ], array_keys($cursor['position']['file_sync_patch_processor_cursor']['position'])); + $this->assertSame([ + 'patch_base_index_file_b64', + 'patch_result_index_file_b64', + 'active_deletion_roots_file_b64', + 'included_index_path_roots_b64', + 'excluded_index_path_roots_b64', 'index_diff_cursor', 'active_deletion_root_byte_offset', - ], array_keys($cursor['position']['file_sync_planner_cursor'])); + ], array_keys($cursor['position']['file_sync_patch_processor_cursor']['position']['file_sync_patch_planner_cursor'])); $this->assertSame(1, $cursor['position']['local_paths_to_push_count']); $this->assertSame(1, $cursor['position']['local_file_bytes_to_push']); $this->assertSame( @@ -640,10 +662,10 @@ public function testCursorKeepsTheFilesystemRootSlash(): void try { $this->assertSame( - '/', + base64_encode('/'), $plan->get_cursor()['position'][ - 'fresh_local_index_cursor' - ]['filesystem_root'] + 'file_sync_patch_processor_cursor' + ]['position']['fresh_local_index_cursor']['filesystem_root_b64'] ); } finally { $plan->close(); @@ -806,7 +828,7 @@ private function startPlan( ); $this->cursor = $plan->get_cursor(); for ($step = 0; $step < 100; ++$step) { - if ($this->planCursor()['phase'] === 'diffing') { + if ($plan->get_phase() === 'planning') { return $plan; } $this->assertTrue($this->nextPlanStep($plan)); diff --git a/tests/PushEndpointsTest.php b/tests/PushEndpointsTest.php index 1234b57d..de96264d 100644 --- a/tests/PushEndpointsTest.php +++ b/tests/PushEndpointsTest.php @@ -1677,7 +1677,7 @@ public function testHighLevelSenderReportsTargetConfirmedPathProgress(): void } } - public function testHighLevelSenderResumesStateWrittenBeforeDocumentRootMappingAndProgressTotals(): void + public function testHighLevelSenderResumesStateWrittenBeforeProgressTotals(): void { $local_docroot = $this->root . '/old-sender-state-local-docroot'; mkdir($local_docroot, 0700, true); @@ -1700,9 +1700,6 @@ public function testHighLevelSenderResumesStateWrittenBeforeDocumentRootMappingA unset($state['local_paths_pushed']); unset($state['local_file_bytes_to_push']); unset($state['local_file_bytes_pushed']); - unset($state['push_plan_cursor']['document_root_local_relative_path']); - unset($state['push_plan_cursor']['position']['local_paths_to_push_count']); - unset($state['push_plan_cursor']['position']['local_file_bytes_to_push']); file_put_contents( $push_state_directory . '/sender.json', json_encode($state, JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) @@ -1885,8 +1882,12 @@ public function testHighLevelSenderRetainsPushPlanAcrossPlanningSteps(): void $push_state_directory = $this->root . '/retained-plan-state'; $fresh_local_index_path = $push_state_directory . '/plan/fresh_local_index.jsonl'; $plan_property = new ReflectionProperty(PushFilesSender::class, 'plan'); - $fresh_local_index_processor_property = new ReflectionProperty( + $file_sync_patch_processor_property = new ReflectionProperty( PushPlan::class, + 'file_sync_patch_processor' + ); + $fresh_local_index_processor_property = new ReflectionProperty( + FileSyncPatchProcessor::class, 'fresh_local_index_processor' ); $fresh_local_index_handle_property = new ReflectionProperty( @@ -1904,8 +1905,12 @@ public function testHighLevelSenderRetainsPushPlanAcrossPlanningSteps(): void $this->assertSame('planning', $create_result['phase']); $plan = $plan_property->getValue($sender); $this->assertInstanceOf(PushPlan::class, $plan); + $file_sync_patch_processor = + $file_sync_patch_processor_property->getValue($plan); $fresh_local_index_processor = - $fresh_local_index_processor_property->getValue($plan); + $fresh_local_index_processor_property->getValue( + $file_sync_patch_processor + ); $fresh_local_index_handle = $fresh_local_index_handle_property->getValue( $fresh_local_index_processor @@ -1960,8 +1965,12 @@ public function testHighLevelSenderBuildsFreshLocalIndexInBoundedSteps(): void $fresh_local_index_path = $push_state_directory . '/plan/fresh_local_index.jsonl'; $options = $this->senderOptions($local_docroot, $push_state_directory); $plan_property = new ReflectionProperty(PushFilesSender::class, 'plan'); - $fresh_local_index_processor_property = new ReflectionProperty( + $file_sync_patch_processor_property = new ReflectionProperty( PushPlan::class, + 'file_sync_patch_processor' + ); + $fresh_local_index_processor_property = new ReflectionProperty( + FileSyncPatchProcessor::class, 'fresh_local_index_processor' ); $file_index_processor_property = new ReflectionProperty( @@ -1986,8 +1995,12 @@ public function testHighLevelSenderBuildsFreshLocalIndexInBoundedSteps(): void ); $plan = $plan_property->getValue($sender); $this->assertInstanceOf(PushPlan::class, $plan); + $file_sync_patch_processor = + $file_sync_patch_processor_property->getValue($plan); $fresh_local_index_processor = - $fresh_local_index_processor_property->getValue($plan); + $fresh_local_index_processor_property->getValue( + $file_sync_patch_processor + ); $file_index_processor = $file_index_processor_property->getValue( $fresh_local_index_processor ); @@ -2003,7 +2016,9 @@ public function testHighLevelSenderBuildsFreshLocalIndexInBoundedSteps(): void $this->assertSame($plan, $plan_property->getValue($sender)); $this->assertSame( $fresh_local_index_processor, - $fresh_local_index_processor_property->getValue($plan) + $fresh_local_index_processor_property->getValue( + $file_sync_patch_processor_property->getValue($plan) + ) ); $this->assertSame( $file_index_processor, @@ -2018,15 +2033,24 @@ public function testHighLevelSenderBuildsFreshLocalIndexInBoundedSteps(): void ) ); $plan_cursor = $this->loadPlanPosition($push_state_directory); - $this->assertSame('indexing', $plan_cursor['phase']); + $file_sync_patch_processor_cursor = + $plan_cursor['file_sync_patch_processor_cursor']; + $this->assertSame( + 'indexing', + $file_sync_patch_processor_cursor['position']['phase'] + ); $this->assertSame( ftell($fresh_local_index_handle), - $plan_cursor['fresh_local_index_cursor']['position'][ + $file_sync_patch_processor_cursor['position'][ + 'fresh_local_index_cursor' + ]['position'][ 'fresh_local_index_byte_offset' ] ); $this->assertNotEmpty( - $plan_cursor['fresh_local_index_cursor']['position'][ + $file_sync_patch_processor_cursor['position'][ + 'fresh_local_index_cursor' + ]['position'][ 'file_index_cursor' ]['stack'] ); @@ -2044,22 +2068,31 @@ public function testHighLevelSenderBuildsFreshLocalIndexInBoundedSteps(): void $this->assertSame('planning', $sender->get_phase()); $resumed_plan = $plan_property->getValue($sender); $resumed_fresh_local_index_processor = - $fresh_local_index_processor_property->getValue($resumed_plan); + $fresh_local_index_processor_property->getValue( + $file_sync_patch_processor_property->getValue( + $resumed_plan + ) + ); $this->assertIsResource( $fresh_local_index_handle_property->getValue( $resumed_fresh_local_index_processor ) ); - $this->takeSenderStepsUntilPlanPhase($sender, $push_state_directory, 'starting_diff'); + $this->takeSenderStepsUntilPlanPhase($sender, $push_state_directory, 'sorting'); $this->assertFileExists($fresh_local_index_path); $this->assertTrue($sender->next_step()); $this->assertSame('planning', $sender->get_phase()); $plan_cursor = $this->loadPlanPosition($push_state_directory); - $this->assertSame('diffing', $plan_cursor['phase']); + $this->assertSame('processing', $plan_cursor['phase']); $this->assertFileExists($fresh_local_index_path); $progress = $sender->get_progress(); $this->assertSame('planning', $progress['phase']); + $this->assertSame('starting_diff', $progress['planning_phase']); + + $this->assertTrue($sender->next_step()); + $progress = $sender->get_progress(); + $this->assertSame('planning', $progress['phase']); $this->assertSame('diffing', $progress['planning_phase']); $this->assertSame(0, $progress['index_bytes_done']); $this->assertGreaterThan(0, $progress['index_bytes_total']); @@ -2124,10 +2157,17 @@ public function testHighLevelSenderContinuesFreshLocalIndexAfterProcessDeath(): $this->assertIsArray($state); $this->assertSame('planning', $state['phase']); $plan_cursor = $this->loadPlanPosition($push_state_directory); - $this->assertSame('indexing', $plan_cursor['phase']); + $file_sync_patch_processor_cursor = + $plan_cursor['file_sync_patch_processor_cursor']; + $this->assertSame( + 'indexing', + $file_sync_patch_processor_cursor['position']['phase'] + ); $this->assertGreaterThan( 0, - $plan_cursor['fresh_local_index_cursor']['position'][ + $file_sync_patch_processor_cursor['position'][ + 'fresh_local_index_cursor' + ]['position'][ 'fresh_local_index_byte_offset' ] ); @@ -4010,7 +4050,10 @@ private function takeSenderStepsUntilPlanPhase( ): void { for ($step = 0; $step < 300; ++$step) { $cursor = $this->loadPlanPosition($push_state_directory); - if ($cursor['phase'] === $phase) { + $current_phase = $cursor['phase'] === 'processing' + ? $cursor['file_sync_patch_processor_cursor']['position']['phase'] + : $cursor['phase']; + if ($current_phase === $phase) { return; } if (!$sender->next_step()) {