diff --git a/.gitattributes b/.gitattributes index c2ca54bbcb6..2c709bba7e8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -72,3 +72,7 @@ src/verification/generated export-ignore src/verification/**/.lake export-ignore config/t32/libXp.so.6 export-ignore config/t32/libjpeg.so.62.0.0 export-ignore + +# Raw bug-registry test inputs deliberately retain CRLF and an invalid CRC control. +# These are copied into temporary .sdn files by the tests; preserve exact bytes. +test/fixtures/database/bug_append_legacy/*.sdn.fixture -text -eol diff --git a/.github/workflows/mail-cli-portability.yml b/.github/workflows/mail-cli-portability.yml index eaa7338497f..749d1f4f65e 100644 --- a/.github/workflows/mail-cli-portability.yml +++ b/.github/workflows/mail-cli-portability.yml @@ -5,6 +5,9 @@ on: paths: - 'tools/mail-cli/**' - 'test/01_unit/tools/mail_cli/**' + - 'src/app/mail_credentials/**' + - 'src/app/devhub/email_config_sdn.spl' + - 'scripts/ci/prepare_mail_config_helper.py' - '.github/workflows/mail-cli-portability.yml' workflow_dispatch: @@ -19,7 +22,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} - timeout-minutes: 10 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 with: @@ -28,6 +31,13 @@ jobs: shell: bash run: | command -v bash jq curl base64 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Prepare the real shared SDN configuration helper + env: + MAIL_CI_PHASE2_PRODUCERS_JSON: ${{ vars.MAIL_CI_PHASE2_PRODUCERS_JSON }} + run: python scripts/ci/prepare_mail_config_helper.py - name: Validate declared shell dialects shell: bash run: | @@ -47,6 +57,15 @@ jobs: if: runner.os == 'Linux' shell: bash run: python3 test/01_unit/tools/mail_cli/fixtures/loopback_pop3_probe.py + - name: Preserve helper build and provenance evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: mail-config-helper-evidence-${{ matrix.os }} + path: | + ${{ runner.temp }}/mail-config-helper/helper-build.log + ${{ runner.temp }}/mail-config-helper/evidence.json + if-no-files-found: ignore # These host fixtures do not replace native Simple helper/SSpec acceptance. # Real crypto and dev-hub execution still require an admitted self-hosted build. diff --git a/doc/05_design/devhub_secured_sqlite_phase2_solution_20261027.md b/doc/05_design/devhub_secured_sqlite_phase2_solution_20261027.md new file mode 100644 index 00000000000..c4c1c03f012 --- /dev/null +++ b/doc/05_design/devhub_secured_sqlite_phase2_solution_20261027.md @@ -0,0 +1,102 @@ + +# DevHub secured SQLite solution with the pinned Phase 2 producer + +Date: 2026-10-27. Status: proposed implementation and acceptance plan; not an implementation or runtime PASS report. + +## Scope and delivery order + +Messenger database access is a required main feature. The user's instruction to land the other fixes first permits an independent first PR; it does not waive the database requirement. Start database implementation on the next branch after that PR lands. This document may travel with the first PR as planning only. + +Phase A delivers secure, working production Messenger reads against a freshly provisioned encrypted database: rooms, members, history, and message fetch. Keep externally owned Messenger databases read-only. Creating an owned acceptance database does not imply adding write commands to Messenger. Phase B improves session reuse and bounded reads. A production owned-database creation API is a separate deliverable if required; it must not delay proving the read path merely to expand the product's write surface. + +Use the existing producer without rebuilding it: + +* Path: `C:/dev/simple-disk-headroom-warm-bootstrap-storage-20261025/output/stage3/simple.exe` +* SHA-256: `13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b` + +Application/library source workarounds and narrowly tested runtime-provider fixes are allowed. Do not substitute a new compiler, sweep in a Rust seed replacement, bypass the codec registry, or replace real assertions/runners with successful stubs. Existing Messenger RTS01 and SSpec investigation limits remain in force; they are unresolved product/tooling defects, not external infrastructure. + +## Research basis and confirmed source observations + +Astra synthesized the following source research from Sol (`sol_db_preresearch`, callback ownership and historical evidence) and Terra (`terra_db_preresearch`, storage geometry, snapshot security, memory ownership, and efficiency). These observations are source findings, not proof of the current native panic's cause. + +| Owner/path | Observation and consequence | +|---|---| +| `src/app/devhub/messenger/adapter/db_common.spl`: `_read_table`, `messenger_db_list_rooms`, `messenger_db_list_members`, `messenger_db_history`, `messenger_db_fetch_message` | Each operation opens, reads a table, and closes. Preserve policy gates, schema selection, typed errors, and read-only intent while changing storage mechanics. | +| `src/lib/nogc_sync_mut/database/server/storage_port.spl`: `storage_backend_sqlcipher4_open`, `DatabaseStoragePortV1.open` | Backend dispatch already uses `(backend.open_fn)(...)`; codec open/decode/close still use method spelling. The encrypted opener reads/decrypts a complete image. | +| `src/lib/nogc_sync_mut/database/codec/codec_registry.spl`: `PageCodecV1`, registry resolution | Factory dispatch uses `entry.factory(codec_id)`. Preserve registry admission and configured codec selection. | +| `doc/08_tracking/bug/runner_callback_field_owner_workaround_2026-10-11.md`, issue #2980 | Historical indirect function-field calls lost aggregate result ownership. Parenthesized field calls fixed parsing; a concrete typed identity-return helper repaired one historical ARM aggregate boundary. A typed local alone did not. This is a lead, not Windows P2 proof. | +| `src/lib/nogc_sync_mut/database/sqlite_format/{reader,pager,btree}.spl` | `sqlite_format_read_table_bytes` materializes rows; `SqlitePager` caches pages but retains the full snapshot. A small top-K result alone does not bound database reads or retained image memory. | +| `src/lib/nogc_sync_mut/database/sqlite_format/wimage.spl`: `sqlite_format_create_image`, `_build_table_pages`, `_build_schema_pages`, `_write_header` | Current writer assumes usable page size equals 4096 and writes reserve byte zero. It cannot safely feed SQLCipher by truncating 80 bytes after packing. Secondary-index creation is not supplied by this writer. | +| `src/lib/nogc_sync_mut/database/codec/sqlcipher4_page.spl` | Profile uses 4096-byte pages with 80 reserved bytes and authenticates ciphertext before CBC decryption. Preserve that ordering and geometry. | +| `src/lib/nogc_sync_mut/database/codec/sqlcipher4_keys.spl`, `sqlcipher4_page.spl` | Comments explicitly admit unwiped frozen KDF-result copies and environment-secret text. Wiping a mutable copy does not erase those originals. | +| `storage_port.spl`: encrypted-open error exits | Later-page authentication failure and decrypted-header rejection can leave previously accumulated plaintext unwiped. A whole-file byte copy also does not establish a consistent live SQLite/WAL snapshot. | + +## Phase A: establish working reads with finite boundary evidence + +Provision a fresh, private, multi-page SQLCipher-compatible database containing realistic room/member/message tables, Unicode, escaped text, equal sequence values, and overflow payloads. Prefer a real reference SQLCipher provisioner for the initial fixture so the current writer's reserve geometry does not confound read diagnosis. Keep secrets out of argv and logs. Reference provisioning is acceptance setup, not a substitute production reader. + +Record the producer hash, source revision, fixture geometry, artifact hash, stage reached, and exact error. Add nonsecret stage evidence at these real boundaries in one planned diagnostic path: + +1. Policy/schema/`target.db` admission. +2. Configured storage-backend dispatch and codec registry factory return. +3. Key-session return and first-page authentication/decode. +4. Later-page decode and SQLite header/schema admission. +5. Concrete row return and Messenger JSON result. + +Do not rerun the exhausted unchanged RTS01 fixture. A new acceptance run must exercise substantive secure-provision/read work and produce new evidence; renaming a fixture or opening a new branch does not reset the six-cycle cap. Record carried-forward cycles and stop when the applicable limit is reached. If it prevents the next needed experiment, report the limitation rather than silently resetting it. + +At a demonstrated function-field boundary, first use explicit `(receiver.callback)(...)` spelling, including the factory when relevant. If evidence still identifies aggregate-owner loss there, use a small concrete identity-return helper matching the actual result type, such as `Result`. Keep original failures and payloads intact. Do not scatter generic wrappers, assume Option unwrapping is the cause, or call a backend directly to skip registry policy. Rebuild only the affected application acceptance artifact with the pinned producer; never rebuild P2. + +Phase A is complete only when actual production entry points return the expected rooms, members, ordered history, and fetched body after the provisioner has closed the database, and negative cases below fail correctly. A passing leaf selector or page decoder alone is insufficient. + +## Secure storage contract + +Introduce a snapshot owner below the existing port, with a proposed interface `open_read_snapshot(path, policy) -> Result`. Names are design proposals, not existing APIs. + +* Open a regular file through a no-follow, read-only native handle and retain its identity for the session. Do not change source bytes, permissions, journals, or checkpoints. +* Support a consistent committed view using actual SQLite-compatible locking and validated WAL overlay, or return a typed `SnapshotUnavailable` for unsupported/live journal states. Checking whether `-wal` exists and then copying the main file is not a race-safe implementation. A quiescent external snapshot requires a real ownership/locking guarantee. +* Admit size, page count, reserve geometry, table/schema bounds, and cache budget before allocating proportionally. Reject malformed page references and overflow chains without unbounded traversal. +* Authenticate each page before releasing its plaintext. Use a common externally safe authentication failure for wrong keys and damaged authenticated pages; do not return partial rows after failure. +* If reads become lazy, explicitly describe their guarantee as authentication of accessed pages. Provide a separate full-integrity operation for detecting tampering in unvisited pages; do not claim that a successful limited query authenticates the entire file. + +Make key, plaintext-page, and snapshot ownership explicit. Route every failure and normal close through cleanup that wipes accumulated plaintext, decoded scratch pages, session keys, and owned password buffers. Eviction must wipe pages before releasing them. Returned message text is deliberately application-visible; define its lifetime rather than pretending that returned copies were erased. + +Close the frozen-copy gap at the actual provider boundary, for example by deriving into a caller-owned wipeable buffer or retaining keys in an opaque native session with explicit destruction. Test that provider independently through its real owner. Avoid manufacturing an additional frozen return value just to wipe its mutable copy. Environment-secret text cannot support a complete-erasure claim while the runtime retains immutable copies; use a bounded secret-provider interface for that claim, or explicitly retain the limitation and leave security completion open. + +## Session and read-efficiency design + +Add a typed `MessengerReadSession` owning the admitted snapshot, codec session, schema profile, schema metadata, and bounded page cache. Proposed operations are `open`, `list_rooms`, `list_members`, `history`, `fetch_message`, `check_integrity`, and `close`; existing one-shot API wrappers may own a short session for compatibility. Long-lived callers must actually reuse the session to obtain the benefit. + +Perform one configured key-derivation setup and one schema discovery per session; do not repeat them for each operation. Derivation setup includes the profile's required key derivations, not literally a single PBKDF invocation. Invalidate the session on snapshot identity change; never reuse authenticated plaintext against a different file generation. + +Replace whole-table row materialization with a bounded iterator and an authenticated random-access pager. A bounded scan is the explicit fallback where no supported index exists. Implement actual index B-tree seek only for supported schemas/indexes, including a stable cursor and correct row lookup; do not advertise indexed latency from a synthetic index that the production reader cannot traverse. + +Preserve `src/app/devhub/messenger/adapter/history_rows.spl` semantics: lowest sequences ascending, source-order stability for equal keys, and validation of every matching sequence even outside the selected limit. Early indexed termination changes the last rule. Either establish validation separately or make a deliberate, documented contract change with acceptance coverage. Do not silently reinterpret history as latest-first. + +Instrument key setups, schema discoveries, physical bytes/pages read, authenticated/decrypted pages, cache hits/evictions, peak owned bytes, and elapsed open/query time. Initial structural budgets are one setup/schema discovery per session and a configurable fixed page-cache bound. Record cold/warm measurements on small and large databases on the same host. For indexed queries, require measured read growth consistent with the implemented seek plus result pages; for scans, report linear reads honestly. Set numeric latency targets after this baseline, before accepting the optimization; there is no measured latency claim in this plan. + +## Owned database creation, when required + +If acceptance must use the repository writer, or a production creation API is required, add an explicit reserved-byte parameter through every writer sizing/packing path: schema, table leaf/interior cells, overflow pages, content offsets, and header reserve byte. For this profile use page size 4096, reserve 80, usable 4016; honor the first page's additional SQLite header offset. Preserve reserve-zero SQLite behavior for existing callers. + +A proposed `create_owned_sqlcipher_database(path, schema, rows, secret_ref)` must use cryptographic randomness for file salt and page IVs, the real configured SQLCipher codec, and private native staging before secret bytes are written. Publish atomically without overwriting an existing destination, check flush/publication failures, and clean up/wipe on failure. A deterministic fixture RNG is not a production default. Do not create a plaintext temporary database on disk. + +Require independent reference SQLCipher open/query/integrity checks of produced files, and production reads of reference-produced files. Include page-boundary and overflow cases that would expose a 4096-versus-4016 packing mistake. Reference tooling's absence is an explicit interoperability qualification gap, not permission to label self-roundtrip as independent verification. + +## Implementation lanes and acceptance + +After the first PR lands, the merge owner freezes shared interfaces and schedules native builds. Suggested lanes retain the research ownership: Sol isolates and fixes the proven callback/result boundary; Terra implements snapshot and wipe ownership; the root owner integrates production sessions and provisioning; Astra reviews security, behavior, and exact final commit. These are assignments for the next phase, not claims that work has started. No additional sidecars are required for this finite scope. + +| Acceptance group | Required evidence | +|---|---| +| Working Messenger feature | Closed fresh encrypted fixture reopened through configured production registry; rooms/members/history/fetch results asserted, including Unicode, escapes, ties, missing records and malformed rows. | +| Authentication and cleanup | Missing secret, wrong key, first/later-page tamper, corrupt decrypted header, and injected read/decode failures produce no partial output; owned sensitive buffers and private staging are checked after failure/close. Full-integrity test includes unvisited-page tamper. | +| Read-only snapshot | Source bytes and metadata remain unchanged; live writer/checkpoint race yields a valid committed snapshot or explicit refusal. Symlink/nonregular input is rejected according to the snapshot contract. | +| Provisioning | Private file permissions/Windows DACL, no overwrite, CSPRNG failure refusal, correct reserved geometry, overflow data, and independent SQLCipher interoperability. Apply production-creator cases only if that API is implemented. | +| Efficiency | Same-session setup/schema counts, bounded cache/row retention, actual cold/warm timings, physical-read counters across database sizes, and supported-index versus fallback-scan evidence. | +| Honest execution | Real production calls and negative controls; captured exit status and stage evidence. SSpec's unresolved producer compatibility is reported separately. A native acceptance artifact can prove its actual operations without claiming SSpec was repaired. | + +Every temporary workaround must carry `@workaround`, a registered canonical BugDB ID, affected producer hash, narrow boundary/reproducer, and a removal condition tied to the eventual producer/provider repair and passing retained regression. Use the canonical append-preserving registration workflow for legacy BugDB rows; do not regenerate the registry or invent tags with unregistered IDs. Reuse #2980 as research evidence without declaring it the current cause before isolation. + +The completion report must separate implemented behavior, actual passing acceptance, unexecuted cases, remaining producer defects, and genuinely external dependencies. Neither this design nor the independent first PR constitutes a working Messenger database claim. diff --git a/doc/08_tracking/bug/bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27.md b/doc/08_tracking/bug/bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27.md new file mode 100644 index 00000000000..49edb11a297 --- /dev/null +++ b/doc/08_tracking/bug/bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27.md @@ -0,0 +1,13 @@ +# Bug database full save truncates opaque legacy row payloads + +Canonical ID: `bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27`. Status: open. Owner: canonical BugDatabase. Created: 2026-10-27. + +The canonical registry contains legacy rows with more fields than their declared table schema. The generic table parser and full save do not preserve every overwide legacy row payload, so a complete BugDatabase.save round trip can discard untouched records or fields. The domain append-only API serializes only new records and preserves the complete original body byte for byte apart from inserted canonical rows. + +## Evidence + +Read-only baseline audit found eleven versus ten fields at lines2452,2453,2463 and additional overwide rows. No real registry write or generic regeneration was performed. Append-only native copy qualification and independent byte preservation must pass before actual registration. + +## Removal condition + +Retire the legacy append compatibility path only after all legacy rows are repaired by their owners or the canonical serializer losslessly retains opaque fields, and a complete scratch round trip preserves every original payload byte and record. Never normalize or truncate records merely to enable registration. diff --git a/doc/08_tracking/bug/bug_db.sdn b/doc/08_tracking/bug/bug_db.sdn index eb93a6730ac..90aa91ba0c5 100644 --- a/doc/08_tracking/bug/bug_db.sdn +++ b/doc/08_tracking/bug/bug_db.sdn @@ -1,4 +1,4 @@ -#sdn-crc32:1763259956 +#sdn-crc32:2967022985 #sdn-crc32:1744891490 #sdn-crc32:2280124460 #sdn-crc32:2139344494 @@ -6,6 +6,15 @@ strings |id, value| bug_descriptions |bug_id, line_num, content| + devhub_secured_db_key_session_open_failure_2026_10_27, 0, Fresh secured DB ledger1 compiled successfully with pinned Phase2 but actual execution returned Err at direct key-session open before encrypted decode. Messenger database access remains an unresolved main-feature blocker; this is not an infrastructure exception. + phase2_octal_permission_mode_lowering_2026_10_27, 0, "Pinned P2 octal-mode source produced read-only final and staging files. The staging file contained the correct replacement token and expiry, so this failure was not a cache parser failure. Decimal 384 is exactly mode0600 and decimal448 is mode0700." + phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27, 0, Native binary upload failed with an invalid array handle before any loopback request. The accelerated u8 SHA256 boundary is incompatible with this producer. The workaround copies octets by index into the established i64 SHA256 engine without changing the public byte contract. + windows_core_http_client_unavailable_2026_10_27, 0, "The Windows core native HTTP branch explicitly returns unavailable. A shared real curl compatibility transport uses bounded execution, private mode384 header/body/response files, no redirects and no TLS bypass; request secrets are absent from argv. Applications retain configured public std interfaces." + phase2_result_tuple_try_binding_metadata_2026_10_27, 0, "Explicit typed Ok and Err bindings preserve u64 and byte-array tuple payload metadata where native Result propagation did not. Related compiler Result tuple repairs are in source, but the pinned producer has not been rebuilt and the compatibility bindings remain." + phase2_bounded_process_text_abi_2026_10_27, 0, The actual HTTP compatibility probe returned missing command although curl8.21.0 was installed. The existing bounded tuple runtime provider accepts explicit command pointer and UTF8 length; a source adapter can preserve that ABI without changing the compiler or provider. + phase2_module_static_array_concat_initialization_2026_10_27, 0, "The compatibility flag catalogs retain literal typed arrays instead of executing array concatenation while initializing module statics. Their public values remain unchanged. This is narrower than ordinary function-local array concatenation, which remains supported by other focused checks." + bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27, 0, "The canonical registry contains legacy rows with more fields than their declared table schema. The generic table parser and full save do not preserve every overwide legacy row payload, so a complete BugDatabase.save round trip can discard untouched records or fields. The domain append-only API serializes only new records and preserves the complete original body byte for byte apart from inserted canonical rows." + phase2_crc32_i64_width_spill_2026_10_27, 0, A native append-only registry copy preserved all original payload bytes but wrote CRC header -2292158518 while independent unsigned zlib computed2002808778. The values differ by2^32. This proves a CRC width or ABI boundary discrepancy; the exact widening or complement cause is not isolated. Masking public i64 CRC outputs to their low32bits preserves the actual CRC register contract. stage3_hir_shard_workers_collide_on_mem_snapshot_file_2026_10_19, 0, "run_ledger_shards spawns 8 HIR workers with the parent env; mem_snapshot_begin opens SIMPLE_MEM_SNAPSHOT_FILE O_EXCL, so every worker failed 'SIMPLE_MEM_SNAPSHOT_FILE could not be established safely', hir_shard_guard_block blocked the ledger and the next claim reported 'HIR shard ledger claim failed'. Fix: guard the retained-path mem_snapshot_begin with 'if not hir_shard_mode:' exactly like the streaming path." test_coverage_wrapper_native_build_refused_by_scv_admission_2026_10_19, 0, "`simple test --coverage` (seed bin/simple, release_phase3 01ff68b9c2c) builds the instrumented wrapper with `native-build --backend llvm --refresh-source-authority --entry-closure --runtime-bundle core-c-bootstrap`, and that build is refused before codegen, so every spec ABORTS BEFORE EXECUTION and the report is empty (Files tracked: 0). The rt_secure_temp_dir_set_parent twin is no longer the blocker. Fresh worktree: `SCV-E-ADMISSION: compile-event-journal-missing`; with SIMPLE_SCV_INVENTORY_COLD_INIT=1: `SCV-E-ADMISSION: git-event-refresh-failed` (inventory_events.spl _compiler_inventory_refresh_locked_v1; which `return nil` in _compiler_inventory_git_events_traced_v1 fires is not yet isolated; enabling SIMPLE_INVENTORY_TRACE_ATTEMPT instead fails with `semantic: invalid assignment: field assignment target is not a place`, i.e. the trace hook itself does not run on the seed). The warm main checkout also aborts (1.7s) with its real cause hidden behind a lint warning line; the diagnostic picker was fixed in the same change so the next run names it. The wrapper stages under test/native-generated--*/ (not gitignored, inside the SCV test family). Single-file mode is separate and also gives no report: `simple test .spl --coverage` exits 0 with NO coverage output (the flag is silently ignored; test_runner_single gates on SIMPLE_COVERAGE only), and with SIMPLE_COVERAGE=1 it fails `error: spawned test process did not publish a valid child APK activation receipt`. UPDATE 2026-10-20 (work/cover-headers-and-coverage): root-cause chain, layer by layer. (1) compile-event-journal-missing: FIXED -- the coverage runner (lib test_runner_execute.spl coverage_scv_cold_init_needed) gives the FIRST wrapper build of a run SIMPLE_SCV_INVENTORY_COLD_INIT=1 only when build/scv has no journal pointer; later builds stay warm and strict. Verified: build/scv/source-inventory published from a fresh checkout. (2) git-event-refresh-failed under cold init was really the seed failing extern `__errno_location` (inventory_aliases.compiler_alias_present_v1 -> path_identity.path_entry_present): native_loader ProcessSymbolProvider/ChainedProvider stripped one leading `_` (Mach-O ABI) BEFORE lookup, so `__errno_location` was looked up as `_errno_location`, the JIT spliced the call to the interpreter, which has no backing. FIXED in source (exact name first, stripped spelling as fallback; native_loader unit tests red->green); takes effect only after a seed rebuild+redeploy -- the deployed bin/simple still has the bug. (3) REMAINING, not coverage-specific: with a private fixed seed the wrapper compile is killed by the unit-test resource-class default budget after ~20s (passes the budget with --no-limits), and then native-build --entry-closure fails in HIR shard workers with `semantic: type mismatch: cannot cast object to i64` -- a hello-world entry fails identically, so native-build of an entry closure on the seed is broken in general. Files tracked > 0 is blocked on that. (4) single-file `test .spl --coverage` now exits 2 with `--coverage needs a directory target` instead of silently ignoring the flag (light-daemon client lane)." bdd_assert_outside_it_silent_false_green_2026_10_19, 0, "Seed dispatches assert_*/fail* as BDD builtins at Priority 3 (interpreter_call/mod.rs:723, bdd.rs eval_bdd_builtin) before user functions; a failure only sets BDD_EXPECT_FAILED, which is read solely at `it` end, so outside any `it` (comment doctest composites, `simple run` scripts, describe-level or before_all bodies) a wrong assert_equal exited 0 and `test x.spl --spl-doctest` reported 1 passed. `use std.spec.*` cannot help: the builtin shadows spec.spl assert_equal. FIXED for assert_*/fail* 2026-10-19: record_assertion_failure returns a hard error when BDD_INSIDE_IT is false. FIXED 2026-10-19 (same class, work/test-runner-leftovers): (1) `expect(2).to_equal(5)` at top level still exits 0 (matcher path, interpreter_method/mod.rs) -- now finish_bdd_matcher() (bdd.rs) makes a failed matcher outside an `it` a hard error, negated forms included; (2) pure-Simple: spec.spl fail_assertion outside a case pushes to current_test_errors with no consumer unless aggregate mode is active, so the self-hosted binary would false-green a `use std.spec.*` doctest too (the lib twin doctest_runner.spl build_spl_doctest_code injects that prelude; the app twin does not) -- now fatal: spec.spl tracks _case_depth around each example and fail_assertion exits 1 outside a case when aggregate mode is off. Remaining: a bare `expect ` (provisional, no matcher) outside an `it` is still not fatal. Not the unregistered-extern class and not name-resolution leniency." @@ -2311,6 +2320,15 @@ bug_descriptions |bug_id, line_num, content| zstd_fse_compressed_block_deferred_2026-06-15, 0, "Registration of unresolved tracking document. Priority: untriaged; P3 is a registration placeholder. The document remains the authority for diagnosis, ownership, and acceptance; this row does not claim implementation or closure." bugs_active |id, severity, status, title, file, line, reproducible_by, created_at, updated_at, valid| + devhub_secured_db_key_session_open_failure_2026_10_27, P1, open, DevHub secured messenger DB key session fails before decode, doc/08_tracking/bug/devhub_secured_db_key_session_open_failure_2026_10_27.md, 1, D:/dev/simple-devhub-secured-db-p2-20261027/build/devhub-completion/fresh-secured-db-ledger1-run.log, 2026-10-27, 2026-10-27, true + phase2_octal_permission_mode_lowering_2026_10_27, P1, open, Pinned Phase 2 lowers octal permission modes incorrectly, src/lib/common/oauth_token_cache.spl, 1, test/fixtures/devhub/native_outlook_token_cache/main.spl, 2026-10-27, 2026-10-27, true + phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27, P1, open, Pinned Phase 2 byte carrier is rejected by accelerated SHA256 FFI, src/app/devhub/adapter_minio.spl, 1, test/fixtures/devhub/native_minio_binary_put/main.spl, 2026-10-27, 2026-10-27, true + windows_core_http_client_unavailable_2026_10_27, P1, open, Windows core runtime native HTTP client provider is unavailable, src/runtime/runtime_native.c, 1, test/fixtures/devhub/native_http_curl/main.spl, 2026-10-27, 2026-10-27, true + phase2_result_tuple_try_binding_metadata_2026_10_27, P1, open, Pinned Phase 2 loses integer and byte tuple metadata across Result propagation, src/lib/nogc_sync_mut/database/sqlite_format/cell.spl, 1, test/fixtures/devhub/native_result_unsigned_tuple/main.spl, 2026-10-27, 2026-10-27, true + phase2_bounded_process_text_abi_2026_10_27, P1, open, Pinned Phase 2 bounded process call loses command text ABI, src/lib/nogc_sync_mut/sffi/process_capture.spl, 1, test/fixtures/devhub/native_http_curl/main.spl, 2026-10-27, 2026-10-27, true + phase2_module_static_array_concat_initialization_2026_10_27, P1, open, Pinned Phase 2 cannot initialize DevHub module static arrays by concatenation, src/app/devhub/gh_compat.spl, 1, test/01_unit/app/devhub/gh_compat_spec.spl, 2026-10-27, 2026-10-27, true + bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27, P1, open, Bug database full save truncates opaque legacy row payloads, src/lib/nogc_sync_mut/database/bug.spl, 1, test/fixtures/devhub/native_bug_workaround_registration/main.spl, 2026-10-27, 2026-10-27, true + phase2_crc32_i64_width_spill_2026_10_27, P1, open, Pinned Phase 2 CRC32 i64 boundary retains unexpected high bits, src/lib/nogc_sync_mut/compression/gzip/crc.spl, 1, test/fixtures/devhub/native_bug_workaround_registration/main.spl, 2026-10-27, 2026-10-27, true native_closure_walks_bind_dotted_imports_differently_2026_10_20, P2, open, "Native closure walks bind imports differently between coordinator and snapshot-root workers", src/compiler/80.driver/driver_source_pipeline_loading.spl, 1060, doc/08_tracking/bug/native_closure_walks_bind_dotted_imports_differently_2026_10_20.md, 2026-10-20, 2026-10-20, true stage3_closure_loop_rss_growth_after_provider_feed_2026_10_20, P1, open, "Stage 3 closure loop RSS grows 20-36 MB per module with the provider feed (52 GiB cap nearly exhausted)", src/compiler/50.mir/_MirLowering/module_lowering.spl, 2394, doc/08_tracking/bug/stage3_closure_loop_rss_growth_after_provider_feed_2026_10_20.md, 2026-10-20, 2026-10-20, true stage3_hir_cache_direct_route_misses_sharded_entries_2026_10_20, P2, open, "Stage 3 HIR cache: direct route misses every entry the sharded route stored", src/compiler/80.driver/driver_hir_cache.spl, 1, doc/08_tracking/bug/stage3_hir_cache_direct_route_misses_sharded_entries_2026_10_20.md, 2026-10-20, 2026-10-20, true @@ -5109,6 +5127,15 @@ bugs |id, severity, status, title, file, line, reproducible_by, created_at, upda bug_investigation_logs |bug_id, line_num, content| + devhub_secured_db_key_session_open_failure_2026_10_27, 0, "Pinned P2 producer13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Ledger1 build0 run1. Memory oracle passed AES NIST, independent KDF/HMAC and SQLite rows/overflow. Execution stopped at STAGE fresh direct key-session return with FAIL fresh secured DB direct key session. Encrypted inputs and metadata stayed unchanged; no plaintext or journal secret files. Exact root cause remains unisolated; no further feature experiments authorized this session." + phase2_octal_permission_mode_lowering_2026_10_27, 0, Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Cache unchanged regression compiled and ran exit0 after decimal-mode workaround; artifact e285b95b2f4250817eec5555aee4d67fea0f1f299d2d3d3568381ae8f9b044fd. Earlier actual filesystem inspection observed both files readonly. The exact compiler lowering cause is not yet isolated. Workaround commit pending integration; underlying bug remains open. + phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27, 0, Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Initial native compile0 run1 artifact3adb13de091efe809b6a571ced270f697ee30de9d36d33939692a758dcbaaa8d; independent server observed zero requests. Final typed-carrier transport qualification is recorded in the parent verification report and is not assumed here. Workaround commit pending integration; underlying bug remains open. + windows_core_http_client_unavailable_2026_10_27, 0, Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Source-confirmed Windows provider gap. Actual curl8.21.0 exists on the qualification host. HTTP text and signed binary loopback acceptance remains separately bound to the parent qualification artifacts; source review alone is not runtime PASS. Workaround commit pending integration; underlying bug remains open. + phase2_result_tuple_try_binding_metadata_2026_10_27, 0, Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Earlier Messenger compilation admitted all169 MIR modules after explicit SQLite tuple bindings. Supplemental unsigned varint and truncation checks passed. Production Messenger adapter RTS01 remains unresolved; this workaround is not proof that the database read works. Workaround commit pending integration; underlying bug remains open. + phase2_bounded_process_text_abi_2026_10_27, 0, Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Frozen-source HTTP and binary probes compiled but returned missing command before any localhost request. The exact tuple-provider source adapter and final actual loopback evidence are owned by the provider lane; no success is inferred by this record. Workaround commit pending integration; underlying bug remains open. + phase2_module_static_array_concat_initialization_2026_10_27, 0, "Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. The retained literal tables were part of the earlier DevHub P2 source repair. Current GH JSON and flag native graphs compile and run successfully, but this is not a new independent replay of the original static-initializer failure. Related older registry issue native_array_concat_dispatch_bootstrap_2026_10_16 remains separate. Workaround commit pending integration; underlying bug remains open." + bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27, 0, "Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Read-only baseline audit found eleven versus ten fields at lines2452,2453,2463 and additional overwide rows. No real registry write or generic regeneration was performed. Append-only native copy qualification and independent byte preservation must pass before actual registration. Workaround commit pending integration; underlying bug remains open." + phase2_crc32_i64_width_spill_2026_10_27, 0, "Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Cycle3 canonical API compile0 and copy-run0 were not passing evidence because the independent strict CRC oracle failed. Exact old-body preservation passed after removing only28newrowlines, including all5265oldrows and20opaqueoverwide rows. Real canonical registry remained untouched. New actual KAT and copy qualification are required before registration. Workaround commit pending integration; underlying bug remains open." block_highlighting_raw_escape_native_hex_2026_10_12, 0, "Actual failed-only GDB found Comment end8 rather than6 and newline constant bytes5c6e. Raw-string language contract confirms source misuse. Cycle3 remaining four cases exit0 with strict paired recovered CI runtime; prior three PASS retained." phase1_docstring_trim_start_dispatch_2026_10_12, 0, "Actual Phase1 source diagnostic7 primitive checks2.014s159MBtree and6 real extractor checks2.527s161MBtree PASS; byteexact Unicode tabs blank repeated trailing and sdoctest assertions. Authority pins stable. No fourth whole listing." optional_array_ifval_type_and_nil_constant_2026_10_12, 0, "Baseline1a9279 MIR failure3.974s; HIR successorad02 codegen failure4.325s; HIR plus narrow nil-copy successorcb8ce176 codegen failure4.304s. Expanded fixture unchanged99c36f. Retained IR evidence and seven authored unexecuted SSpec cases. Three-cycle cap STOP." @@ -6833,6 +6860,15 @@ bug_investigation_logs |bug_id, line_num, content| bug_fix_strategies |bug_id, line_num, content| + devhub_secured_db_key_session_open_failure_2026_10_27, 0, "Repair direct key-session open with a discriminating error regression. Then qualify secure SQLite decode and efficient snapshot paging with memory-only plaintext, unchanged encrypted inputs, cleanup and independent row oracles before claiming Messenger DB support." + phase2_octal_permission_mode_lowering_2026_10_27, 0, "Remove the source workaround only after the deployed producer proves octal and decimal OS modes identical in actual create, replace and permission checks; require credential and OAuth cache regressions. Do not close this underlying compiler bug from workaround success." + phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27, 0, "Retire after this deployed producer and linked crypto provider pass exact octets 0,128,255 plus empty, Unicode byte sequences and independent SHA256 goldens through the accelerated ABI. Keep the byte-exact upload and SigV4 oracle." + windows_core_http_client_unavailable_2026_10_27, 0, "Retire only after the selected Windows native provider supports text and exact binary bodies, status and errors, size and time bounds, redirects/TLS policy, secret handling and independent HTTP/SigV4 acceptance with the deployed compiler. An installed curl workaround does not close the missing provider bug." + phase2_result_tuple_try_binding_metadata_2026_10_27, 0, Retire after the actual deployed producer passes try-propagated u64 maxima and byte arrays plus the full SQLite parser and production Messenger read/wrong-key/tamper regressions. Keep the underlying bug open until then. + phase2_bounded_process_text_abi_2026_10_27, 0, "Remove the source adapter only after the deployed producer correctly marshals command text, argv, deadline and output bound to the normal bounded process ABI and passes real quoted-argument, Unicode, timeout, output-limit and HTTP transport regressions." + phase2_module_static_array_concat_initialization_2026_10_27, 0, "Retire only after actual module-static concatenation preserves every flag value and method/type metadata under the deployed producer, with native command flag regressions and no startup evaluation error." + bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27, 0, "Retire the legacy append compatibility path only after all legacy rows are repaired by their owners or the canonical serializer losslessly retains opaque fields, and a complete scratch round trip preserves every original payload byte and record. Never normalize or truncate records merely to enable registration." + phase2_crc32_i64_width_spill_2026_10_27, 0, "Retire the explicit width normalization only after the deployed producer passes text123456789 CRC3421780262, empty input, chunked update through high-bit-set states and independent unsigned zlib comparisons for real registry bodies. Never accept negative header CRC or relax the external oracle." mir_json_c0_escaping_2026_10_12, 0, "Preserve existing short escapes; iterate chars and encode remaining C0 as ASCII Unicode escapes. Require all32 golden cases, interior NUL byte and length roundtrips, Unicode, literal escape text and real MIR fields on genuine target native producer. No seed tests or memory-fix claim." imported_generic_native_callee_owner_loss_2026_10_12, 0, "Retain focused native receipts, provenance and regression fixtures; complete remaining compiler/core/MCP gates before full qualification. Workaround tag IMPORTED_GENERIC_CALLEE_OWNER" nul_literal_double_box_constructor_2026_10_12, 0, "Keep NUL literal constructor input raw Ptr(U8) with authored byte count; ordinary Opaque string constants unchanged. Original exact regressions retained. Old9fd facet28checks scoped only; current source/core/generation qualification UNEXECUTED, no exhaustedbuildretry." diff --git a/doc/08_tracking/bug/devhub_secured_db_key_session_open_failure_2026_10_27.md b/doc/08_tracking/bug/devhub_secured_db_key_session_open_failure_2026_10_27.md new file mode 100644 index 00000000000..6155f112b2b --- /dev/null +++ b/doc/08_tracking/bug/devhub_secured_db_key_session_open_failure_2026_10_27.md @@ -0,0 +1,9 @@ +# DevHub secured Messenger DB key-session open failure + +Status: OPEN. Severity: P1. Date: 2026-10-27. + +The fresh secured DB ledger1 build passed with the existing Phase2 compiler, but actual execution failed at direct key-session open before encrypted decode. The memory oracle passed NIST AES, independent KDF/HMAC, SQLite integrity, rows and overflow checks. The encrypted inputs and metadata stayed unchanged and no plaintext or journal secret files remained. + +This is the main DevHub Messenger database feature blocker, not an infrastructure exemption. The exact cause remains unisolated. Repair key-session open first; then qualify secure SQLite decode, snapshot lifetime and efficient bounded paging with independent row oracles and memory-only plaintext. Do not claim DB support from the memory oracle alone. + +Evidence: `D:/dev/simple-devhub-secured-db-p2-20261027/build/devhub-completion/fresh-secured-db-ledger1-build.log` and `fresh-secured-db-ledger1-run.log`. Ledger1 build exit0 / run exit1. No additional DB feature experiments in this session. diff --git a/doc/08_tracking/bug/phase2_bounded_process_text_abi_2026_10_27.md b/doc/08_tracking/bug/phase2_bounded_process_text_abi_2026_10_27.md new file mode 100644 index 00000000000..a1633d8850e --- /dev/null +++ b/doc/08_tracking/bug/phase2_bounded_process_text_abi_2026_10_27.md @@ -0,0 +1,21 @@ +# Pinned Phase 2 bounded process call loses command text ABI + +- Canonical ID: `phase2_bounded_process_text_abi_2026_10_27` +- Status: open; source compatibility does not close the underlying bug. +- Owner: DevHub / compiler-runtime owners. +- Created: 2026-10-27. +- Producer: P2 `13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b`. +- Source: `src/lib/nogc_sync_mut/sffi/process_capture.spl`. +- Regression: `test/fixtures/devhub/native_http_curl/main.spl`. + +The actual HTTP compatibility probe returned missing command although curl8.21.0 was installed. The existing bounded tuple runtime provider accepts explicit command pointer and UTF8 length; a source adapter can preserve that ABI without changing the compiler or provider. + +## Evidence + +Frozen-source HTTP and binary probes compiled but returned missing command before any localhost request. The exact tuple-provider source adapter and final actual loopback evidence are owned by the provider lane; no success is inferred by this record. + +## Removal condition + +Remove the source adapter only after the deployed producer correctly marshals command text, argv, deadline and output bound to the normal bounded process ABI and passes real quoted-argument, Unicode, timeout, output-limit and HTTP transport regressions. + +Commit identity and final receipts must be appended through the canonical BugDatabase API after integration. No compiler rebuild, standard SSpec execution or unexecuted matrix is claimed. diff --git a/doc/08_tracking/bug/phase2_crc32_i64_width_spill_2026_10_27.md b/doc/08_tracking/bug/phase2_crc32_i64_width_spill_2026_10_27.md new file mode 100644 index 00000000000..a0728489898 --- /dev/null +++ b/doc/08_tracking/bug/phase2_crc32_i64_width_spill_2026_10_27.md @@ -0,0 +1,15 @@ +# Pinned Phase 2 CRC32 i64 boundary retains unexpected high bits + +Canonical ID: `phase2_crc32_i64_width_spill_2026_10_27`. Status: open. Owner: compiler/runtime CRC boundary. Created: 2026-10-27. + +A native append-only registry copy preserved all original payload bytes but wrote CRC header -2292158518 while independent unsigned zlib computed2002808778. The values differ by2^32. This proves a CRC width or ABI boundary discrepancy; the exact widening or complement cause is not isolated. Masking public i64 CRC outputs to their low32bits preserves the actual CRC register contract. + +## Evidence + +Cycle3 canonical API compile0 and copy-run0 were not passing evidence because the independent strict CRC oracle failed. Exact old-body preservation passed after removing only28newrowlines, including all5265oldrows and20opaqueoverwide rows. Real canonical registry remained untouched. New actual KAT and copy qualification are required before registration. + +## Removal condition + +Retire the explicit width normalization only after the deployed producer passes text123456789 CRC3421780262, empty input, chunked update through high-bit-set states and independent unsigned zlib comparisons for real registry bodies. Never accept negative header CRC or relax the external oracle. + +No compiler rebuild or exact root-cause claim is made. Public CRC values remain unsigned32-bit values carried in i64. diff --git a/doc/08_tracking/bug/phase2_module_static_array_concat_initialization_2026_10_27.md b/doc/08_tracking/bug/phase2_module_static_array_concat_initialization_2026_10_27.md new file mode 100644 index 00000000000..1fd7ca3297a --- /dev/null +++ b/doc/08_tracking/bug/phase2_module_static_array_concat_initialization_2026_10_27.md @@ -0,0 +1,21 @@ +# Pinned Phase 2 cannot initialize DevHub module static arrays by concatenation + +- Canonical ID: `phase2_module_static_array_concat_initialization_2026_10_27` +- Status: open; source compatibility does not close the underlying bug. +- Owner: DevHub / compiler-runtime owners. +- Created: 2026-10-27. +- Producer: P2 `13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b`. +- Source: `src/app/devhub/gh_compat.spl`. +- Regression: `test/01_unit/app/devhub/gh_compat_spec.spl`. + +The compatibility flag catalogs retain literal typed arrays instead of executing array concatenation while initializing module statics. Their public values remain unchanged. This is narrower than ordinary function-local array concatenation, which remains supported by other focused checks. + +## Evidence + +The retained literal tables were part of the earlier DevHub P2 source repair. Current GH JSON and flag native graphs compile and run successfully, but this is not a new independent replay of the original static-initializer failure. Related older registry issue native_array_concat_dispatch_bootstrap_2026_10_16 remains separate. + +## Removal condition + +Retire only after actual module-static concatenation preserves every flag value and method/type metadata under the deployed producer, with native command flag regressions and no startup evaluation error. + +Commit identity and final receipts must be appended through the canonical BugDatabase API after integration. No compiler rebuild, standard SSpec execution or unexecuted matrix is claimed. diff --git a/doc/08_tracking/bug/phase2_octal_permission_mode_lowering_2026_10_27.md b/doc/08_tracking/bug/phase2_octal_permission_mode_lowering_2026_10_27.md new file mode 100644 index 00000000000..d4214f3a36a --- /dev/null +++ b/doc/08_tracking/bug/phase2_octal_permission_mode_lowering_2026_10_27.md @@ -0,0 +1,21 @@ +# Pinned Phase 2 lowers octal permission modes incorrectly + +- Canonical ID: `phase2_octal_permission_mode_lowering_2026_10_27` +- Status: open; an active source workaround is not a compiler/provider fix. +- Owner: DevHub completion / compiler-runtime owners. +- Created: 2026-10-27. +- Producer: existing P2 `13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b`; no compiler rebuild. +- Source: `src/lib/common/oauth_token_cache.spl`. +- Regression: `test/fixtures/devhub/native_outlook_token_cache/main.spl`. + +Pinned P2 octal-mode source produced read-only final and staging files. The staging file contained the correct replacement token and expiry, so this failure was not a cache parser failure. Decimal 384 is exactly mode0600 and decimal448 is mode0700. + +## Evidence + +Cache unchanged regression compiled and ran exit0 after decimal-mode workaround; artifact e285b95b2f4250817eec5555aee4d67fea0f1f299d2d3d3568381ae8f9b044fd. Earlier actual filesystem inspection observed both files readonly. The exact compiler lowering cause is not yet isolated. + +## Removal condition + +Remove the source workaround only after the deployed producer proves octal and decimal OS modes identical in actual create, replace and permission checks; require credential and OAuth cache regressions. Do not close this underlying compiler bug from workaround success. + +The standard SSpec suite, all runtime-family combinations and any claim not explicitly tied to an actual artifact remain unqualified. Record the workaround commit in the canonical BugDatabase investigation log after integration; never retire it from a source-only review. diff --git a/doc/08_tracking/bug/phase2_result_tuple_try_binding_metadata_2026_10_27.md b/doc/08_tracking/bug/phase2_result_tuple_try_binding_metadata_2026_10_27.md new file mode 100644 index 00000000000..332776cb90b --- /dev/null +++ b/doc/08_tracking/bug/phase2_result_tuple_try_binding_metadata_2026_10_27.md @@ -0,0 +1,21 @@ +# Pinned Phase 2 loses integer and byte tuple metadata across Result propagation + +- Canonical ID: `phase2_result_tuple_try_binding_metadata_2026_10_27` +- Status: open; an active source workaround is not a compiler/provider fix. +- Owner: DevHub completion / compiler-runtime owners. +- Created: 2026-10-27. +- Producer: existing P2 `13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b`; no compiler rebuild. +- Source: `src/lib/nogc_sync_mut/database/sqlite_format/cell.spl`. +- Regression: `test/fixtures/devhub/native_result_unsigned_tuple/main.spl`. + +Explicit typed Ok and Err bindings preserve u64 and byte-array tuple payload metadata where native Result propagation did not. Related compiler Result tuple repairs are in source, but the pinned producer has not been rebuilt and the compatibility bindings remain. + +## Evidence + +Earlier Messenger compilation admitted all169 MIR modules after explicit SQLite tuple bindings. Supplemental unsigned varint and truncation checks passed. Production Messenger adapter RTS01 remains unresolved; this workaround is not proof that the database read works. + +## Removal condition + +Retire after the actual deployed producer passes try-propagated u64 maxima and byte arrays plus the full SQLite parser and production Messenger read/wrong-key/tamper regressions. Keep the underlying bug open until then. + +The standard SSpec suite, all runtime-family combinations and any claim not explicitly tied to an actual artifact remain unqualified. Record the workaround commit in the canonical BugDatabase investigation log after integration; never retire it from a source-only review. diff --git a/doc/08_tracking/bug/phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27.md b/doc/08_tracking/bug/phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27.md new file mode 100644 index 00000000000..609dd958d49 --- /dev/null +++ b/doc/08_tracking/bug/phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27.md @@ -0,0 +1,21 @@ +# Pinned Phase 2 byte carrier is rejected by accelerated SHA256 FFI + +- Canonical ID: `phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27` +- Status: open; an active source workaround is not a compiler/provider fix. +- Owner: DevHub completion / compiler-runtime owners. +- Created: 2026-10-27. +- Producer: existing P2 `13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b`; no compiler rebuild. +- Source: `src/app/devhub/adapter_minio.spl`. +- Regression: `test/fixtures/devhub/native_minio_binary_put/main.spl`. + +Native binary upload failed with an invalid array handle before any loopback request. The accelerated u8 SHA256 boundary is incompatible with this producer. The workaround copies octets by index into the established i64 SHA256 engine without changing the public byte contract. + +## Evidence + +Initial native compile0 run1 artifact3adb13de091efe809b6a571ced270f697ee30de9d36d33939692a758dcbaaa8d; independent server observed zero requests. Final typed-carrier transport qualification is recorded in the parent verification report and is not assumed here. + +## Removal condition + +Retire after this deployed producer and linked crypto provider pass exact octets 0,128,255 plus empty, Unicode byte sequences and independent SHA256 goldens through the accelerated ABI. Keep the byte-exact upload and SigV4 oracle. + +The standard SSpec suite, all runtime-family combinations and any claim not explicitly tied to an actual artifact remain unqualified. Record the workaround commit in the canonical BugDatabase investigation log after integration; never retire it from a source-only review. diff --git a/doc/08_tracking/bug/windows_core_http_client_unavailable_2026_10_27.md b/doc/08_tracking/bug/windows_core_http_client_unavailable_2026_10_27.md new file mode 100644 index 00000000000..5a4312a0f7b --- /dev/null +++ b/doc/08_tracking/bug/windows_core_http_client_unavailable_2026_10_27.md @@ -0,0 +1,21 @@ +# Windows core runtime native HTTP client provider is unavailable + +- Canonical ID: `windows_core_http_client_unavailable_2026_10_27` +- Status: open; an active source workaround is not a compiler/provider fix. +- Owner: DevHub completion / compiler-runtime owners. +- Created: 2026-10-27. +- Producer: existing P2 `13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b`; no compiler rebuild. +- Source: `src/runtime/runtime_native.c`. +- Regression: `test/fixtures/devhub/native_http_curl/main.spl`. + +The Windows core native HTTP branch explicitly returns unavailable. A shared real curl compatibility transport uses bounded execution, private mode384 header/body/response files, no redirects and no TLS bypass; request secrets are absent from argv. Applications retain configured public std interfaces. + +## Evidence + +Source-confirmed Windows provider gap. Actual curl8.21.0 exists on the qualification host. HTTP text and signed binary loopback acceptance remains separately bound to the parent qualification artifacts; source review alone is not runtime PASS. + +## Removal condition + +Retire only after the selected Windows native provider supports text and exact binary bodies, status and errors, size and time bounds, redirects/TLS policy, secret handling and independent HTTP/SigV4 acceptance with the deployed compiler. An installed curl workaround does not close the missing provider bug. + +The standard SSpec suite, all runtime-family combinations and any claim not explicitly tied to an actual artifact remain unqualified. Record the workaround commit in the canonical BugDatabase investigation log after integration; never retire it from a source-only review. diff --git a/doc/08_tracking/todo/todo_db.sdn b/doc/08_tracking/todo/todo_db.sdn index 2b201c34ec4..0bf8d69b7f9 100644 --- a/doc/08_tracking/todo/todo_db.sdn +++ b/doc/08_tracking/todo/todo_db.sdn @@ -1,4 +1,6 @@ +#sdn-crc32:2283810727 todos |id, keyword, area, priority, description, file, line, issue, blocked, status, valid| + 377, TODO, devhub, P1, Complete secured Messenger DB key-session open then secure SQLite snapshot and efficient bounded paging qualification, doc/08_tracking/bug/devhub_secured_db_key_session_open_failure_2026_10_27.md, 1, devhub_secured_db_key_session_open_failure_2026_10_27, Actual pinned Phase2 ledger1 key-session Err before decode; main DevHub feature remains unverified, blocked, true 321, TODO, test_runner, P1, "Verify macOS default and explicit max-procs policy plus safe-mode no-limits with source-matched Phase 2 full CLI and test runner", doc/08_tracking/bug/test_runner_ulimit_caps_unusable_on_macos_2026-09-05.md, 1, , "Phase 2 native compilation awaits SCV cold-inventory memory repair; bootstrap compiler has no test command", open, true 0, TODO, general, P3, "wire up hwprobe when available", src/compiler/30.types/simd_capabilities.spl, 492, , "", open, true 1, TODO, general, P3, "promote this to self.error_fatal (with an Unreachable MIR", src/compiler/50.mir/_MirLoweringExpr/switch_operators_calls.spl, 2944, , "", open, true diff --git a/doc/09_report/verification/devhub_close_handoff_20261027.md b/doc/09_report/verification/devhub_close_handoff_20261027.md new file mode 100644 index 00000000000..37ac5ad8b7a --- /dev/null +++ b/doc/09_report/verification/devhub_close_handoff_20261027.md @@ -0,0 +1,18 @@ +# DevHub session close handoff + +The user requested canonical bug/TODO registration, landing eligible work, and closing this session. Feature experiments stopped. The secured Messenger database is not fixed. + +## Preserved work + +- Independent DevHub fixes: PR [5085](https://github.com/ormastes/simple/pull/5085), branch `fix/devhub-completion-20261027`. +- Raw registry fixtures now retain their intentional bytes across Windows/Linux checkout (`abb16238d22`). No CRC gate exemption was added. +- Real mail SDN helper provisioning is saved as WIP (`5eb7aa40ee8`). Focused tests using the existing real Windows helper passed, as did four provisioning admission tests. Qualified host producer profiles are missing, so the hosted matrix is not qualified and provisioning fails closed. +- Secured DB branch `fix/devhub-secured-db-p2-20261027` is pushed. It preserves callback/cleanup changes, a fresh acceptance fixture, an unreferenced/unrun snapshot candidate, and a detailed failure report at `doc/09_report/verification/devhub_secured_db_session_close_20261027.md`. + +## Actual blocker and landing status + +The fresh secured-DB native build succeeded using the unchanged pinned phase-2 compiler. Its native run exited 1 when direct key-session opening returned `Err`, before decoding, registry dispatch, or Messenger rows. The old RTS01 panic was not observed in this run; its repair is not proven. The independent fixture oracle passed and encrypted source bytes/metadata remained unchanged. No second fresh attempt was started. + +Automatic self-review admission rejected PR #5085 at head `6c067bee4725f1d05c5ab1311b300b9f92bda31b`: `immutable semantic restriction matched current content: credential_secret` (check `122061116596`). No bypass or repeated admission is attempted. Landing requires policy-owner resolution and ordinary current-head checks; pushing saved commits does not constitute a merge. The original hosted mail fixture and CRC failures are retained as evidence, with subsequent repairs/WIP distinguished from hosted PASS. + +Remaining DB work includes isolating the typed key-session failure, real rooms/members/history/fetch acceptance, native snapshot qualification, journal/WAL support policy, session reuse and bounded paging/row traversal. Do not claim full security cleanup, efficient database access, formal SSpec execution, all configuration families, or live account qualification. diff --git a/doc/09_report/verification/devhub_phase2_workarounds_20261027.md b/doc/09_report/verification/devhub_phase2_workarounds_20261027.md new file mode 100644 index 00000000000..e75713a87b3 --- /dev/null +++ b/doc/09_report/verification/devhub_phase2_workarounds_20261027.md @@ -0,0 +1,41 @@ +# DevHub phase-2 application repairs and tracked workarounds + +User direction: keep the existing phase-2 compiler, use Astra guidance/review and parallel work, land independent fixes first, and keep Messenger database access an active required feature. Sol and Terra supplied the database pre-research. This report does not claim complete DevHub or SSpec qualification. + +## Producer and actual passing evidence + +The existing producer is `C:/dev/simple-disk-headroom-warm-bootstrap-storage-20261025/output/stage3/simple.exe`, SHA256 `13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b`. Despite its directory name it is phase 2. It was not rebuilt. Application fixtures used LLVM and the real core-C runtime, with the existing real memory-headroom object supplied explicitly because this producer's runtime roster predates it. + +| Focused production check | Actual result | Artifact SHA256 | +|---|---|---| +| RFC header case/body separation | Bash red exit1, then green exit0; three groups | See the checked-in production probe and local evidence | +| GH JSON decoded string/control round trip | Native P2 red exit1, green exit0 | `14decf3e075a994cf10dd98add7f18b45f46f9f48b7660fdf1af4155c5f512bc` | +| Windows private atomic replacement | Actual C provider red exit1, green exit0, final owner/System DACL | `e13b9778a85b6cb23184e515f52a8b9a6e2c6d14dfbd1e6a960d3b4fd9e4461d` | +| History selection, ordering, ties and invalid metadata | Native P2 compile0/run0 | `926cd8fb6d22c5ab6ca0ab929889a5070cf2f8fb1095bbbb8e7bd8bdadfc6044` | +| Flags, MinIO expiry and JSON values | Native P2 compile0/run0 | `5ac5ddf12ba7a6982c338063edc1eee3cf6e65a1811ba3e2cd295a770a4665c8` | +| Actual private auth persistence | Native P2 compile0/run0, escaped Unicode/dotted providers/injection/refusal/deletion | `720834f8a0c613e0401802df257dcb8813f0c3329bcbe2b9d509aa05d1852f02` | +| Actual editor files | Native P2 compile0/run0, Unicode/empty/unique/refusal/cleanup | `5f7e4f4da734fb08eb527187d868541f83195847e746611155529634e755284c` | +| Actual Outlook/OAuth caches and Graph guards | Native P2 compile0/run0 after decimal permission-mode workaround; seven groups | `e285b95b2f4250817eec5555aee4d67fea0f1f299d2d3d3568381ae8f9b044fd` | +| Canonical bug API append on a copy | Native P2 compile0/run0; unsigned CRC known answers; independent zlib and exact original-body preservation | `4e7a0160881d43b5bf54df41b628c5c6ddc5651cb664658494705314f1849b18` | +| Private curl HTTP transport | Native P2 compile0/run0; six exact requests, form/UTF-8, redirect refusal, empty GET, size refusal, measured 30.054s timeout, staging cleanup | `4b3dd930128c9cf6006c0ded17a535a3bce4af77ce13a4c27592b2998b95e052` | +| Signed binary MinIO upload | Native P2 compile0/run0; independent exact octets, payload hashes, SigV4, dot-segment path, zero-byte file and missing-file request count | `3ec2fb24ebead331d4e50e8f68fce54cbd7ba4c49c1e0600b9126499dc8c3247` | + +The registry copy retained all 5265 original rows across six tables, including 20 opaque overwide legacy rows. Removing only the 32 newly inserted row lines reproduced the original payload byte for byte. Duplicate-ID, corrupt-CRC and pending-WAL cases each exited1 without changing input. After syncing release commit `16b7917a2349bb103035326d4883152a2c14177d`, the qualified API registered eight open records in the actual registry. Independent audit preserved all 5267 existing rows and all 20 opaque rows exactly; 32 new rows were inserted. The final unsigned CRC is 3989747835 and the registry SHA256 is `de927717baac960d519afd5c2677a5cbbffc575cc4ff5e0f8714eeca793c2846`. No registry cell or checksum was manually edited. See `devhub_workaround_registration_20261027.md`. + +## Coded behavior + +Production consumers use real configured file providers, checked failure paths and private atomic secret files. OAuth constructs and sends a form request, uses decoded JSON token values, and delegates Windows transport to the private-file curl owner. Both Graph adapters validate pagination URL, cycle, page limit and response shape. Daily-debug captures its watermark before fetching, aborts on errors or missing source configuration, and avoids cache writes in dry-run mode. Editor and pager no longer share predictable temporary files. MinIO preserves binary bodies, uses strict expiry validation and canonical JSON, and returns failure for unhealthy JSON output. Missing provider facade exports are supplied without globally changing selected GC/concurrency family. + +The Windows atomic file provider applies a protected owner/System DACL before any secret bytes are written. The missing Rust mode twin and collision cleanup ownership are coded; Rust runtime verification remains unqualified after the bounded test attempts, including a corrected test import error. Do not describe the C result as Rust parity evidence. + +The decimal masks preserve exact permissions: 384=0600 and 448=0700. An earlier native cache run wrote read-only target and staging files with the octal source form; the staging file contained the correct new value. Replacing the mode carrier with its decimal equivalent made the unchanged regression pass. The exact compiler-lowering cause remains open. + +## Qualification limits and remaining DB work + +The bounded native-process bridge and curl HTTP/binary-upload loopback checks passed against the unchanged phase-2 producer. Independent servers verified exact requests and bytes rather than trusting only process output. Earlier curl attempts returned the observed prelaunch tuple (empty stdout, missing command, exit-1) without a request. The compatibility path retains the normal process API first and permits one bounded native fallback for only that signature. It does not retry arbitrary transport or HTTP errors. + +Formal modern SSpec scenarios are authored but unexecuted, and spipe-docgen has not reported zero stubs. Existing pinned-producer matcher, nested callback and environment tuple failures remain tooling/product compatibility defects, not external infrastructure. The runtime-family matrix and live Microsoft/provider account exchange are not claimed qualified. + +Messenger database access remains required and unresolved. Its old production-adapter RTS01 panic is not fixed by the independent history-selector result. The selector bounds retained rows, not whole-database I/O. Follow `doc/05_design/devhub_secured_sqlite_phase2_solution_20261027.md` for the Sol/Terra research and Astra solution. Do not rerun the capped unchanged fixture or silently reset its cycle count. + +All 30 new workaround markers were applied after canonical API registration and audited to immediately precede code and reference registered records. The final exact commit receives Astra review. The original source-fix commit is `16296fd91449d482797d4b6f42c810cdea716157`; base sync commit `842d92702049268524a39e4d11d937e4106da6d4` retained those changes. Upstream runtime overlaps only added exclusive no-follow copy/link providers; they did not change the qualified atomic-write or process tuple contracts. Comment-only tagging did not change the previously qualified implementations. Never hand-edit or regenerate the derived `.simple/workarounds.sdn` index or truncate legacy bug rows. diff --git a/doc/09_report/verification/devhub_secured_db_tracking_20261027.md b/doc/09_report/verification/devhub_secured_db_tracking_20261027.md new file mode 100644 index 00000000000..5eadfa129ea --- /dev/null +++ b/doc/09_report/verification/devhub_secured_db_tracking_20261027.md @@ -0,0 +1,24 @@ +# Secured Messenger DB blocker registration + +The secured DB main feature remains blocked: fresh ledger1 compiled but returned Err at direct key-session open before decode. Its memory cryptography and SQLite oracle passed; those results do not establish working Messenger DB access. No further feature experiments were run. + +Registered OPEN P1 bug `devhub_secured_db_key_session_open_failure_2026_10_27` and blocked TODO `377`. The TODO requires key-session repair followed by secure SQLite snapshot and efficient bounded paging qualification. Both link the bug report and preserve the unresolved status. + +One metadata-only native utility was compiled with the existing Phase2 producer, without rebuilding the compiler. Astra reviewed the narrow TODO append API with no P0/P1 findings for these records. The API holds the database lock, refuses a nonempty WAL, checks an existing CRC when present, serializes only the new row through SdnTable and publishes atomically. Existing TODO data was unsealed; publication adds a valid CRC while preserving its entire old body. The two database writes are separately atomic, not a cross-file transaction. + +The utility compiled and ran exit0 against isolated copies first. An independent Python zlib audit then required exact original body bytes after removing only the new-ID rows and checked every TODO field. Actual input bytes were rechecked against those backups before one actual API run, which exited0. Actual output matched the qualified copies. + +| Database | Preserved old rows | Added rows | Unsigned CRC32 | +| --- | ---: | ---: | ---: | +| Bug | 5299 | 4 | 367800804 | +| TODO | 367 | 1 | 2283810727 | + +Bug SHA256: `dc58fdeafbb26b072811d35448bd79caa2d7c0d576bbac6aaa7baca69b4c9443`. + +TODO SHA256: `2f8d439c0755304927b07b2d9fe6728baec4e2fec6596263281e92e6f06a71c1`. + +Producer SHA256: `13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b`. + +Metadata artifact SHA256: `22030d9e03036af2cd2320caa7a42660e41519ffa0d2de633205519a9518c24e`. + +Local evidence: `build/devhub-completion/secured-db-tracking-evidence.json`, `secured-db-tracking-build.log`, `secured-db-tracking-copy-run.log`, `secured-db-tracking-actual-run.log`, before-image backups and `verify_secured_tracking.py`. This is metadata-registration evidence, not a full SSpec or Messenger feature PASS. No hand-edited registry rows, checksum repairs, generic full saves or derived-index changes were used. diff --git a/doc/09_report/verification/devhub_workaround_registration_20261027.md b/doc/09_report/verification/devhub_workaround_registration_20261027.md new file mode 100644 index 00000000000..b4a8613666c --- /dev/null +++ b/doc/09_report/verification/devhub_workaround_registration_20261027.md @@ -0,0 +1,42 @@ +# DevHub workaround registration and preservation evidence + +The audited DevHub/P2 compatibility workarounds are linked to canonical open bugs. The main Messenger database panic is not claimed fixed by this tracking work. + +## Canonical registration + +The existing BugDatabase API could not safely regenerate the complete registry: 20 legacy rows have field counts outside their table schema. A narrow append-only domain API now serializes only new records and keeps the entire old payload opaque. It validates the existing CRC, rejects pending WAL and duplicate IDs, holds the existing file lock across read/publication, and uses the real atomic file publisher. No database cells or CRCs were hand edited. The derived `.simple/workarounds.sdn` index was not manually edited. + +After a qualified scratch copy, the API registered eight open underlying bugs on fresh release base `842d92702049268524a39e4d11d937e4106da6d4`. Strict independent inspection confirmed: + +- All 5,267 original rows in six tables, including all 20 opaque legacy rows, remain intact. +- Removing only the 32 newly added row lines yields the exact original payload bytes, including comments and table metadata. +- Every new bug row, description, removal condition and investigation value matches the intended API input. +- The actual new unsigned CRC is 3989747835 and exactly matches Python zlib over the actual stored body. +- Duplicate-ID, invalid-CRC and pending-WAL controls each exited 1 without changing the input. Locks were released and WAL bytes retained. + +The actual before-file SHA256 was `c617ac5ea12dfc2887fe2fb6e4e6433daf459ac8a4c74ef27b3c88e11e85fae4`; after-file SHA256 is `de927717baac960d519afd5c2677a5cbbffc575cc4ff5e0f8714eeca793c2846`. + +## Producer and actual checks + +The existing phase-2 producer remained unchanged: SHA256 `13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b`. No compiler was rebuilt. The qualified registration executable is SHA256 `4e7a0160881d43b5bf54df41b628c5c6ddc5651cb664658494705314f1849b18`. + +The fourth bounded feature cycle compiled and executed successfully. It checked the independent CRC32 known answer 3421780262 for `123456789`, empty input, and chunked updates through high-bit-set register values before registry mutation. An earlier scratch API run internally accepted a negative CRC differing from zlib by 2^32; the independent unsigned oracle rejected it. Public i64 CRC outputs are now explicitly confined to their low 32 bits. The exact compiler widening/complement cause remains unisolated; the external oracle was not relaxed. + +The first compile used the wrong independently copied BugDatabase owner and was corrected to the canonical sync writer with an explicit separate async read-snapshot type. The second compile encountered the already tracked native array.take issue in the sync query dependency; its established ordered-prefix source workaround is now mirrored without changing query limits. These are recorded failures, not passing executions. + +## Source markers and limitations + +Thirty new `@workaround` sites across sixteen files reference the eight actual canonical bug IDs. The sync query prefix loop also uses the already registered `native_array_take_method_missing_2026_10_21` marker. Markers immediately precede affected code; ordinary permanent fixes such as proper Windows DACL enforcement and normal explicit typing are not mislabeled as temporary workarounds. + +The marker audit is static metadata validation, not a full native index or SSpec execution claim. Comment-only tags were inserted after native qualification. Formal append and CRC SSpec regressions are authored but unexecuted on the currently blocked standard SSpec path. All underlying workaround bugs remain open with explicit removal conditions; source-only review does not retire them. + +## Registered underlying bugs + +- `phase2_octal_permission_mode_lowering_2026_10_27`: Pinned Phase 2 lowers octal permission modes incorrectly. +- `phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27`: Pinned Phase 2 byte carrier is rejected by accelerated SHA256 FFI. +- `windows_core_http_client_unavailable_2026_10_27`: Windows core runtime native HTTP client provider is unavailable. +- `phase2_result_tuple_try_binding_metadata_2026_10_27`: Pinned Phase 2 loses integer and byte tuple metadata across Result propagation. +- `phase2_bounded_process_text_abi_2026_10_27`: Pinned Phase 2 bounded process call loses command text ABI. +- `phase2_module_static_array_concat_initialization_2026_10_27`: Pinned Phase 2 cannot initialize DevHub module static arrays by concatenation. +- `bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27`: Bug database full save truncates opaque legacy row payloads. +- `phase2_crc32_i64_width_spill_2026_10_27`: Pinned Phase 2 CRC32 i64 boundary retains unexpected high bits. diff --git a/scripts/check/check-windows-private-atomic-write.ps1 b/scripts/check/check-windows-private-atomic-write.ps1 new file mode 100644 index 00000000000..bad32dd4998 --- /dev/null +++ b/scripts/check/check-windows-private-atomic-write.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Compiler, + [Parameter(Mandatory=$true)][string]$OutputDirectory) + +$ErrorActionPreference = 'Stop' +if (-not (Test-Path -LiteralPath $Compiler) -or -not $env:INCLUDE -or -not $env:LIB) { + throw 'pinned clang-cl and initialized MSVC SDK environment required' +} +$repo = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..')) +$output = [IO.Path]::GetFullPath($OutputDirectory) +if (Test-Path -LiteralPath $output) { throw 'qualification output must be new' } +New-Item -ItemType Directory -Path $output | Out-Null +$source = Join-Path $repo 'test/01_unit/runtime/windows_private_atomic_write_test.c' +$object = Join-Path $output 'windows-private-atomic.obj' +$binary = Join-Path $output 'windows-private-atomic.exe' +$log = Join-Path $output 'build.log' +$previousPreference = $ErrorActionPreference +try { + $ErrorActionPreference = 'Continue' + & $Compiler '/nologo' '/O1' '/std:c11' '/D_CRT_SECURE_NO_WARNINGS' "/I$repo" $source "/Fo$object" "/Fe$binary" '/link' 'advapi32.lib' *> $log + $compileExit = $LASTEXITCODE +} finally { + $ErrorActionPreference = $previousPreference +} +if ($compileExit -ne 0) { throw "native provider build failed; inspect $log" } +& $binary +if ($LASTEXITCODE -ne 0) { throw 'actual Windows private atomic provider regression failed' } +Get-FileHash -LiteralPath $binary -Algorithm SHA256 diff --git a/scripts/ci/prepare_mail_config_helper.py b/scripts/ci/prepare_mail_config_helper.py new file mode 100644 index 00000000000..23d5e49297b --- /dev/null +++ b/scripts/ci/prepare_mail_config_helper.py @@ -0,0 +1,134 @@ +"""Build the real mail SDN bridge with a pinned, already-built Phase-2 producer. + +MAIL_CI_PHASE2_PRODUCERS_JSON maps host triples to immutable producer bundles. +No compiler is rebuilt and no seed/interpreter/parser substitute is admitted. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shutil +import subprocess +import sys +import tarfile +import urllib.request +import zipfile + + +def refuse(message: str) -> None: + raise SystemExit("MAIL_CI_PHASE2_UNAVAILABLE: " + message) + + +def sha256_file(path: Path) -> str: + with path.open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def contained(root: Path, relative: str) -> Path: + target = (root / relative).resolve() + if not target.is_relative_to(root.resolve()): + refuse("producer bundle path escapes its staging directory") + return target + + +def provision(entry: dict, destination: Path) -> Path: + if entry.get("phase") != 2: + refuse("the pinned producer must declare phase 2") + digest = entry.get("sha256", "") + if not re.fullmatch(r"[0-9a-f]{64}", digest): + refuse("a pinned producer bundle SHA-256 is required") + url = entry.get("url", "") + if not isinstance(url, str) or not url.startswith("https://"): + refuse("the existing producer bundle requires an HTTPS URL") + archive = destination / "producer.download" + request = urllib.request.Request(url, headers={"User-Agent": "simple-mail-ci"}) + with urllib.request.urlopen(request, timeout=120) as response, archive.open("wb") as output: + shutil.copyfileobj(response, output) + if sha256_file(archive) != digest: + refuse("the downloaded producer bundle checksum does not match") + bundle = destination / "producer" + bundle.mkdir() + kind = entry.get("format", "file") + executable = entry.get("compiler", "simple.exe" if os.name == "nt" else "simple") + compiler = contained(bundle, executable) + if kind == "file": + compiler.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(archive, compiler) + elif kind == "tar.gz": + with tarfile.open(archive, "r:gz") as source: + source.extractall(bundle, filter="data") + elif kind == "zip": + with zipfile.ZipFile(archive) as source: + for member in source.infolist(): + contained(bundle, member.filename) + if (member.external_attr >> 16) & 0o170000 == 0o120000: + refuse("producer ZIP contains a symbolic link") + source.extractall(bundle) + else: + refuse("producer format must be file, tar.gz, or zip") + if not compiler.is_file(): + refuse("the pinned bundle does not contain its declared compiler") + compiler.chmod(compiler.stat().st_mode | 0o700) + asset_root = contained(bundle, entry.get("asset_root", ".")) + os.environ["SIMPLE_ASSET_ROOT"] = str(asset_root) + os.environ["PATH"] = os.pathsep.join([str(asset_root / "bin"), str(asset_root / "lib"), os.environ["PATH"]]) + return compiler + + +def main() -> None: + machine = platform.machine().lower() + machine = {"amd64": "x86_64", "aarch64": "arm64"}.get(machine, machine) + system = {"Windows": "windows", "Darwin": "darwin", "Linux": "linux"}.get(platform.system()) + host = f"{system}-{machine}" + try: + profiles = json.loads(os.environ.get("MAIL_CI_PHASE2_PRODUCERS_JSON", "{}")) + except json.JSONDecodeError: + refuse("MAIL_CI_PHASE2_PRODUCERS_JSON is not valid JSON") + entry = profiles.get(host) if isinstance(profiles, dict) else None + if not isinstance(entry, dict): + refuse(f"no existing producer is configured for {host}; set MAIL_CI_PHASE2_PRODUCERS_JSON with url, sha256, phase=2, format and compiler") + checkout = Path.cwd().resolve() + destination = Path(os.environ.get("RUNNER_TEMP", str(checkout / "build"))) / "mail-config-helper" + destination.mkdir(parents=True, exist_ok=False) + compiler = provision(entry, destination) + producer_digest = sha256_file(compiler) + helper = destination / ("simple-mail-credentials.exe" if os.name == "nt" else "simple-mail-credentials") + environment = dict(os.environ, SIMPLE_NO_STUB_FALLBACK="1", SIMPLE_NO_BOOTSTRAP_DELEGATE="1", SIMPLE_NATIVE_BUILD_THREADS="2") + arguments = [str(compiler), "native-build", "src/app/mail_credentials/main.spl"] + roots = ["src/app", "src/lib", "src/os", "src/compiler"] + if os.name == "nt": + roots.insert(0, "src/providers/windows_native/lib") + for root in roots: + arguments.extend(["--source", root]) + arguments.extend(["--backend=llvm", "-o", str(helper)]) + with (destination / "helper-build.log").open("wb") as log: + result = subprocess.run(arguments, cwd=checkout, env=environment, stdout=log, stderr=subprocess.STDOUT, timeout=1200) + if result.returncode != 0 or not helper.is_file(): + refuse(f"the existing producer could not build the real helper; see {destination / 'helper-build.log'}") + # Exercise the compiled production parser before exporting it to fixtures. + document = destination / "probe.sdn" + document.write_text('default_account: "ci"\naccounts:\n ci:\n protocol: pop3\n provider: other\n email: "ci@example.test"\n pop3_server: "127.0.0.1"\n tls: implicit\n', encoding="utf-8") + parsed = subprocess.run([str(helper), "config-json", str(document)], capture_output=True, text=True, timeout=20, env=environment) + try: + actual = json.loads(parsed.stdout) + except json.JSONDecodeError: + refuse("the compiled helper returned invalid JSON for the SDN control") + if parsed.returncode != 0 or actual.get("accounts", {}).get("ci", {}).get("protocol") != "pop3": + refuse("the compiled helper did not pass the actual SDN parser control") + document.write_text('accounts: [\n', encoding="utf-8") + rejected = subprocess.run([str(helper), "config-json", str(document)], capture_output=True, text=True, timeout=20, env=environment) + if rejected.returncode == 0 or rejected.stdout.strip(): + refuse("the compiled helper accepted malformed SDN") + helper_digest = sha256_file(helper) + (destination / "evidence.json").write_text(json.dumps({"platform": host, "producer_sha256": producer_digest, "helper_sha256": helper_digest, "compiler_rebuilt": False, "actual_parser_controls": 2}, indent=2) + "\n") + with Path(os.environ["GITHUB_ENV"]).open("a", encoding="utf-8") as output: + output.write(f"MAIL_CONFIG_BIN={helper.as_posix()}\n") + print(f"Prepared real mail SDN helper for {host}; producer={producer_digest}; helper={helper_digest}") + + +if __name__ == "__main__": + main() diff --git a/src/app/devhub/adapter_minio.spl b/src/app/devhub/adapter_minio.spl index 6d6f30ada3b..1079ab738f1 100644 --- a/src/app/devhub/adapter_minio.spl +++ b/src/app/devhub/adapter_minio.spl @@ -6,8 +6,13 @@ use std.common.crypto.types.{rotr32} use std.env.{env_get} -use std.file_system.file_ops.{file_read_text, file_exists} -use std.io.http_client_core.{rt_http_request, rt_http_download} +use std.io.file_ops.{file_exists, file_read_regular_no_follow_bounded_bytes} +use std.io_runtime.{read_file_text_result} +use std.io.http_client_core.{rt_http_request, rt_http_download, http_request_v2_raw} +use std.common.crypto.sha256.{sha256_bytes} +use std.common.http_curl.{curl_request_bytes} +use std.io_runtime.{host_os} +use std.common.text_bytes.{bytes_to_text} use std.aws_sigv4.{ SIGV4_EMPTY_SHA256, SIGV4_UNSIGNED_PAYLOAD, SigV4Header, SigV4QueryParam, @@ -85,9 +90,9 @@ fn load_minio_config() -> MinioConfig: val path = auth_read_path() if not file_exists(path): return MinioConfig.default() - val content = file_read_text(path) - if content == nil: - return MinioConfig.default() + val content: text = match read_file_text_result(path): + case Ok(value): value + case Err(_): return MinioConfig.default() # NOTE: tls_skip_verify enforcement lives at the command boundary # (cmd_minio._require_transport), not here — this loader stays a pure # parser so config-parsing specs can assert on it directly. Memory note: @@ -648,12 +653,9 @@ fn minio_delete_object(cfg: MinioConfig, bucket: text, key: text) -> (bool, text # --------------------------------------------------------------------------- # PutObject # -# IMPORTANT LIMITATION (recorded per issue #10 follow-up): -# `rt_http_request` takes `body: text`, which is UTF-8. Binary PUTs (e.g. FW -# images) cannot round-trip through `text` losslessly. We therefore expose -# `minio_put_text` for text/JSON/XML bodies (≤64 MiB) and reject binary PUTs -# with a clear error. Multipart upload + binary streaming PUT require a new -# `rt_http_request_bytes`/`rt_http_upload_raw` SFFI — see TODO below. +# Legacy text requests remain available. Binary uploads use the already-owned +# v2 HTTP byte-array API, preserving NUL and non-UTF-8 octets without conversion. +# Both single-request APIs are limited to 64 MiB; multipart is a separate feature. # --------------------------------------------------------------------------- fn minio_put_text(cfg: MinioConfig, bucket: text, key: text, body: text, content_type: text) -> (bool, text, text): @@ -799,18 +801,90 @@ fn minio_delete_bucket(cfg: MinioConfig, bucket: text) -> (bool, text): (true, "") +# @workaround bug=phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27 reason=Copy indexed octets into the admitted i64 SHA256 carrier until the deployed P2 byte accelerator ABI is qualified +fn _minio_binary_payload_hash(body: [u8]) -> text: + # Keep the public byte contract. Copy each octet into the established + # typed SHA-256 engine carrier instead of crossing the old [u8] digest FFI. + var words: [i64] = [] + var index = 0 + while index < body.len(): + words.push(body[index] as i64) + index = index + 1 + val digest: [i64] = sha256_bytes(words) + val digits = "0123456789abcdef" + var encoded = "" + index = 0 + while index < digest.len(): + val octet: i64 = digest[index] + val high = octet / 16 + val low = octet % 16 + encoded = encoded + digits[high:high + 1] + digits[low:low + 1] + index = index + 1 + encoded + +fn minio_put_bytes(cfg: MinioConfig, bucket: text, key: text, body: [u8], content_type: text) -> (bool, text, text): + """Lossless bounded binary PUT through native HTTP or the real curl transport.""" + if body.len() > MINIO_MULTIPART_THRESHOLD: + return (false, "", "minio_put_bytes: body exceeds {MINIO_MULTIPART_THRESHOLD} bytes; multipart upload required") + val canonical_uri = _build_object_path(bucket, key) + val amz_dt = _amz_datetime_now() + val host = _host_from_url(cfg.url) + val payload_hash = _minio_binary_payload_hash(body) + val ct = if content_type == "": "application/octet-stream" else: content_type + val sig_headers: [SigV4Header] = [ + SigV4Header(name: "content-length", value: "{body.len()}"), + SigV4Header(name: "content-type", value: ct), + SigV4Header(name: "host", value: host), + SigV4Header(name: "x-amz-content-sha256", value: payload_hash), + SigV4Header(name: "x-amz-date", value: amz_dt) + ] + val auth = sigv4_authorization_header("PUT", canonical_uri, "", sig_headers, + payload_hash, cfg.access_key, cfg.secret_key, cfg.region, MINIO_SERVICE, amz_dt) + val headers: [text] = ["Host: {host}", "Content-Type: {ct}", + "Content-Length: {body.len()}", "x-amz-content-sha256: {payload_hash}", + "x-amz-date: {amz_dt}", "Authorization: {auth}"] + var attempt = 0 + var status = 0 + var response_body = "" + var transport_error = "" + while attempt < ITF_RETRY_MAX_ATTEMPTS: + # @workaround bug=windows_core_http_client_unavailable_2026_10_27 reason=Use the bounded private-file curl transport while the selected Windows core HTTP provider is unavailable + if host_os() == "windows": + # The core Windows runtime explicitly has no native HTTP v2 + # provider. The real bounded compatibility transport keeps binary + # octets and Authorization out of shell/argv interpolation. + val response: (i64, text, text) = curl_request_bytes( + "PUT", "{cfg.url}{canonical_uri}", headers, body) + status = response.0 + response_body = response.1 + transport_error = response.2 + else: + val response: (i64, text, text, [u8], text) = http_request_v2_raw( + "PUT", "{cfg.url}{canonical_uri}", headers, body, 30000) + status = response.0 + val response_bytes: [u8] = response.3 + response_body = bytes_to_text(response_bytes) + transport_error = response.4 + attempt = attempt + 1 + if not itf_should_retry(status, transport_error) or attempt >= ITF_RETRY_MAX_ATTEMPTS: + break + itf_retry_wait(attempt - 1, "") + val mapped = map_s3_error(status, response_body, transport_error) + if mapped.kind != "ok": + return (false, "", mapped.message) + (true, "", response_body) + fn minio_put_object(cfg: MinioConfig, bucket: text, key: text, local_path: text) -> (bool, text, text): - """PUT a local file as an object. + """PUT raw file bytes without interpreting or re-encoding them as UTF-8. - BLOCKER (issue #10 follow-up): the existing `rt_http_request`/`rt_http_upload` - SFFI does not support raw-bytes PUT bodies — `rt_http_request` body is `text` - (UTF-8) and `rt_http_upload` is multipart/form-data. Until a `rt_http_put_file` - or `rt_http_request_bytes` SFFI lands, this returns a clear error rather than - silently corrupting binary uploads. Small text files can use `minio_put_text`. + The admitted read refuses symlinks, nonregular files and bodies over the + current single-request limit. Empty files are successful zero-byte bodies. """ - if not file_exists(local_path): - return (false, "", "local file not found: {local_path}") - (false, "", "minio_put_object: binary PUT not yet implemented — needs raw-bytes HTTP SFFI (issue #10 follow-up). Workaround: `itf minio presign-put {bucket} {key}` then PUT the file's raw bytes to that URL with curl/any HTTP client. Use minio_put_text for ≤64 MiB text bodies instead.") + val body: [u8] = match file_read_regular_no_follow_bounded_bytes(local_path, MINIO_MULTIPART_THRESHOLD): + case Ok(value): value + case Err(reason): return (false, "", reason) + minio_put_bytes(cfg, bucket, key, body, "application/octet-stream") + # --------------------------------------------------------------------------- diff --git a/src/app/devhub/adapter_outlook.spl b/src/app/devhub/adapter_outlook.spl index 0dfbc1f04fd..65beba5a343 100644 --- a/src/app/devhub/adapter_outlook.spl +++ b/src/app/devhub/adapter_outlook.spl @@ -37,7 +37,8 @@ use std.http_client.types.{ } use std.sffi.system.{sleep_secs} use std.oauth2.{oauth2_get_token, oauth2_parse_retry_after} -use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_field, json_value_has, json_value_len, json_value_at, json_value_serialize} +use app.devhub.outlook_pagination.{outlook_page_error, outlook_page_response_error} +use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_field, json_value_has, json_value_len, json_value_at, json_value_serialize, json_value_as_text} use std.common.text_advanced.{escape_json} @@ -197,12 +198,7 @@ fn _outlook_patch(client: OutlookClient, url: text, payload: text) -> (i64, text # ============================================================================ fn _ojson_str(obj: JsonValue, key: text) -> text: - if not json_value_has(obj, key): - return "" - val s = json_value_serialize(json_value_field(obj, key)) - if s.starts_with("\"") and s.ends_with("\"") and s.len() >= 2: - return s[1:s.len() - 1] - s + json_value_as_text(json_value_field(obj, key)) ?? "" fn _ojson_int(obj: JsonValue, key: text) -> i64: @@ -270,7 +266,12 @@ fn outlook_list_folders(client: OutlookClient) -> (bool, [OutlookFolder], text): var folders: [OutlookFolder] = [] var first_body = "" + var visited: [text] = [] while url != "": + val page_error = outlook_page_error(client.base_url, url, visited) + if page_error != "": + return (false, folders, page_error) + visited.push(url) val (status, body) = _outlook_get(client, url) if first_body == "": first_body = body @@ -280,6 +281,9 @@ fn outlook_list_folders(client: OutlookClient) -> (bool, [OutlookFolder], text): if parsed_opt == nil: return (false, folders, "invalid JSON response") val parsed = parsed_opt.unwrap() + val response_error = outlook_page_response_error(parsed) + if response_error != "": + return (false, folders, response_error) val arr = json_value_field(parsed, "value") if json_value_has(parsed, "value"): val n = json_value_len(arr) @@ -288,12 +292,7 @@ fn outlook_list_folders(client: OutlookClient) -> (bool, [OutlookFolder], text): folders = folders + [_parse_folder(json_value_at(arr, i))] i = i + 1 # @odata.nextLink is an absolute URL — use as-is. - if json_value_has(parsed, "@odata.nextLink"): - url = json_value_serialize(json_value_field(parsed, "@odata.nextLink")) - if url.starts_with("\"") and url.ends_with("\"") and url.len() >= 2: - url = url[1:url.len() - 1] - else: - url = "" + url = _ojson_str(parsed, "@odata.nextLink") (true, folders, first_body) @@ -318,7 +317,12 @@ fn outlook_list_messages(client: OutlookClient, folder_id: text, filter: text, t var messages: [OutlookMessage] = [] var first_body = "" + var visited: [text] = [] while url != "": + val page_error = outlook_page_error(client.base_url, url, visited) + if page_error != "": + return (false, messages, page_error) + visited.push(url) val (status, body) = _outlook_get(client, url) if first_body == "": first_body = body @@ -328,6 +332,9 @@ fn outlook_list_messages(client: OutlookClient, folder_id: text, filter: text, t if parsed_opt == nil: return (false, messages, "invalid JSON response") val parsed = parsed_opt.unwrap() + val response_error = outlook_page_response_error(parsed) + if response_error != "": + return (false, messages, response_error) val arr = json_value_field(parsed, "value") if json_value_has(parsed, "value"): val n = json_value_len(arr) @@ -335,12 +342,7 @@ fn outlook_list_messages(client: OutlookClient, folder_id: text, filter: text, t while i < n: messages = messages + [_parse_message(json_value_at(arr, i))] i = i + 1 - if json_value_has(parsed, "@odata.nextLink"): - url = json_value_serialize(json_value_field(parsed, "@odata.nextLink")) - if url.starts_with("\"") and url.ends_with("\"") and url.len() >= 2: - url = url[1:url.len() - 1] - else: - url = "" + url = _ojson_str(parsed, "@odata.nextLink") (true, messages, first_body) diff --git a/src/app/devhub/adapter_outlook_curl.spl b/src/app/devhub/adapter_outlook_curl.spl index 82f16798664..790b0ed5637 100644 --- a/src/app/devhub/adapter_outlook_curl.spl +++ b/src/app/devhub/adapter_outlook_curl.spl @@ -6,19 +6,22 @@ # call Microsoft Graph under /users/{shared_mailbox_upn}/... use std.io.process_ops.{process_run} -use std.file_system.file_ops.{file_read_text, file_write_text, file_exists, file_rename, file_delete} -use std.file_system.permissions.{file_set_mode} +use std.common.http_curl.{curl_request_text_with_program} +use std.common.oauth_token_cache.{oauth_token_cache_read, oauth_token_cache_write, oauth_token_cache_render, oauth_token_cache_parse} use std.env.{env_get} use std.io.time_ops.{current_time_unix} use std.http.url.{url_encode} -use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_field, json_value_has, json_value_len, json_value_at, json_value_serialize} +use app.devhub.outlook_pagination.{outlook_page_error, outlook_page_response_error} +use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_field, json_value_has, json_value_len, json_value_at, json_value_serialize, json_value_as_text} val OUTLOOK_CURL_GRAPH_BASE: text = "https://graph.microsoft.com/v1.0" val OUTLOOK_CURL_DEFAULT_TOP: i64 = 50 val OUTLOOK_CURL_DEFAULT_SCOPE: text = "https://graph.microsoft.com/.default" val OUTLOOK_CURL_DEFAULT_SELECT_MESSAGES: text = "id,subject,from,receivedDateTime,bodyPreview,hasAttachments" -val OUTLOOK_CURL_CACHE_FILE_MODE: i64 = 0o600 +# Decimal spelling preserves 0o600 on the pinned Phase-2 producer. +# @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Keep the exact owner-only mode in decimal until deployed P2 octal permission lowering is qualified +val OUTLOOK_CURL_CACHE_FILE_MODE: i64 = 384 val OUTLOOK_CURL_CACHE_SAFETY_MARGIN: i64 = 60 @@ -76,8 +79,8 @@ fn outlook_curl_get_token_with_secret(tenant_id: text, client_id: text, client_s val curl_path = "curl" val url = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" val body = "grant_type=client_credentials&client_id={_url_form_encode(client_id)}&client_secret={_url_form_encode(client_secret)}&scope={_url_form_encode(OUTLOOK_CURL_DEFAULT_SCOPE)}" - val argv = _build_curl_argv_token(url, body) - val (status, resp_body, stderr) = _run_curl(curl_path, argv) + val (status, resp_body, stderr) = curl_request_text_with_program(curl_path, "POST", url, + ["Content-Type: application/x-www-form-urlencoded", "Accept: application/json"], body) if status != 200: return (false, "") val parsed_opt = json_value_parse_opt(resp_body) @@ -113,7 +116,12 @@ fn outlook_curl_list_folders(client: OutlookCurlClient) -> (bool, [OutlookFolder var first_body = "" var folders: [OutlookFolder] = [] + var visited: [text] = [] while url != "": + val page_error = outlook_page_error(client.base_url, url, visited) + if page_error != "": + return (false, folders, page_error) + visited.push(url) val (status, body, _stderr) = _graph_get(client, url) if first_body == "": first_body = body @@ -123,6 +131,9 @@ fn outlook_curl_list_folders(client: OutlookCurlClient) -> (bool, [OutlookFolder if parsed_opt == nil: return (false, folders, "invalid JSON response") val parsed = parsed_opt.unwrap() + val response_error = outlook_page_response_error(parsed) + if response_error != "": + return (false, folders, response_error) val arr = json_value_field(parsed, "value") if json_value_has(parsed, "value"): val n = json_value_len(arr) @@ -140,7 +151,12 @@ fn outlook_curl_list_messages(client: OutlookCurlClient, folder_id: text, filter var first_body = "" var messages: [OutlookMessage] = [] + var visited: [text] = [] while url != "": + val page_error = outlook_page_error(client.base_url, url, visited) + if page_error != "": + return (false, messages, page_error) + visited.push(url) val (status, body, _stderr) = _graph_get(client, url) if first_body == "": first_body = body @@ -150,6 +166,9 @@ fn outlook_curl_list_messages(client: OutlookCurlClient, folder_id: text, filter if parsed_opt == nil: return (false, messages, "invalid JSON response") val parsed = parsed_opt.unwrap() + val response_error = outlook_page_response_error(parsed) + if response_error != "": + return (false, messages, response_error) val arr = json_value_field(parsed, "value") if json_value_has(parsed, "value"): val n = json_value_len(arr) @@ -182,53 +201,27 @@ fn outlook_curl_token_cache_is_fresh(expires_at: i64, now: i64) -> bool: fn outlook_curl_render_token_cache(token: text, expires_at: i64) -> text: - "# itf outlook curl token cache (auto-managed; do not edit)\n" + - "access_token: \"{_escape_sdn_value(token)}\"\n" + - "expires_at: {expires_at}\n" + oauth_token_cache_render(token, expires_at) fn outlook_curl_parse_token_cache(content: text) -> (text, i64): - if content == nil or content == "": - return ("", 0) - var token = "" - var expires_at: i64 = 0 - val lines = content.split("\n") - for line in lines: - val trimmed = line.trim() - if trimmed == "" or trimmed.starts_with("#"): - pass_dn - else if trimmed.starts_with("access_token:"): - token = _strip_sdn_value(trimmed.split(":", 2)[1]) - else if trimmed.starts_with("expires_at:"): - expires_at = _parse_i64(_strip_sdn_value(trimmed.split(":", 2)[1])) - (token, expires_at) + oauth_token_cache_parse(content) fn outlook_curl_token_cache_load(path: text) -> (text, i64): - if path == "" or not file_exists(path): - return ("", 0) - val content = file_read_text(path) - if content == nil: + val content = oauth_token_cache_read(path) + if content == "": return ("", 0) outlook_curl_parse_token_cache(content) fn outlook_curl_token_cache_save(path: text, token: text, expires_at: i64) -> bool: - if path == "": - return false - val tmp_path = "{path}.tmp" val payload = outlook_curl_render_token_cache(token, expires_at) - if not file_write_text(tmp_path, payload): - return false - if not file_set_mode(tmp_path, OUTLOOK_CURL_CACHE_FILE_MODE): - file_delete(tmp_path) - return false - if not file_rename(tmp_path, path): - file_delete(tmp_path) - return false - true + oauth_token_cache_write(path, payload) +# Legacy argv-rendering helpers retained for compatibility tests. Production +# requests use the bounded private-file transport and never expose secrets here. fn _build_curl_argv_token(url: text, body: text) -> [text]: [ "-s", "-S", @@ -313,7 +306,8 @@ fn _status_message(code: i64, body: text) -> text: fn _graph_get(client: OutlookCurlClient, url: text) -> (i64, text, text): - _run_curl(client.curl_path, _build_curl_argv_get(client.access_token, url)) + curl_request_text_with_program(client.curl_path, "GET", url, + ["Authorization: Bearer {client.access_token}", "Accept: application/json"], "") fn _outlook_user_path(client: OutlookCurlClient, suffix: text) -> text: @@ -351,11 +345,11 @@ fn _parse_message(obj: JsonValue) -> OutlookMessage: fn _ostr(obj: JsonValue, key: text) -> text: - _json_field_text(obj, key) + json_value_as_text(json_value_field(obj, key)) ?? "" fn _oint(obj: JsonValue, key: text) -> i64: - _parse_i64(_json_field_text(obj, key)) + _parse_i64(json_value_serialize(json_value_field(obj, key))) fn _obool(obj: JsonValue, key: text) -> bool: @@ -365,12 +359,7 @@ fn _obool(obj: JsonValue, key: text) -> bool: fn _json_field_text(obj: JsonValue, key: text) -> text: - if not json_value_has(obj, key): - return "" - val s = json_value_serialize(json_value_field(obj, key)) - if s.starts_with("\"") and s.ends_with("\"") and s.len() >= 2: - return s[1:s.len() - 1] - s + json_value_as_text(json_value_field(obj, key)) ?? "" fn _parse_i64(s: text) -> i64: diff --git a/src/app/devhub/auth_token_store.spl b/src/app/devhub/auth_token_store.spl index 7644185aa4f..6cfea81898e 100644 --- a/src/app/devhub/auth_token_store.spl +++ b/src/app/devhub/auth_token_store.spl @@ -3,17 +3,30 @@ # This module owns parsing and rewriting plaintext token sections. Token # resolution policy (environment, command, then file) remains in config.spl. -use std.sosix.host_facade.{sosix_file_read, sosix_file_exists, sosix_file_atomic_write} +use std.io.file_ops.{file_exists} +use std.io_runtime.{read_file_text_result} +use std.sffi.fs.{file_atomic_write_mode} +use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_as_text, json_value_serialize} + +fn auth_provider_is_valid(provider: text) -> bool: + if provider == "": return false + var i = 0 + while i < provider.len(): + val ch = provider.char_at(i) + if not ((ch >= "a" and ch <= "z") or (ch >= "A" and ch <= "Z") or ch == "_" or + (i > 0 and ((ch >= "0" and ch <= "9") or ch == "-" or ch == "."))): + return false + i = i + 1 + true fn load_auth_token_from(path: text, provider: text) -> text: """Load one provider token from an auth.sdn path.""" - if not sosix_file_exists(path): - return "" - - val content = sosix_file_read(path) - if content == nil: + if not auth_provider_is_valid(provider): return "" + val content: text = match read_file_text_result(path): + case Ok(value): value + case Err(_): return "" var section = "" for line in content.split("\n"): @@ -26,8 +39,10 @@ fn load_auth_token_from(path: text, provider: text) -> text: # Split at the first colon so values such as user:pass survive. val colon_idx = trimmed.find(":") val token = trimmed[colon_idx + 1:].trim() - if token.starts_with("\"") and token.ends_with("\""): - return token[1:token.len() - 1] + if token.starts_with("\""): + match json_value_parse_opt(token): + case Some(value): return json_value_as_text(value) ?? "" + case nil: return "" return token "" @@ -46,8 +61,16 @@ fn _without_auth_section(content: text, section_name: text) -> text: fn save_auth_token_to(path: text, provider: text, token: text) -> bool: """Save one provider token while preserving all other auth.sdn sections.""" - val existing = if sosix_file_exists(path): sosix_file_read(path) else: "# ITF Auth Tokens (do not share)\n\n" + if path == "" or not auth_provider_is_valid(provider): + return false + val existing: text = if file_exists(path): + match read_file_text_result(path): + case Ok(value): value + case Err(_): return false + else: + "# ITF Auth Tokens (do not share)\n\n" var content = _without_auth_section(existing, provider) if token != "": - content = content + "\n{provider}:\n token: \"{token}\"\n" - sosix_file_atomic_write(path, content) + val quoted = json_value_serialize(JsonValue.Str(token)) + content = content + "\n{provider}:\n token: {quoted}\n" + file_atomic_write_mode(path, content, 384 as i32) diff --git a/src/app/devhub/backend_resolve.spl b/src/app/devhub/backend_resolve.spl index 3378e8f3e50..3e3fce11aac 100644 --- a/src/app/devhub/backend_resolve.spl +++ b/src/app/devhub/backend_resolve.spl @@ -25,9 +25,9 @@ # Research: doc/01_research/app/tools/devhub/devhub_cli_wrapper_forwarding_2026-09-06.md use std.env.{env_get} -use std.file_system.file_ops.{file_read_text, file_exists} +use std.io.file_ops.{file_exists} use std.io.process_ops.{process_run} -use std.io_runtime.{cwd} +use std.io_runtime.{cwd, read_file_text_result} use app.devhub.config.{ItfConfig, load_config, parse_sdn_sections} use app.devhub.flags.{_extract_flag} @@ -217,21 +217,41 @@ fn parse_repo_devhub_config(content: text) -> RepoDevhubConfig: # ============================================================================ fn find_repo_root() -> text: + find_repo_root_from(cwd()) + + +fn repo_parent_directory(path: text) -> text: + """Return a canonical parent on Unix and native Windows; roots stop.""" + var normalized = path.replace("\\", "/") + while normalized.ends_with("/") and normalized.len() > 1: + if normalized.len() == 3 and normalized.char_at(1) == ":": + return "" + normalized = normalized[0:normalized.len() - 1] + if normalized.starts_with("//"): + val parts = normalized.split("/") + # Do not probe a different share or a server root while ascending UNC. + if parts.len() <= 4: + return "" + val slash = normalized.rfind("/") + if slash < 0 or normalized == "/": return "" + if slash == 0: return "/" + if slash == 2 and normalized.char_at(1) == ":": return normalized[0:3] + normalized[0:slash] + + +fn find_repo_root_from(start: text) -> text: """Walk up from the cwd looking for `.spipe/config.sdn`; "" if not found. Anchored on that file rather than `.git` because devhub's repo-scoped config IS that file — a worktree or a submodule checkout with a `.git` file but no `.spipe/` has nothing here for us to read. """ - var dir = cwd() + var dir = start.replace("\\", "/") var guard: i64 = 0 while dir != "" and guard < 64: if file_exists("{dir}/.spipe/config.sdn"): return dir - val slash = dir.rfind("/") - if slash <= 0: - return "" - dir = dir[0:slash] + dir = repo_parent_directory(dir) guard = guard + 1 "" @@ -241,10 +261,13 @@ fn load_repo_devhub_config() -> RepoDevhubConfig: val root = find_repo_root() if root == "": return RepoDevhubConfig.empty() - val content = file_read_text("{root}/.spipe/config.sdn") - if content == nil: - return RepoDevhubConfig.empty() - parse_repo_devhub_config(content) + load_repo_devhub_config_from(root) + + +fn load_repo_devhub_config_from(root: text) -> RepoDevhubConfig: + match read_file_text_result("{root}/.spipe/config.sdn"): + case Ok(content): parse_repo_devhub_config(content) + case Err(_): RepoDevhubConfig.empty() fn git_remote_origin_url() -> text: diff --git a/src/app/devhub/cmd_auth.spl b/src/app/devhub/cmd_auth.spl index c7b9f40ca12..aff2b290010 100644 --- a/src/app/devhub/cmd_auth.spl +++ b/src/app/devhub/cmd_auth.spl @@ -9,7 +9,7 @@ use app.devhub.atlassian_site.{ATLASSIAN_AUTH_BASIC, ATLASSIAN_AUTH_BEARER, atla use app.devhub.flags.{_extract_flag, _has_flag} use app.devhub.output.{print_error, print_success, print_info, print_warning, should_use_color} use app.ui.render.ansi.{ansi_green, ansi_red, ansi_bold, ansi_reset} -use std.file_system.file_ops.{file_exists, file_write_text, file_read_text} +use std.io.dir_ops.{dir_create_all} use std.io.process_ops.{process_run} @@ -83,7 +83,9 @@ fn _login_confluence(args: [text]) -> i64: return 1 # Ensure config dir exists - _ensure_config_dir() + if not _ensure_config_dir(): + print_error("failed to create authentication configuration directory") + return 1 # Save config var config = load_config_for_target(target) @@ -170,7 +172,9 @@ fn _login_jira(args: [text]) -> i64: if err != "": print_error(err) return 1 - _ensure_config_dir() + if not _ensure_config_dir(): + print_error("failed to create authentication configuration directory") + return 1 var config = load_config() val new_deployment = if deployment != "": deployment else: config.jira_deployment val new_auth = if auth != "": auth else: config.jira_auth @@ -331,11 +335,13 @@ fn _auth_logout(args: [text]) -> i64: 0 -fn _ensure_config_dir(): +fn ensure_auth_config_dir(path: text) -> bool: + dir_create_all(path) + + +fn _ensure_config_dir() -> bool: """Create config directory if it doesn't exist.""" - val dir = config_dir() - # Use mkdir -p via process_run - process_run("mkdir", ["-p", dir]) + ensure_auth_config_dir(config_dir()) fn _print_auth_help(): diff --git a/src/app/devhub/cmd_daily_debug.spl b/src/app/devhub/cmd_daily_debug.spl index 907b12f020d..f707eec2183 100644 --- a/src/app/devhub/cmd_daily_debug.spl +++ b/src/app/devhub/cmd_daily_debug.spl @@ -7,9 +7,9 @@ # adapter until the native http transport extern lands. use std.env.{env_get} -use std.file_system.file_ops.{file_read_text, file_write_text, file_exists, file_rename} +use std.io.file_ops.{file_exists, file_atomic_write} use app.io.mod.{dir_create_all} -use std.io_runtime.{time_now_unix_micros} +use std.io_runtime.{time_now_unix_micros, read_file_text_result, file_write as file_write_text} use std.common.time_utils.{timestamp_get_year, timestamp_get_month, timestamp_get_day, timestamp_get_hour, timestamp_get_minute, timestamp_get_second} use app.devhub.flags.{_has_flag} use app.devhub.output.{print_error, print_info, print_success} @@ -135,7 +135,9 @@ fn load_watermark() -> text: path = "{base}/.config/itf/spipe_daily.sdn" if not file_exists(path): return _iso_minus_24h() - val content = file_read_text(path) ?? "" + val content = match read_file_text_result(path): + case Ok(value): value + case Err(_): return _iso_minus_24h() if content == "": return _iso_minus_24h() val parsed = parse_watermark(content) @@ -144,16 +146,17 @@ fn load_watermark() -> text: parsed fn save_watermark_atomic(iso_now: text) -> bool: - """Atomic save: write to `.tmp`, then rename into place.""" + """Persist through the real provider's exclusive atomic staging path.""" val path = watermark_path() - val tmp = "{path}.tmp" - val body = render_watermark(iso_now) val home = env_get("HOME") val base = if home != "": home else: env_get("USERPROFILE") - val _created = dir_create_all("{base}/.config/devhub") - if not file_write_text(tmp, body): + if not dir_create_all("{base}/.config/devhub"): return false - file_rename(tmp, path) + save_watermark_atomic_at(path, iso_now) + + +fn save_watermark_atomic_at(path: text, iso_now: text) -> bool: + file_atomic_write(path, render_watermark(iso_now)) # --------------------------------------------------------------------------- # Internal helpers @@ -201,7 +204,11 @@ fn _digest_path() -> text: """Resolve `doc/08_tracking/debug/.md`.""" "doc/08_tracking/debug/{_today_yyyy_mm_dd()}.md" -fn _fetch_inbox(since_iso: text) -> [DailyMessage]: +fn daily_debug_inbox_filter(since_iso: text, until_iso: text) -> text: + "receivedDateTime ge {since_iso} and receivedDateTime lt {until_iso}" + + +fn _fetch_inbox(since_iso: text, until_iso: text, persist_cache: bool = true) -> Result<[DailyMessage], text>: """Fetch inbox messages received since the watermark. """ val tenant = env_get("GRAPH_TENANT_ID") @@ -209,9 +216,10 @@ fn _fetch_inbox(since_iso: text) -> [DailyMessage]: val client_secret = env_get("GRAPH_CLIENT_SECRET") val mailbox = env_get("GRAPH_MAILBOX_UPN") if tenant == "" or client_id == "" or client_secret == "" or mailbox == "": - print_info("daily-debug", "outlook skipped (set GRAPH_TENANT_ID/GRAPH_CLIENT_ID/GRAPH_CLIENT_SECRET/GRAPH_MAILBOX_UPN)") - return [] - val cache_path = _graph_cache_path() + return Err("outlook is not configured (set GRAPH_TENANT_ID/GRAPH_CLIENT_ID/GRAPH_CLIENT_SECRET/GRAPH_MAILBOX_UPN)") + # An empty cache path is the Outlook owner's explicit no-cache-I/O mode. + # A dry run may read the inbox but must not persist a refreshed token. + val cache_path = if persist_cache: _graph_cache_path() else: "" val (ok, client) = outlook_curl_client_acquire( tenant, client_id, @@ -220,19 +228,16 @@ fn _fetch_inbox(since_iso: text) -> [DailyMessage]: cache_path, ) if not ok: - print_info("daily-debug", "outlook token acquisition failed") - return [] + return Err("outlook token acquisition failed") val folder_id = _resolve_inbox_folder_id(client) if folder_id == "": - print_info("daily-debug", "Inbox folder not found") - return [] + return Err("Inbox folder not found or unavailable") - val filter = "receivedDateTime ge {since_iso}" + val filter = daily_debug_inbox_filter(since_iso, until_iso) val (success, messages, raw) = outlook_curl_list_messages(client, folder_id, filter, 50) if not success: - print_info("daily-debug", "outlook fetch failed: {raw}") - return [] + return Err("outlook fetch failed") var out: [DailyMessage] = [] for m in messages: @@ -244,7 +249,7 @@ fn _fetch_inbox(since_iso: text) -> [DailyMessage]: jira_keys: extract_jira_keys(combined), minio_urls: extract_minio_urls(combined), )) - out + Ok(out) fn _graph_cache_path() -> text: @@ -354,13 +359,20 @@ fn run_daily_debug(args: [text]) -> i64: val quiet = _has_flag(args, "--quiet") val dry_run = _has_flag(args, "--dry-run") + # Capture the upper bound before authentication or any paginated fetch. + # Arrivals during this run belong to the next closed-open interval. + val fetch_upper_bound = _iso_now() # 1. Load watermark val since = load_watermark() print_info("daily-debug", "since={since} dry_run={dry_run} quiet={quiet}") # 2. List inbox since watermark - val messages = _fetch_inbox(since) + val messages: [DailyMessage] = match _fetch_inbox(since, fetch_upper_bound, not dry_run): + case Ok(value): value + case Err(reason): + print_error(reason) + return 1 # 3+4. Already done per-message at fetch time; aggregate artifacts here. var all_urls: [text] = [] @@ -372,16 +384,15 @@ fn run_daily_debug(args: [text]) -> i64: # 5. Write daily digest (gated on dry-run) val digest = _render_digest(messages, artifacts) if not dry_run: - if not file_write_text(_digest_path(), digest): + if not dir_create_all("doc/08_tracking/debug") or not file_write_text(_digest_path(), digest): print_error("failed to write daily digest") return 1 else: print_info("dry-run", "would write {_digest_path()} ({digest.len()} bytes)") # 6. Save watermark atomically (gated on dry-run) - val now = _iso_now() if not dry_run: - if not save_watermark_atomic(now): + if not save_watermark_atomic(fetch_upper_bound): print_error("failed to save watermark") return 1 diff --git a/src/app/devhub/cmd_lifecycle.spl b/src/app/devhub/cmd_lifecycle.spl index 77bc58e701e..ad3197cc31b 100644 --- a/src/app/devhub/cmd_lifecycle.spl +++ b/src/app/devhub/cmd_lifecycle.spl @@ -8,7 +8,7 @@ use std.scv.lifecycle.store.{lifecycle_store_probe, lifecycle_store_path, lifecy use std.scv.lifecycle.codec.{lifecycle_field_valid} use std.scv.lifecycle.identity.{lifecycle_change_identity} use std.scv.lifecycle.entity_codec.{change_record} -use std.file_system.{dir_walk_files} +use std.io.dir_ops.{dir_walk} fn _lifecycle_json(value: text) -> text: value.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n").replace("\r", "\\r") @@ -144,6 +144,12 @@ fn _lifecycle_positionals(args: [text]) -> [text]: i = i + 1 out +fn lifecycle_source_files(root: text) -> [text]: + # The native walk enumerates real non-directory entries and canonicalizes + # separators. The demonstration file_system walker does not touch disk. + dir_walk(root) + + fn handle_lifecycle(args: [text]) -> i64: if args.len() == 0 or args[0] == "capabilities" or args[0] == "--help" or args[0] == "-h": print lifecycle_capabilities_json() @@ -163,7 +169,7 @@ fn handle_lifecycle(args: [text]) -> i64: print '{"output_version":"devhub/v1","status":"rejected","code":"VERSION_MANIFEST_INVALID","message":"' + _lifecycle_json(manifest.error) + '"}' return 1 var discovered = ["VERSION"] - for consumer in dir_walk_files("src"): + for consumer in lifecycle_source_files("src"): if (consumer.ends_with("/simple.sdn") and _projection_version(consumer) == manifest.semver) or (consumer.ends_with(".spl") and _payload_declares_product_version(file_read(consumer), manifest.semver)): discovered.push(consumer) val undeclared = version_undeclared_consumers(manifest, discovered) diff --git a/src/app/devhub/cmd_minio.spl b/src/app/devhub/cmd_minio.spl index b95d8441e24..edc39891500 100644 --- a/src/app/devhub/cmd_minio.spl +++ b/src/app/devhub/cmd_minio.spl @@ -1,15 +1,17 @@ # ITF CLI — `itf minio {ls,get,put,stat,presign,health}` subcommands. -use std.file_system.file_ops.{file_read_text} +use std.io.file_ops.{file_read_regular_no_follow_bounded_bytes} use std.io_runtime.{time_now_unix_micros} +use app.devhub.minio_output.{minio_stat_json, minio_health_json} +use app.devhub.minio_options.{minio_parse_expiry} use app.devhub.adapter_minio.{ MinioConfig, BucketInfo, ObjectInfo, GetObjectResult, HeadObjectResult, load_minio_config, check_minio_auth, minio_list_buckets, minio_list_objects, minio_get_object, minio_stat_object, - minio_put_text, minio_put_object, minio_presign_get, minio_presign_put, minio_health_live, - minio_download, minio_create_bucket, minio_delete_bucket, minio_delete_object + minio_put_text, minio_put_object, minio_put_bytes, minio_presign_get, minio_presign_put, minio_health_live, + minio_download, minio_create_bucket, minio_delete_bucket, minio_delete_object, MINIO_MULTIPART_THRESHOLD } -use app.devhub.flags.{_extract_flag, _has_flag, _positional_args} +use app.devhub.flags.{_extract_flag, _has_flag, _positional_args_with_booleans} use app.devhub.output.{print_error, print_success, print_info, print_dim, print_table_dynamic, format_json_output, wants_json, format_size, format_transfer_summary, set_no_pager} @@ -74,26 +76,13 @@ fn _require_auth(cfg: MinioConfig) -> bool: true -fn _parse_int(s: text, default_val: i64) -> i64: - var n: i64 = 0 - var i = 0 - var seen_digit = false - while i < s.len(): - val c = s.char_code_at(i) - if c >= 48 and c <= 57: - n = n * 10 + (c - 48) - seen_digit = true - i = i + 1 - if seen_digit: n else: default_val - - fn _minio_ls(args: [text]) -> i64: """`itf minio ls [bucket [--prefix P]] [--no-pager]`""" set_no_pager(_has_flag(args, "--no-pager")) val cfg = load_minio_config() if not _require_auth(cfg): return 1 - val pos = _positional_args(args) + val pos = _positional_args_with_booleans(args, ["--json"]) if pos.len() == 0: val response: (bool, [BucketInfo], text) = minio_list_buckets(cfg) val ok = response.0 @@ -142,7 +131,7 @@ fn _minio_get(args: [text]) -> i64: val cfg = load_minio_config() if not _require_auth(cfg): return 1 - val pos = _positional_args(args) + val pos = _positional_args_with_booleans(args, ["--json"]) if pos.len() < 2: print_error("usage: itf minio get [--out PATH] [--range R]") return 1 @@ -174,14 +163,12 @@ fn _minio_get(args: [text]) -> i64: fn _minio_put(args: [text]) -> i64: """`itf minio put --file PATH [--content-type CT]` - Routes to `minio_put_text` for text bodies (any --file content read as UTF-8 text) - up to 64 MiB. Binary bodies fall through to `minio_put_object` which returns a - clear "needs raw-bytes SFFI" error (issue #10 follow-up). + Read the actual file as bounded bytes and pass them losslessly to HTTP v2. """ val cfg = load_minio_config() if not _require_auth(cfg): return 1 - val pos = _positional_args(args) + val pos = _positional_args_with_booleans(args, ["--json"]) if pos.len() < 2: print_error("usage: itf minio put --file PATH [--content-type CT]") return 1 @@ -192,24 +179,19 @@ fn _minio_put(args: [text]) -> i64: if file_path == "": print_error("--file PATH is required") return 1 - # Try text path first — works for JSON/XML/plain bodies up to 64 MiB. - if val content = file_read_text(file_path): - val started = time_now_unix_micros() - val (ok, _etag, raw) = minio_put_text(cfg, bucket, key, content, ct) - val elapsed_ms = (time_now_unix_micros() - started) / 1000 - if not ok: - print_error(raw) + val content: [u8] = match file_read_regular_no_follow_bounded_bytes(file_path, MINIO_MULTIPART_THRESHOLD): + case Ok(value): value + case Err(reason): + print_error(reason) return 1 - print_success("uploaded {bucket}/{key} ({content.len()} bytes, text path)") - print_dim(format_transfer_summary("uploaded", content.len(), elapsed_ms)) - return 0 - # Fallthrough — binary path (currently surfaces a clear error + workaround; - # the message names `itf minio presign-put`, the real escape hatch). - val (ok, etag, raw) = minio_put_object(cfg, bucket, key, file_path) + val started = time_now_unix_micros() + val (ok, _etag, raw) = minio_put_bytes(cfg, bucket, key, content, ct) + val elapsed_ms = (time_now_unix_micros() - started) / 1000 if not ok: print_error(raw) return 1 - print_success("uploaded {bucket}/{key} (etag {etag})") + print_success("uploaded {bucket}/{key} ({content.len()} bytes)") + print_dim(format_transfer_summary("uploaded", content.len(), elapsed_ms)) 0 @@ -221,7 +203,7 @@ fn _minio_stat(args: [text]) -> i64: val cfg = load_minio_config() if not _require_auth(cfg): return 1 - val pos = _positional_args(args) + val pos = _positional_args_with_booleans(args, ["--json"]) if pos.len() < 2: print_error("usage: itf minio stat ") return 1 @@ -232,7 +214,7 @@ fn _minio_stat(args: [text]) -> i64: print_error(raw) return 1 if wants_json(args): - print "{{\"key\":\"{key}\",\"size\":{head.content_length},\"etag\":\"{head.etag}\",\"last_modified\":\"{head.last_modified}\"}}" + print minio_stat_json(key, head.content_length, head.etag, head.last_modified) return 0 val size_text = if _has_flag(args, "--bytes"): "{head.content_length}" else: format_size(head.content_length) print_info("Size", size_text) @@ -246,7 +228,7 @@ fn _minio_mb(args: [text]) -> i64: val cfg = load_minio_config() if not _require_auth(cfg): return 1 - val pos = _positional_args(args) + val pos = _positional_args_with_booleans(args, ["--json"]) if pos.len() < 1: print_error("usage: itf minio mb ") return 1 @@ -263,7 +245,7 @@ fn _minio_rb(args: [text]) -> i64: val cfg = load_minio_config() if not _require_auth(cfg): return 1 - val pos = _positional_args(args) + val pos = _positional_args_with_booleans(args, ["--json"]) if pos.len() < 1: print_error("usage: itf minio rb ") return 1 @@ -280,7 +262,7 @@ fn _minio_rm(args: [text]) -> i64: val cfg = load_minio_config() if not _require_auth(cfg): return 1 - val pos = _positional_args(args) + val pos = _positional_args_with_booleans(args, ["--json"]) if pos.len() < 2: print_error("usage: itf minio rm ") return 1 @@ -297,11 +279,15 @@ fn _minio_presign(args: [text]) -> i64: val cfg = load_minio_config() if not _require_auth(cfg): return 1 - val pos = _positional_args(args) + val pos = _positional_args_with_booleans(args, ["--json"]) if pos.len() < 2: print_error("usage: itf minio presign [--expires SECONDS]") return 1 - val expires = _parse_int(_extract_flag(args, "--expires", "86400"), 86400) + val expires: i64 = match minio_parse_expiry(_extract_flag(args, "--expires", "86400")): + case Some(value): value + case nil: + print_error("--expires must be a decimal integer from 1 through 604800") + return 1 val (ok, url, err) = minio_presign_get(cfg, pos[0], pos[1], expires) if not ok: print_error(err) @@ -321,11 +307,15 @@ fn _minio_presign_put(args: [text]) -> i64: val cfg = load_minio_config() if not _require_auth(cfg): return 1 - val pos = _positional_args(args) + val pos = _positional_args_with_booleans(args, ["--json"]) if pos.len() < 2: print_error("usage: itf minio presign-put [--expires SECONDS]") return 1 - val expires = _parse_int(_extract_flag(args, "--expires", "86400"), 86400) + val expires: i64 = match minio_parse_expiry(_extract_flag(args, "--expires", "86400")): + case Some(value): value + case nil: + print_error("--expires must be a decimal integer from 1 through 604800") + return 1 val (ok, url, err) = minio_presign_put(cfg, pos[0], pos[1], expires) if not ok: print_error(err) @@ -347,8 +337,8 @@ fn _minio_health(args: [text]) -> i64: print_error(raw) return 1 if wants_json(args): - print "{{\"alive\":{alive},\"status\":\"{raw}\",\"latency_ms\":{latency_ms},\"endpoint\":\"{cfg.url}\"}}" - return 0 + print minio_health_json(alive, raw, latency_ms, cfg.url) + return if alive: 0 else: 1 if alive: print_success("OK — {cfg.url} ({latency_ms}ms, {raw})") return 0 @@ -365,10 +355,10 @@ fn _print_minio_help(): print "Commands:" print " ls [bucket [--prefix P]] List buckets, or objects under a bucket" print " get Fetch object body or stream to --out PATH" - print " put Upload --file PATH (text/JSON/XML; binary blocked, see presign-put)" + print " put Upload --file PATH (lossless text/binary, up to 64 MiB)" print " stat Size/etag/last-modified (via ListObjectsV2 — see NOTE below)" print " presign Generate a presigned GET URL (--expires SECONDS, max 604800)" - print " presign-put Generate a presigned PUT URL (binary-upload workaround)" + print " presign-put Generate a presigned PUT URL for an external uploader" print " mb Create a bucket" print " rb Delete an (empty) bucket" print " rm Delete an object (confirmation-free)" diff --git a/src/app/devhub/cmd_outlook.spl b/src/app/devhub/cmd_outlook.spl index 75f354faea3..ca25950648c 100644 --- a/src/app/devhub/cmd_outlook.spl +++ b/src/app/devhub/cmd_outlook.spl @@ -11,7 +11,7 @@ # This file does NOT modify auth.spl or main.spl — the orchestrator # wires those up after both adapter agents finish. -use std.file_system.file_ops.{file_exists, file_read_text} +use std.sosix.host_facade.{sosix_file_exists as file_exists, sosix_file_read as file_read_text} use app.devhub.config.{ItfConfig, load_config, config_dir, auth_read_path} use app.devhub.flags.{_extract_flag, _has_flag, _positional_args} use app.devhub.output.{print_error, print_success, print_info, print_table_header, print_table_row, format_json_output, wants_json} @@ -280,7 +280,7 @@ fn _load_outlook_section() -> OutlookSection: if not file_exists(path): return empty val content = file_read_text(path) - if content == nil: + if content == "": return empty var section = "" diff --git a/src/app/devhub/editor.spl b/src/app/devhub/editor.spl index 4892a92f577..cc56e05f7bb 100644 --- a/src/app/devhub/editor.spl +++ b/src/app/devhub/editor.spl @@ -3,7 +3,8 @@ # Fetch → tmpfile → $EDITOR → diff → upload # Supports both storage (native) and markdown (lossy) edit modes -use std.file_system.file_ops.{file_read_text, file_write_text, file_exists, file_delete} +use std.io_runtime.{read_file_text_result} +export use app.devhub.editor_file.{editor_temp_parent, prepare_editor_file, cleanup_editor_file} use std.io.process_ops.{process_run} use app.devhub.config.{ItfConfig, resolve_editor} use app.devhub.adapter_confluence.{confluence_get_page, confluence_update_page, PageContent} @@ -42,27 +43,39 @@ fn edit_page(config: ItfConfig, page_id: text, mode: text, message: text) -> i64 file_ext = ".html" # 3. Write to temp file - val tmp_path = "/tmp/itf_edit_{page_id}{file_ext}" - file_write_text(tmp_path, edit_text) + val prepared: (text, text) = match prepare_editor_file(editor_temp_parent(), edit_text, file_ext): + case Ok(value): value + case Err(reason): + print_error(reason) + return 1 + val tmp_directory = prepared.0 + val tmp_path = prepared.1 # 4. Open editor val editor = resolve_editor(config) val (_, _, editor_code) = process_run(editor, [tmp_path]) if editor_code != 0: + if not cleanup_editor_file(tmp_directory): + print_error("could not remove private editor directory") print_error("editor exited with code {editor_code}") return 2 # 5. Read edited content - val edited = file_read_text(tmp_path) - if edited == nil: - print_error("could not read edited file") - return 1 + val edited = match read_file_text_result(tmp_path): + case Ok(value): value + case Err(_): + if not cleanup_editor_file(tmp_directory): + print_error("could not remove private editor directory") + print_error("could not read edited file") + return 1 # 6. Check if changed if edited == edit_text: print "No changes made." # Clean up temp file - file_delete(tmp_path) + if not cleanup_editor_file(tmp_directory): + print_error("could not remove private editor directory") + return 1 return 0 # 7. Convert back if needed @@ -72,13 +85,16 @@ fn edit_page(config: ItfConfig, page_id: text, mode: text, message: text) -> i64 else: new_storage = edited + # The edited text is now in memory. Release the private local copy before + # any remote mutation, and do not report success when cleanup fails. + if not cleanup_editor_file(tmp_directory): + print_error("could not remove private editor directory") + return 1 + # 8. Upload with version tracking val commit_msg = if message != "": message else: "Edited via itf CLI" val (update_ok, updated_ref, update_err) = confluence_update_page(config, page_id, title, new_storage, version, commit_msg) - # Clean up temp file - file_delete(tmp_path) - if not update_ok: print_error("failed to update page: {update_err}") return 1 diff --git a/src/app/devhub/editor_file.spl b/src/app/devhub/editor_file.spl new file mode 100644 index 00000000000..ffb0a448aea --- /dev/null +++ b/src/app/devhub/editor_file.spl @@ -0,0 +1,40 @@ +# Real local-file ownership for the DevHub edit workflow. +# Private workspaces are unique and never include remote page identifiers. +use std.io.file_ops.{secure_temp_dir, file_create_exclusive} +use std.io.dir_ops.{is_dir, dir_remove_all} +use std.env.{env_get} +use std.sffi.fs.{file_atomic_write_mode} + +fn editor_temp_parent() -> text: + var parent = env_get("TMPDIR") + if parent == "": parent = env_get("TEMP") + if parent == "": parent = env_get("TMP") + if parent == "" and is_dir("/tmp"): parent = "/tmp" + parent + + +fn prepare_editor_file(parent: text, content: text, extension: text) -> Result<(text, text), text>: + # Page IDs are remote input and never become local path components. + if extension != ".md" and extension != ".html": + return Err("unsupported editor file extension") + val directory = match secure_temp_dir(parent, "devhub-edit"): + case Ok(path): path + case Err(_): return Err("could not create private editor directory") + val path = directory + "/page" + extension + # Reserve the name without writing the page into a default-DACL file. + # The mode-aware owner publishes the payload only in an owner-only file; + # a private Windows directory alone does not enforce that file DACL. + if not file_create_exclusive(path, ""): + val _cleaned = cleanup_editor_file(directory) + return Err("could not write private editor file") + if not file_atomic_write_mode(path, content, 384): + val _cleaned = cleanup_editor_file(directory) + return Err("could not write private editor file") + Ok((directory, path)) + + +fn cleanup_editor_file(directory: text) -> bool: + # Only a directory obtained from prepare_editor_file is passed here. + dir_remove_all(directory) + + diff --git a/src/app/devhub/flags.spl b/src/app/devhub/flags.spl index 584443ccf7a..d6bea32981d 100644 --- a/src/app/devhub/flags.spl +++ b/src/app/devhub/flags.spl @@ -7,7 +7,11 @@ fn _extract_flag(cmd_args: [text], flag_name: text, default_val: text) -> text: """Extract a flag value from args. E.g., --json fields -> "fields".""" var i = 0 while i < cmd_args.len(): + if cmd_args[i] == "--": + return default_val if cmd_args[i] == flag_name and i + 1 < cmd_args.len(): + if cmd_args[i + 1].starts_with("--") or _flag_is_valueless(cmd_args[i + 1]) or _flag_is_short_value(cmd_args[i + 1]): + return default_val return cmd_args[i + 1] # Support --flag=value syntax if cmd_args[i].starts_with(flag_name + "="): @@ -19,20 +23,43 @@ fn _extract_flag(cmd_args: [text], flag_name: text, default_val: text) -> text: fn _has_flag(cmd_args: [text], flag_name: text) -> bool: """Check if a boolean flag is present. E.g., --yes, --json (no value).""" for arg in cmd_args: + if arg == "--": + return false if arg == flag_name: return true false +fn _flag_is_valueless(arg: text) -> bool: + # Keep optional-value --json and required-value --jq/config options out + # of this table. Merely checking their presence does not make them boolean. + match arg: + "--acli" | "--all" | "--approve" | "--attachments" | "--bytes" | "--confirm" | "--confluence" | "--dry-run" | "--force" | "--help" | "--include" | "--jira" | "--keep-source" | "--no-pager" | "--non-interactive" | "--off" | "--push" | "--quiet" | "--raw" | "--ready-check" | "--recursive" | "--remove" | "--silent" | "--threads" | "--unapprove" | "--unread" | "--verbose" | "--web" | "--yes" | "-a" | "-h" | "-r": true + _: false + +fn _flag_is_short_value(arg: text) -> bool: + match arg: + "-B" | "-H" | "-L" | "-R" | "-b" | "-t": true + _: false + fn _positional_args(cmd_args: [text]) -> [text]: + _positional_args_with_booleans(cmd_args, []) + +fn _positional_args_with_booleans(cmd_args: [text], boolean_flags: [text]) -> [text]: """Extract positional arguments (skip flags and their values).""" var result: [text] = [] var i = 0 while i < cmd_args.len(): val arg = cmd_args[i] - if arg.starts_with("--"): + if arg == "--": + # An explicit terminator makes all remaining values positional, + # including object keys and filenames that begin with dashes. + return result + cmd_args[i + 1:] + if _flag_is_valueless(arg) or boolean_flags.contains(arg): + i = i + 1 + elif arg.starts_with("--") or _flag_is_short_value(arg): # Skip flag + value pair (unless it's a boolean flag) - if i + 1 < cmd_args.len() and not cmd_args[i + 1].starts_with("--"): + if not arg.contains("=") and i + 1 < cmd_args.len() and not cmd_args[i + 1].starts_with("--") and not _flag_is_valueless(cmd_args[i + 1]) and not _flag_is_short_value(cmd_args[i + 1]): i = i + 2 else: i = i + 1 @@ -48,6 +75,8 @@ fn _remove_flags(cmd_args: [text], flag_names: [text]) -> [text]: var i = 0 while i < cmd_args.len(): val arg = cmd_args[i] + if arg == "--": + return result + cmd_args[i:] var is_flag = false for name in flag_names: if arg == name: @@ -55,7 +84,7 @@ fn _remove_flags(cmd_args: [text], flag_names: [text]) -> [text]: if arg.starts_with(name + "="): is_flag = true if is_flag: - if not arg.contains("=") and i + 1 < cmd_args.len() and not cmd_args[i + 1].starts_with("--"): + if not _flag_is_valueless(arg) and not arg.contains("=") and i + 1 < cmd_args.len() and not cmd_args[i + 1].starts_with("--") and not _flag_is_valueless(cmd_args[i + 1]) and not _flag_is_short_value(cmd_args[i + 1]): i = i + 2 else: i = i + 1 diff --git a/src/app/devhub/gh_compat.spl b/src/app/devhub/gh_compat.spl index 4102a2dfe09..3ef6bc6dccc 100644 --- a/src/app/devhub/gh_compat.spl +++ b/src/app/devhub/gh_compat.spl @@ -19,7 +19,7 @@ # Mapping tables and rationale: # doc/01_research/app/tools/devhub/devhub_cli_wrapper_forwarding_2026-09-06.md §3 -use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_field, json_value_has, json_value_at, json_value_len, json_value_serialize} +use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_field, json_value_has, json_value_at, json_value_len, json_value_serialize, json_value_as_text, json_value_escape_string} # ============================================================================ @@ -29,12 +29,13 @@ use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_field, js fn _js(obj: JsonValue, key: text) -> text: """String field off a JSON object; "" when missing.""" - val s = json_value_serialize(json_value_field(obj, key)) - if s.starts_with("\"") and s.ends_with("\""): - return s[1:s.len() - 1] - if s == "null": + val value = json_value_field(obj, key) + if val decoded = json_value_as_text(value): + return decoded + val encoded = json_value_serialize(value) + if encoded == "null": return "" - s + encoded fn _jnested(obj: JsonValue, path: [text]) -> text: @@ -55,21 +56,7 @@ fn _jnested(obj: JsonValue, path: [text]) -> text: fn json_escape(s: text) -> text: """Escape a string for embedding in JSON output.""" - var out = "" - for ch in s: - if ch == "\"": - out = out + "\\\"" - else if ch == "\\": - out = out + "\\\\" - else if ch == "\n": - out = out + "\\n" - else if ch == "\r": - out = out + "\\r" - else if ch == "\t": - out = out + "\\t" - else: - out = out + ch - out + json_value_escape_string(s) fn _quoted(s: text) -> text: @@ -194,6 +181,7 @@ val _COMMON_KNOWN: [text] = ["--json", "--jq", "--no-pager", "--workspace", "--r # Literal tables retain the same public values without requiring executable # array concatenation during native module-static initialization. +# @workaround bug=phase2_module_static_array_concat_initialization_2026_10_27 reason=Keep literal flag catalogs until deployed P2 module-static array concatenation is qualified val PR_CREATE_KNOWN: [text] = ["--json", "--jq", "--no-pager", "--workspace", "--repo", "--backend", "--title", "--head", "--base", "--body", "--reviewer"] val PR_LIST_KNOWN: [text] = ["--json", "--jq", "--no-pager", "--workspace", "--repo", "--backend", "--state", "--limit"] val PR_VIEW_KNOWN: [text] = _COMMON_KNOWN diff --git a/src/app/devhub/messenger/adapter/db_common.spl b/src/app/devhub/messenger/adapter/db_common.spl index e2603624bf8..4219f22bdc0 100644 --- a/src/app/devhub/messenger/adapter/db_common.spl +++ b/src/app/devhub/messenger/adapter/db_common.spl @@ -36,6 +36,8 @@ use std.database.server.storage_port.{ database_storage_port_v1, StorageOpenResult, StoragePortError } use std.database.sql.types.{DbRow, DbValue} +use std.common.json.value.{json_value_escape_string} +use app.devhub.messenger.adapter.history_rows.{messenger_history_rows} val MESSENGER_ADAPTER_HISTORY_DEFAULT_LIMIT: i64 = 50 @@ -51,7 +53,7 @@ fn _rb() -> text: "}" fn _json_escape(value: text) -> text: - value.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n").replace("\r", "\\r") + json_value_escape_string(value) fn _json_str(value: text) -> text: "\"{_json_escape(value)}\"" @@ -63,7 +65,7 @@ fn _json_str(value: text) -> text: # Value of column `name` in a typed row; nil when absent or NULL. fn _col(row: DbRow, name: text) -> DbValue?: var i = 0 - while i < row.columns.len(): + while i < row.columns.len() and i < row.values.len(): if row.columns[i] == name: return Some(row.values[i]) i = i + 1 @@ -243,21 +245,17 @@ fn messenger_db_history(target: MessengerTarget, format_name: text, room: text, case DbReadOutcome.Failed(result): result case DbReadOutcome.Rows(rows): - # Rows arrive in ascending rowid order; history is - # seq-ordered within a room (§7.4: sequence_kind is - # monotonic_within_room for this schema profile). + val ordered: [DbRow] = match messenger_history_rows(rows, profile.message_room_id, profile.message_sequence, room, bounded): + case Ok(selected): selected + case Err(_): return MessengerPortResult.config_error("invalid history sequence metadata") var payload = _lb() + "\"room\":{_json_str(room)},\"messages\":[" var i = 0 - var count = 0 - while i < rows.len(): - if _col_text(rows[i], profile.message_room_id) == room: - if count > 0: - payload = payload + "," - payload = payload + _message_json(rows[i], profile) - count = count + 1 - if count >= bounded: - break + while i < ordered.len(): + if i > 0: + payload = payload + "," + payload = payload + _message_json(ordered[i], profile) i = i + 1 + val count = ordered.len() MessengerPortResult.ok(payload + "],\"count\":{count}" + _rb()) return outcome diff --git a/src/app/devhub/messenger/adapter/history_rows.spl b/src/app/devhub/messenger/adapter/history_rows.spl new file mode 100644 index 00000000000..e628edbd535 --- /dev/null +++ b/src/app/devhub/messenger/adapter/history_rows.spl @@ -0,0 +1,49 @@ +# Pure, bounded sequence selection shared by the read-only database adapter. +use std.database.sql.types.{DbRow, DbValue} + +fn _history_column(row: DbRow, name: text) -> DbValue?: + var i = 0 + while i < row.columns.len() and i < row.values.len(): + if row.columns[i] == name: + return Some(row.values[i]) + i = i + 1 + nil + +fn messenger_history_rows(rows: [DbRow], room_column: text, sequence_column: text, room: text, limit: i64) -> Result<[DbRow], text>: + if limit <= 0: + return Err("history limit must be positive") + var selected: [DbRow] = [] + var sequences: [i64] = [] + for row in rows: + if row.columns.len() != row.values.len(): + return Err("malformed history row") + val row_room: text = match _history_column(row, room_column): + case Some(DbValue.Null): return Err("history room column unavailable") + case Some(value): value.to_text() + case nil: return Err("history room column unavailable") + if row_room != room: + continue + val sequence: i64 = match _history_column(row, sequence_column): + case Some(DbValue.Integer(value)): value + case _: return Err("history sequence must be an integer") + # Retain the lowest `limit` sequences. Validate every matching row, + # including those outside the returned page. Equal keys stay in source + # order, and retained memory stays bounded by the admitted page size. + if selected.len() < limit: + selected.push(row) + sequences.push(sequence) + elif sequence < sequences[sequences.len() - 1]: + selected[selected.len() - 1] = row + sequences[sequences.len() - 1] = sequence + else: + continue + var index = selected.len() - 1 + while index > 0 and sequences[index] < sequences[index - 1]: + val previous_row = selected[index - 1] + val previous_sequence = sequences[index - 1] + selected[index - 1] = selected[index] + sequences[index - 1] = sequences[index] + selected[index] = previous_row + sequences[index] = previous_sequence + index = index - 1 + Ok(selected) diff --git a/src/app/devhub/minio_options.spl b/src/app/devhub/minio_options.spl new file mode 100644 index 00000000000..bbf3271aa43 --- /dev/null +++ b/src/app/devhub/minio_options.spl @@ -0,0 +1,14 @@ +# Exact decimal parsing for the S3 seven-day presign contract. +fn minio_parse_expiry(value: text) -> i64?: + if value == "": return nil + var result: i64 = 0 + var index = 0 + while index < value.len(): + val digit = value.byte_at(index) - 48 + if digit < 0 or digit > 9: return nil + # Admission precedes multiplication, so oversized input cannot wrap. + if result > 60480 or (result == 60480 and digit > 0): return nil + result = result * 10 + digit + index = index + 1 + if result == 0: return nil + Some(result) diff --git a/src/app/devhub/minio_output.spl b/src/app/devhub/minio_output.spl new file mode 100644 index 00000000000..27a0ff1c445 --- /dev/null +++ b/src/app/devhub/minio_output.spl @@ -0,0 +1,14 @@ +# Pure JSON presentation shared by the production MinIO command and specs. +use std.common.json.value.{JsonValue, json_value_object_from, json_value_serialize} + +fn minio_stat_json(key: text, size: i64, etag: text, last_modified: text) -> text: + json_value_serialize(json_value_object_from([ + ("key", JsonValue.Str(key)), ("size", JsonValue.Int(size)), + ("etag", JsonValue.Str(etag)), ("last_modified", JsonValue.Str(last_modified)) + ])) + +fn minio_health_json(alive: bool, status: text, latency_ms: i64, endpoint: text) -> text: + json_value_serialize(json_value_object_from([ + ("alive", JsonValue.Bool(alive)), ("status", JsonValue.Str(status)), + ("latency_ms", JsonValue.Int(latency_ms)), ("endpoint", JsonValue.Str(endpoint)) + ])) diff --git a/src/app/devhub/outlook_pagination.spl b/src/app/devhub/outlook_pagination.spl new file mode 100644 index 00000000000..e84bfb6bfe6 --- /dev/null +++ b/src/app/devhub/outlook_pagination.spl @@ -0,0 +1,31 @@ +# Shared fail-closed Graph pagination contract for both real transports. +use std.common.json.value.{JsonValue, json_value_field, json_value_has, json_value_type, json_value_len, json_value_at} + +fn outlook_page_error(base_url: text, url: text, visited: [text]) -> text: + if base_url == "" or not (url == base_url or url.starts_with(base_url + "/") or url.starts_with(base_url + "?")): + return "Graph pagination URL is outside the configured endpoint" + if url.contains("\r") or url.contains("\n") or url.contains("\t") or url.contains("#"): + return "Graph pagination URL contains invalid control or fragment data" + if visited.len() >= 1000: + return "Graph pagination page limit exceeded" + for previous in visited: + if previous == url: + return "Graph pagination cycle detected" + "" + +fn outlook_page_response_error(parsed: JsonValue) -> text: + if json_value_type(parsed) != "object" or not json_value_has(parsed, "value"): + return "Graph response is missing the value array" + val rows = json_value_field(parsed, "value") + if json_value_type(rows) != "array": + return "Graph response value must be an array" + var index = 0 + while index < json_value_len(rows): + if json_value_type(json_value_at(rows, index)) != "object": + return "Graph response value contains a non-object row" + index = index + 1 + if json_value_has(parsed, "@odata.nextLink"): + val kind = json_value_type(json_value_field(parsed, "@odata.nextLink")) + if kind != "string" and kind != "null": + return "Graph pagination nextLink must be text" + "" diff --git a/src/app/devhub/output.spl b/src/app/devhub/output.spl index c827b671c28..845cd10aa83 100644 --- a/src/app/devhub/output.spl +++ b/src/app/devhub/output.spl @@ -9,7 +9,9 @@ use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_serialize use app.devhub.flags.{_extract_flag, _has_flag} use std.tui.terminal.{terminal_get_size, terminal_stdout_is_tty} use std.io_runtime.{time_now_unix_micros} -use std.file_system.file_ops.{file_write_text, file_delete} +use std.io.file_ops.{file_delete, secure_temp_dir} +use std.sffi.fs.{file_atomic_write_mode} +use std.io.dir_ops.{dir_remove} use std.io.process_ops.{process_run} use app.io.process_ops.{process_run_inherit} @@ -512,12 +514,25 @@ fn _emit_lines(lines: [text]): print line return - val tmp_path = "/tmp/itf_pager_output.txt" - file_write_text(tmp_path, lines.join("\n") + "\n") + var parent = env_get("TMPDIR") + if parent == "": parent = env_get("TEMP") + if parent == "": parent = env_get("TMP") + if parent == "": parent = "/tmp" + val private_dir: text = match secure_temp_dir(parent, "devhub-pager-"): + case Ok(path): path + case Err(_): + for line in lines: print line + return + val tmp_path = private_dir + "/output.txt" + if not file_atomic_write_mode(tmp_path, lines.join("\n") + "\n", 384): + val removed = dir_remove(private_dir, false) + for line in lines: print line + return val (pager_bin, pager_args) = _split_pager_cmd(resolve_pager_cmd()) val code = process_run_inherit(pager_bin, pager_args + [tmp_path]) file_delete(tmp_path) - if code < 0: + val removed = dir_remove(private_dir, false) + if code != 0: # Pager binary missing/unspawnable — never drop the output. for line in lines: print line diff --git a/src/compiler_rust/runtime/build.rs b/src/compiler_rust/runtime/build.rs index 8893d772197..d01e347997f 100644 --- a/src/compiler_rust/runtime/build.rs +++ b/src/compiler_rust/runtime/build.rs @@ -50,6 +50,7 @@ fn main() { println!("cargo:rerun-if-changed=../../runtime/runtime_rocm.c"); println!("cargo:rerun-if-changed=../../runtime/runtime_hosted_signal.c"); println!("cargo:rerun-if-changed=../../runtime/runtime_hosted_fs.c"); + println!("cargo:rerun-if-changed=../../runtime/platform/runtime_windows_private_atomic.h"); println!("cargo:rerun-if-changed=../../runtime/runtime_font.c"); println!("cargo:rerun-if-changed=../../runtime/runtime_value.h"); println!("cargo:rerun-if-changed=../../runtime/runtime_db.c"); diff --git a/src/compiler_rust/runtime/src/value/sffi/file_io/file_ops.rs b/src/compiler_rust/runtime/src/value/sffi/file_io/file_ops.rs index e165c44ed02..0944c550301 100644 --- a/src/compiler_rust/runtime/src/value/sffi/file_io/file_ops.rs +++ b/src/compiler_rust/runtime/src/value/sffi/file_io/file_ops.rs @@ -535,8 +535,48 @@ pub unsafe extern "C" fn rt_file_read_text_rv(path: RuntimeValue) -> RuntimeValu /// (doc/08_tracking/bug/native_link_missing_rt_file_atomic_write_2026-08-17.md). #[no_mangle] pub unsafe extern "C" fn rt_file_atomic_write(path: RuntimeValue, content: RuntimeValue) -> i64 { + rt_file_atomic_write_impl(path, content, None) +} + +/// Explicit-mode core-C twin. Windows owner-only writes use a protected +/// DACL on the staging file before content is written. +#[no_mangle] +pub unsafe extern "C" fn rt_file_atomic_write_mode(path: RuntimeValue, content: RuntimeValue, mode: i32) -> i64 { + rt_file_atomic_write_impl(path, content, Some(mode)) +} + +fn atomic_write_staging_file(temp_path: &Path, content: &[u8], mode: Option) -> std::io::Result<()> { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + // A failed exclusive open never transfers ownership of an existing path. + let mut file = options.open(temp_path)?; + let write_result = (|| -> std::io::Result<()> { + file.write_all(content)?; + #[cfg(unix)] + if let Some(mode) = mode { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode((mode as u32) & 0o7777))?; + } + #[cfg(windows)] + if let Some(mode) = mode { + let mut permissions = file.metadata()?.permissions(); + permissions.set_readonly(mode & 0o200 == 0); + file.set_permissions(permissions)?; + } + file.sync_all()?; + Ok(()) + })(); + drop(file); + if write_result.is_err() { + let _ = std::fs::remove_file(temp_path); + } + write_result +} + +unsafe fn rt_file_atomic_write_impl(path: RuntimeValue, content: RuntimeValue, mode: Option) -> i64 { use crate::value::collections::{rt_string_data, rt_string_len}; - use std::io::Write; let decode = |v: RuntimeValue| -> Option> { if v.is_nil() || v.0 == 0 { @@ -579,6 +619,16 @@ pub unsafe extern "C" fn rt_file_atomic_write(path: RuntimeValue, content: Runti } } + #[cfg(windows)] + if let Some(private_mode) = mode.filter(|value| value & 0o77 == 0) { + extern "C" { + fn rt_hosted_fs_private_atomic_write_utf8(path: *const u8, path_len: u64, + content: *const u8, content_len: u64, mode: i32) -> i32; + } + return i64::from(rt_hosted_fs_private_atomic_write_utf8(path_bytes.as_ptr(), + path_bytes.len() as u64, content_bytes.as_ptr(), content_bytes.len() as u64, private_mode) != 0); + } + static SEQUENCE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let temp_path = std::path::PathBuf::from(format!( "{}.tmp.{}.{}", @@ -587,22 +637,12 @@ pub unsafe extern "C" fn rt_file_atomic_write(path: RuntimeValue, content: Runti SEQUENCE.fetch_add(1, std::sync::atomic::Ordering::Relaxed) )); - let write_result = (|| -> std::io::Result<()> { - let mut file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&temp_path)?; - file.write_all(&content_bytes)?; - file.sync_all()?; - Ok(()) - })(); - if write_result.is_err() { - let _ = std::fs::remove_file(&temp_path); + if atomic_write_staging_file(&temp_path, &content_bytes, mode).is_err() { return 0; } #[cfg(unix)] - if let Some(mode) = existing_mode { + if let Some(mode) = existing_mode.filter(|_| mode.is_none()) { use std::os::unix::fs::PermissionsExt; if std::fs::set_permissions(&temp_path, std::fs::Permissions::from_mode(mode)).is_err() { let _ = std::fs::remove_file(&temp_path); @@ -3365,6 +3405,83 @@ sandbox_lowering: // Similar-problem test: edge semantics shared with the C definition — // missing parent directories are created; an empty path is rejected. + #[cfg(windows)] + fn assert_owner_only_acl(path: &std::path::Path) { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Security::{GetFileSecurityW, GetSecurityDescriptorControl, + GetSecurityDescriptorDacl, GetAce, IsWellKnownSid, WinCreatorOwnerRightsSid, + WinLocalSystemSid, ACCESS_ALLOWED_ACE, + DACL_SECURITY_INFORMATION, INHERITED_ACE, SE_DACL_PROTECTED}; + let wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + unsafe { + let mut required = 0; + GetFileSecurityW(wide.as_ptr(), DACL_SECURITY_INFORMATION, std::ptr::null_mut(), 0, &mut required); + assert!(required > 0); + let mut storage = vec![0u64; (required as usize + 7) / 8]; + let descriptor = storage.as_mut_ptr().cast(); + assert_ne!(GetFileSecurityW(wide.as_ptr(), DACL_SECURITY_INFORMATION, descriptor, + required, &mut required), 0); + let mut control = 0; + let mut revision = 0; + assert_ne!(GetSecurityDescriptorControl(descriptor, &mut control, &mut revision), 0); + assert_ne!(control & SE_DACL_PROTECTED, 0); + let mut present = 0; + let mut defaulted = 0; + let mut acl = std::ptr::null_mut(); + assert_ne!(GetSecurityDescriptorDacl(descriptor, &mut present, &mut acl, &mut defaulted), 0); + assert!(present != 0 && !acl.is_null()); + assert_eq!((*acl).AceCount, 2); + for index in 0..(*acl).AceCount { + let mut raw = std::ptr::null_mut(); + assert_ne!(GetAce(acl, u32::from(index), &mut raw), 0); + let ace = &*raw.cast::(); + // winnt.h ACCESS_ALLOWED_ACE_TYPE = 0; avoid enabling the + // unrelated SystemServices feature just for this constant. + assert_eq!(ace.Header.AceType, 0); + assert_eq!(u32::from(ace.Header.AceFlags) & INHERITED_ACE, 0); + let sid = std::ptr::addr_of!(ace.SidStart).cast_mut().cast(); + assert!(IsWellKnownSid(sid, WinLocalSystemSid) != 0 || + IsWellKnownSid(sid, WinCreatorOwnerRightsSid) != 0); + } + } + } + + #[test] + fn test_rt_file_atomic_write_mode_collision_preserves_foreign_staging() { + let temp_dir = TempDir::new().unwrap(); + let staged = temp_dir.path().join("occupied.tmp"); + fs::write(&staged, "foreign sentinel").unwrap(); + assert!(atomic_write_staging_file(&staged, b"must not replace", Some(0o600)).is_err()); + assert_eq!(fs::read_to_string(&staged).unwrap(), "foreign sentinel"); + } + + #[test] + fn test_rt_file_atomic_write_mode_private_replacement_and_failures() { + let temp_dir = TempDir::new().unwrap(); + let file_path = temp_dir.path().join("private-token.txt"); + fs::write(&file_path, "old inherited value").unwrap(); + let path = rv_text(file_path.to_str().unwrap()); + unsafe { + assert_eq!(rt_file_atomic_write_mode(path, rv_text("first synthetic token"), 0o600), 1); + assert_eq!(fs::read_to_string(&file_path).unwrap(), "first synthetic token"); + #[cfg(windows)] + assert_owner_only_acl(&file_path); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(fs::metadata(&file_path).unwrap().permissions().mode() & 0o7777, 0o600); + } + assert_eq!(rt_file_atomic_write_mode(path, rv_text("second synthetic token"), 0o600), 1); + assert_eq!(fs::read_to_string(&file_path).unwrap(), "second synthetic token"); + #[cfg(windows)] + assert_owner_only_acl(&file_path); + assert_eq!(rt_file_atomic_write_mode(rv_text(""), rv_text("refused"), 0o600), 0); + assert_eq!(rt_file_atomic_write_mode(rv_text(temp_dir.path().to_str().unwrap()), rv_text("refused"), 0o600), 0); + } + assert!(fs::read_dir(temp_dir.path()).unwrap().filter_map(Result::ok) + .all(|entry| !entry.file_name().to_string_lossy().contains(".tmp."))); + } + #[test] fn test_rt_file_atomic_write_creates_parents_and_rejects_empty_path() { let temp_dir = TempDir::new().unwrap(); diff --git a/src/lib/common/http_curl.spl b/src/lib/common/http_curl.spl new file mode 100644 index 00000000000..a97d89b6728 --- /dev/null +++ b/src/lib/common/http_curl.spl @@ -0,0 +1,130 @@ +# Bounded HTTP compatibility transport for hosts without the native provider. +# Caller credentials and payloads stay in private files, never process argv. +use std.env.{env_get} +use std.io.file_ops.{secure_temp_dir, file_write_bytes, file_delete, file_exists, file_read_regular_no_follow_bounded} +use std.io.dir_ops.{dir_remove} +use std.io.process_ops.{process_run_bounded, process_run_bounded_native} +use std.io_runtime.{host_os} +use std.sffi.fs.{file_atomic_write_mode} +use std.common.text_bytes.{text_to_utf8_bytes} + +val CURL_HTTP_MAX_BODY_BYTES = 67108864 +val CURL_HTTP_MAX_RESPONSE_BYTES = 1048576 + +fn _curl_cleanup(private_dir: text) -> bool: + val headers = private_dir + "/headers.txt" + val body = private_dir + "/body.bin" + val response = private_dir + "/response.txt" + val headers_removed = if file_exists(headers): file_delete(headers) else: true + val body_removed = if file_exists(body): file_delete(body) else: true + val response_removed = if file_exists(response): file_delete(response) else: true + val directory_removed = dir_remove(private_dir, false) + headers_removed and body_removed and response_removed and directory_removed + +fn _curl_request_admission(method: text, url: text, headers: [text]) -> text: + if method != "GET" and method != "POST" and method != "PUT" and method != "PATCH" and method != "DELETE" and method != "HEAD" and method != "OPTIONS": + return "unsupported HTTP method" + if not url.starts_with("http://") and not url.starts_with("https://"): + return "HTTP URL must use http or https" + if url.contains("\r") or url.contains("\n") or url.contains("\u{0000}"): + return "HTTP URL contains a control delimiter" + # Userinfo would expose credentials through argv even without verbose curl. + val authority_start = if url.starts_with("https://"): 8 else: 7 + val authority_tail = url[authority_start:] + val authority = authority_tail.split("/", 2)[0] + if authority == "" or authority.contains("@"): + return "HTTP URL authority is empty or contains userinfo" + for header in headers: + if header.contains("\r") or header.contains("\n") or header.contains("\u{0000}"): + return "HTTP header contains a control delimiter" + if not header.contains(":"): + return "HTTP header must contain a name and colon" + "" + +fn _curl_http_status(value: text) -> i64: + val status = value.trim() + if status.len() != 3: return 0 + var result: i64 = 0 + var index = 0 + while index < 3: + val code = status.byte_at(index) as i64 + if code < 48 or code > 57: return 0 + result = result * 10 + code - 48 + index = index + 1 + if result < 100 or result > 599: return 0 + result + +fn curl_request_bytes(method: text, url: text, headers: [text], body: [u8]) -> (i64, text, text): + curl_request_bytes_with_program("curl", method, url, headers, body) + +# @workaround bug=windows_core_http_client_unavailable_2026_10_27 reason=Use the bounded private-file curl transport while the selected Windows core HTTP provider is unavailable +fn curl_request_bytes_with_program(program: text, method: text, url: text, headers: [text], body: [u8]) -> (i64, text, text): + """HTTP status/body/error through real curl, without shell or text conversion. + + HTTP errors retain their status and body. Transport/admission failures use + status zero and a nonempty error. Requests are limited to 64 MiB, replies + to 1 MiB, and execution to 35 seconds including curl's 30-second deadline. + """ + if program == "": return (0, "", "curl HTTP executable is empty") + val refused = _curl_request_admission(method, url, headers) + if refused != "": return (0, "", refused) + if body.len() > CURL_HTTP_MAX_BODY_BYTES: + return (0, "", "HTTP body exceeds the bounded compatibility limit") + var parent = env_get("TMPDIR") + if parent == "": parent = env_get("TEMP") + if parent == "": parent = env_get("TMP") + if parent == "": parent = "/tmp" + val private_dir: text = match secure_temp_dir(parent, "simple-http-"): + case Ok(path): path + case Err(_): return (0, "", "private HTTP staging directory unavailable") + val body_path = private_dir + "/body.bin" + val headers_path = private_dir + "/headers.txt" + val response_path = private_dir + "/response.txt" + # Create owner-only files BEFORE writing bytes. The binary writer opens + # the existing file with wb, retaining its ACL/mode rather than unlinking. + if not file_atomic_write_mode(body_path, "", 384) or not file_atomic_write_mode(headers_path, headers.join("\n") + "\n", 384) or not file_atomic_write_mode(response_path, "", 384): + _curl_cleanup(private_dir) + return (0, "", "private HTTP staging files unavailable") + if not file_write_bytes(body_path, body): + _curl_cleanup(private_dir) + return (0, "", "HTTP binary staging write failed") + val status_format = "%" + "{" + "http_code" + "}" + var arguments: [text] = ["-q", "--silent", "--show-error", "--globoff", "--path-as-is", + "--proto", "=http,https", "--request", method, "--url", url, + "--max-time", "30", "--max-filesize", "{CURL_HTTP_MAX_RESPONSE_BYTES}", + "--header", "@" + headers_path, + "--output", response_path, "--write-out", status_format] + if method == "HEAD": + arguments = arguments + ["--head"] + elif body.len() > 0 or method == "POST" or method == "PUT" or method == "PATCH": + arguments = arguments + ["--data-binary", "@" + body_path] + val initial: (text, text, i64) = process_run_bounded(program, arguments, 35000, 4096) + # The deployed native P2 returned this exact prelaunch fault without any + # loopback request. Retry that observed signature once through the explicit + # native tuple ABI; ordinary interpreter/process results retain their owner. + # This is not a general claim about arbitrary configured executables. + val result: (text, text, i64) = if host_os() == "windows" and initial.0 == "" and initial.1.trim() == "missing command" and initial.2 == -1: + process_run_bounded_native(program, arguments, 35000, 4096) + else: + initial + if result.2 != 0: + _curl_cleanup(private_dir) + return (0, "", "curl HTTP transport failed (exit {result.2}): " + result.1.trim()) + val status = _curl_http_status(result.0) + if status == 0: + _curl_cleanup(private_dir) + return (0, "", "curl HTTP transport returned an invalid status") + val response: text = match file_read_regular_no_follow_bounded(response_path, CURL_HTTP_MAX_RESPONSE_BYTES): + case Ok(value): value + case Err(_): + _curl_cleanup(private_dir) + return (0, "", "HTTP response exceeded its limit or could not be read") + if not _curl_cleanup(private_dir): + return (0, "", "private HTTP staging cleanup failed") + (status, response, "") + +fn curl_request_text(method: text, url: text, headers: [text], body: text) -> (i64, text, text): + curl_request_bytes(method, url, headers, text_to_utf8_bytes(body)) + +fn curl_request_text_with_program(program: text, method: text, url: text, headers: [text], body: text) -> (i64, text, text): + curl_request_bytes_with_program(program, method, url, headers, text_to_utf8_bytes(body)) diff --git a/src/lib/common/oauth_token_cache.spl b/src/lib/common/oauth_token_cache.spl new file mode 100644 index 00000000000..dbbd7f1125e --- /dev/null +++ b/src/lib/common/oauth_token_cache.spl @@ -0,0 +1,50 @@ +# Token-cache I/O owner shared by the configured OAuth implementations. +# The native primitive creates a private same-directory staging file before +# writing secret bytes and publishes it atomically; a mock chmod is never used. +use std.nogc_sync_mut.io_runtime.{read_file_text_result} +use std.nogc_sync_mut.sffi.fs.{file_atomic_write_mode} +use std.common.sdn.value.{SdnValue} +use std.common.sdn.encode.{sdn_encode_canonical} +use std.common.sdn.parser.{parse_with_issues, SdnIssue} + +pub fn oauth_token_cache_read(path: text) -> text: + if path == "": + return "" + match read_file_text_result(path): + case Ok(content): content + case Err(_): "" + +pub fn oauth_token_cache_write(path: text, content: text) -> bool: + if path == "": + return false + # Explicit provider ownership here is shared by all configured families. + # Applications import std.oauth2, never a physical family to select policy. + # Pinned Phase-2 workaround: octal mode lowering produced read-only files. + # Decimal 384 is the exact owner-read/write mask on both old and new producers. + # @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Pass the proven decimal owner-only mode instead of the failing pinned P2 octal carrier + file_atomic_write_mode(path, content, 384 as i32) + +# Use the canonical SDN string writer/parser so escaped or Unicode token bytes +# round-trip without allowing a newline to inject another cache field. +pub fn oauth_token_cache_render(token: text, expires_at: i64) -> text: + val encoded = sdn_encode_canonical(SdnValue.String(token)) + "# OAuth token cache (auto-managed; do not edit)\n" + + "access_token: {encoded}\nexpires_at: {expires_at}\n" + +pub fn oauth_token_cache_parse(content: text) -> (text, i64): + if content == "": + return ("", 0) + val parsed: (SdnValue, [SdnIssue]) = match parse_with_issues(content): + case Ok(value): value + case Err(_): return ("", 0) + val root: SdnValue = parsed.0 + val issues: [SdnIssue] = parsed.1 + if issues.len() > 0 or not root.is_dict(): + return ("", 0) + val token: SdnValue = root.get("access_token") ?? SdnValue.Null + val expiry: SdnValue = root.get("expires_at") ?? SdnValue.Null + if not token.is_string() or not expiry.is_int(): + return ("", 0) + val decoded: text = token.as_str() ?? "" + val expires: i64 = expiry.as_i64() ?? 0 + (decoded, expires) diff --git a/src/lib/gc_async_mut/io/file_ops.spl b/src/lib/gc_async_mut/io/file_ops.spl index 4564a7a7b34..cb112294a3b 100644 --- a/src/lib/gc_async_mut/io/file_ops.spl +++ b/src/lib/gc_async_mut/io/file_ops.spl @@ -6,14 +6,14 @@ The path-based file runtime hooks are owned by `nogc_sync_mut.io.file_ops`. """ export use std.nogc_sync_mut.io.file_ops.{ - file_exists, file_read, file_write, file_copy, file_delete, - file_atomic_write, file_append, file_modified_time, file_remove, + file_exists, file_read, file_write, file_write_bytes, file_copy, file_delete, + file_atomic_write, file_create_exclusive, file_append, file_modified_time, file_remove, file_hash_sha256, file_read_lines, file_lock, file_unlock, file_size, file_size_raw, file_read_text, file_read_bytes, file_mmap_read_text, file_mmap_read_bytes, file_read_text_at, file_write_text_at, file_fsync, file_fsync_cached, file_mmap, file_munmap, file_madvise, file_msync, secure_temp_dir_raw, file_publish_noreplace_raw, secure_temp_dir, file_publish_noreplace, - file_read_regular_no_follow_bounded_bytes + file_read_regular_no_follow_bounded, file_read_regular_no_follow_bounded_bytes } export use std.nogc_sync_mut.io_runtime.{rt_file_rename} diff --git a/src/lib/gc_async_mut/io/http_client_core.spl b/src/lib/gc_async_mut/io/http_client_core.spl new file mode 100644 index 00000000000..eae373399c3 --- /dev/null +++ b/src/lib/gc_async_mut/io/http_client_core.spl @@ -0,0 +1,14 @@ +# Blocking compatibility facade for the configured HTTP client surface. +# The native transport ABI has one provider owner. This facade preserves the +# selected module identity; it does not claim nonblocking execution. +export use std.nogc_sync_mut.io.http_client_core.{ + rt_http_get, rt_http_request, rt_http_request_v2, rt_http_download, + HttpMethod, HttpStatus, HttpResponse, + method_to_string, status_to_code, code_to_status, make_http_response, + url_encode, url_decode, + http_request_raw, http_request_v2_raw, + http_get, http_post, http_post_form, http_put, http_delete, + http_patch, http_head, http_request, http_download, + http_get_json, http_post_json, http_is_success, http_is_redirect, + http_is_client_error, http_is_server_error +} diff --git a/src/lib/gc_async_mut/oauth2.spl b/src/lib/gc_async_mut/oauth2.spl index 00f54536410..485c4545a4c 100644 --- a/src/lib/gc_async_mut/oauth2.spl +++ b/src/lib/gc_async_mut/oauth2.spl @@ -10,12 +10,14 @@ # # This file does NOT depend on aws_sigv4.spl — they are separate concerns. -use std.gc_async_mut.http_client.{http_post, response_get_status_code, response_get_body} -use std.gc_async_mut.file_system.file_ops.{file_read_text, file_write_text, file_exists, file_rename, file_delete} -use std.gc_async_mut.file_system.permissions.{file_set_mode} +use std.gc_async_mut.io.http_client_core.{http_request_raw} +use std.common.http_curl.{curl_request_text} +use std.io_runtime.{platform_name} +use std.common.oauth_token_cache.{oauth_token_cache_read, oauth_token_cache_write, oauth_token_cache_render, oauth_token_cache_parse} use std.gc_async_mut.env.variables.{env_get} use std.gc_async_mut.io.time_ops.{current_time_unix} -use std.json.{json_parse, json_object_get, json_serialize} +use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_has, json_value_field, json_value_serialize, json_value_as_text} +use std.nogc_sync_mut.http.url.{url_encode} # Safety margin (seconds) subtracted from access-token expiry before we @@ -23,13 +25,21 @@ use std.json.{json_parse, json_object_get, json_serialize} val OAUTH2_EXPIRY_SAFETY_MARGIN: i64 = 60 # File mode for the token cache file (owner read/write only). -val OAUTH2_CACHE_FILE_MODE: i64 = 0o600 +# Decimal spelling preserves 0o600 on the pinned Phase-2 producer. +# @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Keep the exact owner-only mode in decimal until deployed P2 octal permission lowering is qualified +val OAUTH2_CACHE_FILE_MODE: i64 = 384 # ============================================================================ # Token endpoint # ============================================================================ +fn oauth2_client_credentials_request(tenant_id: text, client_id: text, client_secret: text, scope: text) -> (text, text, text, text): + val url = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" + val body = "grant_type=client_credentials&client_id={_url_form_encode(client_id)}&client_secret={_url_form_encode(client_secret)}&scope={_url_form_encode(scope)}" + ("POST", url, "Content-Type: application/x-www-form-urlencoded\nAccept: application/json", body) + + fn oauth2_client_credentials_token(tenant_id: text, client_id: text, client_secret: text, scope: text) -> (bool, text, i64): """Request an OAuth2 access token via the client-credentials grant. @@ -44,20 +54,25 @@ fn oauth2_client_credentials_token(tenant_id: text, client_id: text, client_secr if tenant_id == "" or client_id == "" or client_secret == "" or scope == "": return (false, "", 0) - val url = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" - val body = "grant_type=client_credentials&client_id={_url_form_encode(client_id)}&client_secret={_url_form_encode(client_secret)}&scope={_url_form_encode(scope)}" - val now = current_time_unix() - val response = http_post(url, body) - val status = response_get_status_code(response) - val resp_body = response_get_body(response) + val request: (text, text, text, text) = oauth2_client_credentials_request(tenant_id, client_id, client_secret, scope) + # The Windows native HTTP provider explicitly refuses requests. Use the + # shared bounded curl owner there; other hosts retain their native provider. + # @workaround bug=windows_core_http_client_unavailable_2026_10_27 reason=Use the real private-file curl provider while Windows core HTTP is unavailable + val response: (i64, text, text) = if platform_name() == "windows": + curl_request_text(request.0, request.1, request.2.split("\n"), request.3) + else: + http_request_raw(request.0, request.1, request.2, request.3) + val status = response.0 + val resp_body = response.1 if status < 200 or status >= 300: return (false, "", 0) - val parsed = json_parse(resp_body) - if parsed == nil: + val parsed_opt = json_value_parse_opt(resp_body) + if parsed_opt == nil: return (false, "", 0) + val parsed = parsed_opt.unwrap() val tok = _json_text_field(parsed, "access_token") if tok == "": @@ -80,59 +95,21 @@ fn oauth2_token_cache_load(path: text) -> (text, i64): malformed. No expiry check is performed here — the caller decides whether the cached value is still fresh enough. """ - if path == "" or not file_exists(path): - return ("", 0) - - val content = file_read_text(path) - if content == nil or content == "": + val content = oauth_token_cache_read(path) + if content == "": return ("", 0) - var token = "" - var expires_at: i64 = 0 - - val lines = content.split("\n") - for line in lines: - val trimmed = line.trim() - if trimmed == "" or trimmed.starts_with("#"): - pass_dn - else if trimmed.starts_with("access_token:"): - token = _strip_sdn_value(trimmed.split(":", 2)[1]) - else if trimmed.starts_with("expires_at:"): - val raw = _strip_sdn_value(trimmed.split(":", 2)[1]) - expires_at = _parse_i64(raw) - - (token, expires_at) + oauth_token_cache_parse(content) fn oauth2_token_cache_save(path: text, token: text, expires_at: i64) -> bool: - """Atomically write a token cache file with mode 0600. + """Publish a private token cache through the real atomic I/O owner. - Strategy: write temp file (siblings the target with `.tmp` suffix), - chmod 0600 BEFORE rename so there is never a window where the file - is world-readable, then rename over the target. On failure the temp - file is removed. + The shared owner creates a restrictive staging file before writing bytes; + no predictable .tmp path or demonstration permission setter is involved. """ - if path == "": - return false - - val tmp_path = "{path}.tmp" - val payload = "# itf outlook token cache (auto-managed; do not edit)\n" + - "access_token: \"{_escape_sdn_value(token)}\"\n" + - "expires_at: {expires_at}\n" - - if not file_write_text(tmp_path, payload): - return false - - # 0600 BEFORE rename so there is no world-readable window. - if not file_set_mode(tmp_path, OAUTH2_CACHE_FILE_MODE): - file_delete(tmp_path) - return false - - if not file_rename(tmp_path, path): - file_delete(tmp_path) - return false - - true + val payload = oauth_token_cache_render(token, expires_at) + oauth_token_cache_write(path, payload) # ============================================================================ @@ -199,13 +176,13 @@ fn oauth2_parse_retry_after(value: text) -> i64: return 0 val trimmed = value.trim() var n: i64 = 0 - var any = false + var seen_digit = false for ch in trimmed: if ch >= "0" and ch <= "9": n = n * 10 + (ch.to_i64() - "0".to_i64()) - any = true + seen_digit = true else: - return if any: n else: 0 + return if seen_digit: n else: 0 n @@ -215,24 +192,7 @@ fn oauth2_parse_retry_after(value: text) -> i64: fn _url_form_encode(s: text) -> text: """Percent-encode a value for application/x-www-form-urlencoded bodies.""" - var out: [text] = [] - var i = 0 - val len = s.length() - while i < len: - val ch = s.char_at(i) - val code = s.char_code_at(i) - val is_alpha = (code >= 65 and code <= 90) or (code >= 97 and code <= 122) - val is_digit = code >= 48 and code <= 57 - val is_unreserved = is_alpha or is_digit or ch == "-" or ch == "_" or ch == "." or ch == "~" - if is_unreserved: - out.push(ch) - else: - var hex = code.to_string(16).upper() - if hex.length() == 1: - hex = "0" + hex - out.push("%" + hex) - i = i + 1 - out.join("") + url_encode(s) fn _strip_sdn_value(raw: text) -> text: @@ -251,32 +211,21 @@ fn _escape_sdn_value(s: text) -> text: fn _parse_i64(s: text) -> i64: val trimmed = s.trim() var n: i64 = 0 - var any = false + var seen_digit = false for ch in trimmed: if ch >= "0" and ch <= "9": n = n * 10 + (ch.to_i64() - "0".to_i64()) - any = true - else if any: + seen_digit = true + else if seen_digit: return n - if any: n else: 0 + if seen_digit: n else: 0 -fn _json_text_field(obj: any, key: text) -> text: - if obj == nil: - return "" - val value = json_object_get(obj, key) - if value == nil: - return "" - val s = json_serialize(value) - if s.starts_with("\"") and s.ends_with("\"") and s.len() >= 2: - return s[1:s.len() - 1] - s +fn _json_text_field(obj: JsonValue, key: text) -> text: + json_value_as_text(json_value_field(obj, key)) ?? "" -fn _json_int_field(obj: any, key: text) -> i64: - if obj == nil: - return 0 - val value = json_object_get(obj, key) - if value == nil: +fn _json_int_field(obj: JsonValue, key: text) -> i64: + if not json_value_has(obj, key): return 0 - _parse_i64(json_serialize(value)) + _parse_i64(json_value_serialize(json_value_field(obj, key))) diff --git a/src/lib/gc_async_mut/sffi/fs.spl b/src/lib/gc_async_mut/sffi/fs.spl new file mode 100644 index 00000000000..0d0d7cd8cd7 --- /dev/null +++ b/src/lib/gc_async_mut/sffi/fs.spl @@ -0,0 +1,16 @@ +# Filesystem runtime aliases share one native primitive owner. +# Preserve the public facade for configured GC/concurrency family selection. +export use std.nogc_sync_mut.sffi.fs.{ + dir_delete, dir_glob, fd_close, fd_read_until, + fd_write, file_atomic_write_mode, file_canonicalize, file_close, + file_create_excl, file_exists_probe_begin, file_exists_probe_end, file_exists_str, + file_list_dir, file_mmap_read_bytes_raw, file_mmap_raw, file_munmap_raw, + file_mmap_create_raw, file_msync_raw, file_madvise_raw, file_mode, + file_open, file_pread_fd, file_pwrite_fd, file_read_bytes_raw, + file_read_lines_raw, file_read_text_unchecked, file_read_text_at, file_read_text_at_checked, + file_remove_raw, file_sync, file_write_raw, fs_read_text, + io_file_close, io_file_delete, io_file_exists, io_file_flush, + io_file_meta_size, io_file_open, io_file_seek, io_file_write, + io_file_write_all, path_basename, path_dirname, path_ext, + path_separator +} diff --git a/src/lib/gc_async_mut/sffi/process_capture.spl b/src/lib/gc_async_mut/sffi/process_capture.spl new file mode 100644 index 00000000000..cd298cd93d9 --- /dev/null +++ b/src/lib/gc_async_mut/sffi/process_capture.spl @@ -0,0 +1,2 @@ +# Configured facade for the explicit native-only process tuple provider. +export use std.nogc_sync_mut.sffi.process_capture.{process_capture_bounded_native} diff --git a/src/lib/gc_sync_mut/aws_sigv4.spl b/src/lib/gc_sync_mut/aws_sigv4.spl new file mode 100644 index 00000000000..279802ae25c --- /dev/null +++ b/src/lib/gc_sync_mut/aws_sigv4.spl @@ -0,0 +1,10 @@ +# GC sync facade for the shared pure Signature V4 implementation. +export use std.gc_async_mut.aws_sigv4.{ + SIGV4_ALGORITHM, SIGV4_REQUEST_SUFFIX, SIGV4_UNSIGNED_PAYLOAD, SIGV4_EMPTY_SHA256, + SigV4QueryParam, SigV4Header, S3ErrorInfo, + sigv4_sha256_hex, sigv4_uri_encode, sigv4_canonical_uri, + sigv4_derive_signing_key, sigv4_credential_scope, sigv4_string_to_sign, + sigv4_compute_signature, sigv4_canonical_query, sigv4_presign_url, + sigv4_presign_get_url, sigv4_canonical_headers, sigv4_authorization_header, + parse_s3_error +} diff --git a/src/lib/gc_sync_mut/io/http_client_core.spl b/src/lib/gc_sync_mut/io/http_client_core.spl new file mode 100644 index 00000000000..eae373399c3 --- /dev/null +++ b/src/lib/gc_sync_mut/io/http_client_core.spl @@ -0,0 +1,14 @@ +# Blocking compatibility facade for the configured HTTP client surface. +# The native transport ABI has one provider owner. This facade preserves the +# selected module identity; it does not claim nonblocking execution. +export use std.nogc_sync_mut.io.http_client_core.{ + rt_http_get, rt_http_request, rt_http_request_v2, rt_http_download, + HttpMethod, HttpStatus, HttpResponse, + method_to_string, status_to_code, code_to_status, make_http_response, + url_encode, url_decode, + http_request_raw, http_request_v2_raw, + http_get, http_post, http_post_form, http_put, http_delete, + http_patch, http_head, http_request, http_download, + http_get_json, http_post_json, http_is_success, http_is_redirect, + http_is_client_error, http_is_server_error +} diff --git a/src/lib/gc_sync_mut/sffi/fs.spl b/src/lib/gc_sync_mut/sffi/fs.spl new file mode 100644 index 00000000000..0d0d7cd8cd7 --- /dev/null +++ b/src/lib/gc_sync_mut/sffi/fs.spl @@ -0,0 +1,16 @@ +# Filesystem runtime aliases share one native primitive owner. +# Preserve the public facade for configured GC/concurrency family selection. +export use std.nogc_sync_mut.sffi.fs.{ + dir_delete, dir_glob, fd_close, fd_read_until, + fd_write, file_atomic_write_mode, file_canonicalize, file_close, + file_create_excl, file_exists_probe_begin, file_exists_probe_end, file_exists_str, + file_list_dir, file_mmap_read_bytes_raw, file_mmap_raw, file_munmap_raw, + file_mmap_create_raw, file_msync_raw, file_madvise_raw, file_mode, + file_open, file_pread_fd, file_pwrite_fd, file_read_bytes_raw, + file_read_lines_raw, file_read_text_unchecked, file_read_text_at, file_read_text_at_checked, + file_remove_raw, file_sync, file_write_raw, fs_read_text, + io_file_close, io_file_delete, io_file_exists, io_file_flush, + io_file_meta_size, io_file_open, io_file_seek, io_file_write, + io_file_write_all, path_basename, path_dirname, path_ext, + path_separator +} diff --git a/src/lib/gc_sync_mut/sffi/process_capture.spl b/src/lib/gc_sync_mut/sffi/process_capture.spl new file mode 100644 index 00000000000..cd298cd93d9 --- /dev/null +++ b/src/lib/gc_sync_mut/sffi/process_capture.spl @@ -0,0 +1,2 @@ +# Configured facade for the explicit native-only process tuple provider. +export use std.nogc_sync_mut.sffi.process_capture.{process_capture_bounded_native} diff --git a/src/lib/nogc_async_mut/io/file_ops.spl b/src/lib/nogc_async_mut/io/file_ops.spl index fc747bb8b55..cf53ab15401 100644 --- a/src/lib/nogc_async_mut/io/file_ops.spl +++ b/src/lib/nogc_async_mut/io/file_ops.spl @@ -6,8 +6,8 @@ The path-based file runtime hooks are owned by `nogc_sync_mut.io.file_ops`. """ export use std.nogc_sync_mut.io.file_ops.{ - file_exists, file_read, file_write, file_copy, file_delete, - file_atomic_write, file_append, file_modified_time, file_remove, + file_exists, file_read, file_write, file_write_bytes, file_copy, file_delete, + file_atomic_write, file_create_exclusive, file_append, file_modified_time, file_remove, file_hash_sha256, file_read_lines, file_lock, file_unlock, file_size, file_size_raw, file_stat, file_read_text, file_read_bytes, file_mmap_read_text, file_mmap_read_bytes, file_read_text_at, file_write_text_at, file_fsync, diff --git a/src/lib/nogc_async_mut/io/http_client_core.spl b/src/lib/nogc_async_mut/io/http_client_core.spl new file mode 100644 index 00000000000..eae373399c3 --- /dev/null +++ b/src/lib/nogc_async_mut/io/http_client_core.spl @@ -0,0 +1,14 @@ +# Blocking compatibility facade for the configured HTTP client surface. +# The native transport ABI has one provider owner. This facade preserves the +# selected module identity; it does not claim nonblocking execution. +export use std.nogc_sync_mut.io.http_client_core.{ + rt_http_get, rt_http_request, rt_http_request_v2, rt_http_download, + HttpMethod, HttpStatus, HttpResponse, + method_to_string, status_to_code, code_to_status, make_http_response, + url_encode, url_decode, + http_request_raw, http_request_v2_raw, + http_get, http_post, http_post_form, http_put, http_delete, + http_patch, http_head, http_request, http_download, + http_get_json, http_post_json, http_is_success, http_is_redirect, + http_is_client_error, http_is_server_error +} diff --git a/src/lib/nogc_async_mut/io/process_ops.spl b/src/lib/nogc_async_mut/io/process_ops.spl index f31754c6aa8..029fa8848c2 100644 --- a/src/lib/nogc_async_mut/io/process_ops.spl +++ b/src/lib/nogc_async_mut/io/process_ops.spl @@ -6,7 +6,7 @@ Compatibility facade over the no-GC sync process backend. export use std.nogc_sync_mut.io.process_ops.{ ProcessResult, - process_run, process_run_bounded, process_run_bounded_bytes, + process_run, process_run_bounded, process_run_bounded_native, process_run_bounded_bytes, process_run_timeout_unix, process_run_timeout_windows, process_run_timeout, process_run_with_limits, process_output, shell, shell_bool, shell_output_trimmed, shell_lines, diff --git a/src/lib/nogc_async_mut/oauth2.spl b/src/lib/nogc_async_mut/oauth2.spl index 4af87224b00..c0e67b22860 100644 --- a/src/lib/nogc_async_mut/oauth2.spl +++ b/src/lib/nogc_async_mut/oauth2.spl @@ -10,12 +10,14 @@ # # This file does NOT depend on aws_sigv4.spl — they are separate concerns. -use std.nogc_sync_mut.http_client.{http_post, response_get_status_code, response_get_body} -use std.nogc_sync_mut.file_system.file_ops.{file_read_text, file_write_text, file_exists, file_rename, file_delete} -use std.nogc_sync_mut.file_system.permissions.{file_set_mode} +use std.nogc_sync_mut.io.http_client_core.{http_request_raw} +use std.common.http_curl.{curl_request_text} +use std.io_runtime.{platform_name} +use std.common.oauth_token_cache.{oauth_token_cache_read, oauth_token_cache_write, oauth_token_cache_render, oauth_token_cache_parse} use std.nogc_sync_mut.env.variables.{env_get} use std.nogc_sync_mut.io.time_ops.{current_time_unix} -use std.json.{json_parse, json_object_get, json_serialize} +use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_has, json_value_field, json_value_serialize, json_value_as_text} +use std.nogc_sync_mut.http.url.{url_encode} # Safety margin (seconds) subtracted from access-token expiry before we @@ -23,13 +25,21 @@ use std.json.{json_parse, json_object_get, json_serialize} val OAUTH2_EXPIRY_SAFETY_MARGIN: i64 = 60 # File mode for the token cache file (owner read/write only). -val OAUTH2_CACHE_FILE_MODE: i64 = 0o600 +# Decimal spelling preserves 0o600 on the pinned Phase-2 producer. +# @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Keep the exact owner-only mode in decimal until deployed P2 octal permission lowering is qualified +val OAUTH2_CACHE_FILE_MODE: i64 = 384 # ============================================================================ # Token endpoint # ============================================================================ +fn oauth2_client_credentials_request(tenant_id: text, client_id: text, client_secret: text, scope: text) -> (text, text, text, text): + val url = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" + val body = "grant_type=client_credentials&client_id={_url_form_encode(client_id)}&client_secret={_url_form_encode(client_secret)}&scope={_url_form_encode(scope)}" + ("POST", url, "Content-Type: application/x-www-form-urlencoded\nAccept: application/json", body) + + fn oauth2_client_credentials_token(tenant_id: text, client_id: text, client_secret: text, scope: text) -> (bool, text, i64): """Request an OAuth2 access token via the client-credentials grant. @@ -44,20 +54,25 @@ fn oauth2_client_credentials_token(tenant_id: text, client_id: text, client_secr if tenant_id == "" or client_id == "" or client_secret == "" or scope == "": return (false, "", 0) - val url = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" - val body = "grant_type=client_credentials&client_id={_url_form_encode(client_id)}&client_secret={_url_form_encode(client_secret)}&scope={_url_form_encode(scope)}" - val now = current_time_unix() - val response = http_post(url, body) - val status = response_get_status_code(response) - val resp_body = response_get_body(response) + val request: (text, text, text, text) = oauth2_client_credentials_request(tenant_id, client_id, client_secret, scope) + # The Windows native HTTP provider explicitly refuses requests. Use the + # shared bounded curl owner there; other hosts retain their native provider. + # @workaround bug=windows_core_http_client_unavailable_2026_10_27 reason=Use the real private-file curl provider while Windows core HTTP is unavailable + val response: (i64, text, text) = if platform_name() == "windows": + curl_request_text(request.0, request.1, request.2.split("\n"), request.3) + else: + http_request_raw(request.0, request.1, request.2, request.3) + val status = response.0 + val resp_body = response.1 if status < 200 or status >= 300: return (false, "", 0) - val parsed = json_parse(resp_body) - if parsed == nil: + val parsed_opt = json_value_parse_opt(resp_body) + if parsed_opt == nil: return (false, "", 0) + val parsed = parsed_opt.unwrap() val tok = _json_text_field(parsed, "access_token") if tok == "": @@ -80,59 +95,21 @@ fn oauth2_token_cache_load(path: text) -> (text, i64): malformed. No expiry check is performed here — the caller decides whether the cached value is still fresh enough. """ - if path == "" or not file_exists(path): - return ("", 0) - - val content = file_read_text(path) - if content == nil or content == "": + val content = oauth_token_cache_read(path) + if content == "": return ("", 0) - var token = "" - var expires_at: i64 = 0 - - val lines = content.split("\n") - for line in lines: - val trimmed = line.trim() - if trimmed == "" or trimmed.starts_with("#"): - pass_dn - else if trimmed.starts_with("access_token:"): - token = _strip_sdn_value(trimmed.split(":", 2)[1]) - else if trimmed.starts_with("expires_at:"): - val raw = _strip_sdn_value(trimmed.split(":", 2)[1]) - expires_at = _parse_i64(raw) - - (token, expires_at) + oauth_token_cache_parse(content) fn oauth2_token_cache_save(path: text, token: text, expires_at: i64) -> bool: - """Atomically write a token cache file with mode 0600. + """Publish a private token cache through the real atomic I/O owner. - Strategy: write temp file (siblings the target with `.tmp` suffix), - chmod 0600 BEFORE rename so there is never a window where the file - is world-readable, then rename over the target. On failure the temp - file is removed. + The shared owner creates a restrictive staging file before writing bytes; + no predictable .tmp path or demonstration permission setter is involved. """ - if path == "": - return false - - val tmp_path = "{path}.tmp" - val payload = "# itf outlook token cache (auto-managed; do not edit)\n" + - "access_token: \"{_escape_sdn_value(token)}\"\n" + - "expires_at: {expires_at}\n" - - if not file_write_text(tmp_path, payload): - return false - - # 0600 BEFORE rename so there is no world-readable window. - if not file_set_mode(tmp_path, OAUTH2_CACHE_FILE_MODE): - file_delete(tmp_path) - return false - - if not file_rename(tmp_path, path): - file_delete(tmp_path) - return false - - true + val payload = oauth_token_cache_render(token, expires_at) + oauth_token_cache_write(path, payload) # ============================================================================ @@ -199,13 +176,13 @@ fn oauth2_parse_retry_after(value: text) -> i64: return 0 val trimmed = value.trim() var n: i64 = 0 - var any = false + var seen_digit = false for ch in trimmed: if ch >= "0" and ch <= "9": n = n * 10 + (ch.to_i64() - "0".to_i64()) - any = true + seen_digit = true else: - return if any: n else: 0 + return if seen_digit: n else: 0 n @@ -215,24 +192,7 @@ fn oauth2_parse_retry_after(value: text) -> i64: fn _url_form_encode(s: text) -> text: """Percent-encode a value for application/x-www-form-urlencoded bodies.""" - var out: [text] = [] - var i = 0 - val len = s.length() - while i < len: - val ch = s.char_at(i) - val code = s.char_code_at(i) - val is_alpha = (code >= 65 and code <= 90) or (code >= 97 and code <= 122) - val is_digit = code >= 48 and code <= 57 - val is_unreserved = is_alpha or is_digit or ch == "-" or ch == "_" or ch == "." or ch == "~" - if is_unreserved: - out.push(ch) - else: - var hex = code.to_string(16).upper() - if hex.length() == 1: - hex = "0" + hex - out.push("%" + hex) - i = i + 1 - out.join("") + url_encode(s) fn _strip_sdn_value(raw: text) -> text: @@ -251,32 +211,21 @@ fn _escape_sdn_value(s: text) -> text: fn _parse_i64(s: text) -> i64: val trimmed = s.trim() var n: i64 = 0 - var any = false + var seen_digit = false for ch in trimmed: if ch >= "0" and ch <= "9": n = n * 10 + (ch.to_i64() - "0".to_i64()) - any = true - else if any: + seen_digit = true + else if seen_digit: return n - if any: n else: 0 + if seen_digit: n else: 0 -fn _json_text_field(obj: any, key: text) -> text: - if obj == nil: - return "" - val value = json_object_get(obj, key) - if value == nil: - return "" - val s = json_serialize(value) - if s.starts_with("\"") and s.ends_with("\"") and s.len() >= 2: - return s[1:s.len() - 1] - s +fn _json_text_field(obj: JsonValue, key: text) -> text: + json_value_as_text(json_value_field(obj, key)) ?? "" -fn _json_int_field(obj: any, key: text) -> i64: - if obj == nil: - return 0 - val value = json_object_get(obj, key) - if value == nil: +fn _json_int_field(obj: JsonValue, key: text) -> i64: + if not json_value_has(obj, key): return 0 - _parse_i64(json_serialize(value)) + _parse_i64(json_value_serialize(json_value_field(obj, key))) diff --git a/src/lib/nogc_async_mut/sffi/fs.spl b/src/lib/nogc_async_mut/sffi/fs.spl new file mode 100644 index 00000000000..0d0d7cd8cd7 --- /dev/null +++ b/src/lib/nogc_async_mut/sffi/fs.spl @@ -0,0 +1,16 @@ +# Filesystem runtime aliases share one native primitive owner. +# Preserve the public facade for configured GC/concurrency family selection. +export use std.nogc_sync_mut.sffi.fs.{ + dir_delete, dir_glob, fd_close, fd_read_until, + fd_write, file_atomic_write_mode, file_canonicalize, file_close, + file_create_excl, file_exists_probe_begin, file_exists_probe_end, file_exists_str, + file_list_dir, file_mmap_read_bytes_raw, file_mmap_raw, file_munmap_raw, + file_mmap_create_raw, file_msync_raw, file_madvise_raw, file_mode, + file_open, file_pread_fd, file_pwrite_fd, file_read_bytes_raw, + file_read_lines_raw, file_read_text_unchecked, file_read_text_at, file_read_text_at_checked, + file_remove_raw, file_sync, file_write_raw, fs_read_text, + io_file_close, io_file_delete, io_file_exists, io_file_flush, + io_file_meta_size, io_file_open, io_file_seek, io_file_write, + io_file_write_all, path_basename, path_dirname, path_ext, + path_separator +} diff --git a/src/lib/nogc_async_mut/sffi/process_capture.spl b/src/lib/nogc_async_mut/sffi/process_capture.spl new file mode 100644 index 00000000000..cd298cd93d9 --- /dev/null +++ b/src/lib/nogc_async_mut/sffi/process_capture.spl @@ -0,0 +1,2 @@ +# Configured facade for the explicit native-only process tuple provider. +export use std.nogc_sync_mut.sffi.process_capture.{process_capture_bounded_native} diff --git a/src/lib/nogc_sync_mut/compression/gzip/crc.spl b/src/lib/nogc_sync_mut/compression/gzip/crc.spl index 4ea92b383c5..1b7a4e76f68 100644 --- a/src/lib/nogc_sync_mut/compression/gzip/crc.spl +++ b/src/lib/nogc_sync_mut/compression/gzip/crc.spl @@ -10,18 +10,23 @@ use std.common.crypto.crc32.{crc32, crc32_update as _shared_crc32_update} fn crc32_calculate(data: [u8]) -> i64: if data == nil: return 0 - crc32(data).to_i64() + # Keep the public i64 carrier within the CRC32 register width. + # @workaround bug=phase2_crc32_i64_width_spill_2026_10_27 reason=Retain only the CRC32 low word until the deployed P2 unsigned i64 carrier boundary is qualified + crc32(data).to_i64() & 4294967295 # Update running CRC32 with new data (crc is the pre-final-XOR register) fn crc32_update(crc: i64, data: [u8]) -> i64: if data == nil: return crc - _shared_crc32_update(crc.to_u32(), data).to_i64() + # The pinned P2 widening boundary can retain unexpected high bits. + # @workaround bug=phase2_crc32_i64_width_spill_2026_10_27 reason=Retain only the CRC32 low word until the deployed P2 unsigned i64 carrier boundary is qualified + _shared_crc32_update(crc.to_u32(), data).to_i64() & 4294967295 # Finalize CRC32 (XOR with 0xffffffff) # @workaround bug=p4p2_cranelift_verifier_types_2026-10-20 reason=untyped-param fns kept a Unit signature (cranelift return mismatch) fn crc32_finalize(crc: i64) -> i64: - return crc ^ 0xffffffff + # @workaround bug=phase2_crc32_i64_width_spill_2026_10_27 reason=Retain only the CRC32 low word until the deployed P2 unsigned i64 carrier boundary is qualified + return (crc ^ 0xffffffff) & 4294967295 # Calculate CRC32 in chunks # @workaround bug=p4p2_cranelift_verifier_types_2026-10-20 reason=untyped-param fns kept a Unit signature (cranelift return mismatch) @@ -59,4 +64,5 @@ fn crc32_verify(data: [u8], expected: i64) -> bool: # test/01_unit/lib/nogc_sync_mut/compression/crc32_text_crosslang_spec.spl # (binary_runtime_hardening C-MIG tranche 1). fn crc32_text(s: text) -> i64: - crc32(s.bytes()).to_i64() + # @workaround bug=phase2_crc32_i64_width_spill_2026_10_27 reason=Retain only the CRC32 low word until the deployed P2 unsigned i64 carrier boundary is qualified + crc32(s.bytes()).to_i64() & 4294967295 diff --git a/src/lib/nogc_sync_mut/database/bug.spl b/src/lib/nogc_sync_mut/database/bug.spl index b35de9e65eb..958bdd65d74 100644 --- a/src/lib/nogc_sync_mut/database/bug.spl +++ b/src/lib/nogc_sync_mut/database/bug.spl @@ -17,6 +17,11 @@ use nogc_sync_mut.database.core.{SdnDatabase, SdnTable, SdnRow, StringInterner} use nogc_sync_mut.database.query use std.convert (try_parse_int) +use nogc_sync_mut.database.atomic.{FileLock} +use std.nogc_sync_mut.io.file_ops.{file_atomic_write, file_exists} +use std.io_runtime.{read_file_text_result} +use std.nogc_sync_mut.compression.gzip.crc.{crc32_update} +use std.common.text_bytes.{text_to_utf8_bytes} use nogc_sync_mut.database.checker.{DbIssue, check_duplicate_ids, check_valid_column, check_empty_required, check_file_refs, check_enum_values, check_orphan_refs} # Removed 'use app.io' - it shadows core.spl's local function declarations @@ -469,6 +474,17 @@ class BugDatabase: me save() -> bool: self.db.save() + # Append-only migration path for a legacy registry whose opaque rows must + # not be parsed and regenerated. Only new open bugs are serialized; every + # pre-existing payload byte remains intact, including overwide old rows. + # @workaround bug=bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27 reason=Append only new canonical rows until full registry round trips preserve opaque legacy payloads + me append_open_bugs_preserving_legacy(records: [Bug]) -> bool: + var lock: FileLock = FileLock.for_file(self.db.path) + if not lock.try_acquire(5000): return false + val published = _bug_append_open_locked(self.db.path, records) + lock.release() + published + # Create a new bug database without replacing an existing path entry. me save_exclusive() -> bool: self.db.save_exclusive() @@ -751,3 +767,80 @@ fn bug_to_row(bug: Bug) -> SdnRow: val valid_str = if bug.valid: "true" else: "false" row.set("valid", valid_str) row + + +# One physical line is the admitted field contract of the legacy table writer. +fn _bug_append_field_safe(value: text) -> bool: + value == value.trim() and not value.contains("\n") and not value.contains("\r") and not value.contains("\0") and not value.contains("\"") + +fn _bug_append_id_safe(value: text) -> bool: + if value == "": return false + val allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.-" + for character in value.chars(): + if not allowed.contains(character): return false + true + +fn _bug_append_open_locked(path: text, records: [Bug]) -> bool: + if records.len() == 0: return false + # A pending WAL is authoritative. Do not bypass or discard its records. + if file_exists(path + ".wal"): + val wal: text = match read_file_text_result(path + ".wal"): + case Ok(value): value + case Err(_): return false + if wal.trim() != "": return false + val original: text = match read_file_text_result(path): + case Ok(value): value + case Err(_): return false + if not original.starts_with("#sdn-crc32:") or original.len() > 33554432: + return false + val newline = original.index_of("\n") + if newline < 0: return false + val expected = try_parse_int(original.substring(11, newline).trim()) + if expected == nil: return false + val body: text = original.substring(newline + 1) + val initial_crc = crc32_update(0xffffffff, text_to_utf8_bytes(body)) ^ 0xffffffff + if initial_crc != (expected ?? -1): return false + val original_lines: [text] = body.split("\n") + var pending: BugDatabase = create_bug_database(path) + val columns: [text] = ["id", "severity", "status", "title", "file", "line", "reproducible_by", "created_at", "updated_at", "valid"] + pending.db.set_table("bugs_active", SdnTable.new("bugs_active", columns)) + for raw_record in records: + val record: Bug = raw_record + if record.status != BugStatus.Open or not record.valid or not _bug_append_id_safe(record.id): return false + if not _bug_append_field_safe(record.title) or not _bug_append_field_safe(record.file) or not _bug_append_field_safe(record.reproducible_by) or not _bug_append_field_safe(record.created_at) or not _bug_append_field_safe(record.updated_at): return false + for field in record.description + record.fix_strategy + record.investigation_log: + if not _bug_append_field_safe(field): return false + for raw_line in original_lines: + val line: text = raw_line.trim() + if line.starts_with(record.id + ",") or line.starts_with("\"" + record.id + "\","): + return false + # Use the existing domain row and multiline serialization helpers. + if not pending.add_bug(record): return false + var updated: text = body + val names: [text] = ["bugs_active", "bug_descriptions", "bug_fix_strategies", "bug_investigation_logs"] + for name in names: + val table: SdnTable = match pending.db.get_table(name): + case Some(value): value + case None: return false + if table.rows.len() == 0: continue + val encoded: text = table.to_sdn() + val separator = encoded.index_of("\n") + if separator < 0: return false + val header = encoded.substring(0, separator) + var rows = encoded.substring(separator + 1) + val anchored = "\n" + updated + val crlf = "\n" + header + "\r\n" + val lf = "\n" + header + "\n" + val crlf_count = anchored.split(crlf).len() - 1 + val lf_count = anchored.split(lf).len() - 1 + if crlf_count + lf_count != 1: return false + if crlf_count == 1: + rows = rows.replace("\n", "\r\n") + updated = anchored.replace(crlf, crlf + rows).substring(1) + else: + updated = anchored.replace(lf, lf + rows).substring(1) + val final_crc = crc32_update(0xffffffff, text_to_utf8_bytes(updated)) ^ 0xffffffff + val prefix = if original.substring(0, newline).ends_with("\r"): "#sdn-crc32:{final_crc}\r\n" else: "#sdn-crc32:{final_crc}\n" + # FileLock is already held across read/validation/publication. The real + # native atomic publisher creates its own exclusive same-directory stage. + file_atomic_write(path, prefix + updated) diff --git a/src/lib/nogc_sync_mut/database/query.spl b/src/lib/nogc_sync_mut/database/query.spl index 208b10cd75d..3a18813b249 100644 --- a/src/lib/nogc_sync_mut/database/query.spl +++ b/src/lib/nogc_sync_mut/database/query.spl @@ -151,7 +151,14 @@ class QueryBuilder: # Apply limit if self.has_limit_count: - row_ids = row_ids.take(self.limit_count) + # @workaround bug=native_array_take_method_missing_2026_10_21 reason=Use the existing ordered fresh-prefix contract until native array.take is admitted + var limited_row_ids: [i64] = [] + val limit_end = if self.limit_count < row_ids.len(): self.limit_count else: row_ids.len() + var limit_index = 0 + while limit_index < limit_end: + limited_row_ids.push(row_ids[limit_index]) + limit_index = limit_index + 1 + row_ids = limited_row_ids var results: [SdnRow] = [SdnRow.empty(); row_ids.len()] var result_idx = 0 diff --git a/src/lib/nogc_sync_mut/database/sqlite_format/btree.spl b/src/lib/nogc_sync_mut/database/sqlite_format/btree.spl index 17350e6f59e..732100f37b4 100644 --- a/src/lib/nogc_sync_mut/database/sqlite_format/btree.spl +++ b/src/lib/nogc_sync_mut/database/sqlite_format/btree.spl @@ -30,6 +30,7 @@ fn _scan_table_page(pager: SqlitePager, page_no: i64, depth: i64, acc: [(i64, [u var i: i64 = 0 while i < n: val ptr = cell_ptr(page, base, 8, i) + # @workaround bug=phase2_result_tuple_try_binding_metadata_2026_10_27 reason=Bind native Result tuple payloads explicitly until deployed P2 preserves integer and byte metadata through propagation val cell: (i64, [u8]) = match parse_table_leaf_cell(pager, page, ptr): case Ok(value): value case Err(error): return Err(error) diff --git a/src/lib/nogc_sync_mut/database/sqlite_format/cell.spl b/src/lib/nogc_sync_mut/database/sqlite_format/cell.spl index ad0ce65ca26..2ec9db7e56d 100644 --- a/src/lib/nogc_sync_mut/database/sqlite_format/cell.spl +++ b/src/lib/nogc_sync_mut/database/sqlite_format/cell.spl @@ -91,10 +91,12 @@ fn read_overflow_chain(pager: SqlitePager, first_page: i64, needed: i64) -> Resu # Returns (rowid, full record payload). fn parse_table_leaf_cell(pager: SqlitePager, page: [u8], ptr: i64) -> Result<(i64, [u8]), SqliteError>: val usable = pager.usable_size() + # @workaround bug=phase2_result_tuple_try_binding_metadata_2026_10_27 reason=Bind native Result tuple payloads explicitly until deployed P2 preserves integer and byte metadata through propagation val len_dec: (u64, i64) = match sqlite_varint_decode(page, ptr): case Ok(value): value case Err(error): return Err(error) val payload_len = len_dec.0 as i64 + # @workaround bug=phase2_result_tuple_try_binding_metadata_2026_10_27 reason=Bind native Result tuple payloads explicitly until deployed P2 preserves integer and byte metadata through propagation val rowid_dec: (u64, i64) = match sqlite_varint_decode(page, ptr + len_dec.1): case Ok(value): value case Err(error): return Err(error) @@ -122,6 +124,7 @@ fn parse_table_leaf_cell(pager: SqlitePager, page: [u8], ptr: i64) -> Result<(i6 fn parse_table_interior_cell(pager: SqlitePager, page: [u8], ptr: i64) -> Result<(i64, i64), SqliteError>: pager.check_span(ptr, 5)? val child = be_u32_at(page, ptr) + # @workaround bug=phase2_result_tuple_try_binding_metadata_2026_10_27 reason=Bind native Result tuple payloads explicitly until deployed P2 preserves integer and byte metadata through propagation val key: (u64, i64) = match sqlite_varint_decode(page, ptr + 4): case Ok(value): value case Err(error): return Err(error) diff --git a/src/lib/nogc_sync_mut/database/sqlite_format/record.spl b/src/lib/nogc_sync_mut/database/sqlite_format/record.spl index 7a87c84d356..7ff1b8aebd6 100644 --- a/src/lib/nogc_sync_mut/database/sqlite_format/record.spl +++ b/src/lib/nogc_sync_mut/database/sqlite_format/record.spl @@ -90,6 +90,7 @@ fn sqlite_record_decode(payload: [u8], encoding: i64) -> Result<[DbValue], Sqlit if payload.len() == 0: return Err(SqliteError.MalformedRecord(message: "empty record payload")) + # @workaround bug=phase2_result_tuple_try_binding_metadata_2026_10_27 reason=Bind native Result tuple payloads explicitly until deployed P2 preserves integer and byte metadata through propagation val hdr: (u64, i64) = match sqlite_varint_decode(payload, 0): case Ok(value): value case Err(error): return Err(error) @@ -101,6 +102,7 @@ fn sqlite_record_decode(payload: [u8], encoding: i64) -> Result<[DbValue], Sqlit var serials: [i64] = [] var pos = hdr_varint_len while pos < header_len: + # @workaround bug=phase2_result_tuple_try_binding_metadata_2026_10_27 reason=Bind native Result tuple payloads explicitly until deployed P2 preserves integer and byte metadata through propagation val s: (u64, i64) = match sqlite_varint_decode(payload, pos): case Ok(value): value case Err(error): return Err(error) diff --git a/src/lib/nogc_sync_mut/database/todo.spl b/src/lib/nogc_sync_mut/database/todo.spl index 59276d616b5..118cf2c9808 100644 --- a/src/lib/nogc_sync_mut/database/todo.spl +++ b/src/lib/nogc_sync_mut/database/todo.spl @@ -1,11 +1,11 @@ -# Todo Database - Read/validate wrapper for todo_db.sdn +# Todo Database - Read/validate wrapper and preserving append for todo_db.sdn # # The todo_scan app currently writes directly via file_write(), # bypassing the shared database library. This module provides # read-only access and validation for check-dbs/fix-dbs commands. # -# NOTE: Write path migration is out of scope. This module only -# reads and validates existing todo_db.sdn files. +# The scanner write-path migration remains out of scope. The narrow append API +# below serializes new blocked records without regenerating legacy payloads. # # Usage: # use std.database.todo.{TodoDatabase, load_todo_database} @@ -153,3 +153,73 @@ class TodoDatabase: # ============================================================================ # Exports # ============================================================================ + +use nogc_sync_mut.database.atomic.{FileLock} +use std.nogc_sync_mut.io.file_ops.{file_atomic_write} +use std.io_runtime.{read_file_text_result} +use std.nogc_sync_mut.compression.gzip.crc.{crc32_text} +use std.convert (try_parse_int) + +# Serialize only the new row; never parse and regenerate existing TODO payloads. +fn append_todo_preserving_legacy(path: text, row: SdnRow) -> bool: + var lock: FileLock = FileLock.for_file(path) + if not lock.try_acquire(5000): return false + val published = _todo_append_locked(path, row) + lock.release() + published + +fn _todo_append_locked(path: text, row: SdnRow) -> bool: + # A pending WAL is authoritative. Do not bypass or discard its records. + if file_exists(path + ".wal"): + val wal: text = match read_file_text_result(path + ".wal"): + case Ok(value): value + case Err(_): return false + if wal.trim() != "": return false + val original: text = match read_file_text_result(path): + case Ok(value): value + case Err(_): return false + if original.len() > 33554432: return false + var body: text = original + if original.starts_with("#sdn-crc32:"): + val newline = original.index_of("\n") + if newline < 0: return false + val expected = try_parse_int(original.substring(11, newline).trim()) + body = original.substring(newline + 1) + if expected == nil or crc32_text(body) != (expected ?? -1): return false + val columns: [text] = ["id", "keyword", "area", "priority", "description", "file", "line", "issue", "blocked", "status", "valid"] + val id: text = row.get("id") ?? "" + if id == "": return false + for character in id.chars(): + if not "0123456789".contains(character): return false + for column in columns: + val field = row.get(column) + if field == nil: return false + val value: text = field ?? "" + if value != value.trim() or value.contains("\n") or value.contains("\r") or value.contains("\0") or value.contains("\""): return false + val valid: text = row.get("valid") ?? "" + val status: text = row.get("status") ?? "" + if valid != "true" or status != "blocked": return false + for raw in body.split("\n"): + val line: text = raw.trim() + if line.starts_with(id + ",") or line.starts_with("\"" + id + "\","): return false + var pending: SdnTable = SdnTable.new("todos", columns) + if not pending.add_row(row): return false + val encoded: text = pending.to_sdn() + val separator = encoded.index_of("\n") + if separator < 0: return false + val header = encoded.substring(0, separator) + var rows = encoded.substring(separator + 1) + val anchored = "\n" + body + val crlf = "\n" + header + "\r\n" + val lf = "\n" + header + "\n" + val crlf_count = anchored.split(crlf).len() - 1 + val lf_count = anchored.split(lf).len() - 1 + if crlf_count + lf_count != 1: return false + var updated: text = "" + if crlf_count == 1: + rows = rows.replace("\n", "\r\n") + updated = anchored.replace(crlf, crlf + rows).substring(1) + else: + updated = anchored.replace(lf, lf + rows).substring(1) + val checksum = crc32_text(updated) + file_atomic_write(path, "#sdn-crc32:{checksum}\n" + updated) diff --git a/src/lib/nogc_sync_mut/io/process_ops.spl b/src/lib/nogc_sync_mut/io/process_ops.spl index 3a92904f3b4..eb70c40b6e9 100644 --- a/src/lib/nogc_sync_mut/io/process_ops.spl +++ b/src/lib/nogc_sync_mut/io/process_ops.spl @@ -3,6 +3,7 @@ use std.nogc_sync_mut.io.process_governor (proc_slot_acquire, proc_slot_release) use std.nogc_sync_mut.fs.host_path.{host_path_native, host_is_windows_host} +use std.nogc_sync_mut.sffi.process_capture.{process_capture_bounded_native} use std.nogc_sync_mut.io.windows_redirected_process.{WINDOWS_REDIRECTED_TOKEN_BASE, windows_redirected_spawn, windows_redirected_wait, windows_redirected_is_running, windows_redirected_kill} use std.common.process.observation_v1.{ ProcessObservationProviderV1, ObservedProcessStartStatusV1, @@ -1435,6 +1436,25 @@ pub fn process_run_bounded(cmd: text, args: [text], timeout_ms: i64, max_output_ proc_slot_release() (stdout, stderr, exit_code) +pub fn process_run_bounded_native(cmd: text, args: [text], timeout_ms: i64, max_output_bytes: i64) -> (text, text, i64): + """Native tuple compatibility entry with positive timeout/capture limits. + + The ordinary interpreter-capable process API remains separate. Resolution, + command path conversion and process-governor slots keep their real owners. + """ + if timeout_ms <= 0 or max_output_bytes <= 0: + return ("", "native bounded capture requires positive timeout and output limits", -1) + val resolved_command: text = _resolve_process_cmd(cmd + "") + val tool: (text, [text]) = _resolve_posix_tool(resolved_command, args) + val native_command: text = host_path_native(tool.0 + "") + val native_arguments: [text] = tool.1 + if native_command == "": + return ("", "native bounded capture command resolution returned empty text", -1) + proc_slot_acquire() + val result: (text, text, i64) = process_capture_bounded_native(native_command, native_arguments, timeout_ms, max_output_bytes) + proc_slot_release() + result + pub fn process_run_bounded_bytes(cmd: text, args: [text], timeout_ms: i64, max_output_bytes: i64) -> ([u8], [u8], i64): proc_slot_acquire() val result = unsafe(capabilities: [ffi]): diff --git a/src/lib/nogc_sync_mut/oauth2.spl b/src/lib/nogc_sync_mut/oauth2.spl index 15ec49cede8..c0e67b22860 100644 --- a/src/lib/nogc_sync_mut/oauth2.spl +++ b/src/lib/nogc_sync_mut/oauth2.spl @@ -10,12 +10,13 @@ # # This file does NOT depend on aws_sigv4.spl — they are separate concerns. -use std.nogc_sync_mut.http_client.{http_post, response_get_status_code, response_get_body} -use std.nogc_sync_mut.file_system.file_ops.{file_read_text, file_write_text, file_exists, file_rename, file_delete} -use std.nogc_sync_mut.file_system.permissions.{file_set_mode} +use std.nogc_sync_mut.io.http_client_core.{http_request_raw} +use std.common.http_curl.{curl_request_text} +use std.io_runtime.{platform_name} +use std.common.oauth_token_cache.{oauth_token_cache_read, oauth_token_cache_write, oauth_token_cache_render, oauth_token_cache_parse} use std.nogc_sync_mut.env.variables.{env_get} use std.nogc_sync_mut.io.time_ops.{current_time_unix} -use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_has, json_value_field, json_value_serialize} +use std.common.json.value.{JsonValue, json_value_parse_opt, json_value_has, json_value_field, json_value_serialize, json_value_as_text} use std.nogc_sync_mut.http.url.{url_encode} @@ -24,13 +25,21 @@ use std.nogc_sync_mut.http.url.{url_encode} val OAUTH2_EXPIRY_SAFETY_MARGIN: i64 = 60 # File mode for the token cache file (owner read/write only). -val OAUTH2_CACHE_FILE_MODE: i64 = 0o600 +# Decimal spelling preserves 0o600 on the pinned Phase-2 producer. +# @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Keep the exact owner-only mode in decimal until deployed P2 octal permission lowering is qualified +val OAUTH2_CACHE_FILE_MODE: i64 = 384 # ============================================================================ # Token endpoint # ============================================================================ +fn oauth2_client_credentials_request(tenant_id: text, client_id: text, client_secret: text, scope: text) -> (text, text, text, text): + val url = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" + val body = "grant_type=client_credentials&client_id={_url_form_encode(client_id)}&client_secret={_url_form_encode(client_secret)}&scope={_url_form_encode(scope)}" + ("POST", url, "Content-Type: application/x-www-form-urlencoded\nAccept: application/json", body) + + fn oauth2_client_credentials_token(tenant_id: text, client_id: text, client_secret: text, scope: text) -> (bool, text, i64): """Request an OAuth2 access token via the client-credentials grant. @@ -45,13 +54,17 @@ fn oauth2_client_credentials_token(tenant_id: text, client_id: text, client_secr if tenant_id == "" or client_id == "" or client_secret == "" or scope == "": return (false, "", 0) - val url = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" - val body = "grant_type=client_credentials&client_id={_url_form_encode(client_id)}&client_secret={_url_form_encode(client_secret)}&scope={_url_form_encode(scope)}" - val now = current_time_unix() - val response = http_post(url, body) - val status = response_get_status_code(response) - val resp_body = response_get_body(response) + val request: (text, text, text, text) = oauth2_client_credentials_request(tenant_id, client_id, client_secret, scope) + # The Windows native HTTP provider explicitly refuses requests. Use the + # shared bounded curl owner there; other hosts retain their native provider. + # @workaround bug=windows_core_http_client_unavailable_2026_10_27 reason=Use the real private-file curl provider while Windows core HTTP is unavailable + val response: (i64, text, text) = if platform_name() == "windows": + curl_request_text(request.0, request.1, request.2.split("\n"), request.3) + else: + http_request_raw(request.0, request.1, request.2, request.3) + val status = response.0 + val resp_body = response.1 if status < 200 or status >= 300: return (false, "", 0) @@ -82,59 +95,21 @@ fn oauth2_token_cache_load(path: text) -> (text, i64): malformed. No expiry check is performed here — the caller decides whether the cached value is still fresh enough. """ - if path == "" or not file_exists(path): + val content = oauth_token_cache_read(path) + if content == "": return ("", 0) - val content = file_read_text(path) - if content == nil or content == "": - return ("", 0) - - var token = "" - var expires_at: i64 = 0 - - val lines = content.split("\n") - for line in lines: - val trimmed = line.trim() - if trimmed == "" or trimmed.starts_with("#"): - pass_dn - else if trimmed.starts_with("access_token:"): - token = _strip_sdn_value(trimmed.split(":", 2)[1]) - else if trimmed.starts_with("expires_at:"): - val raw = _strip_sdn_value(trimmed.split(":", 2)[1]) - expires_at = _parse_i64(raw) - - (token, expires_at) + oauth_token_cache_parse(content) fn oauth2_token_cache_save(path: text, token: text, expires_at: i64) -> bool: - """Atomically write a token cache file with mode 0600. + """Publish a private token cache through the real atomic I/O owner. - Strategy: write temp file (siblings the target with `.tmp` suffix), - chmod 0600 BEFORE rename so there is never a window where the file - is world-readable, then rename over the target. On failure the temp - file is removed. + The shared owner creates a restrictive staging file before writing bytes; + no predictable .tmp path or demonstration permission setter is involved. """ - if path == "": - return false - - val tmp_path = "{path}.tmp" - val payload = "# itf outlook token cache (auto-managed; do not edit)\n" + - "access_token: \"{_escape_sdn_value(token)}\"\n" + - "expires_at: {expires_at}\n" - - if not file_write_text(tmp_path, payload): - return false - - # 0600 BEFORE rename so there is no world-readable window. - if not file_set_mode(tmp_path, OAUTH2_CACHE_FILE_MODE): - file_delete(tmp_path) - return false - - if not file_rename(tmp_path, path): - file_delete(tmp_path) - return false - - true + val payload = oauth_token_cache_render(token, expires_at) + oauth_token_cache_write(path, payload) # ============================================================================ @@ -247,12 +222,7 @@ fn _parse_i64(s: text) -> i64: fn _json_text_field(obj: JsonValue, key: text) -> text: - if not json_value_has(obj, key): - return "" - val s = json_value_serialize(json_value_field(obj, key)) - if s.starts_with("\"") and s.ends_with("\"") and s.len() >= 2: - return s[1:s.len() - 1] - s + json_value_as_text(json_value_field(obj, key)) ?? "" fn _json_int_field(obj: JsonValue, key: text) -> i64: diff --git a/src/lib/nogc_sync_mut/sffi/process_capture.spl b/src/lib/nogc_sync_mut/sffi/process_capture.spl new file mode 100644 index 00000000000..05b0dee1997 --- /dev/null +++ b/src/lib/nogc_sync_mut/sffi/process_capture.spl @@ -0,0 +1,21 @@ +# Native-only bounded capture ABI. The tuple provider already exists in both +# C and Rust runtimes and returns the native three-word result block. +# This is deliberately separate from the boxed interpreter process API. +@unsafe(reason: "borrowed command byte address retained for synchronous capture", capabilities: [ffi, raw_ptr]) +extern fn rt_string_data(value: T) -> i64 +@unsafe(reason: "UTF8 byte extent for the retained command", capabilities: [ffi]) +extern fn rt_string_len(value: T) -> i64 +@unsafe(reason: "existing native bounded tuple provider takes explicit command pointer and length", capabilities: [ffi, raw_ptr]) +extern fn rt_process_run_bounded_tuple(command_ptr: i64, command_len: i64, args: [text], timeout_ms: i64, max_output_bytes: i64) -> (text, text, i64) + +# @workaround bug=phase2_bounded_process_text_abi_2026_10_27 reason=Split command data and UTF8 length explicitly through the existing bounded tuple provider until the deployed P2 text ABI is qualified +fn process_capture_bounded_native(command: text, args: [text], timeout_ms: i64, max_output_bytes: i64) -> (text, text, i64): + val retained_command = command + "" + val pointer = unsafe(capabilities: [ffi, raw_ptr]): + rt_string_data(retained_command) + val length = unsafe(capabilities: [ffi]): + rt_string_len(retained_command) + if pointer == 0 or length <= 0: + return ("", "native bounded capture command pointer or length is unavailable", -1) + unsafe(capabilities: [ffi, raw_ptr]): + rt_process_run_bounded_tuple(pointer, length, args, timeout_ms, max_output_bytes) diff --git a/src/lib/nogc_sync_mut/terminal/credential/store.spl b/src/lib/nogc_sync_mut/terminal/credential/store.spl index 7ad6214d904..a3339931328 100644 --- a/src/lib/nogc_sync_mut/terminal/credential/store.spl +++ b/src/lib/nogc_sync_mut/terminal/credential/store.spl @@ -78,8 +78,12 @@ val AES_KEY_SIZE: i64 = 32 # AES-256 val AES_IV_SIZE: i64 = 16 # 128-bit IV val KDF_SALT_SIZE: i64 = 16 -val KEY_FILE_MODE: i32 = 0o600 # rw------- (secret: owner only) -val KEY_DIR_MODE: i32 = 0o700 # rwx------ +# Decimal spellings retain 0o600/0o700 with the pinned Phase-2 producer; +# its octal mode lowering does not preserve the intended OS permission mask. +# @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Preserve the intended owner-only OS masks without the pinned P2 octal lowering defect +val KEY_FILE_MODE: i32 = 384 # rw------- (secret: owner only) +# @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Preserve the intended owner-only directory mask without the pinned P2 octal lowering defect +val KEY_DIR_MODE: i32 = 448 # rwx------ # NOTE ON THE OLD FIXED SALT # diff --git a/src/runtime/platform/runtime_windows_private_atomic.h b/src/runtime/platform/runtime_windows_private_atomic.h new file mode 100644 index 00000000000..9e2da2266c3 --- /dev/null +++ b/src/runtime/platform/runtime_windows_private_atomic.h @@ -0,0 +1,81 @@ +/* Windows owner-only atomic file provider shared by core C and Rust. + * The protected owner/System DACL is attached by CREATE_NEW, before bytes + * exist. chmod after publication cannot provide this privacy contract. */ +#ifndef SIMPLE_RUNTIME_WINDOWS_PRIVATE_ATOMIC_H +#define SIMPLE_RUNTIME_WINDOWS_PRIVATE_ATOMIC_H +#if defined(_WIN32) +#include "runtime_win_long_path.h" +#include +#include +#include +#include +#include + +static inline int rt_win_private_atomic_write(const char *path, + const unsigned char *content, size_t length, int mode) { + static atomic_uint_fast64_t sequence = 0; + if (!path || !*path || (!content && length) || (mode & 0077)) return 0; + size_t capacity = strlen(path) + 64; + char *temp = (char *)malloc(capacity); + wchar_t *target = rt_win_long_path_widen(path); + if (!temp || !target) { free(temp); free(target); return 0; } + typedef BOOL (WINAPI *ConvertSddlFn)(const char *, DWORD, PSECURITY_DESCRIPTOR *, ULONG *); + HMODULE advapi = LoadLibraryA("advapi32.dll"); + ConvertSddlFn convert = advapi ? (ConvertSddlFn)GetProcAddress(advapi, + "ConvertStringSecurityDescriptorToSecurityDescriptorA") : NULL; + PSECURITY_DESCRIPTOR descriptor = NULL; + if (!convert || !convert("D:P(A;;FA;;;SY)(A;;FA;;;OW)", 1, &descriptor, NULL)) { + if (advapi) FreeLibrary(advapi); + free(temp); free(target); + return 0; + } + SECURITY_ATTRIBUTES attributes = { sizeof(attributes), descriptor, FALSE }; + int fd = -1; + wchar_t *wide = NULL; + for (int attempt = 0; attempt < 16 && fd < 0; attempt++) { + int n = snprintf(temp, capacity, "%s.tmp.%lu.%llu", path, + (unsigned long)GetCurrentProcessId(), + (unsigned long long)atomic_fetch_add(&sequence, 1)); + if (n < 0 || (size_t)n >= capacity) break; + free(wide); + wide = rt_win_long_path_widen(temp); + if (!wide) break; + HANDLE handle = CreateFileW(wide, GENERIC_WRITE, 0, &attributes, + CREATE_NEW, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OPEN_REPARSE_POINT, NULL); + if (handle == INVALID_HANDLE_VALUE) { + DWORD error = GetLastError(); + errno = error == ERROR_FILE_EXISTS || error == ERROR_ALREADY_EXISTS ? EEXIST : EACCES; + } else { + fd = _open_osfhandle((intptr_t)handle, _O_WRONLY | _O_BINARY); + if (fd < 0) { + CloseHandle(handle); + DeleteFileW(wide); + break; + } + } + if (fd < 0 && errno != EEXIST) break; + } + LocalFree(descriptor); + FreeLibrary(advapi); + int created = fd >= 0; + FILE *file = fd < 0 ? NULL : _fdopen(fd, "wb"); + int ok = file != NULL; + if (ok) ok = fwrite(content, 1, length, file) == length; + if (ok) ok = fflush(file) == 0; + if (ok) ok = _commit(fd) == 0; + if (file && fclose(file) != 0) ok = 0; + else if (!file && fd >= 0) _close(fd); + /* Windows only maps the owner-write bit to its readonly attribute; + * the DACL above independently enforces the owner-only reader set. */ + if (ok) ok = _wchmod(wide, _S_IREAD | ((mode & 0200) ? _S_IWRITE : 0)) == 0; + if (ok) ok = MoveFileExW(wide, target, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0; + if (created && !ok) { + _wchmod(wide, _S_IREAD | _S_IWRITE); + DeleteFileW(wide); + } + free(wide); free(target); free(temp); + return ok; +} +#endif +#endif diff --git a/src/runtime/runtime_hosted_fs.c b/src/runtime/runtime_hosted_fs.c index 231a0254492..a68c3a9d618 100644 --- a/src/runtime/runtime_hosted_fs.c +++ b/src/runtime/runtime_hosted_fs.c @@ -34,3 +34,21 @@ int64_t rt_remove(const char *path) { return unlink(path) == 0 ? 0 : -(int64_t)errno; #endif } + +#if defined(_WIN32) +#include "platform/runtime_windows_private_atomic.h" +/* Rust mode-provider bridge: byte-slice ABI, not tagged text handles. */ +int32_t rt_hosted_fs_private_atomic_write_utf8(const uint8_t *path, uint64_t path_len, + const uint8_t *content, uint64_t content_len, int32_t mode) { + if (!path || !path_len || path_len >= SIZE_MAX || + memchr(path, 0, (size_t)path_len) || (!content && content_len) || + content_len > SIZE_MAX) return 0; + char *terminated = (char *)malloc((size_t)path_len + 1); + if (!terminated) return 0; + memcpy(terminated, path, (size_t)path_len); + terminated[path_len] = 0; + int ok = rt_win_private_atomic_write(terminated, content, (size_t)content_len, mode); + free(terminated); + return ok; +} +#endif diff --git a/src/runtime/runtime_native.c b/src/runtime/runtime_native.c index 8e843078e67..503d57a64a5 100644 --- a/src/runtime/runtime_native.c +++ b/src/runtime/runtime_native.c @@ -27,6 +27,7 @@ const volatile int rt_alloc_observer_native_owner_v1 = 1; #include "runtime_simd_dispatch.h" #include "runtime_memory_guard.h" #include "runtime_startup_args.h" +#include "platform/runtime_windows_private_atomic.h" #include "simple_gui_html_provider_abi_v1.h" #include "simple_gui_event_provider_abi_v1.h" @@ -16175,6 +16176,14 @@ static int64_t rt_file_atomic_write_impl(int64_t path_value, int64_t content_val } } free(parent); +#if defined(_WIN32) + if (have_mode && (mode & 0077) == 0) { + int ok = rt_win_private_atomic_write(path, content_string->data, + (size_t)content_string->len, (int)mode); + free(path); + return ok ? 1 : 0; + } +#endif size_t path_len = strlen(path); char* temp_path = (char*)rt_observed_malloc_v1(path_len + 64); if (!temp_path) { @@ -16226,8 +16235,9 @@ static int64_t rt_file_atomic_write_impl(int64_t path_value, int64_t content_val #endif } #if defined(_WIN32) - /* Win32 CRT permissions express exactly one bit -- read-only -- so the - * POSIX mode is honoured to the only extent the platform allows: the + /* Owner-only modes use the protected-DACL provider above. For other + * modes retain the pre-existing CRT mapping: permissions express exactly + * one bit -- read-only -- so the POSIX mode is honoured only as follows: the * owner-write bit maps to _S_IWRITE, everything else (group/other, * setuid, sticky) has no Win32 equivalent and is silently not applied. * Applied to the TEMP file, before the move, for the same diff --git a/test/01_unit/app/devhub/auth_token_persistence_spec.spl b/test/01_unit/app/devhub/auth_token_persistence_spec.spl new file mode 100644 index 00000000000..9b029b51bed --- /dev/null +++ b/test/01_unit/app/devhub/auth_token_persistence_spec.spl @@ -0,0 +1,56 @@ +# @cover src/app/devhub/auth_token_store.spl +use std.spec.* +use std.spec.step +use std.env.{env_get} +use std.io.file_ops.{secure_temp_dir} +use std.io.dir_ops.{dir_remove_all} +use std.io_runtime.{read_file_text_result} +use app.devhub.auth_token_store.{save_auth_token_to, load_auth_token_from} + +fn auth_store_test_root() -> text: + var parent = env_get("TMPDIR") + if parent == "": parent = env_get("TEMP") + if parent == "": parent = env_get("TMP") + if parent == "": parent = "/tmp" + secure_temp_dir(parent, "devhub-auth-store").unwrap() + +describe "DevHub private auth token persistence": + it "should round trip quoted Unicode tokens and preserve other providers": + step("Write two synthetic provider credentials to the actual private file owner") + val root = auth_store_test_root() + val path = root + "/auth.sdn" + val token = "héllo 日本 \"quote\" \\path\nsecond line" + val first = save_auth_token_to(path, "jira", token) + val second = save_auth_token_to(path, "confluence.primary", "synthetic-other") + val loaded = load_auth_token_from(path, "jira") + val other = load_auth_token_from(path, "confluence.primary") + val cleaned = dir_remove_all(root) + expect(first and second and cleaned).to_equal(true) + expect(loaded).to_equal(token) + expect(other).to_equal("synthetic-other") + + it "should reject provider section injection without changing the file": + step("Attempt to insert a second section through a provider name") + val root = auth_store_test_root() + val path = root + "/auth.sdn" + val saved = save_auth_token_to(path, "jira", "synthetic-before") + val before = read_file_text_result(path).unwrap() + val rejected = save_auth_token_to(path, "jira:\nother", "synthetic-after") + val after = read_file_text_result(path).unwrap() + val cleaned = dir_remove_all(root) + expect(saved and cleaned).to_equal(true) + expect(rejected).to_equal(false) + expect(after).to_equal(before) + + it "should remove one provider and fail closed for an invalid destination": + step("Remove a token and attempt to replace a directory with a token file") + val root = auth_store_test_root() + val path = root + "/auth.sdn" + val first = save_auth_token_to(path, "jira", "synthetic-delete") + val removed = save_auth_token_to(path, "jira", "") + val loaded = load_auth_token_from(path, "jira") + val invalid = save_auth_token_to(root, "jira", "synthetic-value") + val cleaned = dir_remove_all(root) + expect(first and removed and cleaned).to_equal(true) + expect(loaded).to_equal("") + expect(invalid).to_equal(false) diff --git a/test/01_unit/app/devhub/cmd_daily_debug_spec.spl b/test/01_unit/app/devhub/cmd_daily_debug_spec.spl index 058d9ff5845..0aa860525a1 100644 --- a/test/01_unit/app/devhub/cmd_daily_debug_spec.spl +++ b/test/01_unit/app/devhub/cmd_daily_debug_spec.spl @@ -10,6 +10,8 @@ Covers: """ use std.spec +use std.io_runtime.{env_get_opt} +use std.env.{env_set, env_unset} use app.devhub.cmd_daily_debug.{ extract_jira_keys, extract_minio_urls, @@ -22,6 +24,18 @@ use app.devhub.cmd_daily_debug.{ daily_debug_rfc3339_stamp, } +fn run_without_graph_tenant(args: [text]) -> (i64, bool): + val previous = env_get_opt("GRAPH_TENANT_ID") + if not env_set("GRAPH_TENANT_ID", ""): + return (-1, false) + val code = run_daily_debug(args) + var restored = false + if val value = previous: + restored = env_set("GRAPH_TENANT_ID", value) + else: + restored = env_unset("GRAPH_TENANT_ID") + (code, restored) + describe "extract_jira_keys (issue #10 step 3)": it "extracts a single ABC-123 style key": @@ -130,19 +144,17 @@ describe "watermark roundtrip (issue #10 steps 1+6)": describe "run_daily_debug --dry-run (issue #10 AC4)": - it "exits 0 with --dry-run and writes no files": - # In --dry-run mode the driver must not call file_write_text on - # either the digest or the watermark. The outlook adapter is gated - # by `outlook_available = false` so step 2 returns []. The test - # verifies the exit code only; the absence of file writes is - # guaranteed structurally by the single `if not dry_run` gate - # around each `file_write_text` call. - val rc = run_daily_debug(["--dry-run", "--quiet"]) - expect(rc).to_equal(0) - - it "exits 0 with --dry-run alone": - val rc = run_daily_debug(["--dry-run"]) - expect(rc).to_equal(0) + it "should reject an unconfigured inbox during a quiet dry run": + step("Remove one required Graph setting and restore it before assertions") + val result: (i64, bool) = run_without_graph_tenant(["--dry-run", "--quiet"]) + expect(result.0).to_equal(1) + expect(result.1).to_equal(true) + + it "should reject an unconfigured inbox with dry-run alone": + step("Run the production validation path without a tenant") + val result: (i64, bool) = run_without_graph_tenant(["--dry-run"]) + expect(result.0).to_equal(1) + expect(result.1).to_equal(true) it "--help exits 0 and never runs the pipeline": # Regression: --help used to fall through with dry_run=false and diff --git a/test/01_unit/app/devhub/flag_positionals_spec.spl b/test/01_unit/app/devhub/flag_positionals_spec.spl new file mode 100644 index 00000000000..dbb5e7475c4 --- /dev/null +++ b/test/01_unit/app/devhub/flag_positionals_spec.spl @@ -0,0 +1,27 @@ +# @cover src/app/devhub/flags.spl +use std.spec.* +use std.spec.step +use app.devhub.flags.{_positional_args, _positional_args_with_booleans, _extract_flag, _remove_flags} + +describe "DevHub positional flag boundaries": + it "should retain positional values after valueless switches": + step("Parse a MinIO bucket following boolean display flags") + expect(_positional_args(["--bytes", "--no-pager", "bucket", "key"]).join("|")).to_equal("bucket|key") + expect(_positional_args_with_booleans(["--json", "bucket", "key"], ["--json"]).join("|")).to_equal("bucket|key") + + it "should remove value flags in separated and equals forms": + step("Extract a bucket while omitting a prefix option") + expect(_positional_args(["--prefix", "logs/", "bucket"]).join("|")).to_equal("bucket") + expect(_positional_args(["--prefix=logs/", "bucket"]).join("|")).to_equal("bucket") + + it "should preserve option-looking positional values after the terminator": + step("End option parsing before a literal object key") + expect(_positional_args(["--", "--literal", "-file"]).join("|")).to_equal("--literal|-file") + expect(_extract_flag(["--", "--prefix=literal"], "--prefix", "default")).to_equal("default") + expect(_remove_flags(["--", "--bytes", "bucket"], ["--bytes"]).join("|")).to_equal("--|--bytes|bucket") + + it "should not consume another option as a missing value": + step("Read a malformed prefix option followed by a valueless switch") + expect(_positional_args(["--prefix", "--bytes", "bucket"]).join("|")).to_equal("bucket") + expect(_extract_flag(["--prefix", "--bytes"], "--prefix", "default")).to_equal("default") + expect(_remove_flags(["--bytes", "bucket"], ["--bytes"]).join("|")).to_equal("bucket") diff --git a/test/01_unit/app/devhub/gh_json_roundtrip_spec.spl b/test/01_unit/app/devhub/gh_json_roundtrip_spec.spl new file mode 100644 index 00000000000..da6b5695e76 --- /dev/null +++ b/test/01_unit/app/devhub/gh_json_roundtrip_spec.spl @@ -0,0 +1,25 @@ +# @cover src/app/devhub/gh_compat.spl +use std.spec.* +use std.spec.step +use app.devhub.gh_compat.{bb_pr_to_gh_json, json_escape} +use std.common.json.value.{JsonValue, json_value_object_from, json_value_serialize, json_value_parse_opt, json_value_field, json_value_as_text} + +describe "DevHub backend JSON string roundtrip": + it "should preserve quoted titles and Unicode text": + step("Translate a real backend JSON string containing quotes and backslashes") + val title = "quote\" slash\\ and café" + val input = json_value_object_from([("id", JsonValue.Int(42)), ("title", JsonValue.Str(title))]) + val output = json_value_parse_opt(bb_pr_to_gh_json(json_value_serialize(input)))! + expect(json_value_as_text(json_value_field(output, "title"))!).to_equal(title) + expect(json_value_serialize(json_value_field(output, "number"))).to_equal("42") + + it "should preserve decoded newlines and tabs in descriptions": + step("Translate control escapes through the production backend adapter") + val body = "first\nsecond\ttab" + val input = json_value_object_from([("description", JsonValue.Str(body))]) + val output = json_value_parse_opt(bb_pr_to_gh_json(json_value_serialize(input)))! + expect(json_value_as_text(json_value_field(output, "body"))!).to_equal(body) + + it "should escape every JSON control character": + step("Escape low control bytes using the canonical JSON encoder") + expect(json_escape("\u{0001}\u{0008}\u{000c}")).to_equal("\\u0001\\b\\f") diff --git a/test/01_unit/app/devhub/messenger_history_rows_spec.spl b/test/01_unit/app/devhub/messenger_history_rows_spec.spl new file mode 100644 index 00000000000..2c64653fe89 --- /dev/null +++ b/test/01_unit/app/devhub/messenger_history_rows_spec.spl @@ -0,0 +1,54 @@ +# @cover src/app/devhub/messenger/adapter/history_rows.spl +use std.spec.* +use std.spec.step +use std.database.sql.types.{DbRow, DbValue} +use app.devhub.messenger.adapter.history_rows.{messenger_history_rows} + +fn history_test_row(room: text, sequence: i64, id: text) -> DbRow: + DbRow(columns: ["room", "seq", "id"], values: [DbValue.Text(room), DbValue.Integer(sequence), DbValue.Text(id)]) + +describe "Messenger history sequence ordering": + it "should select by sequence before applying the limit": + step("Read rowid order that differs from message sequence order") + val rows = [history_test_row("r", 30, "third"), history_test_row("r", 10, "first"), history_test_row("other", 0, "foreign"), history_test_row("r", 20, "second")] + match messenger_history_rows(rows, "room", "seq", "r", 2): + case Ok(selected): + expect(selected.len()).to_equal(2) + expect(selected[0].values[2].to_text()).to_equal("first") + expect(selected[1].values[2].to_text()).to_equal("second") + case Err(error): fail(error) + + it "should keep stable order for equal sequence values": + step("Select duplicate sequences and an empty room") + val rows = [history_test_row("r", 2, "a"), history_test_row("r", 2, "b"), history_test_row("r", 1, "c")] + match messenger_history_rows(rows, "room", "seq", "r", 3): + case Ok(selected): + expect(selected[0].values[2].to_text()).to_equal("c") + expect(selected[1].values[2].to_text()).to_equal("a") + expect(selected[2].values[2].to_text()).to_equal("b") + case Err(error): fail(error) + match messenger_history_rows(rows, "room", "seq", "absent", 2): + case Ok(selected): expect(selected.len()).to_equal(0) + case Err(error): fail(error) + match messenger_history_rows(rows, "room", "seq", "r", 2): + case Ok(selected): + expect(selected[0].values[2].to_text()).to_equal("c") + expect(selected[1].values[2].to_text()).to_equal("a") + case Err(error): fail(error) + + it "should reject missing or noninteger sequence values even beyond the limit": + step("Keep invalid sequence metadata from being hidden by truncation") + val invalid = DbRow(columns: ["room", "seq"], values: [DbValue.Text("r"), DbValue.Text("2")]) + expect(messenger_history_rows([history_test_row("r", 1, "a"), invalid], "room", "seq", "r", 1).is_err()).to_equal(true) + val missing = DbRow(columns: ["room"], values: [DbValue.Text("r")]) + expect(messenger_history_rows([missing], "room", "seq", "r", 2).is_err()).to_equal(true) + expect(messenger_history_rows([], "room", "seq", "r", 0).is_err()).to_equal(true) + + it "should reject malformed rows and unavailable room identifiers": + step("Reject inconsistent or null metadata before selecting a history page") + val malformed = DbRow(columns: ["room", "seq"], values: [DbValue.Text("r")]) + expect(messenger_history_rows([malformed], "room", "seq", "r", 2).is_err()).to_equal(true) + val missing = DbRow(columns: ["seq"], values: [DbValue.Integer(1)]) + expect(messenger_history_rows([missing], "room", "seq", "r", 2).is_err()).to_equal(true) + val null_room = DbRow(columns: ["room", "seq"], values: [DbValue.Null, DbValue.Integer(1)]) + expect(messenger_history_rows([null_room], "room", "seq", "NULL", 2).is_err()).to_equal(true) diff --git a/test/01_unit/app/devhub/messenger_json_rendering_spec.spl b/test/01_unit/app/devhub/messenger_json_rendering_spec.spl new file mode 100644 index 00000000000..bf8c978d602 --- /dev/null +++ b/test/01_unit/app/devhub/messenger_json_rendering_spec.spl @@ -0,0 +1,23 @@ +# @cover src/app/devhub/messenger/adapter/db_common.spl +use std.spec.* +use std.spec.step +use app.devhub.messenger.adapter.db_common.{_json_str} +use std.common.json.value.{json_value_parse_opt, json_value_as_text} + +describe "Messenger JSON string rendering": + it "should escape tabs and line controls as valid JSON": + step("Render message text containing JSON control characters") + val subject = "first\tsecond\nthird\rfourth" + expect(_json_str(subject)).to_equal("\"first\\tsecond\\nthird\\rfourth\"") + + it "should retain Unicode and escape quotes and backslashes": + step("Round trip message text through the actual JSON renderer and parser") + val subject = "héllo 日本 \"quoted\" \\path" + val parsed = json_value_parse_opt(_json_str(subject)) + match parsed: + case Some(value): expect(json_value_as_text(value).unwrap()).to_equal(subject) + case nil: fail("message renderer emitted invalid JSON") + + it "should render an empty message as a JSON string": + step("Keep an empty message distinct from a missing JSON value") + expect(_json_str("")).to_equal("\"\"") diff --git a/test/01_unit/app/devhub/minio_cli_values_spec.spl b/test/01_unit/app/devhub/minio_cli_values_spec.spl new file mode 100644 index 00000000000..b937fed8d6b --- /dev/null +++ b/test/01_unit/app/devhub/minio_cli_values_spec.spl @@ -0,0 +1,38 @@ +# @cover src/app/devhub/minio_options.spl +# @cover src/app/devhub/minio_output.spl +use std.spec.* +use std.spec.step +use app.devhub.minio_options.{minio_parse_expiry} +use app.devhub.minio_output.{minio_stat_json, minio_health_json} +use std.common.json.value.{json_value_parse_opt, json_value_field, json_value_as_text, json_value_serialize} + +describe "MinIO CLI expiry and JSON contracts": + it "should admit exact presign expiry boundaries": + step("Read the minimum, default and maximum seconds") + expect(minio_parse_expiry("1")!).to_equal(1) + expect(minio_parse_expiry("86400")!).to_equal(86400) + expect(minio_parse_expiry("604800")!).to_equal(604800) + + it "should reject malformed, zero, negative and overflowing expiry values": + step("Refuse input rather than stripping nondigits or wrapping arithmetic") + expect(minio_parse_expiry("")).to_be_nil() + expect(minio_parse_expiry("0")).to_be_nil() + expect(minio_parse_expiry("-60")).to_be_nil() + expect(minio_parse_expiry("6oops0")).to_be_nil() + expect(minio_parse_expiry("604801")).to_be_nil() + expect(minio_parse_expiry("999999999999999999999999")).to_be_nil() + + it "should encode object keys and ETags as JSON strings": + step("Serialize object metadata containing JSON delimiters") + val key = "quote\" slash\\ newline\n" + val output = json_value_parse_opt(minio_stat_json(key, 4, "etag\"", "now"))! + expect(json_value_as_text(json_value_field(output, "key"))!).to_equal(key) + expect(json_value_as_text(json_value_field(output, "etag"))!).to_equal("etag\"") + expect(json_value_serialize(json_value_field(output, "size"))).to_equal("4") + + it "should encode failed health details without corrupting JSON": + step("Serialize a failing endpoint with multiline diagnostics") + val output = json_value_parse_opt(minio_health_json(false, "fail\nreason", 2, "http://host/\""))! + expect(json_value_as_text(json_value_field(output, "status"))!).to_equal("fail\nreason") + expect(json_value_as_text(json_value_field(output, "endpoint"))!).to_equal("http://host/\"") + expect(json_value_serialize(json_value_field(output, "alive"))).to_equal("false") diff --git a/test/01_unit/app/devhub/oauth_form_encoding_spec.spl b/test/01_unit/app/devhub/oauth_form_encoding_spec.spl index e676e335760..f438f2d3fb5 100644 --- a/test/01_unit/app/devhub/oauth_form_encoding_spec.spl +++ b/test/01_unit/app/devhub/oauth_form_encoding_spec.spl @@ -4,7 +4,7 @@ use std.spec.* use std.spec.step use app.devhub.adapter_outlook_curl.{_url_form_encode as outlook_form_encode} -use std.nogc_sync_mut.oauth2.{_url_form_encode as oauth_form_encode} +use std.oauth2.{_url_form_encode as oauth_form_encode} describe "OAuth form value encoding": it "should encode UTF-8 octets consistently in both production clients": diff --git a/test/01_unit/app/devhub/real_file_workflows_spec.spl b/test/01_unit/app/devhub/real_file_workflows_spec.spl new file mode 100644 index 00000000000..d73e0b56624 --- /dev/null +++ b/test/01_unit/app/devhub/real_file_workflows_spec.spl @@ -0,0 +1,158 @@ +# @req REQ-DEVHUB-REAL-FILES +# @cover src/app/devhub/editor.spl +# @cover src/app/devhub/backend_resolve.spl +# @cover src/app/devhub/cmd_daily_debug.spl +# @cover src/app/devhub/cmd_auth.spl +# @cover src/app/devhub/cmd_lifecycle.spl +use std.spec.* +use std.env.{env_get} +use std.io.file_ops.{secure_temp_dir} +use std.io.dir_ops.{dir_create_all, dir_remove_all, is_dir} +use std.io_runtime.{file_write, file_exists, read_file_text_result} +use app.devhub.editor.{prepare_editor_file, cleanup_editor_file} +use app.devhub.backend_resolve.{load_repo_devhub_config_from, repo_parent_directory, find_repo_root_from} +use app.devhub.cmd_daily_debug.{save_watermark_atomic_at, render_watermark, daily_debug_inbox_filter} +use app.devhub.cmd_auth.{ensure_auth_config_dir} +use app.devhub.cmd_lifecycle.{lifecycle_source_files} + +fn workflow_test_workspace() -> text: + var root = env_get("TMPDIR") + if root == "": root = env_get("TEMP") + if root == "": root = env_get("TMP") + if root == "" and is_dir("/tmp"): root = "/tmp" + match secure_temp_dir(root, "devhub-real-files"): + case Ok(path): path.replace("\\", "/") + case Err(reason): + fail(reason) + "" + +describe "DevHub actual file workflows": + it "should bound an inbox read by the timestamp captured before fetching": + step("Build one closed-open watermark interval") + val filter = daily_debug_inbox_filter("2026-10-26T00:00:00Z", "2026-10-27T00:00:00Z") + expect(filter).to_equal("receivedDateTime ge 2026-10-26T00:00:00Z and receivedDateTime lt 2026-10-27T00:00:00Z") + + it "should include the previous upper boundary in the next inbox read": + step("Start the next interval exactly where the preceding query stopped") + val filter = daily_debug_inbox_filter("2026-10-27T00:00:00Z", "2026-10-27T00:00:01Z") + expect(filter).to_start_with("receivedDateTime ge 2026-10-27T00:00:00Z") + expect(filter).to_end_with("receivedDateTime lt 2026-10-27T00:00:01Z") + + it "should represent an unchanged watermark as an empty interval": + step("Use one timestamp for both bounds without widening the query") + val stamp = "2026-10-27T00:00:00Z" + expect(daily_debug_inbox_filter(stamp, stamp)).to_equal("receivedDateTime ge 2026-10-27T00:00:00Z and receivedDateTime lt 2026-10-27T00:00:00Z") + + it "should prepare distinct private editor files with exact Unicode and empty content": + step("Create two edit workspaces below one explicit temporary parent") + val root = workflow_test_workspace() + val first = prepare_editor_file(root, "héllo 日本", ".md").unwrap() + val second = prepare_editor_file(root, "", ".html").unwrap() + val first_text = read_file_text_result(first.1) + val second_text = read_file_text_result(second.1) + val distinct = first.0 != second.0 and first.1 != second.1 + val cleaned_first = cleanup_editor_file(first.0) + val cleaned_second = cleanup_editor_file(second.0) + val clean_root = dir_remove_all(root) + step("Check real bytes and independent workspace ownership after cleanup") + expect(first_text.unwrap()).to_equal("héllo 日本") + expect(second_text.unwrap()).to_equal("") + expect(distinct).to_equal(true) + expect(cleaned_first and cleaned_second and clean_root).to_equal(true) + + it "should reject a missing edit parent before launching an editor": + step("Attempt to prepare an editor file below an absent directory") + val root = workflow_test_workspace() + val result = prepare_editor_file(root + "/absent", "body", ".md") + val cleaned = dir_remove_all(root) + expect(result.is_err()).to_equal(true) + expect(cleaned).to_equal(true) + + it "should reject an editor suffix that could escape the private workspace": + step("Pass an unsupported suffix and inspect the explicit error") + val root = workflow_test_workspace() + val result = prepare_editor_file(root, "body", "/../escaped") + val escaped = file_exists(root + "/escaped") + val cleaned = dir_remove_all(root) + expect(result.is_err()).to_equal(true) + expect(escaped).to_equal(false) + expect(cleaned).to_equal(true) + + it "should load a repository backend from real nested files": + step("Create a repository config and a nested working directory") + val root = workflow_test_workspace() + val made = dir_create_all(root + "/.spipe") and dir_create_all(root + "/a/b") + val wrote = file_write(root + "/.spipe/config.sdn", "devhub:\n git_backend: bitbucket\n") + val found = find_repo_root_from(root + "/a/b") + val config = load_repo_devhub_config_from(root) + val cleaned = dir_remove_all(root) + expect(made and wrote and cleaned).to_equal(true) + expect(found).to_equal(root) + expect(config.git_backend).to_equal("bitbucket") + + it "should distinguish absent repository config from real empty content": + step("Load a missing file and then an actual empty configuration") + val root = workflow_test_workspace() + val missing = load_repo_devhub_config_from(root) + val made = dir_create_all(root + "/.spipe") + val wrote = file_write(root + "/.spipe/config.sdn", "") + val empty = load_repo_devhub_config_from(root) + val cleaned = dir_remove_all(root) + expect(made and wrote and cleaned).to_equal(true) + expect(missing.git_backend).to_equal("") + expect(empty.git_backend).to_equal("") + + it "should walk Windows and Unix parent directories without crossing a root": + step("Normalize native separators and stop at filesystem roots") + expect(repo_parent_directory("C:\\repo\\nested")).to_equal("C:/repo") + expect(repo_parent_directory("C:/repo")).to_equal("C:/") + expect(repo_parent_directory("C:/")).to_equal("") + expect(repo_parent_directory("//server/share/repo")).to_equal("//server/share") + expect(repo_parent_directory("//server/share")).to_equal("") + expect(repo_parent_directory("/repo/nested")).to_equal("/repo") + expect(repo_parent_directory("/")).to_equal("") + + it "should atomically persist a watermark without a predictable sibling staging file": + step("Write two successive actual watermark versions") + val root = workflow_test_workspace() + val path = root + "/watermark.sdn" + val first = save_watermark_atomic_at(path, "2026-10-26T00:00:00Z") + val second = save_watermark_atomic_at(path, "2026-10-27T00:00:00Z") + val content = read_file_text_result(path) + val predictable = file_exists(path + ".tmp") + val cleaned = dir_remove_all(root) + expect(first and second and cleaned).to_equal(true) + expect(content.unwrap()).to_equal(render_watermark("2026-10-27T00:00:00Z")) + expect(predictable).to_equal(false) + + it "should report watermark write failure for a missing parent": + step("Attempt an atomic write to an absent directory") + val root = workflow_test_workspace() + val wrote = save_watermark_atomic_at(root + "/absent/watermark.sdn", "2026-10-27T00:00:00Z") + val cleaned = dir_remove_all(root) + expect(wrote).to_equal(false) + expect(cleaned).to_equal(true) + + it "should create an auth directory with the native provider and reject a file collision": + step("Create a real nested directory and a conflicting regular file") + val root = workflow_test_workspace() + val created = ensure_auth_config_dir(root + "/config/devhub") + val directory = is_dir(root + "/config/devhub") + val wrote = file_write(root + "/occupied", "do not replace") + val refused = ensure_auth_config_dir(root + "/occupied/child") + val original = read_file_text_result(root + "/occupied") + val cleaned = dir_remove_all(root) + expect(created and directory and wrote and cleaned).to_equal(true) + expect(refused).to_equal(false) + expect(original.unwrap()).to_equal("do not replace") + + it "should discover real nested lifecycle consumers instead of mock filenames": + step("Create version consumers under a private source root") + val root = workflow_test_workspace() + val made = dir_create_all(root + "/nested") + val wrote = file_write(root + "/nested/consumer.spl", "val VERSION = \"1.0.0\"\n") + val files = lifecycle_source_files(root) + val cleaned = dir_remove_all(root) + expect(made and wrote and cleaned).to_equal(true) + expect(files).to_contain(root + "/nested/consumer.spl") + expect(files.contains(root + "/nested/example.txt")).to_equal(false) diff --git a/test/01_unit/lib/common/http_curl_admission_spec.spl b/test/01_unit/lib/common/http_curl_admission_spec.spl new file mode 100644 index 00000000000..08138a78ffe --- /dev/null +++ b/test/01_unit/lib/common/http_curl_admission_spec.spl @@ -0,0 +1,24 @@ +# @cover src/lib/common/http_curl.spl +use std.spec.* +use std.spec.step +use std.common.http_curl.{curl_request_text} + +describe "Bounded curl transport admission": + it "should refuse non-HTTP URLs before starting a process": + step("Submit an unsupported local-file protocol") + val result = curl_request_text("POST", "file:///not-http", [], "ignored") + expect(result.0).to_equal(0) + expect(result.1).to_equal("") + expect(result.2).to_contain("http or https") + + it "should refuse headers that introduce an extra line": + step("Submit a header containing a CRLF delimiter") + val result = curl_request_text("POST", "http://127.0.0.1:1", ["X-Value: safe\r\nInjected: yes"], "ignored") + expect(result.0).to_equal(0) + expect(result.2).to_contain("control delimiter") + + it "should refuse URL userinfo so credentials never enter argv": + step("Submit an HTTP URL with a userinfo component") + val result = curl_request_text("POST", "https://user:fixture@127.0.0.1/", [], "ignored") + expect(result.0).to_equal(0) + expect(result.2).to_contain("userinfo") diff --git a/test/01_unit/lib/compression/crc32_unsigned_i64_spec.spl b/test/01_unit/lib/compression/crc32_unsigned_i64_spec.spl new file mode 100644 index 00000000000..8e170bcce38 --- /dev/null +++ b/test/01_unit/lib/compression/crc32_unsigned_i64_spec.spl @@ -0,0 +1,25 @@ +# @cover src/lib/nogc_sync_mut/compression/gzip/crc.spl +use std.spec.* +use std.nogc_sync_mut.compression.gzip.crc.{crc32_calculate, crc32_text, crc32_update, crc32_finalize} +use std.common.text_bytes.{text_to_utf8_bytes} + +describe "CRC32 unsigned words in i64 carriers": + it "should return the independent standard text and byte known answer": + step("Calculate the CRC-32/ISO-HDLC check value and the empty input") + expect(crc32_text("123456789")).to_equal(3421780262) + expect(crc32_calculate(text_to_utf8_bytes("123456789"))).to_equal(3421780262) + expect(crc32_text("")).to_equal(0) + + it "should retain a high-bit-set streaming register across chunks": + step("Continue from the independent pre-final CRC of abc") + val first = crc32_update(4294967295, text_to_utf8_bytes("abc")) + val second = crc32_update(first, text_to_utf8_bytes("def")) + expect(first).to_equal(3403398717) + expect(second).to_equal(3027355152) + expect(crc32_finalize(second)).to_equal(1267612143) + + it "should preserve a valid register for nil input and finalize its low word": + step("Use the no-op input contract and the all-ones initial register") + expect(crc32_update(3403398717, nil)).to_equal(3403398717) + expect(crc32_finalize(4294967295)).to_equal(0) + expect(crc32_finalize(-1)).to_equal(0) diff --git a/test/01_unit/lib/database/bug_append_preserving_spec.spl b/test/01_unit/lib/database/bug_append_preserving_spec.spl new file mode 100644 index 00000000000..1db843e45de --- /dev/null +++ b/test/01_unit/lib/database/bug_append_preserving_spec.spl @@ -0,0 +1,77 @@ +# @cover src/lib/nogc_sync_mut/database/bug.spl +use std.spec.* +use std.env.{env_get} +use std.io.file_ops.{secure_temp_dir} +use std.io.dir_ops.{dir_remove_all} +use std.io_runtime.{file_copy, file_exists, file_write, read_file_text_result} +use std.nogc_sync_mut.database.bug.{Bug, BugSeverity, BugStatus, create_bug_database, load_bug_database} + +fn append_bug_workspace() -> text: + var parent = env_get("TMPDIR") + if parent == "": parent = env_get("TEMP") + if parent == "": parent = env_get("TMP") + if parent == "": parent = "/tmp" + secure_temp_dir(parent, "bug-append-test").unwrap() + +fn append_bug_record(id: text) -> Bug: + Bug(id: id, severity: BugSeverity.P2, status: BugStatus.Open, + title: "Append preserving regression", description: ["new description"], + file: "src/lib/nogc_sync_mut/database/bug.spl", line: 1, + reproducible_by: "test/01_unit/lib/database/bug_append_preserving_spec.spl", + fix_strategy: ["preserve all original bytes"], investigation_log: ["regression control"], + created_at: "2026-10-27", updated_at: "2026-10-27", valid: true) + +describe "Canonical BugDatabase append preserving legacy records": + it "should append a new open bug without losing opaque fields or Unicode": + step("Copy a legacy registry with an extra field and CRLF metadata") + val root = append_bug_workspace() + val path = root + "/registry.sdn" + val copied = file_copy("test/fixtures/database/bug_append_legacy/opaque.sdn.fixture", path) + var database = create_bug_database(path) + val appended = database.append_open_bugs_preserving_legacy([append_bug_record("new_preserving_record")]) + val content = read_file_text_result(path).unwrap() + val reopened = load_bug_database(path) + val lock_left = file_exists(path + ".lock") + val cleaned = dir_remove_all(root) + step("Check exact opaque payload and canonical new record after release") + expect(copied and appended and cleaned).to_equal(true) + expect(content).to_contain("true, EXTRA_UNPARSED_保持\r\n") + expect(content).to_contain(" retained_opaque, 0, Preserve this exact field\r\n") + expect(reopened!.get_bug("new_preserving_record") != nil).to_equal(true) + expect(lock_left).to_equal(false) + + it "should reject an invalid checksum without rewriting any bytes": + step("Copy a deliberately invalid CRC input and retain its original text") + val root = append_bug_workspace() + val path = root + "/registry.sdn" + val copied = file_copy("test/fixtures/database/bug_append_legacy/invalid_crc.sdn.fixture", path) + val before = read_file_text_result(path).unwrap() + var database = create_bug_database(path) + val appended = database.append_open_bugs_preserving_legacy([append_bug_record("rejected_crc_record")]) + val after = read_file_text_result(path).unwrap() + val lock_left = file_exists(path + ".lock") + val cleaned = dir_remove_all(root) + expect(copied and cleaned).to_equal(true) + expect(appended).to_equal(false) + expect(after).to_equal(before) + expect(lock_left).to_equal(false) + + it "should refuse pending WAL and duplicate records without discarding either": + step("Keep an authoritative pending WAL and an existing opaque ID") + val root = append_bug_workspace() + val path = root + "/registry.sdn" + val copied = file_copy("test/fixtures/database/bug_append_legacy/opaque.sdn.fixture", path) + val before = read_file_text_result(path).unwrap() + var database = create_bug_database(path) + val duplicate = database.append_open_bugs_preserving_legacy([append_bug_record("retained_opaque")]) + val wal_written = file_write(path + ".wal", "pending authoritative transaction") + val appended = database.append_open_bugs_preserving_legacy([append_bug_record("rejected_wal_record")]) + val after = read_file_text_result(path).unwrap() + val wal = read_file_text_result(path + ".wal").unwrap() + val lock_left = file_exists(path + ".lock") + val cleaned = dir_remove_all(root) + expect(copied and wal_written and cleaned).to_equal(true) + expect(duplicate or appended).to_equal(false) + expect(after).to_equal(before) + expect(wal).to_equal("pending authoritative transaction") + expect(lock_left).to_equal(false) diff --git a/test/01_unit/lib/oauth_token_cache_spec.spl b/test/01_unit/lib/oauth_token_cache_spec.spl new file mode 100644 index 00000000000..6d656f33bac --- /dev/null +++ b/test/01_unit/lib/oauth_token_cache_spec.spl @@ -0,0 +1,28 @@ +# @cover src/lib/common/oauth_token_cache.spl +use std.spec.* +use std.common.oauth_token_cache.{oauth_token_cache_render, oauth_token_cache_parse} + +describe "OAuth cache canonical SDN payload": + it "should round trip Unicode quotes backslashes and control characters": + step("Render an opaque token containing escaped SDN delimiters") + val token = "café日本🙂 \"quote\" \\slash\nexpires_at: 1\t\r" + val payload = oauth_token_cache_render(token, 4102444800) + step("Parse the canonical SDN document without interpreting token contents as fields") + val parsed: (text, i64) = oauth_token_cache_parse(payload) + expect(parsed.0).to_equal(token) + expect(parsed.1).to_equal(4102444800) + + it "should reject duplicate token fields rather than select a winner": + step("Parse a cache with two conflicting access token fields") + val parsed: (text, i64) = oauth_token_cache_parse("access_token: first\naccess_token: second\nexpires_at: 4102444800\n") + expect(parsed.0).to_equal("") + expect(parsed.1).to_equal(0) + + it "should reject malformed or incorrectly typed cache documents": + step("Parse a scalar root and an incorrectly typed token") + val scalar: (text, i64) = oauth_token_cache_parse("not_a_cache") + val wrong_type: (text, i64) = oauth_token_cache_parse("access_token: 42\nexpires_at: 4102444800\n") + expect(scalar.0).to_equal("") + expect(scalar.1).to_equal(0) + expect(wrong_type.0).to_equal("") + expect(wrong_type.1).to_equal(0) diff --git a/test/01_unit/lib/terminal/credential_upgrade_persist_spec.spl b/test/01_unit/lib/terminal/credential_upgrade_persist_spec.spl index 679295929a6..8a4965f8f28 100644 --- a/test/01_unit/lib/terminal/credential_upgrade_persist_spec.spl +++ b/test/01_unit/lib/terminal/credential_upgrade_persist_spec.spl @@ -1,3 +1,5 @@ +# Mode checks use decimal masks so they test OS semantics on the pinned P2 +# producer rather than repeating its known octal-mode lowering failure. # The v1->v2 credential upgrade PERSISTER, and the mode-carrying write path # that secret-bearing files land through. # @@ -210,6 +212,7 @@ describe "P3 failure leaves the original readable": expect(_write_key(key_path)).to_equal(true) expect(credential_decrypt(v1, key_path)).to_equal("must-survive") + # @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Use exact decimal OS masks so the regression does not repeat the pinned P2 octal lowering defect it "leaves the ORIGINAL byte-intact when the atomic write itself fails": step("Verify: leaves the ORIGINAL byte-intact when the atomic write itself fails") # The case above only proves the NO-OP path (nothing was ever written). @@ -223,7 +226,7 @@ describe "P3 failure leaves the original readable": val key_path = "/tmp/simple_cred_persist_p3ro.key" expect(_write_key(key_path)).to_equal(true) rt_ensure_dir(dir) - rt_package_chmod(dir, 0o700) + rt_package_chmod(dir, 448) val secret = "survives-a-failed-write" val v1 = _v1_record(secret) @@ -237,7 +240,7 @@ describe "P3 failure leaves the original readable": expect(credential_upgrade_record(v1, key_path).starts_with("encrypted:v2:")).to_equal(true) # Now take away write permission on the directory and retry. - rt_package_chmod(dir, 0o500) + rt_package_chmod(dir, 320) expect(credential_upgrade_file(cfg, key_path)).to_equal(-1) # oracle: authoritative expected value documented by this spec's contract # The original file is byte-for-byte intact, and still decrypts. @@ -245,7 +248,7 @@ describe "P3 failure leaves the original readable": expect(credential_decrypt(v1, key_path)).to_equal(secret) # Restore so the tree is not left with an unwritable temp dir. - rt_package_chmod(dir, 0o700) + rt_package_chmod(dir, 448) it "reports -1 for a file that does not exist and creates nothing": step("Verify: reports -1 for a file that does not exist and creates nothing") @@ -260,25 +263,28 @@ describe "P3 failure leaves the original readable": describe "P4 secret files land at the requested mode": + # @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Use exact decimal OS masks so the regression does not repeat the pinned P2 octal lowering defect it "creates the file at 0600 with no wider window, verified on disk": step("Verify: creates the file at 0600 with no wider window, verified on disk") # rt_file_mode is a real stat, not chmod's self-report. Before this # primitive existed there was no way to assert this at all. val p = "/tmp/simple_cred_persist_p4_600.txt" rt_file_remove(p) - expect(rt_file_atomic_write_mode(p, "SECRET", 0o600)).to_equal(true) - expect(rt_file_mode(p)).to_equal(0o600) + expect(rt_file_atomic_write_mode(p, "SECRET", 384)).to_equal(true) + expect(rt_file_mode(p)).to_equal(384) expect(rt_file_read_text(p)).to_equal("SECRET") + # @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Use exact decimal OS masks so the regression does not repeat the pinned P2 octal lowering defect it "honours a mode that is NOT 0600 rather than hardcoding one": step("Verify: honours a mode that is NOT 0600 rather than hardcoding one") # Sabotage guard: a primitive that always chmod'd 0600 would pass the # case above while being wrong. val p = "/tmp/simple_cred_persist_p4_644.txt" rt_file_remove(p) - expect(rt_file_atomic_write_mode(p, "PLAIN", 0o644)).to_equal(true) - expect(rt_file_mode(p)).to_equal(0o644) + expect(rt_file_atomic_write_mode(p, "PLAIN", 420)).to_equal(true) + expect(rt_file_mode(p)).to_equal(420) + # @workaround bug=phase2_octal_permission_mode_lowering_2026_10_27 reason=Use exact decimal OS masks so the regression does not repeat the pinned P2 octal lowering defect it "NARROWS an existing permissive file instead of inheriting its mode": step("Verify: NARROWS an existing permissive file instead of inheriting its mode") # rt_file_atomic_write (no mode) copies the EXISTING target's @@ -286,10 +292,10 @@ describe "P4 secret files land at the requested mode": # file would keep it 0644. The mode argument must override that. val p = "/tmp/simple_cred_persist_p4_narrow.txt" rt_file_remove(p) - expect(rt_file_atomic_write_mode(p, "was-public", 0o644)).to_equal(true) - expect(rt_file_mode(p)).to_equal(0o644) - expect(rt_file_atomic_write_mode(p, "now-secret", 0o600)).to_equal(true) - expect(rt_file_mode(p)).to_equal(0o600) + expect(rt_file_atomic_write_mode(p, "was-public", 420)).to_equal(true) + expect(rt_file_mode(p)).to_equal(420) + expect(rt_file_atomic_write_mode(p, "now-secret", 384)).to_equal(true) + expect(rt_file_mode(p)).to_equal(384) expect(rt_file_read_text(p)).to_equal("now-secret") it "reports -1 for the mode of a file that does not exist": diff --git a/test/01_unit/runtime/windows_private_atomic_write_test.c b/test/01_unit/runtime/windows_private_atomic_write_test.c new file mode 100644 index 00000000000..4143d7bf3df --- /dev/null +++ b/test/01_unit/runtime/windows_private_atomic_write_test.c @@ -0,0 +1,70 @@ +#include "src/runtime/platform/runtime_windows_private_atomic.h" +#if defined(_WIN32) +#include +#include +#include + +#define CHECK(test) do { if (!(test)) { fprintf(stderr, "private atomic check failed at line %d: %s (winerr=%lu)\n", __LINE__, #test, (unsigned long)GetLastError()); return 1; } } while (0) +static int private_acl(const wchar_t *path) { + PSECURITY_DESCRIPTOR descriptor = NULL; + PACL dacl = NULL; + DWORD result = GetNamedSecurityInfoW((LPWSTR)path, SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, NULL, NULL, &dacl, NULL, &descriptor); + if (result != ERROR_SUCCESS || !descriptor || !dacl) return 0; + SECURITY_DESCRIPTOR_CONTROL control = 0; + DWORD revision = 0; + int ok = GetSecurityDescriptorControl(descriptor, &control, &revision) && + (control & SE_DACL_PROTECTED) != 0 && dacl->AceCount > 0; + for (DWORD i = 0; ok && i < dacl->AceCount; i++) { + ACCESS_ALLOWED_ACE *ace = NULL; + LPWSTR sid = NULL; + ok = GetAce(dacl, i, (LPVOID *)&ace) && ace->Header.AceType == ACCESS_ALLOWED_ACE_TYPE && + !(ace->Header.AceFlags & INHERITED_ACE) && + ConvertSidToStringSidW((PSID)&ace->SidStart, &sid); + if (ok) ok = wcscmp(sid, L"S-1-3-4") == 0 || wcscmp(sid, L"S-1-5-18") == 0; + if (sid) LocalFree(sid); + } + LocalFree(descriptor); + return ok; +} +int main(void) { + char parent[MAX_PATH], directory[MAX_PATH], path[MAX_PATH]; + CHECK(GetTempPathA(sizeof(parent), parent) != 0); + CHECK(snprintf(directory, sizeof(directory), "%sdevhub-atomic-acl-%lu", parent, + (unsigned long)GetCurrentProcessId()) > 0); + CHECK(CreateDirectoryA(directory, NULL)); + CHECK(snprintf(path, sizeof(path), "%s\\token.txt", directory) > 0); + /* Replace a pre-existing inherited-ACL file, not just a fresh target. */ + FILE *inherited = fopen(path, "wb"); + CHECK(inherited != NULL && fputs("old public value", inherited) >= 0 && fclose(inherited) == 0); + CHECK(rt_win_private_atomic_write(path, (const unsigned char *)"first synthetic token", strlen("first synthetic token"), 0600) == 1); + wchar_t *wide = rt_widen_long_path_rc(path); + CHECK(wide != NULL); + CHECK(private_acl(wide)); + CHECK(rt_win_private_atomic_write(path, (const unsigned char *)"second synthetic token", strlen("second synthetic token"), 0600) == 1); + CHECK(private_acl(wide)); + FILE *file = fopen(path, "rb"); + CHECK(file != NULL); + char bytes[64] = {0}; + CHECK(fread(bytes, 1, sizeof(bytes), file) == strlen("second synthetic token")); + CHECK(fclose(file) == 0 && strcmp(bytes, "second synthetic token") == 0); + CHECK(rt_win_private_atomic_write(path, (const unsigned char *)"refused", 7, 0644) == 0); + CHECK(rt_win_private_atomic_write("", (const unsigned char *)"refused", 7, 0600) == 0); + CHECK(rt_win_private_atomic_write(directory, (const unsigned char *)"refused", 7, 0600) == 0); + char pattern[MAX_PATH]; + snprintf(pattern, sizeof(pattern), "%s.tmp.*", path); + WIN32_FIND_DATAA found; + HANDLE search = FindFirstFileA(pattern, &found); + CHECK(search == INVALID_HANDLE_VALUE); + snprintf(pattern, sizeof(pattern), "%s.tmp.*", directory); + search = FindFirstFileA(pattern, &found); + CHECK(search == INVALID_HANDLE_VALUE); + CHECK(DeleteFileW(wide)); + free(wide); + CHECK(RemoveDirectoryA(directory)); + puts("PASS actual C atomic mode0600 protected ACL, replacement, content, refusal, cleanup"); + return 0; +} +#else +int main(void) { return 77; } +#endif diff --git a/test/01_unit/tools/mail_cli/fixtures/header_case_probe.shs b/test/01_unit/tools/mail_cli/fixtures/header_case_probe.shs new file mode 100644 index 00000000000..ef75f528f5c --- /dev/null +++ b/test/01_unit/tools/mail_cli/fixtures/header_case_probe.shs @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Production MIME parser regression; no network or alternate parser. +set -euo pipefail +source "${1:?mail-cli library directory required}/format.bash" + +raw=$'FROM: sender@example.test\r\nTO: reader@example.test\r\nSUBJECT: Mixed case\r\nDATE: Mon, 1 Jan 2024 12:00:00 +0000\r\nMESSAGE-ID: \r\nIN-REPLY-TO: \r\nREFERENCES: \r\n\r\nbody' +headers=$(mail_format_parse_headers "$raw") +test "$(jq -r .from <<< "$headers")" = sender@example.test +test "$(jq -r .to <<< "$headers")" = reader@example.test +test "$(jq -r .subject <<< "$headers")" = 'Mixed case' +test "$(jq -r .message_id <<< "$headers")" = '' +test "$(jq -r .in_reply_to <<< "$headers")" = '' +test "$(jq -r .references <<< "$headers")" = '' +printf '%s\n' 'mail_header_case_insensitive_fields: PASS' + +raw=$'From: sender@example.test\nSubject: folded\n\tcontinuation\n\nContent-Transfer-Encoding: base64\nSGVsbG8=' +test "$(mail_format_parse_body "$raw")" = $'Content-Transfer-Encoding: base64\nSGVsbG8=' +headers=$(mail_format_parse_headers "$raw") +test "$(jq -r .subject <<< "$headers")" = 'folded continuation' +printf '%s\n' 'mail_body_header_text_is_not_metadata: PASS' + +raw=$'From: sender@example.test\nContent-Transfer-Encoding: base64\n\nSGVsbG8=' +test "$(mail_format_parse_body "$raw")" = Hello +printf '%s\n' 'mail_actual_transfer_encoding_preserved: PASS' diff --git a/test/01_unit/tools/mail_cli/fixtures/pop3_credentials_probe.shs b/test/01_unit/tools/mail_cli/fixtures/pop3_credentials_probe.shs index 2cde3ca9975..ed2a5e919f7 100644 --- a/test/01_unit/tools/mail_cli/fixtures/pop3_credentials_probe.shs +++ b/test/01_unit/tools/mail_cli/fixtures/pop3_credentials_probe.shs @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Host-fixture evidence only: real client, synthetic curl and credential helper. # Never claims transport interoperability or real encryption correctness. +# SDN groups additionally require MAIL_CONFIG_BIN to select the real compiled +# canonical parser; the encryption fixture never substitutes for that parser. set -euo pipefail umask 077 root=$(cd "$(dirname "$0")/../../../../.." && pwd) @@ -88,6 +90,13 @@ call() { status() { [ "$rc" = "$1" ] || die "exit expected $1, got $rc ($2)"; } change() { jq "$1" "$config" > "$work/config"; mv "$work/config" "$config"; } group="${1:-all}" +case "$group" in + all|config|placeholders) + if [ -z "${MAIL_CONFIG_BIN:-}" ]; then + printf '%s\n' 'error: SDN scenarios require a real compiled MAIL_CONFIG_BIN; the synthetic credential helper does not parse configuration' >&2 + exit 2 + fi ;; +esac if [ "$group" = all ] || [ "$group" = protocol ]; then call inbox --json --limit 1 diff --git a/test/01_unit/tools/mail_cli/mail_header_case_spec.spl b/test/01_unit/tools/mail_cli/mail_header_case_spec.spl new file mode 100644 index 00000000000..b711c102334 --- /dev/null +++ b/test/01_unit/tools/mail_cli/mail_header_case_spec.spl @@ -0,0 +1,14 @@ +# @req REQ-MAIL-HEADER-001 +# @cover tools/mail-cli/lib/format.bash +use std.spec.* +use std.process.{run} + +describe "Mail production header and body parsing": + it "should parse case insensitive fields and keep body text outside metadata": + step("Run the production mail formatter against disposable RFC messages") + val result = run("bash", ["test/01_unit/tools/mail_cli/fixtures/header_case_probe.shs", "tools/mail-cli/lib"]) + step("Check actual field values, body preservation, and declared transfer decoding") + expect(result.exit_code).to_equal(0) + expect(result.stdout).to_contain("mail_header_case_insensitive_fields: PASS") + expect(result.stdout).to_contain("mail_body_header_text_is_not_metadata: PASS") + expect(result.stdout).to_contain("mail_actual_transfer_encoding_preserved: PASS") diff --git a/test/03_system/app/devhub/feature/minio_binary_put_spec.spl b/test/03_system/app/devhub/feature/minio_binary_put_spec.spl new file mode 100644 index 00000000000..f11a78bffb0 --- /dev/null +++ b/test/03_system/app/devhub/feature/minio_binary_put_spec.spl @@ -0,0 +1,45 @@ +# @cover src/app/devhub/adapter_minio.spl +# codex-system-test +# @evidence-display: links +use std.spec.* +use std.spec.step +use std.env.env_get +use std.process.run + +describe "DevHub binary object upload": + it "should preserve binary bytes and authenticate real loopback PUT requests": + step("Select the real compiled production-provider fixture and Python checker") + val binary = env_get("DEVHUB_MINIO_BINARY_BIN") + val python = env_get("DEVHUB_PYTHON_BIN") + expect(binary == "").to_equal(false) + expect(python == "").to_equal(false) + step("Send binary, file and empty-body uploads to the independent loopback checker") + val result = run(python, ["test/fixtures/devhub/native_minio_binary_put/verify_loopback.py", binary]) + step("Check exact bytes, hashes, signatures and missing-file refusal") + expect(result.exit_code).to_equal(0) + expect(result.stdout).to_contain("PASS native MinIO binary, file, empty and missing-file contracts") + expect(result.stdout).to_contain("PASS independent loopback byte-exact payload, SHA256, SigV4 and request-count checks") + + it "should retain failure status when health output is JSON": + step("Select the source-matched DevHub executable and Python checker") + val binary = env_get("DEVHUB_SYSTEM_BIN") + val python = env_get("DEVHUB_PYTHON_BIN") + expect(binary == "").to_equal(false) + expect(python == "").to_equal(false) + step("Run JSON health against a real loopback HTTP 503 response") + val result = run(python, ["test/fixtures/devhub/native_minio_binary_put/verify_health_cli.py", binary]) + step("Check the command returns failure together with valid JSON") + expect(result.exit_code).to_equal(0) + expect(result.stdout).to_contain("PASS real DevHub unhealthy JSON result preserves nonzero exit") + + it "should send exact form and Unicode payloads through the real compatibility transport": + step("Select the compiled common transport fixture and Python checker") + val binary = env_get("DEVHUB_HTTP_CURL_BIN") + val python = env_get("DEVHUB_PYTHON_BIN") + expect(binary == "").to_equal(false) + expect(python == "").to_equal(false) + step("Run actual form and UTF-8 POST requests against a local HTTP checker") + val result = run(python, ["test/fixtures/devhub/native_http_curl/verify_loopback.py", binary]) + step("Check exact wire bytes and rejection of header injection") + expect(result.exit_code).to_equal(0) + expect(result.stdout).to_contain("PASS independent real POST form/UTF-8 and zero-request negative controls") diff --git a/test/fixtures/database/bug_append_legacy/invalid_crc.sdn.fixture b/test/fixtures/database/bug_append_legacy/invalid_crc.sdn.fixture new file mode 100644 index 00000000000..39653aa0778 --- /dev/null +++ b/test/fixtures/database/bug_append_legacy/invalid_crc.sdn.fixture @@ -0,0 +1,15 @@ +#sdn-crc32:0 +strings |id, value| + +bugs_active |id, severity, status, title, file, line, reproducible_by, created_at, updated_at, valid| + retained_opaque, P2, open, Keep this legacy payload, sentinel.spl, 0, N/A, 2026-10-01, 2026-10-01, true, EXTRA_UNPARSED_?? + +bugs |id, severity, status, title, file, line, reproducible_by, created_at, updated_at, valid| + +bug_descriptions |bug_id, line_num, content| + retained_opaque, 0, Preserve this exact field + +bug_investigation_logs |bug_id, line_num, content| + +bug_fix_strategies |bug_id, line_num, content| + diff --git a/test/fixtures/database/bug_append_legacy/opaque.sdn.fixture b/test/fixtures/database/bug_append_legacy/opaque.sdn.fixture new file mode 100644 index 00000000000..35574196df6 --- /dev/null +++ b/test/fixtures/database/bug_append_legacy/opaque.sdn.fixture @@ -0,0 +1,15 @@ +#sdn-crc32:2589356519 +strings |id, value| + +bugs_active |id, severity, status, title, file, line, reproducible_by, created_at, updated_at, valid| + retained_opaque, P2, open, Keep this legacy payload, sentinel.spl, 0, N/A, 2026-10-01, 2026-10-01, true, EXTRA_UNPARSED_?? + +bugs |id, severity, status, title, file, line, reproducible_by, created_at, updated_at, valid| + +bug_descriptions |bug_id, line_num, content| + retained_opaque, 0, Preserve this exact field + +bug_investigation_logs |bug_id, line_num, content| + +bug_fix_strategies |bug_id, line_num, content| + diff --git a/test/fixtures/devhub/native_auth_tokens/main.spl b/test/fixtures/devhub/native_auth_tokens/main.spl new file mode 100644 index 00000000000..b5a07c33429 --- /dev/null +++ b/test/fixtures/devhub/native_auth_tokens/main.spl @@ -0,0 +1,30 @@ +# Supplemental real auth.sdn owner checks, never a replacement SSpec runner. +use std.env.{env_get} +use std.io_runtime.{file_delete, read_file_text_result} +use app.devhub.auth_token_store.{save_auth_token_to, load_auth_token_from} + +fn main() -> i64: + val root = env_get("DEVHUB_AUTH_PROBE_ROOT") + if root == "": return 1 + val path = root + "/auth.sdn" + val synthetic = "héllo 日本 \"quote\" \\path\nsecond line" + assert(save_auth_token_to(path, "jira", synthetic)) + assert(save_auth_token_to(path, "confluence.primary", "synthetic-other")) + assert(load_auth_token_from(path, "jira") == synthetic) + assert(load_auth_token_from(path, "confluence.primary") == "synthetic-other") + print "PASS actual auth token Unicode/escape roundtrip and other-provider preservation" + val before: text = match read_file_text_result(path): + case Ok(value): value + case Err(_): return 1 + assert(not save_auth_token_to(path, "jira:\nother", "synthetic-injection")) + val after: text = match read_file_text_result(path): + case Ok(value): value + case Err(_): return 1 + assert(before == after) + assert(not save_auth_token_to(root, "jira", "synthetic-directory")) + assert(save_auth_token_to(path, "jira", "")) + assert(load_auth_token_from(path, "jira") == "") + assert(load_auth_token_from(path, "confluence.primary") == "synthetic-other") + assert(file_delete(path)) + print "PASS provider injection, invalid destination, deletion and cleanup guards" + 0 diff --git a/test/fixtures/devhub/native_bug_workaround_registration/main.spl b/test/fixtures/devhub/native_bug_workaround_registration/main.spl new file mode 100644 index 00000000000..9d199fc4c9b --- /dev/null +++ b/test/fixtures/devhub/native_bug_workaround_registration/main.spl @@ -0,0 +1,95 @@ +# Canonical BugDatabase API registration utility. Run against a copied root first. +use std.env.{env_get} +use std.nogc_sync_mut.database.bug.{Bug, BugDatabase, BugSeverity, BugStatus, create_bug_database} +use std.nogc_async_mut.database.bug.{BugDatabase as SnapshotBugDatabase} +use app.bug.workaround_store.{workaround_bug_snapshot} +use std.nogc_sync_mut.compression.gzip.crc.{crc32_calculate, crc32_text, crc32_update, crc32_finalize} +use std.common.text_bytes.{text_to_utf8_bytes} + +fn main() -> i64: + if crc32_text("") != 0: return 3 + if crc32_text("123456789") != 3421780262: return 3 + if crc32_calculate(text_to_utf8_bytes("123456789")) != 3421780262: return 3 + val first: i64 = crc32_update(4294967295, text_to_utf8_bytes("abc")) + if first != 3403398717: return 3 + val second: i64 = crc32_update(first, text_to_utf8_bytes("def")) + if second != 3027355152 or crc32_finalize(second) != 1267612143: return 3 + print "PASS actual unsigned CRC32 KAT, empty input and chunked high-bit register" + val root = env_get("WORKAROUND_REGISTRY_ROOT") + if root == "": return 2 + var database: BugDatabase = create_bug_database(root + "/doc/08_tracking/bug/bug_db.sdn") + var records: [Bug] = [] + records.push(Bug( + id: "phase2_octal_permission_mode_lowering_2026_10_27", severity: BugSeverity.P1, status: BugStatus.Open, + title: "Pinned Phase 2 lowers octal permission modes incorrectly", description: ["Pinned P2 octal-mode source produced read-only final and staging files. The staging file contained the correct replacement token and expiry, so this failure was not a cache parser failure. Decimal 384 is exactly mode0600 and decimal448 is mode0700."], + file: "src/lib/common/oauth_token_cache.spl", line: 1, reproducible_by: "test/fixtures/devhub/native_outlook_token_cache/main.spl", + fix_strategy: ["Remove the source workaround only after the deployed producer proves octal and decimal OS modes identical in actual create, replace and permission checks; require credential and OAuth cache regressions. Do not close this underlying compiler bug from workaround success."], + investigation_log: ["Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Cache unchanged regression compiled and ran exit0 after decimal-mode workaround; artifact e285b95b2f4250817eec5555aee4d67fea0f1f299d2d3d3568381ae8f9b044fd. Earlier actual filesystem inspection observed both files readonly. The exact compiler lowering cause is not yet isolated. Workaround commit pending integration; underlying bug remains open."], + created_at: "2026-10-27", updated_at: "2026-10-27", valid: true)) + records.push(Bug( + id: "phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27", severity: BugSeverity.P1, status: BugStatus.Open, + title: "Pinned Phase 2 byte carrier is rejected by accelerated SHA256 FFI", description: ["Native binary upload failed with an invalid array handle before any loopback request. The accelerated u8 SHA256 boundary is incompatible with this producer. The workaround copies octets by index into the established i64 SHA256 engine without changing the public byte contract."], + file: "src/app/devhub/adapter_minio.spl", line: 1, reproducible_by: "test/fixtures/devhub/native_minio_binary_put/main.spl", + fix_strategy: ["Retire after this deployed producer and linked crypto provider pass exact octets 0,128,255 plus empty, Unicode byte sequences and independent SHA256 goldens through the accelerated ABI. Keep the byte-exact upload and SigV4 oracle."], + investigation_log: ["Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Initial native compile0 run1 artifact3adb13de091efe809b6a571ced270f697ee30de9d36d33939692a758dcbaaa8d; independent server observed zero requests. Final typed-carrier transport qualification is recorded in the parent verification report and is not assumed here. Workaround commit pending integration; underlying bug remains open."], + created_at: "2026-10-27", updated_at: "2026-10-27", valid: true)) + records.push(Bug( + id: "windows_core_http_client_unavailable_2026_10_27", severity: BugSeverity.P1, status: BugStatus.Open, + title: "Windows core runtime native HTTP client provider is unavailable", description: ["The Windows core native HTTP branch explicitly returns unavailable. A shared real curl compatibility transport uses bounded execution, private mode384 header/body/response files, no redirects and no TLS bypass; request secrets are absent from argv. Applications retain configured public std interfaces."], + file: "src/runtime/runtime_native.c", line: 1, reproducible_by: "test/fixtures/devhub/native_http_curl/main.spl", + fix_strategy: ["Retire only after the selected Windows native provider supports text and exact binary bodies, status and errors, size and time bounds, redirects/TLS policy, secret handling and independent HTTP/SigV4 acceptance with the deployed compiler. An installed curl workaround does not close the missing provider bug."], + investigation_log: ["Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Source-confirmed Windows provider gap. Actual curl8.21.0 exists on the qualification host. HTTP text and signed binary loopback acceptance remains separately bound to the parent qualification artifacts; source review alone is not runtime PASS. Workaround commit pending integration; underlying bug remains open."], + created_at: "2026-10-27", updated_at: "2026-10-27", valid: true)) + records.push(Bug( + id: "phase2_result_tuple_try_binding_metadata_2026_10_27", severity: BugSeverity.P1, status: BugStatus.Open, + title: "Pinned Phase 2 loses integer and byte tuple metadata across Result propagation", description: ["Explicit typed Ok and Err bindings preserve u64 and byte-array tuple payload metadata where native Result propagation did not. Related compiler Result tuple repairs are in source, but the pinned producer has not been rebuilt and the compatibility bindings remain."], + file: "src/lib/nogc_sync_mut/database/sqlite_format/cell.spl", line: 1, reproducible_by: "test/fixtures/devhub/native_result_unsigned_tuple/main.spl", + fix_strategy: ["Retire after the actual deployed producer passes try-propagated u64 maxima and byte arrays plus the full SQLite parser and production Messenger read/wrong-key/tamper regressions. Keep the underlying bug open until then."], + investigation_log: ["Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Earlier Messenger compilation admitted all169 MIR modules after explicit SQLite tuple bindings. Supplemental unsigned varint and truncation checks passed. Production Messenger adapter RTS01 remains unresolved; this workaround is not proof that the database read works. Workaround commit pending integration; underlying bug remains open."], + created_at: "2026-10-27", updated_at: "2026-10-27", valid: true)) + records.push(Bug( + id: "phase2_bounded_process_text_abi_2026_10_27", severity: BugSeverity.P1, status: BugStatus.Open, + title: "Pinned Phase 2 bounded process call loses command text ABI", description: ["The actual HTTP compatibility probe returned missing command although curl8.21.0 was installed. The existing bounded tuple runtime provider accepts explicit command pointer and UTF8 length; a source adapter can preserve that ABI without changing the compiler or provider."], + file: "src/lib/nogc_sync_mut/sffi/process_capture.spl", line: 1, reproducible_by: "test/fixtures/devhub/native_http_curl/main.spl", + fix_strategy: ["Remove the source adapter only after the deployed producer correctly marshals command text, argv, deadline and output bound to the normal bounded process ABI and passes real quoted-argument, Unicode, timeout, output-limit and HTTP transport regressions."], + investigation_log: ["Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Frozen-source HTTP and binary probes compiled but returned missing command before any localhost request. The exact tuple-provider source adapter and final actual loopback evidence are owned by the provider lane; no success is inferred by this record. Workaround commit pending integration; underlying bug remains open."], + created_at: "2026-10-27", updated_at: "2026-10-27", valid: true)) + records.push(Bug( + id: "phase2_module_static_array_concat_initialization_2026_10_27", severity: BugSeverity.P1, status: BugStatus.Open, + title: "Pinned Phase 2 cannot initialize DevHub module static arrays by concatenation", description: ["The compatibility flag catalogs retain literal typed arrays instead of executing array concatenation while initializing module statics. Their public values remain unchanged. This is narrower than ordinary function-local array concatenation, which remains supported by other focused checks."], + file: "src/app/devhub/gh_compat.spl", line: 1, reproducible_by: "test/01_unit/app/devhub/gh_compat_spec.spl", + fix_strategy: ["Retire only after actual module-static concatenation preserves every flag value and method/type metadata under the deployed producer, with native command flag regressions and no startup evaluation error."], + investigation_log: ["Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. The retained literal tables were part of the earlier DevHub P2 source repair. Current GH JSON and flag native graphs compile and run successfully, but this is not a new independent replay of the original static-initializer failure. Related older registry issue native_array_concat_dispatch_bootstrap_2026_10_16 remains separate. Workaround commit pending integration; underlying bug remains open."], + created_at: "2026-10-27", updated_at: "2026-10-27", valid: true)) + records.push(Bug( + id: "bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27", severity: BugSeverity.P1, status: BugStatus.Open, + title: "Bug database full save truncates opaque legacy row payloads", description: ["The canonical registry contains legacy rows with more fields than their declared table schema. The generic table parser and full save do not preserve every overwide legacy row payload, so a complete BugDatabase.save round trip can discard untouched records or fields. The domain append-only API serializes only new records and preserves the complete original body byte for byte apart from inserted canonical rows."], + file: "src/lib/nogc_sync_mut/database/bug.spl", line: 1, reproducible_by: "test/fixtures/devhub/native_bug_workaround_registration/main.spl", + fix_strategy: ["Retire the legacy append compatibility path only after all legacy rows are repaired by their owners or the canonical serializer losslessly retains opaque fields, and a complete scratch round trip preserves every original payload byte and record. Never normalize or truncate records merely to enable registration."], + investigation_log: ["Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Read-only baseline audit found eleven versus ten fields at lines2452,2453,2463 and additional overwide rows. No real registry write or generic regeneration was performed. Append-only native copy qualification and independent byte preservation must pass before actual registration. Workaround commit pending integration; underlying bug remains open."], + created_at: "2026-10-27", updated_at: "2026-10-27", valid: true)) + records.push(Bug( + id: "phase2_crc32_i64_width_spill_2026_10_27", severity: BugSeverity.P1, status: BugStatus.Open, + title: "Pinned Phase 2 CRC32 i64 boundary retains unexpected high bits", description: ["A native append-only registry copy preserved all original payload bytes but wrote CRC header -2292158518 while independent unsigned zlib computed2002808778. The values differ by2^32. This proves a CRC width or ABI boundary discrepancy; the exact widening or complement cause is not isolated. Masking public i64 CRC outputs to their low32bits preserves the actual CRC register contract."], + file: "src/lib/nogc_sync_mut/compression/gzip/crc.spl", line: 1, reproducible_by: "test/fixtures/devhub/native_bug_workaround_registration/main.spl", + fix_strategy: ["Retire the explicit width normalization only after the deployed producer passes text123456789 CRC3421780262, empty input, chunked update through high-bit-set states and independent unsigned zlib comparisons for real registry bodies. Never accept negative header CRC or relax the external oracle."], + investigation_log: ["Producer P2 SHA256 13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Cycle3 canonical API compile0 and copy-run0 were not passing evidence because the independent strict CRC oracle failed. Exact old-body preservation passed after removing only28newrowlines, including all5265oldrows and20opaqueoverwide rows. Real canonical registry remained untouched. New actual KAT and copy qualification are required before registration. Workaround commit pending integration; underlying bug remains open."], + created_at: "2026-10-27", updated_at: "2026-10-27", valid: true)) + if not database.append_open_bugs_preserving_legacy(records): + print "canonical append-only BugDatabase API refused registration" + return 1 + val checked: SnapshotBugDatabase = match workaround_bug_snapshot(root): + case Ok(value): value + case Err(error): + print "saved registry validation refused: {error}" + return 1 + if checked.get_bug("phase2_octal_permission_mode_lowering_2026_10_27") == nil: return 1 + if checked.get_bug("phase2_u8_accelerated_sha256_ffi_carrier_2026_10_27") == nil: return 1 + if checked.get_bug("windows_core_http_client_unavailable_2026_10_27") == nil: return 1 + if checked.get_bug("phase2_result_tuple_try_binding_metadata_2026_10_27") == nil: return 1 + if checked.get_bug("phase2_bounded_process_text_abi_2026_10_27") == nil: return 1 + if checked.get_bug("phase2_module_static_array_concat_initialization_2026_10_27") == nil: return 1 + if checked.get_bug("bug_database_roundtrip_drops_overwide_legacy_rows_2026_10_27") == nil: return 1 + if checked.get_bug("phase2_crc32_i64_width_spill_2026_10_27") == nil: return 1 + print "canonical append-only BugDatabase API registration completed" + 0 diff --git a/test/fixtures/devhub/native_editor_files/main.spl b/test/fixtures/devhub/native_editor_files/main.spl new file mode 100644 index 00000000000..cdddfbba588 --- /dev/null +++ b/test/fixtures/devhub/native_editor_files/main.spl @@ -0,0 +1,42 @@ +# Phase-2 production-owner check; these assertions are not SSpec case counts. +use std.env.{env_get} +use std.io_runtime.{read_file_text_result, file_exists} +use app.devhub.editor_file.{prepare_editor_file, cleanup_editor_file} + +fn main() -> i64: + # The caller supplies a private existing parent. Never touch real HOME. + val parent = env_get("DEVHUB_EDITOR_PROBE_PARENT") + if parent == "": + print "FAIL explicit private DEVHUB_EDITOR_PROBE_PARENT required" + return 1 + val first: (text, text) = match prepare_editor_file(parent, "héllo 日本 🙂", ".md"): + case Ok(value): value + case Err(reason): + print "FAIL first editor preparation: {reason}" + return 1 + val second: (text, text) = match prepare_editor_file(parent, "", ".html"): + case Ok(value): value + case Err(reason): + val _cleaned = cleanup_editor_file(first.0) + print "FAIL second editor preparation: {reason}" + return 1 + val first_text = read_file_text_result(first.1) + val second_text = read_file_text_result(second.1) + val distinct = first.0 != second.0 and first.1 != second.1 + val invalid = prepare_editor_file(parent, "must not escape", "/../escaped") + val absent = prepare_editor_file(first.0 + "/missing", "must not create parents", ".md") + val escaped = file_exists(parent + "/escaped") + val clean_first = cleanup_editor_file(first.0) + val clean_second = cleanup_editor_file(second.0) + assert(clean_first and clean_second) + assert(not file_exists(first.0) and not file_exists(second.0)) + assert(distinct) + assert(invalid.is_err() and absent.is_err() and not escaped) + match first_text: + case Ok(value): assert(value == "héllo 日本 🙂") + case Err(_): assert(false) + match second_text: + case Ok(value): assert(value == "") + case Err(_): assert(false) + print "PASS actual editor Unicode and empty payloads, distinct workspaces, rejected suffix/missing parent, cleanup" + 0 diff --git a/test/fixtures/devhub/native_flag_positionals/main.spl b/test/fixtures/devhub/native_flag_positionals/main.spl new file mode 100644 index 00000000000..7118a0c1fe6 --- /dev/null +++ b/test/fixtures/devhub/native_flag_positionals/main.spl @@ -0,0 +1,30 @@ +use app.devhub.flags.{_positional_args, _positional_args_with_booleans, _extract_flag, _remove_flags} +use app.devhub.minio_options.{minio_parse_expiry} +use app.devhub.minio_output.{minio_stat_json, minio_health_json} +use std.common.json.value.{json_value_parse_opt, json_value_field, json_value_as_text} + +fn main() -> i64: + assert(_positional_args(["--bytes", "bucket", "key"]).join("|") == "bucket|key") + assert(_positional_args(["--no-pager", "--prefix", "logs/", "bucket"]).join("|") == "bucket") + assert(_positional_args(["--prefix=logs/", "bucket"]).join("|") == "bucket") + assert(_positional_args(["--", "--literal", "-file"]).join("|") == "--literal|-file") + assert(_positional_args(["--prefix", "--bytes", "bucket"]).join("|") == "bucket") + assert(_extract_flag(["--prefix", "--bytes"], "--prefix", "default") == "default") + assert(_extract_flag(["--", "--prefix=literal"], "--prefix", "default") == "default") + assert(_remove_flags(["--bytes", "bucket"], ["--bytes"]).join("|") == "bucket") + assert(_remove_flags(["--", "--bytes", "bucket"], ["--bytes"]).join("|") == "--|--bytes|bucket") + assert(_positional_args_with_booleans(["--json", "bucket", "key"], ["--json"]).join("|") == "bucket|key") + assert(minio_parse_expiry("1")! == 1) + assert(minio_parse_expiry("604800")! == 604800) + assert(minio_parse_expiry("-60") == nil) + assert(minio_parse_expiry("6oops0") == nil) + assert(minio_parse_expiry("604801") == nil) + assert(minio_parse_expiry("999999999999999999999999") == nil) + val key = "quote\" slash\\ newline\n" + val stat = json_value_parse_opt(minio_stat_json(key, 4, "etag\"", "now"))! + assert(json_value_as_text(json_value_field(stat, "key"))! == key) + assert(json_value_as_text(json_value_field(stat, "etag"))! == "etag\"") + val health = json_value_parse_opt(minio_health_json(false, "fail\nreason", 2, "http://host/\""))! + assert(json_value_as_text(json_value_field(health, "status"))! == "fail\nreason") + print "PASS native boolean, valued, equals and terminator flag contracts" + 0 diff --git a/test/fixtures/devhub/native_gh_json_roundtrip/main.spl b/test/fixtures/devhub/native_gh_json_roundtrip/main.spl new file mode 100644 index 00000000000..78a695b44dc --- /dev/null +++ b/test/fixtures/devhub/native_gh_json_roundtrip/main.spl @@ -0,0 +1,19 @@ +use app.devhub.gh_compat.{bb_pr_to_gh_json, json_escape} +use std.common.json.value.{JsonValue, json_value_object_from, json_value_serialize, json_value_parse_opt, json_value_field, json_value_as_text} + +fn main() -> i64: + val title = "quote\" slash\\ and unicode café" + val body = "first\nsecond\ttab" + val original = json_value_object_from([ + ("id", JsonValue.Int(42)), ("title", JsonValue.Str(title)), + ("description", JsonValue.Str(body)), ("state", JsonValue.Str("OPEN")) + ]) + val converted = bb_pr_to_gh_json(json_value_serialize(original)) + val parsed = json_value_parse_opt(converted) + assert(parsed != nil) + assert(json_value_as_text(json_value_field(parsed!, "title"))! == title) + assert(json_value_as_text(json_value_field(parsed!, "body"))! == body) + assert(json_value_serialize(json_value_field(parsed!, "number")) == "42") + assert(json_escape("\u{0001}\u{0008}\u{000c}") == "\\u0001\\b\\f") + print "PASS native gh JSON decoded-string and control-byte roundtrip" + 0 diff --git a/test/fixtures/devhub/native_http_curl/main.spl b/test/fixtures/devhub/native_http_curl/main.spl new file mode 100644 index 00000000000..595e53a4885 --- /dev/null +++ b/test/fixtures/devhub/native_http_curl/main.spl @@ -0,0 +1,38 @@ +use std.common.http_curl.{curl_request_text, curl_request_text_with_program} +use std.env.{env_get} +use std.io_runtime.{time_now_unix_micros} + +fn main() -> i64: + val endpoint = env_get("DEVHUB_HTTP_LOOPBACK_URL") + assert(endpoint != "") + val form: (i64, text, text) = curl_request_text_with_program("curl", "POST", endpoint + "/form", + ["Content-Type: application/x-www-form-urlencoded"], "client_secret=caf%C3%A9&scope=x%20y") + print "native form status={form.0} error={form.2}" + assert(form.0 == 200) + assert(form.1 == "form accepted") + assert(form.2 == "") + val utf8: (i64, text, text) = curl_request_text("POST", endpoint + "/utf8", + ["Content-Type: text/plain; charset=utf-8"], "café日本🙂") + print "native UTF-8 status={utf8.0} error={utf8.2}" + assert(utf8.0 == 200) + assert(utf8.1 == "utf8 accepted") + val redirect: (i64, text, text) = curl_request_text("POST", endpoint + "/redirect", [], "not forwarded") + assert(redirect.0 == 302) + val get: (i64, text, text) = curl_request_text("GET", endpoint + "/empty-get", ["Authorization: Bearer fixture-token"], "") + assert(get.0 == 200 and get.1 == "get accepted") + val oversized: (i64, text, text) = curl_request_text("POST", endpoint + "/oversized", [], "") + assert(oversized.0 == 0 and oversized.2 != "") + val started = time_now_unix_micros() + val stalled: (i64, text, text) = curl_request_text("POST", endpoint + "/stall", [], "") + val elapsed_ms = (time_now_unix_micros() - started) / 1000 + print "native stalled HTTP elapsed_ms={elapsed_ms}" + assert(stalled.0 == 0 and stalled.2 != "") + assert(elapsed_ms >= 25000 and elapsed_ms < 45000) + val injection: (i64, text, text) = curl_request_text("POST", endpoint + "/rejected", + ["Content-Type: text/plain\r\nX-Injected: yes"], "ignored") + assert(injection.0 == 0 and injection.2 != "") + val bad_scheme: (i64, text, text) = curl_request_text("POST", "file:///not-http", [], "ignored") + assert(bad_scheme.0 == 0 and bad_scheme.2 != "") + print "PASS native curl form, UTF-8, header-injection and URL admission" + print "PASS native HTTP response-size and elapsed-time bounds" + 0 diff --git a/test/fixtures/devhub/native_http_curl/verify_loopback.py b/test/fixtures/devhub/native_http_curl/verify_loopback.py new file mode 100644 index 00000000000..dd93619d468 --- /dev/null +++ b/test/fixtures/devhub/native_http_curl/verify_loopback.py @@ -0,0 +1,105 @@ +"""Check the actual Simple curl transport on an independent local HTTP server.""" +import http.server +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import threading +import time + + +def main(): + binary = Path(sys.argv[1]).resolve(strict=True) + attempts = [] + failures = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + attempts.append((self.path, 0)) + try: + assert self.path == "/empty-get" + assert self.headers.get("Content-Type") is None + assert self.headers.get("Content-Length") in (None, "0") + assert self.headers["Authorization"] == "Bearer fixture-token" + reply = b"get accepted" + self.send_response(200) + self.send_header("Content-Length", str(len(reply))) + self.end_headers() + self.wfile.write(reply) + except Exception as exc: + failures.append(str(exc)) + self.send_response(400) + self.send_header("Content-Length", "0") + self.end_headers() + + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + attempts.append((self.path, len(body))) + try: + if self.path == "/oversized": + self.send_response(200) + self.send_header("Content-Length", str(1048577)) + self.end_headers() + return + if self.path == "/stall": + time.sleep(32) + return + if self.path == "/redirect": + self.send_response(302) + self.send_header("Location", "/must-not-follow") + self.send_header("Content-Length", "0") + self.end_headers() + return + if self.path == "/form": + assert self.headers["Content-Type"] == "application/x-www-form-urlencoded" + assert body == b"client_secret=caf%C3%A9&scope=x%20y" + reply = b"form accepted" + else: + assert self.path == "/utf8" + assert self.headers["Content-Type"] == "text/plain; charset=utf-8" + assert body == "café日本🙂".encode("utf-8") + reply = b"utf8 accepted" + self.send_response(200) + self.send_header("Content-Length", str(len(reply))) + self.end_headers() + self.wfile.write(reply) + except Exception as exc: + failures.append(str(exc)) + self.send_response(400) + self.send_header("Content-Length", "0") + self.end_headers() + + with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with tempfile.TemporaryDirectory(prefix="devhub-http-staging-") as temporary: + root = Path(temporary) + env = os.environ.copy() + env.update(DEVHUB_HTTP_LOOPBACK_URL=f"http://127.0.0.1:{server.server_port}", + TMPDIR=str(root), TEMP=str(root), TMP=str(root)) + started = time.monotonic() + result = subprocess.run([str(binary)], env=env, timeout=150, capture_output=True, text=True) + elapsed = time.monotonic() - started + print(result.stdout, end="") + print(result.stderr, end="", file=sys.stderr) + print(f"loopback native_exit={result.returncode} request_attempts={attempts} failures={failures}") + assert result.returncode == 0, f"native transport exited {result.returncode}" + assert not failures, failures + assert [path for path, _ in attempts] == ["/form", "/utf8", "/redirect", "/empty-get", "/oversized", "/stall"], attempts + assert 25 <= elapsed < 55, f"bounded native HTTP run took {elapsed:.3f}s" + assert not list(root.iterdir()), "private HTTP staging was not cleaned" + print("PASS independent real POST form/UTF-8 and zero-request negative controls") + print("PASS redirect refusal and private staging cleanup") + print(f"PASS independent size refusal and deadline ({elapsed:.3f}s)") + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/test/fixtures/devhub/native_messenger_history_rows/main.spl b/test/fixtures/devhub/native_messenger_history_rows/main.spl new file mode 100644 index 00000000000..23ac9e3b505 --- /dev/null +++ b/test/fixtures/devhub/native_messenger_history_rows/main.spl @@ -0,0 +1,47 @@ +# Supplemental production-helper evidence, not the Messenger database or SSpec suite. +use std.database.sql.types.{DbRow, DbValue} +use app.devhub.messenger.adapter.history_rows.{messenger_history_rows} + +fn row(room: text, sequence: i64, id: text) -> DbRow: + DbRow(columns: ["room", "seq", "id"], values: [DbValue.Text(room), DbValue.Integer(sequence), DbValue.Text(id)]) + +fn main() -> i64: + val source = [row("r", 30, "third"), row("r", 10, "first"), row("other", 0, "foreign"), row("r", 20, "second")] + match messenger_history_rows(source, "room", "seq", "r", 2): + case Ok(selected): + assert(selected.len() == 2) + assert(selected[0].values[2].to_text() == "first") + assert(selected[1].values[2].to_text() == "second") + case Err(_): return 1 + print "PASS sequence selection before limit and room isolation" + match messenger_history_rows([row("r", 2, "a"), row("r", 2, "b"), row("r", 1, "c")], "room", "seq", "r", 2): + case Ok(selected): + assert(selected.len() == 2) + assert(selected[0].values[2].to_text() == "c") + assert(selected[1].values[2].to_text() == "a") + case Err(_): return 1 + print "PASS stable equal-sequence eviction at cutoff" + val invalid = DbRow(columns: ["room", "seq"], values: [DbValue.Text("r"), DbValue.Text("2")]) + match messenger_history_rows([row("r", 1, "ok"), invalid], "room", "seq", "r", 1): + case Err(_): () + case Ok(_): return 1 + val malformed = DbRow(columns: ["room", "seq"], values: [DbValue.Text("r")]) + match messenger_history_rows([malformed], "room", "seq", "r", 2): + case Err(_): () + case Ok(_): return 1 + val missing = DbRow(columns: ["seq"], values: [DbValue.Integer(1)]) + match messenger_history_rows([missing], "room", "seq", "r", 2): + case Err(_): () + case Ok(_): return 1 + val null_room = DbRow(columns: ["room", "seq"], values: [DbValue.Null, DbValue.Integer(1)]) + match messenger_history_rows([null_room], "room", "seq", "NULL", 2): + case Err(_): () + case Ok(_): return 1 + match messenger_history_rows([], "room", "seq", "r", 0): + case Err(_): () + case Ok(_): return 1 + match messenger_history_rows(source, "room", "seq", "absent", 2): + case Ok(selected): assert(selected.len() == 0) + case Err(_): return 1 + print "PASS invalid metadata, invalid limit and empty room guards" + 0 diff --git a/test/fixtures/devhub/native_minio_binary_put/main.spl b/test/fixtures/devhub/native_minio_binary_put/main.spl new file mode 100644 index 00000000000..3780eb1e33e --- /dev/null +++ b/test/fixtures/devhub/native_minio_binary_put/main.spl @@ -0,0 +1,27 @@ +# Supplemental production-provider proof; full SSpec remains a separate check. +use std.env.{env_get} +use app.devhub.adapter_minio.{MinioConfig, minio_put_bytes, minio_put_object, _minio_binary_payload_hash} + +fn main() -> i64: + val endpoint = env_get("DEVHUB_MINIO_LOOPBACK_URL") + val path = env_get("DEVHUB_MINIO_BINARY_FILE") + val empty_path = env_get("DEVHUB_MINIO_EMPTY_FILE") + assert(endpoint != "" and path != "" and empty_path != "") + val cfg = MinioConfig(url: endpoint, region: "us-east-1", + access_key: "fixture-access", secret_key: "fixture-signing-key", tls_skip_verify: false) + val bytes: [u8] = [0, 128, 255, 65, 10] + assert(_minio_binary_payload_hash(bytes) == "27041e6e01b219382c3f088ed30b721653d18a97734e7bad9ad05c97fcdff0fb") + print "PASS native binary canonical typed SHA256" + val direct: (bool, text, text) = minio_put_bytes(cfg, "bucket", "dir/../direct.bin", bytes, "application/x-fixture") + print "native binary direct accepted={direct.0} detail={direct.2}" + assert(direct.0) + assert(direct.2 == "uploaded") + val file: (bool, text, text) = minio_put_object(cfg, "bucket", "file.bin", path) + assert(file.0) + val empty: (bool, text, text) = minio_put_object(cfg, "bucket", "empty.bin", empty_path) + assert(empty.0) + val missing: (bool, text, text) = minio_put_object(cfg, "bucket", "missing.bin", path + ".absent") + assert(not missing.0) + assert(missing.2 != "") + print "PASS native MinIO binary, file, empty and missing-file contracts" + 0 diff --git a/test/fixtures/devhub/native_minio_binary_put/verify_health_cli.py b/test/fixtures/devhub/native_minio_binary_put/verify_health_cli.py new file mode 100644 index 00000000000..69897886939 --- /dev/null +++ b/test/fixtures/devhub/native_minio_binary_put/verify_health_cli.py @@ -0,0 +1,50 @@ +"""Check the real DevHub JSON health exit code against a failing HTTP server.""" +import http.server +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import threading + + +def main(): + binary = Path(sys.argv[1]).resolve(strict=True) + requests = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + requests.append(self.path) + self.send_response(503) + self.send_header("Content-Length", "0") + self.end_headers() + + with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with tempfile.TemporaryDirectory(prefix="devhub-minio-health-") as temporary: + root = Path(temporary) + config = root / ".config" / "devhub" + config.mkdir(parents=True) + (config / "auth.sdn").write_text(f"[minio]\nurl = http://127.0.0.1:{server.server_port}\n", encoding="utf-8") + env = os.environ.copy() + env.update(HOME=str(root), USERPROFILE=str(root), NO_COLOR="1") + result = subprocess.run([str(binary), "minio", "health", "--json"], env=env, timeout=30, capture_output=True, text=True) + print(result.stdout, end="") + assert result.returncode == 1, f"unhealthy JSON command exited {result.returncode}" + output = json.loads(result.stdout) + assert output["alive"] is False and output["status"] == "HTTP 503", output + assert requests == ["/minio/health/live"], requests + print("PASS real DevHub unhealthy JSON result preserves nonzero exit") + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/test/fixtures/devhub/native_minio_binary_put/verify_loopback.py b/test/fixtures/devhub/native_minio_binary_put/verify_loopback.py new file mode 100644 index 00000000000..0394928f760 --- /dev/null +++ b/test/fixtures/devhub/native_minio_binary_put/verify_loopback.py @@ -0,0 +1,85 @@ +"""Run the actual native provider against an independent loopback S3 checker.""" +import hashlib +import hmac +import http.server +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import threading + + +def main(): + binary = Path(sys.argv[1]).resolve(strict=True) + observations = [] + failures = [] + attempts = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_PUT(self): + try: + expected = b"" if self.path == "/bucket/empty.bin" else bytes([0, 128, 255, 65, 10]) + body = self.rfile.read(int(self.headers["Content-Length"])) + attempts.append((self.path, len(body), hashlib.sha256(body).hexdigest())) + assert body == expected, "request body changed binary octets" + digest = hashlib.sha256(body).hexdigest() + assert self.headers["x-amz-content-sha256"] == digest + content_type = "application/x-fixture" if self.path == "/bucket/dir/../direct.bin" else "application/octet-stream" + assert self.headers["Content-Type"] == content_type + timestamp = self.headers["x-amz-date"] + signed = "content-length;content-type;host;x-amz-content-sha256;x-amz-date" + canonical_headers = "".join(f"{name}:{self.headers[name].strip()}\n" for name in signed.split(";")) + canonical = "\n".join(["PUT", self.path, "", canonical_headers, signed, digest]) + scope = f"{timestamp[:8]}/us-east-1/s3/aws4_request" + to_sign = "\n".join(["AWS4-HMAC-SHA256", timestamp, scope, hashlib.sha256(canonical.encode()).hexdigest()]) + signing = b"AWS4fixture-signing-key" + for part in [timestamp[:8], "us-east-1", "s3", "aws4_request"]: + signing = hmac.new(signing, part.encode(), hashlib.sha256).digest() + signature = hmac.new(signing, to_sign.encode(), hashlib.sha256).hexdigest() + expected_auth = f"AWS4-HMAC-SHA256 Credential=fixture-access/{scope}, SignedHeaders={signed}, Signature={signature}" + assert self.headers["Authorization"] == expected_auth, "Signature V4 mismatch" + observations.append((self.path, len(body), digest)) + self.send_response(200) + self.send_header("Content-Length", "8") + self.end_headers() + self.wfile.write(b"uploaded") + except Exception as exc: + failures.append(str(exc)) + self.send_response(400) + self.send_header("Content-Length", "0") + self.end_headers() + + with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with tempfile.TemporaryDirectory(prefix="devhub-minio-loopback-") as temporary: + root = Path(temporary) + fixture = root / "binary.bin" + fixture.write_bytes(bytes([0, 128, 255, 65, 10])) + empty = root / "empty.bin" + empty.write_bytes(b"") + env = os.environ.copy() + env.update(DEVHUB_MINIO_LOOPBACK_URL=f"http://127.0.0.1:{server.server_port}", + DEVHUB_MINIO_BINARY_FILE=str(fixture), DEVHUB_MINIO_EMPTY_FILE=str(empty), + TMPDIR=str(root), TEMP=str(root), TMP=str(root)) + result = subprocess.run([str(binary)], env=env, timeout=120, capture_output=True, text=True) + print(result.stdout, end="") + print(result.stderr, end="", file=sys.stderr) + print(f"loopback native_exit={result.returncode} request_attempts={attempts} verified_requests={observations} failures={failures}") + assert result.returncode == 0, f"native provider exited {result.returncode}" + assert not failures, failures + assert [p for p, _, _ in observations] == ["/bucket/dir/../direct.bin", "/bucket/file.bin", "/bucket/empty.bin"], observations + assert not list(root.glob("simple-http-*")), "private HTTP staging was not cleaned" + print("PASS independent loopback byte-exact payload, SHA256, SigV4 and request-count checks") + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/test/fixtures/devhub/native_outlook_token_cache/main.spl b/test/fixtures/devhub/native_outlook_token_cache/main.spl new file mode 100644 index 00000000000..04f56e81b90 --- /dev/null +++ b/test/fixtures/devhub/native_outlook_token_cache/main.spl @@ -0,0 +1,66 @@ +# Real production cache persistence check; no network or alternative parser. +use app.devhub.adapter_outlook_curl.{outlook_curl_token_cache_save, outlook_curl_token_cache_load} +use std.oauth2.{oauth2_token_cache_save, oauth2_token_cache_load, oauth2_client_credentials_request} +use std.sosix.host_facade.{sosix_env_get_opt, sosix_file_read, sosix_file_exists} +use std.io_runtime.{file_delete, file_write} +use app.devhub.outlook_pagination.{outlook_page_error, outlook_page_response_error} +use std.common.json.value.{json_value_parse_opt} + +fn main() -> i64: + val root = sosix_env_get_opt("DEVHUB_TOKEN_CACHE_PROBE_DIR") ?? "" + assert(root != "") + val request: (text, text, text, text) = oauth2_client_credentials_request("fixture-tenant", "fixture-client", "café +", "scope/read") + assert(request.0 == "POST") + assert(request.1 == "https://login.microsoftonline.com/fixture-tenant/oauth2/v2.0/token") + assert(request.2.contains("Content-Type: application/x-www-form-urlencoded")) + assert(request.3.contains("client_secret=caf%C3%A9%20%2B")) + assert(request.3.contains("scope=scope%2Fread")) + print "PASS production OAuth request is an encoded form POST" + val path = root + "/outlook-cache.sdn" + val missing: (text, i64) = outlook_curl_token_cache_load(root + "/absent-cache.sdn") + assert(missing.0 == "" and missing.1 == 0) + val missing_oauth: (text, i64) = oauth2_token_cache_load(root + "/absent-cache.sdn") + assert(missing_oauth.0 == "" and missing_oauth.1 == 0) + assert(outlook_curl_token_cache_save(path, "synthetic-outlook-fixture", 4102444800)) + assert(sosix_file_exists(path)) + val actual = sosix_file_read(path) + assert(actual.contains("synthetic-outlook-fixture")) + val loaded: (text, i64) = outlook_curl_token_cache_load(path) + assert(loaded.0 == "synthetic-outlook-fixture" and loaded.1 == 4102444800) + print "PASS production Outlook cache persists and reloads actual bytes" + assert(oauth2_token_cache_save(path, "synthetic-oauth-fixture", 4102444801)) + val oauth_loaded: (text, i64) = oauth2_token_cache_load(path) + assert(oauth_loaded.0 == "synthetic-oauth-fixture" and oauth_loaded.1 == 4102444801) + assert(not sosix_file_read(path).contains("synthetic-outlook-fixture")) + assert(not sosix_file_exists(path + ".tmp")) + print "PASS production OAuth cache atomically replaces actual bytes" + val opaque = "café日本🙂 \"quote\" \\slash\nexpires_at: 1\t\r" + assert(outlook_curl_token_cache_save(path, opaque, 4102444802)) + val escaped_outlook: (text, i64) = outlook_curl_token_cache_load(path) + assert(escaped_outlook.0 == opaque and escaped_outlook.1 == 4102444802) + assert(oauth2_token_cache_save(path, opaque, 4102444803)) + val escaped_oauth: (text, i64) = oauth2_token_cache_load(path) + assert(escaped_oauth.0 == opaque and escaped_oauth.1 == 4102444803) + print "PASS production token caches preserve Unicode and escaped payloads" + assert(file_write(path, "malformed cache fixture\n")) + val malformed: (text, i64) = outlook_curl_token_cache_load(path) + assert(malformed.0 == "" and malformed.1 == 0) + val malformed_oauth: (text, i64) = oauth2_token_cache_load(path) + assert(malformed_oauth.0 == "" and malformed_oauth.1 == 0) + assert(file_write(path, "access_token: first\naccess_token: second\nexpires_at: 4102444800\n")) + val duplicate: (text, i64) = outlook_curl_token_cache_load(path) + assert(duplicate.0 == "" and duplicate.1 == 0) + print "PASS production token caches reject absent and malformed payloads" + assert(not outlook_curl_token_cache_save("", "synthetic-fixture", 4102444800)) + assert(not oauth2_token_cache_save("", "synthetic-fixture", 4102444800)) + assert(file_delete(path)) + print "PASS production token caches reject absent destinations" + val base = "https://graph.microsoft.com/v1.0" + val page = base + "/users/fixture/messages?$skiptoken=next" + assert(outlook_page_error(base, page, []) == "") + assert(outlook_page_error(base, "https://graph.microsoft.com.evil.test/v1.0/messages", []) != "") + assert(outlook_page_error(base, page, [page]) != "") + assert(outlook_page_response_error(json_value_parse_opt("{\"value\":[]}").unwrap()) == "") + assert(outlook_page_response_error(json_value_parse_opt("{\"value\":{},\"@odata.nextLink\":7}").unwrap()) != "") + print "PASS production Graph pagination guards reject foreign cycles and malformed pages" + 0 diff --git a/test/fixtures/devhub/native_secured_db_tracking/main.spl b/test/fixtures/devhub/native_secured_db_tracking/main.spl new file mode 100644 index 00000000000..ab036ce06a1 --- /dev/null +++ b/test/fixtures/devhub/native_secured_db_tracking/main.spl @@ -0,0 +1,37 @@ +# Metadata-only registration; no secured DB feature execution. +use std.env.{env_get} +use std.nogc_sync_mut.database.bug.{Bug, BugDatabase, BugSeverity, BugStatus, create_bug_database} +use std.nogc_sync_mut.database.core.{SdnRow} +use std.nogc_sync_mut.database.todo.{append_todo_preserving_legacy} +use std.nogc_sync_mut.compression.gzip.crc.{crc32_text} + +fn main() -> i64: + if crc32_text("123456789") != 3421780262: return 3 + val root = env_get("WORKAROUND_REGISTRY_ROOT") + if root == "": return 2 + var database: BugDatabase = create_bug_database(root + "/doc/08_tracking/bug/bug_db.sdn") + val record = Bug( + id: "devhub_secured_db_key_session_open_failure_2026_10_27", severity: BugSeverity.P1, status: BugStatus.Open, + title: "DevHub secured messenger DB key session fails before decode", + description: ["Fresh secured DB ledger1 compiled successfully with pinned Phase2 but actual execution returned Err at direct key-session open before encrypted decode. Messenger database access remains an unresolved main-feature blocker; this is not an infrastructure exception."], + file: "doc/08_tracking/bug/devhub_secured_db_key_session_open_failure_2026_10_27.md", line: 1, + reproducible_by: "D:/dev/simple-devhub-secured-db-p2-20261027/build/devhub-completion/fresh-secured-db-ledger1-run.log", + fix_strategy: ["Repair direct key-session open with a discriminating error regression. Then qualify secure SQLite decode and efficient snapshot paging with memory-only plaintext, unchanged encrypted inputs, cleanup and independent row oracles before claiming Messenger DB support."], + investigation_log: ["Pinned P2 producer13d73267e690b2cdd46438315e2bfe6d4d6c0c37bcfc15591a921fc9728b270b. Ledger1 build0 run1. Memory oracle passed AES NIST, independent KDF/HMAC and SQLite rows/overflow. Execution stopped at STAGE fresh direct key-session return with FAIL fresh secured DB direct key session. Encrypted inputs and metadata stayed unchanged; no plaintext or journal secret files. Exact root cause remains unisolated; no further feature experiments authorized this session."], + created_at: "2026-10-27", updated_at: "2026-10-27", valid: true) + if not database.append_open_bugs_preserving_legacy([record]): return 1 + var row: SdnRow = SdnRow.empty() + row.set("id", "377") + row.set("keyword", "TODO") + row.set("area", "devhub") + row.set("priority", "P1") + row.set("description", "Complete secured Messenger DB key-session open then secure SQLite snapshot and efficient bounded paging qualification") + row.set("file", "doc/08_tracking/bug/devhub_secured_db_key_session_open_failure_2026_10_27.md") + row.set("line", "1") + row.set("issue", "devhub_secured_db_key_session_open_failure_2026_10_27") + row.set("blocked", "Actual pinned Phase2 ledger1 key-session Err before decode; main DevHub feature remains unverified") + row.set("status", "blocked") + row.set("valid", "true") + if not append_todo_preserving_legacy(root + "/doc/08_tracking/todo/todo_db.sdn", row): return 1 + print "PASS canonical BUG and TODO append registration" + 0 diff --git a/test/scripts/test_prepare_mail_config_helper.py b/test/scripts/test_prepare_mail_config_helper.py new file mode 100644 index 00000000000..0d25832233b --- /dev/null +++ b/test/scripts/test_prepare_mail_config_helper.py @@ -0,0 +1,52 @@ +"""Fail-closed CI dependency admission tests; never emulate a Simple compiler.""" +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import zipfile + + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location("prepare_mail_config_helper", ROOT / "scripts/ci/prepare_mail_config_helper.py") +prepare = importlib.util.module_from_spec(spec) +spec.loader.exec_module(prepare) + + +class AdmissionTests(unittest.TestCase): + def test_missing_platform_producer_fails_instead_of_using_a_seed(self): + with patch.dict("os.environ", {"MAIL_CI_PHASE2_PRODUCERS_JSON": "{}"}): + with self.assertRaisesRegex(SystemExit, "no existing producer is configured"): + prepare.main() + + def test_wrong_generation_is_rejected_before_download(self): + with tempfile.TemporaryDirectory() as directory, patch.object(prepare.urllib.request, "urlopen") as download: + with self.assertRaisesRegex(SystemExit, "must declare phase 2"): + prepare.provision({"phase": 1}, Path(directory)) + download.assert_not_called() + + def test_corrupted_bundle_never_reaches_extraction_or_execution(self): + entry = {"phase": 2, "sha256": "0" * 64, "url": "https://example.invalid/pinned-producer", "format": "zip"} + with tempfile.TemporaryDirectory() as directory: + with patch.object(prepare.urllib.request, "urlopen", return_value=io.BytesIO(b"corrupt bundle")): + with self.assertRaisesRegex(SystemExit, "checksum does not match"): + prepare.provision(entry, Path(directory)) + self.assertFalse((Path(directory) / "producer").exists()) + + def test_zip_path_traversal_is_refused_after_checksum_validation(self): + payload = io.BytesIO() + with zipfile.ZipFile(payload, "w") as archive: + archive.writestr("../../escaped-compiler", "not executable") + data = payload.getvalue() + entry = {"phase": 2, "sha256": hashlib.sha256(data).hexdigest(), "url": "https://example.invalid/pinned-producer", "format": "zip"} + with tempfile.TemporaryDirectory() as directory: + with patch.object(prepare.urllib.request, "urlopen", return_value=io.BytesIO(data)): + with self.assertRaisesRegex(SystemExit, "escapes its staging directory"): + prepare.provision(entry, Path(directory)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/mail-cli/README.md b/tools/mail-cli/README.md index 3ffd8230bb4..4fda491b82a 100644 --- a/tools/mail-cli/README.md +++ b/tools/mail-cli/README.md @@ -78,3 +78,45 @@ inherited PATH. On Unix, the executable `bin/mail` can run directly. Never place passwords in command lines or launcher files. See the [mail-cli guide](../../doc/07_guide/app/mail_cli.md) for configuration and password input modes. + +## Hosted fixture dependency + +The host-fixture credential executable deliberately implements only synthetic +encryption orchestration. SDN scenarios require a separate, real +`MAIL_CONFIG_BIN`; that executable must run the canonical Simple parser. +`scripts/ci/prepare_mail_config_helper.py` builds the current helper source with +an existing, pinned Phase-2 compiler and exercises valid and malformed SDN +before exporting its path to subsequent workflow steps. It never rebuilds the +compiler or falls back to a seed, another parser, or skipped assertions. + +Configure the repository variable `MAIL_CI_PHASE2_PRODUCERS_JSON` with qualified +producer bundles for the hosted platforms, for example: + +```json +{ + "linux-x86_64": { + "phase": 2, + "url": "https://example.invalid/existing-qualified-phase2.tar.gz", + "sha256": "", + "format": "tar.gz", + "compiler": "bin/simple", + "asset_root": "." + } +} +``` + +Supported host keys are `linux-x86_64`, `darwin-arm64`, `darwin-x86_64` and +`windows-x86_64`. Formats are `file`, `tar.gz` and `zip`; bundles must include +the producer's runtime dependencies and an available native toolchain. The +digest pins the complete bundle, while the producer generation comes from its +qualification record. A missing profile, checksum mismatch or failed real +helper build remains an infrastructure failure. The workflow preserves build +logs and producer/helper digests instead of reporting the SDN scenarios passed. + +TODO (hosted CI infrastructure, 2026-10-27): provide qualified existing Phase-2 +producer profiles for every matrix host and qualify the resulting current-source +helper builds. No Linux/Darwin profile has been selected. The provisioning +admission tests passed locally; the focused config and placeholder fixtures +passed with the existing real Windows SDN helper. This is not hosted-matrix or +new-helper-build acceptance, and the workflow remains blocked until those +external producer dependencies are provisioned. diff --git a/tools/mail-cli/lib/format.bash b/tools/mail-cli/lib/format.bash index 78dbe2bcb39..48a8f0eb64c 100644 --- a/tools/mail-cli/lib/format.bash +++ b/tools/mail-cli/lib/format.bash @@ -181,13 +181,13 @@ mail_format_parse_headers() { headers=$(echo "$headers" | sed -E -e ':a' -e 'N' -e '$!ba' -e 's/\n[[:space:]]+/ /g') local from to subject date msg_id reply_to references - from=$(echo "$headers" | grep -i '^From:' | head -1 | sed 's/^[Ff]rom:[[:space:]]*//') - to=$(echo "$headers" | grep -i '^To:' | head -1 | sed 's/^[Tt]o:[[:space:]]*//') - subject=$(echo "$headers" | grep -i '^Subject:' | head -1 | sed 's/^[Ss]ubject:[[:space:]]*//') - date=$(echo "$headers" | grep -i '^Date:' | head -1 | sed 's/^[Dd]ate:[[:space:]]*//') - msg_id=$(echo "$headers" | grep -i '^Message-ID:' | head -1 | sed 's/^[Mm]essage-[Ii][Dd]:[[:space:]]*//') - reply_to=$(echo "$headers" | grep -i '^In-Reply-To:' | head -1 | sed 's/^[Ii]n-[Rr]eply-[Tt]o:[[:space:]]*//') - references=$(echo "$headers" | grep -i '^References:' | head -1 | sed 's/^[Rr]eferences:[[:space:]]*//') + from=$(echo "$headers" | grep -i '^From:' | head -1 | sed 's/^[^:]*:[[:space:]]*//') + to=$(echo "$headers" | grep -i '^To:' | head -1 | sed 's/^[^:]*:[[:space:]]*//') + subject=$(echo "$headers" | grep -i '^Subject:' | head -1 | sed 's/^[^:]*:[[:space:]]*//') + date=$(echo "$headers" | grep -i '^Date:' | head -1 | sed 's/^[^:]*:[[:space:]]*//') + msg_id=$(echo "$headers" | grep -i '^Message-ID:' | head -1 | sed 's/^[^:]*:[[:space:]]*//') + reply_to=$(echo "$headers" | grep -i '^In-Reply-To:' | head -1 | sed 's/^[^:]*:[[:space:]]*//') + references=$(echo "$headers" | grep -i '^References:' | head -1 | sed 's/^[^:]*:[[:space:]]*//') jq -n \ --arg from "$from" \ @@ -209,9 +209,10 @@ mail_format_parse_body() { local body body=$(echo "$raw" | sed '1,/^$/d') - # Check Content-Transfer-Encoding + # Only the header section declares transfer encoding; body text may contain + # a line that looks like a header and must remain plain content. local encoding - encoding=$(echo "$raw" | grep -i '^Content-Transfer-Encoding:' | head -1 | sed 's/.*:[[:space:]]*//' | tr '[:upper:]' '[:lower:]') + encoding=$(echo "$raw" | sed '/^$/q' | grep -i '^Content-Transfer-Encoding:' | head -1 | sed 's/.*:[[:space:]]*//' | tr '[:upper:]' '[:lower:]') case "$encoding" in base64)