integrate: merge five reviewed and repaired PRs

.github/workflows/python-ci.yml

@@ -66,12 +66,13 @@ jobs:
           tests/application/test_session_deletion_service.py
           tests/application/test_execution_coordinator.py
 
-      # These suites carry Windows-gated cases (the Job Object backend) that
-      # the ubuntu job can only skip. This is the sole place they actually
-      # execute.
-      - name: Verify Job Object sandbox
+      # These suites carry Windows-gated cases (NTFS ACLs, the Job Object
+      # backend) that the ubuntu job can only skip. This is the sole place
+      # they actually execute.
+      - name: Verify Windows ACLs and Job Object sandbox
         run: >-
           python -m pytest -q
+          tests/test_private_storage_windows.py
           tests/test_harness_sandbox.py
           tests/test_exec_sandbox_wiring.py
 

core/agent_runtime/runner.py

@@ -88,6 +88,49 @@
 # it survives across turns and is not re-summarized every step.
 _BACKFILL_CONTENT = "[Tool result unavailable — call was interrupted or lost]"
 
+# PreCompact checkpoint re-injection (bounded, provider-safe). A PreCompact
+# hook may attach ``additional_contexts`` that must survive a successful
+# compaction so the post-compaction model can restore working context. The
+# re-injection is a plain ``role: user`` message (provider-agnostic) carrying a
+# clearly delimited prefix, with hard limits per context and in total — a
+# runaway hook can never blow the post-compaction window back open.
+_PRECOMPACT_CHECKPOINT_PREFIX = "[PreCompact checkpoint]"
+_PRECOMPACT_CONTEXT_LIMIT = 2000  # chars per additional context
+_PRECOMPACT_TOTAL_LIMIT = 8000  # chars for the whole checkpoint block
+
+
+def _build_precompact_checkpoint(
+    contexts: list[str],
+    *,
+    total_limit: int = _PRECOMPACT_TOTAL_LIMIT,
+) -> str | None:
+    """Bounded, delimited representation of PreCompact hook context.
+
+    Each context is stripped, truncated to ``_PRECOMPACT_CONTEXT_LIMIT`` chars
+    and the combined block capped at ``_PRECOMPACT_TOTAL_LIMIT``. Returns
+    ``None`` when nothing survives (empty input or all contexts blank).
+    """
+    prefix = _PRECOMPACT_CHECKPOINT_PREFIX + "\n"
+    total_limit = min(max(total_limit, 0), _PRECOMPACT_TOTAL_LIMIT)
+    content_limit = total_limit - len(prefix)
+    if not contexts or content_limit <= 0:
+        return None
+    parts: list[str] = []
+    used = 0
+    for ctx in contexts:
+        text = (ctx or "").strip()
+        if not text:
+            continue
+        text = text[:_PRECOMPACT_CONTEXT_LIMIT]
+        room = content_limit - used
+        if room <= 0:
+            break
+        parts.append(text[:room])
+        used += min(len(text), room) + 1  # +1 for the newline separator
+    if not parts:
+        return None
+    return prefix + "\n".join(parts)
+
 
 @dataclass(slots=True)
 class AgentRunSpec:
@@ -1688,15 +1731,16 @@ async def _maybe_compact(
                 if estimate is None or estimate <= trigger:
                     return messages
 
+        pre_contexts: list[str] = []
         signature = history_signature(messages)
         if signature == self._refused_compaction:
             # Already tried on exactly this history and it did not shrink.
             return messages
-
         if spec.pre_compact_hook is not None:
             pre = await self._call_tool_hook(spec.pre_compact_hook, "auto")
             if pre is not None and getattr(pre, "block", False):
                 return messages  # a PreCompact hook aborted compaction this turn
+            pre_contexts = list(getattr(pre, "additional_contexts", None) or [])
 
         summary = await self._summarize(
             spec,
@@ -1723,6 +1767,28 @@ async def _maybe_compact(
                 spec.session_key or "default",
             )
             return messages
+        # Bounded checkpoint re-injection: the PreCompact hook's
+        # ``additional_contexts`` survive a successful compaction as a single
+        # provider-agnostic user message. ``_build_precompact_checkpoint``
+        # caps each context and the total block, so a runaway hook can never
+        # blow the post-compaction window back open. When the hook blocks or
+        # summarization fails we returned above — the checkpoint only ever
+        # appears after a successful compaction.
+        # A checkpoint must not turn successful compaction back into growth.
+        # Bound it to the actual character reduction in addition to the
+        # absolute hook-context cap.
+        checkpoint_room = (
+            self._history_chars(messages) - self._history_chars(compacted) - 1
+        )
+        checkpoint = _build_precompact_checkpoint(
+            pre_contexts,
+            total_limit=checkpoint_room,
+        )
+        if checkpoint:
+            self._append_injected_messages(
+                compacted,
+                [{"role": "user", "content": checkpoint}],
+            )
         self._refused_compaction = None
         if spec.post_compact_hook is not None:
             await self._call_tool_hook(spec.post_compact_hook, "auto")

core/events/session.py

@@ -418,6 +418,8 @@ def __init__(
         # no hooks are configured, so the whole feature is dormant at zero cost.
         self._hooks_engine = hooks_engine
         self._session_started = False
+        self._session_end_fired = False
+        self._session_end_lock = asyncio.Lock()
         # When this session is a spawned sub-agent, (agent_id, agent_type) — its
         # lifecycle fires SubagentStart/SubagentStop instead of SessionStart/Stop.
         self._agent_context = agent_context
@@ -622,6 +624,10 @@ async def submit(self, op: Op) -> None:
             if task is not None and not task.done() and task.cancelling() == 0:
                 task.cancel()
         elif isinstance(op, Shutdown):
+            # SessionEnd fires exactly once, here — at the real session
+            # termination boundary — never per turn. Per-turn notifications
+            # are the Stop event's job (see _EVENTS_WITHOUT_MATCHER).
+            await self._run_end_hook(reason="other")
             self._emit(ShutdownComplete())
         else:  # pragma: no cover - exhaustive guard
             self._emit(ErrorEvent(message=f"unknown op: {op!r}"))
@@ -641,6 +647,7 @@ async def aclose(self) -> None:
             if task.cancelling() == 0:
                 task.cancel()
             await asyncio.gather(task, return_exceptions=True)
+        await self._run_end_hook(reason="other")
         control = getattr(self, "_agent_control", None)
         if control is not None:
             await control.close()
@@ -690,6 +697,30 @@ async def _run_start_hook(self):
             logger.exception("start hook failed")
             return None
 
+    async def _run_end_hook(self, reason: str = "other") -> None:
+        """Run SessionEnd hooks when the session itself terminates.
+
+        Notification-only: a failure is logged and never crashes the
+        shutdown. Fired exactly once from ``submit(Shutdown)`` or the actual
+        resource teardown path, whichever comes first. ``other`` is the
+        compatible exit reason for a DeepCode runtime shutdown; per-turn
+        notifications belong to the Stop event, not SessionEnd.
+        """
+        async with self._session_end_lock:
+            if self._session_end_fired:
+                return
+            self._session_end_fired = True
+            # Spawned agents use their dedicated SubagentStop lifecycle.
+            if self._agent_context is not None:
+                return
+            engine = self._hooks_engine
+            if engine is None or not engine.has_event("SessionEnd"):
+                return
+            try:
+                await engine.run_session_end(reason=reason)
+            except Exception:  # noqa: BLE001 - hooks never crash a shutdown
+                logger.exception("session end hook failed")
+
     async def _run_prompt_hooks(
         self, text: str, hook_contexts: list[str]
     ) -> str | None:
@@ -810,6 +841,10 @@ async def _run_user_input(self, op: UserInput | str) -> None:
             self._active_turn_task = None
             if terminal is not None:
                 self._emit(terminal)
+            # SessionEnd is NOT fired here: this finally block runs after
+            # every turn, and SessionEnd must fire exactly once at session
+            # termination (submit(Shutdown)), not per turn. Per-turn
+            # notifications are the Stop event's responsibility.
 
     async def _execute_turn(
         self,

core/harness/hooks/discovery.py

@@ -38,6 +38,8 @@
 )
 
 _DEFAULT_TIMEOUT_SEC = 600
+_SESSION_END_DEFAULT_TIMEOUT_SEC = 2
+_SESSION_END_MAX_TIMEOUT_SEC = 60
 
 
 @dataclass(slots=True)
@@ -150,12 +152,19 @@ def _append_group(
             warnings.append(f"skipping empty hook command in {path}")
             continue
         timeout = handler.get("timeout")
+        default_timeout = (
+            _SESSION_END_DEFAULT_TIMEOUT_SEC
+            if event_name == "SessionEnd"
+            else _DEFAULT_TIMEOUT_SEC
+        )
         try:
             timeout_sec = (
-                max(1, int(timeout)) if timeout is not None else _DEFAULT_TIMEOUT_SEC
+                max(1, int(timeout)) if timeout is not None else default_timeout
             )
         except (TypeError, ValueError):
-            timeout_sec = _DEFAULT_TIMEOUT_SEC
+            timeout_sec = default_timeout
+        if event_name == "SessionEnd":
+            timeout_sec = min(timeout_sec, _SESSION_END_MAX_TIMEOUT_SEC)
         status_message = handler.get("statusMessage") or handler.get("status_message")
         handlers.append(
             Handler(

core/harness/hooks/engine.py

@@ -176,6 +176,22 @@ async def run_session_start(self, source: str = "startup") -> ContextOutcome:
             additional_contexts=folded.additional_contexts,
         )
 
+    async def run_session_end(self, reason: str = "other") -> ContextOutcome:
+        """Session lifecycle end — fires exactly once when the session terminates.
+
+        Called from ``AgentSession.submit(Shutdown)``, never per turn. The
+        reason doubles as the matcher input. DeepCode runtime shutdown maps to
+        the compatible ``other`` reason. The caller logs failures so a hook
+        can never crash the session close.
+        """
+        payload = {"hook_event_name": "SessionEnd", "reason": reason}
+        folded = await self._dispatch("SessionEnd", reason, payload)
+        return ContextOutcome(
+            block=folded.block,
+            block_reason=folded.block_reason,
+            additional_contexts=folded.additional_contexts,
+        )
+
     async def run_user_prompt_submit(self, prompt: str) -> ContextOutcome:
         payload = {"hook_event_name": "UserPromptSubmit", "prompt": prompt}
         folded = await self._dispatch("UserPromptSubmit", None, payload)
@@ -192,10 +208,18 @@ async def run_stop(self, stop_hook_active: bool = False) -> StopOutcome:
 
     async def run_pre_compact(self, trigger: str = "auto") -> ContextOutcome:
         """Before a summarization pass. A ``block`` (continue:false) asks to skip
-        compaction this turn; the matcher runs against ``trigger`` (auto/manual)."""
+        compaction this turn; the matcher runs against ``trigger`` (auto/manual).
+
+        ``additional_contexts`` from hook ``hookSpecificOutput.additionalContext``
+        are passed through so a PreCompact hook can inject a checkpoint summary
+        (memento-style) that survives the compaction."""
         payload = {"hook_event_name": "PreCompact", "trigger": trigger}
         folded = await self._dispatch("PreCompact", trigger, payload)
-        return ContextOutcome(block=folded.block, block_reason=folded.block_reason)
+        return ContextOutcome(
+            block=folded.block,
+            block_reason=folded.block_reason,
+            additional_contexts=folded.additional_contexts,
+        )
 
     async def run_post_compact(self, trigger: str = "auto") -> ContextOutcome:
         """After a summarization pass — a notification hook (state saved, etc.)."""

core/harness/hooks/events.py

@@ -15,8 +15,8 @@
 - ``Stop``            — when the agent would end its turn (may force-continue)
 
 The remaining reference events (``PermissionRequest``, ``PreCompact``,
-``PostCompact``, ``SubagentStart``, ``SubagentStop``) are recognized by the
-engine but wired opportunistically as DeepCode grows the matching kernel seams.
+``PostCompact``, ``SessionEnd``, ``SubagentStart``, ``SubagentStop``) are
+recognized by the engine and wired at their matching lifecycle seams.
 """
 
 from __future__ import annotations
@@ -31,14 +31,18 @@
     "PreCompact",
     "PostCompact",
     "SessionStart",
+    "SessionEnd",
     "UserPromptSubmit",
     "SubagentStart",
     "SubagentStop",
     "Stop",
 )
 
-# Events whose ``matcher`` field is meaningful. ``UserPromptSubmit`` and ``Stop``
-# fire unconditionally, so their matchers are ignored (mirrors the reference).
+# Events whose ``matcher`` field is meaningful. ``UserPromptSubmit`` and
+# ``Stop`` fire unconditionally, so their matchers are ignored (mirrors the
+# reference). ``SessionEnd`` DOES honour its matcher: the session-exit reason
+# is matched against the ``matcher`` field so hooks can target a specific exit
+# path. DeepCode runtime shutdown uses the compatible ``other`` reason.
 _EVENTS_WITHOUT_MATCHER: frozenset[str] = frozenset({"UserPromptSubmit", "Stop"})
 
 

core/harness/hooks/execution.py

@@ -250,7 +250,9 @@ def _decode_permission_request(obj: dict) -> HandlerDecision:
     "Stop": lambda o: _block_from_decision(o, "Stop"),
     "SubagentStop": lambda o: _block_from_decision(o, "SubagentStop"),
     "SessionStart": _decode_additional_context_only,
+    "SessionEnd": _decode_additional_context_only,
     "SubagentStart": _decode_additional_context_only,
+    "PreCompact": lambda o: _block_from_decision(o, "PreCompact"),
     "PermissionRequest": _decode_permission_request,
 }
 

core/harness/memory.py

@@ -23,6 +23,9 @@
 
 from __future__ import annotations
 
+import os
+import re
+from functools import lru_cache
 from pathlib import Path
 from typing import Any
 
@@ -41,6 +44,69 @@
 _REMINDER_OPEN = "<system-reminder>"
 _REMINDER_CLOSE = "</system-reminder>"
 _REMINDER_CLOSE_ESCAPED = "&lt;/system-reminder&gt;"
+# Comma-separated glob patterns for instruction files that must not be loaded,
+# for example ``code/CLAUDE.md,**/vendor/**``.
+_INSTRUCTION_EXCLUDE_ENV = "DEEPCODE_INSTRUCTION_EXCLUDES"
+
+
+@lru_cache(maxsize=256)
+def _glob_to_re(pattern: str) -> re.Pattern[str]:
+    """Compile a path glob where ``**`` crosses directory boundaries."""
+
+    parts = []
+    i, n = 0, len(pattern)
+    while i < n:
+        c = pattern[i]
+        if c == "*":
+            if i + 1 < n and pattern[i + 1] == "*":
+                if i + 2 < n and pattern[i + 2] in "/\\":
+                    parts.append(r"(?:.*/)?")
+                    i += 3
+                else:
+                    parts.append(".*")
+                    i += 2
+            else:
+                parts.append(r"[^/\\]*")
+                i += 1
+        elif c == "?":
+            parts.append(r"[^/\\]")
+            i += 1
+        else:
+            parts.append(re.escape(c))
+            i += 1
+    flags = re.IGNORECASE if os.name == "nt" else 0
+    return re.compile("^" + "".join(parts) + "$", flags)
+
+
+def _instruction_excluded(candidate: Path, *, root: Path | None = None) -> bool:
+    """Whether the candidate instruction file is excluded by pattern.
+
+    Patterns containing a separator match both the absolute path and, when
+    available, the path relative to the repository root. A bare filename such
+    as ``CLAUDE.md`` matches that filename at any searched level.
+    """
+    patterns = [
+        p.strip()
+        for p in os.environ.get(_INSTRUCTION_EXCLUDE_ENV, "").split(",")
+        if p.strip()
+    ]
+    if not patterns:
+        return False
+    candidates = {str(candidate).replace("\\", "/"), candidate.name}
+    if root is not None:
+        try:
+            candidates.add(candidate.relative_to(root).as_posix())
+        except ValueError:
+            pass
+    for pat in patterns:
+        normalized = pat.replace("\\", "/")
+        try:
+            compiled = _glob_to_re(normalized)
+            if any(compiled.fullmatch(value) for value in candidates):
+                return True
+        except re.error:
+            continue
+    return False
 
 
 def memory_dir(workspace: str | Path) -> Path:
@@ -135,7 +201,10 @@ def project_instructions(workspace: str | Path) -> str:
     for directory in search_dirs:
         for name in _PROJECT_FILES:
             candidate = directory / name
-            if candidate.is_file():
+            if candidate.is_file() and not _instruction_excluded(
+                candidate,
+                root=root or workspace,
+            ):
                 try:
                     body = candidate.read_text(
                         encoding="utf-8", errors="replace"