Require an agent key on /v1 and fail closed for keyless off-loopback callers

Makefile

@@ -5,6 +5,8 @@ WEB_DIR ?= apps/web
 DIST_DIR ?= dist
 REPRO_DIST_DIR ?= $(DIST_DIR)-reproducibility-check
 PROJECT_VERSION = $(shell $(PYTHON) -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')
+# Reload is off unless a developer opts in: APP_RELOAD=true make dev
+APP_RELOAD ?= false
 ALICE_WEB_HOST ?= 127.0.0.1
 ALICE_WEB_PORT ?= 3000
 
@@ -39,14 +41,14 @@ migrate:
 	./scripts/dev_up.sh
 
 api:
-	APP_RELOAD=false ./scripts/api_dev.sh
+	APP_RELOAD=$(APP_RELOAD) ./scripts/api_dev.sh
 
 doctor:
 	$(ALICEBOT) vnext doctor --fix-safe --ci
 
 dev:
 	./scripts/dev_up.sh
-	APP_RELOAD=false ./scripts/api_dev.sh & \
+	APP_RELOAD=$(APP_RELOAD) ./scripts/api_dev.sh & \
 	api_pid=$$!; \
 	$(PNPM) --dir $(WEB_DIR) dev & \
 	web_pid=$$!; \
@@ -56,7 +58,7 @@ dev:
 runtime:
 	./scripts/dev_up.sh
 	$(PNPM) --dir $(WEB_DIR) build
-	APP_RELOAD=false ./scripts/api_dev.sh & \
+	APP_RELOAD=$(APP_RELOAD) ./scripts/api_dev.sh & \
 	api_pid=$$!; \
 	$(PNPM) --dir $(WEB_DIR) start --hostname $(ALICE_WEB_HOST) --port $(ALICE_WEB_PORT) & \
 	web_pid=$$!; \

apps/api/src/alicebot_api/chatgpt_import.py

@@ -21,6 +21,12 @@
     parse_optional_confidence,
     parse_optional_status,
 )
+from alicebot_api.importer_paths import (
+    ImportSourceFile,
+    contained_source_files,
+    read_contained_source_text,
+    snapshot_source_files,
+)
 from alicebot_api.importers.common import ImportPersistenceConfig, import_normalized_batch
 from alicebot_api.store import ContinuityStore, JsonObject, JsonValue
 
@@ -63,9 +69,9 @@ def _build_raw_content(*, object_type: str, text: str) -> str:
     return f"{prefix}: {text}"
 
 
-def _read_json(path: Path) -> object:
+def _parse_json(path: Path, raw_text: str) -> object:
     try:
-        return json.loads(path.read_text(encoding="utf-8"))
+        return json.loads(raw_text)
     except json.JSONDecodeError as exc:
         raise ChatGPTImportValidationError(
             f"invalid JSON at {path}: {exc.msg}"
@@ -77,16 +83,41 @@ def _read_chatgpt_source_files(source: str | Path) -> tuple[Path, list[Path]]:
     if not source_path.exists():
         raise ChatGPTImportValidationError(f"ChatGPT source path does not exist: {source_path}")
 
-    source_files = [source_path] if source_path.is_file() else sorted(source_path.rglob("*.json"))
+    source_files = (
+        [source_path]
+        if source_path.is_file()
+        else contained_source_files(
+            source_path,
+            suffixes=(".json",),
+            recursive=True,
+            error_factory=ChatGPTImportValidationError,
+        )
+    )
     if not source_files:
         raise ChatGPTImportValidationError("no ChatGPT JSON files were found at the source path")
     return source_path, source_files
 
 
-def _relative_source_file(source_root: Path, file_path: Path) -> str:
-    if source_root.is_dir():
-        return str(file_path.relative_to(source_root))
-    return file_path.name
+def _snapshot_chatgpt_source(source: str | Path) -> tuple[Path, list[ImportSourceFile]]:
+    """Select the ChatGPT export files and read each of them exactly once."""
+
+    source_path, source_files = _read_chatgpt_source_files(source)
+    if source_path.is_file():
+        return source_path, [
+            ImportSourceFile(
+                path=source_path,
+                relative_path=source_path.name,
+                text=read_contained_source_text(
+                    source_path,
+                    error_factory=ChatGPTImportValidationError,
+                ),
+            )
+        ]
+    return source_path, snapshot_source_files(
+        source_path,
+        source_files,
+        error_factory=ChatGPTImportValidationError,
+    )
 
 
 def _normalize_message_text(value: object) -> str | None:
@@ -261,16 +292,22 @@ def _message_text(message: JsonObject) -> str | None:
 
 
 def load_chatgpt_payload(source: str | Path) -> ImporterNormalizedBatch:
-    source_path, source_files = _read_chatgpt_source_files(source)
+    source_path, snapshot = _snapshot_chatgpt_source(source)
+    return _load_chatgpt_batch(source_path, snapshot)
 
+
+def _load_chatgpt_batch(
+    source_path: Path,
+    snapshot: list[ImportSourceFile],
+) -> ImporterNormalizedBatch:
     fixture_id: str | None = None
     workspace_id: str | None = None
     workspace_name: str | None = None
 
     items: list[ImporterNormalizedItem] = []
 
-    for source_file in source_files:
-        payload = _read_json(source_file)
+    for source_file in snapshot:
+        payload = _parse_json(source_file.path, source_file.text)
 
         maybe_fixture_id, maybe_workspace_id, maybe_workspace_name = _extract_workspace_metadata(payload)
         if fixture_id is None:
@@ -363,7 +400,7 @@ def load_chatgpt_payload(source: str | Path) -> ImporterNormalizedBatch:
                 items.append(
                     ImporterNormalizedItem(
                         source_item_id=source_item_id,
-                        source_file=_relative_source_file(source_path, source_file),
+                        source_file=source_file.relative_path,
                         source_locator={
                             "conversation_id": conversation_id,
                             "message_id": message_id,
@@ -403,23 +440,27 @@ def import_chatgpt_source(
     user_id: UUID,
     source: str | Path,
 ) -> JsonObject:
-    source_path, source_files = _read_chatgpt_source_files(source)
+    # One snapshot feeds both the evidence archive and the parse, so the
+    # archived text is the text that was imported. It is decoded text and not
+    # the disk bytes: the read is text mode, so CRLF arrives as LF and the
+    # archive will not checksum against the original file.
+    source_path, snapshot = _snapshot_chatgpt_source(source)
     archived_artifacts = archive_import_source_files(
         store,
         user_id=user_id,
         source_kind="chatgpt_import",
         import_source_path=str(source_path),
         files=[
             SourceArtifactArchiveInput(
-                relative_path=_relative_source_file(source_path, file_path),
-                display_name=file_path.name,
+                relative_path=source_file.relative_path,
+                display_name=source_file.path.name,
                 media_type="application/json",
-                content_text=file_path.read_text(encoding="utf-8"),
+                content_text=source_file.text,
             )
-            for file_path in source_files
+            for source_file in snapshot
         ],
     )
-    batch = load_chatgpt_payload(source_path)
+    batch = _load_chatgpt_batch(source_path, snapshot)
     return import_normalized_batch(
         store,
         user_id=user_id,

apps/api/src/alicebot_api/importer_paths.py

@@ -0,0 +1,181 @@
+"""Filesystem containment for importer source trees.
+
+Importers walk a root the operator selected and read whatever they find under
+it. Two ordinary habits break that boundary. ``Path.rglob`` descends directory
+symlinks and ``Path.is_file`` is true for a link whose target is a regular
+file, so one link planted inside the root can pull in bytes from anywhere the
+server process can read. And re-opening a listed path for each pass lets the
+name be swapped between passes, so the bytes that get archived as evidence
+need not be the bytes that were parsed.
+
+Everything here refuses to follow a link out of the selected root, and hands
+back the exact text it read so callers parse and archive one snapshot.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+import errno
+import fcntl
+import os
+from pathlib import Path
+import stat
+
+
+ErrorFactory = Callable[[str], Exception]
+
+# BSD kernels report O_NOFOLLOW on a symlink as EMLINK rather than ELOOP.
+_SYMLINK_OPEN_ERRNOS = frozenset({errno.ELOOP, errno.EMLINK})
+
+
+@dataclass(frozen=True, slots=True)
+class ImportSourceFile:
+    """One import file opened once, with the bytes that were read."""
+
+    path: Path
+    relative_path: str
+    text: str
+
+
+def _relative_source_file(source_root: Path, file_path: Path) -> str:
+    if source_root.is_dir():
+        return str(file_path.relative_to(source_root))
+    return file_path.name
+
+
+def contained_source_files(
+    source_root: Path,
+    *,
+    suffixes: Iterable[str],
+    recursive: bool,
+    error_factory: ErrorFactory,
+) -> list[Path]:
+    """List import candidates under ``source_root`` without following links.
+
+    Walks with ``followlinks=False`` so no directory symlink is descended, and
+    refuses any symlinked directory or candidate file instead of silently
+    importing bytes from outside the root or silently skipping content the
+    operator expected to import.
+    """
+
+    matches: list[Path] = []
+    normalized_suffixes = {suffix.casefold() for suffix in suffixes}
+    for raw_directory, directory_names, file_names in os.walk(source_root, followlinks=False):
+        current_directory = Path(raw_directory)
+        if recursive:
+            for directory_name in sorted(directory_names):
+                candidate_directory = current_directory / directory_name
+                if candidate_directory.is_symlink():
+                    raise error_factory(
+                        f"import source must not contain symlinked directories: {candidate_directory}"
+                    )
+        for file_name in sorted(file_names):
+            candidate = current_directory / file_name
+            if candidate.suffix.casefold() not in normalized_suffixes:
+                continue
+            if candidate.is_symlink():
+                raise error_factory(f"import source must not contain symlinked files: {candidate}")
+            matches.append(candidate)
+        if not recursive:
+            directory_names[:] = []
+    return sorted(matches)
+
+
+def read_contained_source_text(
+    file_path: Path,
+    *,
+    source_root: Path | None = None,
+    error_factory: ErrorFactory,
+) -> str:
+    """Open one import file once and return the exact text that was read.
+
+    ``O_NOFOLLOW`` fails the open when the final path component is a symlink,
+    so a link swapped in after the listing cannot redirect the read.
+    ``O_NONBLOCK`` keeps the open itself from parking forever on a FIFO that
+    has no writer: the descriptor is checked to be a regular file before any
+    bytes are consumed, and that check is worthless if the process never gets
+    to it. The flag is cleared once the descriptor is known to be a regular
+    file, for which it has no defined effect anyway.
+
+    ``source_root``, when given, is enforced: a candidate carrying ``..`` or
+    otherwise resolving outside the selected root is refused rather than read.
+
+    Known limitation: a hard link is not a reference to a file, it is the
+    file, so a hard link planted inside the root to content elsewhere is
+    indistinguishable from ordinary content and is read. Nothing at this layer
+    can separate the two.
+    """
+
+    if ".." in file_path.parts:
+        raise error_factory(f"import source path must not traverse upward: {file_path}")
+    if source_root is not None and not file_path.is_relative_to(source_root):
+        raise error_factory(f"import source path escapes the selected root: {file_path}")
+
+    open_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
+    try:
+        descriptor = os.open(file_path, open_flags)
+    except OSError as exc:
+        if exc.errno in _SYMLINK_OPEN_ERRNOS:
+            raise error_factory(
+                f"import source must not contain symlinked files: {file_path}"
+            ) from exc
+        raise error_factory(f"import source file is not readable: {file_path}") from exc
+
+    try:
+        # Checked on the descriptor rather than the path, so a swap between the
+        # listing and the open cannot change what is measured. Without this a
+        # FIFO under the root imports as an empty document, and on a platform
+        # where the read-open parks, it hangs the import instead.
+        opened_status = os.fstat(descriptor)
+        if not stat.S_ISREG(opened_status.st_mode):
+            raise error_factory(
+                f"import source file is not a regular file: {file_path}"
+            )
+        descriptor_flags = fcntl.fcntl(descriptor, fcntl.F_GETFL)
+        fcntl.fcntl(descriptor, fcntl.F_SETFL, descriptor_flags & ~os.O_NONBLOCK)
+    except BaseException:
+        os.close(descriptor)
+        raise
+
+    with os.fdopen(descriptor, "r", encoding="utf-8") as stream:
+        try:
+            return stream.read()
+        except UnicodeDecodeError as exc:
+            # Decoding is part of reading the file, so a bad byte has to leave
+            # as the caller's own validation error naming the file. Raised bare
+            # it surfaces as a byte offset with no path attached, which tells an
+            # operator nothing about which file to go and look at.
+            raise error_factory(
+                f"import source file is not valid UTF-8 text: {file_path}"
+            ) from exc
+
+
+def snapshot_source_files(
+    source_root: Path,
+    files: Iterable[Path],
+    *,
+    error_factory: ErrorFactory,
+) -> list[ImportSourceFile]:
+    """Read every selected file once, in listing order."""
+
+    return [
+        ImportSourceFile(
+            path=file_path,
+            relative_path=_relative_source_file(source_root, file_path),
+            text=read_contained_source_text(
+                file_path,
+                source_root=source_root,
+                error_factory=error_factory,
+            ),
+        )
+        for file_path in files
+    ]
+
+
+__all__ = [
+    "ImportSourceFile",
+    "contained_source_files",
+    "read_contained_source_text",
+    "snapshot_source_files",
+]

apps/api/src/alicebot_api/local_server.py

@@ -22,7 +22,10 @@ def main() -> int:
         "alicebot_api.main:app",
         host=settings.app_host,
         port=settings.app_port,
-        reload=_env_flag("APP_RELOAD", default=True),
+        # Reload watches the source tree and re-executes on change. That is a
+        # development convenience, never a serving posture, so it stays off
+        # unless APP_RELOAD explicitly turns it on.
+        reload=_env_flag("APP_RELOAD", default=False),
         access_log=settings.app_access_log,
         log_config=build_uvicorn_log_config(settings),
     )