security: make the secret-detection gates actually reject

.env.example

@@ -2,12 +2,12 @@
 # Copy this file to .env and fill in your credentials
 
 # Graphistry/Louie credentials
-GRAPHISTRY_SERVER=graphistry-dev.grph.xyz
-GRAPHISTRY_USERNAME=your_username
-GRAPHISTRY_PASSWORD=your_password
+GRAPHISTRY_SERVER = hub.graphistry.com
+GRAPHISTRY_USERNAME = your_username
+GRAPHISTRY_PASSWORD = your_password
 
 # Alternative server URL (if different)
-# LOUIE_SERVER=louie-dev.grph.xyz
+# LOUIE_SERVER=https://den.louie.ai
 
 # API version (usually 3)
-GRAPHISTRY_API_VERSION=3
+GRAPHISTRY_API_VERSION = 3

.github/workflows/ci.yml

@@ -24,17 +24,21 @@ jobs:
           curl -LsSf https://astral.sh/uv/install.sh | sh
           echo "$HOME/.cargo/bin" >> $GITHUB_PATH
       - name: Install dependencies
-        run: |
-          uv venv --python python3.12
-          uv pip install -e ".[dev]"
+        # Lockfile-pinned: lint/format/type results must be reproducible.
+        # `uv pip install -e ".[dev]"` ignores uv.lock and resolves floors
+        # (ruff>=0.12.0) to whatever is newest, so CI silently drifted to
+        # ruff 0.16 / mypy 2.3 while uv.lock pins 0.12.5 / 1.17.0.
+        # --locked (not --frozen) so a lockfile that has drifted from
+        # pyproject.toml fails the build instead of installing a stale set.
+        run: uv sync --locked --all-extras
       - name: Secret Detection
         run: ./scripts/ci/secret-detection.sh
       - name: Lint (ruff)
-        run: uv run ruff check .
+        run: uv run --locked ruff check .
       - name: Format (ruff)
-        run: uv run ruff format --check .
+        run: uv run --locked ruff format --check .
       - name: Type Check (mypy)
-        run: uv run mypy .
+        run: uv run --locked mypy .
 
   # Stage 2: Light smoke test (quick test run before full matrix)
   light-smoke-test:
@@ -159,7 +163,7 @@ jobs:
         run: |
           # Only run if credentials are configured
           if [ -n "$GRAPHISTRY_USERNAME" ]; then
-            uv run pytest tests/integration/ -v --cov=louieai --cov-report=xml
+            uv run --locked pytest tests/integration/ -v --cov=louieai --cov-report=xml
           else
             echo "⚠️  Skipping integration tests - no credentials configured"
           fi
@@ -187,17 +191,19 @@ jobs:
           curl -LsSf https://astral.sh/uv/install.sh | sh
           echo "$HOME/.cargo/bin" >> $GITHUB_PATH
       - name: Install dependencies
-        run: |
-          uv venv --python python3.12
-          uv pip install -e ".[dev,docs]"
+        # Lockfile-pinned: `mkdocs build --strict` is a determinism gate, so a
+        # floating mkdocs/plugin version must not change pass/fail. See the
+        # matching note in quality-checks.
+        run: uv sync --locked --all-extras
       - name: Test documentation examples
         run: |
-          uv run pytest tests/unit/test_documentation.py -v
+          uv run --locked pytest tests/unit/test_documentation.py -v
       - name: Validate ReadTheDocs config
         run: |
           curl -sSL https://raw.githubusercontent.com/readthedocs/readthedocs.org/main/readthedocs/rtd_tests/fixtures/spec/v2/schema.json -o rtd-schema.json
-          uv pip install jsonschema pyyaml
-          uv run python -c "
+          # jsonschema and pyyaml already come from `uv sync --all-extras`; a
+          # bare `uv pip install` here would perturb the pinned environment.
+          uv run --locked python -c "
           import yaml, json, jsonschema
           with open('.readthedocs.yml') as f:
               config = yaml.safe_load(f)
@@ -207,7 +213,7 @@ jobs:
           print('✅ ReadTheDocs config is valid')
           "
       - name: Build documentation
-        run: uv run mkdocs build --strict
+        run: uv run --locked mkdocs build --strict
       - name: Verify logo included
         run: |
           test -f site/assets/louie-logo.png || (echo "❌ Logo not found in built docs" && exit 1)

.github/workflows/secret-detection-test.yml

@@ -4,12 +4,15 @@ on:
     paths:
       # Only run when secret detection files change
       - 'scripts/ci/secret-detection.sh'
+      - 'scripts/ci/check_credential_literals.py'
+      - 'scripts/ci/check_new_secrets.py'
       - 'scripts/pre-commit-secret-check.sh'
       - 'scripts/test-secret-detection.sh'
       - 'scripts/secrets.sh'
       - '.secrets.baseline'
       - '.secret-patterns.md'
       - 'tests/secret_patterns_reference.py'
+      - 'tests/unit/security/**'
       - '.github/workflows/secret-detection-test.yml'
 
 jobs:
@@ -29,22 +32,26 @@ jobs:
           echo "$HOME/.cargo/bin" >> $GITHUB_PATH
 
       - name: Install dependencies
-        run: |
-          uv venv --python python3.12
-          uv pip install detect-secrets
-      
+        # Lockfile-pinned so detect-secrets' plugin set — which decides what the
+        # harness expectations mean — cannot drift between runs.
+        run: uv sync --locked --all-extras
+
       - name: Test secret detection patterns
-        run: |
-          echo "🔬 Testing secret detection system..."
-          # For now, just verify scripts run without error
-          # Full pattern testing is complex due to detect-secrets heuristics
-          echo "Skipping pattern tests - verifying script execution only"
+        # Previously this step only echoed "Skipping pattern tests", so the
+        # harness was never executed in CI. It was also broken: it scanned an
+        # absolute /tmp path while cd'ed to the repo root, which made every
+        # detect-secrets scan return empty, so all five "unsafe" fixtures
+        # silently looked undetected. Both are fixed; run it for real.
+        run: ./scripts/test-secret-detection.sh
+
+      - name: Test credential-literal gate
+        run: uv run --locked pytest tests/unit/security/ -q
 
       - name: Verify test patterns file
         run: |
           echo "📋 Verifying test patterns file..."
           # Check that safe patterns don't trigger
-          if uv run detect-secrets scan tests/secret_patterns_reference.py | grep -q "SAFE_"; then
+          if uv run --locked detect-secrets scan tests/secret_patterns_reference.py | grep -q "SAFE_"; then
             echo "❌ Safe patterns are triggering detection!"
             exit 1
           fi

.pre-commit-config.yaml

@@ -1,12 +1,16 @@
+# Versions must match uv.lock, which is what CI installs via `uv sync --frozen`.
+# Skew here is the same failure mode CI just hit: ruff 0.16 formats Markdown code
+# fences and 0.12.5 does not, so a hook running a different version produces
+# diffs the CI gate rejects.
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.4
+    rev: v0.12.5
     hooks:
       - id: ruff
         args: [--fix]
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.17.0
     hooks:
       - id: mypy
   - repo: https://github.com/pre-commit/pygrep-hooks

.secret-patterns.md

@@ -23,7 +23,13 @@ uv run detect-secrets scan tests/secret_patterns_reference.py
 - `****` or `********` - Masked placeholder
 
 ## API Keys
-- `sk-XXXXXXXXXXXXXXXX` - Clearly fake key pattern
+- `sk-XXXXXXXXXXXXXXXX` - Clearly fake key pattern. **Caveat:** when assigned to
+  a credential-ish name (`API_KEY = "sk-XXXXXXXXXXXXXXXX"`), detect-secrets'
+  `KeywordDetector` still reports it as `Secret Keyword` — its placeholder
+  filters recognise `token-XXXX-XXXX-XXXX` and `your-api-key-here` but not this
+  form. Either append `# pragma: allowlist secret` or keep the value out of a
+  keyword-adjacent assignment. `scripts/test-secret-detection.sh` asserts this
+  real behaviour.
 - `api-key-XXXX-XXXX-XXXX` - Placeholder pattern
 - `<your-api-key>` - Clear placeholder
 - `YOUR_API_KEY_HERE` - Uppercase placeholder
@@ -36,9 +42,78 @@ uv run detect-secrets scan tests/secret_patterns_reference.py
 ## Other
 - `secret-XXXX` - Generic secret placeholder
 - `<your-secret>` - Clear placeholder
+- `<your-personal-key-id>` - Graphistry personal key ID placeholder
+- `<your-personal-key-secret>` - Graphistry personal key secret placeholder
 - `localhost` or `example.com` - Safe domains
 - `user@example.com` - Example email
 
+Graphistry `personal_key_id` and `personal_key_secret` assignments receive an
+additional deterministic tracked-file check because short keys may not meet
+generic entropy thresholds. The checker reports only the file, line, and key
+name; it never prints the value.
+
+## Internal names: what is screened where
+
+`scripts/ci/check_credential_literals.py` rejects internal hostnames
+(`*.grph.xyz`, `*.louie.internal`) in tracked files. Use RFC 2606 example
+domains (`louie.example.com`) in fixtures, or the public endpoints
+(`hub.graphistry.com`, `https://den.louie.ai`) in user-facing docs.
+
+The rule is deliberately **domain-level, not a list of known hosts** — an
+enumerated list misses the next subdomain nobody thought of.
+
+Specific account and organisation names are screened **locally only**, in
+`.git/hooks/pre-commit`. That is not an oversight: this repository is public, so
+a tracked denylist of the private strings we are trying to keep out would
+publish them. Anything generalizable enough to express as a pattern belongs in
+the tracked gate; anything that is itself the sensitive value stays local. The
+local hook is not installed by default and CI does not run it, so treat it as a
+convenience, not a control.
+
+## Writing tests that contain deliberate fake secrets
+
+Security tests must plant realistic secrets to prove the gates reject them — and
+the gates then flag those fixtures. This is the gates working, not a bug. Test
+files are deliberately **not** exempt: the original credential leak in this repo
+was duplicated into `tests/unit/test_org_auth_flow.py`, so a blanket exclusion
+for `tests/` would have hidden it.
+
+Three ways to write a fixture, in order of preference:
+
+1. **Assemble it at runtime** so no scannable literal exists in the file:
+
+   ```python
+   FAKE_SECRET = "A1B2C3D4" + "E5F6G7H8"  # pragma: allowlist secret
+   KEY_SECRET = "personal_key_" + "secret"  # pragma: allowlist secret
+   ```
+
+   This is what `tests/unit/security/test_credential_literals.py` does throughout.
+
+   Runtime assembly defeats the *value*-matching rules, but `detect-secrets`'
+   `KeywordDetector` fires on the **variable name** (anything containing `key`,
+   `secret`, `password`, `token`) regardless of the value. So a constant named
+   `FAKE_SECRET` still needs a pragma even when its value is assembled — the two
+   techniques compose rather than substitute.
+
+2. **Add a same-line pragma** when the literal must be inline:
+
+   ```python
+   token = "zQ3RtP8xL2mN7vB4kW9jH6dF1sA5gY0c"  # pragma: allowlist secret
+   ```
+
+   It must be on the **same line** as the value — both `detect-secrets` and
+   `check_credential_literals.py` match per line. A pragma on the preceding line
+   does nothing.
+
+3. **Exclude the path** for files that cannot carry a comment at all — an
+   `.ipynb` data URI, a CSV row — via
+   `check_credential_literals.py --exclude-files <regex>`. Prefer 1 or 2; an
+   exclusion is a permanent hole.
+
+Never re-baseline to silence a fixture. `.secrets.baseline` accepts a value
+everywhere in the tree, so baselining a fake secret also accepts a real one that
+happens to match.
+
 Note: Real patterns to avoid:
 - Actual API keys (even expired ones)
 - Real passwords (even old ones)