feat(gitlab): add project wiki support to the GitLab connector

CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [Unreleased]
+
+### Added
+
+- **GitLab project wikis**: add `?wiki=true` to a `gitlab:` source to sync a project's wiki instead of its repository (e.g. `gitlab:owner/repo?wiki=true`). Each wiki page becomes a file, preserving the page hierarchy from its slug, via the Project Wikis API. Reuses `GITLAB_TOKEN`/`GITLAB_URL`; no new dependency.
+- **GitLab numeric project IDs**: a `gitlab:` source may be a numeric project ID (e.g. `gitlab:42`, `gitlab:42?wiki=true`), used directly as the API `:id` — handy when the namespace path is deep or awkward to encode.
+- **GitLab subgroup paths**: sources now accept the full nested namespace (e.g. `gitlab:group/subgroup/project?wiki=true`), URL-encoding the whole project path.
+### Changed
+
+- GitLab wiki sync uses `with_content=1` on the wikis list endpoint when available and falls back to fetching each page via `GET /wikis/:slug` on older GitLab instances that don't return page content inline.
+
 ## [0.3.6] - 2026-05-28
 
 ### Added

docs/guide.md

@@ -242,6 +242,21 @@ oikb sync bitbucket:owner/repo --kb-id your-kb-id
 
 Requires `GITLAB_TOKEN` or `BITBUCKET_TOKEN` respectively.
 
+To sync a project's **wiki** instead of its repository, add `?wiki=true` to the
+GitLab source. Each wiki page becomes one file, preserving the page hierarchy
+(an `api/auth` slug syncs as `api/auth.md`). `GITLAB_TOKEN`/`GITLAB_URL` are the
+same as for repositories; `--branch` and `--path` don't apply in wiki mode.
+
+The project can be given as a path or as a numeric project ID — the numeric form
+avoids namespace/subgroup encoding entirely (find it under the project's
+Settings → General in GitLab):
+
+```bash
+oikb sync "gitlab:owner/repo?wiki=true" --kb-id your-kb-id
+oikb sync "gitlab:group/subgroup/project?wiki=true" --kb-id your-kb-id
+oikb sync "gitlab:42?wiki=true" --kb-id your-kb-id
+```
+
 ### Confluence
 
 ```bash

src/oikb/cli.py

@@ -53,7 +53,14 @@ def _resolve_connector(source: str, branch: str | None = None, path: str | None
     if source.startswith("gitlab:"):
         from oikb.connectors.gitlab import GitLabConnector, parse_gitlab_source
         parsed = parse_gitlab_source(source)
-        return GitLabConnector(owner=parsed["owner"], repo=parsed["repo"], branch=branch, path=path or parsed.get("path"))
+        return GitLabConnector(
+            owner=parsed.get("owner"),
+            repo=parsed.get("repo"),
+            project_id=parsed.get("project_id"),
+            branch=branch,
+            path=path or parsed.get("path"),
+            is_wiki=bool(parsed.get("wiki")),
+        )
 
     if source.startswith("s3://"):
         from oikb.connectors.s3 import S3Connector, parse_s3_source

src/oikb/connectors/gitlab.py

@@ -1,44 +1,64 @@
-"""GitLab connector — sync a GitLab repo to a Knowledge Base via the API.
+"""GitLab connector — sync a GitLab repo (or its project wiki) to a KB via the API.
 
-Requires: pip install oikb[gitlab]  (uses httpx, already a core dep)
-Uses the GitLab Repository Tree API — no local clone needed.
+Uses the GitLab Repository Tree API for repositories and the Project Wikis API
+for wikis — no local clone needed. Set GITLAB_TOKEN (and GITLAB_URL for a
+self-managed instance).
 """
 
 from __future__ import annotations
 
+import hashlib
 import os
-from typing import Any
+import urllib.parse
 
 import httpx
 
 from oikb.connectors import BaseConnector, ManifestEntry
 
+# GitLab wiki markup formats → file extension (wiki mode only).
+_WIKI_FORMAT_EXT = {
+    "markdown": ".md",
+    "rdoc": ".rdoc",
+    "asciidoc": ".adoc",
+    "org": ".org",
+}
+
 
 class GitLabConnector(BaseConnector):
     """Sync files from a GitLab repository.
 
     Args:
-        owner:    Project namespace (e.g. "open-webui").
-        repo:     Project name (e.g. "docs").
-        branch:   Branch to sync from (default: project default branch).
-        path:     Subdirectory to scope to (e.g. "docs/").
-        token:    GitLab personal access token (or GITLAB_TOKEN env var).
-        base_url: GitLab instance URL (default: https://gitlab.com).
+        owner:      Project namespace (e.g. "open-webui"). Omit if project_id
+                    is given.
+        repo:       Project name (e.g. "docs"). Omit if project_id is given.
+        branch:     Branch to sync from (default: project default branch).
+        path:       Subdirectory to scope to (e.g. "docs/").
+        token:      GitLab personal access token (or GITLAB_TOKEN env var).
+        base_url:   GitLab instance URL (default: https://gitlab.com).
+        is_wiki:    Sync the project wiki instead of the repository. In wiki
+                    mode ``branch`` and ``path`` are ignored (default: False).
+        project_id: Explicit GitLab project ID (numeric) or pre-encoded path,
+                    used verbatim instead of owner/repo.
     """
 
     def __init__(
         self,
-        owner: str,
-        repo: str,
+        owner: str | None = None,
+        repo: str | None = None,
         branch: str | None = None,
         path: str | None = None,
         token: str | None = None,
         base_url: str | None = None,
+        is_wiki: bool = False,
+        project_id: str | None = None,
     ):
         self.owner = owner
         self.repo = repo
         self.branch = branch
         self.path = path.strip("/") if path else None
+        self.is_wiki = is_wiki
+        # Populated by build_manifest() in wiki mode; read_file() serves from it.
+        self._cache: dict[str, str] = {}
         self._token = token or os.environ.get("GITLAB_TOKEN")
         self._base_url = (base_url or os.environ.get("GITLAB_URL", "https://gitlab.com")).rstrip("/")
 
@@ -52,14 +72,24 @@ def __init__(
             timeout=60.0,
         )
 
-        # URL-encode the project path for GitLab's API.
-        self._project_id = f"{self.owner}%2F{self.repo}"
+        # GitLab's :id accepts a numeric project ID or the URL-encoded project
+        # path. An explicit project_id (e.g. "42") is used as-is and avoids any
+        # namespace/subgroup guesswork; otherwise encode the whole
+        # "namespace/project" so subgroup paths stay correct.
+        if project_id is not None:
+            self._project_id = str(project_id)
+        else:
+            self._project_id = urllib.parse.quote(f"{self.owner}/{self.repo}", safe="")
 
     def build_manifest(self) -> list[ManifestEntry]:
-        """Fetch the repo tree and build a manifest.
+        """Fetch the repo tree (or wiki pages) and build a manifest.
 
-        Uses the recursive tree API. Blob IDs are content-addressable hashes.
+        Repo mode uses the recursive tree API; blob IDs are content-addressable
+        hashes. Wiki mode uses the Project Wikis API.
         """
+        if self.is_wiki:
+            return self._build_wiki_manifest()
+
         ref = self.branch or self._get_default_branch()
         entries: list[ManifestEntry] = []
 
@@ -115,8 +145,10 @@ def build_manifest(self) -> list[ManifestEntry]:
         return entries
 
     def read_file(self, path: str, filename: str) -> bytes:
-        """Download a file's raw content via the GitLab Repository Files API."""
-        import urllib.parse
+        """Return raw file content (repo) or the cached wiki page body."""
+        if self.is_wiki:
+            key = f"{path}/{filename}" if path else filename
+            return (self._cache.get(key) or "").encode("utf-8")
 
         file_path = f"{path}/{filename}" if path else filename
         if self.path:
@@ -132,6 +164,60 @@ def read_file(self, path: str, filename: str) -> bytes:
         resp.raise_for_status()
         return resp.content
 
+    def _build_wiki_manifest(self) -> list[ManifestEntry]:
+        """Fetch every wiki page and build a manifest.
+
+        The list endpoint returns all pages; with_content=1 includes each body
+        in a single request on GitLab 16.4+. Older instances silently ignore the
+        flag, so any page returned without content is fetched individually via
+        GET /wikis/:slug.
+        """
+        resp = self._http.get(
+            f"/projects/{self._project_id}/wikis",
+            params={"with_content": 1},
+        )
+        resp.raise_for_status()
+
+        entries: list[ManifestEntry] = []
+        for page in resp.json():
+            slug = page.get("slug")
+            if not slug:
+                continue
+
+            content = page.get("content")
+            if content is None:  # older GitLab ignored with_content — fetch it.
+                content = self._fetch_wiki_page(slug)
+
+            title = page.get("title") or slug
+            text = f"# {title}\n\n{content or ''}"
+
+            ext = _WIKI_FORMAT_EXT.get(page.get("format", "markdown"), ".md")
+            if "/" in slug:  # Wiki slugs are hierarchical, e.g. "api/auth".
+                dir_path, base = slug.rsplit("/", 1)
+            else:
+                dir_path, base = "", slug
+
+            entry = ManifestEntry(
+                filename=f"{base}{ext}",
+                path=dir_path,
+                checksum=hashlib.sha256(text.encode()).hexdigest()[:16],
+                size=len(text.encode()),
+            )
+            entries.append(entry)
+            # Key by full display path — two pages can share a basename
+            # (api/index vs guide/index).
+            self._cache[entry.display_path] = text
+
+        entries.sort(key=lambda e: e.display_path)
+        return entries
+
+    def _fetch_wiki_page(self, slug: str) -> str:
+        """Fetch one wiki page's content by slug (GET /wikis/:slug)."""
+        encoded = urllib.parse.quote(slug, safe="")
+        resp = self._http.get(f"/projects/{self._project_id}/wikis/{encoded}")
+        resp.raise_for_status()
+        return resp.json().get("content") or ""
+
     def _get_default_branch(self) -> str:
         """Fetch the project's default branch name."""
         resp = self._http.get(f"/projects/{self._project_id}")
@@ -142,21 +228,44 @@ def close(self) -> None:
         self._http.close()
 
 
-def parse_gitlab_source(source: str) -> dict[str, str | None]:
-    """Parse a gitlab:owner/repo[/path] source string.
+def parse_gitlab_source(source: str) -> dict[str, str | bool | None]:
+    """Parse a gitlab source string.
 
-    Examples:
-        gitlab:open-webui/docs
-        gitlab:open-webui/docs/api
+    Forms:
+        gitlab:owner/repo                  # repository
+        gitlab:owner/repo/subdir           # repository, scoped to a subdirectory
+        gitlab:owner/repo?wiki=true         # project wiki
+        gitlab:group/subgroup/proj?wiki=true  # wiki of a subgroup project
+        gitlab:42                          # repository by numeric project ID
+        gitlab:42?wiki=true                # wiki by numeric project ID
     """
-    source = source.removeprefix("gitlab:")
+    raw = source.removeprefix("gitlab:")
+    base, _, query = raw.partition("?")
+
+    is_wiki = False
+    for param in query.split("&"):
+        key, _, value = param.partition("=")
+        if key == "wiki":
+            is_wiki = value.lower() in ("", "1", "true", "yes")
+
+    # A bare numeric project ID is used directly as GitLab's :id — handy when
+    # the namespace path is deep, awkward to encode, or simply unknown.
+    if base.isdigit():
+        return {"owner": None, "repo": None, "path": None, "project_id": base, "wiki": is_wiki}
 
-    parts = source.split("/", 2)
-    if len(parts) < 2:
+    if "/" not in base:
         raise ValueError(f"Invalid GitLab source: {source}. Expected: gitlab:owner/repo")
 
+    if is_wiki:
+        # Wikis have no subdirectory concept, and self-managed projects often
+        # live under nested subgroups (group/subgroup/project) — so treat the
+        # entire path as the project: everything up to the last "/" is the
+        # namespace, the final segment is the project.
+        owner, repo = base.rsplit("/", 1)
+        return {"owner": owner, "repo": repo, "path": None, "project_id": None, "wiki": True}
+
+    parts = base.split("/", 2)
     owner = parts[0]
     repo = parts[1]
     path = parts[2] if len(parts) > 2 else None
-
-    return {"owner": owner, "repo": repo, "path": path}
+    return {"owner": owner, "repo": repo, "path": path, "project_id": None, "wiki": False}