diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c2ca60..0fb522d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [Unreleased] + +### Added + +- **GitLab project wikis**: add `?wiki=true` to a `gitlab:` source to sync a project's wiki instead of its repository (e.g. `gitlab:owner/repo?wiki=true`). Each wiki page becomes a file, preserving the page hierarchy from its slug, via the Project Wikis API. Reuses `GITLAB_TOKEN`/`GITLAB_URL`; no new dependency. +- **GitLab numeric project IDs**: a `gitlab:` source may be a numeric project ID (e.g. `gitlab:42`, `gitlab:42?wiki=true`), used directly as the API `:id` — handy when the namespace path is deep or awkward to encode. +- **GitLab subgroup paths**: sources now accept the full nested namespace (e.g. `gitlab:group/subgroup/project?wiki=true`), URL-encoding the whole project path. +### Changed + +- GitLab wiki sync uses `with_content=1` on the wikis list endpoint when available and falls back to fetching each page via `GET /wikis/:slug` on older GitLab instances that don't return page content inline. + ## [0.3.6] - 2026-05-28 ### Added diff --git a/docs/guide.md b/docs/guide.md index 905d74a..4d39a87 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -242,6 +242,21 @@ oikb sync bitbucket:owner/repo --kb-id your-kb-id Requires `GITLAB_TOKEN` or `BITBUCKET_TOKEN` respectively. +To sync a project's **wiki** instead of its repository, add `?wiki=true` to the +GitLab source. Each wiki page becomes one file, preserving the page hierarchy +(an `api/auth` slug syncs as `api/auth.md`). `GITLAB_TOKEN`/`GITLAB_URL` are the +same as for repositories; `--branch` and `--path` don't apply in wiki mode. + +The project can be given as a path or as a numeric project ID — the numeric form +avoids namespace/subgroup encoding entirely (find it under the project's +Settings → General in GitLab): + +```bash +oikb sync "gitlab:owner/repo?wiki=true" --kb-id your-kb-id +oikb sync "gitlab:group/subgroup/project?wiki=true" --kb-id your-kb-id +oikb sync "gitlab:42?wiki=true" --kb-id your-kb-id +``` + ### Confluence ```bash diff --git a/src/oikb/cli.py b/src/oikb/cli.py index 39250e2..e26b746 100644 --- a/src/oikb/cli.py +++ b/src/oikb/cli.py @@ -53,7 +53,14 @@ def _resolve_connector(source: str, branch: str | None = None, path: str | None if source.startswith("gitlab:"): from oikb.connectors.gitlab import GitLabConnector, parse_gitlab_source parsed = parse_gitlab_source(source) - return GitLabConnector(owner=parsed["owner"], repo=parsed["repo"], branch=branch, path=path or parsed.get("path")) + return GitLabConnector( + owner=parsed.get("owner"), + repo=parsed.get("repo"), + project_id=parsed.get("project_id"), + branch=branch, + path=path or parsed.get("path"), + is_wiki=bool(parsed.get("wiki")), + ) if source.startswith("s3://"): from oikb.connectors.s3 import S3Connector, parse_s3_source diff --git a/src/oikb/connectors/gitlab.py b/src/oikb/connectors/gitlab.py index da17686..13fe476 100644 --- a/src/oikb/connectors/gitlab.py +++ b/src/oikb/connectors/gitlab.py @@ -1,44 +1,64 @@ -"""GitLab connector — sync a GitLab repo to a Knowledge Base via the API. +"""GitLab connector — sync a GitLab repo (or its project wiki) to a KB via the API. -Requires: pip install oikb[gitlab] (uses httpx, already a core dep) -Uses the GitLab Repository Tree API — no local clone needed. +Uses the GitLab Repository Tree API for repositories and the Project Wikis API +for wikis — no local clone needed. Set GITLAB_TOKEN (and GITLAB_URL for a +self-managed instance). """ from __future__ import annotations +import hashlib import os -from typing import Any +import urllib.parse import httpx from oikb.connectors import BaseConnector, ManifestEntry +# GitLab wiki markup formats → file extension (wiki mode only). +_WIKI_FORMAT_EXT = { + "markdown": ".md", + "rdoc": ".rdoc", + "asciidoc": ".adoc", + "org": ".org", +} + class GitLabConnector(BaseConnector): """Sync files from a GitLab repository. Args: - owner: Project namespace (e.g. "open-webui"). - repo: Project name (e.g. "docs"). - branch: Branch to sync from (default: project default branch). - path: Subdirectory to scope to (e.g. "docs/"). - token: GitLab personal access token (or GITLAB_TOKEN env var). - base_url: GitLab instance URL (default: https://gitlab.com). + owner: Project namespace (e.g. "open-webui"). Omit if project_id + is given. + repo: Project name (e.g. "docs"). Omit if project_id is given. + branch: Branch to sync from (default: project default branch). + path: Subdirectory to scope to (e.g. "docs/"). + token: GitLab personal access token (or GITLAB_TOKEN env var). + base_url: GitLab instance URL (default: https://gitlab.com). + is_wiki: Sync the project wiki instead of the repository. In wiki + mode ``branch`` and ``path`` are ignored (default: False). + project_id: Explicit GitLab project ID (numeric) or pre-encoded path, + used verbatim instead of owner/repo. """ def __init__( self, - owner: str, - repo: str, + owner: str | None = None, + repo: str | None = None, branch: str | None = None, path: str | None = None, token: str | None = None, base_url: str | None = None, + is_wiki: bool = False, + project_id: str | None = None, ): self.owner = owner self.repo = repo self.branch = branch self.path = path.strip("/") if path else None + self.is_wiki = is_wiki + # Populated by build_manifest() in wiki mode; read_file() serves from it. + self._cache: dict[str, str] = {} self._token = token or os.environ.get("GITLAB_TOKEN") self._base_url = (base_url or os.environ.get("GITLAB_URL", "https://gitlab.com")).rstrip("/") @@ -52,14 +72,24 @@ def __init__( timeout=60.0, ) - # URL-encode the project path for GitLab's API. - self._project_id = f"{self.owner}%2F{self.repo}" + # GitLab's :id accepts a numeric project ID or the URL-encoded project + # path. An explicit project_id (e.g. "42") is used as-is and avoids any + # namespace/subgroup guesswork; otherwise encode the whole + # "namespace/project" so subgroup paths stay correct. + if project_id is not None: + self._project_id = str(project_id) + else: + self._project_id = urllib.parse.quote(f"{self.owner}/{self.repo}", safe="") def build_manifest(self) -> list[ManifestEntry]: - """Fetch the repo tree and build a manifest. + """Fetch the repo tree (or wiki pages) and build a manifest. - Uses the recursive tree API. Blob IDs are content-addressable hashes. + Repo mode uses the recursive tree API; blob IDs are content-addressable + hashes. Wiki mode uses the Project Wikis API. """ + if self.is_wiki: + return self._build_wiki_manifest() + ref = self.branch or self._get_default_branch() entries: list[ManifestEntry] = [] @@ -115,8 +145,10 @@ def build_manifest(self) -> list[ManifestEntry]: return entries def read_file(self, path: str, filename: str) -> bytes: - """Download a file's raw content via the GitLab Repository Files API.""" - import urllib.parse + """Return raw file content (repo) or the cached wiki page body.""" + if self.is_wiki: + key = f"{path}/{filename}" if path else filename + return (self._cache.get(key) or "").encode("utf-8") file_path = f"{path}/{filename}" if path else filename if self.path: @@ -132,6 +164,60 @@ def read_file(self, path: str, filename: str) -> bytes: resp.raise_for_status() return resp.content + def _build_wiki_manifest(self) -> list[ManifestEntry]: + """Fetch every wiki page and build a manifest. + + The list endpoint returns all pages; with_content=1 includes each body + in a single request on GitLab 16.4+. Older instances silently ignore the + flag, so any page returned without content is fetched individually via + GET /wikis/:slug. + """ + resp = self._http.get( + f"/projects/{self._project_id}/wikis", + params={"with_content": 1}, + ) + resp.raise_for_status() + + entries: list[ManifestEntry] = [] + for page in resp.json(): + slug = page.get("slug") + if not slug: + continue + + content = page.get("content") + if content is None: # older GitLab ignored with_content — fetch it. + content = self._fetch_wiki_page(slug) + + title = page.get("title") or slug + text = f"# {title}\n\n{content or ''}" + + ext = _WIKI_FORMAT_EXT.get(page.get("format", "markdown"), ".md") + if "/" in slug: # Wiki slugs are hierarchical, e.g. "api/auth". + dir_path, base = slug.rsplit("/", 1) + else: + dir_path, base = "", slug + + entry = ManifestEntry( + filename=f"{base}{ext}", + path=dir_path, + checksum=hashlib.sha256(text.encode()).hexdigest()[:16], + size=len(text.encode()), + ) + entries.append(entry) + # Key by full display path — two pages can share a basename + # (api/index vs guide/index). + self._cache[entry.display_path] = text + + entries.sort(key=lambda e: e.display_path) + return entries + + def _fetch_wiki_page(self, slug: str) -> str: + """Fetch one wiki page's content by slug (GET /wikis/:slug).""" + encoded = urllib.parse.quote(slug, safe="") + resp = self._http.get(f"/projects/{self._project_id}/wikis/{encoded}") + resp.raise_for_status() + return resp.json().get("content") or "" + def _get_default_branch(self) -> str: """Fetch the project's default branch name.""" resp = self._http.get(f"/projects/{self._project_id}") @@ -142,21 +228,44 @@ def close(self) -> None: self._http.close() -def parse_gitlab_source(source: str) -> dict[str, str | None]: - """Parse a gitlab:owner/repo[/path] source string. +def parse_gitlab_source(source: str) -> dict[str, str | bool | None]: + """Parse a gitlab source string. - Examples: - gitlab:open-webui/docs - gitlab:open-webui/docs/api + Forms: + gitlab:owner/repo # repository + gitlab:owner/repo/subdir # repository, scoped to a subdirectory + gitlab:owner/repo?wiki=true # project wiki + gitlab:group/subgroup/proj?wiki=true # wiki of a subgroup project + gitlab:42 # repository by numeric project ID + gitlab:42?wiki=true # wiki by numeric project ID """ - source = source.removeprefix("gitlab:") + raw = source.removeprefix("gitlab:") + base, _, query = raw.partition("?") + + is_wiki = False + for param in query.split("&"): + key, _, value = param.partition("=") + if key == "wiki": + is_wiki = value.lower() in ("", "1", "true", "yes") + + # A bare numeric project ID is used directly as GitLab's :id — handy when + # the namespace path is deep, awkward to encode, or simply unknown. + if base.isdigit(): + return {"owner": None, "repo": None, "path": None, "project_id": base, "wiki": is_wiki} - parts = source.split("/", 2) - if len(parts) < 2: + if "/" not in base: raise ValueError(f"Invalid GitLab source: {source}. Expected: gitlab:owner/repo") + if is_wiki: + # Wikis have no subdirectory concept, and self-managed projects often + # live under nested subgroups (group/subgroup/project) — so treat the + # entire path as the project: everything up to the last "/" is the + # namespace, the final segment is the project. + owner, repo = base.rsplit("/", 1) + return {"owner": owner, "repo": repo, "path": None, "project_id": None, "wiki": True} + + parts = base.split("/", 2) owner = parts[0] repo = parts[1] path = parts[2] if len(parts) > 2 else None - - return {"owner": owner, "repo": repo, "path": path} + return {"owner": owner, "repo": repo, "path": path, "project_id": None, "wiki": False} \ No newline at end of file