From e31a79f52174318893ced31fed82ebe9f19ad0ae Mon Sep 17 00:00:00 2001 From: yiaany Date: Sat, 29 Aug 2026 21:06:01 +0500 Subject: [PATCH 01/12] docs: add full-site version snapshots --- .gitattributes | 2 + .github/workflows/docs.yaml | 37 ++- .github/workflows/docs_pr.yaml | 37 ++- dev/release/README.md | 23 ++ docs/README.md | 141 +++++++--- docs/scripts/assemble_site.py | 147 +++++++++++ docs/scripts/release_conf.py | 60 +++++ docs/scripts/snapshot_site.py | 282 ++++++++++++++++++++ docs/scripts/validate_site.py | 420 ++++++++++++++++++++++++++++++ docs/scripts/versioned_docs.py | 116 +++++++++ docs/source/_static/versions.json | 14 + docs/source/conf.py | 11 +- docs/tests/test_versioned_docs.py | 330 +++++++++++++++++++++++ 13 files changed, 1575 insertions(+), 45 deletions(-) create mode 100644 docs/scripts/assemble_site.py create mode 100644 docs/scripts/release_conf.py create mode 100644 docs/scripts/snapshot_site.py create mode 100644 docs/scripts/validate_site.py create mode 100644 docs/scripts/versioned_docs.py create mode 100644 docs/source/_static/versions.json create mode 100644 docs/tests/test_versioned_docs.py diff --git a/.gitattributes b/.gitattributes index 84b47a6fc56e1..75ac548699177 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,6 @@ .github/ export-ignore +docs/*.sh text eol=lf +docs/scripts/*.sh text eol=lf datafusion/core/tests/data/newlines_in_values.csv text eol=lf datafusion/proto/src/generated/prost.rs linguist-generated datafusion/proto/src/generated/pbjson.rs linguist-generated diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index c52ed5681c64f..217c6f01b5601 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,12 +20,19 @@ on: branches: - main paths: + - .gitattributes - .asf.yaml - .github/workflows/docs.yaml - docs/** + - pyproject.toml + - uv.lock name: Deploy DataFusion site +concurrency: + group: docs-deployment + cancel-in-progress: false + jobs: build-docs: permissions: @@ -34,16 +41,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout docs sources - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Checkout asf-site branch - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: asf-site path: asf-site - name: Setup uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install dependencies run: uv sync --package datafusion-docs @@ -53,7 +60,7 @@ jobs: sudo apt-get update sudo apt-get install -y graphviz - name: Install cargo-depgraph - uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 + uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 with: tool: cargo-depgraph@1.6 @@ -63,7 +70,21 @@ jobs: cd docs uv run --package datafusion-docs ./build.sh - - name: Copy & push the generated HTML + - name: Assemble deployment site + run: | + uv run --package datafusion-docs python docs/scripts/assemble_site.py \ + --current-site docs/build/html \ + --published-site asf-site \ + --output-site deployment-site + cp .asf.yaml deployment-site/ + touch deployment-site/.nojekyll + + - name: Validate deployment site + run: >- + uv run --package datafusion-docs python docs/scripts/validate_site.py + --site-root deployment-site --require-snapshots + + - name: Copy and push the generated HTML run: | set -x cd asf-site/ @@ -71,15 +92,13 @@ jobs: -a \ --delete \ --exclude '/.git/' \ - ../docs/build/html/ \ + ../deployment-site/ \ ./ - cp ../.asf.yaml . - touch .nojekyll git status --porcelain if [ "$(git status --porcelain)" != "" ]; then git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add --all git commit -m 'Publish built docs triggered by ${{ github.sha }}' - git push || git push --force + git push fi diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml index 2cbb64274a526..7035111b70fb0 100644 --- a/.github/workflows/docs_pr.yaml +++ b/.github/workflows/docs_pr.yaml @@ -25,13 +25,23 @@ concurrency: on: push: paths: + - ".gitattributes" - "docs/**" + - ".github/workflows/docs.yaml" + - ".github/workflows/docs_pr.yaml" + - "pyproject.toml" + - "uv.lock" branches: - main - branch-* pull_request: paths: + - ".gitattributes" - "docs/**" + - ".github/workflows/docs.yaml" + - ".github/workflows/docs_pr.yaml" + - "pyproject.toml" + - "uv.lock" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: @@ -45,12 +55,12 @@ jobs: name: Test doc build runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: submodules: true fetch-depth: 1 - name: Setup uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install doc dependencies run: uv sync --package datafusion-docs - name: Install Graphviz @@ -59,7 +69,7 @@ jobs: sudo apt-get update sudo apt-get install -y graphviz - name: Install cargo-depgraph - uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 + uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 with: tool: cargo-depgraph@1.6 - name: Build docs html and check for warnings @@ -67,3 +77,24 @@ jobs: set -x cd docs uv run --package datafusion-docs ./build.sh # fails on errors + - name: Fetch exact release tag + run: git fetch --depth=1 origin tag 55.0.0 + - name: Test versioned documentation tooling + run: uv run --package datafusion-docs python -m unittest discover -s docs/tests + - name: Validate current site + run: >- + uv run --package datafusion-docs python docs/scripts/validate_site.py + --site-root docs/build/html + - name: Build exact release snapshot + run: | + uv run --package datafusion-docs python docs/scripts/snapshot_site.py \ + 55.0.0 55.0.0 /tmp/datafusion-release-site + - name: Validate release deployment + run: | + uv run --package datafusion-docs python docs/scripts/assemble_site.py \ + --current-site docs/build/html \ + --published-site /tmp/datafusion-release-site \ + --output-site docs/build/validation-site + uv run --package datafusion-docs python docs/scripts/validate_site.py \ + --site-root docs/build/validation-site \ + --require-snapshots diff --git a/dev/release/README.md b/dev/release/README.md index d49c1c9125235..bc2d6beaad6b5 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -301,6 +301,29 @@ git tag 50.3.0 git push apache 50.3.0 ``` +#### Publish the versioned documentation + +For the initial `55.0.0` proof of concept, make the final exact tag available +locally and build its complete documentation site into a temporary publication +package: + +```shell +uv run --package datafusion-docs python docs/scripts/snapshot_site.py 55.0.0 55.0.0 /tmp/datafusion-release-site +``` + +The command verifies the tag's peeled commit and creates the one immutable +publication unit `versions/55.0.0/`. Manually copy that directory to the same +location on the `asf-site` branch, then review, commit, and push the branch. The +command never commits or pushes and refuses to replace an existing release. + +Do not simply publish the snapshot before merging the feature PR: the old docs +deployment can erase it with `rsync --delete`. Keep the feature PR in draft until +maintainers choose either a preliminary retention-only merge before publication, +or a coordinated window that confirms no old docs deployment is running or +queued, publishes the snapshot, merges the retention and picker change, and +verifies `asf-site/versions/55.0.0/` after the new deployment. Automating future +release publication and backfilling older releases remain follow-up work. + ### 10. Publish on Crates.io Only approved releases of the tarball should be published to diff --git a/docs/README.md b/docs/README.md index 48fdcefdeae1a..53a704a61543f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,63 +19,142 @@ # DataFusion Documentation -This folder contains the source content of the [User Guide](./source/user-guide) -and [Contributor Guide](./source/contributor-guide). These are both published to -https://datafusion.apache.org/ as part of the release process. +This folder contains the sources for https://datafusion.apache.org/. The root +site is built continuously from `main`. Complete, immutable release sites are +published under `/versions//`. ## Dependencies -Install build dependencies and build the documentation using +From the repository root, install the documentation dependencies using [uv](https://docs.astral.sh/uv/): ```sh -uv sync -uv run bash build.sh +uv sync --package datafusion-docs ``` The docs build regenerates the workspace dependency graph via `docs/scripts/generate_dependency_graph.sh`, so ensure `cargo`, `cargo-depgraph` (`cargo install cargo-depgraph --version ^1.6 --locked`), and Graphviz `dot` (`brew install graphviz` or `sudo apt-get install -y graphviz`) are available. +`.gitattributes` keeps documentation shell scripts LF-terminated so the same +scripts run from Linux and WSL checkouts. -## Build & Preview +## Build and Preview -Run the provided script to build the HTML pages. +Build the current complete site from the repository root: ```bash -# If using venv, ensure you have activated it -./build.sh +uv run --package datafusion-docs docs/build.sh ``` -The HTML will be generated into a `build` directory. Open `build/html/index.html` -in your preferred browser, e.g. +The HTML is generated in `docs/build/html`. Serve it over HTTP because browsers +do not load the version manifest from `file:` URLs: -Preview the site on Linux by running this command. +```bash +python3 -m http.server --directory docs/build/html 8000 +``` + +Then open http://localhost:8000/. + +The public and assembled layouts are: + +```text +https://datafusion.apache.org/ +|-- user-guide/ +|-- library-user-guide/ +|-- contributor-guide/ +|-- _static/versions.json +`-- versions/55.0.0/ + |-- user-guide/ + |-- library-user-guide/ + |-- contributor-guide/ + |-- download.html + |-- search.html + |-- _sources/ + |-- _static/ + `-- sitemap.xml +``` + +## Release Snapshots + +`docs/source/_static/versions.json` is both the PyData version-picker manifest +and the release catalog. It contains `Development` at the site root and records +each release's semantic version, exact tag, and exact 40-character commit. + +The first snapshot is the lightweight tag `55.0.0`, which peels to +`d5552342012888b7d1a3ab88d92e3d292fc0cde0`. Create its publication package +from the repository root with the tag available locally: ```bash -# On macOS -open build/html/index.html -# On Linux with Firefox -firefox build/html/index.html +uv run --package datafusion-docs python docs/scripts/snapshot_site.py \ + 55.0.0 55.0.0 /tmp/datafusion-release-site ``` -## Making Changes +The command creates only +`/tmp/datafusion-release-site/versions/55.0.0/`. It refuses to overwrite that +directory, rejects output overlapping the repository, builds in an isolated +detached worktree, and does not commit or push. + +The snapshot uses the tag's complete documentation tree, templates, static +files, helper extension, build script dependency graph, and locked documentation +dependencies. `release_conf.py` supplies only the publication prefix, canonical +base URL, exactly pinned `sphinx-sitemap` extension, version picker, exact GitHub +tag, corrected repository name, and a release-local redirect. After Sphinx +builds, `snapshot_site.py` rewrites only published links that would otherwise +escape to current DataFusion docs or mutable DataFusion `main` and `latest` +targets, including links in the agent-facing `llms.txt`. This includes one narrow +fix for the tagged broken `/contributor-guide/gsoc_application_guidelines.html` +link, which is redirected to the tagged +`contributor-guide/gsoc/gsoc_application_guidelines_2025.html` page. Tagged +source files and generated `_sources` files are never modified. In particular, +the tagged statement that 55.0.0 has not been released yet remains unchanged. + +Validate a complete assembled site with: + +```bash +uv run --package datafusion-docs python docs/scripts/assemble_site.py \ + --current-site docs/build/html \ + --published-site /tmp/datafusion-release-site \ + --output-site /tmp/datafusion-site +uv run --package datafusion-docs python docs/scripts/validate_site.py \ + --site-root /tmp/datafusion-site --require-snapshots +``` + +The assembler starts with a fresh current build, copies the existing complete +`asf-site/versions/` archive unchanged, and creates a root sitemap index. It does +not use picker entries as a deletion list, so removing an old release from the +picker cannot erase its archive. + +## Publication Bootstrap + +The old deployment uses unrestricted `rsync --delete`, so publishing a snapshot +before snapshot retention reaches `main` is not race-free. Keep the feature PR +in draft until maintainers agree on one of these bootstrap procedures: -To make changes to the docs, simply make a Pull Request with your -proposed changes as normal. When the PR is merged the docs will be -automatically updated. +1. Merge a preliminary retention-only workflow change, wait for it to deploy, + publish `versions/55.0.0/` manually to `asf-site`, then merge the picker and + validation changes. +2. Use a coordinated window: confirm no old documentation deployment is running + or queued, publish the snapshot, merge this change immediately, wait for the + new serialized deployment, and verify `asf-site/versions/55.0.0/` afterward. -## Release Process +For manual publication, copy the single generated `versions/55.0.0/` directory +to the same location in an `asf-site` worktree. Review, commit, and push that +branch manually. Never replace an existing release directory. The snapshot +tooling never commits, pushes, or publishes. -This documentation is hosted at https://datafusion.apache.org/ +## Site Deployment -When the PR is merged to the `main` branch of the DataFusion -repository, a [github workflow](https://github.com/apache/datafusion/blob/main/.github/workflows/docs.yaml) which: +When a documentation change reaches `main`, the deployment workflow: -1. Builds the html content -2. Pushes the html content to the [`asf-site`](https://github.com/apache/datafusion/tree/asf-site) branch in this repository. +1. Builds the current complete site and replaces all current root files. +2. Copies the entire pre-existing `asf-site/versions/` archive without using the + picker as a retention registry. +3. Builds `sitemap.xml` as an index over `sitemap-main.xml` and every retained + complete release sitemap. +4. Deliberately installs `.asf.yaml` and `.nojekyll`, then uses `rsync --delete` + while excluding `.git` to remove stale current output. +5. Serializes deployments and pushes a normal, non-force commit. -The Apache Software Foundation provides https://datafusion.apache.org/, -which serves content based on the configuration in -[.asf.yaml](https://github.com/apache/datafusion/blob/main/.asf.yaml), -which specifies the target as https://datafusion.apache.org/. +The Apache Software Foundation serves the branch according to +[`.asf.yaml`](https://github.com/apache/datafusion/blob/main/.asf.yaml). diff --git a/docs/scripts/assemble_site.py b/docs/scripts/assemble_site.py new file mode 100644 index 0000000000000..1723397f7471b --- /dev/null +++ b/docs/scripts/assemble_site.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Replace the current site while retaining immutable full-site snapshots.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import shutil +import sys +import tempfile +import xml.etree.ElementTree as ET + +from versioned_docs import SITE_URL, VERSION_PATTERN, is_link, reject_links + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--current-site", type=Path, required=True) + parser.add_argument("--published-site", type=Path, required=True) + parser.add_argument("--output-site", type=Path, required=True) + return parser.parse_args() + + +def validate_paths(current_site: Path, published_site: Path, output_site: Path) -> None: + if not current_site.is_dir(): + raise RuntimeError(f"current site does not exist: {current_site}") + if not published_site.is_dir(): + raise RuntimeError(f"published site does not exist: {published_site}") + if not (current_site / "index.html").is_file(): + raise RuntimeError(f"current site has no index.html: {current_site}") + if is_link(output_site): + raise RuntimeError(f"output site must not be a link: {output_site}") + for left, right in ( + (current_site, output_site), + (published_site, output_site), + ): + left = left.resolve() + right = right.resolve() + if left == right or left.is_relative_to(right) or right.is_relative_to(left): + raise RuntimeError( + "output site and input sites must not contain one another" + ) + + +def archived_versions(published_site: Path) -> list[str]: + versions_root = published_site / "versions" + if not versions_root.exists(): + return [] + if is_link(versions_root) or not versions_root.is_dir(): + raise RuntimeError(f"published versions must be a directory: {versions_root}") + versions: list[str] = [] + for child in versions_root.iterdir(): + if is_link(child) or not child.is_dir(): + raise RuntimeError(f"invalid published version directory: {child}") + if not VERSION_PATTERN.fullmatch(child.name): + raise RuntimeError(f"unsafe published version directory name: {child.name}") + if not (child / "index.html").is_file(): + raise RuntimeError(f"published version has no index.html: {child}") + if not (child / "sitemap.xml").is_file(): + raise RuntimeError(f"published version has no sitemap.xml: {child}") + reject_links(child) + versions.append(child.name) + return sorted(versions, key=lambda value: tuple(map(int, value.split(".")))) + + +def write_sitemap_index(site: Path, versions: list[str]) -> None: + current_sitemap = site / "sitemap.xml" + if not current_sitemap.is_file(): + raise RuntimeError(f"current site sitemap is missing: {current_sitemap}") + current_sitemap.rename(site / "sitemap-main.xml") + locations = [f"{SITE_URL}/sitemap-main.xml"] + [ + f"{SITE_URL}/versions/{version}/sitemap.xml" for version in versions + ] + root = ET.Element( + "sitemapindex", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" + ) + for location in locations: + sitemap = ET.SubElement(root, "sitemap") + ET.SubElement(sitemap, "loc").text = location + ET.indent(root, space=" ") + ET.ElementTree(root).write(current_sitemap, encoding="utf-8", xml_declaration=True) + + +def assemble(current_site: Path, published_site: Path, output_site: Path) -> None: + validate_paths(current_site, published_site, output_site) + current_site = current_site.resolve() + published_site = published_site.resolve() + output_site = output_site.resolve() + versions = archived_versions(published_site) + output_site.parent.mkdir(parents=True, exist_ok=True) + temporary_root = Path( + tempfile.mkdtemp(prefix=f".{output_site.name}-", dir=output_site.parent) + ) + staged_site = temporary_root / "site" + previous_site = temporary_root / "previous-site" + recovery_failed = False + try: + shutil.copytree(current_site, staged_site) + if versions: + shutil.copytree(published_site / "versions", staged_site / "versions") + write_sitemap_index(staged_site, versions) + if output_site.exists(): + if not output_site.is_dir(): + raise RuntimeError(f"output site is not a directory: {output_site}") + output_site.rename(previous_site) + try: + staged_site.rename(output_site) + except BaseException as error: + if previous_site.exists(): + try: + previous_site.rename(output_site) + except OSError as recovery_error: + recovery_failed = True + raise RuntimeError( + f"could not restore {output_site}; previous output remains at " + f"{previous_site}: {recovery_error}" + ) from error + raise + finally: + if not recovery_failed: + shutil.rmtree(temporary_root, ignore_errors=True) + + +if __name__ == "__main__": + try: + args = parse_args() + assemble(args.current_site, args.published_site, args.output_site) + except (OSError, RuntimeError) as error: + print(f"error: {error}", file=sys.stderr) + raise SystemExit(1) from error diff --git a/docs/scripts/release_conf.py b/docs/scripts/release_conf.py new file mode 100644 index 0000000000000..355a3c3264e2d --- /dev/null +++ b/docs/scripts/release_conf.py @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Minimal Sphinx configuration overlay for a full release site.""" + +from __future__ import annotations + +import os +from pathlib import Path +import sys + +source_dir = Path(os.environ["DATAFUSION_RELEASE_SOURCE"]) +docs_dir = source_dir.parent +sys.path.insert(0, str(docs_dir)) +tagged_conf = source_dir / "conf.py" +exec(compile(tagged_conf.read_bytes(), str(tagged_conf), "exec")) + +version = release = os.environ["DATAFUSION_RELEASE_VERSION"] +base_url = f"https://datafusion.apache.org/versions/{version}/" +html_baseurl = base_url +sitemap_url_scheme = "{link}" +extensions = [*extensions, "sphinx_sitemap"] +templates_path = [str(source_dir / path) for path in templates_path] +html_static_path = [str(source_dir / path) for path in html_static_path] +html_extra_path = [str(source_dir / path) for path in html_extra_path] +html_logo = str(source_dir / html_logo) +html_favicon = str(source_dir / html_favicon) + +html_context = dict(html_context) +html_context.update( + github_user="apache", + github_repo="datafusion", + github_version=os.environ["DATAFUSION_RELEASE_TAG"], +) +html_theme_options = dict(html_theme_options) +html_theme_options.update( + check_switcher=False, + navbar_end=["version-switcher", "theme-switcher"], + switcher={ + "json_url": "https://datafusion.apache.org/_static/versions.json", + "version_match": version, + }, +) + +redirects = dict(redirects) +redirects["library-user-guide/upgrading"] = "upgrading/index.html" diff --git a/docs/scripts/snapshot_site.py b/docs/scripts/snapshot_site.py new file mode 100644 index 0000000000000..6ec43684ae025 --- /dev/null +++ b/docs/scripts/snapshot_site.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Build one complete documentation snapshot from an exact release tag.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile + +from versioned_docs import DOCS_DIR, VERSION_PATTERN, is_link, release_entry + +REPOSITORY = DOCS_DIR.parent +INTERNAL_DOC_LINK = re.compile( + rb"https://datafusion\.apache\.org/" + rb"(?=(?:user-guide|library-user-guide|contributor-guide)/)" +) +GITHUB_MAIN_LINK = re.compile( + rb"https://github\.com/apache/(?:arrow-)?datafusion/(blob|tree)/main" + rb"(?=/|[\"'#<)\s]|$)" +) +DOCS_RS_LATEST_LINK = re.compile( + rb"https://docs\.rs/(datafusion(?:-[a-z0-9-]+)?)/latest(?=/|[\"'#<])" +) +TAGGED_GSOC_GUIDELINES_LINK = re.compile( + rb"https://datafusion\.apache\.org/contributor-guide/" + rb"gsoc_application_guidelines\.html" +) + + +def git(*arguments: str, capture: bool = False) -> str: + result = subprocess.run( + ["git", *arguments], + cwd=REPOSITORY, + check=True, + text=True, + stdout=subprocess.PIPE if capture else None, + ) + return result.stdout.strip() if capture else "" + + +def verify_tag(tag: str, expected_commit: str) -> None: + tag_ref = f"refs/tags/{tag}" + try: + object_type = git("cat-file", "-t", tag_ref, capture=True) + commit = git("rev-parse", f"{tag_ref}^{{commit}}", capture=True) + peeled = git("rev-parse", f"{tag_ref}^{{}}", capture=True) + except subprocess.CalledProcessError as error: + raise RuntimeError( + f"tag {tag} is missing; fetch it with: git fetch origin tag {tag}" + ) from error + if object_type not in {"commit", "tag"}: + raise RuntimeError(f"tag {tag} has unexpected object type {object_type}") + if commit != expected_commit or peeled != expected_commit: + raise RuntimeError(f"tag {tag} peels to {commit}, expected {expected_commit}") + + +def validate_package_root(package_root: Path) -> Path: + package_root = package_root.resolve() + repository = REPOSITORY.resolve() + if package_root.is_relative_to(repository) or repository.is_relative_to( + package_root + ): + raise RuntimeError( + "snapshot output must be outside and must not contain the repository" + ) + return package_root + + +def adapt_generated_output(site: Path, version: str, tag: str) -> None: + """Pin published links without changing tagged sources or ``_sources``.""" + prefix = f"https://datafusion.apache.org/versions/{version}/".encode() + tag_bytes = tag.encode() + paths = [*site.rglob("*.html"), site / "llms.txt"] + for path in paths: + if not path.is_file(): + continue + content = path.read_bytes() + if path.suffix == ".html": + content = TAGGED_GSOC_GUIDELINES_LINK.sub( + prefix + + b"contributor-guide/gsoc/gsoc_application_guidelines_2025.html", + content, + ) + content = INTERNAL_DOC_LINK.sub(prefix, content) + content = GITHUB_MAIN_LINK.sub( + rb"https://github.com/apache/datafusion/\1/" + tag_bytes, + content, + ) + content = DOCS_RS_LATEST_LINK.sub( + lambda match: b"https://docs.rs/" + + match.group(1) + + b"/" + + version.encode(), + content, + ) + path.write_bytes(content) + + +def generate_dependency_graph(docs_dir: Path) -> None: + script = docs_dir / "scripts" / "generate_dependency_graph.sh" + if os.name != "nt": + subprocess.run(["bash", str(script)], cwd=docs_dir, check=True) + return + + for command in ("cargo", "dot"): + if shutil.which(command) is None: + raise RuntimeError( + f"{command} is required to build the tagged dependency graph" + ) + output = docs_dir / "source" / "_static" / "data" / "deps.svg" + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as destination: + cargo = subprocess.Popen( + [ + "cargo", + "depgraph", + "--workspace-only", + "--all-deps", + "--dedup-transitive-deps", + "--exclude", + "gen,gen-common", + ], + cwd=docs_dir.parent, + stdout=subprocess.PIPE, + ) + assert cargo.stdout is not None + dot = subprocess.run( + [ + "dot", + "-Grankdir=TB", + "-Gconcentrate=true", + "-Goverlap=false", + "-Tsvg", + ], + stdin=cargo.stdout, + stdout=destination, + check=False, + ) + cargo.stdout.close() + cargo_status = cargo.wait() + if cargo_status != 0: + raise subprocess.CalledProcessError(cargo_status, cargo.args) + if dot.returncode != 0: + raise subprocess.CalledProcessError(dot.returncode, dot.args) + + +def publish_snapshot(staged_site: Path, package_root: Path, version: str) -> Path: + versions_root = package_root / "versions" + if is_link(versions_root): + raise RuntimeError(f"versions output must not be a link: {versions_root}") + versions_root.mkdir(parents=True, exist_ok=True) + destination = versions_root / version + if destination.exists() or is_link(destination): + raise RuntimeError(f"refusing to overwrite {destination}") + temporary_root = Path(tempfile.mkdtemp(prefix=f".{version}-", dir=versions_root)) + local_stage = temporary_root / "site" + try: + shutil.copytree(staged_site, local_stage) + if destination.exists() or is_link(destination): + raise RuntimeError(f"refusing to overwrite {destination}") + local_stage.rename(destination) + finally: + shutil.rmtree(temporary_root, ignore_errors=True) + return destination + + +def build_snapshot(version: str, tag: str, package_root: Path) -> Path: + if not VERSION_PATTERN.fullmatch(version) or tag != version: + raise RuntimeError("version and tag must be the same exact X.Y.Z release") + release = release_entry(version, tag) + expected_commit = str(release["commit"]) + verify_tag(tag, expected_commit) + package_root = validate_package_root(package_root) + destination = package_root / "versions" / version + if destination.exists() or is_link(destination): + raise RuntimeError(f"refusing to overwrite {destination}") + + temporary_parent = Path(tempfile.mkdtemp(prefix="datafusion-release-site-")) + worktree = temporary_parent / "worktree" + built_site = temporary_parent / "html" + worktree_added = False + try: + git("worktree", "add", "--detach", str(worktree), expected_commit) + worktree_added = True + actual_commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=worktree, + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + if actual_commit != expected_commit: + raise RuntimeError( + f"temporary worktree is {actual_commit}, expected {expected_commit}" + ) + + docs_dir = worktree / "docs" + generate_dependency_graph(docs_dir) + environment = os.environ.copy() + environment.update( + DATAFUSION_RELEASE_SOURCE=str(docs_dir / "source"), + DATAFUSION_RELEASE_VERSION=version, + DATAFUSION_RELEASE_TAG=tag, + ) + config_dir = temporary_parent / "config" + config_dir.mkdir() + shutil.copy2(DOCS_DIR / "scripts" / "release_conf.py", config_dir / "conf.py") + subprocess.run( + [ + "uv", + "run", + "--project", + str(worktree), + "--package", + "datafusion-docs", + "--with", + "sphinx-sitemap==2.9.0", + "python", + "-m", + "sphinx", + "-W", + "-b", + "html", + "-c", + str(config_dir), + str(docs_dir / "source"), + str(built_site), + ], + cwd=docs_dir, + env=environment, + check=True, + ) + adapt_generated_output(built_site, version, tag) + return publish_snapshot(built_site, package_root, version) + finally: + if worktree_added: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=REPOSITORY, + check=False, + ) + shutil.rmtree(temporary_parent, ignore_errors=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("version") + parser.add_argument("tag") + parser.add_argument("package_root", type=Path) + args = parser.parse_args() + destination = build_snapshot(args.version, args.tag, args.package_root) + print(f"snapshot created: {destination}") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as error: + print(f"error: {error}", file=sys.stderr) + raise SystemExit(1) from error diff --git a/docs/scripts/validate_site.py b/docs/scripts/validate_site.py new file mode 100644 index 0000000000000..7756b30d169ff --- /dev/null +++ b/docs/scripts/validate_site.py @@ -0,0 +1,420 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Validate current and immutable full-site documentation output.""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from functools import partial +from html.parser import HTMLParser +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import re +import sys +import threading +from typing import Iterator +import xml.etree.ElementTree as ET +from urllib.error import HTTPError, URLError +from urllib.parse import unquote, urljoin, urlsplit +from urllib.request import Request, urlopen + +from versioned_docs import SITE_URL, load_versions + +PICKER_URL = re.compile( + r"DOCUMENTATION_OPTIONS\.theme_switcher_json_url\s*=\s*['\"]([^'\"]+)" +) +PICKER_VERSION = re.compile( + r"DOCUMENTATION_OPTIONS\.theme_switcher_version_match\s*=\s*['\"]([^'\"]+)" +) +MUTABLE_RELEASE_LINK = re.compile( + r"https://(?:" + r"github\.com/apache/(?:arrow-)?datafusion/(?:blob|tree)/main(?:/|[\"'#<)\s]|$)|" + r"docs\.rs/datafusion(?:-[a-z0-9-]+)?/latest(?:/|[\"'#<])" + r")" +) +CSS_URL = re.compile(r"url\(\s*(['\"]?)(.*?)\1\s*\)") +META_REFRESH_URL = re.compile(r"(?:^|;)\s*url\s*=\s*(['\"]?)(.*?)\1\s*$", re.I) + + +class ValidationError(RuntimeError): + pass + + +class HTMLReferences(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.references: list[str] = [] + self.canonicals: list[str] = [] + self.ids: set[str] = set() + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + values = {name.lower(): value for name, value in attrs if value is not None} + for attribute in ("href", "src", "action", "poster", "data"): + if attribute in values: + self.references.append(values[attribute]) + for value in values.get("srcset", "").split(","): + if value.strip(): + self.references.append(value.strip().split()[0]) + if tag.lower() == "meta" and values.get("http-equiv", "").lower() == "refresh": + refresh = META_REFRESH_URL.search(values.get("content", "")) + if refresh: + self.references.append(refresh.group(2)) + for identifier in (values.get("id"), values.get("name")): + if identifier: + self.ids.add(identifier) + if tag.lower() == "link" and "canonical" in values.get("rel", "").split(): + self.canonicals.append(values["href"]) + + +def read_html(path: Path) -> tuple[str, HTMLReferences]: + text = path.read_text(encoding="utf-8") + parsed = HTMLReferences() + parsed.feed(text) + return text, parsed + + +def site_target(site_root: Path, page: Path, url: str) -> Path | None: + parsed = urlsplit(url) + if parsed.scheme not in ("", "http", "https"): + return None + if parsed.netloc and parsed.netloc != urlsplit(SITE_URL).netloc: + return None + if parsed.netloc and parsed.path.startswith( + ("/blog/", "/python/", "/java/", "/comet/", "/ballista/") + ): + return None + if not parsed.path: + return page + target = ( + site_root / unquote(parsed.path.lstrip("/")) + if parsed.netloc or parsed.path.startswith("/") + else page.parent / unquote(parsed.path) + ).resolve() + try: + target.relative_to(site_root.resolve()) + except ValueError as error: + raise ValidationError( + f"local target escapes site root: {page}: {url}" + ) from error + if parsed.path.endswith("/"): + target /= "index.html" + return target + + +def switch_candidate(page: str, source_version: str, target_url: str) -> str: + path = urlsplit(page).path.lstrip("/") + prefix = "" if source_version == "main" else f"versions/{source_version}/" + if not path.startswith(prefix): + raise ValidationError(f"page is outside its version prefix: {page}") + relative = path[len(prefix) :] + candidate = urljoin(target_url, relative) + if f"/versions/{source_version}/versions/" in candidate: + raise ValidationError(f"version path is duplicated: {candidate}") + return candidate + + +def validate_page( + site_root: Path, relative: str, version: str, errors: list[str] +) -> None: + path = site_root / relative + if not path.is_file(): + errors.append(f"representative page is missing: {path}") + return + text, parsed = read_html(path) + canonical = SITE_URL + "/" + relative.replace("\\", "/") + if parsed.canonicals != [canonical]: + errors.append(f"incorrect canonical in {path}: expected {canonical}") + picker_url = PICKER_URL.search(text) + if picker_url is None or picker_url.group(1) != f"{SITE_URL}/_static/versions.json": + errors.append(f"incorrect picker manifest URL in {path}") + picker_version = PICKER_VERSION.search(text) + if picker_version is None or picker_version.group(1) != version: + errors.append(f"incorrect picker version in {path}") + + +def validate_local_links(site_root: Path, root: Path, errors: list[str]) -> None: + parsed_pages: dict[Path, HTMLReferences] = {} + for page in root.rglob("*.html"): + if "_static" in page.relative_to(root).parts: + continue + _, parsed = read_html(page) + parsed_pages[page.resolve()] = parsed + expected_canonical = f"{SITE_URL}/{page.relative_to(site_root).as_posix()}" + if parsed.canonicals and parsed.canonicals != [expected_canonical]: + errors.append( + f"incorrect canonical in {page}: expected {expected_canonical}" + ) + for url in parsed.references: + parsed_url = urlsplit(url) + if parsed_url.netloc == urlsplit(SITE_URL).netloc and root == site_root: + # Current-source absolute links are checked by the repository's + # Markdown link checker, not by snapshot hosting validation. + continue + try: + target = site_target(site_root, page, url) + except ValidationError as error: + errors.append(str(error)) + continue + if target is not None and not target.exists(): + errors.append(f"missing local target in {page}: {url}") + continue + fragment = urlsplit(url).fragment + if target is not None and fragment and target.suffix == ".html": + target_parsed = parsed_pages.get(target.resolve()) + if target_parsed is None: + _, target_parsed = read_html(target) + parsed_pages[target.resolve()] = target_parsed + if unquote(fragment) not in target_parsed.ids: + errors.append(f"missing local fragment in {page}: {url}") + for stylesheet in root.rglob("*.css"): + text = stylesheet.read_text(encoding="utf-8") + for _, url in CSS_URL.findall(text): + target = site_target(site_root, stylesheet, url) + if target is not None and not target.exists(): + errors.append(f"missing CSS target in {stylesheet}: {url}") + + +def validate_sitemaps(site_root: Path, versions: list[str], errors: list[str]) -> None: + namespace = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} + if versions: + try: + tree = ET.parse(site_root / "sitemap.xml") + except (OSError, ET.ParseError) as error: + errors.append(f"invalid sitemap index: {error}") + return + locations = [item.text for item in tree.findall("sm:sitemap/sm:loc", namespace)] + expected = [f"{SITE_URL}/sitemap-main.xml"] + [ + f"{SITE_URL}/versions/{version}/sitemap.xml" for version in versions + ] + if locations != expected: + errors.append("sitemap index has incorrect entries") + for version in versions: + sitemap = site_root / "versions" / version / "sitemap.xml" + try: + tree = ET.parse(sitemap) + except (OSError, ET.ParseError) as error: + errors.append(f"invalid release sitemap {sitemap}: {error}") + continue + prefix = f"{SITE_URL}/versions/{version}/" + for location in tree.findall("sm:url/sm:loc", namespace): + if location.text is None or not location.text.startswith(prefix): + errors.append(f"release sitemap URL escapes {prefix}: {location.text}") + + +class QuietHandler(SimpleHTTPRequestHandler): + def log_message(self, format: str, *args: object) -> None: + pass + + +@contextmanager +def local_server(site_root: Path) -> Iterator[str]: + server = ThreadingHTTPServer( + ("127.0.0.1", 0), partial(QuietHandler, directory=str(site_root)) + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + server.server_close() + thread.join() + + +def validate_http(site_root: Path, paths: list[str], errors: list[str]) -> None: + with local_server(site_root) as server: + for relative in paths: + for method in ("GET", "HEAD"): + try: + with urlopen( + Request(f"{server}/{relative}", method=method), timeout=10 + ) as response: + if response.status != 200: + errors.append( + f"HTTP {method} {relative}: {response.status}" + ) + except (HTTPError, URLError, TimeoutError) as error: + errors.append(f"HTTP {method} {relative} failed: {error}") + + +def validate_site( + site_root: Path, *, require_snapshots: bool = False, http_checks: bool = True +) -> None: + site_root = site_root.resolve() + if not site_root.is_dir(): + raise ValidationError(f"site root does not exist: {site_root}") + entries = load_versions() + errors: list[str] = [] + manifest = site_root / "_static" / "versions.json" + if ( + not manifest.is_file() + or json.loads(manifest.read_text(encoding="utf-8")) != entries + ): + errors.append(f"site has a missing or stale versions manifest: {manifest}") + + releases = [str(entry["version"]) for entry in entries[1:]] + versions_root = site_root / "versions" + present = [] + if versions_root.is_dir(): + present = sorted( + ( + child.name + for child in versions_root.iterdir() + if child.is_dir() + and re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", child.name) + ), + key=lambda value: tuple(map(int, value.split("."))), + ) + if require_snapshots and not set(releases).issubset(present): + errors.append("required release snapshots are missing") + + representatives = ( + "index.html", + "user-guide/introduction.html", + "library-user-guide/index.html", + "contributor-guide/index.html", + "download.html", + "search.html", + "_static/theme_overrides.css", + "_sources/index.rst.txt", + "llms.txt", + ) + redirects = ( + "library-user-guide/adding-udfs.html", + "library-user-guide/upgrading.html", + "user-guide/runtime_configs.html", + ) + http_paths = ["_static/versions.json"] + for version in ["main", *present]: + prefix = "" if version == "main" else f"versions/{version}/" + root = site_root if version == "main" else site_root / "versions" / version + if not (root / "searchindex.js").is_file(): + errors.append(f"search index is missing: {root}") + elif (root / "searchindex.js").stat().st_size == 0: + errors.append(f"search index is empty: {root}") + if not (root / "objects.inv").is_file(): + errors.append(f"Sphinx inventory is missing: {root}") + elif (root / "objects.inv").stat().st_size == 0: + errors.append(f"Sphinx inventory is empty: {root}") + for representative in representatives: + relative = prefix + representative + if representative.endswith((".html",)): + validate_page(site_root, relative, version, errors) + elif not (site_root / relative).is_file(): + errors.append(f"representative file is missing: {relative}") + elif (site_root / relative).stat().st_size == 0: + errors.append(f"representative file is empty: {relative}") + http_paths.append(relative) + for redirect in redirects: + relative = prefix + redirect + if not (site_root / relative).is_file(): + errors.append(f"redirect is missing: {relative}") + http_paths.append(relative) + validate_local_links(site_root, root, errors) + + page = f"{SITE_URL}/{prefix}user-guide/sql/select.html" + for entry in entries: + candidate = switch_candidate(page, version, str(entry["url"])) + if "/versions/55.0.0/versions/" in candidate: + errors.append(f"picker duplicates a versions path: {candidate}") + + if version != "main": + release_files = [*root.rglob("*.html"), root / "llms.txt"] + for page_path in release_files: + if page_path.suffix == ".html": + text, parsed = read_html(page_path) + else: + text = page_path.read_text(encoding="utf-8") + parsed = HTMLReferences() + mutable = MUTABLE_RELEASE_LINK.search(text) + if mutable: + errors.append( + f"mutable release link in {page_path}: {mutable.group(0)}" + ) + for url in parsed.references: + parsed_url = urlsplit(url) + if ( + parsed_url.netloc == urlsplit(SITE_URL).netloc + and parsed_url.path.startswith( + ( + "/user-guide/", + "/library-user-guide/", + "/contributor-guide/", + ) + ) + and (site_root / parsed_url.path.lstrip("/")).exists() + ): + errors.append( + f"release documentation link escapes in {page_path}: {url}" + ) + + current_only = site_root / "library-user-guide" / "upgrading" / "56.0.0.html" + release_current_only = ( + site_root + / "versions" + / "55.0.0" + / "library-user-guide" + / "upgrading" + / "56.0.0.html" + ) + if not current_only.is_file(): + errors.append(f"current-only page is missing: {current_only}") + if release_current_only.exists(): + errors.append(f"release contains current-only page: {release_current_only}") + if "55.0.0" in present: + tagged_source = ( + site_root + / "versions" + / "55.0.0" + / "_sources" + / "library-user-guide" + / "upgrading" + / "55.0.0.md.txt" + ) + if ( + not tagged_source.is_file() + or "DataFusion `55.0.0` has not been released yet." + not in tagged_source.read_text(encoding="utf-8") + ): + errors.append("exact tagged 55.0.0 release-note prose was not preserved") + + validate_sitemaps(site_root, present, errors) + if http_checks and not errors: + validate_http(site_root, http_paths, errors) + if errors: + raise ValidationError("site validation failed:\n- " + "\n- ".join(errors)) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--site-root", type=Path, required=True) + parser.add_argument("--require-snapshots", action="store_true") + args = parser.parse_args() + validate_site(args.site_root, require_snapshots=args.require_snapshots) + print("site validation passed") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, ValidationError) as error: + print(f"error: {error}", file=sys.stderr) + raise SystemExit(1) from error diff --git a/docs/scripts/versioned_docs.py b/docs/scripts/versioned_docs.py new file mode 100644 index 0000000000000..d52b6e2be564c --- /dev/null +++ b/docs/scripts/versioned_docs.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared catalog handling for full-site documentation snapshots.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import re +import stat + +DOCS_DIR = Path(__file__).resolve().parent.parent +MANIFEST = DOCS_DIR / "source" / "_static" / "versions.json" +SITE_URL = "https://datafusion.apache.org" +VERSION_PATTERN = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+") +EXPECTED_55_COMMIT = "d5552342012888b7d1a3ab88d92e3d292fc0cde0" + + +def is_link(path: Path) -> bool: + """Return whether a path is a symlink or Windows reparse point.""" + try: + status = path.lstat() + except FileNotFoundError: + return False + reparse_point = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + file_attributes = getattr(status, "st_file_attributes", 0) + return stat.S_ISLNK(status.st_mode) or bool(file_attributes & reparse_point) + + +def reject_links(root: Path) -> None: + """Reject links before copying an immutable archive tree.""" + if is_link(root): + raise RuntimeError(f"archived path must not be a link: {root}") + for directory, directories, files in os.walk(root, followlinks=False): + for name in [*directories, *files]: + path = Path(directory) / name + if is_link(path): + raise RuntimeError(f"archived path must not be a link: {path}") + + +def load_versions(path: Path = MANIFEST) -> list[dict[str, object]]: + try: + entries = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"cannot read versions manifest {path}: {error}") from error + if not isinstance(entries, list) or not entries: + raise ValueError("versions manifest must contain a non-empty list") + + versions: set[str] = set() + urls: set[str] = set() + preferred = 0 + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise ValueError("each versions manifest entry must be an object") + version = entry.get("version") + url = entry.get("url") + if not isinstance(version, str) or version in versions: + raise ValueError(f"invalid or duplicate version: {version!r}") + if not isinstance(url, str) or url in urls: + raise ValueError(f"invalid or duplicate version URL: {url!r}") + versions.add(version) + urls.add(url) + preferred += entry.get("preferred") is True + + if index == 0: + if entry != { + "name": "Development", + "url": f"{SITE_URL}/", + "version": "main", + }: + raise ValueError( + "the first entry must be Development/main at site root" + ) + continue + if not VERSION_PATTERN.fullmatch(version): + raise ValueError(f"invalid semantic release version: {version!r}") + expected_url = f"{SITE_URL}/versions/{version}/" + if entry.get("name") != version or url != expected_url: + raise ValueError(f"release {version} has an invalid name or URL") + if entry.get("tag") != version or not re.fullmatch( + r"[0-9a-f]{40}", str(entry.get("commit", "")) + ): + raise ValueError(f"release {version} must have an exact tag and commit") + if preferred > 1: + raise ValueError("at most one versions manifest entry may be preferred") + + release_55 = next( + (entry for entry in entries if entry["version"] == "55.0.0"), None + ) + if release_55 is None or release_55.get("commit") != EXPECTED_55_COMMIT: + raise ValueError("55.0.0 must resolve to its expected exact commit") + return entries + + +def release_entry(version: str, tag: str) -> dict[str, object]: + for entry in load_versions(): + if entry["version"] == version and entry.get("tag") == tag: + return entry + raise RuntimeError("version and tag must exactly match versions.json") diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json new file mode 100644 index 0000000000000..d78cddec398c1 --- /dev/null +++ b/docs/source/_static/versions.json @@ -0,0 +1,14 @@ +[ + { + "name": "Development", + "url": "https://datafusion.apache.org/", + "version": "main" + }, + { + "commit": "d5552342012888b7d1a3ab88d92e3d292fc0cde0", + "name": "55.0.0", + "tag": "55.0.0", + "url": "https://datafusion.apache.org/versions/55.0.0/", + "version": "55.0.0" + } +] diff --git a/docs/source/conf.py b/docs/source/conf.py index 68ba628bba1f2..a4f7460dfd0a4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -37,6 +37,7 @@ project = "Apache DataFusion" copyright = "2019-2025, Apache Software Foundation" author = "Apache Software Foundation" +version = release = "main" # -- General configuration --------------------------------------------------- @@ -95,12 +96,18 @@ }, "use_edit_page_button": True, "navbar_center": [], - "navbar_end": ["theme-switcher"], + "navbar_end": ["version-switcher", "theme-switcher"], + # The persistent production URL is intentionally not fetched at build time. + "check_switcher": False, + "switcher": { + "json_url": "https://datafusion.apache.org/_static/versions.json", + "version_match": version, + }, } html_context = { "github_user": "apache", - "github_repo": "arrow-datafusion", + "github_repo": "datafusion", "github_version": "main", "doc_path": "docs/source", } diff --git a/docs/tests/test_versioned_docs.py b/docs/tests/test_versioned_docs.py new file mode 100644 index 0000000000000..87abe8137e64e --- /dev/null +++ b/docs/tests/test_versioned_docs.py @@ -0,0 +1,330 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +DOCS_DIR = Path(__file__).resolve().parents[1] +SCRIPTS_DIR = DOCS_DIR / "scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) + + +def load_script(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS_DIR / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +versioned_docs = load_script("versioned_docs") +snapshot_site = load_script("snapshot_site") +assemble_site = load_script("assemble_site") +validate_site = load_script("validate_site") + + +def write(path: Path, content: str = "file") -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +class VersionedDocsTest(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def test_manifest_has_one_site_wide_exact_release(self) -> None: + entries = versioned_docs.load_versions() + self.assertEqual( + entries, + [ + { + "name": "Development", + "url": f"{versioned_docs.SITE_URL}/", + "version": "main", + }, + { + "commit": versioned_docs.EXPECTED_55_COMMIT, + "name": "55.0.0", + "tag": "55.0.0", + "url": f"{versioned_docs.SITE_URL}/versions/55.0.0/", + "version": "55.0.0", + }, + ], + ) + + def test_manifest_is_appendable_and_rejects_duplicates(self) -> None: + manifest = self.root / "versions.json" + entries = versioned_docs.load_versions() + entries.append( + { + "commit": "a" * 40, + "name": "56.0.0", + "tag": "56.0.0", + "url": f"{versioned_docs.SITE_URL}/versions/56.0.0/", + "version": "56.0.0", + } + ) + write(manifest, json.dumps(entries)) + self.assertEqual(len(versioned_docs.load_versions(manifest)), 3) + entries[-1]["version"] = "55.0.0" + write(manifest, json.dumps(entries)) + with self.assertRaisesRegex(ValueError, "duplicate version"): + versioned_docs.load_versions(manifest) + + def test_exact_lightweight_tag_and_peeled_commit(self) -> None: + snapshot_site.verify_tag("55.0.0", versioned_docs.EXPECTED_55_COMMIT) + self.assertEqual( + snapshot_site.git("cat-file", "-t", "refs/tags/55.0.0", capture=True), + "commit", + ) + + def test_exact_tag_keeps_unreleased_55_prose(self) -> None: + prose = snapshot_site.git( + "show", + "55.0.0:docs/source/library-user-guide/upgrading/55.0.0.md", + capture=True, + ) + self.assertIn("DataFusion `55.0.0` has not been released yet.", prose) + + def test_missing_tag_error_is_actionable(self) -> None: + repository = self.root / "repository" + subprocess.run( + ["git", "init", str(repository)], check=True, capture_output=True + ) + with mock.patch.object(snapshot_site, "REPOSITORY", repository): + with self.assertRaisesRegex(RuntimeError, "git fetch origin tag missing"): + snapshot_site.verify_tag("missing", "0" * 40) + + def test_generated_html_adaptations_leave_sources_unchanged(self) -> None: + site = self.root / "site" + original = ( + "https://datafusion.apache.org/contributor-guide/testing.html\n" + "https://datafusion.apache.org/contributor-guide/gsoc_application_guidelines.html\n" + "https://github.com/apache/datafusion/blob/main/docs/source/index.rst\n" + "https://docs.rs/datafusion/latest/datafusion/\n" + ) + write(site / "index.html", original) + write(site / "_sources" / "index.rst.txt", original) + write( + site / "llms.txt", + original + "https://github.com/apache/datafusion/tree/main\n", + ) + snapshot_site.adapt_generated_output(site, "55.0.0", "55.0.0") + output = (site / "index.html").read_text(encoding="utf-8") + self.assertIn("/versions/55.0.0/contributor-guide/", output) + self.assertIn( + "/versions/55.0.0/contributor-guide/gsoc/gsoc_application_guidelines_2025.html", + output, + ) + self.assertIn("/blob/55.0.0/docs/source/", output) + self.assertIn("docs.rs/datafusion/55.0.0/datafusion/", output) + self.assertEqual( + (site / "_sources" / "index.rst.txt").read_text(encoding="utf-8"), + original, + ) + llms = (site / "llms.txt").read_text(encoding="utf-8") + self.assertIn("/versions/55.0.0/contributor-guide/", llms) + self.assertIn("/tree/55.0.0", llms) + self.assertNotIn("/tree/main", llms) + + def test_publication_is_one_directory_and_refuses_overwrite(self) -> None: + staged = self.root / "staged" + package = self.root / "package with spaces" + write(staged / "index.html") + destination = snapshot_site.publish_snapshot(staged, package, "55.0.0") + self.assertEqual(destination, package / "versions" / "55.0.0") + self.assertTrue((destination / "index.html").is_file()) + write(staged / "index.html") + with self.assertRaisesRegex(RuntimeError, "refusing to overwrite"): + snapshot_site.publish_snapshot(staged, package, "55.0.0") + + def test_publication_failure_leaves_no_release_directory(self) -> None: + staged = self.root / "staged" + package = self.root / "package" + write(staged / "index.html") + with mock.patch.object( + snapshot_site.shutil, + "copytree", + side_effect=OSError("simulated copy failure"), + ): + with self.assertRaisesRegex(OSError, "simulated copy failure"): + snapshot_site.publish_snapshot(staged, package, "55.0.0") + self.assertFalse((package / "versions" / "55.0.0").exists()) + self.assertEqual(list((package / "versions").iterdir()), []) + + def test_snapshot_rejects_unsafe_output_and_version(self) -> None: + with self.assertRaisesRegex(RuntimeError, "must not contain the repository"): + snapshot_site.validate_package_root(snapshot_site.REPOSITORY.parent) + with self.assertRaisesRegex(RuntimeError, "must be outside"): + snapshot_site.validate_package_root(snapshot_site.REPOSITORY / "docs/build") + with self.assertRaisesRegex(RuntimeError, "exact X.Y.Z"): + snapshot_site.build_snapshot("../55", "../55", self.root / "output") + + def test_assembly_replaces_current_and_retains_complete_archive(self) -> None: + current = self.root / "current" + published = self.root / "published" + output = self.root / "output" + write(current / "index.html", "new") + write(current / "sitemap.xml", "") + write(published / "obsolete.html", "old current") + for version in ("54.0.0", "55.0.0"): + write(published / "versions" / version / "index.html", version) + write(published / "versions" / version / "sitemap.xml", "") + assemble_site.assemble(current, published, output) + self.assertEqual((output / "index.html").read_text(encoding="utf-8"), "new") + self.assertFalse((output / "obsolete.html").exists()) + self.assertTrue((output / "versions" / "54.0.0" / "index.html").is_file()) + self.assertTrue((output / "versions" / "55.0.0" / "index.html").is_file()) + sitemap = (output / "sitemap.xml").read_text(encoding="utf-8") + self.assertIn("sitemap-main.xml", sitemap) + self.assertIn("versions/54.0.0/sitemap.xml", sitemap) + errors: list[str] = [] + validate_site.validate_sitemaps(output, ["54.0.0", "55.0.0"], errors) + self.assertEqual(errors, []) + + def test_removed_picker_entry_does_not_delete_archive(self) -> None: + current = self.root / "current" + published = self.root / "published" + output = self.root / "output" + write(current / "index.html") + write(current / "sitemap.xml", "") + write(published / "versions" / "54.0.0" / "index.html") + write(published / "versions" / "54.0.0" / "sitemap.xml") + assemble_site.assemble(current, published, output) + self.assertTrue((output / "versions" / "54.0.0" / "index.html").is_file()) + + def test_assembly_rejects_unsafe_archive_name_and_path_overlap(self) -> None: + published = self.root / "published" + write(published / "versions" / "latest" / "index.html") + write(published / "versions" / "latest" / "sitemap.xml") + with self.assertRaisesRegex(RuntimeError, "unsafe published version"): + assemble_site.archived_versions(published) + current = self.root / "container" / "current" + write(current / "index.html") + with self.assertRaisesRegex(RuntimeError, "must not contain one another"): + assemble_site.validate_paths(current, published, self.root / "container") + + def test_assembly_rejects_links_in_archive_and_output(self) -> None: + current = self.root / "current" + published = self.root / "published" + external = self.root / "external" + output_link = self.root / "output-link" + write(current / "index.html") + write(published / "versions" / "55.0.0" / "index.html") + write(published / "versions" / "55.0.0" / "sitemap.xml") + write(external / "secret.txt") + archive_link = published / "versions" / "55.0.0" / "external" + try: + archive_link.symlink_to(external, target_is_directory=True) + output_link.symlink_to(external, target_is_directory=True) + except OSError as error: + self.skipTest(f"directory symlinks are unavailable: {error}") + with self.assertRaisesRegex(RuntimeError, "archived path must not be a link"): + assemble_site.archived_versions(published) + archive_link.unlink() + with self.assertRaisesRegex(RuntimeError, "output site must not be a link"): + assemble_site.assemble(current, published, output_link) + self.assertTrue((external / "secret.txt").is_file()) + + def test_assembly_restores_previous_output_on_final_rename_failure(self) -> None: + current = self.root / "current" + published = self.root / "published" + output = self.root / "output" + write(current / "index.html") + write(current / "sitemap.xml", "") + published.mkdir() + write(output / "sentinel.txt", "previous") + real_rename = Path.rename + + def fail_staged(source: Path, destination: Path) -> Path: + if source.name == "site" and destination == output: + raise OSError("simulated failure") + return real_rename(source, destination) + + with mock.patch.object(Path, "rename", fail_staged): + with self.assertRaisesRegex(OSError, "simulated failure"): + assemble_site.assemble(current, published, output) + self.assertEqual( + (output / "sentinel.txt").read_text(encoding="utf-8"), "previous" + ) + + def test_picker_equivalent_paths_and_fallback(self) -> None: + release = f"{versioned_docs.SITE_URL}/versions/55.0.0/" + candidate = validate_site.switch_candidate( + f"{versioned_docs.SITE_URL}/user-guide/sql/select.html", "main", release + ) + self.assertEqual(candidate, release + "user-guide/sql/select.html") + reverse = validate_site.switch_candidate( + candidate, "55.0.0", f"{versioned_docs.SITE_URL}/" + ) + self.assertEqual( + reverse, f"{versioned_docs.SITE_URL}/user-guide/sql/select.html" + ) + current_only = validate_site.switch_candidate( + f"{versioned_docs.SITE_URL}/library-user-guide/upgrading/56.0.0.html", + "main", + release, + ) + self.assertEqual( + current_only, release + "library-user-guide/upgrading/56.0.0.html" + ) + self.assertNotIn("/versions/55.0.0/versions/", current_only) + + @unittest.skipUnless(shutil.which("rsync"), "rsync is not installed") + def test_rsync_keeps_git_and_deletes_stale_current_files(self) -> None: + source = self.root / "source" + destination = self.root / "destination" + write(source / "index.html", "new") + write(source / "versions" / "55.0.0" / "index.html") + write(source / ".asf.yaml") + write(source / ".nojekyll") + write(destination / "stale.html") + write(destination / ".git" / "HEAD", "git") + subprocess.run( + [ + "rsync", + "-a", + "--delete", + "--exclude", + "/.git/", + f"{source}/", + f"{destination}/", + ], + check=True, + ) + self.assertFalse((destination / "stale.html").exists()) + self.assertTrue((destination / "versions" / "55.0.0" / "index.html").is_file()) + self.assertEqual( + (destination / ".git" / "HEAD").read_text(encoding="utf-8"), "git" + ) + + +if __name__ == "__main__": + unittest.main() From d7ac573854c1987944d1fc51138c4402913412e7 Mon Sep 17 00:00:00 2001 From: yiaany Date: Sun, 30 Aug 2026 02:11:34 +0500 Subject: [PATCH 02/12] docs: harden versioned docs publication --- docs/scripts/assemble_site.py | 27 ++++++------------------ docs/scripts/snapshot_site.py | 4 ++++ docs/tests/test_versioned_docs.py | 35 +++++++++++++++++++++++-------- 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/docs/scripts/assemble_site.py b/docs/scripts/assemble_site.py index 1723397f7471b..0536e49607d26 100644 --- a/docs/scripts/assemble_site.py +++ b/docs/scripts/assemble_site.py @@ -57,6 +57,8 @@ def validate_paths(current_site: Path, published_site: Path, output_site: Path) raise RuntimeError( "output site and input sites must not contain one another" ) + if output_site.exists(): + raise RuntimeError(f"output site already exists: {output_site}") def archived_versions(published_site: Path) -> list[str]: @@ -109,33 +111,16 @@ def assemble(current_site: Path, published_site: Path, output_site: Path) -> Non tempfile.mkdtemp(prefix=f".{output_site.name}-", dir=output_site.parent) ) staged_site = temporary_root / "site" - previous_site = temporary_root / "previous-site" - recovery_failed = False try: shutil.copytree(current_site, staged_site) if versions: shutil.copytree(published_site / "versions", staged_site / "versions") write_sitemap_index(staged_site, versions) - if output_site.exists(): - if not output_site.is_dir(): - raise RuntimeError(f"output site is not a directory: {output_site}") - output_site.rename(previous_site) - try: - staged_site.rename(output_site) - except BaseException as error: - if previous_site.exists(): - try: - previous_site.rename(output_site) - except OSError as recovery_error: - recovery_failed = True - raise RuntimeError( - f"could not restore {output_site}; previous output remains at " - f"{previous_site}: {recovery_error}" - ) from error - raise + if output_site.exists() or is_link(output_site): + raise RuntimeError(f"output site already exists: {output_site}") + staged_site.rename(output_site) finally: - if not recovery_failed: - shutil.rmtree(temporary_root, ignore_errors=True) + shutil.rmtree(temporary_root, ignore_errors=True) if __name__ == "__main__": diff --git a/docs/scripts/snapshot_site.py b/docs/scripts/snapshot_site.py index 6ec43684ae025..c9673cc8e87b3 100644 --- a/docs/scripts/snapshot_site.py +++ b/docs/scripts/snapshot_site.py @@ -90,6 +90,8 @@ def validate_package_root(package_root: Path) -> Path: def adapt_generated_output(site: Path, version: str, tag: str) -> None: """Pin published links without changing tagged sources or ``_sources``.""" + shutil.rmtree(site / ".doctrees", ignore_errors=True) + (site / ".buildinfo").unlink(missing_ok=True) prefix = f"https://datafusion.apache.org/versions/{version}/".encode() tag_bytes = tag.encode() paths = [*site.rglob("*.html"), site / "llms.txt"] @@ -243,6 +245,8 @@ def build_snapshot(version: str, tag: str, package_root: Path) -> Path: "-W", "-b", "html", + "-d", + str(temporary_parent / "doctrees"), "-c", str(config_dir), str(docs_dir / "source"), diff --git a/docs/tests/test_versioned_docs.py b/docs/tests/test_versioned_docs.py index 87abe8137e64e..f4193bd1427d2 100644 --- a/docs/tests/test_versioned_docs.py +++ b/docs/tests/test_versioned_docs.py @@ -132,6 +132,8 @@ def test_generated_html_adaptations_leave_sources_unchanged(self) -> None: ) write(site / "index.html", original) write(site / "_sources" / "index.rst.txt", original) + write(site / ".buildinfo", "environment-specific metadata") + write(site / ".doctrees" / "environment.pickle", "temporary paths") write( site / "llms.txt", original + "https://github.com/apache/datafusion/tree/main\n", @@ -153,6 +155,8 @@ def test_generated_html_adaptations_leave_sources_unchanged(self) -> None: self.assertIn("/versions/55.0.0/contributor-guide/", llms) self.assertIn("/tree/55.0.0", llms) self.assertNotIn("/tree/main", llms) + self.assertFalse((site / ".buildinfo").exists()) + self.assertFalse((site / ".doctrees").exists()) def test_publication_is_one_directory_and_refuses_overwrite(self) -> None: staged = self.root / "staged" @@ -253,7 +257,7 @@ def test_assembly_rejects_links_in_archive_and_output(self) -> None: assemble_site.assemble(current, published, output_link) self.assertTrue((external / "secret.txt").is_file()) - def test_assembly_restores_previous_output_on_final_rename_failure(self) -> None: + def test_assembly_refuses_to_replace_existing_output(self) -> None: current = self.root / "current" published = self.root / "published" output = self.root / "output" @@ -261,18 +265,31 @@ def test_assembly_restores_previous_output_on_final_rename_failure(self) -> None write(current / "sitemap.xml", "") published.mkdir() write(output / "sentinel.txt", "previous") - real_rename = Path.rename + with self.assertRaisesRegex(RuntimeError, "output site already exists"): + assemble_site.assemble(current, published, output) + self.assertEqual( + (output / "sentinel.txt").read_text(encoding="utf-8"), "previous" + ) - def fail_staged(source: Path, destination: Path) -> Path: - if source.name == "site" and destination == output: - raise OSError("simulated failure") - return real_rename(source, destination) + def test_assembly_refuses_output_created_while_staging(self) -> None: + current = self.root / "current" + published = self.root / "published" + output = self.root / "output" + write(current / "index.html") + write(current / "sitemap.xml", "") + published.mkdir() + real_copytree = shutil.copytree - with mock.patch.object(Path, "rename", fail_staged): - with self.assertRaisesRegex(OSError, "simulated failure"): + def create_output(source: Path, destination: Path) -> Path: + copied = real_copytree(source, destination) + write(output / "sentinel.txt", "concurrent") + return copied + + with mock.patch.object(assemble_site.shutil, "copytree", create_output): + with self.assertRaisesRegex(RuntimeError, "output site already exists"): assemble_site.assemble(current, published, output) self.assertEqual( - (output / "sentinel.txt").read_text(encoding="utf-8"), "previous" + (output / "sentinel.txt").read_text(encoding="utf-8"), "concurrent" ) def test_picker_equivalent_paths_and_fallback(self) -> None: From d022e1e559b42b09489ce05aaea44383d3f18265 Mon Sep 17 00:00:00 2001 From: yiaany Date: Tue, 29 Sep 2026 22:17:34 +0500 Subject: [PATCH 03/12] docs: simplify versioned documentation workflow --- .gitattributes | 2 - .github/workflows/docs.yaml | 37 +-- .github/workflows/docs_pr.yaml | 37 +-- dev/release/README.md | 51 ++-- docs/README.md | 148 +++-------- docs/scripts/assemble_site.py | 132 ---------- docs/scripts/release/conf.py | 53 ++++ docs/scripts/release_conf.py | 60 ----- docs/scripts/snapshot_site.py | 286 -------------------- docs/scripts/validate_site.py | 420 ------------------------------ docs/scripts/versioned_docs.py | 116 --------- docs/source/_static/versions.json | 7 +- docs/source/conf.py | 19 +- docs/tests/test_versioned_docs.py | 347 ------------------------ 14 files changed, 150 insertions(+), 1565 deletions(-) delete mode 100644 docs/scripts/assemble_site.py create mode 100644 docs/scripts/release/conf.py delete mode 100644 docs/scripts/release_conf.py delete mode 100644 docs/scripts/snapshot_site.py delete mode 100644 docs/scripts/validate_site.py delete mode 100644 docs/scripts/versioned_docs.py delete mode 100644 docs/tests/test_versioned_docs.py diff --git a/.gitattributes b/.gitattributes index 75ac548699177..84b47a6fc56e1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,4 @@ .github/ export-ignore -docs/*.sh text eol=lf -docs/scripts/*.sh text eol=lf datafusion/core/tests/data/newlines_in_values.csv text eol=lf datafusion/proto/src/generated/prost.rs linguist-generated datafusion/proto/src/generated/pbjson.rs linguist-generated diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 217c6f01b5601..c52ed5681c64f 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,19 +20,12 @@ on: branches: - main paths: - - .gitattributes - .asf.yaml - .github/workflows/docs.yaml - docs/** - - pyproject.toml - - uv.lock name: Deploy DataFusion site -concurrency: - group: docs-deployment - cancel-in-progress: false - jobs: build-docs: permissions: @@ -41,16 +34,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout docs sources - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Checkout asf-site branch - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: asf-site path: asf-site - name: Setup uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install dependencies run: uv sync --package datafusion-docs @@ -60,7 +53,7 @@ jobs: sudo apt-get update sudo apt-get install -y graphviz - name: Install cargo-depgraph - uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 + uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 with: tool: cargo-depgraph@1.6 @@ -70,21 +63,7 @@ jobs: cd docs uv run --package datafusion-docs ./build.sh - - name: Assemble deployment site - run: | - uv run --package datafusion-docs python docs/scripts/assemble_site.py \ - --current-site docs/build/html \ - --published-site asf-site \ - --output-site deployment-site - cp .asf.yaml deployment-site/ - touch deployment-site/.nojekyll - - - name: Validate deployment site - run: >- - uv run --package datafusion-docs python docs/scripts/validate_site.py - --site-root deployment-site --require-snapshots - - - name: Copy and push the generated HTML + - name: Copy & push the generated HTML run: | set -x cd asf-site/ @@ -92,13 +71,15 @@ jobs: -a \ --delete \ --exclude '/.git/' \ - ../deployment-site/ \ + ../docs/build/html/ \ ./ + cp ../.asf.yaml . + touch .nojekyll git status --porcelain if [ "$(git status --porcelain)" != "" ]; then git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add --all git commit -m 'Publish built docs triggered by ${{ github.sha }}' - git push + git push || git push --force fi diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml index 7035111b70fb0..2cbb64274a526 100644 --- a/.github/workflows/docs_pr.yaml +++ b/.github/workflows/docs_pr.yaml @@ -25,23 +25,13 @@ concurrency: on: push: paths: - - ".gitattributes" - "docs/**" - - ".github/workflows/docs.yaml" - - ".github/workflows/docs_pr.yaml" - - "pyproject.toml" - - "uv.lock" branches: - main - branch-* pull_request: paths: - - ".gitattributes" - "docs/**" - - ".github/workflows/docs.yaml" - - ".github/workflows/docs_pr.yaml" - - "pyproject.toml" - - "uv.lock" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: @@ -55,12 +45,12 @@ jobs: name: Test doc build runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: submodules: true fetch-depth: 1 - name: Setup uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install doc dependencies run: uv sync --package datafusion-docs - name: Install Graphviz @@ -69,7 +59,7 @@ jobs: sudo apt-get update sudo apt-get install -y graphviz - name: Install cargo-depgraph - uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 + uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 with: tool: cargo-depgraph@1.6 - name: Build docs html and check for warnings @@ -77,24 +67,3 @@ jobs: set -x cd docs uv run --package datafusion-docs ./build.sh # fails on errors - - name: Fetch exact release tag - run: git fetch --depth=1 origin tag 55.0.0 - - name: Test versioned documentation tooling - run: uv run --package datafusion-docs python -m unittest discover -s docs/tests - - name: Validate current site - run: >- - uv run --package datafusion-docs python docs/scripts/validate_site.py - --site-root docs/build/html - - name: Build exact release snapshot - run: | - uv run --package datafusion-docs python docs/scripts/snapshot_site.py \ - 55.0.0 55.0.0 /tmp/datafusion-release-site - - name: Validate release deployment - run: | - uv run --package datafusion-docs python docs/scripts/assemble_site.py \ - --current-site docs/build/html \ - --published-site /tmp/datafusion-release-site \ - --output-site docs/build/validation-site - uv run --package datafusion-docs python docs/scripts/validate_site.py \ - --site-root docs/build/validation-site \ - --require-snapshots diff --git a/dev/release/README.md b/dev/release/README.md index bc2d6beaad6b5..64bb69370ec0e 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -303,26 +303,45 @@ git push apache 50.3.0 #### Publish the versioned documentation -For the initial `55.0.0` proof of concept, make the final exact tag available -locally and build its complete documentation site into a temporary publication -package: +After the final release tag is available, build its complete documentation +using the configuration from the tag. The small configuration overlay adds the +version picker and the correct URL prefix to older tags without changing their +documentation source. For example, from the repository root, for `55.0.0`: ```shell -uv run --package datafusion-docs python docs/scripts/snapshot_site.py 55.0.0 55.0.0 /tmp/datafusion-release-site +git fetch origin tag 55.0.0 +git worktree add --detach /tmp/datafusion-55.0.0 55.0.0 +cd /tmp/datafusion-55.0.0/docs +DATAFUSION_DOCS_SOURCE="$PWD/source" DATAFUSION_DOCS_VERSION=55.0.0 \ + SPHINXOPTS="-W -c /path/to/current/datafusion/docs/scripts/release" \ + uv run --package datafusion-docs --with sphinx-sitemap ./build.sh ``` -The command verifies the tag's peeled commit and creates the one immutable -publication unit `versions/55.0.0/`. Manually copy that directory to the same -location on the `asf-site` branch, then review, commit, and push the branch. The -command never commits or pushes and refuses to replace an existing release. - -Do not simply publish the snapshot before merging the feature PR: the old docs -deployment can erase it with `rsync --delete`. Keep the feature PR in draft until -maintainers choose either a preliminary retention-only merge before publication, -or a coordinated window that confirms no old docs deployment is running or -queued, publishes the snapshot, merges the retention and picker change, and -verifies `asf-site/versions/55.0.0/` after the new deployment. Automating future -release publication and backfilling older releases remain follow-up work. +Replace the version and paths for each release. Use an absolute path to the +overlay in your current checkout. The existing `build.sh` generates the tagged +dependency graph and builds the HTML with warnings treated as errors. The +`--with sphinx-sitemap` option supplies the sitemap extension for older tags. + +From the current checkout, prepare a separate PR targeting `asf-site`: + +```shell +git fetch origin asf-site +git worktree add -b docs/publish-55.0.0 /tmp/datafusion-asf-site origin/asf-site +mkdir -p /tmp/datafusion-asf-site/versions/55.0.0 +rsync -a --exclude '/.buildinfo' \ + /tmp/datafusion-55.0.0/docs/build/html/ \ + /tmp/datafusion-asf-site/versions/55.0.0/ +``` + +Check that the release directory does not already exist before copying it. +Review the generated files and open the publication PR for review. +Once it is published, add its entry to `docs/source/_static/versions.json` on +`main` so the picker offers only working destinations. Remove the temporary +worktrees afterward. + +Merge the separate documentation-retention workflow change before publishing +the first release documentation; otherwise the next `main` deployment would +delete `/versions/`. ### 10. Publish on Crates.io diff --git a/docs/README.md b/docs/README.md index 53a704a61543f..ac9fa681a48be 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,142 +19,72 @@ # DataFusion Documentation -This folder contains the sources for https://datafusion.apache.org/. The root -site is built continuously from `main`. Complete, immutable release sites are -published under `/versions//`. +This folder contains the source content of the [User Guide](./source/user-guide) +and [Contributor Guide](./source/contributor-guide). The site root shows the +development documentation built from `main`. Released versions of the complete +site are available under `/versions//`. ## Dependencies -From the repository root, install the documentation dependencies using +Install build dependencies and build the documentation using [uv](https://docs.astral.sh/uv/): ```sh -uv sync --package datafusion-docs +uv sync +uv run bash build.sh ``` The docs build regenerates the workspace dependency graph via `docs/scripts/generate_dependency_graph.sh`, so ensure `cargo`, `cargo-depgraph` (`cargo install cargo-depgraph --version ^1.6 --locked`), and Graphviz `dot` (`brew install graphviz` or `sudo apt-get install -y graphviz`) are available. -`.gitattributes` keeps documentation shell scripts LF-terminated so the same -scripts run from Linux and WSL checkouts. -## Build and Preview +## Build & Preview -Build the current complete site from the repository root: +Run the provided script to build the HTML pages. ```bash -uv run --package datafusion-docs docs/build.sh +# If using venv, ensure you have activated it +./build.sh ``` -The HTML is generated in `docs/build/html`. Serve it over HTTP because browsers -do not load the version manifest from `file:` URLs: +The HTML will be generated into a `build` directory. Serve the site over HTTP +to test the version switcher (it cannot fetch JSON from a `file:` URL): ```bash -python3 -m http.server --directory docs/build/html 8000 +python3 -m http.server --directory build/html 8000 ``` -Then open http://localhost:8000/. - -The public and assembled layouts are: - -```text -https://datafusion.apache.org/ -|-- user-guide/ -|-- library-user-guide/ -|-- contributor-guide/ -|-- _static/versions.json -`-- versions/55.0.0/ - |-- user-guide/ - |-- library-user-guide/ - |-- contributor-guide/ - |-- download.html - |-- search.html - |-- _sources/ - |-- _static/ - `-- sitemap.xml -``` - -## Release Snapshots - -`docs/source/_static/versions.json` is both the PyData version-picker manifest -and the release catalog. It contains `Development` at the site root and records -each release's semantic version, exact tag, and exact 40-character commit. - -The first snapshot is the lightweight tag `55.0.0`, which peels to -`d5552342012888b7d1a3ab88d92e3d292fc0cde0`. Create its publication package -from the repository root with the tag available locally: - -```bash -uv run --package datafusion-docs python docs/scripts/snapshot_site.py \ - 55.0.0 55.0.0 /tmp/datafusion-release-site -``` - -The command creates only -`/tmp/datafusion-release-site/versions/55.0.0/`. It refuses to overwrite that -directory, rejects output overlapping the repository, builds in an isolated -detached worktree, and does not commit or push. - -The snapshot uses the tag's complete documentation tree, templates, static -files, helper extension, build script dependency graph, and locked documentation -dependencies. `release_conf.py` supplies only the publication prefix, canonical -base URL, exactly pinned `sphinx-sitemap` extension, version picker, exact GitHub -tag, corrected repository name, and a release-local redirect. After Sphinx -builds, `snapshot_site.py` rewrites only published links that would otherwise -escape to current DataFusion docs or mutable DataFusion `main` and `latest` -targets, including links in the agent-facing `llms.txt`. This includes one narrow -fix for the tagged broken `/contributor-guide/gsoc_application_guidelines.html` -link, which is redirected to the tagged -`contributor-guide/gsoc/gsoc_application_guidelines_2025.html` page. Tagged -source files and generated `_sources` files are never modified. In particular, -the tagged statement that 55.0.0 has not been released yet remains unchanged. - -Validate a complete assembled site with: - -```bash -uv run --package datafusion-docs python docs/scripts/assemble_site.py \ - --current-site docs/build/html \ - --published-site /tmp/datafusion-release-site \ - --output-site /tmp/datafusion-site -uv run --package datafusion-docs python docs/scripts/validate_site.py \ - --site-root /tmp/datafusion-site --require-snapshots -``` - -The assembler starts with a fresh current build, copies the existing complete -`asf-site/versions/` archive unchanged, and creates a root sitemap index. It does -not use picker entries as a deletion list, so removing an old release from the -picker cannot erase its archive. +The version switcher reads `source/_static/versions.json` from the site root. +Add an entry only when that release's documentation is published. For the manual +release build and publication procedure, see +[the release guide](../dev/release/README.md#publish-the-versioned-documentation). -## Publication Bootstrap +For a local or fork preview, set `DATAFUSION_DOCS_BASE_URL` to the preview's root +URL when building both development and released docs, and change the URLs in +the preview's `_static/versions.json` to that same root. For example, use +`http://localhost:8000/` locally or `https://.github.io/datafusion/` +on GitHub Pages. Serve the combined site, with released docs under `versions/`, +and check the picker in both directions, a page missing from a release, search, +and static assets. Only the preview manifest should contain preview URLs. -The old deployment uses unrestricted `rsync --delete`, so publishing a snapshot -before snapshot retention reaches `main` is not race-free. Keep the feature PR -in draft until maintainers agree on one of these bootstrap procedures: +## Making Changes -1. Merge a preliminary retention-only workflow change, wait for it to deploy, - publish `versions/55.0.0/` manually to `asf-site`, then merge the picker and - validation changes. -2. Use a coordinated window: confirm no old documentation deployment is running - or queued, publish the snapshot, merge this change immediately, wait for the - new serialized deployment, and verify `asf-site/versions/55.0.0/` afterward. +To make changes to the docs, simply make a Pull Request with your +proposed changes as normal. When the PR is merged the docs will be +automatically updated. -For manual publication, copy the single generated `versions/55.0.0/` directory -to the same location in an `asf-site` worktree. Review, commit, and push that -branch manually. Never replace an existing release directory. The snapshot -tooling never commits, pushes, or publishes. +## Release Process -## Site Deployment +This documentation is hosted at https://datafusion.apache.org/ -When a documentation change reaches `main`, the deployment workflow: +When a PR is merged to the `main` branch of the DataFusion +repository, a [github workflow](https://github.com/apache/datafusion/blob/main/.github/workflows/docs.yaml) which: -1. Builds the current complete site and replaces all current root files. -2. Copies the entire pre-existing `asf-site/versions/` archive without using the - picker as a retention registry. -3. Builds `sitemap.xml` as an index over `sitemap-main.xml` and every retained - complete release sitemap. -4. Deliberately installs `.asf.yaml` and `.nojekyll`, then uses `rsync --delete` - while excluding `.git` to remove stale current output. -5. Serializes deployments and pushes a normal, non-force commit. +1. Builds the html content +2. Pushes the html content to the [`asf-site`](https://github.com/apache/datafusion/tree/asf-site) branch in this repository. -The Apache Software Foundation serves the branch according to -[`.asf.yaml`](https://github.com/apache/datafusion/blob/main/.asf.yaml). +The Apache Software Foundation provides https://datafusion.apache.org/, +which serves content based on the configuration in +[.asf.yaml](https://github.com/apache/datafusion/blob/main/.asf.yaml), +which specifies the target as https://datafusion.apache.org/. diff --git a/docs/scripts/assemble_site.py b/docs/scripts/assemble_site.py deleted file mode 100644 index 0536e49607d26..0000000000000 --- a/docs/scripts/assemble_site.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Replace the current site while retaining immutable full-site snapshots.""" - -from __future__ import annotations - -import argparse -from pathlib import Path -import shutil -import sys -import tempfile -import xml.etree.ElementTree as ET - -from versioned_docs import SITE_URL, VERSION_PATTERN, is_link, reject_links - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--current-site", type=Path, required=True) - parser.add_argument("--published-site", type=Path, required=True) - parser.add_argument("--output-site", type=Path, required=True) - return parser.parse_args() - - -def validate_paths(current_site: Path, published_site: Path, output_site: Path) -> None: - if not current_site.is_dir(): - raise RuntimeError(f"current site does not exist: {current_site}") - if not published_site.is_dir(): - raise RuntimeError(f"published site does not exist: {published_site}") - if not (current_site / "index.html").is_file(): - raise RuntimeError(f"current site has no index.html: {current_site}") - if is_link(output_site): - raise RuntimeError(f"output site must not be a link: {output_site}") - for left, right in ( - (current_site, output_site), - (published_site, output_site), - ): - left = left.resolve() - right = right.resolve() - if left == right or left.is_relative_to(right) or right.is_relative_to(left): - raise RuntimeError( - "output site and input sites must not contain one another" - ) - if output_site.exists(): - raise RuntimeError(f"output site already exists: {output_site}") - - -def archived_versions(published_site: Path) -> list[str]: - versions_root = published_site / "versions" - if not versions_root.exists(): - return [] - if is_link(versions_root) or not versions_root.is_dir(): - raise RuntimeError(f"published versions must be a directory: {versions_root}") - versions: list[str] = [] - for child in versions_root.iterdir(): - if is_link(child) or not child.is_dir(): - raise RuntimeError(f"invalid published version directory: {child}") - if not VERSION_PATTERN.fullmatch(child.name): - raise RuntimeError(f"unsafe published version directory name: {child.name}") - if not (child / "index.html").is_file(): - raise RuntimeError(f"published version has no index.html: {child}") - if not (child / "sitemap.xml").is_file(): - raise RuntimeError(f"published version has no sitemap.xml: {child}") - reject_links(child) - versions.append(child.name) - return sorted(versions, key=lambda value: tuple(map(int, value.split(".")))) - - -def write_sitemap_index(site: Path, versions: list[str]) -> None: - current_sitemap = site / "sitemap.xml" - if not current_sitemap.is_file(): - raise RuntimeError(f"current site sitemap is missing: {current_sitemap}") - current_sitemap.rename(site / "sitemap-main.xml") - locations = [f"{SITE_URL}/sitemap-main.xml"] + [ - f"{SITE_URL}/versions/{version}/sitemap.xml" for version in versions - ] - root = ET.Element( - "sitemapindex", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" - ) - for location in locations: - sitemap = ET.SubElement(root, "sitemap") - ET.SubElement(sitemap, "loc").text = location - ET.indent(root, space=" ") - ET.ElementTree(root).write(current_sitemap, encoding="utf-8", xml_declaration=True) - - -def assemble(current_site: Path, published_site: Path, output_site: Path) -> None: - validate_paths(current_site, published_site, output_site) - current_site = current_site.resolve() - published_site = published_site.resolve() - output_site = output_site.resolve() - versions = archived_versions(published_site) - output_site.parent.mkdir(parents=True, exist_ok=True) - temporary_root = Path( - tempfile.mkdtemp(prefix=f".{output_site.name}-", dir=output_site.parent) - ) - staged_site = temporary_root / "site" - try: - shutil.copytree(current_site, staged_site) - if versions: - shutil.copytree(published_site / "versions", staged_site / "versions") - write_sitemap_index(staged_site, versions) - if output_site.exists() or is_link(output_site): - raise RuntimeError(f"output site already exists: {output_site}") - staged_site.rename(output_site) - finally: - shutil.rmtree(temporary_root, ignore_errors=True) - - -if __name__ == "__main__": - try: - args = parse_args() - assemble(args.current_site, args.published_site, args.output_site) - except (OSError, RuntimeError) as error: - print(f"error: {error}", file=sys.stderr) - raise SystemExit(1) from error diff --git a/docs/scripts/release/conf.py b/docs/scripts/release/conf.py new file mode 100644 index 0000000000000..0f41956926526 --- /dev/null +++ b/docs/scripts/release/conf.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Use the tagged Sphinx configuration when manually building release docs.""" + +import os +from pathlib import Path +import sys + +source = Path(os.environ["DATAFUSION_DOCS_SOURCE"]).resolve() +sys.path.insert(0, str(source.parent)) +exec(compile((source / "conf.py").read_bytes(), str(source / "conf.py"), "exec")) + +# Resolve paths relative to the tag, not this configuration overlay. +templates_path = [str(source / path) for path in templates_path] +html_static_path = [str(source / path) for path in html_static_path] +html_extra_path = [str(source / path) for path in html_extra_path] +html_logo = str(source / html_logo) +html_favicon = str(source / html_favicon) + +version = release = os.environ["DATAFUSION_DOCS_VERSION"] +docs_base_url = os.environ.get("DATAFUSION_DOCS_BASE_URL", "https://datafusion.apache.org/") +docs_base_url = docs_base_url.rstrip("/") + "/" +html_baseurl = f"{docs_base_url}versions/{version}/" +sitemap_url_scheme = "{link}" +if "sphinx_sitemap" not in extensions: + extensions.append("sphinx_sitemap") +html_context = {**html_context, "github_repo": "datafusion", "github_version": version} +html_theme_options = { + **html_theme_options, + "navbar_end": ["version-switcher", "theme-switcher"], + "check_switcher": False, + "switcher": { + "json_url": docs_base_url + "_static/versions.json", + "version_match": version, + }, +} +# The old tag's absolute redirect would leave the release documentation. +redirects = {**redirects, "library-user-guide/upgrading": "upgrading/index.html"} diff --git a/docs/scripts/release_conf.py b/docs/scripts/release_conf.py deleted file mode 100644 index 355a3c3264e2d..0000000000000 --- a/docs/scripts/release_conf.py +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Minimal Sphinx configuration overlay for a full release site.""" - -from __future__ import annotations - -import os -from pathlib import Path -import sys - -source_dir = Path(os.environ["DATAFUSION_RELEASE_SOURCE"]) -docs_dir = source_dir.parent -sys.path.insert(0, str(docs_dir)) -tagged_conf = source_dir / "conf.py" -exec(compile(tagged_conf.read_bytes(), str(tagged_conf), "exec")) - -version = release = os.environ["DATAFUSION_RELEASE_VERSION"] -base_url = f"https://datafusion.apache.org/versions/{version}/" -html_baseurl = base_url -sitemap_url_scheme = "{link}" -extensions = [*extensions, "sphinx_sitemap"] -templates_path = [str(source_dir / path) for path in templates_path] -html_static_path = [str(source_dir / path) for path in html_static_path] -html_extra_path = [str(source_dir / path) for path in html_extra_path] -html_logo = str(source_dir / html_logo) -html_favicon = str(source_dir / html_favicon) - -html_context = dict(html_context) -html_context.update( - github_user="apache", - github_repo="datafusion", - github_version=os.environ["DATAFUSION_RELEASE_TAG"], -) -html_theme_options = dict(html_theme_options) -html_theme_options.update( - check_switcher=False, - navbar_end=["version-switcher", "theme-switcher"], - switcher={ - "json_url": "https://datafusion.apache.org/_static/versions.json", - "version_match": version, - }, -) - -redirects = dict(redirects) -redirects["library-user-guide/upgrading"] = "upgrading/index.html" diff --git a/docs/scripts/snapshot_site.py b/docs/scripts/snapshot_site.py deleted file mode 100644 index c9673cc8e87b3..0000000000000 --- a/docs/scripts/snapshot_site.py +++ /dev/null @@ -1,286 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Build one complete documentation snapshot from an exact release tag.""" - -from __future__ import annotations - -import argparse -import os -from pathlib import Path -import re -import shutil -import subprocess -import sys -import tempfile - -from versioned_docs import DOCS_DIR, VERSION_PATTERN, is_link, release_entry - -REPOSITORY = DOCS_DIR.parent -INTERNAL_DOC_LINK = re.compile( - rb"https://datafusion\.apache\.org/" - rb"(?=(?:user-guide|library-user-guide|contributor-guide)/)" -) -GITHUB_MAIN_LINK = re.compile( - rb"https://github\.com/apache/(?:arrow-)?datafusion/(blob|tree)/main" - rb"(?=/|[\"'#<)\s]|$)" -) -DOCS_RS_LATEST_LINK = re.compile( - rb"https://docs\.rs/(datafusion(?:-[a-z0-9-]+)?)/latest(?=/|[\"'#<])" -) -TAGGED_GSOC_GUIDELINES_LINK = re.compile( - rb"https://datafusion\.apache\.org/contributor-guide/" - rb"gsoc_application_guidelines\.html" -) - - -def git(*arguments: str, capture: bool = False) -> str: - result = subprocess.run( - ["git", *arguments], - cwd=REPOSITORY, - check=True, - text=True, - stdout=subprocess.PIPE if capture else None, - ) - return result.stdout.strip() if capture else "" - - -def verify_tag(tag: str, expected_commit: str) -> None: - tag_ref = f"refs/tags/{tag}" - try: - object_type = git("cat-file", "-t", tag_ref, capture=True) - commit = git("rev-parse", f"{tag_ref}^{{commit}}", capture=True) - peeled = git("rev-parse", f"{tag_ref}^{{}}", capture=True) - except subprocess.CalledProcessError as error: - raise RuntimeError( - f"tag {tag} is missing; fetch it with: git fetch origin tag {tag}" - ) from error - if object_type not in {"commit", "tag"}: - raise RuntimeError(f"tag {tag} has unexpected object type {object_type}") - if commit != expected_commit or peeled != expected_commit: - raise RuntimeError(f"tag {tag} peels to {commit}, expected {expected_commit}") - - -def validate_package_root(package_root: Path) -> Path: - package_root = package_root.resolve() - repository = REPOSITORY.resolve() - if package_root.is_relative_to(repository) or repository.is_relative_to( - package_root - ): - raise RuntimeError( - "snapshot output must be outside and must not contain the repository" - ) - return package_root - - -def adapt_generated_output(site: Path, version: str, tag: str) -> None: - """Pin published links without changing tagged sources or ``_sources``.""" - shutil.rmtree(site / ".doctrees", ignore_errors=True) - (site / ".buildinfo").unlink(missing_ok=True) - prefix = f"https://datafusion.apache.org/versions/{version}/".encode() - tag_bytes = tag.encode() - paths = [*site.rglob("*.html"), site / "llms.txt"] - for path in paths: - if not path.is_file(): - continue - content = path.read_bytes() - if path.suffix == ".html": - content = TAGGED_GSOC_GUIDELINES_LINK.sub( - prefix - + b"contributor-guide/gsoc/gsoc_application_guidelines_2025.html", - content, - ) - content = INTERNAL_DOC_LINK.sub(prefix, content) - content = GITHUB_MAIN_LINK.sub( - rb"https://github.com/apache/datafusion/\1/" + tag_bytes, - content, - ) - content = DOCS_RS_LATEST_LINK.sub( - lambda match: b"https://docs.rs/" - + match.group(1) - + b"/" - + version.encode(), - content, - ) - path.write_bytes(content) - - -def generate_dependency_graph(docs_dir: Path) -> None: - script = docs_dir / "scripts" / "generate_dependency_graph.sh" - if os.name != "nt": - subprocess.run(["bash", str(script)], cwd=docs_dir, check=True) - return - - for command in ("cargo", "dot"): - if shutil.which(command) is None: - raise RuntimeError( - f"{command} is required to build the tagged dependency graph" - ) - output = docs_dir / "source" / "_static" / "data" / "deps.svg" - output.parent.mkdir(parents=True, exist_ok=True) - with output.open("wb") as destination: - cargo = subprocess.Popen( - [ - "cargo", - "depgraph", - "--workspace-only", - "--all-deps", - "--dedup-transitive-deps", - "--exclude", - "gen,gen-common", - ], - cwd=docs_dir.parent, - stdout=subprocess.PIPE, - ) - assert cargo.stdout is not None - dot = subprocess.run( - [ - "dot", - "-Grankdir=TB", - "-Gconcentrate=true", - "-Goverlap=false", - "-Tsvg", - ], - stdin=cargo.stdout, - stdout=destination, - check=False, - ) - cargo.stdout.close() - cargo_status = cargo.wait() - if cargo_status != 0: - raise subprocess.CalledProcessError(cargo_status, cargo.args) - if dot.returncode != 0: - raise subprocess.CalledProcessError(dot.returncode, dot.args) - - -def publish_snapshot(staged_site: Path, package_root: Path, version: str) -> Path: - versions_root = package_root / "versions" - if is_link(versions_root): - raise RuntimeError(f"versions output must not be a link: {versions_root}") - versions_root.mkdir(parents=True, exist_ok=True) - destination = versions_root / version - if destination.exists() or is_link(destination): - raise RuntimeError(f"refusing to overwrite {destination}") - temporary_root = Path(tempfile.mkdtemp(prefix=f".{version}-", dir=versions_root)) - local_stage = temporary_root / "site" - try: - shutil.copytree(staged_site, local_stage) - if destination.exists() or is_link(destination): - raise RuntimeError(f"refusing to overwrite {destination}") - local_stage.rename(destination) - finally: - shutil.rmtree(temporary_root, ignore_errors=True) - return destination - - -def build_snapshot(version: str, tag: str, package_root: Path) -> Path: - if not VERSION_PATTERN.fullmatch(version) or tag != version: - raise RuntimeError("version and tag must be the same exact X.Y.Z release") - release = release_entry(version, tag) - expected_commit = str(release["commit"]) - verify_tag(tag, expected_commit) - package_root = validate_package_root(package_root) - destination = package_root / "versions" / version - if destination.exists() or is_link(destination): - raise RuntimeError(f"refusing to overwrite {destination}") - - temporary_parent = Path(tempfile.mkdtemp(prefix="datafusion-release-site-")) - worktree = temporary_parent / "worktree" - built_site = temporary_parent / "html" - worktree_added = False - try: - git("worktree", "add", "--detach", str(worktree), expected_commit) - worktree_added = True - actual_commit = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=worktree, - check=True, - text=True, - stdout=subprocess.PIPE, - ).stdout.strip() - if actual_commit != expected_commit: - raise RuntimeError( - f"temporary worktree is {actual_commit}, expected {expected_commit}" - ) - - docs_dir = worktree / "docs" - generate_dependency_graph(docs_dir) - environment = os.environ.copy() - environment.update( - DATAFUSION_RELEASE_SOURCE=str(docs_dir / "source"), - DATAFUSION_RELEASE_VERSION=version, - DATAFUSION_RELEASE_TAG=tag, - ) - config_dir = temporary_parent / "config" - config_dir.mkdir() - shutil.copy2(DOCS_DIR / "scripts" / "release_conf.py", config_dir / "conf.py") - subprocess.run( - [ - "uv", - "run", - "--project", - str(worktree), - "--package", - "datafusion-docs", - "--with", - "sphinx-sitemap==2.9.0", - "python", - "-m", - "sphinx", - "-W", - "-b", - "html", - "-d", - str(temporary_parent / "doctrees"), - "-c", - str(config_dir), - str(docs_dir / "source"), - str(built_site), - ], - cwd=docs_dir, - env=environment, - check=True, - ) - adapt_generated_output(built_site, version, tag) - return publish_snapshot(built_site, package_root, version) - finally: - if worktree_added: - subprocess.run( - ["git", "worktree", "remove", "--force", str(worktree)], - cwd=REPOSITORY, - check=False, - ) - shutil.rmtree(temporary_parent, ignore_errors=True) - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("version") - parser.add_argument("tag") - parser.add_argument("package_root", type=Path) - args = parser.parse_args() - destination = build_snapshot(args.version, args.tag, args.package_root) - print(f"snapshot created: {destination}") - - -if __name__ == "__main__": - try: - main() - except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as error: - print(f"error: {error}", file=sys.stderr) - raise SystemExit(1) from error diff --git a/docs/scripts/validate_site.py b/docs/scripts/validate_site.py deleted file mode 100644 index 7756b30d169ff..0000000000000 --- a/docs/scripts/validate_site.py +++ /dev/null @@ -1,420 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Validate current and immutable full-site documentation output.""" - -from __future__ import annotations - -import argparse -from contextlib import contextmanager -from functools import partial -from html.parser import HTMLParser -from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer -import json -from pathlib import Path -import re -import sys -import threading -from typing import Iterator -import xml.etree.ElementTree as ET -from urllib.error import HTTPError, URLError -from urllib.parse import unquote, urljoin, urlsplit -from urllib.request import Request, urlopen - -from versioned_docs import SITE_URL, load_versions - -PICKER_URL = re.compile( - r"DOCUMENTATION_OPTIONS\.theme_switcher_json_url\s*=\s*['\"]([^'\"]+)" -) -PICKER_VERSION = re.compile( - r"DOCUMENTATION_OPTIONS\.theme_switcher_version_match\s*=\s*['\"]([^'\"]+)" -) -MUTABLE_RELEASE_LINK = re.compile( - r"https://(?:" - r"github\.com/apache/(?:arrow-)?datafusion/(?:blob|tree)/main(?:/|[\"'#<)\s]|$)|" - r"docs\.rs/datafusion(?:-[a-z0-9-]+)?/latest(?:/|[\"'#<])" - r")" -) -CSS_URL = re.compile(r"url\(\s*(['\"]?)(.*?)\1\s*\)") -META_REFRESH_URL = re.compile(r"(?:^|;)\s*url\s*=\s*(['\"]?)(.*?)\1\s*$", re.I) - - -class ValidationError(RuntimeError): - pass - - -class HTMLReferences(HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.references: list[str] = [] - self.canonicals: list[str] = [] - self.ids: set[str] = set() - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - values = {name.lower(): value for name, value in attrs if value is not None} - for attribute in ("href", "src", "action", "poster", "data"): - if attribute in values: - self.references.append(values[attribute]) - for value in values.get("srcset", "").split(","): - if value.strip(): - self.references.append(value.strip().split()[0]) - if tag.lower() == "meta" and values.get("http-equiv", "").lower() == "refresh": - refresh = META_REFRESH_URL.search(values.get("content", "")) - if refresh: - self.references.append(refresh.group(2)) - for identifier in (values.get("id"), values.get("name")): - if identifier: - self.ids.add(identifier) - if tag.lower() == "link" and "canonical" in values.get("rel", "").split(): - self.canonicals.append(values["href"]) - - -def read_html(path: Path) -> tuple[str, HTMLReferences]: - text = path.read_text(encoding="utf-8") - parsed = HTMLReferences() - parsed.feed(text) - return text, parsed - - -def site_target(site_root: Path, page: Path, url: str) -> Path | None: - parsed = urlsplit(url) - if parsed.scheme not in ("", "http", "https"): - return None - if parsed.netloc and parsed.netloc != urlsplit(SITE_URL).netloc: - return None - if parsed.netloc and parsed.path.startswith( - ("/blog/", "/python/", "/java/", "/comet/", "/ballista/") - ): - return None - if not parsed.path: - return page - target = ( - site_root / unquote(parsed.path.lstrip("/")) - if parsed.netloc or parsed.path.startswith("/") - else page.parent / unquote(parsed.path) - ).resolve() - try: - target.relative_to(site_root.resolve()) - except ValueError as error: - raise ValidationError( - f"local target escapes site root: {page}: {url}" - ) from error - if parsed.path.endswith("/"): - target /= "index.html" - return target - - -def switch_candidate(page: str, source_version: str, target_url: str) -> str: - path = urlsplit(page).path.lstrip("/") - prefix = "" if source_version == "main" else f"versions/{source_version}/" - if not path.startswith(prefix): - raise ValidationError(f"page is outside its version prefix: {page}") - relative = path[len(prefix) :] - candidate = urljoin(target_url, relative) - if f"/versions/{source_version}/versions/" in candidate: - raise ValidationError(f"version path is duplicated: {candidate}") - return candidate - - -def validate_page( - site_root: Path, relative: str, version: str, errors: list[str] -) -> None: - path = site_root / relative - if not path.is_file(): - errors.append(f"representative page is missing: {path}") - return - text, parsed = read_html(path) - canonical = SITE_URL + "/" + relative.replace("\\", "/") - if parsed.canonicals != [canonical]: - errors.append(f"incorrect canonical in {path}: expected {canonical}") - picker_url = PICKER_URL.search(text) - if picker_url is None or picker_url.group(1) != f"{SITE_URL}/_static/versions.json": - errors.append(f"incorrect picker manifest URL in {path}") - picker_version = PICKER_VERSION.search(text) - if picker_version is None or picker_version.group(1) != version: - errors.append(f"incorrect picker version in {path}") - - -def validate_local_links(site_root: Path, root: Path, errors: list[str]) -> None: - parsed_pages: dict[Path, HTMLReferences] = {} - for page in root.rglob("*.html"): - if "_static" in page.relative_to(root).parts: - continue - _, parsed = read_html(page) - parsed_pages[page.resolve()] = parsed - expected_canonical = f"{SITE_URL}/{page.relative_to(site_root).as_posix()}" - if parsed.canonicals and parsed.canonicals != [expected_canonical]: - errors.append( - f"incorrect canonical in {page}: expected {expected_canonical}" - ) - for url in parsed.references: - parsed_url = urlsplit(url) - if parsed_url.netloc == urlsplit(SITE_URL).netloc and root == site_root: - # Current-source absolute links are checked by the repository's - # Markdown link checker, not by snapshot hosting validation. - continue - try: - target = site_target(site_root, page, url) - except ValidationError as error: - errors.append(str(error)) - continue - if target is not None and not target.exists(): - errors.append(f"missing local target in {page}: {url}") - continue - fragment = urlsplit(url).fragment - if target is not None and fragment and target.suffix == ".html": - target_parsed = parsed_pages.get(target.resolve()) - if target_parsed is None: - _, target_parsed = read_html(target) - parsed_pages[target.resolve()] = target_parsed - if unquote(fragment) not in target_parsed.ids: - errors.append(f"missing local fragment in {page}: {url}") - for stylesheet in root.rglob("*.css"): - text = stylesheet.read_text(encoding="utf-8") - for _, url in CSS_URL.findall(text): - target = site_target(site_root, stylesheet, url) - if target is not None and not target.exists(): - errors.append(f"missing CSS target in {stylesheet}: {url}") - - -def validate_sitemaps(site_root: Path, versions: list[str], errors: list[str]) -> None: - namespace = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - if versions: - try: - tree = ET.parse(site_root / "sitemap.xml") - except (OSError, ET.ParseError) as error: - errors.append(f"invalid sitemap index: {error}") - return - locations = [item.text for item in tree.findall("sm:sitemap/sm:loc", namespace)] - expected = [f"{SITE_URL}/sitemap-main.xml"] + [ - f"{SITE_URL}/versions/{version}/sitemap.xml" for version in versions - ] - if locations != expected: - errors.append("sitemap index has incorrect entries") - for version in versions: - sitemap = site_root / "versions" / version / "sitemap.xml" - try: - tree = ET.parse(sitemap) - except (OSError, ET.ParseError) as error: - errors.append(f"invalid release sitemap {sitemap}: {error}") - continue - prefix = f"{SITE_URL}/versions/{version}/" - for location in tree.findall("sm:url/sm:loc", namespace): - if location.text is None or not location.text.startswith(prefix): - errors.append(f"release sitemap URL escapes {prefix}: {location.text}") - - -class QuietHandler(SimpleHTTPRequestHandler): - def log_message(self, format: str, *args: object) -> None: - pass - - -@contextmanager -def local_server(site_root: Path) -> Iterator[str]: - server = ThreadingHTTPServer( - ("127.0.0.1", 0), partial(QuietHandler, directory=str(site_root)) - ) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - yield f"http://127.0.0.1:{server.server_port}" - finally: - server.shutdown() - server.server_close() - thread.join() - - -def validate_http(site_root: Path, paths: list[str], errors: list[str]) -> None: - with local_server(site_root) as server: - for relative in paths: - for method in ("GET", "HEAD"): - try: - with urlopen( - Request(f"{server}/{relative}", method=method), timeout=10 - ) as response: - if response.status != 200: - errors.append( - f"HTTP {method} {relative}: {response.status}" - ) - except (HTTPError, URLError, TimeoutError) as error: - errors.append(f"HTTP {method} {relative} failed: {error}") - - -def validate_site( - site_root: Path, *, require_snapshots: bool = False, http_checks: bool = True -) -> None: - site_root = site_root.resolve() - if not site_root.is_dir(): - raise ValidationError(f"site root does not exist: {site_root}") - entries = load_versions() - errors: list[str] = [] - manifest = site_root / "_static" / "versions.json" - if ( - not manifest.is_file() - or json.loads(manifest.read_text(encoding="utf-8")) != entries - ): - errors.append(f"site has a missing or stale versions manifest: {manifest}") - - releases = [str(entry["version"]) for entry in entries[1:]] - versions_root = site_root / "versions" - present = [] - if versions_root.is_dir(): - present = sorted( - ( - child.name - for child in versions_root.iterdir() - if child.is_dir() - and re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", child.name) - ), - key=lambda value: tuple(map(int, value.split("."))), - ) - if require_snapshots and not set(releases).issubset(present): - errors.append("required release snapshots are missing") - - representatives = ( - "index.html", - "user-guide/introduction.html", - "library-user-guide/index.html", - "contributor-guide/index.html", - "download.html", - "search.html", - "_static/theme_overrides.css", - "_sources/index.rst.txt", - "llms.txt", - ) - redirects = ( - "library-user-guide/adding-udfs.html", - "library-user-guide/upgrading.html", - "user-guide/runtime_configs.html", - ) - http_paths = ["_static/versions.json"] - for version in ["main", *present]: - prefix = "" if version == "main" else f"versions/{version}/" - root = site_root if version == "main" else site_root / "versions" / version - if not (root / "searchindex.js").is_file(): - errors.append(f"search index is missing: {root}") - elif (root / "searchindex.js").stat().st_size == 0: - errors.append(f"search index is empty: {root}") - if not (root / "objects.inv").is_file(): - errors.append(f"Sphinx inventory is missing: {root}") - elif (root / "objects.inv").stat().st_size == 0: - errors.append(f"Sphinx inventory is empty: {root}") - for representative in representatives: - relative = prefix + representative - if representative.endswith((".html",)): - validate_page(site_root, relative, version, errors) - elif not (site_root / relative).is_file(): - errors.append(f"representative file is missing: {relative}") - elif (site_root / relative).stat().st_size == 0: - errors.append(f"representative file is empty: {relative}") - http_paths.append(relative) - for redirect in redirects: - relative = prefix + redirect - if not (site_root / relative).is_file(): - errors.append(f"redirect is missing: {relative}") - http_paths.append(relative) - validate_local_links(site_root, root, errors) - - page = f"{SITE_URL}/{prefix}user-guide/sql/select.html" - for entry in entries: - candidate = switch_candidate(page, version, str(entry["url"])) - if "/versions/55.0.0/versions/" in candidate: - errors.append(f"picker duplicates a versions path: {candidate}") - - if version != "main": - release_files = [*root.rglob("*.html"), root / "llms.txt"] - for page_path in release_files: - if page_path.suffix == ".html": - text, parsed = read_html(page_path) - else: - text = page_path.read_text(encoding="utf-8") - parsed = HTMLReferences() - mutable = MUTABLE_RELEASE_LINK.search(text) - if mutable: - errors.append( - f"mutable release link in {page_path}: {mutable.group(0)}" - ) - for url in parsed.references: - parsed_url = urlsplit(url) - if ( - parsed_url.netloc == urlsplit(SITE_URL).netloc - and parsed_url.path.startswith( - ( - "/user-guide/", - "/library-user-guide/", - "/contributor-guide/", - ) - ) - and (site_root / parsed_url.path.lstrip("/")).exists() - ): - errors.append( - f"release documentation link escapes in {page_path}: {url}" - ) - - current_only = site_root / "library-user-guide" / "upgrading" / "56.0.0.html" - release_current_only = ( - site_root - / "versions" - / "55.0.0" - / "library-user-guide" - / "upgrading" - / "56.0.0.html" - ) - if not current_only.is_file(): - errors.append(f"current-only page is missing: {current_only}") - if release_current_only.exists(): - errors.append(f"release contains current-only page: {release_current_only}") - if "55.0.0" in present: - tagged_source = ( - site_root - / "versions" - / "55.0.0" - / "_sources" - / "library-user-guide" - / "upgrading" - / "55.0.0.md.txt" - ) - if ( - not tagged_source.is_file() - or "DataFusion `55.0.0` has not been released yet." - not in tagged_source.read_text(encoding="utf-8") - ): - errors.append("exact tagged 55.0.0 release-note prose was not preserved") - - validate_sitemaps(site_root, present, errors) - if http_checks and not errors: - validate_http(site_root, http_paths, errors) - if errors: - raise ValidationError("site validation failed:\n- " + "\n- ".join(errors)) - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--site-root", type=Path, required=True) - parser.add_argument("--require-snapshots", action="store_true") - args = parser.parse_args() - validate_site(args.site_root, require_snapshots=args.require_snapshots) - print("site validation passed") - - -if __name__ == "__main__": - try: - main() - except (OSError, ValueError, ValidationError) as error: - print(f"error: {error}", file=sys.stderr) - raise SystemExit(1) from error diff --git a/docs/scripts/versioned_docs.py b/docs/scripts/versioned_docs.py deleted file mode 100644 index d52b6e2be564c..0000000000000 --- a/docs/scripts/versioned_docs.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Shared catalog handling for full-site documentation snapshots.""" - -from __future__ import annotations - -import json -import os -from pathlib import Path -import re -import stat - -DOCS_DIR = Path(__file__).resolve().parent.parent -MANIFEST = DOCS_DIR / "source" / "_static" / "versions.json" -SITE_URL = "https://datafusion.apache.org" -VERSION_PATTERN = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+") -EXPECTED_55_COMMIT = "d5552342012888b7d1a3ab88d92e3d292fc0cde0" - - -def is_link(path: Path) -> bool: - """Return whether a path is a symlink or Windows reparse point.""" - try: - status = path.lstat() - except FileNotFoundError: - return False - reparse_point = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) - file_attributes = getattr(status, "st_file_attributes", 0) - return stat.S_ISLNK(status.st_mode) or bool(file_attributes & reparse_point) - - -def reject_links(root: Path) -> None: - """Reject links before copying an immutable archive tree.""" - if is_link(root): - raise RuntimeError(f"archived path must not be a link: {root}") - for directory, directories, files in os.walk(root, followlinks=False): - for name in [*directories, *files]: - path = Path(directory) / name - if is_link(path): - raise RuntimeError(f"archived path must not be a link: {path}") - - -def load_versions(path: Path = MANIFEST) -> list[dict[str, object]]: - try: - entries = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: - raise ValueError(f"cannot read versions manifest {path}: {error}") from error - if not isinstance(entries, list) or not entries: - raise ValueError("versions manifest must contain a non-empty list") - - versions: set[str] = set() - urls: set[str] = set() - preferred = 0 - for index, entry in enumerate(entries): - if not isinstance(entry, dict): - raise ValueError("each versions manifest entry must be an object") - version = entry.get("version") - url = entry.get("url") - if not isinstance(version, str) or version in versions: - raise ValueError(f"invalid or duplicate version: {version!r}") - if not isinstance(url, str) or url in urls: - raise ValueError(f"invalid or duplicate version URL: {url!r}") - versions.add(version) - urls.add(url) - preferred += entry.get("preferred") is True - - if index == 0: - if entry != { - "name": "Development", - "url": f"{SITE_URL}/", - "version": "main", - }: - raise ValueError( - "the first entry must be Development/main at site root" - ) - continue - if not VERSION_PATTERN.fullmatch(version): - raise ValueError(f"invalid semantic release version: {version!r}") - expected_url = f"{SITE_URL}/versions/{version}/" - if entry.get("name") != version or url != expected_url: - raise ValueError(f"release {version} has an invalid name or URL") - if entry.get("tag") != version or not re.fullmatch( - r"[0-9a-f]{40}", str(entry.get("commit", "")) - ): - raise ValueError(f"release {version} must have an exact tag and commit") - if preferred > 1: - raise ValueError("at most one versions manifest entry may be preferred") - - release_55 = next( - (entry for entry in entries if entry["version"] == "55.0.0"), None - ) - if release_55 is None or release_55.get("commit") != EXPECTED_55_COMMIT: - raise ValueError("55.0.0 must resolve to its expected exact commit") - return entries - - -def release_entry(version: str, tag: str) -> dict[str, object]: - for entry in load_versions(): - if entry["version"] == version and entry.get("tag") == tag: - return entry - raise RuntimeError("version and tag must exactly match versions.json") diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json index d78cddec398c1..3abc7e6db4f99 100644 --- a/docs/source/_static/versions.json +++ b/docs/source/_static/versions.json @@ -5,9 +5,12 @@ "version": "main" }, { - "commit": "d5552342012888b7d1a3ab88d92e3d292fc0cde0", + "name": "55.1.0", + "url": "https://datafusion.apache.org/versions/55.1.0/", + "version": "55.1.0" + }, + { "name": "55.0.0", - "tag": "55.0.0", "url": "https://datafusion.apache.org/versions/55.0.0/", "version": "55.0.0" } diff --git a/docs/source/conf.py b/docs/source/conf.py index a4f7460dfd0a4..818b328a9cd66 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -88,6 +88,8 @@ # a list of builtin themes. # html_theme = "pydata_sphinx_theme" +html_baseurl = os.environ.get("DATAFUSION_DOCS_BASE_URL", "https://datafusion.apache.org/") +html_baseurl = html_baseurl.rstrip("/") + "/" html_theme_options = { "logo": { @@ -97,17 +99,17 @@ "use_edit_page_button": True, "navbar_center": [], "navbar_end": ["version-switcher", "theme-switcher"], - # The persistent production URL is intentionally not fetched at build time. + # Release docs are published separately; the manifest may not be live yet. "check_switcher": False, "switcher": { - "json_url": "https://datafusion.apache.org/_static/versions.json", + "json_url": html_baseurl + "_static/versions.json", "version_match": version, }, } html_context = { "github_user": "apache", - "github_repo": "datafusion", + "github_repo": "arrow-datafusion", "github_version": "main", "doc_path": "docs/source", } @@ -121,16 +123,7 @@ # resolve at the conventional URL `https://datafusion.apache.org/llms.txt`. html_extra_path = ["llms.txt", "robots.txt"] -# Base URL of the published site. Required by sphinx-sitemap to build -# absolute entries in the generated sitemap.xml. -html_baseurl = "https://datafusion.apache.org/" - -# The site is published as a single flat, single-language tree (see -# .github/workflows/docs.yaml, which rsyncs docs/build/html/ straight to the -# site root) — there is no /en/ or version path segment. Override -# sphinx-sitemap's default "{lang}{version}{link}" scheme, which assumes a -# Read the Docs-style multi-language/multi-version layout, so entries -# match the real URLs instead of a nonexistent /en/... prefix. +# html_baseurl already includes the release prefix when building tagged docs. sitemap_url_scheme = "{link}" html_logo = "_static/images/2x_bgwhite_original.png" diff --git a/docs/tests/test_versioned_docs.py b/docs/tests/test_versioned_docs.py deleted file mode 100644 index f4193bd1427d2..0000000000000 --- a/docs/tests/test_versioned_docs.py +++ /dev/null @@ -1,347 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import annotations - -import importlib.util -import json -from pathlib import Path -import shutil -import subprocess -import sys -import tempfile -import unittest -from unittest import mock - -DOCS_DIR = Path(__file__).resolve().parents[1] -SCRIPTS_DIR = DOCS_DIR / "scripts" -sys.path.insert(0, str(SCRIPTS_DIR)) - - -def load_script(name: str): - spec = importlib.util.spec_from_file_location(name, SCRIPTS_DIR / f"{name}.py") - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -versioned_docs = load_script("versioned_docs") -snapshot_site = load_script("snapshot_site") -assemble_site = load_script("assemble_site") -validate_site = load_script("validate_site") - - -def write(path: Path, content: str = "file") -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") - - -class VersionedDocsTest(unittest.TestCase): - def setUp(self) -> None: - self.temporary = tempfile.TemporaryDirectory() - self.root = Path(self.temporary.name) - - def tearDown(self) -> None: - self.temporary.cleanup() - - def test_manifest_has_one_site_wide_exact_release(self) -> None: - entries = versioned_docs.load_versions() - self.assertEqual( - entries, - [ - { - "name": "Development", - "url": f"{versioned_docs.SITE_URL}/", - "version": "main", - }, - { - "commit": versioned_docs.EXPECTED_55_COMMIT, - "name": "55.0.0", - "tag": "55.0.0", - "url": f"{versioned_docs.SITE_URL}/versions/55.0.0/", - "version": "55.0.0", - }, - ], - ) - - def test_manifest_is_appendable_and_rejects_duplicates(self) -> None: - manifest = self.root / "versions.json" - entries = versioned_docs.load_versions() - entries.append( - { - "commit": "a" * 40, - "name": "56.0.0", - "tag": "56.0.0", - "url": f"{versioned_docs.SITE_URL}/versions/56.0.0/", - "version": "56.0.0", - } - ) - write(manifest, json.dumps(entries)) - self.assertEqual(len(versioned_docs.load_versions(manifest)), 3) - entries[-1]["version"] = "55.0.0" - write(manifest, json.dumps(entries)) - with self.assertRaisesRegex(ValueError, "duplicate version"): - versioned_docs.load_versions(manifest) - - def test_exact_lightweight_tag_and_peeled_commit(self) -> None: - snapshot_site.verify_tag("55.0.0", versioned_docs.EXPECTED_55_COMMIT) - self.assertEqual( - snapshot_site.git("cat-file", "-t", "refs/tags/55.0.0", capture=True), - "commit", - ) - - def test_exact_tag_keeps_unreleased_55_prose(self) -> None: - prose = snapshot_site.git( - "show", - "55.0.0:docs/source/library-user-guide/upgrading/55.0.0.md", - capture=True, - ) - self.assertIn("DataFusion `55.0.0` has not been released yet.", prose) - - def test_missing_tag_error_is_actionable(self) -> None: - repository = self.root / "repository" - subprocess.run( - ["git", "init", str(repository)], check=True, capture_output=True - ) - with mock.patch.object(snapshot_site, "REPOSITORY", repository): - with self.assertRaisesRegex(RuntimeError, "git fetch origin tag missing"): - snapshot_site.verify_tag("missing", "0" * 40) - - def test_generated_html_adaptations_leave_sources_unchanged(self) -> None: - site = self.root / "site" - original = ( - "https://datafusion.apache.org/contributor-guide/testing.html\n" - "https://datafusion.apache.org/contributor-guide/gsoc_application_guidelines.html\n" - "https://github.com/apache/datafusion/blob/main/docs/source/index.rst\n" - "https://docs.rs/datafusion/latest/datafusion/\n" - ) - write(site / "index.html", original) - write(site / "_sources" / "index.rst.txt", original) - write(site / ".buildinfo", "environment-specific metadata") - write(site / ".doctrees" / "environment.pickle", "temporary paths") - write( - site / "llms.txt", - original + "https://github.com/apache/datafusion/tree/main\n", - ) - snapshot_site.adapt_generated_output(site, "55.0.0", "55.0.0") - output = (site / "index.html").read_text(encoding="utf-8") - self.assertIn("/versions/55.0.0/contributor-guide/", output) - self.assertIn( - "/versions/55.0.0/contributor-guide/gsoc/gsoc_application_guidelines_2025.html", - output, - ) - self.assertIn("/blob/55.0.0/docs/source/", output) - self.assertIn("docs.rs/datafusion/55.0.0/datafusion/", output) - self.assertEqual( - (site / "_sources" / "index.rst.txt").read_text(encoding="utf-8"), - original, - ) - llms = (site / "llms.txt").read_text(encoding="utf-8") - self.assertIn("/versions/55.0.0/contributor-guide/", llms) - self.assertIn("/tree/55.0.0", llms) - self.assertNotIn("/tree/main", llms) - self.assertFalse((site / ".buildinfo").exists()) - self.assertFalse((site / ".doctrees").exists()) - - def test_publication_is_one_directory_and_refuses_overwrite(self) -> None: - staged = self.root / "staged" - package = self.root / "package with spaces" - write(staged / "index.html") - destination = snapshot_site.publish_snapshot(staged, package, "55.0.0") - self.assertEqual(destination, package / "versions" / "55.0.0") - self.assertTrue((destination / "index.html").is_file()) - write(staged / "index.html") - with self.assertRaisesRegex(RuntimeError, "refusing to overwrite"): - snapshot_site.publish_snapshot(staged, package, "55.0.0") - - def test_publication_failure_leaves_no_release_directory(self) -> None: - staged = self.root / "staged" - package = self.root / "package" - write(staged / "index.html") - with mock.patch.object( - snapshot_site.shutil, - "copytree", - side_effect=OSError("simulated copy failure"), - ): - with self.assertRaisesRegex(OSError, "simulated copy failure"): - snapshot_site.publish_snapshot(staged, package, "55.0.0") - self.assertFalse((package / "versions" / "55.0.0").exists()) - self.assertEqual(list((package / "versions").iterdir()), []) - - def test_snapshot_rejects_unsafe_output_and_version(self) -> None: - with self.assertRaisesRegex(RuntimeError, "must not contain the repository"): - snapshot_site.validate_package_root(snapshot_site.REPOSITORY.parent) - with self.assertRaisesRegex(RuntimeError, "must be outside"): - snapshot_site.validate_package_root(snapshot_site.REPOSITORY / "docs/build") - with self.assertRaisesRegex(RuntimeError, "exact X.Y.Z"): - snapshot_site.build_snapshot("../55", "../55", self.root / "output") - - def test_assembly_replaces_current_and_retains_complete_archive(self) -> None: - current = self.root / "current" - published = self.root / "published" - output = self.root / "output" - write(current / "index.html", "new") - write(current / "sitemap.xml", "") - write(published / "obsolete.html", "old current") - for version in ("54.0.0", "55.0.0"): - write(published / "versions" / version / "index.html", version) - write(published / "versions" / version / "sitemap.xml", "") - assemble_site.assemble(current, published, output) - self.assertEqual((output / "index.html").read_text(encoding="utf-8"), "new") - self.assertFalse((output / "obsolete.html").exists()) - self.assertTrue((output / "versions" / "54.0.0" / "index.html").is_file()) - self.assertTrue((output / "versions" / "55.0.0" / "index.html").is_file()) - sitemap = (output / "sitemap.xml").read_text(encoding="utf-8") - self.assertIn("sitemap-main.xml", sitemap) - self.assertIn("versions/54.0.0/sitemap.xml", sitemap) - errors: list[str] = [] - validate_site.validate_sitemaps(output, ["54.0.0", "55.0.0"], errors) - self.assertEqual(errors, []) - - def test_removed_picker_entry_does_not_delete_archive(self) -> None: - current = self.root / "current" - published = self.root / "published" - output = self.root / "output" - write(current / "index.html") - write(current / "sitemap.xml", "") - write(published / "versions" / "54.0.0" / "index.html") - write(published / "versions" / "54.0.0" / "sitemap.xml") - assemble_site.assemble(current, published, output) - self.assertTrue((output / "versions" / "54.0.0" / "index.html").is_file()) - - def test_assembly_rejects_unsafe_archive_name_and_path_overlap(self) -> None: - published = self.root / "published" - write(published / "versions" / "latest" / "index.html") - write(published / "versions" / "latest" / "sitemap.xml") - with self.assertRaisesRegex(RuntimeError, "unsafe published version"): - assemble_site.archived_versions(published) - current = self.root / "container" / "current" - write(current / "index.html") - with self.assertRaisesRegex(RuntimeError, "must not contain one another"): - assemble_site.validate_paths(current, published, self.root / "container") - - def test_assembly_rejects_links_in_archive_and_output(self) -> None: - current = self.root / "current" - published = self.root / "published" - external = self.root / "external" - output_link = self.root / "output-link" - write(current / "index.html") - write(published / "versions" / "55.0.0" / "index.html") - write(published / "versions" / "55.0.0" / "sitemap.xml") - write(external / "secret.txt") - archive_link = published / "versions" / "55.0.0" / "external" - try: - archive_link.symlink_to(external, target_is_directory=True) - output_link.symlink_to(external, target_is_directory=True) - except OSError as error: - self.skipTest(f"directory symlinks are unavailable: {error}") - with self.assertRaisesRegex(RuntimeError, "archived path must not be a link"): - assemble_site.archived_versions(published) - archive_link.unlink() - with self.assertRaisesRegex(RuntimeError, "output site must not be a link"): - assemble_site.assemble(current, published, output_link) - self.assertTrue((external / "secret.txt").is_file()) - - def test_assembly_refuses_to_replace_existing_output(self) -> None: - current = self.root / "current" - published = self.root / "published" - output = self.root / "output" - write(current / "index.html") - write(current / "sitemap.xml", "") - published.mkdir() - write(output / "sentinel.txt", "previous") - with self.assertRaisesRegex(RuntimeError, "output site already exists"): - assemble_site.assemble(current, published, output) - self.assertEqual( - (output / "sentinel.txt").read_text(encoding="utf-8"), "previous" - ) - - def test_assembly_refuses_output_created_while_staging(self) -> None: - current = self.root / "current" - published = self.root / "published" - output = self.root / "output" - write(current / "index.html") - write(current / "sitemap.xml", "") - published.mkdir() - real_copytree = shutil.copytree - - def create_output(source: Path, destination: Path) -> Path: - copied = real_copytree(source, destination) - write(output / "sentinel.txt", "concurrent") - return copied - - with mock.patch.object(assemble_site.shutil, "copytree", create_output): - with self.assertRaisesRegex(RuntimeError, "output site already exists"): - assemble_site.assemble(current, published, output) - self.assertEqual( - (output / "sentinel.txt").read_text(encoding="utf-8"), "concurrent" - ) - - def test_picker_equivalent_paths_and_fallback(self) -> None: - release = f"{versioned_docs.SITE_URL}/versions/55.0.0/" - candidate = validate_site.switch_candidate( - f"{versioned_docs.SITE_URL}/user-guide/sql/select.html", "main", release - ) - self.assertEqual(candidate, release + "user-guide/sql/select.html") - reverse = validate_site.switch_candidate( - candidate, "55.0.0", f"{versioned_docs.SITE_URL}/" - ) - self.assertEqual( - reverse, f"{versioned_docs.SITE_URL}/user-guide/sql/select.html" - ) - current_only = validate_site.switch_candidate( - f"{versioned_docs.SITE_URL}/library-user-guide/upgrading/56.0.0.html", - "main", - release, - ) - self.assertEqual( - current_only, release + "library-user-guide/upgrading/56.0.0.html" - ) - self.assertNotIn("/versions/55.0.0/versions/", current_only) - - @unittest.skipUnless(shutil.which("rsync"), "rsync is not installed") - def test_rsync_keeps_git_and_deletes_stale_current_files(self) -> None: - source = self.root / "source" - destination = self.root / "destination" - write(source / "index.html", "new") - write(source / "versions" / "55.0.0" / "index.html") - write(source / ".asf.yaml") - write(source / ".nojekyll") - write(destination / "stale.html") - write(destination / ".git" / "HEAD", "git") - subprocess.run( - [ - "rsync", - "-a", - "--delete", - "--exclude", - "/.git/", - f"{source}/", - f"{destination}/", - ], - check=True, - ) - self.assertFalse((destination / "stale.html").exists()) - self.assertTrue((destination / "versions" / "55.0.0" / "index.html").is_file()) - self.assertEqual( - (destination / ".git" / "HEAD").read_text(encoding="utf-8"), "git" - ) - - -if __name__ == "__main__": - unittest.main() From 59b9cf643cfb1da363c4ea2d3c6c69c20d2982c2 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 8 Oct 2026 16:28:37 -0400 Subject: [PATCH 04/12] trim unecssary docs, remove PR comment pretending to be documetnation --- dev/release/README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/dev/release/README.md b/dev/release/README.md index 6385a29cd2a32..8244fbbe79089 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -315,10 +315,8 @@ git push apache 55.2.0 #### Publish the versioned documentation -After the final release tag is available, build its complete documentation -using the configuration from the tag. The small configuration overlay adds the -version picker and the correct URL prefix to older tags without changing their -documentation source. For example, from the repository root, for `55.0.0`: +After the final release tag is available, build the documentation for this release +For example, from the repository root, for `55.0.0`: ```shell git fetch origin tag 55.0.0 @@ -351,10 +349,6 @@ Once it is published, add its entry to `docs/source/_static/versions.json` on `main` so the picker offers only working destinations. Remove the temporary worktrees afterward. -Merge the separate documentation-retention workflow change before publishing -the first release documentation; otherwise the next `main` deployment would -delete `/versions/`. - ### 10. Publish on Crates.io Only approved releases of the tarball should be published to From a1fc0119d1907383a5acdf3aa91bd48a8c9545e1 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 8 Oct 2026 16:39:55 -0400 Subject: [PATCH 05/12] simplify and document local publish workflow --- docs/README.md | 27 +++++++++++++-------------- docs/source/_static/versions.json | 17 +++++++++-------- docs/source/conf.py | 15 +++++++++++---- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/docs/README.md b/docs/README.md index ac9fa681a48be..78cd865d895ce 100644 --- a/docs/README.md +++ b/docs/README.md @@ -48,25 +48,24 @@ Run the provided script to build the HTML pages. ./build.sh ``` -The HTML will be generated into a `build` directory. Serve the site over HTTP -to test the version switcher (it cannot fetch JSON from a `file:` URL): +The HTML will be generated into a `build` directory. Open `build/html/index.html` +in your preferred browser, for example on Linux via + +```bash +# On macOS +open build/html/index.html +# On Linux with Firefox +firefox build/html/index.html +``` + +Note that some features of the site, such as the "Version Picker" do not work +when read from files. To preview such features, start a local webserver: ```bash python3 -m http.server --directory build/html 8000 ``` -The version switcher reads `source/_static/versions.json` from the site root. -Add an entry only when that release's documentation is published. For the manual -release build and publication procedure, see -[the release guide](../dev/release/README.md#publish-the-versioned-documentation). - -For a local or fork preview, set `DATAFUSION_DOCS_BASE_URL` to the preview's root -URL when building both development and released docs, and change the URLs in -the preview's `_static/versions.json` to that same root. For example, use -`http://localhost:8000/` locally or `https://.github.io/datafusion/` -on GitHub Pages. Serve the combined site, with released docs under `versions/`, -and check the picker in both directions, a page missing from a release, search, -and static assets. Only the preview manifest should contain preview URLs. +And then open http://localhost:8000/ in your browser ## Making Changes diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json index 3abc7e6db4f99..23470a79f42d9 100644 --- a/docs/source/_static/versions.json +++ b/docs/source/_static/versions.json @@ -1,17 +1,18 @@ [ - { - "name": "Development", - "url": "https://datafusion.apache.org/", - "version": "main" - }, { "name": "55.1.0", - "url": "https://datafusion.apache.org/versions/55.1.0/", - "version": "55.1.0" + "url": "/versions/55.1.0/", + "version": "55.1.0", + "preferred": true }, { "name": "55.0.0", - "url": "https://datafusion.apache.org/versions/55.0.0/", + "url": "/versions/55.0.0/", "version": "55.0.0" + }, + { + "name": "Development", + "url": "/", + "version": "main" } ] diff --git a/docs/source/conf.py b/docs/source/conf.py index 818b328a9cd66..7df696576ab3d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -88,8 +88,6 @@ # a list of builtin themes. # html_theme = "pydata_sphinx_theme" -html_baseurl = os.environ.get("DATAFUSION_DOCS_BASE_URL", "https://datafusion.apache.org/") -html_baseurl = html_baseurl.rstrip("/") + "/" html_theme_options = { "logo": { @@ -102,7 +100,7 @@ # Release docs are published separately; the manifest may not be live yet. "check_switcher": False, "switcher": { - "json_url": html_baseurl + "_static/versions.json", + "json_url": "/_static/versions.json", "version_match": version, }, } @@ -123,7 +121,16 @@ # resolve at the conventional URL `https://datafusion.apache.org/llms.txt`. html_extra_path = ["llms.txt", "robots.txt"] -# html_baseurl already includes the release prefix when building tagged docs. +# Base URL of the published site. Required by sphinx-sitemap to build +# absolute entries in the generated sitemap.xml. +html_baseurl = "https://datafusion.apache.org/" + +# The site is published as a single flat, single-language tree (see +# .github/workflows/docs.yaml, which rsyncs docs/build/html/ straight to the +# site root) — there is no /en/ or version path segment. Override +# sphinx-sitemap's default "{lang}{version}{link}" scheme, which assumes a +# Read the Docs-style multi-language/multi-version layout, so entries +# match the real URLs instead of a nonexistent /en/... prefix. sitemap_url_scheme = "{link}" html_logo = "_static/images/2x_bgwhite_original.png" From 5a7ee5214d914f5adf90e7e75c6231a633344b50 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 8 Oct 2026 16:41:35 -0400 Subject: [PATCH 06/12] Update versions --- docs/source/_static/versions.json | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json index 23470a79f42d9..24dd2e8e09467 100644 --- a/docs/source/_static/versions.json +++ b/docs/source/_static/versions.json @@ -1,18 +1,17 @@ [ + { + "name": "Latest", + "url": "https://datafusion.apache.org/", + "version": "main" + }, { "name": "55.1.0", - "url": "/versions/55.1.0/", - "version": "55.1.0", - "preferred": true + "url": "https://datafusion.apache.org/versions/55.1.0/", + "version": "55.1.0" }, { "name": "55.0.0", - "url": "/versions/55.0.0/", + "url": "https://datafusion.apache.org/versions/55.0.0/", "version": "55.0.0" - }, - { - "name": "Development", - "url": "/", - "version": "main" } ] From 98afe601a3b903c86508653df332210a64104221 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 8 Oct 2026 16:42:32 -0400 Subject: [PATCH 07/12] fix urls --- docs/source/_static/versions.json | 6 +++--- docs/source/index.md | 0 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 docs/source/index.md diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json index 24dd2e8e09467..993ac84c9770e 100644 --- a/docs/source/_static/versions.json +++ b/docs/source/_static/versions.json @@ -1,17 +1,17 @@ [ { "name": "Latest", - "url": "https://datafusion.apache.org/", + "url": "/", "version": "main" }, { "name": "55.1.0", - "url": "https://datafusion.apache.org/versions/55.1.0/", + "url": "/versions/55.1.0/", "version": "55.1.0" }, { "name": "55.0.0", - "url": "https://datafusion.apache.org/versions/55.0.0/", + "url": "/versions/55.0.0/", "version": "55.0.0" } ] diff --git a/docs/source/index.md b/docs/source/index.md new file mode 100644 index 0000000000000..e69de29bb2d1d From 6ac6765615b223fc379bef22d57ff619d7628072 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 8 Oct 2026 16:44:16 -0400 Subject: [PATCH 08/12] remove index --- docs/source/index.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/source/index.md diff --git a/docs/source/index.md b/docs/source/index.md deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 53f8bb3fd87a243d7c054db068f816ce35519494 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 8 Oct 2026 16:51:07 -0400 Subject: [PATCH 09/12] update build readme --- docs/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index 78cd865d895ce..35892673b18f8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -59,10 +59,10 @@ firefox build/html/index.html ``` Note that some features of the site, such as the "Version Picker" do not work -when read from files. To preview such features, start a local webserver: +when read from files. To preview such features, you can use the sphinx like this: ```bash -python3 -m http.server --directory build/html 8000 +uv run --with sphinx-autobuild sphinx-autobuild source build/html --port 8000 ``` And then open http://localhost:8000/ in your browser From f22bba2386f0aea877739cb794972d6efb047d88 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 8 Oct 2026 16:59:13 -0400 Subject: [PATCH 10/12] Update release/conf.py script and docs --- docs/scripts/release/conf.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/scripts/release/conf.py b/docs/scripts/release/conf.py index 0f41956926526..8776449cb9711 100644 --- a/docs/scripts/release/conf.py +++ b/docs/scripts/release/conf.py @@ -15,7 +15,26 @@ # specific language governing permissions and limitations # under the License. -"""Use the tagged Sphinx configuration when manually building release docs.""" +"""Sphinx configuration overlay for building docs from a release tag. + +Loads ``conf.py`` from a``docs/source`` directory located at +``DATAFUSION_DOCS_SOURCE`` and overrides the following settings: +- the version string (``DATAFUSION_DOCS_VERSION``), +- the site base URL used for the sitemap +- the version switcher. + +Why? + +Documentation from older release tags do not have the version picker code and +expect their base URL to be ``/`` rather than ```versions//``, so we can +not publish docs built for the archived version without modification. + +Rather than modifying the documentation in the old release branch, +the release process builds docs with the following command from a main +checkout: + +sphinx-build -c docs/scripts/release +""" import os from pathlib import Path @@ -33,9 +52,7 @@ html_favicon = str(source / html_favicon) version = release = os.environ["DATAFUSION_DOCS_VERSION"] -docs_base_url = os.environ.get("DATAFUSION_DOCS_BASE_URL", "https://datafusion.apache.org/") -docs_base_url = docs_base_url.rstrip("/") + "/" -html_baseurl = f"{docs_base_url}versions/{version}/" +html_baseurl = f"https://datafusion.apache.org/versions/{version}/" sitemap_url_scheme = "{link}" if "sphinx_sitemap" not in extensions: extensions.append("sphinx_sitemap") @@ -45,7 +62,7 @@ "navbar_end": ["version-switcher", "theme-switcher"], "check_switcher": False, "switcher": { - "json_url": docs_base_url + "_static/versions.json", + "json_url": "/_static/versions.json", "version_match": version, }, } From acd7dbaedb66c0b6135622d1b86bc751b07c21c2 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 8 Oct 2026 17:08:23 -0400 Subject: [PATCH 11/12] try and simplicy --- .github/workflows/docs.yaml | 1 + dev/release/README.md | 60 ++++++++++++++++++++++++++----------- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index f5a5b3d4b9ab0..abaffc3a5dc06 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -71,6 +71,7 @@ jobs: -a \ --delete \ --exclude '/.git/' \ + --exclude '/versions/' \ ../docs/build/html/ \ ./ cp ../.asf.yaml . diff --git a/dev/release/README.md b/dev/release/README.md index 8244fbbe79089..5811170becd28 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -315,39 +315,65 @@ git push apache 55.2.0 #### Publish the versioned documentation -After the final release tag is available, build the documentation for this release -For example, from the repository root, for `55.0.0`: +**1. Build the documentation from the release tagm using the a configuration +overlay (`docs/scripts/release/conf.py`) from the main checkout. For example, +for `55.0.0`, from the repository root: ```shell -git fetch origin tag 55.0.0 +export DATAFUSION_DOCS_OVERLAY="$PWD/docs/scripts/release" +git fetch apache tag 55.0.0 git worktree add --detach /tmp/datafusion-55.0.0 55.0.0 cd /tmp/datafusion-55.0.0/docs DATAFUSION_DOCS_SOURCE="$PWD/source" DATAFUSION_DOCS_VERSION=55.0.0 \ - SPHINXOPTS="-W -c /path/to/current/datafusion/docs/scripts/release" \ + SPHINXOPTS="-W -c $DATAFUSION_DOCS_OVERLAY" \ uv run --package datafusion-docs --with sphinx-sitemap ./build.sh +cd - ``` -Replace the version and paths for each release. Use an absolute path to the -overlay in your current checkout. The existing `build.sh` generates the tagged -dependency graph and builds the HTML with warnings treated as errors. The -`--with sphinx-sitemap` option supplies the sitemap extension for older tags. +Notes: +1. You need ot use an absolute path to the overlay in your current checkout. +2. The `--with sphinx-sitemap` option supplies the sitemap extension for older tags. -From the current checkout, prepare a separate PR targeting `asf-site`: +prepare a PR targeting `asf-site` that contains the +documentation for this release. (For example +[#25881](https://github.com/apache/datafusion/pull/25881)). ```shell -git fetch origin asf-site -git worktree add -b docs/publish-55.0.0 /tmp/datafusion-asf-site origin/asf-site -mkdir -p /tmp/datafusion-asf-site/versions/55.0.0 +mkdir -p docs/build/html/versions/55.0.0 +rsync -a /tmp/datafusion-55.0.0/docs/build/html/ docs/build/html/versions/55.0.0/ +python3 -m http.server --directory docs/build/html 8000 +``` + +**2. Add the built site to the `asf-site` branch.** Open a PR against +`asf-site` that adds the release under `versions//`: + +```shell +git fetch apache asf-site +git worktree add -b docs/publish-55.0.0 /tmp/datafusion-asf-site apache/asf-site rsync -a --exclude '/.buildinfo' \ /tmp/datafusion-55.0.0/docs/build/html/ \ /tmp/datafusion-asf-site/versions/55.0.0/ +cd /tmp/datafusion-asf-site +git add versions/55.0.0 +git commit -m "Publish documentation for 55.0.0" +git push origin docs/publish-55.0.0 ``` -Check that the release directory does not already exist before copying it. -Review the generated files and open the publication PR for review. -Once it is published, add its entry to `docs/source/_static/versions.json` on -`main` so the picker offers only working destinations. Remove the temporary -worktrees afterward. +The release is live at `https://datafusion.apache.org/versions/55.0.0/` once +the PR is merged. + +**3. Add the release to the version picker.** Open a PR against `main` that +adds the release to `docs/source/_static/versions.json`. Put the new release +first and move `"preferred": true` to it, so that it is the default shown. +Only add releases that are already published, since the picker links to them +directly. + +Finally, remove the temporary worktrees: + +```shell +git worktree remove /tmp/datafusion-55.0.0 +git worktree remove /tmp/datafusion-asf-site +``` ### 10. Publish on Crates.io From 0baa2481f6b57813f2424eaa4c7d3fd8c350d84a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 9 Oct 2026 17:28:38 -0400 Subject: [PATCH 12/12] Update dev/release/README.md Co-authored-by: Martin Grigorov --- dev/release/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/README.md b/dev/release/README.md index 5811170becd28..5c7654249f833 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -315,7 +315,7 @@ git push apache 55.2.0 #### Publish the versioned documentation -**1. Build the documentation from the release tagm using the a configuration +**1. Build the documentation from the release tag using the a configuration overlay (`docs/scripts/release/conf.py`) from the main checkout. For example, for `55.0.0`, from the repository root: