From 26fc404fbd2ca5349558ca84316f7f3f3e6ef480 Mon Sep 17 00:00:00 2001 From: Marek Honzal Date: Mon, 21 Sep 2026 13:56:27 +0200 Subject: [PATCH 1/3] fix: collapse doubled section prefix in docs .md cross-links --- nginx.conf | 44 +++++++++++++++++++++++++++++++++++++++ scripts/joinLlmsFiles.mjs | 11 +++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/nginx.conf b/nginx.conf index 1996d680fa..779c460f66 100644 --- a/nginx.conf +++ b/nginx.conf @@ -17,6 +17,15 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + # Rewrite the doubled section prefix out of the proxied Markdown itself (see the + # WORKAROUND note below). Applied per location, on the routes that serve Markdown. + # Each of those locations also resets Accept-Encoding, because sub_filter cannot + # rewrite a compressed body and GitHub Pages compresses even for the HTTP/1.0 + # requests proxy_pass makes. That reset drops the inherited X-Real-IP and + # X-Forwarded-* headers in those locations, which the Pages origin ignores anyway. + sub_filter_types text/markdown text/plain; + sub_filter_once off; + # comment out the resolver and use localhost:3000 for local development set $backend "https://apify.github.io/apify-docs"; resolver 1.1.1.1 8.8.8.8 valid=30s ipv6=off; @@ -63,6 +72,16 @@ server { rewrite ^(.+)/$ $1$is_args$args? redirect; } + # WORKAROUND: @signalwire/docusaurus-plugin-llms-txt 1.2.2 doubles the section + # prefix in .md cross-links on sites served under a sub-path, so links such as + # /sdk/python/sdk/python/docs/... 404. Fixed upstream in 2.0.0-alpha.6; the five + # sub-sites still pin ^1.2.2. Collapse the doubled prefix for links already in + # the wild (search indexes, agent caches). + # REMOVE once all five sections build on the plugin's 2.x line. + location ~ "^/(sdk/js|sdk/python|api/client/js|api/client/python|cli)/\1(/.*)?$" { + return 301 /$1$2$is_args$args; + } + location / { set $rewrite_condition "$serve_markdown$has_no_extension"; set $proxy_path $request_uri; @@ -101,6 +120,9 @@ server { } location ~ ^/sdk/js/(llms|llms-full)\.txt$ { + proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body + sub_filter_types *; # Content-Type is hidden and re-added below + sub_filter "docs.apify.com/sdk/js/sdk/js/" "docs.apify.com/sdk/js/"; rewrite ^/sdk/js(/.*)$ $1 break; proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; @@ -108,6 +130,8 @@ server { } location ~ ^/sdk/js/(.*)$ { + proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body + sub_filter "docs.apify.com/sdk/js/sdk/js/" "docs.apify.com/sdk/js/"; set $path_suffix $1; set $proxy_path "/$path_suffix"; @@ -148,6 +172,9 @@ server { } location ~ ^/sdk/python/(llms|llms-full)\.txt$ { + proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body + sub_filter_types *; # Content-Type is hidden and re-added below + sub_filter "docs.apify.com/sdk/python/sdk/python/" "docs.apify.com/sdk/python/"; rewrite ^/sdk/python(/.*)$ $1 break; proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; @@ -155,6 +182,8 @@ server { } location ~ ^/sdk/python/(.*)$ { + proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body + sub_filter "docs.apify.com/sdk/python/sdk/python/" "docs.apify.com/sdk/python/"; set $path_suffix $1; set $proxy_path "/$path_suffix"; @@ -195,6 +224,9 @@ server { } location ~ ^/api/client/js/(llms|llms-full)\.txt$ { + proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body + sub_filter_types *; # Content-Type is hidden and re-added below + sub_filter "docs.apify.com/api/client/js/api/client/js/" "docs.apify.com/api/client/js/"; rewrite ^/api/client/js(/.*)$ $1 break; proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; @@ -202,6 +234,8 @@ server { } location ~ ^/api/client/js/(.*)$ { + proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body + sub_filter "docs.apify.com/api/client/js/api/client/js/" "docs.apify.com/api/client/js/"; set $path_suffix $1; set $proxy_path "/$path_suffix"; @@ -242,6 +276,9 @@ server { } location ~ ^/api/client/python/(llms|llms-full)\.txt$ { + proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body + sub_filter_types *; # Content-Type is hidden and re-added below + sub_filter "docs.apify.com/api/client/python/api/client/python/" "docs.apify.com/api/client/python/"; rewrite ^/api/client/python(/.*)$ $1 break; proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; @@ -249,6 +286,8 @@ server { } location ~ ^/api/client/python/(.*)$ { + proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body + sub_filter "docs.apify.com/api/client/python/api/client/python/" "docs.apify.com/api/client/python/"; set $path_suffix $1; set $proxy_path "/$path_suffix"; @@ -289,6 +328,9 @@ server { } location ~ ^/cli/(llms|llms-full)\.txt$ { + proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body + sub_filter_types *; # Content-Type is hidden and re-added below + sub_filter "docs.apify.com/cli/cli/" "docs.apify.com/cli/"; rewrite ^/cli(/.*)$ $1 break; proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; @@ -296,6 +338,8 @@ server { } location ~ ^/cli/(.*)$ { + proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body + sub_filter "docs.apify.com/cli/cli/" "docs.apify.com/cli/"; set $path_suffix $1; set $proxy_path "/$path_suffix"; diff --git a/scripts/joinLlmsFiles.mjs b/scripts/joinLlmsFiles.mjs index 34130cb539..c53013d81c 100644 --- a/scripts/joinLlmsFiles.mjs +++ b/scripts/joinLlmsFiles.mjs @@ -253,9 +253,18 @@ async function joinFiles() { console.log('Wrote llms-full.txt to build/'); } +// WORKAROUND: @signalwire/docusaurus-plugin-llms-txt 1.2.2 doubles the section prefix in +// cross-links on sites served under a sub-path, so the SDK/client/CLI sections we fetch above +// arrive with links such as /sdk/python/sdk/python/docs/... Fixed upstream in 2.0.0-alpha.6; +// those sites still pin ^1.2.2. Remove once they build on the plugin's 2.x line. +const DOUBLED_SECTION_PREFIX = + /(https:\/\/docs\.apify\.com\/(sdk\/js|sdk\/python|api\/client\/js|api\/client\/python|cli))\/\2\//g; + async function sanitizeFile(filePath) { const content = await fs.readFile(filePath, 'utf8'); - const sanitizedContent = content.replace(/<[^>]*>/g, ''); // Remove HTML tags + const sanitizedContent = content + .replace(/<[^>]*>/g, '') // Remove HTML tags + .replace(DOUBLED_SECTION_PREFIX, '$1/'); await fs.writeFile(filePath, sanitizedContent, 'utf8'); console.log(`Sanitized ${filePath}`); } From 08be038f08643894b91803e2b0b39c06116266be Mon Sep 17 00:00:00 2001 From: Marek Honzal Date: Mon, 21 Sep 2026 14:29:56 +0200 Subject: [PATCH 2/3] test: cover doubled section prefix redirect and Markdown rewrite --- .github/workflows/test.yaml | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8f16da9eb8..d4c90d923e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -138,6 +138,29 @@ jobs: fi } + # Asserts that the body at $url contains $required and does not + # contain $forbidden. Sends Accept-Encoding: gzip on purpose: the + # sub_filter rewrite only works when nginx strips that header + # before proxying, so this also guards the Accept-Encoding reset. + function assert_body() { + url=$1 + required=$2 + forbidden=$3 + shift 3 + extra_args=("$@") + : > "$body_sink" + curl -s --compressed -o "$body_sink" "${retry_opts[@]}" -H "Accept-Encoding: gzip" "${extra_args[@]}" "$url" || true + echo "โ†’ $url โ†’ $(wc -c < "$body_sink") bytes" + if ! grep -qF "$required" "$body_sink"; then + echo "โŒ Expected '$required' in body for $url" + exit 1 + fi + if grep -qF "$forbidden" "$body_sink"; then + echo "โŒ Found '$forbidden' in body for $url" + exit 1 + fi + } + echo "๐Ÿงช Checking open redirect protection..." # Backslash URLs must not produce redirects (the redirect Location # would contain \, which browsers normalize to /, creating @@ -230,6 +253,27 @@ jobs: assert_header "http://localhost:8080/cli/llms.txt" "Content-Type" "text/markdown" assert_header "http://localhost:8080/cli/llms-full.txt" "Content-Type" "text/markdown" + echo "๐Ÿงช Checking doubled section prefix workaround..." + # Regression tests for the llms-txt plugin workaround in nginx.conf. + # Doubled prefixes redirect to the canonical URL and keep the query + # string, and the proxied Markdown has the doubled links rewritten. + # Remove together with the workaround. + assert_status "http://localhost:8080/sdk/js/sdk/js/docs/introduction/quick-start" "301" + assert_header "http://localhost:8080/sdk/js/sdk/js/docs/introduction/quick-start" "Location" "^Location: http://[^/]*/sdk/js/docs/introduction/quick-start$" + assert_header "http://localhost:8080/sdk/js/sdk/js/docs/introduction/quick-start.md?x=1" "Location" "^Location: http://[^/]*/sdk/js/docs/introduction/quick-start.md?x=1$" + assert_header "http://localhost:8080/sdk/python/sdk/python/docs/changelog" "Location" "^Location: http://[^/]*/sdk/python/docs/changelog$" + assert_header "http://localhost:8080/api/client/js/api/client/js/docs/changelog" "Location" "^Location: http://[^/]*/api/client/js/docs/changelog$" + assert_header "http://localhost:8080/api/client/python/api/client/python/docs/changelog" "Location" "^Location: http://[^/]*/api/client/python/docs/changelog$" + assert_header "http://localhost:8080/cli/cli/docs/changelog" "Location" "^Location: http://[^/]*/cli/docs/changelog$" + assert_body "http://localhost:8080/sdk/js/llms-full.txt" "docs.apify.com/sdk/js/" "docs.apify.com/sdk/js/sdk/js/" + assert_body "http://localhost:8080/sdk/js/docs/introduction/quick-start.md" "docs.apify.com/sdk/js/" "docs.apify.com/sdk/js/sdk/js/" + assert_body "http://localhost:8080/sdk/python/llms-full.txt" "docs.apify.com/sdk/python/" "docs.apify.com/sdk/python/sdk/python/" + assert_body "http://localhost:8080/sdk/python/docs/changelog" "docs.apify.com/sdk/python/" "docs.apify.com/sdk/python/sdk/python/" -H "Accept: text/markdown" + assert_body "http://localhost:8080/api/client/js/llms-full.txt" "docs.apify.com/api/client/js/" "docs.apify.com/api/client/js/api/client/js/" + assert_body "http://localhost:8080/api/client/python/llms-full.txt" "docs.apify.com/api/client/python/" "docs.apify.com/api/client/python/api/client/python/" + assert_body "http://localhost:8080/cli/llms-full.txt" "docs.apify.com/cli/" "docs.apify.com/cli/cli/" + assert_body "http://localhost:8080/cli/docs/integrating-scrapy.md" "docs.apify.com/cli/" "docs.apify.com/cli/cli/" + echo "โœ… All Nginx header checks passed." - name: Verify per-page .md navigation headers From ebb801682fc8cab14071d15eb1c5e628f62ea5fc Mon Sep 17 00:00:00 2001 From: Marek Honzal Date: Mon, 21 Sep 2026 18:01:48 +0200 Subject: [PATCH 3/3] fix: drop Markdown body rewrite, keep the doubled prefix redirect --- .github/workflows/test.yaml | 38 +++--------------------------- nginx.conf | 47 ++++++------------------------------- scripts/checkLlmsSize.mjs | 14 +++++++++++ scripts/joinLlmsFiles.mjs | 2 +- 4 files changed, 25 insertions(+), 76 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d4c90d923e..f28ba79d88 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -138,29 +138,6 @@ jobs: fi } - # Asserts that the body at $url contains $required and does not - # contain $forbidden. Sends Accept-Encoding: gzip on purpose: the - # sub_filter rewrite only works when nginx strips that header - # before proxying, so this also guards the Accept-Encoding reset. - function assert_body() { - url=$1 - required=$2 - forbidden=$3 - shift 3 - extra_args=("$@") - : > "$body_sink" - curl -s --compressed -o "$body_sink" "${retry_opts[@]}" -H "Accept-Encoding: gzip" "${extra_args[@]}" "$url" || true - echo "โ†’ $url โ†’ $(wc -c < "$body_sink") bytes" - if ! grep -qF "$required" "$body_sink"; then - echo "โŒ Expected '$required' in body for $url" - exit 1 - fi - if grep -qF "$forbidden" "$body_sink"; then - echo "โŒ Found '$forbidden' in body for $url" - exit 1 - fi - } - echo "๐Ÿงช Checking open redirect protection..." # Backslash URLs must not produce redirects (the redirect Location # would contain \, which browsers normalize to /, creating @@ -253,26 +230,17 @@ jobs: assert_header "http://localhost:8080/cli/llms.txt" "Content-Type" "text/markdown" assert_header "http://localhost:8080/cli/llms-full.txt" "Content-Type" "text/markdown" - echo "๐Ÿงช Checking doubled section prefix workaround..." + echo "๐Ÿงช Checking doubled section prefix redirect..." # Regression tests for the llms-txt plugin workaround in nginx.conf. # Doubled prefixes redirect to the canonical URL and keep the query - # string, and the proxied Markdown has the doubled links rewritten. - # Remove together with the workaround. + # string. Remove together with the workaround. assert_status "http://localhost:8080/sdk/js/sdk/js/docs/introduction/quick-start" "301" assert_header "http://localhost:8080/sdk/js/sdk/js/docs/introduction/quick-start" "Location" "^Location: http://[^/]*/sdk/js/docs/introduction/quick-start$" - assert_header "http://localhost:8080/sdk/js/sdk/js/docs/introduction/quick-start.md?x=1" "Location" "^Location: http://[^/]*/sdk/js/docs/introduction/quick-start.md?x=1$" + assert_header "http://localhost:8080/sdk/js/sdk/js/docs/introduction/quick-start.md?x=1" "Location" "^Location: http://[^/]*/sdk/js/docs/introduction/quick-start[.]md[?]x=1$" assert_header "http://localhost:8080/sdk/python/sdk/python/docs/changelog" "Location" "^Location: http://[^/]*/sdk/python/docs/changelog$" assert_header "http://localhost:8080/api/client/js/api/client/js/docs/changelog" "Location" "^Location: http://[^/]*/api/client/js/docs/changelog$" assert_header "http://localhost:8080/api/client/python/api/client/python/docs/changelog" "Location" "^Location: http://[^/]*/api/client/python/docs/changelog$" assert_header "http://localhost:8080/cli/cli/docs/changelog" "Location" "^Location: http://[^/]*/cli/docs/changelog$" - assert_body "http://localhost:8080/sdk/js/llms-full.txt" "docs.apify.com/sdk/js/" "docs.apify.com/sdk/js/sdk/js/" - assert_body "http://localhost:8080/sdk/js/docs/introduction/quick-start.md" "docs.apify.com/sdk/js/" "docs.apify.com/sdk/js/sdk/js/" - assert_body "http://localhost:8080/sdk/python/llms-full.txt" "docs.apify.com/sdk/python/" "docs.apify.com/sdk/python/sdk/python/" - assert_body "http://localhost:8080/sdk/python/docs/changelog" "docs.apify.com/sdk/python/" "docs.apify.com/sdk/python/sdk/python/" -H "Accept: text/markdown" - assert_body "http://localhost:8080/api/client/js/llms-full.txt" "docs.apify.com/api/client/js/" "docs.apify.com/api/client/js/api/client/js/" - assert_body "http://localhost:8080/api/client/python/llms-full.txt" "docs.apify.com/api/client/python/" "docs.apify.com/api/client/python/api/client/python/" - assert_body "http://localhost:8080/cli/llms-full.txt" "docs.apify.com/cli/" "docs.apify.com/cli/cli/" - assert_body "http://localhost:8080/cli/docs/integrating-scrapy.md" "docs.apify.com/cli/" "docs.apify.com/cli/cli/" echo "โœ… All Nginx header checks passed." diff --git a/nginx.conf b/nginx.conf index 779c460f66..857dea168f 100644 --- a/nginx.conf +++ b/nginx.conf @@ -17,15 +17,6 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - # Rewrite the doubled section prefix out of the proxied Markdown itself (see the - # WORKAROUND note below). Applied per location, on the routes that serve Markdown. - # Each of those locations also resets Accept-Encoding, because sub_filter cannot - # rewrite a compressed body and GitHub Pages compresses even for the HTTP/1.0 - # requests proxy_pass makes. That reset drops the inherited X-Real-IP and - # X-Forwarded-* headers in those locations, which the Pages origin ignores anyway. - sub_filter_types text/markdown text/plain; - sub_filter_once off; - # comment out the resolver and use localhost:3000 for local development set $backend "https://apify.github.io/apify-docs"; resolver 1.1.1.1 8.8.8.8 valid=30s ipv6=off; @@ -72,12 +63,13 @@ server { rewrite ^(.+)/$ $1$is_args$args? redirect; } - # WORKAROUND: @signalwire/docusaurus-plugin-llms-txt 1.2.2 doubles the section - # prefix in .md cross-links on sites served under a sub-path, so links such as - # /sdk/python/sdk/python/docs/... 404. Fixed upstream in 2.0.0-alpha.6; the five - # sub-sites still pin ^1.2.2. Collapse the doubled prefix for links already in - # the wild (search indexes, agent caches). - # REMOVE once all five sections build on the plugin's 2.x line. + # @signalwire/docusaurus-plugin-llms-txt 1.2.2 repeats the section prefix in the + # .md files and llms*.txt it generates for sites served under a sub-path, so links + # such as /sdk/python/sdk/python/docs/... 404. The five sub-sites still pin ^1.2.2. + # This redirect makes those links resolve in one hop. Keep it after the sub-sites + # stop generating them: the doubled URLs stay in search indexes and agent caches. + # It only collapses a repeated same-section prefix. The plugin also prefixes + # cross-section links (/sdk/js/cli/docs/...), which this cannot detect. location ~ "^/(sdk/js|sdk/python|api/client/js|api/client/python|cli)/\1(/.*)?$" { return 301 /$1$2$is_args$args; } @@ -120,9 +112,6 @@ server { } location ~ ^/sdk/js/(llms|llms-full)\.txt$ { - proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body - sub_filter_types *; # Content-Type is hidden and re-added below - sub_filter "docs.apify.com/sdk/js/sdk/js/" "docs.apify.com/sdk/js/"; rewrite ^/sdk/js(/.*)$ $1 break; proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; @@ -130,8 +119,6 @@ server { } location ~ ^/sdk/js/(.*)$ { - proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body - sub_filter "docs.apify.com/sdk/js/sdk/js/" "docs.apify.com/sdk/js/"; set $path_suffix $1; set $proxy_path "/$path_suffix"; @@ -172,9 +159,6 @@ server { } location ~ ^/sdk/python/(llms|llms-full)\.txt$ { - proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body - sub_filter_types *; # Content-Type is hidden and re-added below - sub_filter "docs.apify.com/sdk/python/sdk/python/" "docs.apify.com/sdk/python/"; rewrite ^/sdk/python(/.*)$ $1 break; proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; @@ -182,8 +166,6 @@ server { } location ~ ^/sdk/python/(.*)$ { - proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body - sub_filter "docs.apify.com/sdk/python/sdk/python/" "docs.apify.com/sdk/python/"; set $path_suffix $1; set $proxy_path "/$path_suffix"; @@ -224,9 +206,6 @@ server { } location ~ ^/api/client/js/(llms|llms-full)\.txt$ { - proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body - sub_filter_types *; # Content-Type is hidden and re-added below - sub_filter "docs.apify.com/api/client/js/api/client/js/" "docs.apify.com/api/client/js/"; rewrite ^/api/client/js(/.*)$ $1 break; proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; @@ -234,8 +213,6 @@ server { } location ~ ^/api/client/js/(.*)$ { - proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body - sub_filter "docs.apify.com/api/client/js/api/client/js/" "docs.apify.com/api/client/js/"; set $path_suffix $1; set $proxy_path "/$path_suffix"; @@ -276,9 +253,6 @@ server { } location ~ ^/api/client/python/(llms|llms-full)\.txt$ { - proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body - sub_filter_types *; # Content-Type is hidden and re-added below - sub_filter "docs.apify.com/api/client/python/api/client/python/" "docs.apify.com/api/client/python/"; rewrite ^/api/client/python(/.*)$ $1 break; proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; @@ -286,8 +260,6 @@ server { } location ~ ^/api/client/python/(.*)$ { - proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body - sub_filter "docs.apify.com/api/client/python/api/client/python/" "docs.apify.com/api/client/python/"; set $path_suffix $1; set $proxy_path "/$path_suffix"; @@ -328,9 +300,6 @@ server { } location ~ ^/cli/(llms|llms-full)\.txt$ { - proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body - sub_filter_types *; # Content-Type is hidden and re-added below - sub_filter "docs.apify.com/cli/cli/" "docs.apify.com/cli/"; rewrite ^/cli(/.*)$ $1 break; proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; @@ -338,8 +307,6 @@ server { } location ~ ^/cli/(.*)$ { - proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a compressed body - sub_filter "docs.apify.com/cli/cli/" "docs.apify.com/cli/"; set $path_suffix $1; set $proxy_path "/$path_suffix"; diff --git a/scripts/checkLlmsSize.mjs b/scripts/checkLlmsSize.mjs index fad4bc3f03..1ff1c8ef97 100644 --- a/scripts/checkLlmsSize.mjs +++ b/scripts/checkLlmsSize.mjs @@ -27,6 +27,20 @@ if (llmsChars === null || llmsFullChars === null) { process.exit(1); } +// Guards the substitution in joinLlmsFiles.mjs. The sub-sites generate links with their +// section prefix repeated; if the substitution stops matching, the joined files silently +// fill up with URLs that only resolve through the redirect in nginx.conf. +const DOUBLED_SECTION_PREFIX = + /https:\/\/docs\.apify\.com\/(sdk\/js|sdk\/python|api\/client\/js|api\/client\/python|cli)\/\1\//g; + +for (const filePath of [llmsPath, llmsFullPath]) { + const matches = (await fs.readFile(filePath, 'utf8')).match(DOUBLED_SECTION_PREFIX); + if (matches) { + console.error(`\nERROR: ${filePath} has ${matches.length} links with a doubled section prefix`); + process.exitCode = 1; + } +} + console.log(`llms.txt: ${llmsChars.toLocaleString()} characters`); console.log(`llms-full.txt: ${llmsFullChars.toLocaleString()} characters`); diff --git a/scripts/joinLlmsFiles.mjs b/scripts/joinLlmsFiles.mjs index c53013d81c..f43699c2a6 100644 --- a/scripts/joinLlmsFiles.mjs +++ b/scripts/joinLlmsFiles.mjs @@ -256,7 +256,7 @@ async function joinFiles() { // WORKAROUND: @signalwire/docusaurus-plugin-llms-txt 1.2.2 doubles the section prefix in // cross-links on sites served under a sub-path, so the SDK/client/CLI sections we fetch above // arrive with links such as /sdk/python/sdk/python/docs/... Fixed upstream in 2.0.0-alpha.6; -// those sites still pin ^1.2.2. Remove once they build on the plugin's 2.x line. +// those sites still pin ^1.2.2. Remove once they stop generating the doubled prefix. const DOUBLED_SECTION_PREFIX = /(https:\/\/docs\.apify\.com\/(sdk\/js|sdk\/python|api\/client\/js|api\/client\/python|cli))\/\2\//g;