From 5de6b4b104ca87cd5bb2c03877b798ff271cffc6 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 8 Jun 2026 10:24:08 -0400 Subject: [PATCH 01/10] feat(dev): xloader relative uris; - Do not store absolute URIs in the database. --- ckan/logic/action/get.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a6acb2cd030..5dace7b76fb 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1154,6 +1154,8 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS res_dict.get('url', '').startswith('/') ): res_dict['url'] = current_domain + res_dict['url'] + if res_dict.get('original_url'): + res_dict['original_url'] = current_domain + res_dict['original_url'] # XLoader field return package_dict From 5601f3f1ae148d5bb64be7ea7445a6ccbe4f7852 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 8 Jun 2026 11:09:00 -0400 Subject: [PATCH 02/10] feat(misc): changelog; - Added change log file. --- changes/229.canada.changes | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/229.canada.changes diff --git a/changes/229.canada.changes b/changes/229.canada.changes new file mode 100644 index 00000000000..62ab2473788 --- /dev/null +++ b/changes/229.canada.changes @@ -0,0 +1 @@ +Prepends the requesting host and scheme to the `original_url` XLoader Resource field, supporting relative URIs for XLoader fields in the database. From ad98f5b55aaef279a34dcedb4a8c52e69ae30995 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 8 Jun 2026 15:56:55 -0400 Subject: [PATCH 03/10] feat(logic): lang domain support; - Added support for the language_domains plugin. --- ...29.canada.changes => 229.a.canada.changes} | 0 changes/229.b.canada.changes | 1 + ckan/logic/action/get.py | 25 +++++++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) rename changes/{229.canada.changes => 229.a.canada.changes} (100%) create mode 100644 changes/229.b.canada.changes diff --git a/changes/229.canada.changes b/changes/229.a.canada.changes similarity index 100% rename from changes/229.canada.changes rename to changes/229.a.canada.changes diff --git a/changes/229.b.canada.changes b/changes/229.b.canada.changes new file mode 100644 index 00000000000..beb61eca4e6 --- /dev/null +++ b/changes/229.b.canada.changes @@ -0,0 +1 @@ +Added support for the `language_domains` plugin to output the correct relative Resource download URIs. diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 5dace7b76fb..4db4736d8bc 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -16,8 +16,9 @@ # (canada fork only): non-qualified res_url for lang domain support from flask import request +import re from urllib.parse import urlparse, urlunparse - +from ckan.lib import helpers import ckan import ckan.lib.dictization @@ -1137,17 +1138,31 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS if not context.get('for_index'): try: current_domain = request.host_url.rstrip('/') + current_locale = plugins.toolkit.h.lang() except RuntimeError: current_domain = config['ckan.site_url'].rstrip('/') + current_locale = config.get('ckan.locale_default', 'en') # upscale to configured scheme if different... # NOTE: this is very important for firewall and middleware stuff... - configured_domain = config['ckan.site_url'] - configured_parts = urlparse(configured_domain) + configured_scheme, configured_host = helpers.get_site_protocol_and_host(current_locale) requesting_parts = urlparse(current_domain) - if configured_parts.scheme != requesting_parts.scheme: + if configured_scheme != requesting_parts.scheme: requesting_parts = requesting_parts._replace( - scheme=configured_parts.scheme) + scheme=configured_scheme) current_domain = urlunparse(requesting_parts) + if plugins.plugin_loaded('language_domains'): + root_paths = config.get('ckanext.language_domains.root_paths', {}) + keep_lang_paths = config.get( + 'ckanext.language_domains.keep_lang_paths', False) + root_path = root_paths.get(configured_host, '').rstrip('/') + if root_path and keep_lang_paths: + root_path = re.sub('{{LANG}}', current_locale, root_path) + else: + root_path = re.sub('{{LANG}}', '', root_path).rstrip('/') + if root_path: + current_domain = f'{current_domain}{root_path}' + elif keep_lang_paths: + current_domain = f'{current_domain}/{current_locale}' for res_dict in package_dict['resources']: if( res_dict.get('url_type') == 'upload' and From 80ee953dc0a5744758039b8a9097aea9c497ba15 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Mon, 15 Jun 2026 13:50:57 -0400 Subject: [PATCH 04/10] stash --- ckan/logic/action/get.py | 7 +++++++ ckan/views/__init__.py | 9 +++++++++ ckanext/datastore/backend/postgres.py | 6 ++++++ 3 files changed, 22 insertions(+) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 4db4736d8bc..f41a4104196 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1171,6 +1171,13 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS res_dict['url'] = current_domain + res_dict['url'] if res_dict.get('original_url'): res_dict['original_url'] = current_domain + res_dict['original_url'] # XLoader field + else: + # (canada fork only): never index locale sub dirs + locales = config.get('ckan.locales_offered', 'en') + for res_dict in package_dict['resources']: + for locale in locales: + if res_dict.get('url', '').startswith(f'/{locale}/'): + res_dict['url'] = res_dict['url'].replace(f'/{locale}/', '/') return package_dict diff --git a/ckan/views/__init__.py b/ckan/views/__init__.py index dbdab3beaeb..f938768e831 100644 --- a/ckan/views/__init__.py +++ b/ckan/views/__init__.py @@ -220,7 +220,16 @@ def set_ckan_current_url(environ: Any) -> None: u'/'.join(quote(pce, u'') for pce in path_info.split(u'/')) qs = environ.get(u'QUERY_STRING') + if qs: environ[u'CKAN_CURRENT_URL'] = u'%s?%s' % (path_info, qs) else: environ[u'CKAN_CURRENT_URL'] = path_info + + log.info(' ') + log.info('DEBUGGING::STEP SURPRISE!!!') + log.info(' ') + log.info(environ['CKAN_CURRENT_URL']) + log.info(environ['REQUEST_URI']) + # log.info(plugins.toolkit.request.environ['CKAN_CURRENT_URL']) + log.info(' ') diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py index b278b9b56c0..48c2b1a11e5 100644 --- a/ckanext/datastore/backend/postgres.py +++ b/ckanext/datastore/backend/postgres.py @@ -1031,6 +1031,12 @@ def _insert_links(data_dict: dict[str, Any], limit: int, offset: int): except (KeyError, TypeError, RuntimeError): return # no links required for local actions + log.info(' ') + log.info('DEBUGGING::STEP 3') + log.info(' ') + log.info(urlstring) + log.info(' ') + # change the offset in the url parsed = list(urlparse(urlstring)) query = parsed[4] From 0e9ae8cec4046438464c82ea5d26bdfd0964c0ab Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Tue, 16 Jun 2026 09:46:43 -0400 Subject: [PATCH 05/10] stash --- ckan/logic/action/get.py | 8 ++++---- ckan/views/__init__.py | 9 --------- ckanext/datastore/backend/postgres.py | 6 ------ 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index f41a4104196..a4e6963408d 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1141,7 +1141,7 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS current_locale = plugins.toolkit.h.lang() except RuntimeError: current_domain = config['ckan.site_url'].rstrip('/') - current_locale = config.get('ckan.locale_default', 'en') + current_locale = config['ckan.locale_default'] # upscale to configured scheme if different... # NOTE: this is very important for firewall and middleware stuff... configured_scheme, configured_host = helpers.get_site_protocol_and_host(current_locale) @@ -1156,9 +1156,9 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS 'ckanext.language_domains.keep_lang_paths', False) root_path = root_paths.get(configured_host, '').rstrip('/') if root_path and keep_lang_paths: - root_path = re.sub('{{LANG}}', current_locale, root_path) + root_path = root_path.replace('{{LANG}}', current_locale) else: - root_path = re.sub('{{LANG}}', '', root_path).rstrip('/') + root_path = root_path.replace('{{LANG}}', '').rstrip('/') if root_path: current_domain = f'{current_domain}{root_path}' elif keep_lang_paths: @@ -1173,7 +1173,7 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS res_dict['original_url'] = current_domain + res_dict['original_url'] # XLoader field else: # (canada fork only): never index locale sub dirs - locales = config.get('ckan.locales_offered', 'en') + locales = config['ckan.locales_offered'] for res_dict in package_dict['resources']: for locale in locales: if res_dict.get('url', '').startswith(f'/{locale}/'): diff --git a/ckan/views/__init__.py b/ckan/views/__init__.py index f938768e831..288ca9d2b43 100644 --- a/ckan/views/__init__.py +++ b/ckan/views/__init__.py @@ -218,18 +218,9 @@ def set_ckan_current_url(environ: Any) -> None: # sort out weird encodings path_info = \ u'/'.join(quote(pce, u'') for pce in path_info.split(u'/')) - qs = environ.get(u'QUERY_STRING') if qs: environ[u'CKAN_CURRENT_URL'] = u'%s?%s' % (path_info, qs) else: environ[u'CKAN_CURRENT_URL'] = path_info - - log.info(' ') - log.info('DEBUGGING::STEP SURPRISE!!!') - log.info(' ') - log.info(environ['CKAN_CURRENT_URL']) - log.info(environ['REQUEST_URI']) - # log.info(plugins.toolkit.request.environ['CKAN_CURRENT_URL']) - log.info(' ') diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py index 48c2b1a11e5..b278b9b56c0 100644 --- a/ckanext/datastore/backend/postgres.py +++ b/ckanext/datastore/backend/postgres.py @@ -1031,12 +1031,6 @@ def _insert_links(data_dict: dict[str, Any], limit: int, offset: int): except (KeyError, TypeError, RuntimeError): return # no links required for local actions - log.info(' ') - log.info('DEBUGGING::STEP 3') - log.info(' ') - log.info(urlstring) - log.info(' ') - # change the offset in the url parsed = list(urlparse(urlstring)) query = parsed[4] From faf70cabafe8d72e6c2251f9345198d413578072 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Tue, 16 Jun 2026 15:27:40 -0400 Subject: [PATCH 06/10] stash --- ckan/logic/action/get.py | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a4e6963408d..b369d291b42 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1138,31 +1138,17 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS if not context.get('for_index'): try: current_domain = request.host_url.rstrip('/') - current_locale = plugins.toolkit.h.lang() except RuntimeError: current_domain = config['ckan.site_url'].rstrip('/') - current_locale = config['ckan.locale_default'] # upscale to configured scheme if different... # NOTE: this is very important for firewall and middleware stuff... - configured_scheme, configured_host = helpers.get_site_protocol_and_host(current_locale) + configured_domain = config['ckan.site_url'] + configured_parts = urlparse(configured_domain) requesting_parts = urlparse(current_domain) - if configured_scheme != requesting_parts.scheme: + if configured_parts.scheme != requesting_parts.scheme: requesting_parts = requesting_parts._replace( - scheme=configured_scheme) + scheme=configured_parts.scheme) current_domain = urlunparse(requesting_parts) - if plugins.plugin_loaded('language_domains'): - root_paths = config.get('ckanext.language_domains.root_paths', {}) - keep_lang_paths = config.get( - 'ckanext.language_domains.keep_lang_paths', False) - root_path = root_paths.get(configured_host, '').rstrip('/') - if root_path and keep_lang_paths: - root_path = root_path.replace('{{LANG}}', current_locale) - else: - root_path = root_path.replace('{{LANG}}', '').rstrip('/') - if root_path: - current_domain = f'{current_domain}{root_path}' - elif keep_lang_paths: - current_domain = f'{current_domain}/{current_locale}' for res_dict in package_dict['resources']: if( res_dict.get('url_type') == 'upload' and @@ -1171,13 +1157,6 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS res_dict['url'] = current_domain + res_dict['url'] if res_dict.get('original_url'): res_dict['original_url'] = current_domain + res_dict['original_url'] # XLoader field - else: - # (canada fork only): never index locale sub dirs - locales = config['ckan.locales_offered'] - for res_dict in package_dict['resources']: - for locale in locales: - if res_dict.get('url', '').startswith(f'/{locale}/'): - res_dict['url'] = res_dict['url'].replace(f'/{locale}/', '/') return package_dict From 1f476f361d862f8d6ce8a307981ee04ebe7c2983 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Tue, 16 Jun 2026 15:31:05 -0400 Subject: [PATCH 07/10] stash --- changes/229.b.canada.changes | 1 - changes/{229.a.canada.changes => 229.canada.changes} | 0 ckan/logic/action/get.py | 4 +--- 3 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 changes/229.b.canada.changes rename changes/{229.a.canada.changes => 229.canada.changes} (100%) diff --git a/changes/229.b.canada.changes b/changes/229.b.canada.changes deleted file mode 100644 index beb61eca4e6..00000000000 --- a/changes/229.b.canada.changes +++ /dev/null @@ -1 +0,0 @@ -Added support for the `language_domains` plugin to output the correct relative Resource download URIs. diff --git a/changes/229.a.canada.changes b/changes/229.canada.changes similarity index 100% rename from changes/229.a.canada.changes rename to changes/229.canada.changes diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index b369d291b42..49ed9fec0b0 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -16,9 +16,7 @@ # (canada fork only): non-qualified res_url for lang domain support from flask import request -import re from urllib.parse import urlparse, urlunparse -from ckan.lib import helpers import ckan import ckan.lib.dictization @@ -1155,7 +1153,7 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS res_dict.get('url', '').startswith('/') ): res_dict['url'] = current_domain + res_dict['url'] - if res_dict.get('original_url'): + if res_dict.get('original_url') and res_dict.get('original_url', '').startswith('/'): res_dict['original_url'] = current_domain + res_dict['original_url'] # XLoader field return package_dict From ac37126afbfd28543c387c1002315c6e6a9f372d Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Tue, 16 Jun 2026 15:31:48 -0400 Subject: [PATCH 08/10] stash --- ckan/views/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/views/__init__.py b/ckan/views/__init__.py index 288ca9d2b43..dbdab3beaeb 100644 --- a/ckan/views/__init__.py +++ b/ckan/views/__init__.py @@ -218,8 +218,8 @@ def set_ckan_current_url(environ: Any) -> None: # sort out weird encodings path_info = \ u'/'.join(quote(pce, u'') for pce in path_info.split(u'/')) - qs = environ.get(u'QUERY_STRING') + qs = environ.get(u'QUERY_STRING') if qs: environ[u'CKAN_CURRENT_URL'] = u'%s?%s' % (path_info, qs) else: From f4234e7860281c6e08004d6b66499d61bb2c95a7 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Tue, 16 Jun 2026 15:49:13 -0400 Subject: [PATCH 09/10] stash --- ckan/logic/action/get.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 49ed9fec0b0..0f393348349 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1148,12 +1148,10 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS scheme=configured_parts.scheme) current_domain = urlunparse(requesting_parts) for res_dict in package_dict['resources']: - if( - res_dict.get('url_type') == 'upload' and - res_dict.get('url', '').startswith('/') - ): - res_dict['url'] = current_domain + res_dict['url'] - if res_dict.get('original_url') and res_dict.get('original_url', '').startswith('/'): + if res_dict.get('url_type') == 'upload': + if res_dict.get('url', '').startswith('/'): + res_dict['url'] = current_domain + res_dict['url'] + if res_dict.get('original_url', '').startswith('/'): res_dict['original_url'] = current_domain + res_dict['original_url'] # XLoader field return package_dict From b31d8ba2bc61fd52172f80b34aa2c7f77cbd2a08 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Tue, 16 Jun 2026 16:00:17 -0400 Subject: [PATCH 10/10] stash --- ckan/logic/action/get.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 0f393348349..6817cde2e46 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1148,11 +1148,12 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS scheme=configured_parts.scheme) current_domain = urlunparse(requesting_parts) for res_dict in package_dict['resources']: - if res_dict.get('url_type') == 'upload': - if res_dict.get('url', '').startswith('/'): - res_dict['url'] = current_domain + res_dict['url'] - if res_dict.get('original_url', '').startswith('/'): - res_dict['original_url'] = current_domain + res_dict['original_url'] # XLoader field + if res_dict.get('url_type') != 'upload': + continue + if res_dict.get('url', '').startswith('/'): + res_dict['url'] = current_domain + res_dict['url'] + if res_dict.get('original_url', '').startswith('/'): + res_dict['original_url'] = current_domain + res_dict['original_url'] # XLoader field return package_dict