No Store Abs URIs

changes/36.canada.bugfix

@@ -0,0 +1 @@
+Added `--resource-id` option to the `xloader submit` command. Fixed various TypeErrors and missing metadata in the CLI methods.

changes/36.canada.changes

@@ -0,0 +1 @@
+Now only stores the relative URIs in the database for XLoader Resource fields `original_url` (only for Uploads). No longer stores the `ckan_url` value in the database.

ckanext/xloader/action.py

@@ -6,6 +6,11 @@
 import json
 import logging
 
+# (canada fork only): non-qualified res_url for lang domain support
+# TODO: upstream contrib!!
+from urllib.parse import urlparse
+from ckan.plugins.toolkit import request
+
 import ckan.lib.jobs as rq_jobs
 import ckan.lib.navl.dictization_functions
 from ckan.logic import side_effect_free
@@ -134,15 +139,26 @@ def xloader_submit(context, data_dict):
         task
     )
 
+    # (canada fork only): non-qualified res_url for lang domain support
+    #                     only for upload types.
+    # TODO: upstream contrib!!
+    original_url = resource_dict.get('url')
+    if resource_dict.get('url_type') == 'upload':
+        original_url_parts = urlparse(original_url)
+        if original_url_parts.netloc:
+            original_url = original_url_parts._replace(scheme='', netloc='').geturl()
+
     data = {
         'job_type': 'xloader_to_datastore',
         'metadata': {
             'ignore_hash': data_dict.get('ignore_hash', False),
-            'ckan_url': config['ckan.site_url'],
+            # (canada fork only): do not store site_url
+            'ckan_url': None,
             'resource_id': res_id,
             'set_url_type': data_dict.get('set_url_type', False),
             'task_created': task['last_updated'],
-            'original_url': resource_dict.get('url'),
+            # (canada fork only): non-qualified res_url for lang domain support
+            'original_url': original_url,
         }
     }
 

ckanext/xloader/cli.py

@@ -6,7 +6,7 @@
 # (canada fork only): add db init
 from ckanext.xloader.db import init
 # (canada fork only): ckan.plugins.toolkit
-from ckan.plugins.toolkit import config
+from ckan.plugins.toolkit import config, get_action, ObjectNotFound
 
 # Click commands for CKAN 2.9 and above
 
@@ -27,29 +27,52 @@ def status():
 
 
 @xloader.command()
-@click.argument(u'dataset-spec')
+# (canada fork only): support resource IDs
+# TODO: upstream contrib??
+@click.argument(u'dataset-spec', required=False)
 @click.option('-y', is_flag=True, default=False, help='Always answer yes to questions')
 @click.option('--dry-run', is_flag=True, default=False, help='Don\'t actually submit any resources')
 @click.option('--queue', help='Queue name for asynchronous processing, unused if executing immediately')
 @click.option('--sync', is_flag=True, default=False,
               help='Execute immediately instead of enqueueing for asynchronous processing')
-def submit(dataset_spec, y, dry_run, queue, sync):
+# (canada fork only): support resource IDs
+# TODO: upstream contrib??
+@click.option('-r', '--resource-id', type=click.STRING, help='A CKAN Resource ID.', required=False)
+def submit(dataset_spec, y, dry_run, queue, sync, resource_id):
     """
         xloader submit [options] <dataset-spec>
     """
     cmd = XloaderCmd(dry_run)
 
-    if dataset_spec == 'all':
+    # (canada fork only): support resource IDs
+    # TODO: upstream contrib??
+    if resource_id:
+        cmd._setup_xloader_logger()
+        try:
+            res_dict = get_action('resource_show')(
+                {'ignore_auth': True}, {'id': resource_id})
+            user = get_action('get_site_user')(
+                {'ignore_auth': True}, {})
+        except ObjectNotFound:
+            click.echo('Resource %s not found')
+            raise click.Abort()
+        cmd._submit_resource(res_dict, user=user, sync=sync, queue=queue)
+    elif dataset_spec == 'all':
         cmd._setup_xloader_logger()
         cmd._submit_all(sync=sync, queue=queue)
     elif dataset_spec == 'all-existing':
         _confirm_or_abort(y, dry_run)
         cmd._setup_xloader_logger()
         cmd._submit_all_existing(sync=sync, queue=queue)
-    else:
+    elif dataset_spec:
         pkg_name_or_id = dataset_spec
         cmd._setup_xloader_logger()
         cmd._submit_package(pkg_name_or_id, sync=sync, queue=queue)
+    else:
+        # (canada fork only): support resource IDs
+        # TODO: upstream contrib??
+        click.echo('No <dataset-spec> or --resource-id supplied.')
+        raise click.Abort()
 
     if cmd.error_occured:
         print('Finished but saw errors - see above for details')

ckanext/xloader/command.py

@@ -1,5 +1,10 @@
 # -*- coding: utf-8 -*-
 
+# (canada fork only): fix TypeErrors
+# TODO: upstream contrib??
+import datetime
+from urllib.parse import urlparse
+
 import sys
 import logging
 # (canada fork only): ckan.plugins.toolkit
@@ -115,13 +120,37 @@ def _submit_resource(self, resource, user, indent=0, sync=False, queue=None):
             'ignore_hash': True,
         }
         if sync:
-            data_dict['ckan_url'] = tk.config.get('ckan.site_url')
+            # (canada fork only): do not store site_url
+            data_dict['ckan_url'] = None
             input_dict = {
                 'metadata': data_dict,
                 'api_key': 'TODO'
             }
+            # (canada fork only): non-qualified res_url for lang domain support
+            #                     only for upload types.
+            # TODO: upstream contrib!!
+            original_url = resource.get('url')
+            if resource.get('url_type') == 'upload':
+                original_url_parts = urlparse(original_url)
+                if original_url_parts.netloc:
+                    original_url = original_url_parts._replace(scheme='', netloc='').geturl()
+            # (canada fork only): fix TypeErrors
+            # TODO: upstream contrib??
+            job_dict = {
+                'status': 'complete',
+                'metadata': {
+                    'ignore_hash': True,
+                    # (canada fork only): do not store site_url
+                    'ckan_url': None,
+                    'resource_id': resource['id'],
+                    'set_url_type': data_dict.get('set_url_type', False),
+                    'task_created': str(datetime.datetime.utcnow()),
+                    # (canada fork only): non-qualified res_url for lang domain support
+                    'original_url': original_url,
+                }
+            }
             logger = logging.getLogger('ckanext.xloader.cli')
-            xloader_data_into_datastore_(input_dict, None, logger)
+            xloader_data_into_datastore_(input_dict, job_dict, logger)
         else:
             if queue:
                 data_dict['queue'] = queue