|
| 1 | +"""Functions to download files directly from a CaltechData record. |
| 2 | +
|
| 3 | +A CaltechData record ID is the ten characters, separated into two groups |
| 4 | +of five by a dash, found at the end of a CaltechData URL or default DOI. |
| 5 | +For instance, in the URL ``https://data.caltech.edu/records/4rgh7-zss31`` |
| 6 | +or DOI ``10.22002/4rgh7-zss31``, the record ID is ``4rgh7-zss31``. |
| 7 | +""" |
| 8 | +import os |
| 9 | +import requests |
| 10 | +from tqdm.auto import tqdm |
| 11 | + |
| 12 | +from typing import Any, Dict, Optional, Sequence |
| 13 | + |
| 14 | + |
| 15 | +def get_files_from_record(record_id: str) -> Dict[str, Any]: |
| 16 | + """Get a dictionary of files associated with a record. |
| 17 | +
|
| 18 | + Parameters |
| 19 | + ---------- |
| 20 | + record_id |
| 21 | + The record ID for which to obtain the list of files. See |
| 22 | + the module documentation for how to find the record ID. |
| 23 | +
|
| 24 | + Returns |
| 25 | + ------- |
| 26 | + dict |
| 27 | + A dictionary with the file names as keys and the metadata |
| 28 | + associated with those files as values. |
| 29 | + """ |
| 30 | + with requests.get(f'https://data.caltech.edu/api/records/{record_id}/files') as r: |
| 31 | + r.raise_for_status() |
| 32 | + files = dict() |
| 33 | + for entry in r.json().get('entries', []): |
| 34 | + key = entry.pop('key') |
| 35 | + files[key] = entry |
| 36 | + |
| 37 | + return files |
| 38 | + |
| 39 | + |
| 40 | +def download_files_from_record( |
| 41 | + record_id: str, output_path: os.PathLike, |
| 42 | + filenames: Optional[Sequence[str]] = None, |
| 43 | + max_redirects: int = 5 |
| 44 | +): |
| 45 | + """Download one or more files from a record. |
| 46 | +
|
| 47 | + Parameters |
| 48 | + ---------- |
| 49 | + record_id |
| 50 | + The record ID from which to download the file. See |
| 51 | + the module documentation for how to find the record ID. |
| 52 | +
|
| 53 | + output_path |
| 54 | + Where to download the files. Must be an existing directory. |
| 55 | + If you want control over exactly how each file is named, |
| 56 | + use :func:`get_files_from_record` to get the URLs for each |
| 57 | + file, then manually call :func:`download_content` for each |
| 58 | + file you wish to download. |
| 59 | +
|
| 60 | + filenames |
| 61 | + The name of the files in the record. Must match exactly. |
| 62 | + If you are not sure of the filenames in the record, |
| 63 | + they are the keys of the dictionary returned by |
| 64 | + :func:`get_files_from_record`. If omitted, all files |
| 65 | + are downloaded. |
| 66 | +
|
| 67 | + max_redirects |
| 68 | + If the record's link to the file content returns a |
| 69 | + redirection to another location, this function will follow |
| 70 | + that redirection. This parameter sets the maximum number of |
| 71 | + hops that the function can take. Usually, only a single |
| 72 | + redirection should be necessary (from the record to the file |
| 73 | + provider), but the default allows for a few extra hops. |
| 74 | + """ |
| 75 | + if not os.path.isdir(output_path): |
| 76 | + raise IOError(f'{output_path} is not an extant directory') |
| 77 | + |
| 78 | + files = get_files_from_record(record_id) |
| 79 | + if filenames is None: |
| 80 | + filenames = sorted(files.keys()) |
| 81 | + |
| 82 | + for filename in filenames: |
| 83 | + output_file = os.path.join(output_path, filename) |
| 84 | + if filename not in files: |
| 85 | + raise KeyError( |
| 86 | + f'File "{filename}" not found in record {record_id}') |
| 87 | + |
| 88 | + entry = files[filename] |
| 89 | + if 'links' not in entry or 'content' not in entry['links']: |
| 90 | + raise NotImplementedError( |
| 91 | + f'Metadata for file "{filename}" in record {record_id} is ' |
| 92 | + 'missing the content link' |
| 93 | + ) |
| 94 | + |
| 95 | + content_url = entry['links']['content'] |
| 96 | + download_content(content_url, output_file, max_redirects=max_redirects) |
| 97 | + |
| 98 | + |
| 99 | +def download_content(content_url: str, fname: os.PathLike, max_redirects=5): |
| 100 | + """Download the contents of a file. |
| 101 | +
|
| 102 | + Parameters |
| 103 | + ---------- |
| 104 | + content_url |
| 105 | + The URL pointing to the contents of the file. In the dictionary |
| 106 | + returned by :func:`get_files_from_record` (call it ``D``), this |
| 107 | + is usually the value at ``D[FILENAME]["links"]["content"]``. |
| 108 | +
|
| 109 | + fname |
| 110 | + The path to which to save the file contents. Will be overwritten! |
| 111 | +
|
| 112 | +
|
| 113 | + max_redirects |
| 114 | + If the ``content_url`` returns a redirection to another location, |
| 115 | + this function will follow that redirection. This parameter sets |
| 116 | + the maximum number of hops that the function can take. Usually, |
| 117 | + only a single redirection should be necessary (from the record to |
| 118 | + the file provider), but the default allows for a few extra hops. |
| 119 | + """ |
| 120 | + url = content_url |
| 121 | + # If max_redirects == 0, then this loop will never run. Since the |
| 122 | + # likely expected behavior for ``max_redirects = 0`` is to only |
| 123 | + # look at the link directly given back by the record, we add 1 |
| 124 | + # for the loop to ensure it runs once in that case. |
| 125 | + for _ in range(max_redirects+1): |
| 126 | + with requests.get(url, stream=True) as r: |
| 127 | + r.raise_for_status() |
| 128 | + if 'Location' in r.headers: |
| 129 | + # Redirection - follow to the next URL |
| 130 | + url = r.headers['Location'] |
| 131 | + else: |
| 132 | + with open(fname, 'wb') as f: |
| 133 | + content_length = r.headers.get('content-length', None) |
| 134 | + if content_length is not None: |
| 135 | + content_length = int(content_length) |
| 136 | + pbar = tqdm(total=content_length // 1024, unit='kB') |
| 137 | + for chunk in r.iter_content(chunk_size=1024): |
| 138 | + if chunk: |
| 139 | + pbar.update() |
| 140 | + f.write(chunk) |
| 141 | + print(f'Download to {fname} complete.') |
| 142 | + return |
| 143 | + raise RuntimeError( |
| 144 | + f'Exceeded maximum number of redirects ({max_redirects})' |
| 145 | + ) |
0 commit comments