|
9 | 9 | ``authors=True``, and from the test (development) instances of either |
10 | 10 | repository by passing ``production=False``. |
11 | 11 | """ |
| 12 | + |
12 | 13 | import argparse |
13 | 14 | import os |
14 | 15 | import requests |
@@ -36,12 +37,12 @@ def get_base_url(production: bool = True, authors: bool = False) -> str: |
36 | 37 | """ |
37 | 38 | if production: |
38 | 39 | if authors: |
39 | | - return 'https://authors.library.caltech.edu' |
40 | | - return 'https://data.caltech.edu' |
| 40 | + return "https://authors.library.caltech.edu" |
| 41 | + return "https://data.caltech.edu" |
41 | 42 | else: |
42 | 43 | if authors: |
43 | | - return 'https://authors.caltechlibrary.dev' |
44 | | - return 'https://data.caltechlibrary.dev' |
| 44 | + return "https://authors.caltechlibrary.dev" |
| 45 | + return "https://data.caltechlibrary.dev" |
45 | 46 |
|
46 | 47 |
|
47 | 48 | def get_files_from_record( |
@@ -69,22 +70,23 @@ def get_files_from_record( |
69 | 70 | associated with those files as values. |
70 | 71 | """ |
71 | 72 | base_url = get_base_url(production=production, authors=authors) |
72 | | - with requests.get(f'{base_url}/api/records/{record_id}/files') as r: |
| 73 | + with requests.get(f"{base_url}/api/records/{record_id}/files") as r: |
73 | 74 | r.raise_for_status() |
74 | 75 | files = dict() |
75 | | - for entry in r.json().get('entries', []): |
76 | | - key = entry.pop('key') |
| 76 | + for entry in r.json().get("entries", []): |
| 77 | + key = entry.pop("key") |
77 | 78 | files[key] = entry |
78 | 79 |
|
79 | 80 | return files |
80 | 81 |
|
81 | 82 |
|
82 | 83 | def download_files_from_record( |
83 | | - record_id: str, output_path: os.PathLike, |
| 84 | + record_id: str, |
| 85 | + output_path: os.PathLike, |
84 | 86 | filenames: Optional[Sequence[str]] = None, |
85 | 87 | max_redirects: int = 5, |
86 | 88 | production: bool = True, |
87 | | - authors: bool = False |
| 89 | + authors: bool = False, |
88 | 90 | ): |
89 | 91 | """Download one or more files from a record. |
90 | 92 |
|
@@ -124,27 +126,25 @@ def download_files_from_record( |
124 | 126 | Whether to download from CaltechAUTHORS instead of CaltechDATA. |
125 | 127 | """ |
126 | 128 | if not os.path.isdir(output_path): |
127 | | - raise IOError(f'{output_path} is not an extant directory') |
| 129 | + raise IOError(f"{output_path} is not an extant directory") |
128 | 130 |
|
129 | | - files = get_files_from_record( |
130 | | - record_id, production=production, authors=authors) |
| 131 | + files = get_files_from_record(record_id, production=production, authors=authors) |
131 | 132 | if filenames is None: |
132 | 133 | filenames = sorted(files.keys()) |
133 | 134 |
|
134 | 135 | for filename in filenames: |
135 | 136 | output_file = os.path.join(output_path, filename) |
136 | 137 | if filename not in files: |
137 | | - raise KeyError( |
138 | | - f'File "{filename}" not found in record {record_id}') |
| 138 | + raise KeyError(f'File "{filename}" not found in record {record_id}') |
139 | 139 |
|
140 | 140 | entry = files[filename] |
141 | | - if 'links' not in entry or 'content' not in entry['links']: |
| 141 | + if "links" not in entry or "content" not in entry["links"]: |
142 | 142 | raise NotImplementedError( |
143 | 143 | f'Metadata for file "{filename}" in record {record_id} is ' |
144 | | - 'missing the content link' |
| 144 | + "missing the content link" |
145 | 145 | ) |
146 | 146 |
|
147 | | - content_url = entry['links']['content'] |
| 147 | + content_url = entry["links"]["content"] |
148 | 148 | download_content(content_url, output_file, max_redirects=max_redirects) |
149 | 149 |
|
150 | 150 |
|
@@ -174,55 +174,55 @@ def download_content(content_url: str, fname: os.PathLike, max_redirects=5): |
174 | 174 | # likely expected behavior for ``max_redirects = 0`` is to only |
175 | 175 | # look at the link directly given back by the record, we add 1 |
176 | 176 | # for the loop to ensure it runs once in that case. |
177 | | - for _ in range(max_redirects+1): |
| 177 | + for _ in range(max_redirects + 1): |
178 | 178 | with requests.get(url, stream=True) as r: |
179 | 179 | r.raise_for_status() |
180 | | - if 'Location' in r.headers: |
| 180 | + if "Location" in r.headers: |
181 | 181 | # Redirection - follow to the next URL |
182 | | - url = r.headers['Location'] |
| 182 | + url = r.headers["Location"] |
183 | 183 | else: |
184 | | - with open(fname, 'wb') as f: |
185 | | - content_length = r.headers.get('content-length', None) |
| 184 | + with open(fname, "wb") as f: |
| 185 | + content_length = r.headers.get("content-length", None) |
186 | 186 | if content_length is not None: |
187 | 187 | content_length = int(content_length) |
188 | | - pbar = tqdm(total=content_length // 1024, unit='kB') |
| 188 | + pbar = tqdm(total=content_length // 1024, unit="kB") |
189 | 189 | for chunk in r.iter_content(chunk_size=1024): |
190 | 190 | if chunk: |
191 | 191 | pbar.update() |
192 | 192 | f.write(chunk) |
193 | | - print(f'Download to {fname} complete.') |
| 193 | + print(f"Download to {fname} complete.") |
194 | 194 | return |
195 | | - raise RuntimeError( |
196 | | - f'Exceeded maximum number of redirects ({max_redirects})' |
197 | | - ) |
| 195 | + raise RuntimeError(f"Exceeded maximum number of redirects ({max_redirects})") |
198 | 196 |
|
199 | 197 |
|
200 | | -if __name__ == '__main__': |
| 198 | +if __name__ == "__main__": |
201 | 199 | parser = argparse.ArgumentParser( |
202 | | - description='download_from_record downloads the files attached to a ' |
203 | | - 'CaltechDATA or CaltechAUTHORS record' |
| 200 | + description="download_from_record downloads the files attached to a " |
| 201 | + "CaltechDATA or CaltechAUTHORS record" |
204 | 202 | ) |
205 | 203 | parser.add_argument( |
206 | | - 'record_id', |
207 | | - help='The record ID for the record to download files from' |
| 204 | + "record_id", help="The record ID for the record to download files from" |
208 | 205 | ) |
209 | 206 | parser.add_argument( |
210 | | - 'filenames', |
211 | | - nargs='*', |
| 207 | + "filenames", |
| 208 | + nargs="*", |
212 | 209 | default=None, |
213 | | - help='The names of the files to download. If omitted, all files in ' |
214 | | - 'the record are downloaded.' |
| 210 | + help="The names of the files to download. If omitted, all files in " |
| 211 | + "the record are downloaded.", |
215 | 212 | ) |
216 | 213 | parser.add_argument( |
217 | | - '-output_path', default='.', |
218 | | - help='Existing directory to download the files into' |
| 214 | + "-output_path", |
| 215 | + default=".", |
| 216 | + help="Existing directory to download the files into", |
219 | 217 | ) |
220 | 218 | parser.add_argument( |
221 | | - '-max_redirects', type=int, default=5, |
222 | | - help='Maximum number of redirects to follow per file' |
| 219 | + "-max_redirects", |
| 220 | + type=int, |
| 221 | + default=5, |
| 222 | + help="Maximum number of redirects to follow per file", |
223 | 223 | ) |
224 | | - parser.add_argument('-test', dest='production', action='store_false') |
225 | | - parser.add_argument('-authors', dest='authors', action='store_true') |
| 224 | + parser.add_argument("-test", dest="production", action="store_false") |
| 225 | + parser.add_argument("-authors", dest="authors", action="store_true") |
226 | 226 |
|
227 | 227 | args = parser.parse_args() |
228 | 228 |
|
|
0 commit comments