Skip to content

Commit b7251fb

Browse files
tmorrellclaude
andcommitted
Add authors, production, and cli option to download_from_record
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent ee9bbcd commit b7251fb

1 file changed

Lines changed: 95 additions & 4 deletions

File tree

‎caltechdata_api/download_from_record.py‎

Lines changed: 95 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,49 @@
44
of five by a dash, found at the end of a CaltechData URL or default DOI.
55
For instance, in the URL ``https://data.caltech.edu/records/4rgh7-zss31``
66
or DOI ``10.22002/4rgh7-zss31``, the record ID is ``4rgh7-zss31``.
7+
8+
The same functions can download from CaltechAUTHORS by passing
9+
``authors=True``, and from the test (development) instances of either
10+
repository by passing ``production=False``.
711
"""
12+
import argparse
813
import os
914
import requests
1015
from tqdm.auto import tqdm
1116

1217
from typing import Any, Dict, Optional, Sequence
1318

1419

15-
def get_files_from_record(record_id: str) -> Dict[str, Any]:
20+
def get_base_url(production: bool = True, authors: bool = False) -> str:
21+
"""Get the base URL of the repository to download from.
22+
23+
Parameters
24+
----------
25+
production
26+
Whether to use the production repository. If ``False``, the
27+
test (development) instance is used instead.
28+
29+
authors
30+
Whether to use CaltechAUTHORS instead of CaltechDATA.
31+
32+
Returns
33+
-------
34+
str
35+
The base URL, without a trailing slash.
36+
"""
37+
if production:
38+
if authors:
39+
return 'https://authors.library.caltech.edu'
40+
return 'https://data.caltech.edu'
41+
else:
42+
if authors:
43+
return 'https://authors.caltechlibrary.dev'
44+
return 'https://data.caltechlibrary.dev'
45+
46+
47+
def get_files_from_record(
48+
record_id: str, production: bool = True, authors: bool = False
49+
) -> Dict[str, Any]:
1650
"""Get a dictionary of files associated with a record.
1751
1852
Parameters
@@ -21,13 +55,21 @@ def get_files_from_record(record_id: str) -> Dict[str, Any]:
2155
The record ID for which to obtain the list of files. See
2256
the module documentation for how to find the record ID.
2357
58+
production
59+
Whether to query the production repository. If ``False``, the
60+
test (development) instance is used instead.
61+
62+
authors
63+
Whether to query CaltechAUTHORS instead of CaltechDATA.
64+
2465
Returns
2566
-------
2667
dict
2768
A dictionary with the file names as keys and the metadata
2869
associated with those files as values.
2970
"""
30-
with requests.get(f'https://data.caltech.edu/api/records/{record_id}/files') as r:
71+
base_url = get_base_url(production=production, authors=authors)
72+
with requests.get(f'{base_url}/api/records/{record_id}/files') as r:
3173
r.raise_for_status()
3274
files = dict()
3375
for entry in r.json().get('entries', []):
@@ -40,7 +82,9 @@ def get_files_from_record(record_id: str) -> Dict[str, Any]:
4082
def download_files_from_record(
4183
record_id: str, output_path: os.PathLike,
4284
filenames: Optional[Sequence[str]] = None,
43-
max_redirects: int = 5
85+
max_redirects: int = 5,
86+
production: bool = True,
87+
authors: bool = False
4488
):
4589
"""Download one or more files from a record.
4690
@@ -71,11 +115,19 @@ def download_files_from_record(
71115
hops that the function can take. Usually, only a single
72116
redirection should be necessary (from the record to the file
73117
provider), but the default allows for a few extra hops.
118+
119+
production
120+
Whether to download from the production repository. If ``False``,
121+
the test (development) instance is used instead.
122+
123+
authors
124+
Whether to download from CaltechAUTHORS instead of CaltechDATA.
74125
"""
75126
if not os.path.isdir(output_path):
76127
raise IOError(f'{output_path} is not an extant directory')
77128

78-
files = get_files_from_record(record_id)
129+
files = get_files_from_record(
130+
record_id, production=production, authors=authors)
79131
if filenames is None:
80132
filenames = sorted(files.keys())
81133

@@ -143,3 +195,42 @@ def download_content(content_url: str, fname: os.PathLike, max_redirects=5):
143195
raise RuntimeError(
144196
f'Exceeded maximum number of redirects ({max_redirects})'
145197
)
198+
199+
200+
if __name__ == '__main__':
201+
parser = argparse.ArgumentParser(
202+
description='download_from_record downloads the files attached to a '
203+
'CaltechDATA or CaltechAUTHORS record'
204+
)
205+
parser.add_argument(
206+
'record_id',
207+
help='The record ID for the record to download files from'
208+
)
209+
parser.add_argument(
210+
'filenames',
211+
nargs='*',
212+
default=None,
213+
help='The names of the files to download. If omitted, all files in '
214+
'the record are downloaded.'
215+
)
216+
parser.add_argument(
217+
'-output_path', default='.',
218+
help='Existing directory to download the files into'
219+
)
220+
parser.add_argument(
221+
'-max_redirects', type=int, default=5,
222+
help='Maximum number of redirects to follow per file'
223+
)
224+
parser.add_argument('-test', dest='production', action='store_false')
225+
parser.add_argument('-authors', dest='authors', action='store_true')
226+
227+
args = parser.parse_args()
228+
229+
download_files_from_record(
230+
args.record_id,
231+
args.output_path,
232+
filenames=args.filenames or None,
233+
max_redirects=args.max_redirects,
234+
production=args.production,
235+
authors=args.authors,
236+
)

0 commit comments

Comments
 (0)