44of five by a dash, found at the end of a CaltechData URL or default DOI.
55For instance, in the URL ``https://data.caltech.edu/records/4rgh7-zss31``
66or DOI ``10.22002/4rgh7-zss31``, the record ID is ``4rgh7-zss31``.
7+
8+ The same functions can download from CaltechAUTHORS by passing
9+ ``authors=True``, and from the test (development) instances of either
10+ repository by passing ``production=False``.
711"""
12+ import argparse
813import os
914import requests
1015from tqdm .auto import tqdm
1116
1217from typing import Any , Dict , Optional , Sequence
1318
1419
15- def get_files_from_record (record_id : str ) -> Dict [str , Any ]:
20+ def get_base_url (production : bool = True , authors : bool = False ) -> str :
21+ """Get the base URL of the repository to download from.
22+
23+ Parameters
24+ ----------
25+ production
26+ Whether to use the production repository. If ``False``, the
27+ test (development) instance is used instead.
28+
29+ authors
30+ Whether to use CaltechAUTHORS instead of CaltechDATA.
31+
32+ Returns
33+ -------
34+ str
35+ The base URL, without a trailing slash.
36+ """
37+ if production :
38+ if authors :
39+ return 'https://authors.library.caltech.edu'
40+ return 'https://data.caltech.edu'
41+ else :
42+ if authors :
43+ return 'https://authors.caltechlibrary.dev'
44+ return 'https://data.caltechlibrary.dev'
45+
46+
47+ def get_files_from_record (
48+ record_id : str , production : bool = True , authors : bool = False
49+ ) -> Dict [str , Any ]:
1650 """Get a dictionary of files associated with a record.
1751
1852 Parameters
@@ -21,13 +55,21 @@ def get_files_from_record(record_id: str) -> Dict[str, Any]:
2155 The record ID for which to obtain the list of files. See
2256 the module documentation for how to find the record ID.
2357
58+ production
59+ Whether to query the production repository. If ``False``, the
60+ test (development) instance is used instead.
61+
62+ authors
63+ Whether to query CaltechAUTHORS instead of CaltechDATA.
64+
2465 Returns
2566 -------
2667 dict
2768 A dictionary with the file names as keys and the metadata
2869 associated with those files as values.
2970 """
30- with requests .get (f'https://data.caltech.edu/api/records/{ record_id } /files' ) as r :
71+ base_url = get_base_url (production = production , authors = authors )
72+ with requests .get (f'{ base_url } /api/records/{ record_id } /files' ) as r :
3173 r .raise_for_status ()
3274 files = dict ()
3375 for entry in r .json ().get ('entries' , []):
@@ -40,7 +82,9 @@ def get_files_from_record(record_id: str) -> Dict[str, Any]:
4082def download_files_from_record (
4183 record_id : str , output_path : os .PathLike ,
4284 filenames : Optional [Sequence [str ]] = None ,
43- max_redirects : int = 5
85+ max_redirects : int = 5 ,
86+ production : bool = True ,
87+ authors : bool = False
4488):
4589 """Download one or more files from a record.
4690
@@ -71,11 +115,19 @@ def download_files_from_record(
71115 hops that the function can take. Usually, only a single
72116 redirection should be necessary (from the record to the file
73117 provider), but the default allows for a few extra hops.
118+
119+ production
120+ Whether to download from the production repository. If ``False``,
121+ the test (development) instance is used instead.
122+
123+ authors
124+ Whether to download from CaltechAUTHORS instead of CaltechDATA.
74125 """
75126 if not os .path .isdir (output_path ):
76127 raise IOError (f'{ output_path } is not an extant directory' )
77128
78- files = get_files_from_record (record_id )
129+ files = get_files_from_record (
130+ record_id , production = production , authors = authors )
79131 if filenames is None :
80132 filenames = sorted (files .keys ())
81133
@@ -143,3 +195,42 @@ def download_content(content_url: str, fname: os.PathLike, max_redirects=5):
143195 raise RuntimeError (
144196 f'Exceeded maximum number of redirects ({ max_redirects } )'
145197 )
198+
199+
200+ if __name__ == '__main__' :
201+ parser = argparse .ArgumentParser (
202+ description = 'download_from_record downloads the files attached to a '
203+ 'CaltechDATA or CaltechAUTHORS record'
204+ )
205+ parser .add_argument (
206+ 'record_id' ,
207+ help = 'The record ID for the record to download files from'
208+ )
209+ parser .add_argument (
210+ 'filenames' ,
211+ nargs = '*' ,
212+ default = None ,
213+ help = 'The names of the files to download. If omitted, all files in '
214+ 'the record are downloaded.'
215+ )
216+ parser .add_argument (
217+ '-output_path' , default = '.' ,
218+ help = 'Existing directory to download the files into'
219+ )
220+ parser .add_argument (
221+ '-max_redirects' , type = int , default = 5 ,
222+ help = 'Maximum number of redirects to follow per file'
223+ )
224+ parser .add_argument ('-test' , dest = 'production' , action = 'store_false' )
225+ parser .add_argument ('-authors' , dest = 'authors' , action = 'store_true' )
226+
227+ args = parser .parse_args ()
228+
229+ download_files_from_record (
230+ args .record_id ,
231+ args .output_path ,
232+ filenames = args .filenames or None ,
233+ max_redirects = args .max_redirects ,
234+ production = args .production ,
235+ authors = args .authors ,
236+ )
0 commit comments