Skip to content

Commit 0745076

Browse files
joshua-laughnertmorrell
authored andcommitted
Added the download_from_record.py module
1 parent ebcf3bb commit 0745076

1 file changed

Lines changed: 145 additions & 0 deletions

File tree

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
"""Functions to download files directly from a CaltechData record.
2+
3+
A CaltechData record ID is the ten characters, separated into two groups
4+
of five by a dash, found at the end of a CaltechData URL or default DOI.
5+
For instance, in the URL ``https://data.caltech.edu/records/4rgh7-zss31``
6+
or DOI ``10.22002/4rgh7-zss31``, the record ID is ``4rgh7-zss31``.
7+
"""
8+
import os
9+
import requests
10+
from tqdm.auto import tqdm
11+
12+
from typing import Any, Dict, Optional, Sequence
13+
14+
15+
def get_files_from_record(record_id: str) -> Dict[str, Any]:
16+
"""Get a dictionary of files associated with a record.
17+
18+
Parameters
19+
----------
20+
record_id
21+
The record ID for which to obtain the list of files. See
22+
the module documentation for how to find the record ID.
23+
24+
Returns
25+
-------
26+
dict
27+
A dictionary with the file names as keys and the metadata
28+
associated with those files as values.
29+
"""
30+
with requests.get(f'https://data.caltech.edu/api/records/{record_id}/files') as r:
31+
r.raise_for_status()
32+
files = dict()
33+
for entry in r.json().get('entries', []):
34+
key = entry.pop('key')
35+
files[key] = entry
36+
37+
return files
38+
39+
40+
def download_files_from_record(
41+
record_id: str, output_path: os.PathLike,
42+
filenames: Optional[Sequence[str]] = None,
43+
max_redirects: int = 5
44+
):
45+
"""Download one or more files from a record.
46+
47+
Parameters
48+
----------
49+
record_id
50+
The record ID from which to download the file. See
51+
the module documentation for how to find the record ID.
52+
53+
output_path
54+
Where to download the files. Must be an existing directory.
55+
If you want control over exactly how each file is named,
56+
use :func:`get_files_from_record` to get the URLs for each
57+
file, then manually call :func:`download_content` for each
58+
file you wish to download.
59+
60+
filenames
61+
The name of the files in the record. Must match exactly.
62+
If you are not sure of the filenames in the record,
63+
they are the keys of the dictionary returned by
64+
:func:`get_files_from_record`. If omitted, all files
65+
are downloaded.
66+
67+
max_redirects
68+
If the record's link to the file content returns a
69+
redirection to another location, this function will follow
70+
that redirection. This parameter sets the maximum number of
71+
hops that the function can take. Usually, only a single
72+
redirection should be necessary (from the record to the file
73+
provider), but the default allows for a few extra hops.
74+
"""
75+
if not os.path.isdir(output_path):
76+
raise IOError(f'{output_path} is not an extant directory')
77+
78+
files = get_files_from_record(record_id)
79+
if filenames is None:
80+
filenames = sorted(files.keys())
81+
82+
for filename in filenames:
83+
output_file = os.path.join(output_path, filename)
84+
if filename not in files:
85+
raise KeyError(
86+
f'File "{filename}" not found in record {record_id}')
87+
88+
entry = files[filename]
89+
if 'links' not in entry or 'content' not in entry['links']:
90+
raise NotImplementedError(
91+
f'Metadata for file "{filename}" in record {record_id} is '
92+
'missing the content link'
93+
)
94+
95+
content_url = entry['links']['content']
96+
download_content(content_url, output_file, max_redirects=max_redirects)
97+
98+
99+
def download_content(content_url: str, fname: os.PathLike, max_redirects=5):
100+
"""Download the contents of a file.
101+
102+
Parameters
103+
----------
104+
content_url
105+
The URL pointing to the contents of the file. In the dictionary
106+
returned by :func:`get_files_from_record` (call it ``D``), this
107+
is usually the value at ``D[FILENAME]["links"]["content"]``.
108+
109+
fname
110+
The path to which to save the file contents. Will be overwritten!
111+
112+
113+
max_redirects
114+
If the ``content_url`` returns a redirection to another location,
115+
this function will follow that redirection. This parameter sets
116+
the maximum number of hops that the function can take. Usually,
117+
only a single redirection should be necessary (from the record to
118+
the file provider), but the default allows for a few extra hops.
119+
"""
120+
url = content_url
121+
# If max_redirects == 0, then this loop will never run. Since the
122+
# likely expected behavior for ``max_redirects = 0`` is to only
123+
# look at the link directly given back by the record, we add 1
124+
# for the loop to ensure it runs once in that case.
125+
for _ in range(max_redirects+1):
126+
with requests.get(url, stream=True) as r:
127+
r.raise_for_status()
128+
if 'Location' in r.headers:
129+
# Redirection - follow to the next URL
130+
url = r.headers['Location']
131+
else:
132+
with open(fname, 'wb') as f:
133+
content_length = r.headers.get('content-length', None)
134+
if content_length is not None:
135+
content_length = int(content_length)
136+
pbar = tqdm(total=content_length // 1024, unit='kB')
137+
for chunk in r.iter_content(chunk_size=1024):
138+
if chunk:
139+
pbar.update()
140+
f.write(chunk)
141+
print(f'Download to {fname} complete.')
142+
return
143+
raise RuntimeError(
144+
f'Exceeded maximum number of redirects ({max_redirects})'
145+
)

0 commit comments

Comments
 (0)