Repository navigation
Expand file tree
/
Copy pathcloud-init.yaml
More file actions
865 lines (808 loc) · 42.2 KB
/
Copy pathcloud-init.yaml
File metadata and controls
865 lines (808 loc) · 42.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
#cloud-config
# Bring-up machine for CaltechTHESIS running under Granian instead of uWSGI
# (repo `caltechthesis`, branch `granian`). Adapted from CaltechAUTHORS's
# cloud-init.yaml (repo `caltechauthors`, branch `granian`), itself adapted
# from the `granian-rdm-v14` research project. Not a scaffold: this clones
# and deploys the real caltechthesis repo -- see setup_caltechthesis_granian.bash.
#
# Scope decisions for this repo (DR-0001 in agents/projects/caltechthesis):
# - x86_64 (amd64), per the retired caltechthesis-rdm project's DR-0002.
# - Python 3.14 via this repo's own .python-version and requires-python
# (~=3.14.0), standard GIL build -- the free-threaded 3.14t build was
# tried and abandoned in granian-rdm-v14.
# - Node 26 via nvm, RDM v14's documented floor being 24+.
# - invenio-app-rdm 14.0.4 (stable v14 release) via this repo's own
# pyproject.toml/uv.lock; uWSGI is gone, Granian serves UI (5000) and
# REST (5001, --url-path-prefix /api per caltechauthors DR-0007).
# - uv is the curl-installed /home/ubuntu/.local/bin/uv, matching
# CaltechAUTHORS production and the committed systemd units.
# - No production snapshot exists to restore, so setup runs
# `invenio-cli services setup --no-demo-data` rather than leaving the
# database for a clasm restore as CaltechAUTHORS does.
# - docker-services.yml sets OpenSearch path.repo and bind-mounts
# /opt/rdm_opensearch_backups, the same as CaltechAUTHORS and CaltechDATA,
# so clasm can snapshot this instance. The setup script creates that
# directory (uid 1000) before starting the containers.
# - File storage: local disk while testing, S3 in production (like
# CaltechAUTHORS). invenio.cfg selects the S3 factory; .env overrides it to
# the local one here via INVENIO_FILES_REST_STORAGE_FACTORY, and the setup
# script creates a local default location. Going to S3 means deleting that
# .env line and creating the location for the real bucket.
# - Granian sizing (2 workers x 2 threads) carries over this repo's old
# uWSGI tuning; unverified under load.
#
# UNVERIFIED: this file has not been booted yet (Phase 6). The nginx site
# config, TLS setup and verify_rdm_boot.bash checks are carried over from
# CaltechAUTHORS as-is; the /static pdf.js paths in particular may differ in
# RDM v14.
package_update: true
package_upgrade: true
fqdn: caltechthesis-granian.local
packages:
- sshfs
- build-essential
- git
- lynx
- tmux
- curl
- rsync
- nginx
- imagemagick
- pkg-config
- libxml2-dev
- libxmlsec1-dev
- docker.io
- docker-compose-v2
- makepasswd
- shellcheck
- jq
- unzip # needed to install AWS CLI v2's official installer bundle --
# found missing 2026-08-18 setting up a clasm-launched
# restore-test instance (caltechdata-restore-test)
groups:
- docker
- staff
- www-data
runcmd:
- adduser ubuntu docker
- adduser ubuntu staff
- adduser ubuntu www-data
- /usr/local/bin/install_rdm_toolchain.bash
snap:
commands:
- snap install --classic certbot
write_files:
- content: |
#!/bin/bash
# Installs uv, a uv-managed Python 3.14, invenio-cli (via "uv tool
# install", not pip -- no PEP 668 externally-managed-environment
# dance needed), and Node 26 -- everything
# setup_caltechthesis_granian.bash needs. Run once at boot via
# runcmd; idempotent if rerun.
set -euo pipefail
# Installed as the ubuntu user (not root, which is what this script
# otherwise runs as via runcmd) so everything lands in
# /home/ubuntu/.local/bin, which is where
# setup_caltechthesis_granian.bash's PATH and the systemd units'
# ExecStart both expect to find it.
echo "Installing uv for the ubuntu user..."
sudo -u ubuntu -H bash -c 'curl -LsSf https://astral.sh/uv/install.sh | sh'
echo "Installing Python 3.14 via uv..."
sudo -u ubuntu -H bash -c 'export PATH="$HOME/.local/bin:$PATH"; uv python install 3.14'
echo "Installing invenio-cli as a uv tool (isolated venv, own Python -- independent of any project's Python pin)..."
sudo -u ubuntu -H bash -c 'export PATH="$HOME/.local/bin:$PATH"; uv tool install "invenio-cli==1.11.0"'
echo "Installing nvm + Node 26 for the ubuntu user..."
# Node 24+ is InvenioRDM v14's documented requirement ("tested with
# 26+ too"), so 26 matches what upstream validated. This is stricter
# than invenio-cli 1.11.0's own gate, which only asserts Node >=18.
#
# Real bug found via live testing on granian-rdm-v14 (2026-07-27,
# carried over as a precaution here too): git-cloning nvm directly
# (instead of running its official installer) skips the step that
# normally appends nvm's loader snippet to .bashrc. Only a shell
# that explicitly sources nvm.sh gets node/npm on PATH -- this
# script's own subshell does, via the export/source lines below,
# but any *later*, separate interactive SSH session would not, so
# append the same loader snippet to .bashrc for every future shell
# too.
sudo -u ubuntu -H bash -c '
set -euo pipefail
if [ ! -d "$HOME/.nvm" ]; then
git clone --depth 1 https://github.com/nvm-sh/nvm "$HOME/.nvm"
fi
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
nvm install 26
nvm alias default 26
# The asset build in RDM v14 (invenio-assets via pynpm) runs pnpm, not npm.
# Node 25+ no longer bundles corepack, so `corepack enable` is not an
# option; install it from npm. Found live on the first Multipass boot:
# `invenio-cli install` died with FileNotFoundError: 'pnpm'.
npm install -g pnpm
echo "" >> "$HOME/.bashrc"
echo "export NVM_DIR=\"\$HOME/.nvm\"" >> "$HOME/.bashrc"
echo "[ -s \"\$NVM_DIR/nvm.sh\" ] && . \"\$NVM_DIR/nvm.sh\"" >> "$HOME/.bashrc"
'
echo "Setting vm.max_map_count for OpenSearch..."
cat <<EOT >/etc/sysctl.d/rdm_sysctl.conf
vm.max_map_count=262144
EOT
sysctl --system
# AWS CLI v2, last so a failed download cannot stop the toolchain above.
# cloud-init has installed `unzip` "for the AWS CLI v2 installer" since
# 2026-08-18 but nothing ever ran the installer, so clasm's backup and
# restore workflows, which run `aws s3` on the instance, failed their
# preflight on every new instance (third occurrence, 2026-09-29). Same
# commands as clasm's instance_install_awscli.bash, which worked on
# caltechauthors-test-v13; arch-aware because the Multipass images are
# aarch64 on Apple silicon and production is x86_64.
echo "Installing AWS CLI v2..."
if command -v aws >/dev/null 2>&1; then
echo "AWS CLI already installed: $(aws --version 2>&1)"
else
case "$(uname -m)" in
x86_64) AWSCLI_ARCH=x86_64 ;;
aarch64) AWSCLI_ARCH=aarch64 ;;
*) echo "Unsupported architecture for the AWS CLI: $(uname -m)" >&2; exit 1 ;;
esac
AWSCLI_TMP=$(mktemp -d /var/tmp/awscli-install.XXXXXX)
curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-${AWSCLI_ARCH}.zip" -o "$AWSCLI_TMP/awscliv2.zip"
unzip -q "$AWSCLI_TMP/awscliv2.zip" -d "$AWSCLI_TMP"
"$AWSCLI_TMP/aws/install"
rm -r "$AWSCLI_TMP"
echo "Installed: $(aws --version 2>&1)"
fi
path: /usr/local/bin/install_rdm_toolchain.bash
owner: 'root:root'
permissions: '0775'
- content: |
#!/bin/bash
# ExecStartPre readiness gate for the rdm/rdm_rest/rdm_celery systemd
# units (see ../cloud-init.yaml). A Docker container reporting "started"
# is not the same as the service inside it actually accepting
# connections -- that gap is what makes a unit's ExecStart race ahead
# against services that are not ready after a reboot. This blocks
# until Postgres/Redis/RabbitMQ/OpenSearch on
# 127.0.0.1 all respond, or exits non-zero after a bounded timeout so
# systemd's Restart=on-failure retries the whole unit instead of the
# ExecStart racing ahead against services that aren't ready yet.
set -uo pipefail
TIMEOUT_SECONDS=180
INTERVAL_SECONDS=2
DEADLINE=$(($(date +%s) + TIMEOUT_SECONDS))
tcp_ready() {
local host="$1" port="$2"
(exec 3<>"/dev/tcp/${host}/${port}") 2>/dev/null
}
http_ready() {
curl -fsS -o /dev/null --max-time 5 "$1"
}
wait_for() {
local description="$1"
shift
while ! "$@"; do
if [ "$(date +%s)" -ge "$DEADLINE" ]; then
echo "Timed out after ${TIMEOUT_SECONDS}s waiting for ${description}" >&2
return 1
fi
sleep "$INTERVAL_SECONDS"
done
echo "${description} is ready"
}
wait_for "Postgres (127.0.0.1:5432)" tcp_ready 127.0.0.1 5432
wait_for "Redis/cache (127.0.0.1:6379)" tcp_ready 127.0.0.1 6379
wait_for "RabbitMQ (127.0.0.1:5672)" tcp_ready 127.0.0.1 5672
wait_for "OpenSearch (127.0.0.1:9200)" http_ready http://127.0.0.1:9200
path: /usr/local/bin/wait_for_rdm_services.bash
owner: 'root:root'
permissions: '0775'
- content: |
#!/bin/bash
# Boot acceptance script. Run this manually right after `sudo reboot` -- not just
# after a manual `systemctl start` sequence -- to confirm the instance
# actually comes back up unattended. Exit 0 = every check passed;
# nonzero = something is still broken.
#
# Deliberately standalone from wait_for_rdm_services.bash (the
# production readiness gate baked into the systemd units via
# ExecStartPre): this script is the outside observer confirming the gate
# actually worked, not part of the gate itself. Keep this in the repo as
# a standing regression check for any future edit to cloud-init.yaml's
# systemd unit section.
#
# USAGE: verify_rdm_boot.bash [INSTANCE_NAME]
set -uo pipefail
INSTANCE_NAME="${1:-caltechthesis}"
FAILURES=0
check() {
local description="$1"
shift
if "$@"; then
echo "OK -- ${description}"
else
echo "FAIL -- ${description}"
FAILURES=$((FAILURES + 1))
fi
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
units_active() {
systemctl is-active --quiet docker rdm rdm_rest rdm_celery nginx
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
api_responds() {
local code
code="$(curl -sk -o /dev/null -w '%{http_code}' --max-time 15 https://localhost/api/records)"
[ "$code" = "200" ]
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
api_prefix_preserved() {
# Regression check for the 2026-09-10 administration Banners 404. The
# REST app has to know it is mounted at /api -- Granian's
# --url-path-prefix in rdm_rest.service, paired with nginx forwarding
# /api/... unmodified -- or every path it generates itself comes out
# without the prefix and lands in the UI app. /api/banners carries no
# trailing slash, so Werkzeug answers with its strict-slashes
# redirect: exactly the case that regressed. Note that api_responds
# above stayed green the whole time this was broken, because a plain
# 200 never exercises URL generation.
local location
location="$(curl -sk -D - -o /dev/null --max-time 15 https://localhost/api/banners \
| tr -d '\r' | awk 'BEGIN { IGNORECASE = 1 } /^location:/ { print $2 }')"
# No Location at all means no redirect was issued, which is fine. A
# redirect that has lost the /api prefix is the bug.
[ -z "$location" ] || [[ "$location" == */api/* ]]
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
single_frame_options_header() {
# Regression check for the 2026-09-10 file-preview breakage. The
# application owns the security headers (DR-0005); nginx's add_header
# appends rather than replaces, so an X-Frame-Options here as well as
# Talisman's leaves the response carrying two conflicting values,
# which browsers treat as one invalid directive and refuse to frame.
local count
count="$(curl -sk -D - -o /dev/null --max-time 15 https://localhost/ \
| grep -ci '^x-frame-options:')"
[ "$count" = "1" ]
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
static_mime_types() {
# Regression check for 2026-09-10. nginx's mime.types carries no .mjs
# entry, so pdf.js's ES modules were served as
# application/octet-stream and Firefox refused to load them as
# modules. It was a 200 with the wrong Content-Type, which is exactly
# why no status-code check could ever have seen it.
#
# The .css control is the other half of the check. The types block
# that adds .mjs sits at http level because repeated types blocks in
# one context add to the map, while a types block in a server or
# location context REPLACES the inherited one -- and that failure
# would show up as every other extension under /static turning into
# application/octet-stream. One line asserts the fix, the other
# asserts the fix cost nothing.
local mjs css
mjs="$(curl -sk -o /dev/null -w '%{http_code} %{content_type}' --max-time 15 \
https://localhost/static/js/pdfjs/build/pdf.mjs)"
css="$(curl -sk -o /dev/null -w '%{http_code} %{content_type}' --max-time 15 \
https://localhost/static/js/pdfjs/web/pdf_viewer.css)"
[ "$mjs" = "200 text/javascript" ] && [ "$css" = "200 text/css" ]
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
celery_responds() {
# Explicitly as ubuntu, regardless of which user runs this script --
# uv and the project venv live under /home/ubuntu, and running this
# as root (e.g. via `sudo`, or SSM's default execution user) silently
# can't find `uv` at all, which looks identical to celery being dead.
# Found live 2026-07-27: a manual ping as ubuntu succeeded instantly
# while this check kept failing, because it was never really running
# celery at all.
#
# Second bug found live the same day: piping straight into `grep -q`
# (which exits the instant it finds a match, closing the pipe early)
# sends SIGPIPE to the still-running celery process upstream -- with
# `set -o pipefail` active, THAT non-zero exit, not grep's successful
# match, became the pipeline's reported status. Capture the output via
# command substitution first (fully waits, no live pipe to the
# subprocess) and match against the captured string instead.
local output
output="$(sudo -u ubuntu -H bash -c "
cd '/Sites/${INSTANCE_NAME}' &&
export PATH=\"\$HOME/.local/bin:\$PATH\" &&
timeout 15 uv run --no-sync celery --app invenio_app.celery inspect ping
" 2>/dev/null)"
grep -q "pong" <<< "$output"
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
frontpage_renders() {
# The front page, header and footer come from this repository's
# templates/ overrides, so a stale override shows up here as a 500
# (TemplateNotFound) rather than as a failed service. A status-only
# check on /api/records never touches the UI app's templates.
local code
code="$(curl -sk -o /dev/null -w '%{http_code}' --max-time 30 https://localhost/)"
[ "$code" = "200" ]
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
template_overrides_resolve() {
# Regression check for 2026-11-20. templates/.../records/detail.html was
# a half-updated copy that included a template v14 no longer ships, so
# every record landing page returned 500 while every check above stayed
# green: an empty instance has no record page to request. This parses
# each override with the application's own Jinja environment and
# confirms every template it includes, extends or imports resolves, so
# it needs no records. Run as ubuntu for the same reason as
# celery_responds. The unresolved references are printed under the
# failing check.
local output status
output="$(sudo -u ubuntu -H bash -c "
cd '/Sites/${INSTANCE_NAME}' &&
export PATH=\"\$HOME/.local/bin:\$PATH\" &&
timeout 120 uv run --no-sync python scripts/check_template_overrides.py
" 2>&1)"
status=$?
[ "$status" -eq 0 ] || grep '^FAIL' <<< "$output" | sed 's/^/ /'
return "$status"
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
vocab_has() {
# vocab_has TYPE ID... Succeeds when every ID is an entry of the
# vocabulary TYPE, read through the REST API as an anonymous visitor.
# The ids that are missing are printed under the failing check. The
# body is parsed, not just the status, so an HTML 200 does not pass.
local type="$1" id body missing=0
shift
for id in "$@"; do
body="$(curl -sk --max-time 15 "https://localhost/api/vocabularies/${type}/${id}")"
if ! jq -e --arg id "$id" '.id == $id and ((.title.en // "") | length > 0)' \
<<< "$body" > /dev/null 2>&1; then
echo " missing from ${type}: ${id}"
missing=1
fi
done
return "$missing"
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
stock_vocabularies_loaded() {
# The control for the two checks below. RDM loads its fixtures through
# about 32,000 Celery tasks that drain for several minutes after
# `services setup`, and until they do every vocabulary is empty. A
# stock entry that is missing means "not loaded yet", so the local
# entries below would fail for the wrong reason.
vocab_has contributorsroles supervisor &&
vocab_has datetypes issued available
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
thesis_roles_defined() {
# DR-0002 and its addendum: advisor, committee chair and committee
# member, plus the co-advisor and co-chair roles that the EPrints
# shape report found. Stock RDM has none of them.
vocab_has contributorsroles advisor co_advisor committee_chair \
committee_co_chair committee_member
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
thesis_date_types_defined() {
# DR-0002: the three local date types, and the stock coverage type
# that the local copy of the list has to keep (v14.0 added it, so a
# copy made from an older list silently drops it).
vocab_has datetypes degree_date approved_date gradofc_approval_date coverage
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
caltech_fields_mapped() {
# The caltech: custom fields have to be in the records search mapping,
# not only in invenio.cfg: a field that is configured and not mapped
# loads and then fails on the first record that carries it. Run as
# ubuntu for the same reason as celery_responds. caltech:groups is
# defined on every version of this repository, so it shows whether the
# command itself works when the new fields are missing.
#
# `custom-fields exists` exits 0 whether or not the field exists and
# reports only in its text ("Field X exists" / "Field X does not
# exist"), so the output is what is matched. The first version of this
# check tested the exit status and passed with four fields missing.
local field output missing=0
for field in caltech:groups caltech:option_major caltech:option_minor \
caltech:division caltech:embargo_until; do
output="$(sudo -u ubuntu -H bash -c "
cd '/Sites/${INSTANCE_NAME}' &&
export PATH=\"\$HOME/.local/bin:\$PATH\" &&
timeout 60 uv run --no-sync invenio rdm-records custom-fields exists -f '${field}'
" 2>&1)"
if ! grep -qx "Field ${field} exists" <<< "$output"; then
echo " not in the records mapping: ${field}"
missing=1
fi
done
return "$missing"
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
custom_fields_configured() {
# scripts/check_custom_fields.py reads the application's configuration
# and schemas and writes nothing. Its FAIL lines are printed under the
# failing check.
local output status
output="$(sudo -u ubuntu -H bash -c "
cd '/Sites/${INSTANCE_NAME}' &&
export PATH=\"\$HOME/.local/bin:\$PATH\" &&
timeout 120 uv run --no-sync python scripts/check_custom_fields.py
" 2>&1)"
status=$?
[ "$status" -eq 0 ] || grep '^FAIL' <<< "$output" | sed 's/^/ /'
return "$status"
}
# shellcheck disable=SC2329 # called indirectly via check()'s "$@"
thesis_options_defined() {
# The option vocabulary behind caltech:option_major and
# caltech:option_minor. Two ids from COLD's thesis_options list.
vocab_has thesisoptions gps aeroeng
}
check "docker/rdm/rdm_rest/rdm_celery/nginx are all systemctl active" units_active
check "https://localhost/api/records responds 200" api_responds
check "redirects generated by the REST app keep the /api prefix" api_prefix_preserved
check "exactly one X-Frame-Options header on a framed response" single_frame_options_header
check "/static serves .mjs as JavaScript and .css as CSS" static_mime_types
check "celery worker responds to inspect ping" celery_responds
check "front page renders (header, footer and front-page overrides)" frontpage_renders
check "template overrides resolve against the installed RDM" template_overrides_resolve
check "stock roles and date types are loaded (fixture queue drained)" stock_vocabularies_loaded
check "thesis contributor roles are defined (advisor, committee chair and member)" thesis_roles_defined
check "thesis date types are defined (degree, approved, grad office approval) and coverage kept" thesis_date_types_defined
check "caltech: custom fields are in the records search mapping" caltech_fields_mapped
check "thesis custom fields are configured and validate (option, division, embargo, degree date)" custom_fields_configured
check "thesis option vocabulary is defined" thesis_options_defined
echo
if [ "$FAILURES" -eq 0 ]; then
echo "All checks passed."
exit 0
else
echo "${FAILURES} check(s) failed."
exit 1
fi
path: /usr/local/bin/verify_rdm_boot.bash
owner: 'root:root'
permissions: '0775'
- content: |
#!/bin/bash
# Deploys the real caltechthesis repo (not a cookiecutter scaffold --
# this repo already exists, with its own pinned pyproject.toml/
# uv.lock and its own committed systemd units) running under
# Granian on Python 3.14. Not run automatically by cloud-init -- ssh
# in after boot and run this manually so each step can be watched/
# debugged.
#
# USAGE: setup_caltechthesis_granian.bash [BRANCH] [REPO_URL]
set -euo pipefail
BRANCH="${1:-granian}"
REPO_URL="${2:-https://github.com/caltechlibrary/caltechthesis.git}"
INSTANCE_NAME="caltechthesis"
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
nvm use 26
export PATH="$HOME/.local/bin:$PATH"
# Real bug found via live testing on granian-rdm-v14 (2026-07-27,
# carried over as a precaution here too): invenio-assets'
# devDependencies pin @rspack/core <1.7.0 while @rspack/cli 2.1.5's
# own peer dependency wants ^2.0.0-0 -- an inconsistency in that
# package's own package.json, not anything specific to this
# project. npm (10.x, strict-by-default peer resolution) refuses to
# install at all otherwise. Setting this persists to ~/.npmrc, so
# it applies automatically to the npm install invenio-cli runs
# internally during "invenio-cli install".
echo "Configuring npm to tolerate invenio-assets' rspack peer-dependency conflict"
npm config set legacy-peer-deps true
sudo mkdir -p /Sites
sudo chown "$USER" /Sites
cd /Sites
echo "Cloning ${REPO_URL} (branch ${BRANCH}) into /Sites/${INSTANCE_NAME}"
git clone --branch "$BRANCH" "$REPO_URL" "$INSTANCE_NAME"
cd "/Sites/${INSTANCE_NAME}"
echo "Syncing dependencies via uv -- pyproject.toml/uv.lock already"
echo "pin invenio-app-rdm 14.0.4 and granian, nothing to 'uv add'"
uv sync --frozen
echo "Building assets and translations"
invenio-cli install
echo "Creating the OpenSearch snapshot directory. docker-services.yml"
echo "bind-mounts /opt/rdm_opensearch_backups as path.repo (so clasm can"
echo "back up and restore the index), owned by uid 1000 inside the"
echo "container. If it does not exist first, Docker creates it as root"
echo "and OpenSearch cannot write snapshots."
sudo mkdir -p /opt/rdm_opensearch_backups
sudo chown 1000:1000 /opt/rdm_opensearch_backups
echo "Starting backing services (Postgres/Redis/RabbitMQ/OpenSearch)"
echo "and initialising them -- unlike CaltechAUTHORS there is no"
echo "production snapshot to restore here, so this runs the normal"
echo "'services setup' (schema, indices, fixtures including the static"
echo "pages) and skips demo records. OpenSearch data lives in the"
echo "search_data named volume from docker-services.yml, so unlike the"
echo "CaltechAUTHORS bind mounts there are no host directories to"
echo "create first."
echo "File storage: local, for testing. invenio.cfg selects the S3"
echo "storage factory (production stores uploads in S3), and .invenio"
echo "says file_storage = S3, which would make 'services setup' create a"
echo "location named s3://default -- a bucket that does not exist."
echo "Temporarily flip .invenio to local for the setup run (setup then"
echo "creates a default location under the instance path), and restore"
echo "the tracked file afterwards. .env below overrides the storage"
echo "factory to match."
sed -i 's/^file_storage = S3$/file_storage = local/' .invenio
invenio-cli services setup --no-demo-data || { git checkout -- .invenio; exit 1; }
git checkout -- .invenio
echo "Generating self-signed TLS cert (internal test instance behind"
echo "a restricted security group, not a public endpoint -- a real"
echo "cert isn't needed, just one nginx can load)"
if [ ! -f /etc/ssl/certs/nginx-selfsigned.crt ]; then
sudo openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \
-keyout /etc/ssl/private/nginx-selfsigned.key \
-out /etc/ssl/certs/nginx-selfsigned.crt \
-subj "/CN=${INSTANCE_NAME}.local"
fi
if [ ! -f /etc/nginx/dhparam.pem ]; then
sudo openssl dhparam -out /etc/nginx/dhparam.pem 2048
fi
sudo mkdir -p /etc/nginx/snippets
cat <<EOT | sudo tee /etc/nginx/snippets/self-signed.conf >/dev/null
ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt;
ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key;
EOT
cat <<EOT | sudo tee /etc/nginx/snippets/ssl-params.conf >/dev/null
ssl_protocols TLSv1.2 TLSv1.3;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/nginx/dhparam.pem;
ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384;
ssl_ecdh_curve secp384r1;
ssl_session_cache shared:SSL:10m;
ssl_session_tickets off;
ssl_stapling off;
ssl_stapling_verify off;
resolver 127.0.0.53 valid=300s;
resolver_timeout 5s;
EOT
# This snippet deliberately carries no add_header of its own. It used to
# end with "add_header X-Frame-Options DENY;" and
# "add_header X-Content-Type-Options nosniff;", which the application
# already sends via Talisman (APP_DEFAULT_SECURE_HEADERS in invenio.cfg,
# frame_options "sameorigin"). nginx's add_header appends rather than
# replaces, so every 2xx response carried two X-Frame-Options values --
# "sameorigin" from the app and "DENY" from here. Browsers collapse
# those into one malformed value and refuse to frame the document, which
# broke the record file preview (reported live on caltechauthors-v13,
# 2026-09-10; the response was confirmed to carry both headers). See
# DR-0005: the application owns the security headers, nginx does not
# duplicate a subset of them.
echo "Writing nginx site config"
# Real bug found via live testing 2026-07-27 (Step 7 verification):
# the proxy "location" blocks used to live in a server block with
# NO "listen" directive of its own -- nginx defaults that to
# "listen 80", which then collided with the *separate*, explicit
# "listen 80" block below it (both declaring "server_name _;" on
# the same socket), logged as "conflicting server name \"_\" on
# 0.0.0.0:80, ignored". Whichever block nginx picked, the TLS
# (443, ssl) block had NO location blocks of its own at all -- so
# every HTTPS request (the actual intended access path, and what
# verify_rdm_boot.bash checks) fell through to nginx's own compiled-
# in default root ("/usr/share/nginx/html"), a bare 404, not our
# app. Fixed by collapsing to two server blocks: plain HTTP (80)
# that redirects to HTTPS, and the actual TLS (443) block carrying
# every "location" -- this is the only block that now needs them.
cat <<EOT >"${INSTANCE_NAME}_nginx.conf"
# nginx 1.24's mime.types (Ubuntu) has no entry for .mjs, so pdf.js's seven ES
# modules under /static/js/pdfjs fall through to nginx.conf's
# "default_type application/octet-stream" and Firefox refuses to load them:
# "Loading module from ... was blocked because of a disallowed MIME type
# (application/octet-stream)". Reported live 2026-09-10, after the
# X-Frame-Options fix -- the PDF previewer had two independent faults, and
# fixing the header alone was necessary but not sufficient.
#
# Placed here, at http level, deliberately. nginx.conf's own
# "include /etc/nginx/mime.types" is a types block in this same context, and
# repeated types blocks in one context add to the map. A types block in a
# nested context (server or location) REPLACES the inherited map instead, so
# putting this inside "location /static" would serve every other extension
# there as octet-stream. Verify after any change to this block by curling a
# .mjs *and* a control .css -- see verify_rdm_boot.bash.
#
# .wasm is already mapped by the distro file; .map, .ttf, .bcmap and .xsl are
# not, and are left alone. Only .mjs is fetched in a context that enforces its
# MIME type. Note that nginx's map sends .ts to video/mp2t, which is right for
# a transport stream and wrong for TypeScript -- irrelevant here only because
# nothing loads the .ts sources in the browser.
types {
text/javascript mjs;
}
server {
server_name _;
listen 80;
listen [::]:80;
return 301 https://\$host\$request_uri;
}
server {
server_name _;
include snippets/self-signed.conf;
include snippets/ssl-params.conf;
add_header Strict-Transport-Security "max-age=15768000";
add_header X-Request-ID \$request_id;
listen 443 ssl;
listen [::]:443 ssl;
proxy_request_buffering off;
proxy_buffering off;
proxy_http_version 1.1;
location / {
proxy_pass http://127.0.0.1:5000;
proxy_set_header Host \$host;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto \$scheme;
proxy_set_header X-Request-ID \$request_id;
proxy_hide_header X-Session-ID;
proxy_hide_header X-User-ID;
client_max_body_size 100m;
}
# Real bug found via live testing 2026-07-27 (Step 7 verification):
# every /api/* request 404'd, including paths with confirmed
# indexed data (e.g. /api/vocabularies/languages) -- ruled out
# both "no demo data" and "OpenSearch still indexing" (a 200
# with an empty result, or a 500, would look different; this was
# a bare Werkzeug "no URL rule" 404). Root cause: confirmed live
# that invenio_app.wsgi_rest:application (port 5001, the REST-
# only Granian process) mounts every API blueprint at the root
# ("/records" returns 200 with real data), NOT under "/api" --
# that prefix is nginx's job to strip before proxying, which
# "proxy_pass http://127.0.0.1:5001;" (no trailing slash, so
# nginx forwards the full unmodified URI) never did. A trailing
# slash on proxy_pass makes nginx replace the matched location
# prefix with it instead of preserving the full path.
#
# Superseded 2026-09-10: the prefix strip moved out of nginx and
# into Granian's --url-path-prefix (rdm_rest.service), because
# stripping it here left SCRIPT_NAME empty and every path the REST
# app generated itself came out without "/api" -- a 404 on the
# administration Banners page, among others. No trailing slash on
# proxy_pass now: nginx forwards the full unmodified URI again.
location /api {
proxy_pass http://127.0.0.1:5001;
proxy_set_header Host \$host;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto \$scheme;
proxy_set_header X-Request-ID \$request_id;
proxy_hide_header X-Session-ID;
proxy_hide_header X-User-ID;
client_max_body_size 100m;
}
# This location used to drop the /api/ prefix with a capture
# group, because a regex location can't use proxy_pass's
# trailing-slash prefix-strip trick. With Granian owning the
# strip there is nothing left to rewrite, so the capture group
# is gone. Optional trailing /<part-number> and trailing slash
# match caltechauthors' own nginx-caltechauthors.conf (multipart
# file-content requests carry a numeric part suffix).
location ~ ^/api/records/.+/draft/files/.+/content(?:/[0-9]+)?/?$ {
gzip off;
proxy_pass http://127.0.0.1:5001;
proxy_set_header Host \$host;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto \$scheme;
proxy_set_header X-Request-ID \$request_id;
proxy_hide_header X-Session-ID;
proxy_hide_header X-User-ID;
client_max_body_size 500G;
}
# Static content served directly by nginx, matching this repo's
# own nginx-sites-enabled-default (path reflects uv's venv layout,
# not the old Pipenv virtualenvs path).
location /static {
alias /Sites/${INSTANCE_NAME}/.venv/var/instance/static;
autoindex off;
# nginx serves these itself, so the application never sees them and
# Talisman never gets to set a header on them. A location-level
# add_header REPLACES every inherited one rather than adding to it,
# so the two server-level headers have to be restated here or they
# are silently lost for /static.
add_header Strict-Transport-Security "max-age=15768000";
add_header X-Request-ID \$request_id;
add_header X-Content-Type-Options nosniff;
}
}
EOT
sudo mv -v "${INSTANCE_NAME}_nginx.conf" "/etc/nginx/sites-available/${INSTANCE_NAME}.conf"
sudo chown root:root "/etc/nginx/sites-available/${INSTANCE_NAME}.conf"
sudo chmod 664 "/etc/nginx/sites-available/${INSTANCE_NAME}.conf"
sudo ln -sf "/etc/nginx/sites-available/${INSTANCE_NAME}.conf" "/etc/nginx/sites-enabled/${INSTANCE_NAME}.conf"
sudo rm -f "/etc/nginx/sites-enabled/default"
sudo systemctl restart nginx
echo "Writing .env"
# invenio.cfg has NO Redis/RabbitMQ/OpenSearch settings at all (only
# SQLALCHEMY_DATABASE_URI happens to default to localhost) -- those
# come from invenio-app-rdm's own upstream defaults, which assume
# Docker-network hostnames (cache/mq/search), since the stock
# cookiecutter template assumes the app itself also runs inside
# docker-compose.full.yml. This repo's app runs bare-metal (matching
# production's own topology), with only the backing services
# dockerized (docker-services.yml) and port-mapped to localhost --
# same situation as this repo's own (gitignored) local dev .env,
# whose comment says exactly this ("localhost because we're running
# outside Docker"). Redis DB-index split (0=cache/1=session/
# 2=celery-result/3=ratelimit/4=communities-identities-cache) matches
# cookiecutter-invenio-rdm's own conventional default split; RabbitMQ
# has no custom auth in docker-services.yml, so guest/guest applies.
INVENIO_SECRET_KEY="$(makepasswd -char 64)"
cat <<EOT >.env
INVENIO_SECRET_KEY=${INVENIO_SECRET_KEY}
INVENIO_LOGGING_CONSOLE_LEVEL=INFO
FLASK_ENV=production
INVENIO_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://caltechthesis:caltechthesis@localhost/caltechthesis
INVENIO_CACHE_TYPE=redis
INVENIO_CACHE_REDIS_URL=redis://localhost:6379/0
INVENIO_ACCOUNTS_SESSION_REDIS_URL=redis://localhost:6379/1
INVENIO_CELERY_RESULT_BACKEND=redis://localhost:6379/2
INVENIO_RATELIMIT_STORAGE_URL=redis://localhost:6379/3
INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL=redis://localhost:6379/4
INVENIO_BROKER_URL=amqp://guest:guest@localhost:5672/
INVENIO_CELERY_BROKER_URL=amqp://guest:guest@localhost:5672/
INVENIO_SEARCH_HOSTS=["localhost:9200"]
INVENIO_DATACITE_ENABLED=false
INVENIO_FILES_REST_STORAGE_FACTORY=invenio_files_rest.storage.pyfs_storage_factory
EOT
chmod 600 .env
echo "NOTE: uploads go to local disk on this instance (see the"
echo "INVENIO_FILES_REST_STORAGE_FACTORY line in .env). For production-"
echo "shaped S3 storage, delete that line, set INVENIO_S3_* credentials,"
echo "and create the default location for the real bucket with"
echo "'invenio files location create --default default-location s3://BUCKET'."
echo "Production does not run 'services setup'; it restores data."
echo "Installing this repo's own committed systemd units (rdm.service,"
echo "rdm_rest.service, rdm_celery.service). As of 2026-09-10 those"
echo "units ship with the ubuntu user and /home/ubuntu/.local/bin/uv"
echo "that development instances and production both actually use, so"
echo "the user and uv-path rewrites this step used to apply are gone."
echo "The one adjustment left is the ExecStartPre readiness gate the"
echo "committed units don't carry -- production's Postgres/OpenSearch"
echo "don't race Docker startup the way this single-node"
echo "docker-compose'd test instance does."
for unit in rdm.service rdm_rest.service rdm_celery.service; do
sed -e '/^\[Service\]/a ExecStartPre=/usr/local/bin/wait_for_rdm_services.bash' \
"$unit" | sudo tee "/etc/systemd/system/$unit" >/dev/null
done
sudo chown root:root /etc/systemd/system/rdm.service /etc/systemd/system/rdm_rest.service /etc/systemd/system/rdm_celery.service
sudo chmod 664 /etc/systemd/system/rdm.service /etc/systemd/system/rdm_rest.service /etc/systemd/system/rdm_celery.service
echo "Writing rdm.target (single start/stop/enable point for"
echo "so nginx orders itself after the app units on boot"
cat <<EOT >rdm.target
[Unit]
Description=CaltechTHESIS Granian bring-up (rdm + rdm_rest + rdm_celery + nginx)
Wants=rdm.service rdm_rest.service rdm_celery.service nginx.service
After=rdm.service rdm_rest.service rdm_celery.service nginx.service
[Install]
WantedBy=multi-user.target
EOT
sudo mv -v rdm.target /etc/systemd/system/
sudo chown root:root /etc/systemd/system/rdm.target
sudo chmod 664 /etc/systemd/system/rdm.target
sudo mkdir -p /etc/systemd/system/nginx.service.d
cat <<EOT | sudo tee /etc/systemd/system/nginx.service.d/rdm-ordering.conf >/dev/null
[Unit]
# Soft ordering only (no Requires=) -- nginx doesn't need the app up
# to itself start, this just avoids confusing early 502s on boot.
After=rdm.service rdm_rest.service
# PartOf= so `systemctl stop rdm.target` takes nginx down too.
PartOf=rdm.target
EOT
sudo systemctl daemon-reload
sudo systemctl enable rdm rdm_rest rdm_celery rdm.target
cat <<EOT
CaltechTHESIS Granian test instance deployed at /Sites/${INSTANCE_NAME}
Python: 3.14 (via uv) | Node: 26 | Branch: ${BRANCH}
Postgres/OpenSearch: local Docker volumes, freshly initialised by
'invenio-cli services setup --no-demo-data' -- empty of records.
The static pages are the placeholders from app_data/pages/.
File storage: local disk (testing). Production uses S3 -- see the
NOTE printed when .env was written.
cd /Sites/${INSTANCE_NAME}
sudo systemctl start rdm.target # or: start rdm_celery rdm_rest rdm nginx
Check:
invenio-cli services status
sudo systemctl status rdm.target
/usr/local/bin/verify_rdm_boot.bash # full acceptance check, esp. after a reboot
Stop:
sudo systemctl stop rdm.target
invenio-cli services stop
EOT
path: /usr/local/bin/setup_caltechthesis_granian.bash
owner: 'root:root'
permissions: '0775'