ar(feat) dms: envs and cleanup

.gitignore

@@ -2,6 +2,7 @@
 .env
 *.private
 
+kubesec/**/*.private
 
 _*/
 matrix/*

ingress/_docker-compose.yml

@@ -37,7 +37,7 @@ services:
       CF_API_KEY: ${HERMES_CLOUDFLARE_API_KEY}
     volumes:
       - "../certs/data:/etc/certs/:ro"
-      - "${PWD}/certs/acme/acme.json:/etc/traefik/acme/acme.json"
+      - "${PWD}/certs/acme/acme.json:/etc/traefik/acme/acme.json:rw"
       - "./data:/data"
       - "./traefik.yml:/etc/traefik/traefik.yml:ro"
       - "./dynamic.yml:/etc/traefik/dynamic.yml:ro"

.env.common.public → kubesec/next/.env.common.public

@@ -52,7 +52,7 @@ HERMES_MODEL_CLOUD_GATEWAY=123.123.6.248
 HERMES_REMOTE=
 HERMES_REMOTE_ROOT=/home/
 HERMES_REMOTE_HOME=/home/
-HERMES_BRANCH=
+HERMES_REMOTE_BRANCH=
 HERMES_PORT_PREFIX=77
 HERMES_CLOUD_DB_HOST=hermes-model-mariadb:7717
 HERMES_ENV=local

kubesec/prod/.env.auth.public

@@ -0,0 +1,5 @@
+# domquixote@example.org, you@example.org
+HERMES_LDAP_ADMIN_USER=
+HERMES_LDAP_ADMIN_PASS=
+HERMES_FIRST_USER=
+HERMES_FIRST_USER_PASS=

kubesec/prod/.env.chat.public

@@ -0,0 +1,25 @@
+# external storage volume (Vultr, S3, GCP Buckets, etc)
+HERMES_CHAT_S3_BUCKET=
+HERMES_CHAT_S3_REGION=
+HERMES_CHAT_S3_URL=
+HERMES_CHAT_S3_KEY=
+HERMES_CHAT_S3_SECRET=
+
+# postgres
+HERMES_DB_USER=
+HERMES_DB_PASS=
+
+# passwords
+HERMES_SECRET_GUEST=
+HERMES_SECRET_FORM=
+HERMES_SECRET_CORE=
+
+# stmp
+HERMES_CHAT_EMAIL_HOST=
+HERMES_CHAT_EMAIL_PORT=
+HERMES_CHAT_EMAIL_USER=
+HERMES_CHAT_EMAIL_PASS=
+HERMES_CHAT_EMAIL_FROM=
+
+# legacy
+HERMES_MATRIX_INTERNAL_PORT=8408

kubesec/prod/.env.cloud.public

@@ -0,0 +1,22 @@
+# if subpath mounted
+HERMES_CLOUD_BASEPATH=/cloud
+
+# external storage volume (Vultr, S3, GCP Buckets, etc)
+HERMES_CLOUD_S3_BUCKET=
+HERMES_CLOUD_S3_REGION=
+HERMES_CLOUD_S3_HOST=
+HERMES_CLOUD_S3_KEY=
+HERMES_CLOUD_S3_SECRET=
+
+# nextcloud maria db
+HERMES_CLOUD_DB_USER=
+HERMES_CLOUD_DB_PASS=
+HERMES_CLOUD_DB_NAME=nextcloud
+HERMES_CLOUD_DB_REDIS=redis
+
+# collabora user
+HERMES_CLOUD_OFFICE_PASS=
+HERMES_CLOUD_OFFICE_USER=
+
+# agnostic
+HERMES_CLOUD_LOG_PATH=/cloud/data/cloud.log

kubesec/prod/.env.common.public

@@ -0,0 +1,64 @@
+# fqdn
+HERMES_APEX=example.org
+HERMES_HOSTNAME=domain.example.org
+HERMES_CERT_FILE=domain.example.org
+HERMES_CERT_KEY_FILE=domain.example.org
+HERMES_ROOT_HANDLE=domain.example.org
+HERMES_SERVER_NAME=domain.example.org
+HERMES_CLOUD_OFFICE_HOSTNAME=domain.example.org
+HERMES_GRAFANA_URL=domain.example.org
+HERMES_TRAEFIK_URL=domain.example.org
+HERMES_PROM_URL=domain.example.org
+
+HERMES_MAIL_DOMAINS=example.org,example.com
+
+# superuser domquixote@example.org
+HERMES_ROOT_HANDLE=domquixote
+
+HERMES_MAIL_MAIN_HOSTNAME=domain.example.org
+
+# admin emails
+HERMES_INGRESS_SRE_ALERT_EMAIL=sre@example.org
+
+# ingress ip: main public ip (wan/lan)
+HERMES_IP=
+
+# tls
+HERMES_TLS=true
+
+# dns/resolver
+HERMES_DNS_RESOLVER=123.123.0.248
+
+HERMES_INGRESS_SUBNET=123.123.1.0/24
+HERMES_INGRESS_GATEWAY=123.123.1.248
+
+HERMES_MAIL_SUBNET=123.123.2.0/24
+HERMES_MAIL_GATEWAY=123.123.2.248
+
+HERMES_CLOUD_SUBNET=123.123.3.0/24
+HERMES_CLOUD_GATEWAY=123.123.3.248
+
+HERMES_CHAT_SUBNET=123.123.4.0/24
+HERMES_CHAT_GATEWAY=123.123.4.248
+
+# sensitive
+HERMES_MODEL_CHAT_SUBNET=123.123.5.0/24
+HERMES_MODEL_CHAT_GATEWAY=123.123.5.248
+
+HERMES_MODEL_CLOUD_SUBNET=123.123.6.0/24
+HERMES_MODEL_CLOUD_GATEWAY=123.123.6.248
+
+# experimental features (should probably be left as is)
+HERMES_REMOTE=
+HERMES_REMOTE_ROOT=/home/
+HERMES_REMOTE_HOME=/home/
+HERMES_REMOTE_BRANCH=
+HERMES_PORT_PREFIX=77
+HERMES_CLOUD_DB_HOST=hermes-model-mariadb:7717
+HERMES_ENV=local
+HERMES_COOLDOWN_POINTER=1
+
+
+# tmp: later LDAP and csv lists
+# ALL DOMAINS SHARE THE SAME ALIASES IN THIS SETUP
+HERMES_DOMAIN_ALIASES=john,jane

kubesec/prod/.env.ingress.public

@@ -0,0 +1,17 @@
+HERMES_CERT_RESOLVER=letsencrypt
+HERMES_CERT_FILE=
+HERMES_CERT_KEY_FILE=
+HERMES_ALT_CERT_FILE=
+HERMES_ALT_CERT_KEY_FILE=
+HERMES_CERT_STORAGE_FILENAME=acme.json
+HERMES_CERT_STORAGE_DIRNAME=./data/
+
+HERMES_PROM_VOLUME=./data/prom
+
+HERMES_INGRESS_STMP_FROM=mailserver
+
+# passwords
+HERMES_TRAEFIK_BASE_AUTH=
+
+HERMES_CLOUDFLARE_EMAIL=
+HERMES_CLOUDFLARE_API_KEY=

kubesec/prod/.env.mail.public

@@ -0,0 +1,9 @@
+# nothing specific yet
+HERMES_MAIN_MAILBOX=
+HERMES_MAIL_INITIAL_BOXES=
+HERMES_MAIL_INITIAL_BOXES_DEFAULT_PASSWORD=
+HERMES_MAIL_CERT_TYPE=manual
+HERMES_MAIL_S3_BUCKET=
+HERMES_MAIL_S3_HOST=
+HERMES_MAIL_S3_KEY=
+HERMES_MAIL_S3_SECRET=

kubesec/prod/.env.model.public

@@ -0,0 +1,2 @@
+# chat db networking
+HERMES_MODEL_DB_FIXED_IP=

deploy-secrets.sh → scripts/deploy-secrets.sh

@@ -45,13 +45,22 @@ take () {
 
 log "dp::hermes::ci::(busy):: Deploying secrets." 2
 
-if [ "$1" == "install" ]; then
+if [ "$1" == "install:next" ]; then
 	log "dp::hermes::ci::(busy):: Sending to installation dir."
+	cd kubesec/next
   scp .env.*.private $HERMES_REMOTE:$HERMES_REMOTE_HOME
-else
+elif [ "$1" == "install:prod" ]; then
 	log "dp::hermes::ci::(busy):: Sending to hermes dir."
+	cd kubesec/prod
+  scp .env.*.private $HERMES_REMOTE:$HERMES_REMOTE_HOME
+else
+	log "dp::hermes::ci::(busy):: Please specify an environment (first flag) Copying from staging instead."
+	cd kubesec/next
   scp .env.*.private $HERMES_REMOTE:$HERMES_REMOTE_ROOT
 fi
+cd $root_dir
 
+log "dp::hermes::${HERMES_ENV}::ingress::(busy):: Copying .PEM certificates (CA)."
+scp ./.env.*cert*.private $HERMES_REMOTE:$HERMES_REMOTE_HOME
 
 log "dp::hermes::ci::(idle)::all good." 0

init-mail.sh → scripts/init-mail.sh

@@ -57,6 +57,12 @@ setup_mailbox () {
 		log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Mailboxes created." 0
 }
 
+setup_aliases () {
+		log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Mailbox Setup: Creating initial aliases." 2
+		docker exec -it hermes-mail-mailserver setup alias add $1@$HERMES_MAIL_MAIN_HOSTNAME $2
+		log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Mailboxes created." 0
+}
+
 setup_storage () {
 		log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Mailbox Setup: Preparing cloud email storage." 2
 		echo $HERMES_MAIL_S3_KEY:$HERMES_MAIL_S3_SECRET > ~/.passwd-s3fs
@@ -94,6 +100,8 @@ fi
 take 5 "dp::hermes::mail::(busy):: Launching Docker Compose Swarms."
 
 cd mail
+mkdir mail/data
+mkdir mail/data/email-data
 docker compose up -d
 cd $root_dir
 
@@ -117,6 +125,10 @@ if [ "$1" == "setup:mailboxes" ]; then
 	for box in ${HERMES_MAIL_INITIAL_BOXES//,/ }
 	do
 	    setup_mailbox $box $HERMES_MAIL_INITIAL_BOXES_DEFAULT_PASSWORD
+	    for alias in ${HERMES_DOMAIN_ALIASES//,/ }
+	    do
+	    	setup_aliases $alias $box
+	    done
 	done
 else
 	log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Skipping Mailboxes setup."

init-weagle.sh → scripts/init-weagle.sh

@@ -76,6 +76,7 @@ tmpfile=$(mktemp --tmpdir=.)
 cp -p $origin $tmpfile
 cat $origin | envsubst > $tmpfile && mv $tmpfile $destination
 
+for f in *.cert.* ; do mv -- "$f" ".env.$f" ; done
 
 # dir setup
 log "dp::hermes::${HERMES_ENV}::ingress::(busy):: Adding .PEM certificates (CA)." 2
@@ -86,6 +87,8 @@ for file in .env.cert.*; do cp -a "$file" "${file#.env.cert.}";done;
 cd $root_dir
 
 take 5 "dp::hermes::${HERMES_ENV}::ingress::(busy):: Launching Docker Compose Swarms."
+mkdir certs/acme
+touch certs/acme/acme.json
 chmod 600 certs/acme/acme.json
 cd ingress
 docker compose up -d

init.sh → scripts/init.sh

@@ -45,25 +45,25 @@ take () {
 
 # docker setup
 log "dp::hermes::init::(busy):: Configuring networks."
-./init-networks.sh
+./scripts/init-networks.sh
 
 
 # docker setup
 log "dp::hermes::${HERMES_ENV}::init::(busy)::Initiating ${HERMES_ENV} env."
 
 log "dp::hermes::${HERMES_ENV}::init::(busy)::Starting: Weagle (Ingress)." 2
-./init-weagle.sh
+./scripts/init-weagle.sh
 
 log "dp::hermes::${HERMES_ENV}::init::(busy)::Starting: Daegis (Databases)." 2
-./init-model.sh
+./scripts/init-model.sh
 
 # log "dp::hermes::${HERMES_ENV}::init::(busy)::Starting: Drew: (Auth)." 2
-# ./init-auth.sh
+# ./scripts/init-auth.sh
 
 log "dp::hermes::${HERMES_ENV}::init::(busy)::Starting: Aemilia: (Mail)." 2
-./init-mail.sh
+./scripts/init-mail.sh
 
 log "dp::hermes::${HERMES_ENV}::init::(busy)::Starting: Claudia: (Storage, Calendar, MWC)." 2
-./init-cloud.sh
+./scripts/init-cloud.sh
 
 log "dp::hermes::${HERMES_ENV}::init::(idle)::all good." 0

init6.sh → scripts/init6.sh

@@ -46,7 +46,7 @@ take () {
 log "dp::hermes::${HERMES_ENV}::(busy):: Reboot. Are you sure?."
 take 10 "dp::hermes::${HERMES_ENV}::reboot::(busy):: Gracefully rebooting: shutting down ${HERMES_ENV}."
 
-./init0.sh
+./scripts/init0.sh
 
 if [ "$1" == "wipe" ]; then
 	take 5 "dp::hermes::${HERMES_ENV}::(busy):: WARNING, WIPING ALL DATA."
@@ -55,7 +55,7 @@ fi
 
 log "dp::hermes::${HERMES_ENV}::reboot::(busy):: Gracefully rebooting: booting up ${HERMES_ENV}."
 
-./init.sh
+./scripts/init.sh
 
 
 log "dp::hermes::${HERMES_ENV}::reboot::(idle)::all good." 0

install-deps.sh → scripts/install-deps.sh

@@ -44,7 +44,7 @@ take () {
 }
 
 log "dp::hermes::ci::(busy):: Installing dependencies: Deploying secrets." 2
-if [[ "$(which pv)" == "" || "$(which docker)" == "" || "$(which s3fs)" == ""]]; then
+if [[ "$(which pv)" == "" || "$(which docker)" == "" || "$(which s3fs)" == "" ]]; then
         if [ "$(uname)" == "Darwin" ]; then
                         log "dp::hermes::ci::(busy)::installing for MacOS."
                         brew install docker --cask

install.sh → scripts/install.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 # init install
 echo -e "\033[0;62m\033[0;49;35m"
-set -a && source .env.common.private && set +a
 root_dir="$(pwd)"
 
 log () {
@@ -43,8 +42,19 @@ take () {
 	while true; do echo -n .; sleep 1; done | pv -s $1  -S -F '%t %p' > /dev/null
 }
 
-log "dp::hermes::ci::(busy):: Installing on server: Deploying secrets."
-./deploy-secrets.sh install
+if [ "$1" == "install:prod" ]; then
+	cp kubesec/prod/.env.*.private .
+	log "dp::hermes::ci::(busy):: Installing on prod server: Deploying secrets."
+	./scripts/deploy-secrets.sh install:prod
+elif [ "$1" == "install:next" ]; then
+	cp kubesec/next/.env.*.private .
+	log "dp::hermes::ci::(busy):: Installing on next server: Deploying secrets."
+	./scripts/deploy-secrets.sh install:next
+else
+	log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Skipping installation, pleace specify environment."
+fi
+
+set -a && source .env.common.private && set +a
 
 # sudo rm -rf /var/lib/rancher/k3s/server/manifests/traefik.yaml
 # helm uninstall traefik traefik-crd -n kube-system
@@ -57,10 +67,10 @@ ssh ${HERMES_REMOTE} "mkdir dp; \
 											git clone https://github.com/dreampipcom/hermes.git; \
 											mv ../.env.*.private hermes/; \
 											cd hermes; \
-											git checkout ${HERMES_BRANCH}; \
+											git checkout ${HERMES_REMOTE_BRANCH}; \
 											git pull; \
-											chmod +x ./install-deps.sh; \
-											./install-deps.sh;
+											chmod +x ./scripts/install-deps.sh; \
+											./scripts/install-deps.sh;
 											"