diff --git a/.gitignore b/.gitignore index 6d66c63..0582782 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .env *.private +kubesec/**/*.private _*/ matrix/* diff --git a/ingress/_docker-compose.yml b/ingress/_docker-compose.yml index 092f039..68dd6b6 100644 --- a/ingress/_docker-compose.yml +++ b/ingress/_docker-compose.yml @@ -37,7 +37,7 @@ services: CF_API_KEY: ${HERMES_CLOUDFLARE_API_KEY} volumes: - "../certs/data:/etc/certs/:ro" - - "${PWD}/certs/acme/acme.json:/etc/traefik/acme/acme.json" + - "${PWD}/certs/acme/acme.json:/etc/traefik/acme/acme.json:rw" - "./data:/data" - "./traefik.yml:/etc/traefik/traefik.yml:ro" - "./dynamic.yml:/etc/traefik/dynamic.yml:ro" diff --git a/.env.auth.public b/kubesec/next/.env.auth.public similarity index 100% rename from .env.auth.public rename to kubesec/next/.env.auth.public diff --git a/.env.chat.public b/kubesec/next/.env.chat.public similarity index 100% rename from .env.chat.public rename to kubesec/next/.env.chat.public diff --git a/.env.cloud.public b/kubesec/next/.env.cloud.public similarity index 100% rename from .env.cloud.public rename to kubesec/next/.env.cloud.public diff --git a/.env.common.public b/kubesec/next/.env.common.public similarity index 98% rename from .env.common.public rename to kubesec/next/.env.common.public index c3fcb6e..95ee42c 100644 --- a/.env.common.public +++ b/kubesec/next/.env.common.public @@ -52,7 +52,7 @@ HERMES_MODEL_CLOUD_GATEWAY=123.123.6.248 HERMES_REMOTE= HERMES_REMOTE_ROOT=/home/ HERMES_REMOTE_HOME=/home/ -HERMES_BRANCH= +HERMES_REMOTE_BRANCH= HERMES_PORT_PREFIX=77 HERMES_CLOUD_DB_HOST=hermes-model-mariadb:7717 HERMES_ENV=local diff --git a/.env.ingress.public b/kubesec/next/.env.ingress.public similarity index 100% rename from .env.ingress.public rename to kubesec/next/.env.ingress.public diff --git a/.env.mail.public b/kubesec/next/.env.mail.public similarity index 100% rename from .env.mail.public rename to kubesec/next/.env.mail.public diff --git a/.env.model.public b/kubesec/next/.env.model.public similarity index 100% rename from .env.model.public rename to kubesec/next/.env.model.public diff --git a/kubesec/prod/.env.auth.public b/kubesec/prod/.env.auth.public new file mode 100644 index 0000000..01ae770 --- /dev/null +++ b/kubesec/prod/.env.auth.public @@ -0,0 +1,5 @@ +# domquixote@example.org, you@example.org +HERMES_LDAP_ADMIN_USER= +HERMES_LDAP_ADMIN_PASS= +HERMES_FIRST_USER= +HERMES_FIRST_USER_PASS= \ No newline at end of file diff --git a/kubesec/prod/.env.chat.public b/kubesec/prod/.env.chat.public new file mode 100644 index 0000000..f1b7255 --- /dev/null +++ b/kubesec/prod/.env.chat.public @@ -0,0 +1,25 @@ +# external storage volume (Vultr, S3, GCP Buckets, etc) +HERMES_CHAT_S3_BUCKET= +HERMES_CHAT_S3_REGION= +HERMES_CHAT_S3_URL= +HERMES_CHAT_S3_KEY= +HERMES_CHAT_S3_SECRET= + +# postgres +HERMES_DB_USER= +HERMES_DB_PASS= + +# passwords +HERMES_SECRET_GUEST= +HERMES_SECRET_FORM= +HERMES_SECRET_CORE= + +# stmp +HERMES_CHAT_EMAIL_HOST= +HERMES_CHAT_EMAIL_PORT= +HERMES_CHAT_EMAIL_USER= +HERMES_CHAT_EMAIL_PASS= +HERMES_CHAT_EMAIL_FROM= + +# legacy +HERMES_MATRIX_INTERNAL_PORT=8408 \ No newline at end of file diff --git a/kubesec/prod/.env.cloud.public b/kubesec/prod/.env.cloud.public new file mode 100644 index 0000000..ace783f --- /dev/null +++ b/kubesec/prod/.env.cloud.public @@ -0,0 +1,22 @@ +# if subpath mounted +HERMES_CLOUD_BASEPATH=/cloud + +# external storage volume (Vultr, S3, GCP Buckets, etc) +HERMES_CLOUD_S3_BUCKET= +HERMES_CLOUD_S3_REGION= +HERMES_CLOUD_S3_HOST= +HERMES_CLOUD_S3_KEY= +HERMES_CLOUD_S3_SECRET= + +# nextcloud maria db +HERMES_CLOUD_DB_USER= +HERMES_CLOUD_DB_PASS= +HERMES_CLOUD_DB_NAME=nextcloud +HERMES_CLOUD_DB_REDIS=redis + +# collabora user +HERMES_CLOUD_OFFICE_PASS= +HERMES_CLOUD_OFFICE_USER= + +# agnostic +HERMES_CLOUD_LOG_PATH=/cloud/data/cloud.log \ No newline at end of file diff --git a/kubesec/prod/.env.common.public b/kubesec/prod/.env.common.public new file mode 100644 index 0000000..33ee25c --- /dev/null +++ b/kubesec/prod/.env.common.public @@ -0,0 +1,64 @@ +# fqdn +HERMES_APEX=example.org +HERMES_HOSTNAME=domain.example.org +HERMES_CERT_FILE=domain.example.org +HERMES_CERT_KEY_FILE=domain.example.org +HERMES_ROOT_HANDLE=domain.example.org +HERMES_SERVER_NAME=domain.example.org +HERMES_CLOUD_OFFICE_HOSTNAME=domain.example.org +HERMES_GRAFANA_URL=domain.example.org +HERMES_TRAEFIK_URL=domain.example.org +HERMES_PROM_URL=domain.example.org + +HERMES_MAIL_DOMAINS=example.org,example.com + +# superuser domquixote@example.org +HERMES_ROOT_HANDLE=domquixote + +HERMES_MAIL_MAIN_HOSTNAME=domain.example.org + +# admin emails +HERMES_INGRESS_SRE_ALERT_EMAIL=sre@example.org + +# ingress ip: main public ip (wan/lan) +HERMES_IP= + +# tls +HERMES_TLS=true + +# dns/resolver +HERMES_DNS_RESOLVER=123.123.0.248 + +HERMES_INGRESS_SUBNET=123.123.1.0/24 +HERMES_INGRESS_GATEWAY=123.123.1.248 + +HERMES_MAIL_SUBNET=123.123.2.0/24 +HERMES_MAIL_GATEWAY=123.123.2.248 + +HERMES_CLOUD_SUBNET=123.123.3.0/24 +HERMES_CLOUD_GATEWAY=123.123.3.248 + +HERMES_CHAT_SUBNET=123.123.4.0/24 +HERMES_CHAT_GATEWAY=123.123.4.248 + +# sensitive +HERMES_MODEL_CHAT_SUBNET=123.123.5.0/24 +HERMES_MODEL_CHAT_GATEWAY=123.123.5.248 + +HERMES_MODEL_CLOUD_SUBNET=123.123.6.0/24 +HERMES_MODEL_CLOUD_GATEWAY=123.123.6.248 + +# experimental features (should probably be left as is) +HERMES_REMOTE= +HERMES_REMOTE_ROOT=/home/ +HERMES_REMOTE_HOME=/home/ +HERMES_REMOTE_BRANCH= +HERMES_PORT_PREFIX=77 +HERMES_CLOUD_DB_HOST=hermes-model-mariadb:7717 +HERMES_ENV=local +HERMES_COOLDOWN_POINTER=1 + + +# tmp: later LDAP and csv lists +# ALL DOMAINS SHARE THE SAME ALIASES IN THIS SETUP +HERMES_DOMAIN_ALIASES=john,jane \ No newline at end of file diff --git a/kubesec/prod/.env.ingress.public b/kubesec/prod/.env.ingress.public new file mode 100644 index 0000000..05192c9 --- /dev/null +++ b/kubesec/prod/.env.ingress.public @@ -0,0 +1,17 @@ +HERMES_CERT_RESOLVER=letsencrypt +HERMES_CERT_FILE= +HERMES_CERT_KEY_FILE= +HERMES_ALT_CERT_FILE= +HERMES_ALT_CERT_KEY_FILE= +HERMES_CERT_STORAGE_FILENAME=acme.json +HERMES_CERT_STORAGE_DIRNAME=./data/ + +HERMES_PROM_VOLUME=./data/prom + +HERMES_INGRESS_STMP_FROM=mailserver + +# passwords +HERMES_TRAEFIK_BASE_AUTH= + +HERMES_CLOUDFLARE_EMAIL= +HERMES_CLOUDFLARE_API_KEY= \ No newline at end of file diff --git a/kubesec/prod/.env.mail.public b/kubesec/prod/.env.mail.public new file mode 100644 index 0000000..8b8cb2c --- /dev/null +++ b/kubesec/prod/.env.mail.public @@ -0,0 +1,9 @@ +# nothing specific yet +HERMES_MAIN_MAILBOX= +HERMES_MAIL_INITIAL_BOXES= +HERMES_MAIL_INITIAL_BOXES_DEFAULT_PASSWORD= +HERMES_MAIL_CERT_TYPE=manual +HERMES_MAIL_S3_BUCKET= +HERMES_MAIL_S3_HOST= +HERMES_MAIL_S3_KEY= +HERMES_MAIL_S3_SECRET= \ No newline at end of file diff --git a/kubesec/prod/.env.model.public b/kubesec/prod/.env.model.public new file mode 100644 index 0000000..05f2a43 --- /dev/null +++ b/kubesec/prod/.env.model.public @@ -0,0 +1,2 @@ +# chat db networking +HERMES_MODEL_DB_FIXED_IP= \ No newline at end of file diff --git a/config.sh b/scripts/config.sh similarity index 100% rename from config.sh rename to scripts/config.sh diff --git a/deploy-secrets.sh b/scripts/deploy-secrets.sh similarity index 72% rename from deploy-secrets.sh rename to scripts/deploy-secrets.sh index 5bf6915..f863f50 100755 --- a/deploy-secrets.sh +++ b/scripts/deploy-secrets.sh @@ -45,13 +45,22 @@ take () { log "dp::hermes::ci::(busy):: Deploying secrets." 2 -if [ "$1" == "install" ]; then +if [ "$1" == "install:next" ]; then log "dp::hermes::ci::(busy):: Sending to installation dir." + cd kubesec/next scp .env.*.private $HERMES_REMOTE:$HERMES_REMOTE_HOME -else +elif [ "$1" == "install:prod" ]; then log "dp::hermes::ci::(busy):: Sending to hermes dir." + cd kubesec/prod + scp .env.*.private $HERMES_REMOTE:$HERMES_REMOTE_HOME +else + log "dp::hermes::ci::(busy):: Please specify an environment (first flag) Copying from staging instead." + cd kubesec/next scp .env.*.private $HERMES_REMOTE:$HERMES_REMOTE_ROOT fi +cd $root_dir +log "dp::hermes::${HERMES_ENV}::ingress::(busy):: Copying .PEM certificates (CA)." +scp ./.env.*cert*.private $HERMES_REMOTE:$HERMES_REMOTE_HOME log "dp::hermes::ci::(idle)::all good." 0 diff --git a/init-auth.sh b/scripts/init-auth.sh similarity index 100% rename from init-auth.sh rename to scripts/init-auth.sh diff --git a/init-chat.sh b/scripts/init-chat.sh similarity index 100% rename from init-chat.sh rename to scripts/init-chat.sh diff --git a/init-cloud.sh b/scripts/init-cloud.sh similarity index 100% rename from init-cloud.sh rename to scripts/init-cloud.sh diff --git a/init-k8s-config.sh b/scripts/init-k8s-config.sh similarity index 100% rename from init-k8s-config.sh rename to scripts/init-k8s-config.sh diff --git a/init-local.sh b/scripts/init-local.sh similarity index 100% rename from init-local.sh rename to scripts/init-local.sh diff --git a/init-mail.sh b/scripts/init-mail.sh similarity index 89% rename from init-mail.sh rename to scripts/init-mail.sh index ff1ff7b..4285eb6 100755 --- a/init-mail.sh +++ b/scripts/init-mail.sh @@ -57,6 +57,12 @@ setup_mailbox () { log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Mailboxes created." 0 } +setup_aliases () { + log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Mailbox Setup: Creating initial aliases." 2 + docker exec -it hermes-mail-mailserver setup alias add $1@$HERMES_MAIL_MAIN_HOSTNAME $2 + log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Mailboxes created." 0 +} + setup_storage () { log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Mailbox Setup: Preparing cloud email storage." 2 echo $HERMES_MAIL_S3_KEY:$HERMES_MAIL_S3_SECRET > ~/.passwd-s3fs @@ -94,6 +100,8 @@ fi take 5 "dp::hermes::mail::(busy):: Launching Docker Compose Swarms." cd mail +mkdir mail/data +mkdir mail/data/email-data docker compose up -d cd $root_dir @@ -117,6 +125,10 @@ if [ "$1" == "setup:mailboxes" ]; then for box in ${HERMES_MAIL_INITIAL_BOXES//,/ } do setup_mailbox $box $HERMES_MAIL_INITIAL_BOXES_DEFAULT_PASSWORD + for alias in ${HERMES_DOMAIN_ALIASES//,/ } + do + setup_aliases $alias $box + done done else log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Skipping Mailboxes setup." diff --git a/init-model.sh b/scripts/init-model.sh similarity index 100% rename from init-model.sh rename to scripts/init-model.sh diff --git a/init-networks.sh b/scripts/init-networks.sh similarity index 100% rename from init-networks.sh rename to scripts/init-networks.sh diff --git a/init-weagle-ca.sh b/scripts/init-weagle-ca.sh similarity index 100% rename from init-weagle-ca.sh rename to scripts/init-weagle-ca.sh diff --git a/init-weagle-local.sh b/scripts/init-weagle-local.sh similarity index 100% rename from init-weagle-local.sh rename to scripts/init-weagle-local.sh diff --git a/init-weagle.sh b/scripts/init-weagle.sh similarity index 96% rename from init-weagle.sh rename to scripts/init-weagle.sh index 6e0557d..82e52b5 100644 --- a/init-weagle.sh +++ b/scripts/init-weagle.sh @@ -76,6 +76,7 @@ tmpfile=$(mktemp --tmpdir=.) cp -p $origin $tmpfile cat $origin | envsubst > $tmpfile && mv $tmpfile $destination +for f in *.cert.* ; do mv -- "$f" ".env.$f" ; done # dir setup log "dp::hermes::${HERMES_ENV}::ingress::(busy):: Adding .PEM certificates (CA)." 2 @@ -86,6 +87,8 @@ for file in .env.cert.*; do cp -a "$file" "${file#.env.cert.}";done; cd $root_dir take 5 "dp::hermes::${HERMES_ENV}::ingress::(busy):: Launching Docker Compose Swarms." +mkdir certs/acme +touch certs/acme/acme.json chmod 600 certs/acme/acme.json cd ingress docker compose up -d diff --git a/init.sh b/scripts/init.sh similarity index 91% rename from init.sh rename to scripts/init.sh index 9ab82ee..3a9ad89 100755 --- a/init.sh +++ b/scripts/init.sh @@ -45,25 +45,25 @@ take () { # docker setup log "dp::hermes::init::(busy):: Configuring networks." -./init-networks.sh +./scripts/init-networks.sh # docker setup log "dp::hermes::${HERMES_ENV}::init::(busy)::Initiating ${HERMES_ENV} env." log "dp::hermes::${HERMES_ENV}::init::(busy)::Starting: Weagle (Ingress)." 2 -./init-weagle.sh +./scripts/init-weagle.sh log "dp::hermes::${HERMES_ENV}::init::(busy)::Starting: Daegis (Databases)." 2 -./init-model.sh +./scripts/init-model.sh # log "dp::hermes::${HERMES_ENV}::init::(busy)::Starting: Drew: (Auth)." 2 -# ./init-auth.sh +# ./scripts/init-auth.sh log "dp::hermes::${HERMES_ENV}::init::(busy)::Starting: Aemilia: (Mail)." 2 -./init-mail.sh +./scripts/init-mail.sh log "dp::hermes::${HERMES_ENV}::init::(busy)::Starting: Claudia: (Storage, Calendar, MWC)." 2 -./init-cloud.sh +./scripts/init-cloud.sh log "dp::hermes::${HERMES_ENV}::init::(idle)::all good." 0 diff --git a/init0.sh b/scripts/init0.sh similarity index 100% rename from init0.sh rename to scripts/init0.sh diff --git a/init6.sh b/scripts/init6.sh similarity index 97% rename from init6.sh rename to scripts/init6.sh index dd6b19c..5102060 100644 --- a/init6.sh +++ b/scripts/init6.sh @@ -46,7 +46,7 @@ take () { log "dp::hermes::${HERMES_ENV}::(busy):: Reboot. Are you sure?." take 10 "dp::hermes::${HERMES_ENV}::reboot::(busy):: Gracefully rebooting: shutting down ${HERMES_ENV}." -./init0.sh +./scripts/init0.sh if [ "$1" == "wipe" ]; then take 5 "dp::hermes::${HERMES_ENV}::(busy):: WARNING, WIPING ALL DATA." @@ -55,7 +55,7 @@ fi log "dp::hermes::${HERMES_ENV}::reboot::(busy):: Gracefully rebooting: booting up ${HERMES_ENV}." -./init.sh +./scripts/init.sh log "dp::hermes::${HERMES_ENV}::reboot::(idle)::all good." 0 diff --git a/install-deps.sh b/scripts/install-deps.sh similarity index 99% rename from install-deps.sh rename to scripts/install-deps.sh index 12fbbbb..f0b7ff3 100644 --- a/install-deps.sh +++ b/scripts/install-deps.sh @@ -44,7 +44,7 @@ take () { } log "dp::hermes::ci::(busy):: Installing dependencies: Deploying secrets." 2 -if [[ "$(which pv)" == "" || "$(which docker)" == "" || "$(which s3fs)" == ""]]; then +if [[ "$(which pv)" == "" || "$(which docker)" == "" || "$(which s3fs)" == "" ]]; then if [ "$(uname)" == "Darwin" ]; then log "dp::hermes::ci::(busy)::installing for MacOS." brew install docker --cask diff --git a/install.sh b/scripts/install.sh similarity index 67% rename from install.sh rename to scripts/install.sh index 6779adb..bf6b92d 100755 --- a/install.sh +++ b/scripts/install.sh @@ -1,7 +1,6 @@ #!/bin/bash # init install echo -e "\033[0;62m\033[0;49;35m" -set -a && source .env.common.private && set +a root_dir="$(pwd)" log () { @@ -43,8 +42,19 @@ take () { while true; do echo -n .; sleep 1; done | pv -s $1 -S -F '%t %p' > /dev/null } -log "dp::hermes::ci::(busy):: Installing on server: Deploying secrets." -./deploy-secrets.sh install +if [ "$1" == "install:prod" ]; then + cp kubesec/prod/.env.*.private . + log "dp::hermes::ci::(busy):: Installing on prod server: Deploying secrets." + ./scripts/deploy-secrets.sh install:prod +elif [ "$1" == "install:next" ]; then + cp kubesec/next/.env.*.private . + log "dp::hermes::ci::(busy):: Installing on next server: Deploying secrets." + ./scripts/deploy-secrets.sh install:next +else + log "dp::hermes::mail::(busy):: Preparing Aemilia (Mail: DMS): Skipping installation, pleace specify environment." +fi + +set -a && source .env.common.private && set +a # sudo rm -rf /var/lib/rancher/k3s/server/manifests/traefik.yaml # helm uninstall traefik traefik-crd -n kube-system @@ -57,10 +67,10 @@ ssh ${HERMES_REMOTE} "mkdir dp; \ git clone https://github.com/dreampipcom/hermes.git; \ mv ../.env.*.private hermes/; \ cd hermes; \ - git checkout ${HERMES_BRANCH}; \ + git checkout ${HERMES_REMOTE_BRANCH}; \ git pull; \ - chmod +x ./install-deps.sh; \ - ./install-deps.sh; + chmod +x ./scripts/install-deps.sh; \ + ./scripts/install-deps.sh; "