From c150d7446f3ab54afab51178833235f7edff1d28 Mon Sep 17 00:00:00 2001 From: Tyler Chong Date: Mon, 17 Jun 2024 18:18:52 -1000 Subject: [PATCH 01/22] .devcontainer --- .devcontainer/devcontainer.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..5e788cfe6 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,21 @@ +{ + "name": "UIPA DevBox", + "dockerComposeFile": "../docker-compose.yml", + "service": "elasticsearch", + "workspaceFolder": "/workspaces/uipa", + "forwardPorts": [8000, 5432, 9200], + // "postCreateCommand": "until docker info > /dev/null 2>&1; do echo 'Waiting for Docker to start...'; sleep 1; done && docker-compose up -d && python -m venv venv && source venv/bin/activate && pip install -r requirements.txt && bash data/seed/init_db.sh && python manage.py check && sleep infinity", + // "remoteUser": "root", + "remoteUser": "devuser", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-azuretools.vscode-docker" + ], + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + } + } + } +} \ No newline at end of file From 3bb9f7f5943ba41ed4a3ccf4a8b1ba0afbe15f58 Mon Sep 17 00:00:00 2001 From: Tyler Chong Date: Mon, 17 Jun 2024 18:20:48 -1000 Subject: [PATCH 02/22] remove remoteUser override --- .devcontainer/devcontainer.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5e788cfe6..a98cf7fd4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,6 @@ "forwardPorts": [8000, 5432, 9200], // "postCreateCommand": "until docker info > /dev/null 2>&1; do echo 'Waiting for Docker to start...'; sleep 1; done && docker-compose up -d && python -m venv venv && source venv/bin/activate && pip install -r requirements.txt && bash data/seed/init_db.sh && python manage.py check && sleep infinity", // "remoteUser": "root", - "remoteUser": "devuser", "customizations": { "vscode": { "extensions": [ From c5dbb77b0a41030fd3e636d31248a6bab4395519 Mon Sep 17 00:00:00 2001 From: Tyler Chong Date: Mon, 17 Jun 2024 18:39:38 -1000 Subject: [PATCH 03/22] testing --- .devcontainer/devcontainer.json | 6 +++--- Dockerfile | 36 +++++++++++++++++++++++++++++++++ docker-compose.yml | 4 ++++ 3 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 Dockerfile diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a98cf7fd4..c4e8f48e9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,9 +1,9 @@ { "name": "UIPA DevBox", "dockerComposeFile": "../docker-compose.yml", - "service": "elasticsearch", - "workspaceFolder": "/workspaces/uipa", - "forwardPorts": [8000, 5432, 9200], + "service": "app", + // "workspaceFolder": "/workspaces/uipa", + // "forwardPorts": [8000, 5432, 9200], // "postCreateCommand": "until docker info > /dev/null 2>&1; do echo 'Waiting for Docker to start...'; sleep 1; done && docker-compose up -d && python -m venv venv && source venv/bin/activate && pip install -r requirements.txt && bash data/seed/init_db.sh && python manage.py check && sleep infinity", // "remoteUser": "root", "customizations": { diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..769dad745 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Note: You can use any Debian/Ubuntu based image you want. +FROM mcr.microsoft.com/vscode/devcontainers/base:bullseye + +# [Option] Install zsh +ARG INSTALL_ZSH="true" +# [Option] Upgrade OS packages to their latest versions +ARG UPGRADE_PACKAGES="false" +# [Option] Enable non-root Docker access in container +ARG ENABLE_NONROOT_DOCKER="true" +# [Option] Use the OSS Moby CLI instead of the licensed Docker CLI +ARG USE_MOBY="true" + +# Enable new "BUILDKIT" mode for Docker CLI +ENV DOCKER_BUILDKIT=1 + +# Install needed packages and setup non-root user. Use a separate RUN statement to add your +# own dependencies. A user of "automatic" attempts to reuse an user ID if one already exists. +ARG USERNAME=automatic +ARG USER_UID=1000 +ARG USER_GID=$USER_UID +COPY library-scripts/*.sh /tmp/library-scripts/ +RUN apt-get update \ + && /bin/bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \ + # Use Docker script from script library to set things up + && /bin/bash /tmp/library-scripts/docker-debian.sh "${ENABLE_NONROOT_DOCKER}" "/var/run/docker-host.sock" "/var/run/docker.sock" "${USERNAME}" \ + # Clean up + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/library-scripts/ + +# Setting the ENTRYPOINT to docker-init.sh will configure non-root access +# to the Docker socket. The script will also execute CMD as needed. +ENTRYPOINT [ "/usr/local/share/docker-init.sh" ] +CMD [ "sleep", "infinity" ] + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 8689d4386..ef27ec8a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,8 @@ services: + app: + build: + context: . + dockerfile: Dockerfile db: image: postgis/postgis:14-3.1 volumes: From 185e2f2d558a1e15aab384ac3a7a6b5ef894762c Mon Sep 17 00:00:00 2001 From: Tyler Chong Date: Mon, 17 Jun 2024 18:45:32 -1000 Subject: [PATCH 04/22] test again --- Dockerfile => .devcontainer/Dockerfile | 0 .devcontainer/devcontainer.json | 2 +- .../docker-compose.yml | 0 .../library-scripts/common-debian.sh | 454 ++++++++++++++++++ .../library-scripts/docker-debian.sh | 355 ++++++++++++++ 5 files changed, 810 insertions(+), 1 deletion(-) rename Dockerfile => .devcontainer/Dockerfile (100%) rename docker-compose.yml => .devcontainer/docker-compose.yml (100%) create mode 100644 .devcontainer/library-scripts/common-debian.sh create mode 100644 .devcontainer/library-scripts/docker-debian.sh diff --git a/Dockerfile b/.devcontainer/Dockerfile similarity index 100% rename from Dockerfile rename to .devcontainer/Dockerfile diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c4e8f48e9..b2689d688 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "UIPA DevBox", - "dockerComposeFile": "../docker-compose.yml", + "dockerComposeFile": "docker-compose.yml", "service": "app", // "workspaceFolder": "/workspaces/uipa", // "forwardPorts": [8000, 5432, 9200], diff --git a/docker-compose.yml b/.devcontainer/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to .devcontainer/docker-compose.yml diff --git a/.devcontainer/library-scripts/common-debian.sh b/.devcontainer/library-scripts/common-debian.sh new file mode 100644 index 000000000..bf1f9e2ed --- /dev/null +++ b/.devcontainer/library-scripts/common-debian.sh @@ -0,0 +1,454 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. +#------------------------------------------------------------------------------------------------------------- +# +# Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/common.md +# Maintainer: The VS Code and Codespaces Teams +# +# Syntax: ./common-debian.sh [install zsh flag] [username] [user UID] [user GID] [upgrade packages flag] [install Oh My Zsh! flag] [Add non-free packages] + +set -e + +INSTALL_ZSH=${1:-"true"} +USERNAME=${2:-"automatic"} +USER_UID=${3:-"automatic"} +USER_GID=${4:-"automatic"} +UPGRADE_PACKAGES=${5:-"true"} +INSTALL_OH_MYS=${6:-"true"} +ADD_NON_FREE_PACKAGES=${7:-"false"} +SCRIPT_DIR="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)" +MARKER_FILE="/usr/local/etc/vscode-dev-containers/common" + +if [ "$(id -u)" -ne 0 ]; then + echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' + exit 1 +fi + +# Ensure that login shells get the correct path if the user updated the PATH using ENV. +rm -f /etc/profile.d/00-restore-env.sh +echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh +chmod +x /etc/profile.d/00-restore-env.sh + +# If in automatic mode, determine if a user already exists, if not use vscode +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in ${POSSIBLE_USERS[@]}; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=vscode + fi +elif [ "${USERNAME}" = "none" ]; then + USERNAME=root + USER_UID=0 + USER_GID=0 +fi + +# Load markers to see which steps have already run +if [ -f "${MARKER_FILE}" ]; then + echo "Marker file found:" + cat "${MARKER_FILE}" + source "${MARKER_FILE}" +fi + +# Ensure apt is in non-interactive to avoid prompts +export DEBIAN_FRONTEND=noninteractive + +# Function to call apt-get if needed +apt_get_update_if_needed() +{ + if [ ! -d "/var/lib/apt/lists" ] || [ "$(ls /var/lib/apt/lists/ | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update + else + echo "Skipping apt-get update." + fi +} + +# Run install apt-utils to avoid debconf warning then verify presence of other common developer tools and dependencies +if [ "${PACKAGES_ALREADY_INSTALLED}" != "true" ]; then + + package_list="apt-utils \ + openssh-client \ + gnupg2 \ + dirmngr \ + iproute2 \ + procps \ + lsof \ + htop \ + net-tools \ + psmisc \ + curl \ + wget \ + rsync \ + ca-certificates \ + unzip \ + zip \ + nano \ + vim-tiny \ + less \ + jq \ + lsb-release \ + apt-transport-https \ + dialog \ + libc6 \ + libgcc1 \ + libkrb5-3 \ + libgssapi-krb5-2 \ + libicu[0-9][0-9] \ + liblttng-ust[0-9] \ + libstdc++6 \ + zlib1g \ + locales \ + sudo \ + ncdu \ + man-db \ + strace \ + manpages \ + manpages-dev \ + init-system-helpers" + + # Needed for adding manpages-posix and manpages-posix-dev which are non-free packages in Debian + if [ "${ADD_NON_FREE_PACKAGES}" = "true" ]; then + # Bring in variables from /etc/os-release like VERSION_CODENAME + . /etc/os-release + sed -i -E "s/deb http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME} main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME} main contrib non-free/" /etc/apt/sources.list + sed -i -E "s/deb-src http:\/\/(deb|httredir)\.debian\.org\/debian ${VERSION_CODENAME} main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME} main contrib non-free/" /etc/apt/sources.list + sed -i -E "s/deb http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list + sed -i -E "s/deb-src http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb-src http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list + # Handle bullseye location for security https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html + sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list + echo "Running apt-get update..." + apt-get update + package_list="${package_list} manpages-posix manpages-posix-dev" + else + apt_get_update_if_needed + fi + + # Install libssl1.1 if available + if [[ ! -z $(apt-cache --names-only search ^libssl1.1$) ]]; then + package_list="${package_list} libssl1.1" + fi + + # Install appropriate version of libssl1.0.x if available + libssl_package=$(dpkg-query -f '${db:Status-Abbrev}\t${binary:Package}\n' -W 'libssl1\.0\.?' 2>&1 || echo '') + if [ "$(echo "$LIlibssl_packageBSSL" | grep -o 'libssl1\.0\.[0-9]:' | uniq | sort | wc -l)" -eq 0 ]; then + if [[ ! -z $(apt-cache --names-only search ^libssl1.0.2$) ]]; then + # Debian 9 + package_list="${package_list} libssl1.0.2" + elif [[ ! -z $(apt-cache --names-only search ^libssl1.0.0$) ]]; then + # Ubuntu 18.04, 16.04, earlier + package_list="${package_list} libssl1.0.0" + fi + fi + + echo "Packages to verify are installed: ${package_list}" + apt-get -y install --no-install-recommends ${package_list} 2> >( grep -v 'debconf: delaying package configuration, since apt-utils is not installed' >&2 ) + + # Install git if not already installed (may be more recent than distro version) + if ! type git > /dev/null 2>&1; then + apt-get -y install --no-install-recommends git + fi + + PACKAGES_ALREADY_INSTALLED="true" +fi + +# Get to latest versions of all packages +if [ "${UPGRADE_PACKAGES}" = "true" ]; then + apt_get_update_if_needed + apt-get -y upgrade --no-install-recommends + apt-get autoremove -y +fi + +# Ensure at least the en_US.UTF-8 UTF-8 locale is available. +# Common need for both applications and things like the agnoster ZSH theme. +if [ "${LOCALE_ALREADY_SET}" != "true" ] && ! grep -o -E '^\s*en_US.UTF-8\s+UTF-8' /etc/locale.gen > /dev/null; then + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen + locale-gen + LOCALE_ALREADY_SET="true" +fi + +# Create or update a non-root user to match UID/GID. +group_name="${USERNAME}" +if id -u ${USERNAME} > /dev/null 2>&1; then + # User exists, update if needed + if [ "${USER_GID}" != "automatic" ] && [ "$USER_GID" != "$(id -g $USERNAME)" ]; then + group_name="$(id -gn $USERNAME)" + groupmod --gid $USER_GID ${group_name} + usermod --gid $USER_GID $USERNAME + fi + if [ "${USER_UID}" != "automatic" ] && [ "$USER_UID" != "$(id -u $USERNAME)" ]; then + usermod --uid $USER_UID $USERNAME + fi +else + # Create user + if [ "${USER_GID}" = "automatic" ]; then + groupadd $USERNAME + else + groupadd --gid $USER_GID $USERNAME + fi + if [ "${USER_UID}" = "automatic" ]; then + useradd -s /bin/bash --gid $USERNAME -m $USERNAME + else + useradd -s /bin/bash --uid $USER_UID --gid $USERNAME -m $USERNAME + fi +fi + +# Add sudo support for non-root user +if [ "${USERNAME}" != "root" ] && [ "${EXISTING_NON_ROOT_USER}" != "${USERNAME}" ]; then + echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME + chmod 0440 /etc/sudoers.d/$USERNAME + EXISTING_NON_ROOT_USER="${USERNAME}" +fi + +# ** Shell customization section ** +if [ "${USERNAME}" = "root" ]; then + user_rc_path="/root" +else + user_rc_path="/home/${USERNAME}" +fi + +# Restore user .bashrc defaults from skeleton file if it doesn't exist or is empty +if [ ! -f "${user_rc_path}/.bashrc" ] || [ ! -s "${user_rc_path}/.bashrc" ] ; then + cp /etc/skel/.bashrc "${user_rc_path}/.bashrc" +fi + +# Restore user .profile defaults from skeleton file if it doesn't exist or is empty +if [ ! -f "${user_rc_path}/.profile" ] || [ ! -s "${user_rc_path}/.profile" ] ; then + cp /etc/skel/.profile "${user_rc_path}/.profile" +fi + +# .bashrc/.zshrc snippet +rc_snippet="$(cat << 'EOF' + +if [ -z "${USER}" ]; then export USER=$(whoami); fi +if [[ "${PATH}" != *"$HOME/.local/bin"* ]]; then export PATH="${PATH}:$HOME/.local/bin"; fi + +# Display optional first run image specific notice if configured and terminal is interactive +if [ -t 1 ] && [[ "${TERM_PROGRAM}" = "vscode" || "${TERM_PROGRAM}" = "codespaces" ]] && [ ! -f "$HOME/.config/vscode-dev-containers/first-run-notice-already-displayed" ]; then + if [ -f "/usr/local/etc/vscode-dev-containers/first-run-notice.txt" ]; then + cat "/usr/local/etc/vscode-dev-containers/first-run-notice.txt" + elif [ -f "/workspaces/.codespaces/shared/first-run-notice.txt" ]; then + cat "/workspaces/.codespaces/shared/first-run-notice.txt" + fi + mkdir -p "$HOME/.config/vscode-dev-containers" + # Mark first run notice as displayed after 10s to avoid problems with fast terminal refreshes hiding it + ((sleep 10s; touch "$HOME/.config/vscode-dev-containers/first-run-notice-already-displayed") &) +fi + +# Set the default git editor if not already set +if [ -z "$(git config --get core.editor)" ] && [ -z "${GIT_EDITOR}" ]; then + if [ "${TERM_PROGRAM}" = "vscode" ]; then + if [[ -n $(command -v code-insiders) && -z $(command -v code) ]]; then + export GIT_EDITOR="code-insiders --wait" + else + export GIT_EDITOR="code --wait" + fi + fi +fi + +EOF +)" + +# code shim, it fallbacks to code-insiders if code is not available +cat << 'EOF' > /usr/local/bin/code +#!/bin/sh + +get_in_path_except_current() { + which -a "$1" | grep -A1 "$0" | grep -v "$0" +} + +code="$(get_in_path_except_current code)" + +if [ -n "$code" ]; then + exec "$code" "$@" +elif [ "$(command -v code-insiders)" ]; then + exec code-insiders "$@" +else + echo "code or code-insiders is not installed" >&2 + exit 127 +fi +EOF +chmod +x /usr/local/bin/code + +# systemctl shim - tells people to use 'service' if systemd is not running +cat << 'EOF' > /usr/local/bin/systemctl +#!/bin/sh +set -e +if [ -d "/run/systemd/system" ]; then + exec /bin/systemctl "$@" +else + echo '\n"systemd" is not running in this container due to its overhead.\nUse the "service" command to start services instead. e.g.: \n\nservice --status-all' +fi +EOF +chmod +x /usr/local/bin/systemctl + +# Codespaces bash and OMZ themes - partly inspired by https://github.com/ohmyzsh/ohmyzsh/blob/master/themes/robbyrussell.zsh-theme +codespaces_bash="$(cat \ +<<'EOF' + +# Codespaces bash prompt theme +__bash_prompt() { + local userpart='`export XIT=$? \ + && [ ! -z "${GITHUB_USER}" ] && echo -n "\[\033[0;32m\]@${GITHUB_USER} " || echo -n "\[\033[0;32m\]\u " \ + && [ "$XIT" -ne "0" ] && echo -n "\[\033[1;31m\]➜" || echo -n "\[\033[0m\]➜"`' + local gitbranch='`\ + if [ "$(git config --get codespaces-theme.hide-status 2>/dev/null)" != 1 ]; then \ + export BRANCH=$(git symbolic-ref --short HEAD 2>/dev/null || git rev-parse --short HEAD 2>/dev/null); \ + if [ "${BRANCH}" != "" ]; then \ + echo -n "\[\033[0;36m\](\[\033[1;31m\]${BRANCH}" \ + && if git ls-files --error-unmatch -m --directory --no-empty-directory -o --exclude-standard ":/*" > /dev/null 2>&1; then \ + echo -n " \[\033[1;33m\]✗"; \ + fi \ + && echo -n "\[\033[0;36m\]) "; \ + fi; \ + fi`' + local lightblue='\[\033[1;34m\]' + local removecolor='\[\033[0m\]' + PS1="${userpart} ${lightblue}\w ${gitbranch}${removecolor}\$ " + unset -f __bash_prompt +} +__bash_prompt + +EOF +)" + +codespaces_zsh="$(cat \ +<<'EOF' +# Codespaces zsh prompt theme +__zsh_prompt() { + local prompt_username + if [ ! -z "${GITHUB_USER}" ]; then + prompt_username="@${GITHUB_USER}" + else + prompt_username="%n" + fi + PROMPT="%{$fg[green]%}${prompt_username} %(?:%{$reset_color%}➜ :%{$fg_bold[red]%}➜ )" # User/exit code arrow + PROMPT+='%{$fg_bold[blue]%}%(5~|%-1~/…/%3~|%4~)%{$reset_color%} ' # cwd + PROMPT+='$([ "$(git config --get codespaces-theme.hide-status 2>/dev/null)" != 1 ] && git_prompt_info)' # Git status + PROMPT+='%{$fg[white]%}$ %{$reset_color%}' + unset -f __zsh_prompt +} +ZSH_THEME_GIT_PROMPT_PREFIX="%{$fg_bold[cyan]%}(%{$fg_bold[red]%}" +ZSH_THEME_GIT_PROMPT_SUFFIX="%{$reset_color%} " +ZSH_THEME_GIT_PROMPT_DIRTY=" %{$fg_bold[yellow]%}✗%{$fg_bold[cyan]%})" +ZSH_THEME_GIT_PROMPT_CLEAN="%{$fg_bold[cyan]%})" +__zsh_prompt + +EOF +)" + +# Add RC snippet and custom bash prompt +if [ "${RC_SNIPPET_ALREADY_ADDED}" != "true" ]; then + echo "${rc_snippet}" >> /etc/bash.bashrc + echo "${codespaces_bash}" >> "${user_rc_path}/.bashrc" + echo 'export PROMPT_DIRTRIM=4' >> "${user_rc_path}/.bashrc" + if [ "${USERNAME}" != "root" ]; then + echo "${codespaces_bash}" >> "/root/.bashrc" + echo 'export PROMPT_DIRTRIM=4' >> "/root/.bashrc" + fi + chown ${USERNAME}:${group_name} "${user_rc_path}/.bashrc" + RC_SNIPPET_ALREADY_ADDED="true" +fi + +# Optionally install and configure zsh and Oh My Zsh! +if [ "${INSTALL_ZSH}" = "true" ]; then + if ! type zsh > /dev/null 2>&1; then + apt_get_update_if_needed + apt-get install -y zsh + fi + if [ "${ZSH_ALREADY_INSTALLED}" != "true" ]; then + echo "${rc_snippet}" >> /etc/zsh/zshrc + ZSH_ALREADY_INSTALLED="true" + fi + + # Adapted, simplified inline Oh My Zsh! install steps that adds, defaults to a codespaces theme. + # See https://github.com/ohmyzsh/ohmyzsh/blob/master/tools/install.sh for official script. + oh_my_install_dir="${user_rc_path}/.oh-my-zsh" + if [ ! -d "${oh_my_install_dir}" ] && [ "${INSTALL_OH_MYS}" = "true" ]; then + template_path="${oh_my_install_dir}/templates/zshrc.zsh-template" + user_rc_file="${user_rc_path}/.zshrc" + umask g-w,o-w + mkdir -p ${oh_my_install_dir} + git clone --depth=1 \ + -c core.eol=lf \ + -c core.autocrlf=false \ + -c fsck.zeroPaddedFilemode=ignore \ + -c fetch.fsck.zeroPaddedFilemode=ignore \ + -c receive.fsck.zeroPaddedFilemode=ignore \ + "https://github.com/ohmyzsh/ohmyzsh" "${oh_my_install_dir}" 2>&1 + echo -e "$(cat "${template_path}")\nDISABLE_AUTO_UPDATE=true\nDISABLE_UPDATE_PROMPT=true" > ${user_rc_file} + sed -i -e 's/ZSH_THEME=.*/ZSH_THEME="codespaces"/g' ${user_rc_file} + + mkdir -p ${oh_my_install_dir}/custom/themes + echo "${codespaces_zsh}" > "${oh_my_install_dir}/custom/themes/codespaces.zsh-theme" + # Shrink git while still enabling updates + cd "${oh_my_install_dir}" + git repack -a -d -f --depth=1 --window=1 + # Copy to non-root user if one is specified + if [ "${USERNAME}" != "root" ]; then + cp -rf "${user_rc_file}" "${oh_my_install_dir}" /root + chown -R ${USERNAME}:${group_name} "${user_rc_path}" + fi + fi +fi + +# Persist image metadata info, script if meta.env found in same directory +meta_info_script="$(cat << 'EOF' +#!/bin/sh +. /usr/local/etc/vscode-dev-containers/meta.env + +# Minimal output +if [ "$1" = "version" ] || [ "$1" = "image-version" ]; then + echo "${VERSION}" + exit 0 +elif [ "$1" = "release" ]; then + echo "${GIT_REPOSITORY_RELEASE}" + exit 0 +elif [ "$1" = "content" ] || [ "$1" = "content-url" ] || [ "$1" = "contents" ] || [ "$1" = "contents-url" ]; then + echo "${CONTENTS_URL}" + exit 0 +fi + +#Full output +echo +echo "Development container image information" +echo +if [ ! -z "${VERSION}" ]; then echo "- Image version: ${VERSION}"; fi +if [ ! -z "${DEFINITION_ID}" ]; then echo "- Definition ID: ${DEFINITION_ID}"; fi +if [ ! -z "${VARIANT}" ]; then echo "- Variant: ${VARIANT}"; fi +if [ ! -z "${GIT_REPOSITORY}" ]; then echo "- Source code repository: ${GIT_REPOSITORY}"; fi +if [ ! -z "${GIT_REPOSITORY_RELEASE}" ]; then echo "- Source code release/branch: ${GIT_REPOSITORY_RELEASE}"; fi +if [ ! -z "${BUILD_TIMESTAMP}" ]; then echo "- Timestamp: ${BUILD_TIMESTAMP}"; fi +if [ ! -z "${CONTENTS_URL}" ]; then echo && echo "More info: ${CONTENTS_URL}"; fi +echo +EOF +)" +if [ -f "${SCRIPT_DIR}/meta.env" ]; then + mkdir -p /usr/local/etc/vscode-dev-containers/ + cp -f "${SCRIPT_DIR}/meta.env" /usr/local/etc/vscode-dev-containers/meta.env + echo "${meta_info_script}" > /usr/local/bin/devcontainer-info + chmod +x /usr/local/bin/devcontainer-info +fi + +# Write marker file +mkdir -p "$(dirname "${MARKER_FILE}")" +echo -e "\ + PACKAGES_ALREADY_INSTALLED=${PACKAGES_ALREADY_INSTALLED}\n\ + LOCALE_ALREADY_SET=${LOCALE_ALREADY_SET}\n\ + EXISTING_NON_ROOT_USER=${EXISTING_NON_ROOT_USER}\n\ + RC_SNIPPET_ALREADY_ADDED=${RC_SNIPPET_ALREADY_ADDED}\n\ + ZSH_ALREADY_INSTALLED=${ZSH_ALREADY_INSTALLED}" > "${MARKER_FILE}" + +echo "Done!" \ No newline at end of file diff --git a/.devcontainer/library-scripts/docker-debian.sh b/.devcontainer/library-scripts/docker-debian.sh new file mode 100644 index 000000000..0559c5560 --- /dev/null +++ b/.devcontainer/library-scripts/docker-debian.sh @@ -0,0 +1,355 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. +#------------------------------------------------------------------------------------------------------------- +# +# Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/docker.md +# Maintainer: The VS Code and Codespaces Teams +# +# Syntax: ./docker-debian.sh [enable non-root docker socket access flag] [source socket] [target socket] [non-root user] [use moby] [CLI version] [Major version for docker-compose] + +ENABLE_NONROOT_DOCKER=${1:-"true"} +SOURCE_SOCKET=${2:-"/var/run/docker-host.sock"} +TARGET_SOCKET=${3:-"/var/run/docker.sock"} +USERNAME=${4:-"automatic"} +USE_MOBY=${5:-"true"} +DOCKER_VERSION=${6:-"latest"} +DOCKER_DASH_COMPOSE_VERSION=${7:-"v1"} # v1 or v2 +MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" +DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="buster bullseye bionic focal jammy" +DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="buster bullseye bionic focal hirsute impish jammy" + +set -e + +if [ "$(id -u)" -ne 0 ]; then + echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' + exit 1 +fi + +# Determine the appropriate non-root user +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in ${POSSIBLE_USERS[@]}; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi +elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root +fi + +# Get central common setting +get_common_setting() { + if [ "${common_settings_file_loaded}" != "true" ]; then + curl -sfL "https://aka.ms/vscode-dev-containers/script-library/settings.env" 2>/dev/null -o /tmp/vsdc-settings.env || echo "Could not download settings file. Skipping." + common_settings_file_loaded=true + fi + if [ -f "/tmp/vsdc-settings.env" ]; then + local multi_line="" + if [ "$2" = "true" ]; then multi_line="-z"; fi + local result="$(grep ${multi_line} -oP "$1=\"?\K[^\"]+" /tmp/vsdc-settings.env | tr -d '\0')" + if [ ! -z "${result}" ]; then declare -g $1="${result}"; fi + fi + echo "$1=${!1}" +} + +# Function to run apt-get if needed +apt_get_update_if_needed() +{ + if [ ! -d "/var/lib/apt/lists" ] || [ "$(ls /var/lib/apt/lists/ | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update + else + echo "Skipping apt-get update." + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update_if_needed + apt-get -y install --no-install-recommends "$@" + fi +} + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Ensure apt is in non-interactive to avoid prompts +export DEBIAN_FRONTEND=noninteractive + +# Install dependencies +check_packages apt-transport-https curl ca-certificates gnupg2 dirmngr +if ! type git > /dev/null 2>&1; then + apt_get_update_if_needed + apt-get -y install git +fi + +# Source /etc/os-release to get OS info +. /etc/os-release +# Fetch host/container arch. +architecture="$(dpkg --print-architecture)" + +# Check if distro is suppported +if [ "${USE_MOBY}" = "true" ]; then + # 'get_common_setting' allows attribute to be updated remotely + get_common_setting DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES + if [[ "${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}" != *"${VERSION_CODENAME}"* ]]; then + err "Unsupported distribution version '${VERSION_CODENAME}'. To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS distribution" + err "Support distributions include: ${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}" + exit 1 + fi + echo "Distro codename '${VERSION_CODENAME}' matched filter '${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}'" +else + get_common_setting DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES + if [[ "${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}" != *"${VERSION_CODENAME}"* ]]; then + err "Unsupported distribution version '${VERSION_CODENAME}'. To resolve, please choose a compatible OS distribution" + err "Support distributions include: ${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}" + exit 1 + fi + echo "Distro codename '${VERSION_CODENAME}' matched filter '${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}'" +fi + +# Set up the necessary apt repos (either Microsoft's or Docker's) +if [ "${USE_MOBY}" = "true" ]; then + + cli_package_name="moby-cli" + + # Import key safely and import Microsoft apt repo + get_common_setting MICROSOFT_GPG_KEYS_URI + curl -sSL ${MICROSOFT_GPG_KEYS_URI} | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg + echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list +else + # Name of proprietary engine package + cli_package_name="docker-ce-cli" + + # Import key safely and import Docker apt repo + curl -fsSL https://download.docker.com/linux/${ID}/gpg | gpg --dearmor > /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/${ID} ${VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list +fi + +# Refresh apt lists +apt-get update + +# Soft version matching for CLI +if [ "${DOCKER_VERSION}" = "latest" ] || [ "${DOCKER_VERSION}" = "lts" ] || [ "${DOCKER_VERSION}" = "stable" ]; then + # Empty, meaning grab whatever "latest" is in apt repo + cli_version_suffix="" +else + # Fetch a valid version from the apt-cache (eg: the Microsoft repo appends +azure, breakfix, etc...) + docker_version_dot_escaped="${DOCKER_VERSION//./\\.}" + docker_version_dot_plus_escaped="${docker_version_dot_escaped//+/\\+}" + # Regex needs to handle debian package version number format: https://www.systutorials.com/docs/linux/man/5-deb-version/ + docker_version_regex="^(.+:)?${docker_version_dot_plus_escaped}([\\.\\+ ~:-]|$)" + set +e # Don't exit if finding version fails - will handle gracefully + cli_version_suffix="=$(apt-cache madison ${cli_package_name} | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${docker_version_regex}")" + set -e + if [ -z "${cli_version_suffix}" ] || [ "${cli_version_suffix}" = "=" ]; then + echo "(!) No full or partial Docker / Moby version match found for \"${DOCKER_VERSION}\" on OS ${ID} ${VERSION_CODENAME} (${architecture}). Available versions:" + apt-cache madison ${cli_package_name} | awk -F"|" '{print $2}' | grep -oP '^(.+:)?\K.+' + exit 1 + fi + echo "cli_version_suffix ${cli_version_suffix}" +fi + +# Install Docker / Moby CLI if not already installed +if type docker > /dev/null 2>&1; then + echo "Docker / Moby CLI already installed." +else + if [ "${USE_MOBY}" = "true" ]; then + apt-get -y install --no-install-recommends moby-cli${cli_version_suffix} moby-buildx + apt-get -y install --no-install-recommends moby-compose || echo "(*) Package moby-compose (Docker Compose v2) not available for OS ${ID} ${VERSION_CODENAME} (${architecture}). Skipping." + else + apt-get -y install --no-install-recommends docker-ce-cli${cli_version_suffix} + apt-get -y install --no-install-recommends docker-compose-plugin || echo "(*) Package docker-compose-plugin (Docker Compose v2) not available for OS ${ID} ${VERSION_CODENAME} (${architecture}). Skipping." + fi +fi + +# Install Docker Compose if not already installed and is on a supported architecture +if type docker-compose > /dev/null 2>&1; then + echo "Docker Compose already installed." +else + TARGET_COMPOSE_ARCH="$(uname -m)" + if [ "${TARGET_COMPOSE_ARCH}" = "amd64" ]; then + TARGET_COMPOSE_ARCH="x86_64" + fi + if [ "${TARGET_COMPOSE_ARCH}" != "x86_64" ]; then + # Use pip to get a version that runns on this architecture + if ! dpkg -s python3-minimal python3-pip libffi-dev python3-venv > /dev/null 2>&1; then + apt_get_update_if_needed + apt-get -y install python3-minimal python3-pip libffi-dev python3-venv + fi + export PIPX_HOME=/usr/local/pipx + mkdir -p ${PIPX_HOME} + export PIPX_BIN_DIR=/usr/local/bin + export PYTHONUSERBASE=/tmp/pip-tmp + export PIP_CACHE_DIR=/tmp/pip-tmp/cache + pipx_bin=pipx + if ! type pipx > /dev/null 2>&1; then + pip3 install --disable-pip-version-check --no-cache-dir --user pipx + pipx_bin=/tmp/pip-tmp/bin/pipx + fi + ${pipx_bin} install --pip-args '--no-cache-dir --force-reinstall' docker-compose + rm -rf /tmp/pip-tmp + else + compose_v1_version="1" + find_version_from_git_tags compose_v1_version "https://github.com/docker/compose" "tags/" + echo "(*) Installing docker-compose ${compose_v1_version}..." + curl -fsSL "https://github.com/docker/compose/releases/download/${compose_v1_version}/docker-compose-Linux-x86_64" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + fi +fi + +# Install docker-compose switch if not already installed - https://github.com/docker/compose-switch#manual-installation +current_v1_compose_path="$(which docker-compose)" +target_v1_compose_path="$(dirname "${current_v1_compose_path}")/docker-compose-v1" +if ! type compose-switch > /dev/null 2>&1; then + echo "(*) Installing compose-switch..." + compose_switch_version="latest" + find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch + chmod +x /usr/local/bin/compose-switch + # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 + + # Setup v1 CLI as alternative in addition to compose-switch (which maps to v2) + mv "${current_v1_compose_path}" "${target_v1_compose_path}" + update-alternatives --install /usr/local/bin/docker-compose docker-compose /usr/local/bin/compose-switch 99 + update-alternatives --install /usr/local/bin/docker-compose docker-compose "${target_v1_compose_path}" 1 +fi +if [ "${DOCKER_DASH_COMPOSE_VERSION}" = "v1" ]; then + update-alternatives --set docker-compose "${target_v1_compose_path}" +else + update-alternatives --set docker-compose /usr/local/bin/compose-switch +fi + +# If init file already exists, exit +if [ -f "/usr/local/share/docker-init.sh" ]; then + exit 0 +fi +echo "docker-init doesnt exist, adding..." + +# By default, make the source and target sockets the same +if [ "${SOURCE_SOCKET}" != "${TARGET_SOCKET}" ]; then + touch "${SOURCE_SOCKET}" + ln -s "${SOURCE_SOCKET}" "${TARGET_SOCKET}" +fi + +# Add a stub if not adding non-root user access, user is root +if [ "${ENABLE_NONROOT_DOCKER}" = "false" ] || [ "${USERNAME}" = "root" ]; then + echo -e '#!/usr/bin/env bash\nexec "$@"' > /usr/local/share/docker-init.sh + chmod +x /usr/local/share/docker-init.sh + exit 0 +fi + +# Setup a docker group in the event the docker socket's group is not root +if ! grep -qE '^docker:' /etc/group; then + groupadd --system docker +fi +usermod -aG docker "${USERNAME}" +DOCKER_GID="$(grep -oP '^docker:x:\K[^:]+' /etc/group)" + +# If enabling non-root access and specified user is found, setup socat and add script +chown -h "${USERNAME}":root "${TARGET_SOCKET}" +if ! dpkg -s socat > /dev/null 2>&1; then + apt_get_update_if_needed + apt-get -y install socat +fi +tee /usr/local/share/docker-init.sh > /dev/null \ +<< EOF +#!/usr/bin/env bash +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. +#------------------------------------------------------------------------------------------------------------- + +set -e + +SOCAT_PATH_BASE=/tmp/vscr-docker-from-docker +SOCAT_LOG=\${SOCAT_PATH_BASE}.log +SOCAT_PID=\${SOCAT_PATH_BASE}.pid + +# Wrapper function to only use sudo if not already root +sudoIf() +{ + if [ "\$(id -u)" -ne 0 ]; then + sudo "\$@" + else + "\$@" + fi +} + +# Log messages +log() +{ + echo -e "[\$(date)] \$@" | sudoIf tee -a \${SOCAT_LOG} > /dev/null +} + +echo -e "\n** \$(date) **" | sudoIf tee -a \${SOCAT_LOG} > /dev/null +log "Ensuring ${USERNAME} has access to ${SOURCE_SOCKET} via ${TARGET_SOCKET}" + +# If enabled, try to update the docker group with the right GID. If the group is root, +# fall back on using socat to forward the docker socket to another unix socket so +# that we can set permissions on it without affecting the host. +if [ "${ENABLE_NONROOT_DOCKER}" = "true" ] && [ "${SOURCE_SOCKET}" != "${TARGET_SOCKET}" ] && [ "${USERNAME}" != "root" ] && [ "${USERNAME}" != "0" ]; then + SOCKET_GID=\$(stat -c '%g' ${SOURCE_SOCKET}) + if [ "\${SOCKET_GID}" != "0" ] && [ "\${SOCKET_GID}" != "${DOCKER_GID}" ] && ! grep -E ".+:x:\${SOCKET_GID}" /etc/group; then + sudoIf groupmod --gid "\${SOCKET_GID}" docker + else + # Enable proxy if not already running + if [ ! -f "\${SOCAT_PID}" ] || ! ps -p \$(cat \${SOCAT_PID}) > /dev/null; then + log "Enabling socket proxy." + log "Proxying ${SOURCE_SOCKET} to ${TARGET_SOCKET} for vscode" + sudoIf rm -rf ${TARGET_SOCKET} + (sudoIf socat UNIX-LISTEN:${TARGET_SOCKET},fork,mode=660,user=${USERNAME} UNIX-CONNECT:${SOURCE_SOCKET} 2>&1 | sudoIf tee -a \${SOCAT_LOG} > /dev/null & echo "\$!" | sudoIf tee \${SOCAT_PID} > /dev/null) + else + log "Socket proxy already running." + fi + fi + log "Success" +fi + +# Execute whatever commands were passed in (if any). This allows us +# to set this script to ENTRYPOINT while still executing the default CMD. +set +e +exec "\$@" +EOF +chmod +x /usr/local/share/docker-init.sh +chown ${USERNAME}:root /usr/local/share/docker-init.sh +echo "Done!" \ No newline at end of file From f7ef992645d1e31c0e5544db555ce55c521e3eec Mon Sep 17 00:00:00 2001 From: Tyler Chong Date: Mon, 17 Jun 2024 18:55:02 -1000 Subject: [PATCH 05/22] test --- .devcontainer/devcontainer.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b2689d688..a59ff428b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -10,11 +10,12 @@ "vscode": { "extensions": [ "ms-python.python", - "ms-azuretools.vscode-docker" + "ms-azuretools.vscode-docker" ], "settings": { "terminal.integrated.shell.linux": "/bin/bash" } } - } + }, + "remoteUser": "vscode" } \ No newline at end of file From e7a10250c7f705a1ecebf0005fcdc6c84eae4176 Mon Sep 17 00:00:00 2001 From: Tyler Chong Date: Tue, 18 Jun 2024 05:27:24 +0000 Subject: [PATCH 06/22] working basics --- .devcontainer/Dockerfile | 2 ++ .devcontainer/devcontainer.json | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 769dad745..791eef3ac 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -26,6 +26,8 @@ RUN apt-get update \ # Clean up && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/library-scripts/ +# Setup the app here + # Setting the ENTRYPOINT to docker-init.sh will configure non-root access # to the Docker socket. The script will also execute CMD as needed. ENTRYPOINT [ "/usr/local/share/docker-init.sh" ] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a59ff428b..7b5b518f7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,10 +2,7 @@ "name": "UIPA DevBox", "dockerComposeFile": "docker-compose.yml", "service": "app", - // "workspaceFolder": "/workspaces/uipa", - // "forwardPorts": [8000, 5432, 9200], - // "postCreateCommand": "until docker info > /dev/null 2>&1; do echo 'Waiting for Docker to start...'; sleep 1; done && docker-compose up -d && python -m venv venv && source venv/bin/activate && pip install -r requirements.txt && bash data/seed/init_db.sh && python manage.py check && sleep infinity", - // "remoteUser": "root", + "forwardPorts": [8000, 5432, 9200], "customizations": { "vscode": { "extensions": [ From adbe771f9fbcc60e3af37ee445554d0f1014115e Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Tue, 2 Jul 2024 20:19:31 -1000 Subject: [PATCH 07/22] base working + docker compose up functions locally --- .devcontainer/Dockerfile | 14 +++++++++++++- .devcontainer/docker-compose.yml | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 791eef3ac..eeb587f8e 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,6 +1,9 @@ # Note: You can use any Debian/Ubuntu based image you want. FROM mcr.microsoft.com/vscode/devcontainers/base:bullseye +# root acess for beginning of the script +USER root + # [Option] Install zsh ARG INSTALL_ZSH="true" # [Option] Upgrade OS packages to their latest versions @@ -23,9 +26,18 @@ RUN apt-get update \ && /bin/bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \ # Use Docker script from script library to set things up && /bin/bash /tmp/library-scripts/docker-debian.sh "${ENABLE_NONROOT_DOCKER}" "/var/run/docker-host.sock" "/var/run/docker.sock" "${USERNAME}" \ + # Create the baseuser user and group + && groupadd -r baseuser && useradd -r -g baseuser baseuser \ # Clean up && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/library-scripts/ +# Create directories for Elasticsearch and add necessary permissions +RUN mkdir -p /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch \ + && chown -R baseuser:baseuser /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch + +# Switch back to the baseuser +USER baseuser + # Setup the app here # Setting the ENTRYPOINT to docker-init.sh will configure non-root access @@ -35,4 +47,4 @@ CMD [ "sleep", "infinity" ] # [Optional] Uncomment this section to install additional OS packages. # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ -# && apt-get -y install --no-install-recommends \ No newline at end of file +# && apt-get -y install --no-install-recommends diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index ef27ec8a4..039ebffe6 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -15,6 +15,7 @@ services: - "127.0.0.1:5432:5432" elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0 + user: root volumes: - es-data:/usr/share/elasticsearch/data - es-logs:/var/log From ccc3fcceb22b02ba28ba4fa772baa3fe67360968 Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Tue, 2 Jul 2024 20:33:55 -1000 Subject: [PATCH 08/22] locally installed requirements through dockerfile - no error spinning up sudo docker-compose up --build --- .devcontainer/Dockerfile | 32 ++++++++++++++++++++ .devcontainer/codespaces-requirements.txt | 36 +++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 .devcontainer/codespaces-requirements.txt diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index eeb587f8e..25119c7e8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -4,6 +4,18 @@ FROM mcr.microsoft.com/vscode/devcontainers/base:bullseye # root acess for beginning of the script USER root +# Install main dependencies in one step to reduce layers +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get install -y --no-install-recommends \ + libpoppler-cpp-dev \ + python-is-python3 \ + gdal-bin \ + libgdal-dev \ + imagemagick \ + libmagickwand-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + # [Option] Install zsh ARG INSTALL_ZSH="true" # [Option] Upgrade OS packages to their latest versions @@ -35,6 +47,26 @@ RUN apt-get update \ RUN mkdir -p /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch \ && chown -R baseuser:baseuser /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch + # Copy the requirements file to the working directory +COPY codespaces-requirements.txt . + +# Display the contents of requirements.txt for debugging +RUN echo "Displaying requirements.txt:" && cat codespaces-requirements.txt + +# Update package list and install pip for Python 3 +RUN apt-get update && \ + apt-get install -y python3-pip + +# Display Python and pip versions for debugging +RUN python3 --version && pip3 --version + +# Upgrade pip +RUN pip3 install --upgrade pip + + +# Install dependencies globally +RUN pip install -r codespaces-requirements.txt + # Switch back to the baseuser USER baseuser diff --git a/.devcontainer/codespaces-requirements.txt b/.devcontainer/codespaces-requirements.txt new file mode 100644 index 000000000..5ec6899c0 --- /dev/null +++ b/.devcontainer/codespaces-requirements.txt @@ -0,0 +1,36 @@ +django==4.2.4 +Markdown==3.4.3 +celery==5.2.7 +django-celery-email==3.0.0 +django-taggit==4.0.0 +pytz==2023.3 +requests==2.31.0 +django-floppyforms==1.9.0 +python-magic==0.4.27 +python-mimeparse==1.6.0 +django-configurations==2.5.1 +django-storages==1.13.2 +dj-database-url==2.0.0 +django-contrib-comments==2.2.0 +unicodecsv==0.14.1 +django-tinymce==3.6.1 +python-docx==0.8.11 +elasticsearch==8.11.1 + +-e git+https://github.com/codewithaloha/froide.git@main#egg=froide + +lxml==5.2.1 +channels==4.0.0 +django-treebeard==4.4 +django-leaflet==0.29.0 +django-json-widget==1.1.1 +django-celery-beat==2.5.0 +django-mfa3==0.11.0 +psycopg[binary]==3.1.18 +psycopg-binary==3.1.18 +-e git+https://github.com/okfde/django-filingcabinet.git@main#egg=django-filingcabinet +oauthlib==3.2.2 +django-oauth-toolkit==1.7.1 +django-fsm==2.8.1 +websockets==11.0.3 +bleach==6.0.0 From 00a239128c8a5eaae260f9f3c07339dbcbcbd2a0 Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Tue, 2 Jul 2024 21:04:28 -1000 Subject: [PATCH 09/22] functional docker-compose up, next is testing the bash data/seed/init_db.sh --- .devcontainer/Dockerfile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 25119c7e8..0ed1f48a4 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -4,7 +4,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/base:bullseye # root acess for beginning of the script USER root -# Install main dependencies in one step to reduce layers +# Install system wide dependencies that cannot be done in a venv RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get install -y --no-install-recommends \ libpoppler-cpp-dev \ @@ -13,6 +13,9 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ libgdal-dev \ imagemagick \ libmagickwand-dev \ + python3-dev \ + pkg-config \ + build-essential \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -53,19 +56,17 @@ COPY codespaces-requirements.txt . # Display the contents of requirements.txt for debugging RUN echo "Displaying requirements.txt:" && cat codespaces-requirements.txt -# Update package list and install pip for Python 3 -RUN apt-get update && \ - apt-get install -y python3-pip +# Install Python virtual environment package and create a virtual environment +RUN sudo apt-get update && sudo apt-get install -y python3-venv -# Display Python and pip versions for debugging -RUN python3 --version && pip3 --version +# Create virtual environment +RUN python3 -m venv venv -# Upgrade pip -RUN pip3 install --upgrade pip +# Activate the virtual environment and install dependencies +RUN /bin/bash -c "source venv/bin/activate && pip install --upgrade pip && pip install -r codespaces-requirements.txt" - -# Install dependencies globally -RUN pip install -r codespaces-requirements.txt +# Debugging: Display Python and pip versions +RUN /bin/bash -c "source venv/bin/activate && python --version && pip --version" # Switch back to the baseuser USER baseuser From 3caba9b8e3a1c614983f77297eaac51cd16929f8 Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Tue, 2 Jul 2024 22:34:12 -1000 Subject: [PATCH 10/22] trying to fix port 8000, before it would run but kept closing --- .devcontainer/Dockerfile | 37 +++++++++++++++++++++++++++++--- .devcontainer/docker-compose.yml | 4 ++++ .devcontainer/entrypoint.sh | 19 ++++++++++++++++ .devcontainer/init_db.sh | 18 ++++++++++++++++ 4 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 .devcontainer/entrypoint.sh create mode 100644 .devcontainer/init_db.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 0ed1f48a4..35fe5521d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -50,7 +50,19 @@ RUN apt-get update \ RUN mkdir -p /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch \ && chown -R baseuser:baseuser /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch - # Copy the requirements file to the working directory + +# Copy your application code from the root directory +COPY .. . + + +# Copy the init file to the working directory + +COPY init_db.sh . + +# Ensure the init_db.sh script is executable +# RUN chmod +x /data/seed/init_db.sh + +# Copy the requirements file to the working directory COPY codespaces-requirements.txt . # Display the contents of requirements.txt for debugging @@ -63,10 +75,17 @@ RUN sudo apt-get update && sudo apt-get install -y python3-venv RUN python3 -m venv venv # Activate the virtual environment and install dependencies +# RUN /bin/bash -c "source venv/bin/activate && pip install --upgrade pip && pip install -r codespaces-requirements.txt && bash init_db.sh" RUN /bin/bash -c "source venv/bin/activate && pip install --upgrade pip && pip install -r codespaces-requirements.txt" # Debugging: Display Python and pip versions -RUN /bin/bash -c "source venv/bin/activate && python --version && pip --version" +RUN /bin/bash -c "source venv/bin/activate && python --version && pip --version" + + + +# Copy entrypoint script +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh # Switch back to the baseuser USER baseuser @@ -75,9 +94,21 @@ USER baseuser # Setting the ENTRYPOINT to docker-init.sh will configure non-root access # to the Docker socket. The script will also execute CMD as needed. -ENTRYPOINT [ "/usr/local/share/docker-init.sh" ] +# ENTRYPOINT [ "/usr/local/share/docker-init.sh" ] +ENTRYPOINT [ "/entrypoint.sh" ] + CMD [ "sleep", "infinity" ] + + + + +# might be able to do the pip install for requirements this way as well if this functions correctly +# Run database migrations and start the server +# CMD ["/bin/bash", "-c", "cd .. && source /app/venv/bin/activate && bash data/seed/init_db.sh && python manage.py migrate && python manage.py runserver 0.0.0.0:8000"] +# CMD ["/bin/bash",]# + + # [Optional] Uncomment this section to install additional OS packages. # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # && apt-get -y install --no-install-recommends diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 039ebffe6..24331c93a 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -3,6 +3,10 @@ services: build: context: . dockerfile: Dockerfile + ports: + # - "8000:8000" # Add this line to map port 8000 + - "127.0.0.1:8000:8000" # Only accessible from the host machine + db: image: postgis/postgis:14-3.1 volumes: diff --git a/.devcontainer/entrypoint.sh b/.devcontainer/entrypoint.sh new file mode 100644 index 000000000..3c1662172 --- /dev/null +++ b/.devcontainer/entrypoint.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Navigate to the project root directory +cd .. + +# Activate the virtual environment +source venv/bin/activate + +# Run the initial setup script +bash data/seed/init_db.sh + +python manage.py check + +# Apply database migrations +# python manage.py migrate + +# Start the development server +# python manage.py runserver 0.0.0.0:8000 +python manage.py runserver diff --git a/.devcontainer/init_db.sh b/.devcontainer/init_db.sh new file mode 100644 index 000000000..2a25133ec --- /dev/null +++ b/.devcontainer/init_db.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# init_db.sh - Initialize the database to the point before uploading public +# body data. +# + +echo "Run initial migration..." +python manage.py migrate + +echo +echo "Create and populate the search index..." +python manage.py search_index --populate + +echo +echo "Load the seed data..." +python manage.py loaddata uipa_org/fixtures/* + +echo +echo "Now you can start the dev web server." From 67d8c1269b48ad87bedf463316c7d60d40ade4aa Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Tue, 2 Jul 2024 23:07:25 -1000 Subject: [PATCH 11/22] further testing needing for automation of server setup --- .devcontainer/Dockerfile | 7 +++--- .devcontainer/devcontainer.json | 1 + .devcontainer/docker-compose.yml | 5 ++++- .devcontainer/entrypoint.sh | 38 +++++++++++++++++++++++++++----- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 35fe5521d..ec1f19af5 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -95,9 +95,9 @@ USER baseuser # Setting the ENTRYPOINT to docker-init.sh will configure non-root access # to the Docker socket. The script will also execute CMD as needed. # ENTRYPOINT [ "/usr/local/share/docker-init.sh" ] -ENTRYPOINT [ "/entrypoint.sh" ] +# ENTRYPOINT [ "/entrypoint.sh" ] -CMD [ "sleep", "infinity" ] +# CMD [ "sleep", "infinity" ] @@ -105,8 +105,9 @@ CMD [ "sleep", "infinity" ] # might be able to do the pip install for requirements this way as well if this functions correctly # Run database migrations and start the server -# CMD ["/bin/bash", "-c", "cd .. && source /app/venv/bin/activate && bash data/seed/init_db.sh && python manage.py migrate && python manage.py runserver 0.0.0.0:8000"] +CMD ["/bin/bash", "-c", "cd .. && source /app/venv/bin/activate && bash data/seed/init_db.sh && python manage.py migrate && python manage.py runserver"] # CMD ["/bin/bash",]# +# CMD [ "sleep", "infinity" ] # [Optional] Uncomment this section to install additional OS packages. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7b5b518f7..bb12d5982 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,7 @@ { "name": "UIPA DevBox", "dockerComposeFile": "docker-compose.yml", + // "postStartCommand": "bash entrypoint.sh", "service": "app", "forwardPorts": [8000, 5432, 9200], "customizations": { diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 24331c93a..b18c03383 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -5,7 +5,10 @@ services: dockerfile: Dockerfile ports: # - "8000:8000" # Add this line to map port 8000 - - "127.0.0.1:8000:8000" # Only accessible from the host machine + # - "127.0.0.1:8000:8000" + # entrypoint: ["/bin/bash", "/entrypoint.sh"] + + db: image: postgis/postgis:14-3.1 diff --git a/.devcontainer/entrypoint.sh b/.devcontainer/entrypoint.sh index 3c1662172..4fb64ae1a 100644 --- a/.devcontainer/entrypoint.sh +++ b/.devcontainer/entrypoint.sh @@ -1,19 +1,47 @@ #!/bin/bash +echo "Starting entrypoint script..." + # Navigate to the project root directory +echo "Changing directory to project root..." cd .. # Activate the virtual environment +echo "Activating virtual environment..." source venv/bin/activate # Run the initial setup script -bash data/seed/init_db.sh +echo "Running initial setup script..." +if bash data/seed/init_db.sh; then + echo "Initial setup script completed successfully." +else + echo "Initial setup script failed." >&2 + exit 1 +fi -python manage.py check +# Check the Django project +echo "Checking Django project..." +if python manage.py check; then + echo "Django project check completed successfully." +else + echo "Django project check failed." >&2 + exit 1 +fi # Apply database migrations -# python manage.py migrate +echo "Applying database migrations..." +if python manage.py migrate; then + echo "Database migrations applied successfully." +else + echo "Database migrations failed." >&2 + exit 1 +fi # Start the development server -# python manage.py runserver 0.0.0.0:8000 -python manage.py runserver +echo "Starting the development server..." +if python manage.py runserver 0.0.0.0:8000; then + echo "Development server started successfully." +else + echo "Failed to start the development server." >&2 + exit 1 +fi From c759734c6a65ec2123d3319860c37b52d7cfce40 Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Mon, 8 Jul 2024 21:05:19 -1000 Subject: [PATCH 12/22] copied entire root dir into docker for testing, 1 of the containers is not passing health check --- .devcontainer/Dockerfile | 66 +++++++++++++------------------- .devcontainer/docker-compose.yml | 41 +++++++++++++------- 2 files changed, 53 insertions(+), 54 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index ec1f19af5..99c96509c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,10 +1,10 @@ -# Note: You can use any Debian/Ubuntu based image you want. +# Use any Debian/Ubuntu based image FROM mcr.microsoft.com/vscode/devcontainers/base:bullseye -# root acess for beginning of the script +# root access for the beginning of the script USER root -# Install system wide dependencies that cannot be done in a venv +# Install system-wide dependencies that cannot be done in a venv RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get install -y --no-install-recommends \ libpoppler-cpp-dev \ @@ -16,76 +16,59 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ python3-dev \ pkg-config \ build-essential \ + python3-venv \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -# [Option] Install zsh +# Optional arguments ARG INSTALL_ZSH="true" -# [Option] Upgrade OS packages to their latest versions ARG UPGRADE_PACKAGES="false" -# [Option] Enable non-root Docker access in container ARG ENABLE_NONROOT_DOCKER="true" -# [Option] Use the OSS Moby CLI instead of the licensed Docker CLI ARG USE_MOBY="true" # Enable new "BUILDKIT" mode for Docker CLI ENV DOCKER_BUILDKIT=1 -# Install needed packages and setup non-root user. Use a separate RUN statement to add your -# own dependencies. A user of "automatic" attempts to reuse an user ID if one already exists. +# Install needed packages and setup non-root user ARG USERNAME=automatic ARG USER_UID=1000 ARG USER_GID=$USER_UID -COPY library-scripts/*.sh /tmp/library-scripts/ + +COPY .devcontainer/library-scripts/ /tmp/library-scripts/ + RUN apt-get update \ && /bin/bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \ - # Use Docker script from script library to set things up && /bin/bash /tmp/library-scripts/docker-debian.sh "${ENABLE_NONROOT_DOCKER}" "/var/run/docker-host.sock" "/var/run/docker.sock" "${USERNAME}" \ - # Create the baseuser user and group && groupadd -r baseuser && useradd -r -g baseuser baseuser \ - # Clean up && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/library-scripts/ # Create directories for Elasticsearch and add necessary permissions RUN mkdir -p /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch \ && chown -R baseuser:baseuser /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch +WORKDIR /home/app # Copy your application code from the root directory -COPY .. . - - -# Copy the init file to the working directory - -COPY init_db.sh . +# COPY .. /home/app/ +COPY . /home/app/ -# Ensure the init_db.sh script is executable -# RUN chmod +x /data/seed/init_db.sh +RUN chmod +x /home/app/data/seed/init_db.sh -# Copy the requirements file to the working directory -COPY codespaces-requirements.txt . +# Display the contents of codespaces-requirements.txt for debugging +# RUN echo "Displaying codespaces-requirements.txt:" && cat /home/app/codespaces-requirements.txt +RUN echo "Displaying requirements.txt:" && cat /home/app/requirements.txt -# Display the contents of requirements.txt for debugging -RUN echo "Displaying requirements.txt:" && cat codespaces-requirements.txt +# Create the virtual environment +RUN python3 -m venv /home/app/venv -# Install Python virtual environment package and create a virtual environment -RUN sudo apt-get update && sudo apt-get install -y python3-venv - -# Create virtual environment -RUN python3 -m venv venv +# Debugging: Display directory structure to check if venv is created correctly +RUN ls -R /home/app # Activate the virtual environment and install dependencies -# RUN /bin/bash -c "source venv/bin/activate && pip install --upgrade pip && pip install -r codespaces-requirements.txt && bash init_db.sh" -RUN /bin/bash -c "source venv/bin/activate && pip install --upgrade pip && pip install -r codespaces-requirements.txt" +RUN /bin/bash -c "source /home/app/venv/bin/activate && pip install --upgrade pip && pip install -r /home/app/requirements.txt" # Debugging: Display Python and pip versions -RUN /bin/bash -c "source venv/bin/activate && python --version && pip --version" - - - -# Copy entrypoint script -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh +RUN /bin/bash -c "source /home/app/venv/bin/activate && python --version && pip --version" # Switch back to the baseuser USER baseuser @@ -105,9 +88,12 @@ USER baseuser # might be able to do the pip install for requirements this way as well if this functions correctly # Run database migrations and start the server -CMD ["/bin/bash", "-c", "cd .. && source /app/venv/bin/activate && bash data/seed/init_db.sh && python manage.py migrate && python manage.py runserver"] +# CMD ["/bin/bash", "-c", "cd .. && source /app/venv/bin/activate && bash data/seed/init_db.sh && python manage.py migrate && python manage.py runserver"] # CMD ["/bin/bash",]# # CMD [ "sleep", "infinity" ] +# Set the CMD to run your application directly +CMD ["/bin/bash", "-c", "source /home/app/venv/bin/activate && bash /home/app/data/seed/init_db.sh && python /home/app/manage.py migrate && python /home/app/manage.py runserver 0.0.0.0:8000"] + # [Optional] Uncomment this section to install additional OS packages. diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index b18c03383..560c4c974 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,14 +1,16 @@ +# version: '3.8' services: app: build: - context: . - dockerfile: Dockerfile + context: .. + dockerfile: .devcontainer/Dockerfile ports: - # - "8000:8000" # Add this line to map port 8000 - # - "127.0.0.1:8000:8000" - # entrypoint: ["/bin/bash", "/entrypoint.sh"] - - + - "127.0.0.1:8000:8000" + depends_on: + db: + condition: service_healthy + elasticsearch: + condition: service_healthy db: image: postgis/postgis:14-3.1 @@ -20,6 +22,12 @@ services: POSTGRES_PASSWORD: froide ports: - "127.0.0.1:5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U froide -d froide"] + interval: 10s + timeout: 5s + retries: 5 + elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0 user: root @@ -27,15 +35,20 @@ services: - es-data:/usr/share/elasticsearch/data - es-logs:/var/log environment: - - "discovery.type=single-node" - - "xpack.security.enabled=true" - - "ELASTIC_PASSWORD=froide" - - "cluster.routing.allocation.disk.threshold_enabled=false" - - "cluster.routing.allocation.disk.watermark.low=3gb" - - "cluster.routing.allocation.disk.watermark.high=2gb" - - "cluster.routing.allocation.disk.watermark.flood_stage=1gb" + - discovery.type=single-node + - xpack.security.enabled=true + - ELASTIC_PASSWORD=froide + - cluster.routing.allocation.disk.threshold_enabled=false + - cluster.routing.allocation.disk.watermark.low=3gb + - cluster.routing.allocation.disk.watermark.high=2gb + - cluster.routing.allocation.disk.watermark.flood_stage=1gb ports: - "127.0.0.1:9200:9200" + healthcheck: + test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"] + interval: 10s + timeout: 5s + retries: 5 volumes: es-data: {} From 6af8a790b2f56478426a0ce0907c80a96ea37297 Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Mon, 8 Jul 2024 21:22:22 -1000 Subject: [PATCH 13/22] still unhealthy with extra timer, will need more investigation --- .devcontainer/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 560c4c974..401b26726 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,4 +1,4 @@ -# version: '3.8' +version: '3.8' services: app: build: @@ -45,10 +45,10 @@ services: ports: - "127.0.0.1:9200:9200" healthcheck: - test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"] - interval: 10s - timeout: 5s - retries: 5 + test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"' || curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"yellow\"'"] + interval: 30s + timeout: 10s + retries: 10 volumes: es-data: {} From 7c81652f577a4e579d8d99c476c0d46a74e2607f Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Sun, 21 Jul 2024 21:40:42 -0400 Subject: [PATCH 14/22] updated health check to provide creds using ' -u ' --- .devcontainer/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 401b26726..8bc989895 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -45,7 +45,7 @@ services: ports: - "127.0.0.1:9200:9200" healthcheck: - test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"' || curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"yellow\"'"] + test: ["CMD-SHELL", "curl -s -u elastic:froide http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"' || curl -s -u elastic:froide http://localhost:9200/_cluster/health | grep -q '\"status\":\"yellow\"'"] interval: 30s timeout: 10s retries: 10 From a424f34e2339dcbaa6f296dd3b7237ee4f447989 Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Sun, 21 Jul 2024 22:03:27 -0400 Subject: [PATCH 15/22] changes to compose to connect postgres and elastic to app 0 currently an error with django-configurations --- .devcontainer/docker-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 8bc989895..1b74a6738 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -6,11 +6,15 @@ services: dockerfile: .devcontainer/Dockerfile ports: - "127.0.0.1:8000:8000" + environment: + - DATABASE_URL=postgres://froide:froide@db:5432/froide + - ELASTICSEARCH_URL=http://elastic:froide@elasticsearch:9200 depends_on: db: condition: service_healthy elasticsearch: condition: service_healthy + command: bash -c "while ! Date: Mon, 22 Jul 2024 14:57:33 -0400 Subject: [PATCH 16/22] working on the #! | AttributeError: 'DatabaseOperations' object has no attribute 'geo_db_type' --- .devcontainer/docker-compose.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 1b74a6738..1b2b1bc8d 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -14,7 +14,10 @@ services: condition: service_healthy elasticsearch: condition: service_healthy - command: bash -c "while ! Date: Mon, 22 Jul 2024 15:24:25 -0400 Subject: [PATCH 17/22] changes to get the static files error and to ensure readyness for containers --- .devcontainer/Dockerfile | 19 +++++++++++++++++++ .devcontainer/docker-compose.yml | 14 ++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 99c96509c..35fc86c2f 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -17,6 +17,10 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ pkg-config \ build-essential \ python3-venv \ + libproj-dev \ + proj-data \ + proj-bin \ + libgeos-dev \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -46,6 +50,21 @@ RUN apt-get update \ RUN mkdir -p /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch \ && chown -R baseuser:baseuser /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch + + + + + + + +# Create the static files directory +RUN mkdir -p /home/app/build + + + + + + WORKDIR /home/app # Copy your application code from the root directory diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 1b2b1bc8d..19dbd9208 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -15,10 +15,10 @@ services: elasticsearch: condition: service_healthy # command: bash -c "while ! + bash -c " + docker-entrypoint.sh postgres & + sleep 10; + psql -U froide -d froide -c 'CREATE EXTENSION IF NOT EXISTS postgis;' + " elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0 From ee5426473930baf5e467263111be9d034c06275c Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Mon, 22 Jul 2024 16:12:47 -0400 Subject: [PATCH 18/22] app trying to connect to DB, but unable to establish a connection --- .devcontainer/Dockerfile | 21 ++++++++++++++++++--- .devcontainer/docker-compose.yml | 20 +++++++++++++------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 35fc86c2f..4da6b332a 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -56,7 +56,7 @@ RUN mkdir -p /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsea - + # Create the static files directory RUN mkdir -p /home/app/build @@ -111,8 +111,23 @@ USER baseuser # CMD ["/bin/bash",]# # CMD [ "sleep", "infinity" ] # Set the CMD to run your application directly -CMD ["/bin/bash", "-c", "source /home/app/venv/bin/activate && bash /home/app/data/seed/init_db.sh && python /home/app/manage.py migrate && python /home/app/manage.py runserver 0.0.0.0:8000"] - +# CMD ["/bin/bash", "-c", "source /home/app/venv/bin/activate && bash /home/app/data/seed/init_db.sh && python /home/app/manage.py migrate && python /home/app/manage.py runserver 0.0.0.0:8000"] +# CMD ["/bin/bash", "-c", "while ! /dev/null 2>&1; do \ + echo 'Waiting for db...'; \ + sleep 5; \ +done; \ +while ! ping -c 1 elasticsearch > /dev/null 2>&1; do \ + echo 'Waiting for elasticsearch...'; \ + sleep 5; \ +done; \ +source /home/app/venv/bin/activate; \ +bash /home/app/data/seed/init_db.sh; \ +python /home/app/manage.py migrate; \ +python /home/app/manage.py runserver 0.0.0.0:8000; \ +"] # [Optional] Uncomment this section to install additional OS packages. diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 19dbd9208..189caa578 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: app: - build: + build: context: .. dockerfile: .devcontainer/Dockerfile ports: @@ -14,11 +14,9 @@ services: condition: service_healthy elasticsearch: condition: service_healthy - # command: bash -c "while ! Date: Mon, 22 Jul 2024 17:09:16 -0400 Subject: [PATCH 19/22] installed postgresql-client and curl onto dockerfile dependencies, result: Confirmed that both are accessible with: sudo docker exec -it devcontainer_app_1 /bin/bash psql --version curl --version --- .devcontainer/Dockerfile | 41 ++------------------------------ .devcontainer/docker-compose.yml | 3 ++- .devcontainer/postgresql.conf | 1 + 3 files changed, 5 insertions(+), 40 deletions(-) create mode 100644 .devcontainer/postgresql.conf diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4da6b332a..57281d337 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -21,6 +21,8 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ proj-data \ proj-bin \ libgeos-dev \ + postgresql-client \ + curl \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -50,33 +52,16 @@ RUN apt-get update \ RUN mkdir -p /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch \ && chown -R baseuser:baseuser /usr/share/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch - - - - - - - # Create the static files directory RUN mkdir -p /home/app/build - - - - - WORKDIR /home/app # Copy your application code from the root directory -# COPY .. /home/app/ COPY . /home/app/ RUN chmod +x /home/app/data/seed/init_db.sh -# Display the contents of codespaces-requirements.txt for debugging -# RUN echo "Displaying codespaces-requirements.txt:" && cat /home/app/codespaces-requirements.txt -RUN echo "Displaying requirements.txt:" && cat /home/app/requirements.txt - # Create the virtual environment RUN python3 -m venv /home/app/venv @@ -92,28 +77,11 @@ RUN /bin/bash -c "source /home/app/venv/bin/activate && python --version && pip # Switch back to the baseuser USER baseuser -# Setup the app here - # Setting the ENTRYPOINT to docker-init.sh will configure non-root access # to the Docker socket. The script will also execute CMD as needed. # ENTRYPOINT [ "/usr/local/share/docker-init.sh" ] -# ENTRYPOINT [ "/entrypoint.sh" ] - -# CMD [ "sleep", "infinity" ] - - - - -# might be able to do the pip install for requirements this way as well if this functions correctly -# Run database migrations and start the server -# CMD ["/bin/bash", "-c", "cd .. && source /app/venv/bin/activate && bash data/seed/init_db.sh && python manage.py migrate && python manage.py runserver"] -# CMD ["/bin/bash",]# -# CMD [ "sleep", "infinity" ] # Set the CMD to run your application directly -# CMD ["/bin/bash", "-c", "source /home/app/venv/bin/activate && bash /home/app/data/seed/init_db.sh && python /home/app/manage.py migrate && python /home/app/manage.py runserver 0.0.0.0:8000"] -# CMD ["/bin/bash", "-c", "while ! /dev/null 2>&1; do \ echo 'Waiting for db...'; \ @@ -128,8 +96,3 @@ bash /home/app/data/seed/init_db.sh; \ python /home/app/manage.py migrate; \ python /home/app/manage.py runserver 0.0.0.0:8000; \ "] - - -# [Optional] Uncomment this section to install additional OS packages. -# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ -# && apt-get -y install --no-install-recommends diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 189caa578..b3c962e2e 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -21,6 +21,7 @@ services: image: postgis/postgis:14-3.1 volumes: - pg-data:/var/lib/postgresql/data/ + - ./postgresql.conf:/var/lib/postgresql/data/postgresql.conf environment: POSTGRES_USER: froide POSTGRES_PASSWORD: froide @@ -34,7 +35,7 @@ services: retries: 5 command: > bash -c " - docker-entrypoint.sh postgres & + docker-entrypoint.sh postgres -c 'config_file=/var/lib/postgresql/data/postgresql.conf' & sleep 10; psql -U froide -d froide -c 'CREATE EXTENSION IF NOT EXISTS postgis;'; tail -f /dev/null diff --git a/.devcontainer/postgresql.conf b/.devcontainer/postgresql.conf new file mode 100644 index 000000000..128ef1aa8 --- /dev/null +++ b/.devcontainer/postgresql.conf @@ -0,0 +1 @@ +listen_addresses = '*' From f51e839d09c20302fc06ecc62006293a0affc60f Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Mon, 22 Jul 2024 17:27:44 -0400 Subject: [PATCH 20/22] added a password for base user to test out pings and other installs --- .devcontainer/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 57281d337..a0814dd6c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -23,6 +23,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ libgeos-dev \ postgresql-client \ curl \ + passwd \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -46,6 +47,7 @@ RUN apt-get update \ && /bin/bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \ && /bin/bash /tmp/library-scripts/docker-debian.sh "${ENABLE_NONROOT_DOCKER}" "/var/run/docker-host.sock" "/var/run/docker.sock" "${USERNAME}" \ && groupadd -r baseuser && useradd -r -g baseuser baseuser \ + && echo 'baseuser:password' | chpasswd \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/library-scripts/ # Create directories for Elasticsearch and add necessary permissions From 1b2b47a15e0337b48e75f550d858ac0c8a414708 Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Mon, 22 Jul 2024 19:10:07 -0400 Subject: [PATCH 21/22] missing geo_db_type --- .devcontainer/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a0814dd6c..907a1d360 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,3 @@ -# Use any Debian/Ubuntu based image FROM mcr.microsoft.com/vscode/devcontainers/base:bullseye # root access for the beginning of the script @@ -23,7 +22,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ libgeos-dev \ postgresql-client \ curl \ - passwd \ + iputils-ping \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -47,7 +46,6 @@ RUN apt-get update \ && /bin/bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \ && /bin/bash /tmp/library-scripts/docker-debian.sh "${ENABLE_NONROOT_DOCKER}" "/var/run/docker-host.sock" "/var/run/docker.sock" "${USERNAME}" \ && groupadd -r baseuser && useradd -r -g baseuser baseuser \ - && echo 'baseuser:password' | chpasswd \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/library-scripts/ # Create directories for Elasticsearch and add necessary permissions From 30eb9b74a0a3482ab061de64a3745988daf56b7c Mon Sep 17 00:00:00 2001 From: Kobe Buckley Date: Mon, 22 Jul 2024 19:28:45 -0400 Subject: [PATCH 22/22] still issues with app_1 | AttributeError: 'DatabaseOperations' object has no attribute 'geo_db_type' --- .devcontainer/Dockerfile | 3 +++ .devcontainer/docker-compose.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 907a1d360..507d91053 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -3,6 +3,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/base:bullseye # root access for the beginning of the script USER root +# Install system-wide dependencies that cannot be done in a venv # Install system-wide dependencies that cannot be done in a venv RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get install -y --no-install-recommends \ @@ -23,6 +24,8 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ postgresql-client \ curl \ iputils-ping \ + postgis \ + postgresql-postgis \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index b3c962e2e..19a0aa6e3 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,4 +1,5 @@ version: '3.8' + services: app: build: