diff --git a/README.md b/README.md index 6322e31aa1..1f44e55569 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,31 @@ Doing this in us-east-1 - Virginia and used the base name `doccano`, so for inst ### Populate Secrets needed by the EC2 - add useful secrets to secrets manager: - quay_io_creds (quay.io login creds) - - doccano_creds (all the information needed in the .env file, mostly doccano and DB credentials) + ``` + { + "user":"", + "password":"" + } + ``` + - doccano_creds + ``` + { + "ADMIN_PASSWORD":"", + "RABBITMQ_DEFAULT_PASS":"", + "POSTGRES_PASSWORD":"", + "FLOWER_BASIC_AUTH":"", + "POSTGRES_HOST":"" + } + ``` + - gearbox_creds + ``` + { + "FENCE_CLIENT_ID":"", + "FENCE_CLIENT_SECRET":"", + "FENCE_TOKEN_URL":"https://gearbox-dev.pedscommons.org/user/oauth2/token?grant_type=client_credentials&scope=openid", + "GEARBOX_RAW_CRITERIA_URL":"https://gearbox-dev.pedscommons.org/gearbox/raw-criteria" + } + ``` ### Create VPC Select the following options: @@ -292,7 +316,7 @@ and add the ec2 created previously - add listener to listen to 80 and redirect (redirect to url) 301 to 443 - Redirect to HTTPS://#{host}:443/#{path}?#{query} ### Deploy new version -- Update the ec2_user_data.sh file with the new tag +- Update the docker-compose.prod.yml file with the new tag/s - Repeat the step `Create EC2 instance` - Repeat the step `Add instance/s to the target group` - Remove old instance from the target group diff --git a/backend/api/gearbox_client.py b/backend/api/gearbox_client.py index 73e9c74704..aeca9c159e 100644 --- a/backend/api/gearbox_client.py +++ b/backend/api/gearbox_client.py @@ -30,5 +30,9 @@ def submit_to_gearbox(jsonl_content: bytes, filename: str = "annotations.jsonl") files={"file": ("annotations.zip", zip_buffer, "application/zip")}, timeout=30, ) - response.raise_for_status() + try: + response.raise_for_status() + except requests.exceptions.HTTPError as e: + logger.error("Gearbox returned %s: %s", e.response.status_code, e.response.text[:500]) + raise return response diff --git a/backend/examples/celery_tasks.py b/backend/examples/celery_tasks.py index e72e22d7b3..2392e22f3e 100644 --- a/backend/examples/celery_tasks.py +++ b/backend/examples/celery_tasks.py @@ -39,6 +39,9 @@ def _transform_to_gearbox_format(jsonl_bytes: bytes) -> bytes: entities = [] pre_annotated = [] for entry in raw_labels: + if len(entry) < 3: + logger.warning("Skipping malformed label entry for example: %s", entry) + continue start_offset, end_offset, label_name = entry[0], entry[1], entry[2] meta = entry[3] if len(entry) > 3 else {} entities.append( @@ -92,6 +95,11 @@ def submit_example_to_gearbox(self, example_id, project_id): example.gearbox_status = "failed" example.save(update_fields=["gearbox_status"]) raise + except Exception: + example.gearbox_status = "failed" + example.save(update_fields=["gearbox_status"]) + logger.exception("Unexpected error submitting example %s to gearbox", example.pk) + raise example.gearbox_status = "success" example.save(update_fields=["gearbox_status"]) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 0461ff1a1f..95d152488a 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -2,7 +2,7 @@ version: "3.7" services: backend: - image: quay.io/pcdc/doccano:be_20250618 + image: quay.io/pcdc/doccano:be_20260701 volumes: - static_volume:/backend/staticfiles - media:/backend/media @@ -27,7 +27,7 @@ services: - network-frontend celery: - image: quay.io/pcdc/doccano:be_20250618 + image: quay.io/pcdc/doccano:be_20260701 volumes: - media:/backend/media - tmp_file:/backend/filepond-temp-uploads @@ -48,7 +48,7 @@ services: - network-backend flower: - image: quay.io/pcdc/doccano:be_20250618 + image: quay.io/pcdc/doccano:be_20260701 entrypoint: ["/opt/bin/prod-flower.sh"] environment: PYTHONUNBUFFERED: "1" @@ -75,7 +75,7 @@ services: - network-backend nginx: - image: quay.io/pcdc/doccano:fe_20250618 + image: quay.io/pcdc/doccano:fe_20260701 command: > /bin/sh -c "envsubst ' diff --git a/ec2_user_data.sh b/ec2_user_data.sh index 1416cdb3ae..73087a77cb 100644 --- a/ec2_user_data.sh +++ b/ec2_user_data.sh @@ -25,6 +25,14 @@ POSTGRES_HOST="$(echo $doccano_secrets | jq -r '.POSTGRES_HOST')" ENCODED_POSTGRES_PASSWORD=$(jq -rn --arg pwd $POSTGRES_PASSWORD '$pwd|@uri') +gearbox_secrets_str=$(aws secretsmanager get-secret-value --secret-id gearbox_creds) +gearbox_secrets="$(echo $gearbox_secrets_str | jq '.SecretString' | jq '. | fromjson')" +FENCE_CLIENT_ID="$(echo $gearbox_secrets | jq -r '.FENCE_CLIENT_ID')" +FENCE_CLIENT_SECRET="$(echo $gearbox_secrets | jq -r '.FENCE_CLIENT_SECRET')" +FENCE_TOKEN_URL="$(echo $gearbox_secrets | jq -r '.FENCE_TOKEN_URL')" +GEARBOX_RAW_CRITERIA_URL="$(echo $gearbox_secrets | jq -r '.GEARBOX_RAW_CRITERIA_URL')" + + #define the template. env_file=$(cat << EOF # platform settings @@ -44,6 +52,12 @@ POSTGRES_DB=postgres # Flower settings FLOWER_BASIC_AUTH='$FLOWER_BASIC_AUTH' + +# GEARBOx settings +FENCE_CLIENT_ID='$FENCE_CLIENT_ID' +FENCE_CLIENT_SECRET='$FENCE_CLIENT_SECRET' +FENCE_TOKEN_URL='$FENCE_TOKEN_URL' +GEARBOX_RAW_CRITERIA_URL='$GEARBOX_RAW_CRITERIA_URL' EOF ) echo "$env_file" > ./.env @@ -54,13 +68,3 @@ docker login quay.io -p "$(echo $quay_secrets | jq '.SecretString' | jq '. | fro sudo docker-compose -f docker/docker-compose.prod.yml --env-file .env up -d - -#old command -# sed -i s/password/D4cGTech/g .env -# sed -i s/admin/admin/g .env -# sed -i s/FLOWER_BASIC_AUTH=""/FLOWER_BASIC_AUTH="admin:D4cGTech"/g .env -# sed -i s/FLOWER_BASIC_AUTH=""/FLOWER_BASIC_AUTH=\"admin:D4cGTech\"/g .env -# sed -i s/FLOWER_BASIC_AUTH=\"\"/FLOWER_BASIC_AUTH=\"admin:D4cGTech\"/g .env - - -