Pcdc dev

.github/workflows/ci.yml

@@ -10,10 +10,10 @@ jobs:
         working-directory: ./backend
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/pypi-publish.yml

@@ -7,7 +7,11 @@ on:
 jobs:
   deploy:
     runs-on: ubuntu-latest
-
+    environment:
+      name: pypi
+      url: https://pypi.org/p/doccano
+    permissions:
+      id-token: write
     steps:
       - uses: actions/checkout@v4
       - name: Preparation
@@ -27,10 +31,10 @@ jobs:
         working-directory: ./frontend
         env:
           PUBLIC_PATH: '/static/_nuxt/'
-      - name: Setup Python 3.8
+      - name: Setup Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -46,8 +50,7 @@ jobs:
           sed -e "s/, from = \"..\"//g" backend/pyproject.toml > pyproject.toml
           poetry build
       - name: Publish a Python distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@master
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          user: ${{ secrets.PYPI_USERNAME }}
-          password: ${{ secrets.PYPI_PASSWORD }}
+          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           packages_dir: ./dist/

.github/workflows/test-installation.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ['3.10', '3.11', '3.12']
     steps:
       - name: Set up Python
         uses: actions/setup-python@v5

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 TIS inc.
+Copyright (c) 2018 Hiroki Nakayama
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -226,7 +226,31 @@ Doing this in us-east-1 - Virginia and used the base name `doccano`, so for inst
 ### Populate Secrets needed by the EC2
 - add useful secrets to secrets manager:
   - quay_io_creds (quay.io login creds)
-  - doccano_creds (all the information needed in the .env file, mostly doccano and DB credentials)
+    ```
+    {
+      "user":"",
+      "password":""
+    }
+    ```
+  - doccano_creds
+    ```
+    {
+      "ADMIN_PASSWORD":"",
+      "RABBITMQ_DEFAULT_PASS":"",
+      "POSTGRES_PASSWORD":"",
+      "FLOWER_BASIC_AUTH":"",
+      "POSTGRES_HOST":""
+    }
+    ```
+  - gearbox_creds
+    ```
+    {
+      "FENCE_CLIENT_ID":"",
+      "FENCE_CLIENT_SECRET":"",
+      "FENCE_TOKEN_URL":"https://gearbox-dev.pedscommons.org/user/oauth2/token?grant_type=client_credentials&scope=openid",
+      "GEARBOX_RAW_CRITERIA_URL":"https://gearbox-dev.pedscommons.org/gearbox/raw-criteria"
+    }
+    ```
 
 ### Create VPC
 Select the following options:
@@ -282,8 +306,9 @@ Select the following options:
 - click on create
 
 ### Add instance/s to the target group
-and add the ec2 created previously
-  - For some reason if the instance is only internal it will now turn healthy in the target group. You can just attach a public IP, wait 2/3 minutes for it to turn healthy and then you can remove the elastic IP. That seems to be solving the issue. (TRY added ALB's security group on port 80 in EC2's security group. By doing this, the issue will be resolved.)
+- add the ec2 created previously
+  - Make sure the ALB's security group is added on port 80 in EC2's security group.
+- wait for it to be healthy.
 
 ### Update SSL Cert and listeners
 - get a cert from certificate manager for the ALB
@@ -292,7 +317,7 @@ and add the ec2 created previously
 - add listener to listen to 80 and redirect (redirect to url) 301 to 443 - Redirect to HTTPS://#{host}:443/#{path}?#{query}
 
 ### Deploy new version
-- Update the ec2_user_data.sh file with the new tag
+- Update the docker-compose.prod.yml file with the new tag/s and make sure all the changes are on pcdc_dev branch (this is the branch used in the ec2 user data to start the app, we may want to update that to master for stability and use pcdc_dev for testing TODO)
 - Repeat the step `Create EC2 instance`
 - Repeat the step `Add instance/s to the target group`
 - Remove old instance from the target group

backend/api/gearbox_client.py

@@ -0,0 +1,41 @@
+import io
+import zipfile
+import logging
+
+import requests
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+
+def get_access_token():
+    response = requests.post(
+        settings.FENCE_TOKEN_URL,
+        data={"grant_type": "client_credentials", "scope": "openid user"},
+        auth=(settings.FENCE_CLIENT_ID, settings.FENCE_CLIENT_SECRET),
+        timeout=30,
+    )
+    response.raise_for_status()
+    return response.json()["access_token"]
+
+
+def submit_to_gearbox(jsonl_content: bytes, filename: str = "annotations.jsonl"):
+    token = get_access_token()
+
+    zip_buffer = io.BytesIO()
+    with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
+        zf.writestr(filename, jsonl_content)
+    zip_buffer.seek(0)
+
+    response = requests.post(
+        settings.GEARBOX_RAW_CRITERIA_URL,
+        headers={"Authorization": f"Bearer {token}"},
+        files={"file": ("annotations.zip", zip_buffer, "application/zip")},
+        timeout=30,
+    )
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        logger.error("Gearbox returned %s: %s", e.response.status_code, e.response.text[:500])
+        raise
+    return response

backend/api/urls.py

@@ -3,5 +3,5 @@
 from .views import TaskStatus
 
 urlpatterns = [
-    path(route="tasks/status/<task_id>", view=TaskStatus.as_view(), name="task_status"),
+    path(route="tasks/status/<uuid:task_id>", view=TaskStatus.as_view(), name="task_status"),
 ]

backend/config/settings/base.py

@@ -293,3 +293,8 @@
 }
 
 SITE_ID = 1
+
+FENCE_CLIENT_ID = env("FENCE_CLIENT_ID", "")
+FENCE_CLIENT_SECRET = env("FENCE_CLIENT_SECRET", "")
+FENCE_TOKEN_URL = env("FENCE_TOKEN_URL", "")
+GEARBOX_RAW_CRITERIA_URL = env("GEARBOX_RAW_CRITERIA_URL", "")

backend/examples/assignment/strategies.py

@@ -32,8 +32,7 @@ def create_assignment_strategy(strategy_name: StrategyName, dataset_size: int, w
 
 class BaseStrategy(abc.ABC):
     @abc.abstractmethod
-    def assign(self) -> List[Assignment]:
-        ...
+    def assign(self) -> List[Assignment]: ...
 
 
 class WeightedSequentialStrategy(BaseStrategy):

backend/examples/celery_tasks.py

@@ -0,0 +1,106 @@
+import json
+import tempfile
+
+import requests
+from celery import shared_task
+from celery.utils.log import get_task_logger
+from django.shortcuts import get_object_or_404
+
+from api.gearbox_client import submit_to_gearbox
+from data_export.models import ExportedExample
+from data_export.pipeline.dataset import Dataset
+from data_export.pipeline.factories import (
+    create_comment,
+    create_formatter,
+    create_labels,
+    create_writer,
+)
+from data_export.pipeline.services import ExportApplicationService
+from examples.models import Example
+from projects.models import Project
+
+logger = get_task_logger(__name__)
+
+
+def _transform_to_gearbox_format(jsonl_bytes: bytes) -> bytes:
+    """Convert doccano JSONL export format to gearbox RawCriteriaIn format.
+
+    Doccano exports label as [[start, end, name, meta], ...].
+    Gearbox expects entities as [{"start_offset":..., "end_offset":..., "label":..., "meta":...}]
+    and pre_annotated as [{"span":[start,end,name], "matched_models":[...], "is_standard_gb_var":...}].
+    """
+    lines = jsonl_bytes.decode("utf-8").strip().split("\n")
+    out = []
+    for line in lines:
+        if not line.strip():
+            continue
+        obj = json.loads(line)
+        raw_labels = obj.pop("label", [])
+        entities = []
+        pre_annotated = []
+        for entry in raw_labels:
+            if len(entry) < 3:
+                logger.warning("Skipping malformed label entry for example: %s", entry)
+                continue
+            start_offset, end_offset, label_name = entry[0], entry[1], entry[2]
+            meta = entry[3] if len(entry) > 3 else {}
+            entities.append(
+                {
+                    "start_offset": start_offset,
+                    "end_offset": end_offset,
+                    "label": label_name,
+                    "meta": meta,
+                }
+            )
+            pre_annotated.append(
+                {
+                    "span": [start_offset, end_offset, label_name],
+                    "matched_models": meta.get("matched_models"),
+                    "is_standard_gb_var": meta.get("is_standard_gb_var"),
+                }
+            )
+        obj["entities"] = entities
+        obj["pre_annotated"] = pre_annotated
+        out.append(json.dumps(obj))
+    return "\n".join(out).encode("utf-8")
+
+
+@shared_task(
+    bind=True,
+    autoretry_for=(requests.exceptions.RequestException,),
+    retry_backoff=True,
+    max_retries=5,
+)
+def submit_example_to_gearbox(self, example_id, project_id):
+    example = get_object_or_404(Example, pk=example_id)
+    project = get_object_or_404(Project, pk=project_id)
+    examples = ExportedExample.objects.filter(pk=example.pk)
+    labels = create_labels(project, examples)
+    comments = create_comment(examples)
+    dataset = Dataset(examples, labels, comments, project.is_text_project)
+    formatters = create_formatter(project, "JSONL")
+    writer = create_writer("JSONL")
+
+    service = ExportApplicationService(dataset, formatters, writer)
+    with tempfile.NamedTemporaryFile(suffix=".jsonl") as tmp:
+        service.export(tmp.name)
+        tmp.seek(0)
+        jsonl_bytes = tmp.read()
+
+    try:
+        jsonl_bytes = _transform_to_gearbox_format(jsonl_bytes)
+        submit_to_gearbox(jsonl_bytes, filename=f"example_{example.pk}.jsonl")
+    except requests.exceptions.RequestException:
+        if self.request.retries == self.max_retries:
+            example.gearbox_status = "failed"
+            example.save(update_fields=["gearbox_status"])
+        raise
+    except Exception:
+        example.gearbox_status = "failed"
+        example.save(update_fields=["gearbox_status"])
+        logger.exception("Unexpected error submitting example %s to gearbox", example.pk)
+        raise
+
+    example.gearbox_status = "success"
+    example.save(update_fields=["gearbox_status"])
+    logger.info("Successfully submitted example %s to gearbox", example.pk)

backend/examples/migrations/0009_example_gearbox_status.py

@@ -0,0 +1,16 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("examples", "0008_assignment"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="example",
+            name="gearbox_status",
+            field=models.CharField(blank=True, max_length=20, null=True),
+        ),
+    ]

backend/examples/models.py

@@ -20,6 +20,7 @@ class Example(models.Model):
     annotations_approved_by = models.ForeignKey(to=User, on_delete=models.SET_NULL, null=True, blank=True)
     text = models.TextField(null=True, blank=True)
     score = models.FloatField(default=100)
+    gearbox_status = models.CharField(max_length=20, null=True, blank=True)
     created_at = models.DateTimeField(auto_now_add=True, db_index=True)
     updated_at = models.DateTimeField(auto_now=True)
 

backend/examples/views/example_state.py

@@ -2,6 +2,7 @@
 from rest_framework import generics
 from rest_framework.permissions import IsAuthenticated
 
+from examples.celery_tasks import submit_example_to_gearbox
 from examples.models import Example, ExampleState
 from examples.serializers import ExampleStateSerializer
 from projects.models import Project
@@ -27,6 +28,12 @@ def perform_create(self, serializer):
         queryset = self.get_queryset()
         if queryset.exists():
             queryset.delete()
+            example = get_object_or_404(Example, pk=self.kwargs["example_id"])
+            example.gearbox_status = None
+            example.save(update_fields=["gearbox_status"])
         else:
             example = get_object_or_404(Example, pk=self.kwargs["example_id"])
+            example.gearbox_status = "pending"
+            example.save(update_fields=["gearbox_status"])
             serializer.save(example=example, confirmed_by=self.request.user)
+            submit_example_to_gearbox.delay(example.pk, self.kwargs["project_id"])

backend/labels/managers.py

@@ -29,7 +29,9 @@ def calc_label_distribution(self, examples, members, labels):
             username = item["user__username"]
             label = item[f"{self.label_type_field}__text"]
             count = item["count"]
-            distribution[username][label] = count
+            # Skip users who are no longer members of the project
+            if username in distribution:
+                distribution[username][label] = count
         return distribution
 
     def get_labels(self, label, project):

backend/metrics/tests.py

@@ -5,7 +5,7 @@
 from api.tests.utils import CRUDMixin
 from examples.tests.utils import make_doc
 from label_types.tests.utils import make_label
-from projects.models import ProjectType
+from projects.models import Member, ProjectType
 from projects.tests.utils import prepare_project
 
 
@@ -78,3 +78,17 @@ def test_fetch_distribution(self):
         expected = {member.username: {self.label.text: 0} for member in self.project.members}
         expected[self.project.admin.username][self.label.text] = 1
         self.assertEqual(response.data, expected)
+
+    def test_fetch_distribution_with_removed_user(self):
+        # Create a category by annotator
+        mommy.make("Category", example=self.example, label=self.label, user=self.project.annotator)
+        # Remove the annotator from the project
+        Member.objects.filter(user=self.project.annotator, project=self.project.item).delete()
+        # Should not raise KeyError
+        response = self.assert_fetch(self.project.admin, status.HTTP_200_OK)
+        # The response should only include current members (after removal)
+        current_members = Member.objects.filter(project=self.project.item).select_related("user")
+        expected = {member.user.username: {self.label.text: 0} for member in current_members}
+        expected[self.project.admin.username][self.label.text] = 1
+        # The removed user's annotations should not be in the distribution
+        self.assertEqual(response.data, expected)