Refactor to django-anymail from resend client and add tests

Author: webstackdev

.env.example

@@ -11,6 +11,7 @@ OPENROUTER_API_KEY=
 OPENROUTER_API_BASE=https://openrouter.ai/api/v1
 OPENROUTER_APP_URL=
 OPENROUTER_APP_NAME=newsletter-maker
+
 AI_CLASSIFICATION_MODEL=meta-llama/llama-3.1-70b-instruct
 AI_RELEVANCE_MODEL=qwen/qwen-2.5-72b-instruct
 AI_SUMMARIZATION_MODEL=google/gemma-3-27b-it
@@ -21,14 +22,39 @@ AI_RELEVANCE_REVIEW_THRESHOLD=0.4
 AI_RELEVANCE_SUMMARIZE_THRESHOLD=0.7
 AI_MAX_NODE_RETRIES=2
 AI_REQUEST_TIMEOUT_SECONDS=60
+
 EMBEDDING_PROVIDER=sentence-transformers
 EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
 EMBEDDING_TRUST_REMOTE_CODE=false
+
 OLLAMA_URL=http://localhost:11434
+
 REDDIT_CLIENT_ID=
 REDDIT_CLIENT_SECRET=
 REDDIT_USER_AGENT=newsletter-maker/0.1
+
+# Outbound mail provider. Use Resend or Amazon SES.
+EMAIL_BACKEND=anymail.backends.resend.EmailBackend
+DEFAULT_FROM_EMAIL=onboarding@resend.dev
+SERVER_EMAIL=onboarding@resend.dev
+
+# Resend outbound + inbound
+RESEND_API_KEY=
+RESEND_FROM_EMAIL=onboarding@resend.dev
+RESEND_INBOUND_SECRET=
+
+# Amazon SES outbound + inbound
+# EMAIL_BACKEND=anymail.backends.amazon_ses.EmailBackend
+# AWS_ACCESS_KEY_ID=
+# AWS_SECRET_ACCESS_KEY=
+# AWS_DEFAULT_REGION=us-east-1
+
+# Shared webhook basic auth for providers that need it.
+# Format must be username:password.
+ANYMAIL_WEBHOOK_SECRET=
+
 LOG_LEVEL=INFO
+
 CELERY_TASK_ALWAYS_EAGER=false
 
 DJANGO_SUPERUSER_USERNAME=admin
@@ -40,5 +66,7 @@ NEWSLETTER_API_USERNAME=admin
 NEWSLETTER_API_PASSWORD=adminpass
 
 DEBUG=True
+
 ALLOWED_HOSTS=localhost,127.0.0.1,newslettermaker.tech
+
 FRONTEND_URL=http://localhost:3000

.vscode/settings.json

@@ -1,10 +1,12 @@
 {
   "cSpell.words": [
     "ASGI",
+    "botocore",
     "buildx",
     "cbranch",
     "cfgv",
     "cstat",
+    "dateutil",
     "djlint",
     "FAVICONS",
     "Feedly",
@@ -33,6 +35,7 @@
     "readyz",
     "Referer",
     "simplejwt",
+    "svix",
     "Unparseable",
     "unstub",
     "upserted",

README.md

@@ -177,3 +177,9 @@ For the default local bootstrap, `.env` also seeds an `admin` superuser in the c
 This repository is licensed under the GNU Affero General Public License v3.0 or later. See [LICENSE](LICENSE).
 
 Based on the current direct dependencies, AGPL is a reasonable fit: the packages in use are permissive or LGPL-compatible licenses such as BSD, MIT, Apache-2.0, and LGPLv3. That said, this is a practical compatibility check, not legal advice, so review it with counsel if you need a formal licensing opinion.
+
+In our skills/relevance_scoring/SKILL.md, I see that we're passing the tenant id. Are we evaluating the content item's relevance against all content items for a project (tenant)? Or how is the relevance actually calculated?
+
+I notice too we're still using "tenant_id". We changed our naming scheme from "tenant" to "project". We need to update the skills and anywhere else we're still referring to the old "tenant" naming scheme.
+
+How many Embedding Dimensions are we using to create embeddings? Where are we specifying that value for Qdrant records?

core/apps.py

@@ -4,3 +4,6 @@
 class CoreConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "core"
+
+    def ready(self) -> None:
+        import core.signals  # noqa: F401

core/newsletters.py

@@ -1,15 +1,16 @@
 from __future__ import annotations
 
-import hashlib
-import hmac
 import re
 from dataclasses import dataclass
 from email.utils import parseaddr
 from html.parser import HTMLParser
-from typing import Any, cast
+from typing import Any, Iterable, cast
 
 from django.conf import settings as django_settings
+from django.core.mail import EmailMultiAlternatives
+from django.urls import reverse
 
+from core.models import IntakeAllowlist, NewsletterIntake, Project
 from core.settings_types import CoreSettings
 
 settings = cast(CoreSettings, django_settings)
@@ -29,17 +30,6 @@ def sanitize_newsletter_html(raw_html: str) -> str:
     return INLINE_HANDLER_PATTERN.sub("", without_scripts)
 
 
-def compute_resend_signature(payload: bytes, secret: str) -> str:
-    return hmac.new(secret.encode("utf-8"), payload, hashlib.sha256).hexdigest()
-
-
-def verify_resend_signature(payload: bytes, provided_signature: str) -> bool:
-    if not settings.RESEND_WEBHOOK_SECRET or not provided_signature:
-        return False
-    expected_signature = compute_resend_signature(payload, settings.RESEND_WEBHOOK_SECRET)
-    return hmac.compare_digest(expected_signature, provided_signature)
-
-
 def extract_project_token(recipient: str) -> str | None:
     _, email_address = parseaddr(recipient)
     local_part = email_address.partition("@")[0]
@@ -50,24 +40,101 @@ def extract_project_token(recipient: str) -> str | None:
 
 
 def send_confirmation_email(*, to_email: str, confirm_url: str, project_name: str) -> None:
-    if not settings.RESEND_API_KEY:
-        raise RuntimeError("RESEND_API_KEY must be configured to send newsletter confirmation emails.")
-
-    import resend
-
-    resend.api_key = settings.RESEND_API_KEY
-    resend.Emails.send(
-        {
-            "from": settings.RESEND_FROM_EMAIL,
-            "to": [to_email],
-            "subject": f"Confirm newsletter intake for {project_name}",
-            "html": (
-                "<p>Confirm this sender for newsletter ingestion.</p>"
-                f'<p><a href="{confirm_url}">Confirm sender</a></p>'
-            ),
-        }
+    subject = f"Confirm newsletter intake for {project_name}"
+    text_body = (
+        "Confirm this sender for newsletter ingestion.\n\n"
+        f"Confirm sender: {confirm_url}"
+    )
+    html_body = (
+        "<p>Confirm this sender for newsletter ingestion.</p>"
+        f'<p><a href="{confirm_url}">Confirm sender</a></p>'
     )
 
+    message = EmailMultiAlternatives(
+        subject=subject,
+        body=text_body,
+        from_email=settings.DEFAULT_FROM_EMAIL,
+        to=[to_email],
+    )
+    message.attach_alternative(html_body, "text/html")
+    message.send()
+
+
+def build_confirmation_url(token: str) -> str:
+    base_url = settings.NEWSLETTER_API_BASE_URL.rstrip("/")
+    return f"{base_url}{reverse('confirm-newsletter-sender', kwargs={'token': token})}"
+
+
+def process_inbound_newsletter(
+    *,
+    recipients: Iterable[str],
+    sender_email: str,
+    subject: str,
+    raw_html: str,
+    raw_text: str,
+    message_id: str,
+) -> dict[str, Any]:
+    project = _find_intake_project(recipients)
+    if project is None:
+        return {"status": "ignored", "reason": "no_matching_project"}
+
+    normalized_sender_email = normalize_sender_email(sender_email)
+    normalized_message_id = message_id.strip()
+    if not normalized_sender_email or not normalized_message_id:
+        return {"status": "ignored", "reason": "missing_sender_or_message_id"}
+
+    defaults = {
+        "project": project,
+        "sender_email": normalized_sender_email,
+        "subject": subject[:512],
+        "raw_html": sanitize_newsletter_html(raw_html),
+        "raw_text": raw_text,
+    }
+    intake, created = NewsletterIntake.objects.get_or_create(
+        message_id=normalized_message_id,
+        defaults=defaults,
+    )
+    if not created:
+        return {"id": intake.id, "status": intake.status, "duplicate": True}
+
+    allowlist, allowlist_created = IntakeAllowlist.objects.get_or_create(
+        project=project,
+        sender_email=normalized_sender_email,
+    )
+
+    if allowlist.is_confirmed:
+        queue_newsletter_intake(intake.id)
+        return {"id": intake.id, "status": intake.status}
+
+    if allowlist_created:
+        send_confirmation_email(
+            to_email=normalized_sender_email,
+            confirm_url=build_confirmation_url(allowlist.confirmation_token),
+            project_name=project.name,
+        )
+
+    return {"id": intake.id, "status": intake.status, "confirmation_required": True}
+
+
+def queue_newsletter_intake(intake_id: int) -> None:
+    from core.tasks import process_newsletter_intake
+
+    if settings.CELERY_TASK_ALWAYS_EAGER:
+        process_newsletter_intake(intake_id)
+    else:
+        process_newsletter_intake.delay(intake_id)
+
+
+def _find_intake_project(recipients: Iterable[str]) -> Project | None:
+    for recipient in recipients:
+        token = extract_project_token(recipient)
+        if token is None:
+            continue
+        project = Project.objects.filter(intake_token=token, intake_enabled=True).first()
+        if project is not None:
+            return project
+    return None
+
 
 @dataclass(slots=True)
 class ExtractedNewsletterItem:
@@ -146,10 +213,3 @@ def extract_newsletter_items(*, subject: str, raw_html: str, raw_text: str) -> l
         )
 
     return extracted_items
-
-
-def get_resend_payload_data(payload: dict[str, Any]) -> dict[str, Any]:
-    data = payload.get("data")
-    if isinstance(data, dict):
-        return data
-    return payload

core/settings_types.py

@@ -3,6 +3,8 @@
 
 class CoreSettings(Protocol):
     CELERY_TASK_ALWAYS_EAGER: bool
+    DEFAULT_FROM_EMAIL: str
+    NEWSLETTER_API_BASE_URL: str
     QDRANT_URL: str
     EMBEDDING_MODEL: str
     EMBEDDING_PROVIDER: str
@@ -13,5 +15,3 @@ class CoreSettings(Protocol):
     OPENROUTER_APP_URL: str
     OPENROUTER_APP_NAME: str
     RESEND_API_KEY: str
-    RESEND_FROM_EMAIL: str
-    RESEND_WEBHOOK_SECRET: str

core/signals.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from anymail.signals import inbound
+from django.dispatch import receiver
+
+from core.newsletters import process_inbound_newsletter
+
+
+def _address_to_string(address) -> str:
+    if address is None:
+        return ""
+    addr_spec = getattr(address, "addr_spec", None)
+    if isinstance(addr_spec, str):
+        return addr_spec.strip()
+    return str(address).strip()
+
+
+@receiver(inbound)
+def handle_anymail_inbound(sender, event, esp_name, **kwargs):
+    message = event.message
+
+    recipients: list[str] = []
+    if message.envelope_recipient:
+        recipients.append(message.envelope_recipient)
+    recipients.extend(
+        address.addr_spec
+        for address in getattr(message, "to", [])
+        if getattr(address, "addr_spec", "")
+    )
+
+    process_inbound_newsletter(
+        recipients=recipients,
+        sender_email=message.envelope_sender or _address_to_string(getattr(message, "from_email", None)),
+        subject=message.subject or "",
+        raw_html=message.html or "",
+        raw_text=message.text or "",
+        message_id=str(message.get("Message-ID", "") or event.event_id or ""),
+    )