Covert all uses of Submit Nodes to Groups

alembic/env.py

@@ -27,10 +27,10 @@
 # Import your Base and all models to ensure they're registered
 from userapp.core.models.main import Base
 from userapp.core.models.tables import (
-    Group, Note, Project, SubmitNode, User, UserGroup,
-    UserNote, UserProject, UserSubmit, Token, Access
+    Group, Note, Project, User, UserGroup,
+    UserNote, UserProject, Token, Access
 )
-from userapp.core.models.views import JoinedProjectView, UserSubmitNodesView  # Import views if needed
+from userapp.core.models.views import JoinedProjectView  # Import views if needed
 
 target_metadata = Base.metadata
 

alembic/versions/026216ef4d8a_add_type_enum_and_label_string_to_groups.py

@@ -0,0 +1,209 @@
+"""add group type/description and submit node groups
+
+Revision ID: 026216ef4d8a
+Revises: f2ec55925c4c
+Create Date: 2026-05-29
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+
+# revision identifiers, used by Alembic.
+revision: str = '026216ef4d8a'
+down_revision: Union[str, Sequence[str], None] = 'f2ec55925c4c'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+GROUPS_DATA = [
+    {'node_name': 'wright-ap4000.chtc.wisc.edu', 'group_name': 'wright-ap-login', 'description': 'Access to wright-ap4000.chtc.wisc.edu'},
+    {'node_name': 'townsend-submit.chtc.wisc.edu', 'group_name': 'townsend-ap-login', 'description': 'Access to townsend-submit.chtc.wisc.edu'},
+    {'node_name': 'hpclogin1.chtc.wisc.edu', 'group_name': 'spark-login', 'description': 'Access to hpclogin1.chtc.wisc.edu'},
+    {'node_name': 'osg-sw-submit.chtc.wisc.edu', 'group_name': 'osgsw-ap-login', 'description': 'Access to osg-sw-submit.chtc.wisc.edu'},
+    {'node_name': 'oconnor-ap.chtc.wisc.edu', 'group_name': 'oconnor-ap-login', 'description': 'Access to oconnor-ap.chtc.wisc.edu'},
+    {'node_name': 'learn.chtc.wisc.edu', 'group_name': 'learn-ap-login', 'description': 'Access to learn.chtc.wisc.edu'},
+    {'node_name': 'htc_transfer', 'group_name': 'htc-transfer-login', 'description': 'Access to htc_transfer'},
+    {'node_name': 'ap2002.chtc.wisc.edu', 'group_name': 'ap2002-login', 'description': 'Access to ap2002.chtc.wisc.edu'},
+    {'node_name': 'ap2001.chtc.wisc.edu', 'group_name': 'ap2001-login', 'description': 'Access to ap2001.chtc.wisc.edu'},
+]
+
+group_type_enum = postgresql.ENUM(
+    'SUBMIT_NODE',
+    name='group_type_enum',
+    create_type=False,
+)
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    bind = op.get_bind()
+    group_type_enum.create(bind, checkfirst=True)
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('groups', sa.Column('description', sa.String(length=255), nullable=True))
+    op.add_column('groups', sa.Column('type', group_type_enum, nullable=True))
+    # ### end Alembic commands ###
+
+    # ### commands generated by human - please check for typos! ###
+    values_list_string = ",\n".join(
+        f"('{g['group_name']}', '{g['description']}', 'SUBMIT_NODE'::group_type_enum, FALSE)"
+        for g in GROUPS_DATA
+    )
+
+    op.execute(f"""
+        INSERT INTO groups (name, description, type, has_groupdir)
+        VALUES
+        {values_list_string}
+        ON CONFLICT (name) DO NOTHING
+    """)
+
+    mapping_list_string = ",\n".join(
+        f"('{g['node_name']}', '{g['group_name']}')" 
+        for g in GROUPS_DATA
+    )
+
+    op.execute(f"""
+        INSERT INTO user_groups (user_id, group_id, managed_by)
+        SELECT DISTINCT
+            us.user_id,
+            g.id,
+            'APPLICATION'::entity_manager_enum
+        FROM user_submits us
+        INNER JOIN submit_nodes sn ON sn.id = us.submit_node_id
+        INNER JOIN (
+            VALUES
+            {mapping_list_string}
+        ) AS mapping(node_name, group_name) ON sn.name = mapping.node_name
+        INNER JOIN groups g ON g.name = mapping.group_name
+        ON CONFLICT ON CONSTRAINT user_groups_distinct DO NOTHING
+    """)
+
+    op.execute("DROP VIEW IF EXISTS user_group_memberships")
+    op.execute("""
+        CREATE VIEW user_group_memberships AS
+        SELECT
+            ug.group_id,
+            ug.user_id,
+            ug.managed_by,
+            ug.created_at,
+            ug.updated_at,
+            g.name,
+            g.description,
+            g.point_of_contact,
+            g.unix_gid,
+            g.has_groupdir,
+            g.type
+        FROM user_groups ug
+        JOIN groups g ON ug.group_id = g.id
+    """)
+
+    # Drop the now-replaced submit node objects. Submit node access is represented as
+    # membership in a SUBMIT_NODE-typed group (backfilled above), so these are obsolete.
+    # Order matters: dependent view first, then child table, then parent table.
+    op.execute("DROP VIEW IF EXISTS user_submit_nodes")
+    op.execute("DROP TABLE IF EXISTS user_submits")
+    op.execute("DROP TABLE IF EXISTS submit_nodes")
+    # ### end commands generated by human ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+
+    # ### commands generated by human - please check for typos! ###
+
+    # Recreate the submit node objects that upgrade() dropped, restoring the pre-migration
+    # schema (reverse order of the drops: parent table, child table, then the view).
+    # NOTE: this restores schema only, not the dropped data (it was migrated into groups).
+    # NOTE: the user_submit_nodes view body is reconstructed from the ORM model columns -- the
+    # original was created pre-Alembic and is in no migration. Confirm against
+    # `SELECT pg_get_viewdef('user_submit_nodes'::regclass, true);` if exact fidelity is needed.
+    op.execute("""
+        CREATE TABLE IF NOT EXISTS submit_nodes (
+            id SERIAL PRIMARY KEY,
+            name VARCHAR(60)
+        )
+    """)
+
+    op.execute("""
+        CREATE TABLE IF NOT EXISTS user_submits (
+            id SERIAL PRIMARY KEY,
+            user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+            submit_node_id INTEGER NOT NULL REFERENCES submit_nodes(id) ON DELETE CASCADE,
+            for_auth_netid BOOLEAN,
+            disk_quota INTEGER,
+            hpc_diskquota INTEGER NOT NULL DEFAULT 100,
+            hpc_inodequota INTEGER NOT NULL DEFAULT 50000,
+            hpc_joblimit INTEGER NOT NULL DEFAULT 10,
+            hpc_corelimit INTEGER NOT NULL DEFAULT 720,
+            hpc_fairshare INTEGER NOT NULL DEFAULT 100,
+            CONSTRAINT user_submits_distinct UNIQUE (user_id, submit_node_id, for_auth_netid)
+        )
+    """)
+    op.execute("CREATE INDEX IF NOT EXISTS ix_user_submits_submit_node_id ON user_submits (submit_node_id)")
+    op.execute("""
+        CREATE INDEX IF NOT EXISTS idx_user_submits_userid_submitnodeid_incl
+        ON user_submits (user_id, submit_node_id)
+        INCLUDE (disk_quota, hpc_diskquota, hpc_inodequota, hpc_joblimit, hpc_corelimit, hpc_fairshare)
+    """)
+
+    op.execute("""
+        CREATE OR REPLACE VIEW user_submit_nodes AS
+        SELECT
+            us.user_id,
+            us.submit_node_id,
+            sn.name AS submit_node_name,
+            us.disk_quota,
+            us.hpc_diskquota,
+            us.hpc_inodequota,
+            us.hpc_joblimit,
+            us.hpc_corelimit,
+            us.hpc_fairshare
+        FROM user_submits us
+        JOIN submit_nodes sn ON sn.id = us.submit_node_id
+    """)
+
+    group_names_string = ', '.join(f"'{group['group_name']}'" for group in GROUPS_DATA)
+
+    op.execute(f"""
+        DELETE FROM user_groups
+        WHERE group_id IN (
+            SELECT id FROM groups
+            WHERE name IN ({group_names_string})
+              AND type = 'SUBMIT_NODE'::group_type_enum
+        )
+    """)
+
+    op.execute(f"""
+        DELETE FROM groups
+        WHERE name IN ({group_names_string})
+          AND type = 'SUBMIT_NODE'::group_type_enum
+    """)
+
+    op.execute("DROP VIEW IF EXISTS user_group_memberships")
+    op.execute("""
+        CREATE VIEW user_group_memberships AS
+        SELECT
+            ug.group_id,
+            ug.user_id,
+            ug.managed_by,
+            ug.created_at,
+            ug.updated_at,
+            g.name,
+            g.point_of_contact,
+            g.unix_gid,
+            g.has_groupdir
+        FROM user_groups ug
+        JOIN groups g ON ug.group_id = g.id
+    """)
+    # ### end commands generated by human ###
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('groups', 'type')
+    op.drop_column('groups', 'description')
+    # ### end Alembic commands ###
+
+    bind = op.get_bind()
+    group_type_enum.drop(bind, checkfirst=True)

userapp/api/load_options.py

@@ -9,5 +9,4 @@
     selectinload(UserTable.projects).selectinload(JoinedProjectViewTable.staff1_user),
     selectinload(UserTable.projects).selectinload(JoinedProjectViewTable.staff2_user),
     selectinload(UserTable.user_forms),
-    selectinload(UserTable.submit_nodes)
 ]

userapp/api/routes/__init__.py

@@ -4,7 +4,6 @@
 from .pi_projects import router as pi_projects_router
 from .projects import router as projects_router
 from .security import router as security_router
-from .submit_nodes import router as submit_nodes_router
 from .users import router as users_router
 from .tokens import router as tokens_router
 from .routes import router as routes_router
@@ -17,7 +16,6 @@
     pi_projects_router,
     projects_router,
     security_router,
-    submit_nodes_router,
     users_router,
     tokens_router
 ]

userapp/api/routes/_util.py

@@ -3,40 +3,37 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from userapp.api.util import create_one_endpoint
-from userapp.core.models.tables import UserSubmit, User, UserProject, UserGroup
+from userapp.core.models.enum import GroupTypeEnum
+from userapp.core.models.tables import User, UserProject, UserGroup
 from userapp.core.models.views import JoinedProjectView, UserGroupView
 from userapp.core.schemas.user_project import UserProjectPatch
 from userapp.core.schemas.user_group import UserGroupPatch
-from userapp.core.schemas.user_submit import UserSubmitTableSchema, UserSubmitPost
+from userapp.core.schemas.user_group import UserGroupCreate  # see below
 
 
-async def _patch_user_submit_nodes(session: AsyncSession, user: User, new_submit_nodes: list[UserSubmitPost]):
+async def _patch_user_submit_node_groups(session: AsyncSession, user: User, new_submit_node_group_ids: list[int]):
     """Updates the passed in user to match the provided list of submit_nodes"""
 
-    # Delete the submit nodes that are not in the list of new submit nodes
-    for existing_submit_node in user.submit_nodes:
-        if existing_submit_node.submit_node_id not in [sn.submit_node_id for sn in new_submit_nodes]:
-            delete_stmt = (
-                UserSubmit.__table__.delete()
-                .where(UserSubmit.user_id == user.id)
-                .where(UserSubmit.submit_node_id == existing_submit_node.submit_node_id)
-            )
-            await session.execute(delete_stmt)
-
-    # Add the missing submit nodes
-    for submit_node in new_submit_nodes:
-
-        if submit_node.submit_node_id in [sn.submit_node_id for sn in user.submit_nodes]:
-            continue  # Already exists
+    existing_group_ids = [g.group_id for g in user.groups]
 
-        # Create nodes for both auth_netid True and False to simplify logic
-        for for_auth_netid in [True, False]:
-            user_submit_model = UserSubmitTableSchema(
-                user_id=user.id,
-                for_auth_netid=for_auth_netid,
-                **submit_node.model_dump(),
+    # Remove memberships not in the new list
+    for existing_group in user.groups:
+        if existing_group.type == GroupTypeEnum.SUBMIT_NODE and existing_group.group_id not in new_submit_node_group_ids:
+            await session.execute(
+                UserGroup.__table__.delete()
+                .where(UserGroup.user_id == user.id)
+                .where(UserGroup.group_id == existing_group.group_id)
             )
-            await create_one_endpoint(session, UserSubmit, user_submit_model)
+
+    # Add memberships that are missing
+    for group_id in new_submit_node_group_ids:
+        if group_id in existing_group_ids:
+            continue
+        await create_one_endpoint(
+            session,
+            UserGroup,
+            UserGroupCreate(user_id=user.id, group_id=group_id),
+        )
 
 
 async def _patch_user_project(

userapp/api/routes/forms/_util.py

@@ -7,7 +7,7 @@
 from userapp.core.models.tables import UserForm as UserFormTable, UserGroup, UserProject
 from userapp.core.schemas.user_application_form import UserFormPatch
 from userapp.core.schemas.user_project import UserProjectTableSchema
-from userapp.api.routes._util import _patch_user_submit_nodes
+from userapp.api.routes._util import _patch_user_submit_node_groups
 
 CHTC_NO_REPLY_EMAIL = "no-reply@chtc.wisc.edu"
 CHTC_TICKETING_EMAIL = "chtc@cs.wisc.edu"
@@ -80,10 +80,10 @@ async def on_user_form_accept(session: AsyncSession, form_id: int, form: UserFor
 
         user.position = form.user_position
 
-        if not (form.project_id and form.user_position and form.submit_nodes):
+        if not (form.project_id and form.user_position and form.submit_node_group_ids):
             raise HTTPException(
                 status_code=400,
-                detail="project_id, user_position, and submit_nodes must be provided to accept a user form",
+                detail="project_id, user_position, and submit_node_group_ids must be provided to accept a user form",
             )
 
         # Clear out projects, and groups
@@ -101,7 +101,7 @@ async def on_user_form_accept(session: AsyncSession, form_id: int, form: UserFor
             ),
         )
         # Update to the set of approved submit nodes
-        await _patch_user_submit_nodes(session, user, form.submit_nodes)
+        await _patch_user_submit_node_groups(session, user, form.submit_node_group_ids)
 
     # Try sending the user an email
     try:

userapp/api/routes/forms/user_application.py

@@ -8,13 +8,12 @@
 from userapp.api.util import create_one_endpoint, list_endpoint, list_select_stmt, update_one_endpoint, get_one_endpoint
 from userapp.core.models.enum import FormStatusEnum, FormTypeEnum
 from userapp.core.models.tables import BaseForm as BaseFormTable, Project as ProjectTable, \
-    SubmitNode as SubmitNodeTable, User as UserTable, UserForm as UserFormTable, UserProject, UserSubmit
+    User as UserTable, UserForm as UserFormTable, UserProject
 from userapp.core.models.views import UserApplicationView as UserApplicationViewTable
 from userapp.core.schemas.general import UserApplicationViewFull as UserApplicationViewFullSchema
 from userapp.core.schemas.forms import BaseFormTableSchema
 from userapp.core.schemas.user_application_form import UserFormPost, UserFormPatch, UserFormTableSchema
 from userapp.core.schemas.user_project import UserProjectTableSchema
-from userapp.core.schemas.user_submit import UserSubmitTableSchema
 from userapp.core.schemas.users import UserGet
 from userapp.db import session_generator
 from userapp.query_parser import get_filter_query_params