From 8dc009cf5cf8d772ef4b00475f8c9353145f14c8 Mon Sep 17 00:00:00 2001 From: Jan-Kazlouski-elastic Date: Fri, 18 Sep 2026 15:29:16 +0300 Subject: [PATCH] fix(sharepoint): add compact site-group DLS mode to reduce ACL memory (#4396) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Closes https://github.com/elastic/chat-program/issues/48 Part of https://github.com/elastic/chat-program/issues/47. SharePoint Online DLS expands every site group member onto each document's `_allow_access_control` array. Large site groups can produce multi-megabyte ACLs per document and OOM Elasticsearch during `_reindex`. This PR adds `expand_site_group_members` (default `true` to preserve existing behavior). When disabled (compact mode): - Content documents store a compact `site_group::` token instead of every site group member - Direct user, Entra group, and site user permissions are unchanged on the document - ACL sync builds a site-group membership index and enriches identity docs so DLS term overlap still resolves access (including nested Entra groups and EEEU guest exclusion) - If a compact token cannot be written safely (e.g. missing `site_id`), the connector falls back to expanding site group members and logs a warning Changing the setting requires a full content sync and access control sync. ## Checklists #### Pre-Review Checklist - [x] this PR does NOT contain credentials of any kind, such as API keys or username/passwords (double check `config.yml.example`) - [x] this PR has a meaningful title - [x] this PR links to all relevant github issues that it fixes or partially addresses - [x] if there is no GH issue, please create it. Each PR should have a link to an issue - [x] this PR has a thorough description - [x] Covered the changes with automated tests - [x] Tested the changes locally - [ ] Added a label for each target release version (example: `v7.13.2`, `v7.14.0`, `v8.0.0`) - [ ] For bugfixes: backport safely to all minor branches still receiving patch releases - [ ] Considered corresponding documentation changes - [ ] Contributed any configuration settings changes to the configuration reference - [x] if you added or changed Rich Configurable Fields for a Native Connector, you made a corresponding PR in [Kibana](https://github.com/elastic/kibana/blob/main/packages/kbn-search-connectors/types/native_connectors.ts) #### Changes Requiring Extra Attention - [x] Security-related changes (encryption, TLS, SSRF, etc) - [ ] New external service dependencies added. ## Related Pull Requests * https://github.com/elastic/connectors/pull/4392 — ServiceNow compact role DLS (same tracking issue) * https://github.com/elastic/kibana/pull/288070 — Kibana RCF for `expand_site_group_members` ## Release Note SharePoint Online document-level security can store compact site-group tokens on documents instead of expanding every site group member onto each record, reducing memory use for large site groups. Disable **Expand site group members** to enable compact mode. The default preserves the previous behavior. Changing the setting requires a full content sync and access control sync. --------- Co-authored-by: Elastic Machine --- .../sharepoint/sharepoint_online/client.py | 16 +- .../sharepoint_online/datasource.py | 244 ++++++++- .../sharepoint/sharepoint_online/utils.py | 18 + .../fixtures/sharepoint_online/connector.json | 20 + .../tests/sources/test_sharepoint_online.py | 466 ++++++++++++++++++ 5 files changed, 746 insertions(+), 18 deletions(-) diff --git a/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/client.py b/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/client.py index 6fbb49ece..4af909415 100644 --- a/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/client.py +++ b/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/client.py @@ -674,6 +674,20 @@ async def site_admins(self, site_web_url): self._logger.debug(f"No site admins found for site: '${site_web_url}'") return + async def site_groups(self, site_web_url): + self._validate_sharepoint_rest_url(site_web_url) + + select_ = "Id,Title" + url = f"{site_web_url}/_api/web/sitegroups?$select={select_}" + + try: + async for page in self._rest_api_client.scroll(url): + for site_group in page: + yield site_group + except NotFound: + self._logger.debug(f"No site groups found for site: '{site_web_url}'") + return + async def site_groups_users(self, site_web_url, site_group_id): self._validate_sharepoint_rest_url(site_web_url) @@ -694,7 +708,7 @@ async def active_users_with_groups(self): expand = "transitiveMemberOf($select=id)" top = 999 # this is accepted, but does not get taken literally. Response size seems to max out at 100 filter_ = "accountEnabled eq true" - select = "UserName,userPrincipalName,Email,mail,transitiveMemberOf,id,createdDateTime" + select = "UserName,userPrincipalName,Email,mail,transitiveMemberOf,id,createdDateTime,userType" url = f"{GRAPH_API_URL}/users?$expand={expand}&$top={top}&$filter={filter_}&$select={select}" try: diff --git a/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/datasource.py b/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/datasource.py index eb9f5367c..92fca1b5e 100644 --- a/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/datasource.py +++ b/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/datasource.py @@ -46,9 +46,11 @@ DeltaLinkExpired, SyncCursorEmpty, _get_login_name, + _is_guest_user, _parse_created_date_time, _prefix_email, _prefix_group, + _prefix_site_group, _prefix_user, _prefix_user_id, ) @@ -253,6 +255,15 @@ def get_default_configuration(cls): "type": "bool", "value": True, }, + "expand_site_group_members": { + "depends_on": [{"field": "use_document_level_security", "value": True}], + "display": "toggle", + "label": "Expand site group members", + "order": 17, + "tooltip": "When enabled, SharePoint site group members are written individually onto each document's access control list. Disable this for large site groups to store a compact site group token on documents instead, and resolve membership during access control syncs. Changing this setting requires a full content sync and access control sync.", + "type": "bool", + "value": True, + }, } async def validate_config(self): @@ -358,7 +369,11 @@ def _is_site_admin(user): member = role_assignment["Member"] member_access_control = set() member_access_control.update( - await self._get_access_control_from_role_assignment(role_assignment) + await self._get_access_control_from_role_assignment( + role_assignment, + site_id=site["id"], + site_web_url=site["webUrl"], + ) ) editors_access_control.update( @@ -408,6 +423,52 @@ def _dls_enabled(self): return self.configuration["use_document_level_security"] + def _expand_site_group_members(self): + """Whether site groups are expanded into individual users on document ACLs. + + Default True preserves legacy behavior. When False (compact mode), documents + receive a single site_group token and membership is resolved on identity docs. + """ + return self.configuration.get("expand_site_group_members", True) + + async def _expand_site_group_members_access_control( + self, site_web_url, site_group_id, role_assignment=None + ): + access_control = [] + if role_assignment is not None: + users = nested_get_from_dict(role_assignment, ["Member", "Users"], []) + for user in users: # pyright: ignore + access_control.extend(await self._access_control_for_member(user)) + return access_control + + users = await self.site_group_users(site_web_url, site_group_id) + for site_group_user in users: + access_control.extend( + await self._access_control_for_member(site_group_user) + ) + return access_control + + async def _site_group_access_control( + self, site_web_url, site_group_id, site_id, role_assignment=None + ): + """Resolve site-group permissions, falling back to member expansion if compact + tokens cannot be written safely.""" + if self._expand_site_group_members(): + return await self._expand_site_group_members_access_control( + site_web_url, site_group_id, role_assignment=role_assignment + ) + + if site_id is not None: + return [_prefix_site_group(site_id, site_group_id)] + + self._logger.warning( + "Cannot write compact site_group token because site_id is missing " + f"(group_id={site_group_id}); falling back to expanding site group members" + ) + return await self._expand_site_group_members_access_control( + site_web_url, site_group_id, role_assignment=role_assignment + ) + def access_control_query(self, access_control): return es_access_control_query(access_control) @@ -491,6 +552,10 @@ async def get_access_control(self): """Yields an access control document for every user of a site. Note: this method will cache users and emails it has already and skip the ingestion for those. + When expand_site_group_members is False, also enriches identity docs with + site_group tokens for SharePoint site group memberships so document ACLs + can stay compact. + Yields: dict: dictionary representing a user access control document """ @@ -530,11 +595,111 @@ async def process_user(user): return person_access_control_doc self._logger.info("Fetching all users") + if self._expand_site_group_members(): + async for user in self.client.active_users_with_groups(): + user_doc = await process_user(user) + if user_doc: + yield user_doc + return + + ( + site_group_tokens_by_acl_token, + eeeu_site_group_tokens, + ) = await self._build_site_group_token_index() + async for user in self.client.active_users_with_groups(): user_doc = await process_user(user) if user_doc: + self._apply_site_group_tokens_to_identity( + user_doc, + user, + site_group_tokens_by_acl_token, + eeeu_site_group_tokens, + ) yield user_doc + def _apply_site_group_tokens_to_identity( + self, + user_doc, + user, + site_group_tokens_by_acl_token, + eeeu_site_group_tokens, + ): + site_group_tokens = set() + if not _is_guest_user(user): + site_group_tokens.update(eeeu_site_group_tokens) + + access_control = user_doc["query"]["template"]["params"].get( + "access_control", [] + ) + for token in access_control: + site_group_tokens.update(site_group_tokens_by_acl_token.get(token, set())) + + if site_group_tokens: + user_doc["query"]["template"]["params"]["access_control"] = list( + set(access_control).union(site_group_tokens) + ) + + async def _build_site_group_token_index(self): + """Map identity ACL tokens to compact site_group tokens from site memberships.""" + site_group_tokens_by_acl_token = {} + eeeu_site_group_tokens = set() + unmatched_members = 0 + indexed_members = 0 + everyone_except_external_users = "group:EveryoneExceptExternalUsers" + + self._logger.info( + "Compact site-group DLS enabled: building site group membership index" + ) + + async for site_collection in self.site_collections(): + async for site in self.sites( + site_collection["siteCollection"]["hostname"], + self.configuration["site_collections"], + ): + site_id = site["id"] + site_web_url = site["webUrl"] + + async for site_group in self.client.site_groups(site_web_url): + group_id = site_group.get("Id") + if group_id is None: + continue + + site_group_token = _prefix_site_group(site_id, group_id) + for member in await self.site_group_users(site_web_url, group_id): + member_tokens = await self._access_control_for_member(member) + if not member_tokens: + unmatched_members += 1 + self._logger.debug( + "Site group member could not be resolved to access " + f"control tokens (site={site_web_url}, group={group_id}, " + f"title={member.get('Title')})" + ) + continue + + if everyone_except_external_users in member_tokens: + eeeu_site_group_tokens.add(site_group_token) + else: + for token in member_tokens: + site_group_tokens_by_acl_token.setdefault( + token, set() + ).add(site_group_token) + indexed_members += 1 + + self._logger.info( + "Site group membership index complete: " + f"{indexed_members} members indexed, {unmatched_members} members unresolved, " + f"{len(eeeu_site_group_tokens)} EEEU site groups" + ) + if unmatched_members: + self._logger.warning( + f"{unmatched_members} SharePoint site group members could not be resolved " + "to access control tokens; users may not see documents protected by the " + "related compact site_group tokens" + ) + + return site_group_tokens_by_acl_token, eeeu_site_group_tokens + async def site_group_users(self, site_web_url, site_group_id): """ Fetches the users of a given site group. Checks in-memory cache before making an API call. @@ -568,13 +733,15 @@ async def site_group_users(self, site_web_url, site_group_id): return users async def _drive_items_batch_with_permissions( - self, drive_id, drive_items_batch, site_web_url + self, drive_id, drive_items_batch, site_web_url, site_id=None ): """Decorate a batch of drive items with their permissions using one API request. Args: drive_id (int): id of the drive, where the drive items reside drive_items_batch (list): list of drive items to decorate with permissions + site_web_url (str): web URL of the parent site + site_id (str): Graph site id (needed for compact site_group tokens) Yields: drive_item (dict): drive item with or without permissions depending on the config value of `fetch_drive_item_permissions` @@ -620,7 +787,7 @@ def _is_item_deleted(drive_item): if drive_item: yield await self._with_drive_item_permissions( - drive_item, permissions, site_web_url + drive_item, permissions, site_web_url, site_id=site_id ) async def get_docs(self, filtering=None): @@ -665,7 +832,10 @@ async def get_docs(self, filtering=None): async for ( drive_item ) in self._drive_items_batch_with_permissions( - site_drive["id"], drive_items_batch, site["webUrl"] + site_drive["id"], + drive_items_batch, + site["webUrl"], + site_id=site["id"], ): drive_item["_id"] = drive_item["id"] drive_item["object_type"] = "drive_item" @@ -776,7 +946,10 @@ async def get_docs_incrementally(self, sync_cursor, filtering=None): async for ( drive_item ) in self._drive_items_batch_with_permissions( - site_drive["id"], drive_items_batch, site["webUrl"] + site_drive["id"], + drive_items_batch, + site["webUrl"], + site_id=site["id"], ): drive_item["_id"] = drive_item["id"] drive_item["object_type"] = "drive_item" @@ -878,13 +1051,15 @@ async def site_drives(self, site, check_timestamp=False): yield site_drive async def _with_drive_item_permissions( - self, drive_item, drive_item_permissions, site_web_url + self, drive_item, drive_item_permissions, site_web_url, site_id=None ): """Decorates a drive item with its permissions. Args: drive_item (dict): drive item to fetch the permissions for. drive_item_permissions (list): drive item permissions to add to the drive_item. + site_web_url (str): web URL of the parent site. + site_id (str): Graph site id (needed for compact site_group tokens). Returns: drive_item (dict): drive item decorated with its permissions. @@ -976,11 +1151,13 @@ def _get_login_name(permissions, label): access_control.append(_prefix_user(site_user_username)) if site_group_id: - users = await self.site_group_users(site_web_url, site_group_id) - for site_group_user in users: # note, 'users' might contain groups. - access_control.extend( - await self._access_control_for_member(site_group_user) + access_control.extend( + await self._site_group_access_control( + site_web_url, + site_group_id, + site_id, ) + ) return self._decorate_with_access_control(drive_item, access_control) @@ -1055,7 +1232,9 @@ async def site_list_items( ): list_item_access_control.extend( await self._get_access_control_from_role_assignment( - role_assignment + role_assignment, + site_id=site_id, + site_web_url=site_web_url, ) ) @@ -1151,7 +1330,9 @@ async def site_lists(self, site, site_access_control, check_timestamp=False): ): site_list_access_control.extend( await self._get_access_control_from_role_assignment( - role_assignment + role_assignment, + site_id=site["id"], + site_web_url=site_url, ) ) @@ -1167,12 +1348,18 @@ async def site_lists(self, site, site_access_control, check_timestamp=False): yield site_list async def _get_access_control_from_role_assignment( - self, role_assignment, require_edit_access=False + self, + role_assignment, + site_id=None, + site_web_url=None, + require_edit_access=False, ): """Extracts access control from a role assignment. Args: role_assignment (dict): dictionary representing a role assignment. + site_id (str): Graph site id (needed for compact site_group tokens). + site_web_url (str): SharePoint site URL (needed for site group fallback). require_edit_access (bool): when True, return only members with edit access. Returns: @@ -1224,10 +1411,31 @@ def _grants_access(role_assignment): is_user = identity_type == "SP.User" if is_group: - users = nested_get_from_dict(role_assignment, ["Member", "Users"], []) - - for user in users: # pyright: ignore - access_control.extend(await self._access_control_for_member(user)) + group_id = nested_get_from_dict(role_assignment, ["Member", "Id"]) + if group_id is None: + group_id = role_assignment.get("PrincipalId") + if group_id is not None: + access_control.extend( + await self._site_group_access_control( + site_web_url=site_web_url, + site_group_id=group_id, + site_id=site_id, + role_assignment=role_assignment, + ) + ) + elif nested_get_from_dict(role_assignment, ["Member", "Users"], []): + access_control.extend( + await self._expand_site_group_members_access_control( + site_web_url=site_web_url, + site_group_id=group_id, + role_assignment=role_assignment, + ) + ) + else: + self._logger.warning( + "Cannot resolve site group for role assignment " + f"'{role_assignment.get('odata.id')}': group_id is missing" + ) elif is_user: member = role_assignment.get("Member", {}) access_control.extend(await self._access_control_for_member(member)) @@ -1296,6 +1504,8 @@ async def site_pages( page_access_control.extend( await self._get_access_control_from_role_assignment( role_assignment, + site_id=site_id, + site_web_url=url, require_edit_access=not published, ) ) diff --git a/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/utils.py b/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/utils.py index 4052af755..2bdbd4ec6 100644 --- a/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/utils.py +++ b/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/utils.py @@ -43,6 +43,24 @@ def _prefix_email(email): return prefix_identity("email", email) +def _prefix_site_group(site_id, group_id): + """Stable token for a SharePoint site group (unique within a Graph site).""" + return prefix_identity("site_group", f"{site_id}:{group_id}") + + +def _is_guest_user(user): + """Return True when a Graph user represents an external/guest account.""" + if user.get("userType") == "Guest": + return True + + for field in ("userPrincipalName", "UserName", "mail", "Email"): + value = user.get(field) + if value and "#EXT#" in value: + return True + + return False + + def _get_login_name(raw_login_name): if raw_login_name and ( raw_login_name.startswith("i:0#.f|membership|") diff --git a/app/connectors_service/tests/sources/fixtures/sharepoint_online/connector.json b/app/connectors_service/tests/sources/fixtures/sharepoint_online/connector.json index 8342f06b8..957cce139 100644 --- a/app/connectors_service/tests/sources/fixtures/sharepoint_online/connector.json +++ b/app/connectors_service/tests/sources/fixtures/sharepoint_online/connector.json @@ -263,6 +263,26 @@ "value": true, "order": 14, "ui_restrictions": [] + }, + "expand_site_group_members": { + "depends_on": [ + { + "field": "use_document_level_security", + "value": true + } + ], + "display": "toggle", + "tooltip": "When enabled, SharePoint site group members are written individually onto each document's access control list. Disable this for large site groups to store a compact site group token on documents instead, and resolve membership during access control syncs. Changing this setting requires a full content sync and access control sync.", + "default_value": true, + "label": "Expand site group members", + "sensitive": false, + "type": "bool", + "required": true, + "options": [], + "validations": [], + "value": true, + "order": 15, + "ui_restrictions": [] } } } diff --git a/app/connectors_service/tests/sources/test_sharepoint_online.py b/app/connectors_service/tests/sources/test_sharepoint_online.py index 473e8476c..652cc8d71 100644 --- a/app/connectors_service/tests/sources/test_sharepoint_online.py +++ b/app/connectors_service/tests/sources/test_sharepoint_online.py @@ -56,8 +56,10 @@ DeltaLinkExpired, SyncCursorEmpty, _get_login_name, + _is_guest_user, _prefix_email, _prefix_group, + _prefix_site_group, _prefix_user, _prefix_user_id, ) @@ -269,6 +271,7 @@ async def create_spo_source( fetch_drive_item_permissions=True, fetch_unique_list_permissions=True, enumerate_all_sites=False, + expand_site_group_members=True, ): async with create_source( SharepointOnlineDataSource, @@ -285,6 +288,7 @@ async def create_spo_source( fetch_drive_item_permissions=fetch_drive_item_permissions, fetch_unique_list_permissions=fetch_unique_list_permissions, enumerate_all_sites=enumerate_all_sites, + expand_site_group_members=expand_site_group_members, ) as source: source.set_features( Features( @@ -1706,6 +1710,32 @@ async def test_tenant_details(self, client, patch_fetch): assert http_call_result == actual_result + @pytest.mark.asyncio + async def test_site_groups(self, client, patch_scroll): + site_web_url = f"https://{self.tenant_name}.sharepoint.com/sites/test" + groups = [{"Id": 3, "Title": "Visitors"}, {"Id": 5, "Title": "Members"}] + + actual_groups = await self._execute_scrolling_method( + client.site_groups, + patch_scroll, + groups, + site_web_url, + ) + + assert actual_groups == groups + + @pytest.mark.asyncio + async def test_site_groups_not_found(self, client, patch_scroll): + site_web_url = f"https://{self.tenant_name}.sharepoint.com/sites/test" + + patch_scroll.side_effect = NotFound() + + returned_items = [] + async for item in client.site_groups(site_web_url): + returned_items.append(item) + + assert len(returned_items) == 0 + @pytest.mark.asyncio async def test_site_group_users(self, client, patch_scroll): site_group_id = 42 @@ -4013,6 +4043,442 @@ def test_prefix_user_id(self): assert _prefix_user_id(user_id) == "user_id:user id" + def test_prefix_site_group(self): + assert _prefix_site_group("site-abc", 5) == "site_group:site-abc:5" + + @pytest.mark.parametrize( + "user,expected", + [ + ({"userType": "Guest"}, True), + ({"userType": "Member"}, False), + ({"userPrincipalName": "guest_user#EXT#@contoso.onmicrosoft.com"}, True), + ({"userPrincipalName": USER_ONE_EMAIL}, False), + ({}, False), + ], + ) + def test_is_guest_user(self, user, expected): + assert _is_guest_user(user) is expected + + @pytest.mark.asyncio + async def test_with_drive_item_permissions_compact_site_group( + self, patch_sharepoint_client + ): + site_id = "graph-site-id" + site_group_id = "3" + async with create_spo_source( + use_document_level_security=True, expand_site_group_members=False + ) as source: + drive_item = {"id": 1} + source.site_group_users = AsyncMock( + side_effect=AssertionError("should not expand site group members") + ) + + drive_item_with_access_control = await source._with_drive_item_permissions( + drive_item, + [{"grantedToV2": {"siteGroup": {"id": site_group_id}}}], + "dummy_site_web_url", + site_id=site_id, + ) + + assert drive_item_with_access_control[ACCESS_CONTROL] == [ + _prefix_site_group(site_id, site_group_id) + ] + source.site_group_users.assert_not_called() + + @pytest.mark.asyncio + async def test_with_drive_item_permissions_compact_falls_back_without_site_id( + self, patch_sharepoint_client + ): + site_group_id = "3" + async with create_spo_source( + use_document_level_security=True, expand_site_group_members=False + ) as source: + drive_item = {"id": 1} + source.site_group_users = AsyncMock( + return_value=[ + { + "Title": "demo.user", + "Email": USER_ONE_EMAIL, + "LoginName": f"i:0#.f|membership|{USER_ONE_EMAIL}", + } + ] + ) + + drive_item_with_access_control = await source._with_drive_item_permissions( + drive_item, + [{"grantedToV2": {"siteGroup": {"id": site_group_id}}}], + "dummy_site_web_url", + site_id=None, + ) + + assert ( + _prefix_user(USER_ONE_EMAIL) + in drive_item_with_access_control[ACCESS_CONTROL] + ) + source.site_group_users.assert_called_once() + + @pytest.mark.asyncio + async def test_get_access_control_from_role_assignment_compact_site_group(self): + site_id = "graph-site-id" + group_id = 42 + role_assignment = { + "Member": { + "odata.type": "SP.Group", + "Id": group_id, + "Users": [ + { + "odata.type": "SP.User", + "UserPrincipalName": USER_ONE_EMAIL, + }, + ], + }, + "RoleDefinitionBindings": READ_BINDING, + } + + async with create_spo_source( + use_document_level_security=True, expand_site_group_members=False + ) as source: + access_control = await source._get_access_control_from_role_assignment( + role_assignment, site_id=site_id + ) + + assert access_control == [_prefix_site_group(site_id, group_id)] + + @pytest.mark.asyncio + async def test_get_access_control_from_role_assignment_compact_uses_principal_id( + self, + ): + site_id = "graph-site-id" + principal_id = 7 + role_assignment = { + "Member": { + "odata.type": "SP.Group", + "Users": [], + }, + "PrincipalId": principal_id, + "RoleDefinitionBindings": READ_BINDING, + } + + async with create_spo_source( + use_document_level_security=True, expand_site_group_members=False + ) as source: + access_control = await source._get_access_control_from_role_assignment( + role_assignment, site_id=site_id + ) + + assert access_control == [_prefix_site_group(site_id, principal_id)] + + @pytest.mark.asyncio + async def test_get_access_control_compact_enriches_identity_docs( + self, patch_sharepoint_client + ): + site_id = "graph-site-id" + group_id = 5 + site_web_url = "https://contoso.sharepoint.com/sites/hr" + member_email = SITEGROUP_USER_ONE_EMAIL + member_upn = "sitegroup.user1@spo.com" + + user = { + "id": "entra-user-1", + "userPrincipalName": member_upn, + "mail": member_email, + "transitiveMemberOf": [], + } + + async with create_spo_source( + use_document_level_security=True, expand_site_group_members=False + ) as source: + patch_sharepoint_client.active_users_with_groups = AsyncIterator([user]) + patch_sharepoint_client.site_collections = AsyncIterator( + [ + { + "webUrl": site_web_url, + "siteCollection": {"hostname": "contoso.sharepoint.com"}, + } + ] + ) + patch_sharepoint_client.sites = AsyncIterator( + [ + { + "id": site_id, + "webUrl": site_web_url, + "lastModifiedDateTime": "2024-01-01T00:00:00Z", + } + ] + ) + patch_sharepoint_client.site_groups = AsyncIterator( + [{"Id": group_id, "Title": "Visitors"}] + ) + patch_sharepoint_client.site_groups_users = AsyncIterator( + [ + { + "Email": member_email, + "UserPrincipalName": member_upn, + "LoginName": f"i:0#.f|membership|{member_email}", + "Title": "Site Group User", + } + ] + ) + + docs = [] + async for doc in source.get_access_control(): + docs.append(doc) + + assert len(docs) == 1 + access_control = docs[0]["query"]["template"]["params"]["access_control"] + assert _prefix_site_group(site_id, group_id) in access_control + assert _prefix_email(member_email) in access_control + + @pytest.mark.asyncio + async def test_get_access_control_compact_enriches_identity_docs_via_nested_group( + self, patch_sharepoint_client + ): + site_id = "graph-site-id" + group_id = 5 + site_web_url = "https://contoso.sharepoint.com/sites/hr" + member_email = USER_ONE_EMAIL + + user = { + "id": "entra-user-1", + "userPrincipalName": member_email, + "mail": member_email, + "transitiveMemberOf": [{"id": GROUP_ONE_ID}], + } + + async with create_spo_source( + use_document_level_security=True, expand_site_group_members=False + ) as source: + patch_sharepoint_client.active_users_with_groups = AsyncIterator([user]) + patch_sharepoint_client.site_collections = AsyncIterator( + [ + { + "webUrl": site_web_url, + "siteCollection": {"hostname": "contoso.sharepoint.com"}, + } + ] + ) + patch_sharepoint_client.sites = AsyncIterator( + [ + { + "id": site_id, + "webUrl": site_web_url, + "lastModifiedDateTime": "2024-01-01T00:00:00Z", + } + ] + ) + patch_sharepoint_client.site_groups = AsyncIterator( + [{"Id": group_id, "Title": "Visitors"}] + ) + patch_sharepoint_client.site_groups_users = AsyncIterator( + [ + { + "LoginName": ( + f"c:0o.c|federateddirectoryclaimprovider|{GROUP_ONE_ID}" + ), + "Title": "Nested Entra Group", + } + ] + ) + + docs = [] + async for doc in source.get_access_control(): + docs.append(doc) + + assert len(docs) == 1 + access_control = docs[0]["query"]["template"]["params"]["access_control"] + assert _prefix_site_group(site_id, group_id) in access_control + assert _prefix_group(GROUP_ONE_ID) in access_control + + @pytest.mark.asyncio + async def test_get_access_control_compact_eeeu_skips_guests( + self, patch_sharepoint_client + ): + site_id = "graph-site-id" + group_id = 5 + site_web_url = "https://contoso.sharepoint.com/sites/hr" + internal_email = USER_ONE_EMAIL + guest_email = "guest_user#EXT#@contoso.onmicrosoft.com" + + internal_user = { + "id": "entra-user-internal", + "userPrincipalName": internal_email, + "mail": internal_email, + "userType": "Member", + "transitiveMemberOf": [], + } + guest_user = { + "id": "entra-user-guest", + "userPrincipalName": guest_email, + "mail": guest_email, + "userType": "Guest", + "transitiveMemberOf": [], + } + + async with create_spo_source( + use_document_level_security=True, expand_site_group_members=False + ) as source: + patch_sharepoint_client.active_users_with_groups = AsyncIterator( + [internal_user, guest_user] + ) + patch_sharepoint_client.site_collections = AsyncIterator( + [ + { + "webUrl": site_web_url, + "siteCollection": {"hostname": "contoso.sharepoint.com"}, + } + ] + ) + patch_sharepoint_client.sites = AsyncIterator( + [ + { + "id": site_id, + "webUrl": site_web_url, + "lastModifiedDateTime": "2024-01-01T00:00:00Z", + } + ] + ) + patch_sharepoint_client.site_groups = AsyncIterator( + [{"Id": group_id, "Title": "Visitors"}] + ) + patch_sharepoint_client.site_groups_users = AsyncIterator( + [ + { + "LoginName": "c:0-.f|rolemanager|spo-grid-all-users", + "Title": "Everyone except external users", + } + ] + ) + + docs_by_id = {} + async for doc in source.get_access_control(): + docs_by_id[doc["_id"]] = doc + + internal_access_control = docs_by_id[internal_email]["query"]["template"][ + "params" + ]["access_control"] + guest_access_control = docs_by_id[guest_email]["query"]["template"][ + "params" + ]["access_control"] + + assert _prefix_site_group(site_id, group_id) in internal_access_control + assert _prefix_site_group(site_id, group_id) not in guest_access_control + + @pytest.mark.asyncio + async def test_get_access_control_compact_unmatched_member_warns( + self, patch_sharepoint_client + ): + site_id = "graph-site-id" + group_id = 5 + site_web_url = "https://contoso.sharepoint.com/sites/hr" + + user = { + "id": "entra-user-1", + "userPrincipalName": USER_ONE_EMAIL, + "mail": USER_ONE_EMAIL, + "transitiveMemberOf": [], + } + + async with create_spo_source( + use_document_level_security=True, expand_site_group_members=False + ) as source: + patch_sharepoint_client.active_users_with_groups = AsyncIterator([user]) + patch_sharepoint_client.site_collections = AsyncIterator( + [ + { + "webUrl": site_web_url, + "siteCollection": {"hostname": "contoso.sharepoint.com"}, + } + ] + ) + patch_sharepoint_client.sites = AsyncIterator( + [ + { + "id": site_id, + "webUrl": site_web_url, + "lastModifiedDateTime": "2024-01-01T00:00:00Z", + } + ] + ) + patch_sharepoint_client.site_groups = AsyncIterator( + [{"Id": group_id, "Title": "Visitors"}] + ) + patch_sharepoint_client.site_groups_users = AsyncIterator( + [ + { + "Title": "Unknown User", + } + ] + ) + + with patch.object(source._logger, "warning") as warning_mock: + docs = [] + async for doc in source.get_access_control(): + docs.append(doc) + + assert len(docs) == 1 + access_control = docs[0]["query"]["template"]["params"]["access_control"] + assert _prefix_site_group(site_id, group_id) not in access_control + warning_mock.assert_called() + warning_text = " ".join( + str(call.args[0]) for call in warning_mock.call_args_list if call.args + ) + assert "could not be resolved" in warning_text + + @pytest.mark.asyncio + async def test_get_access_control_compact_streams_users_after_site_group_index( + self, patch_sharepoint_client + ): + site_id = "graph-site-id" + group_id = 5 + site_web_url = "https://contoso.sharepoint.com/sites/hr" + call_order = [] + + user = { + "id": "entra-user-1", + "userPrincipalName": USER_ONE_EMAIL, + "mail": USER_ONE_EMAIL, + "transitiveMemberOf": [], + } + + async def track_site_groups(site_web_url): + call_order.append("site_groups") + yield {"Id": group_id, "Title": "Visitors"} + + async def track_users(): + call_order.append("users") + yield user + + async with create_spo_source( + use_document_level_security=True, expand_site_group_members=False + ) as source: + patch_sharepoint_client.site_collections = AsyncIterator( + [ + { + "webUrl": site_web_url, + "siteCollection": {"hostname": "contoso.sharepoint.com"}, + } + ] + ) + patch_sharepoint_client.sites = AsyncIterator( + [ + { + "id": site_id, + "webUrl": site_web_url, + "lastModifiedDateTime": "2024-01-01T00:00:00Z", + } + ] + ) + patch_sharepoint_client.site_groups = track_site_groups + patch_sharepoint_client.site_groups_users = AsyncIterator([]) + patch_sharepoint_client.active_users_with_groups = track_users + + docs = [] + async for doc in source.get_access_control(): + docs.append(doc) + + assert call_order == ["site_groups", "users"] + assert len(docs) == 1 + @pytest.mark.parametrize( "role_assignment, expected_access_control", [