From adaab800bd84c1ea415180d371531a1ed9dc054b Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 31 Oct 2025 10:58:28 -0500 Subject: [PATCH 01/35] adding support for vector download --- fence/blueprints/data/indexd.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index c8293044c..f2c3fafe3 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -65,7 +65,7 @@ "az": {"upload": "PUT", "download": "GET"}, } -SUPPORTED_PROTOCOLS = ["s3", "http", "ftp", "https", "gs", "az"] +SUPPORTED_PROTOCOLS = ["s3", "http", "ftp", "https", "gs", "az", "vec"] SUPPORTED_ACTIONS = ["upload", "download"] ANONYMOUS_USER_ID = "-1" ANONYMOUS_USERNAME = "anonymous" @@ -643,6 +643,9 @@ def _get_signed_url( except IndexError: raise NotFound("Can't find any file locations.") + if protocol == "vec": + return file_location.get_vector(action, expires_in) + for file_location in self.indexed_file_locations: # allow file location to be https, even if they specific http if (file_location.protocol == protocol) or ( @@ -865,6 +868,16 @@ def get_signed_url( ): return self.url + def get_vector(self, action, expires_in): + # need to return the vector from the embedding management service + emsID = str(self.url).replace("vec://") + emsUrl = f"{config.get("BASE_URL")}/ems/{emsID}" + try: + req = requests.get(emsUrl) + return req.json() + except Exception as e: + raise NotFound(f"No embedding found with id {emsID}") + class S3IndexedFileLocation(IndexedFileLocation): """ From a18b512dbdc74093f73483fa7fa495b9785e9983 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 4 Nov 2025 11:19:04 -0600 Subject: [PATCH 02/35] fix fstring escape --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index f2c3fafe3..fa3627346 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -871,7 +871,7 @@ def get_signed_url( def get_vector(self, action, expires_in): # need to return the vector from the embedding management service emsID = str(self.url).replace("vec://") - emsUrl = f"{config.get("BASE_URL")}/ems/{emsID}" + emsUrl = f"{config.get('BASE_URL')}/ems/{emsID}" try: req = requests.get(emsUrl) return req.json() From 9255e1341e023276fbaa6473413b3962b37c6818 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 4 Nov 2025 11:23:55 -0600 Subject: [PATCH 03/35] add model to vec get --- fence/blueprints/data/indexd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index fa3627346..b14b611ea 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -870,8 +870,10 @@ def get_signed_url( def get_vector(self, action, expires_in): # need to return the vector from the embedding management service - emsID = str(self.url).replace("vec://") - emsUrl = f"{config.get('BASE_URL')}/ems/{emsID}" + model, emsID = str(self.url).replace("vec://", "").split("/") + emsUrl = ( + f"{config.get('BASE_URL')}/ems//vector/indexes/{model}/embeddings/{emsID}" + ) try: req = requests.get(emsUrl) return req.json() From ad409e04804dca19264f83cff2102f5c6cc1398c Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 4 Nov 2025 12:03:04 -0600 Subject: [PATCH 04/35] change route for vector download --- fence/blueprints/data/indexd.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index b14b611ea..d82c9d92d 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -871,9 +871,7 @@ def get_signed_url( def get_vector(self, action, expires_in): # need to return the vector from the embedding management service model, emsID = str(self.url).replace("vec://", "").split("/") - emsUrl = ( - f"{config.get('BASE_URL')}/ems//vector/indexes/{model}/embeddings/{emsID}" - ) + emsUrl = f"gen3-embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" try: req = requests.get(emsUrl) return req.json() From 98abb8fb10130e0c065c31e16c5a6b10341006c5 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 4 Nov 2025 14:25:36 -0600 Subject: [PATCH 05/35] update vector return --- fence/blueprints/data/indexd.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index d82c9d92d..faa99e919 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -630,6 +630,17 @@ def _get_signed_url( bucket=bucket, ) + print( + f"====================== THIS IS OUR PROTOCOL ======================================" + ) + print(protocol) + print( + "=========================== End of Protocol =======================================" + ) + + if protocol == "vec": + return file_location.get_vector(action, expires_in) + if not protocol: # no protocol specified, return first location as signed url try: @@ -643,9 +654,6 @@ def _get_signed_url( except IndexError: raise NotFound("Can't find any file locations.") - if protocol == "vec": - return file_location.get_vector(action, expires_in) - for file_location in self.indexed_file_locations: # allow file location to be https, even if they specific http if (file_location.protocol == protocol) or ( From 15a5cf8acd152f57ae4112c8256060bfd2d6bd6f Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 4 Nov 2025 15:00:04 -0600 Subject: [PATCH 06/35] add vec class --- fence/blueprints/data/indexd.py | 39 +++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index faa99e919..893478955 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -630,16 +630,16 @@ def _get_signed_url( bucket=bucket, ) - print( - f"====================== THIS IS OUR PROTOCOL ======================================" - ) - print(protocol) - print( - "=========================== End of Protocol =======================================" - ) + # print( + # f"====================== THIS IS OUR PROTOCOL ======================================" + # ) + # print(protocol) + # print( + # "=========================== End of Protocol =======================================" + # ) - if protocol == "vec": - return file_location.get_vector(action, expires_in) + # if protocol == "vec": + # return file_location.get_vector(action, expires_in) if not protocol: # no protocol specified, return first location as signed url @@ -864,6 +864,8 @@ def from_url(url): return GoogleStorageIndexedFileLocation(url) elif protocol == "az": return AzureBlobStorageIndexedFileLocation(url) + elif protocol == "vec": + return VecIndexdFileLocation(url) return IndexedFileLocation(url) def get_signed_url( @@ -887,6 +889,25 @@ def get_vector(self, action, expires_in): raise NotFound(f"No embedding found with id {emsID}") +class VecIndexdFileLocation(IndexedFileLocation): + def get_signed_url( + self, + action, + expires_in, + force_signed_url=True, + r_pays_project=None, + authorized_user=None, + ): + # need to return the vector from the embedding management service + model, emsID = str(self.url).replace("vec://", "").split("/") + emsUrl = f"gen3-embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" + try: + req = requests.get(emsUrl) + return req.json() + except Exception as e: + raise NotFound(f"No embedding found with id {emsID}") + + class S3IndexedFileLocation(IndexedFileLocation): """ An indexed file that lives in an AWS S3 bucket. From a5804e3bcfffb6db689ebc33a0a6f32dba083301 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 4 Nov 2025 15:12:25 -0600 Subject: [PATCH 07/35] add debug statement for vec --- fence/blueprints/data/indexd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 893478955..409637ac9 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -886,6 +886,7 @@ def get_vector(self, action, expires_in): req = requests.get(emsUrl) return req.json() except Exception as e: + print(f"error {e}") raise NotFound(f"No embedding found with id {emsID}") From f9ec6b078e8292adeb56f0d27a5a735275b3f776 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 4 Nov 2025 15:21:52 -0600 Subject: [PATCH 08/35] fix requests url --- fence/blueprints/data/indexd.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 409637ac9..748d74390 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -878,16 +878,16 @@ def get_signed_url( ): return self.url - def get_vector(self, action, expires_in): - # need to return the vector from the embedding management service - model, emsID = str(self.url).replace("vec://", "").split("/") - emsUrl = f"gen3-embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" - try: - req = requests.get(emsUrl) - return req.json() - except Exception as e: - print(f"error {e}") - raise NotFound(f"No embedding found with id {emsID}") + # def get_vector(self, action, expires_in): + # # need to return the vector from the embedding management service + # model, emsID = str(self.url).replace("vec://", "").split("/") + # emsUrl = f"gen3-embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" + # try: + # req = requests.get(emsUrl) + # return req.json() + # except Exception as e: + # print(f"error {e}") + # raise NotFound(f"No embedding found with id {emsID}") class VecIndexdFileLocation(IndexedFileLocation): @@ -901,11 +901,12 @@ def get_signed_url( ): # need to return the vector from the embedding management service model, emsID = str(self.url).replace("vec://", "").split("/") - emsUrl = f"gen3-embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" + emsUrl = f"https://gen3-embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" try: req = requests.get(emsUrl) return req.json() except Exception as e: + print(f"error {e}") raise NotFound(f"No embedding found with id {emsID}") From 98ccf744845389d854dcd6d74103b94bc67c71ff Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 4 Nov 2025 15:37:07 -0600 Subject: [PATCH 09/35] fix request protocol for vec signed url --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 748d74390..f8e00fc91 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -901,7 +901,7 @@ def get_signed_url( ): # need to return the vector from the embedding management service model, emsID = str(self.url).replace("vec://", "").split("/") - emsUrl = f"https://gen3-embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" + emsUrl = f"http://gen3-embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" try: req = requests.get(emsUrl) return req.json() From 589d2e0c9da61191f66e0ad431a18a6be5c4117a Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 5 Nov 2025 16:45:43 -0600 Subject: [PATCH 10/35] adding vector upload --- fence/blueprints/data/blueprint.py | 139 +++++++++++++++++++++++++++++ fence/blueprints/data/indexd.py | 114 +++++++++++++++++++++++ 2 files changed, 253 insertions(+) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index d8041b949..200bc2b43 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -1,4 +1,6 @@ import flask +import json +import hashlib from cdislogging import get_logger from cdispyutils.config import get_value @@ -197,6 +199,143 @@ def upload_data_file(): return flask.jsonify(response), 201 +@blueprint.route("/upload/vector", methods=["POST"]) +@require_auth_header(scope={"data"}) +@login_required({"data"}) +def upload_vector(): + """ + Return a presigned URL for use with uploading a data file. + + See the documentation on the entire flow here for more info: + + https://github.com/uc-cdis/cdis-wiki/tree/master/dev/gen3/data_upload + + """ + # make new record in indexd, with just the `uploader` field (and a GUID) + + # { + # "authz": [ + # "/programs/DEV" + # ], + # "model": "expr", + # "file_id": "1234" + # "embedding": [0,1,2,3,] + # } + + params = flask.request.get_json() + if not params: + raise UserError("wrong Content-Type; expected application/json") + + # if "file_name" not in params: + # raise UserError("missing required argument `file_name`") + if "model" not in params: + raise UserError("missing required argument `model`") + + if "embedding" not in params: + raise UserError("missing required argument `embedding`") + + if "file_id" not in params: + raise UserError("missing required argument `file_id`") + + if "authz" not in params: + raise UserError("missing required argument `authz`") + + authorized = False + authz_err_msg = "Auth error when attempting to get a presigned URL for upload. User must have '{}' access on '{}'." + + authz = params.get("authz") + uploader = None + + guid = params.get("guid") + + if authz: + # if requesting an authz field, using new authorization method which doesn't + # rely on uploader field, so clear it out + uploader = "" + authorized = flask.current_app.arborist.auth_request( + jwt=get_jwt(), + service="fence", + methods=["create", "write-storage"], + resources=authz, + ) + if not authorized: + logger.error(authz_err_msg.format("create' and 'write-storage", authz)) + else: + # no 'authz' was provided, so fall back on 'file_upload' logic + authorized = flask.current_app.arborist.auth_request( + jwt=get_jwt(), + service="fence", + methods=["file_upload"], + resources=["/data_file"], + ) + if not authorized: + logger.error(authz_err_msg.format("file_upload", "/data_file")) + + if not authorized: + raise Forbidden( + "You do not have access to upload data. You either need " + "general file uploader permissions or create and write-storage permissions " + "on the authz resources you specified (if you specified any)." + ) + + # token = get_jwt() + + model = params.get("model") + embedding = params.get("embedding") + file_id = params.get("file_id") + + EMS = EmbeddingIndex( + authz=authz, + model=model, + embedding=embedding, + file_id=file_id, + uploader=uploader, + ) + + # create embedding record and get id from service + embedding_id, md5_hash = EMS.create_embedding_record() + + guid = EMS.create_index_record(embedding_id, md5_hash) + + response = { + "message": "An embedding was successfully added to the embedding management service and inserted into indexd", + "guid": guid, + } + + # create embedding in emebedding management service + # embedding_management_url = "http://" + # embedding_management_service_url = f"http://gen3-embedding-management-service/vector/indexes/{model}/embeddings" + + # blank_index = BlankIndex( + # authz=authz, + # uploader=uploader, + # guid=guid, + # ) + # default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600) + + # expires_in = get_valid_expiration( + # params.get("expires_in"), + # max_limit=default_expires_in, + # default=default_expires_in, + # ) + + protocol = params["protocol"] if "protocol" in params else None + bucket = params.get("bucket") + if bucket: + verify_data_upload_bucket_configuration(bucket) + response = { + "guid": blank_index.guid, + "url": blank_index.make_signed_url( + file_name=params["file_name"], + protocol=protocol, + expires_in=expires_in, + bucket=bucket, + ), + } + + return flask.jsonify(response), 201 + + @blueprint.route("/multipart/init", methods=["POST"]) @require_auth_header(scope={"data"}) @login_required({"data"}) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index f8e00fc91..b3701bc3c 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1,6 +1,7 @@ import re import time import json +import hashlib import boto3 from botocore.client import Config from urllib.parse import urlparse, ParseResult, urlunparse @@ -261,6 +262,119 @@ def prepare_presigned_url_audit_log(protocol, indexed_file): flask.g.audit_data["protocol"] = protocol +class EmbeddingIndex(object): + def __init__( + self, + authz=None, + model=None, + embedding=None, + file_id=None, + uploader=None, + logger_=None, + ): + self.logger = logger_ or logger + self.indexd = ( + flask.current_app.config.get("INDEXD") + or flask.current_app.config["BASE_URL"] + "/index" + ) + + # allow passing "" empty string to signify you do NOT want + # uploader to be populated. If nothing is provided, default + # to parsing from token + if uploader == "": + self.uploader = None + elif uploader: + self.uploader = uploader + else: + self.uploader = current_token["context"]["user"]["name"] + + self.authz = authz + self.model = model + self.embedding = embedding + self.file_id = file_id + + def create_embedding_record(self): + """ + create a record in the embedding management service + """ + + embedding_management_service_url = f"http://gen3-embedding-management-service/vector/indexes/{self.model}/embeddings" + params = { + "file_id": self.file_id, + "vector": self.embedding, + } + + ems_response = requests.post(embedding_management_service_url, json=params) + if ems_response.status_code not in [200, 201]: + try: + data = ems_response.json() + except ValueError: + data = ems_response.text + self.logger.error( + "could not create new record in embedding management service; got response: {}".format( + data + ) + ) + raise InternalError("received error from embedding management service") + else: + dict_str = json.dumps(params, sort_keys=True) + dict_bytes = dict_str.encode("utf-8") + md5_hash = hashlib.md5(dict_bytes).hexdigest() + + emsID = ems_response.text + + return emsID, md5_hash + + def create_indexd_record(self, embedding_id, md5_hash): + """ + create an indexd record for a vector corresponding to the embedding_id + """ + index_url = self.indexd.rstrip("/") + "/index/" + vec_url = f"vec://{self.model}/{embedding_id}" + + params = { + "authz": self.authz, + "form": "vector", + "hashes": {"md5": md5_hash}, + "size": 0, + "urls": [vec_url], + } + params = {"uploader": self.uploader, "file_name": self.file_name} + + if self.authz: + params["authz"] = self.authz + token = get_jwt() + + auth = None + headers = {"Authorization": f"bearer {token}"} + logger.info("passing users authorization header to create blank record") + else: + logger.info("using indexd basic auth to create blank record") + auth = (config["INDEXD_USERNAME"], config["INDEXD_PASSWORD"]) + headers = {} + + indexd_response = requests.post( + index_url, json=params, headers=headers, auth=auth + ) + if indexd_response.status_code not in [200, 201]: + try: + data = indexd_response.json() + except ValueError: + data = indexd_response.text + self.logger.error( + "could not create new record in indexd; got response: {}".format(data) + ) + raise InternalError( + "received error from indexd trying to create blank record" + ) + + document = indexd_response.json() + guid = document["did"] + self.logger.info("created an index record with GUID {} for upload".format(guid)) + + return guid + + class BlankIndex(object): """ A blank record in indexd, to use for the data upload flow. From fcda7dd55a89f7ca58e2fb985414c4aa9dba84db Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 5 Nov 2025 17:03:50 -0600 Subject: [PATCH 11/35] fix import and add logging --- fence/blueprints/data/blueprint.py | 5 +++-- fence/blueprints/data/indexd.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index 200bc2b43..fc32e297e 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -9,6 +9,7 @@ from fence.authz.auth import check_arborist_auth from fence.blueprints.data.indexd import ( BlankIndex, + EmbeddingIndex, IndexedFile, get_signed_url_for_file, verify_data_upload_bucket_configuration, @@ -218,8 +219,8 @@ def upload_vector(): # "/programs/DEV" # ], # "model": "expr", - # "file_id": "1234" - # "embedding": [0,1,2,3,] + # "file_id": "1234", + # "embedding": [0,1,2,3] # } params = flask.request.get_json() diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index b3701bc3c..d9e8f1dcf 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -322,6 +322,9 @@ def create_embedding_record(self): md5_hash = hashlib.md5(dict_bytes).hexdigest() emsID = ems_response.text + logger.info( + f"successfully added vector to embedding service with id {emsID}" + ) return emsID, md5_hash From 9ff13a75468927d40ad74dc0d4c1ecfb36b6201d Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 5 Nov 2025 17:16:10 -0600 Subject: [PATCH 12/35] fix function typo --- fence/blueprints/data/blueprint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index fc32e297e..9f8bb879e 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -296,7 +296,7 @@ def upload_vector(): # create embedding record and get id from service embedding_id, md5_hash = EMS.create_embedding_record() - guid = EMS.create_index_record(embedding_id, md5_hash) + guid = EMS.create_indexd_record(embedding_id, md5_hash) response = { "message": "An embedding was successfully added to the embedding management service and inserted into indexd", From 5485623fc02cc8a7b4a9471b8f400c6d83d1e02b Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 5 Nov 2025 17:27:00 -0600 Subject: [PATCH 13/35] remove old code --- fence/blueprints/data/indexd.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index d9e8f1dcf..4c97735d3 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -342,7 +342,6 @@ def create_indexd_record(self, embedding_id, md5_hash): "size": 0, "urls": [vec_url], } - params = {"uploader": self.uploader, "file_name": self.file_name} if self.authz: params["authz"] = self.authz From 36386138762b08193ecb5bc1b5f7fd3e6c93a3e3 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 5 Nov 2025 17:35:52 -0600 Subject: [PATCH 14/35] update indexd post params --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 4c97735d3..43be9a006 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -337,7 +337,7 @@ def create_indexd_record(self, embedding_id, md5_hash): params = { "authz": self.authz, - "form": "vector", + "form": "object", "hashes": {"md5": md5_hash}, "size": 0, "urls": [vec_url], From 0b237b6008e5c32389ff504b0b8bf810764f695a Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 5 Nov 2025 17:47:00 -0600 Subject: [PATCH 15/35] remove more old code --- fence/blueprints/data/blueprint.py | 31 ------------------------------ 1 file changed, 31 deletions(-) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index 9f8bb879e..66188c27e 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -303,37 +303,6 @@ def upload_vector(): "guid": guid, } - # create embedding in emebedding management service - # embedding_management_url = "http://" - # embedding_management_service_url = f"http://gen3-embedding-management-service/vector/indexes/{model}/embeddings" - - # blank_index = BlankIndex( - # authz=authz, - # uploader=uploader, - # guid=guid, - # ) - # default_expires_in = flask.current_app.config.get("MAX_PRESIGNED_URL_TTL", 3600) - - # expires_in = get_valid_expiration( - # params.get("expires_in"), - # max_limit=default_expires_in, - # default=default_expires_in, - # ) - - protocol = params["protocol"] if "protocol" in params else None - bucket = params.get("bucket") - if bucket: - verify_data_upload_bucket_configuration(bucket) - response = { - "guid": blank_index.guid, - "url": blank_index.make_signed_url( - file_name=params["file_name"], - protocol=protocol, - expires_in=expires_in, - bucket=bucket, - ), - } - return flask.jsonify(response), 201 From f0d09fdc8425b810d357149acadda96f503a5a55 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 5 Nov 2025 18:03:23 -0600 Subject: [PATCH 16/35] fix string in url --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 43be9a006..9bca1104b 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -333,7 +333,7 @@ def create_indexd_record(self, embedding_id, md5_hash): create an indexd record for a vector corresponding to the embedding_id """ index_url = self.indexd.rstrip("/") + "/index/" - vec_url = f"vec://{self.model}/{embedding_id}" + vec_url = f"vec://{self.model}/{embedding_id.strip('"')}" params = { "authz": self.authz, From 6dcb43c08db08f8c3231bf953e6941a1f47e2b68 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 5 Nov 2025 18:07:47 -0600 Subject: [PATCH 17/35] fix string escape typo --- fence/blueprints/data/indexd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 9bca1104b..9e4e18ece 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -333,7 +333,8 @@ def create_indexd_record(self, embedding_id, md5_hash): create an indexd record for a vector corresponding to the embedding_id """ index_url = self.indexd.rstrip("/") + "/index/" - vec_url = f"vec://{self.model}/{embedding_id.strip('"')}" + escaped_id = embedding_id.strip('"') + vec_url = f"vec://{self.model}/{escaped_id}" params = { "authz": self.authz, From 52ab7e92d98943c6d2eb364cad27b9e6cf5b41f1 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 7 Nov 2025 10:51:42 -0600 Subject: [PATCH 18/35] ensure md5 is not used for security --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 9e4e18ece..647d70381 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -319,7 +319,7 @@ def create_embedding_record(self): else: dict_str = json.dumps(params, sort_keys=True) dict_bytes = dict_str.encode("utf-8") - md5_hash = hashlib.md5(dict_bytes).hexdigest() + md5_hash = hashlib.md5(dict_bytes, usedforsecurity=False).hexdigest() emsID = ems_response.text logger.info( From 68392f8f56ecac7f5e1211c1cc959ab90d318eba Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 13 Nov 2025 13:20:22 -0600 Subject: [PATCH 19/35] add bulk download --- fence/blueprints/data/blueprint.py | 38 +++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index 66188c27e..62662fd03 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -214,21 +214,10 @@ def upload_vector(): """ # make new record in indexd, with just the `uploader` field (and a GUID) - # { - # "authz": [ - # "/programs/DEV" - # ], - # "model": "expr", - # "file_id": "1234", - # "embedding": [0,1,2,3] - # } - params = flask.request.get_json() if not params: raise UserError("wrong Content-Type; expected application/json") - # if "file_name" not in params: - # raise UserError("missing required argument `file_name`") if "model" not in params: raise UserError("missing required argument `model`") @@ -457,6 +446,33 @@ def download_file(file_id): return flask.redirect(result["url"]) +@blueprint.route("/download/bulk", methods=["POST"]) +def download_bulk_files(): + """ + Get a presigned url to download a file given by file_id. + """ + # {"guids": ["1234", "4567"]} + params = flask.request.get_json() + if not params: + raise UserError("wrong Content-Type; expected application/json") + + if "guids" not in params: + raise UserError("missing required argument `model`") + + guids = params["guids"] + results = {} + results["urls"] = [] + for g in guids: + result = get_signed_url_for_file("download", file_id) + if not "redirect" in flask.request.args or not "url" in result: + # return flask.jsonify(result) + results["urls"].append(result) + else: + results["urls"].append(result["url"]) + return flask.jsonify(results) + # return flask.redirect(result["url"]) + + @blueprint.route( "/buckets", methods=["GET"], From 24ac1108e65c3b7191e1d1e7fab6e17e80d3f5e9 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 13 Nov 2025 13:20:37 -0600 Subject: [PATCH 20/35] add bulk download --- fence/blueprints/data/blueprint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index 62662fd03..5b42434d9 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -463,7 +463,7 @@ def download_bulk_files(): results = {} results["urls"] = [] for g in guids: - result = get_signed_url_for_file("download", file_id) + result = get_signed_url_for_file("download", g) if not "redirect" in flask.request.args or not "url" in result: # return flask.jsonify(result) results["urls"].append(result) From 59e02348b9a69dcd63705ea51ec7a29c9fdd3db8 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 10:36:20 -0600 Subject: [PATCH 21/35] bulk download with authz role caching --- fence/blueprints/data/blueprint.py | 24 +-- fence/blueprints/data/indexd.py | 295 +++++++++++++++++++++++++++-- 2 files changed, 297 insertions(+), 22 deletions(-) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index 5b42434d9..35a33b38b 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -457,20 +457,22 @@ def download_bulk_files(): raise UserError("wrong Content-Type; expected application/json") if "guids" not in params: - raise UserError("missing required argument `model`") + raise UserError("missing required argument `guids`") guids = params["guids"] - results = {} - results["urls"] = [] - for g in guids: - result = get_signed_url_for_file("download", g) - if not "redirect" in flask.request.args or not "url" in result: - # return flask.jsonify(result) - results["urls"].append(result) - else: - results["urls"].append(result["url"]) + result = bulk_get_signed_url_for_file(guids) + return flask.jsonify(results) - # return flask.redirect(result["url"]) + # results = {} + # results["urls"] = [] + # for g in guids: + # result = get_signed_url_for_file("download", g) + # if not "redirect" in flask.request.args or not "url" in result: + # # return flask.jsonify(result) + # results["urls"].append(result) + # else: + # results["urls"].append(result["url"]) + # return flask.jsonify(results) @blueprint.route( diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 647d70381..a8da684f6 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -183,6 +183,58 @@ def get_signed_url_for_file( return {"url": signed_url} +@enable_audit_logging +def bulk_get_signed_url_for_file( + file_ids, + requested_protocol=None, + db_session=None, + bucket=None, + drs="False", +): + requested_protocol = requested_protocol or flask.request.args.get("protocol", None) + r_pays_project = flask.request.args.get("userProject", None) + db_session = db_session or current_app.scoped_session() + + # default to signing the URL + force_signed_url = True + no_force_sign_param = flask.request.args.get("no_force_sign") + if no_force_sign_param and no_force_sign_param.lower() == "true": + force_signed_url = False + + # Collect X-Forwarded headers + x_forwarded_headers = [ + f"{header}:{value}" for header, value in flask.request.headers if "X-" in header + ] + + auth_info = _get_auth_info_for_id_or_from_request( + sub_type=int, db_session=db_session + ) + flask.g.audit_data = { + "username": auth_info["username"], + "sub": auth_info["user_id"], + "additional_data": x_forwarded_headers, + } + + indexed_files = BulkIndexedFile(file_ids) + + default_expires_in = config.get("MAX_PRESIGNED_URL_TTL", 3600) + expires_in = get_valid_expiration_from_request( + max_limit=default_expires_in, + default=default_expires_in, + ) + + # prepare_presigned_url_audit_log(requested_protocol, indexed_file) + signed_urls = indexed_files.get_signed_urls( + requested_protocol, + expires_in, + force_signed_url=force_signed_url, + r_pays_project=r_pays_project, + bucket=bucket, + ) + + return signed_urls + + def get_bucket_from_urls(urls, protocol): """ Return the bucket name from the first of the provided URLs that starts with the given protocol (usually `gs`, `s3`, `az`...) @@ -587,6 +639,238 @@ def generate_aws_presigned_url_for_part( ) +class BulkIndexedFiles(object): + """ + For bulk handeling of files + + Args: + file_ids (list(str)): A list of GUIDs coresponding to files + """ + + def __init__(self, file_ids): + self.file_ids = file_ids + self.auth_roles = [] + + @cached_property + def indexd_server(self): + indexd_server = ( + flask.current_app.config.get("INDEXD") + or flask.current_app.config["BASE_URL"] + "/index" + ) + return indexd_server.rstrip("/") + + @cached_property + def index_document(self): + index_dict = {} + for file_id in self.file_ids: + indexd_server = config.get("INDEXD") or config["BASE_URL"] + "/index" + url = indexd_server + "/index/" + try: + res = requests.get(url + file_id) + except Exception as e: + logger.error( + "failed to reach indexd at {0}: {1}".format(url + file_id, e) + ) + raise UnavailableError("Fail to reach id service to find data location") + if res.status_code == 200: + try: + json_response = res.json() + if "urls" not in json_response: + logger.error( + "URLs are not included in response from " + "indexd: {}".format(url + file_id) + ) + raise InternalError("URLs and metadata not found") + + # indexd can resolve GUIDs without prefix, but cannot perform other operations + # (such as delete) without the prefix, so make sure `file_id` is the whole GUID + real_guid = json_response.get("did") + if real_guid and real_guid != file_id: + file_id = real_guid + + index_dict[file_id] = json_response + except Exception as e: + logger.error( + "indexd response missing JSON field {}".format(url + file_id) + ) + raise InternalError("internal error from indexd: {}".format(e)) + elif res.status_code == 404: + logger.error( + "Not Found. indexd could not find {}: {}".format( + url + file_id, res.text + ) + ) + raise NotFound("No indexed document found with id {}".format(file_id)) + else: + raise UnavailableError(res.text) + + return index_dict + + def get_signed_urls( + self, + protocol, + expires_in, + force_signed_url=True, + r_pays_project=None, + users_from_passports=None, + ): + users_from_passports = users_from_passports or {} + signed_urls = [] + for file_id in file_ids: + authorized_user = None + file_authz = self.index_document.get(file_id).get("authz") + + if file_authz: + # check if user has already been authorized in this bulk operation + if self.index_document.get(file_id).get("authz") not in self.auth_roles: + action_to_permission = { + "download": "read-storage", + } + is_authorized, authorized_username = ( + self.get_authorized_with_username( + action_to_permission[action], + file_id, + # keys are usernames + usernames_from_passports=list(users_from_passports.keys()), + ) + ) + if not is_authorized: + msg = ( + f"Either you weren't authenticated successfully or you don't have " + f"{action_to_permission[action]} permission " + f"on authorization resource: {self.index_document['authz']}." + ) + logger.debug( + f"denied. authorized_username: {authorized_username}\nmsg:\n{msg}" + ) + raise Unauthorized(msg) + self.auth_roles.append( + self.index_document.get(file_id).get("authz") + ) + authorized_user = users_from_passports.get(authorized_username) + + if action is not None and action not in SUPPORTED_ACTIONS: + raise NotSupported("action {} is not supported".format(action)) + signed_url_tuple = ( + self._get_signed_urls( + protocol, + file_id, + expires_in, + force_signed_url, + r_pays_project, + authorized_user, + ), + authorized_user, + ) + signed_urls.append(signed_url_tuple) + return signed_urls + + def _get_signed_urls( + self, + protocol, + file_id, + expires_in, + force_signed_url, + r_pays_project, + authorized_user=None, + ): + # we only allow for download with bulk operations + file_urls = self.index_document.get(file_id).get("urls", []) + file_locations = list(map(IndexedFileLocation.from_url, file_urls)) + if not protocol: + # no protocol specified, return first location as signed url + try: + return file_locations[0].get_signed_url( + action, + expires_in, + force_signed_url=force_signed_url, + r_pays_project=r_pays_project, + authorized_user=authorized_user, + ) + except IndexError: + raise NotFound("Can't find any file locations.") + + for file_location in file_locations: + # allow file location to be https, even if they specific http + if (file_location.protocol == protocol) or ( + protocol == "http" and file_location.protocol == "https" + ): + return file_location.get_signed_url( + action, + expires_in, + force_signed_url=force_signed_url, + r_pays_project=r_pays_project, + authorized_user=authorized_user, + ) + + raise NotFound( + "File {} does not have a location with specified " + "protocol {}.".format(file_id, protocol) + ) + + def get_authorized_with_username( + self, action, file_id, usernames_from_passports=None + ): + """ + Return a tuple of (boolean, str) which represents whether they're authorized + and their username. username is only returned if `usernames_from_passports` + is provided and one of the usernames from the passports is authorized. + + Args: + action (str): Authorization action being performed + usernames_from_passports (list[str], optional): List of user usernames parsed + from validated passports + + Returns: + tuple of (boolean, str): which represents whether they're authorized + and their username. username is only returned if `usernames_from_passports` + is provided and one of the usernames from the passports is authorized. + """ + if not self.index_document.get(file_id).get("authz"): + raise ValueError("index record missing `authz`") + + logger.debug( + f"authz check can user {action} on {self.index_document[file_id]['authz']} for fence? " + f"if passport provided, IDs parsed: {usernames_from_passports}" + ) + + # handle multiple GA4GH passports as a means of authn/z + if usernames_from_passports: + authorized = False + for username in usernames_from_passports: + authorized = flask.current_app.arborist.auth_request( + jwt=None, + user_id=username, + service="fence", + methods=action, + resources=self.index_document[file_id]["authz"], + ) + # if any passport provides access, user is authorized + if authorized: + # for google proxy groups and future use: we need to know which + # user_id actually gave access + return authorized, username + return authorized, None + else: + try: + token = get_jwt() + except Unauthorized: + # get_jwt raises an Unauthorized error when user is anonymous (no + # available token), so to allow anonymous users possible access to + # public data, we still make the request to Arborist + token = None + + return ( + flask.current_app.arborist.auth_request( + jwt=token, + service="fence", + methods=action, + resources=self.index_document[file_id]["authz"], + ), + None, + ) + + class IndexedFile(object): """ A file from the index service that will contain information about access and where @@ -747,17 +1031,6 @@ def _get_signed_url( bucket=bucket, ) - # print( - # f"====================== THIS IS OUR PROTOCOL ======================================" - # ) - # print(protocol) - # print( - # "=========================== End of Protocol =======================================" - # ) - - # if protocol == "vec": - # return file_location.get_vector(action, expires_in) - if not protocol: # no protocol specified, return first location as signed url try: From 1f2e60fa1214b54feac4cdbc75a0d3c9b60a54ba Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 13:56:44 -0600 Subject: [PATCH 22/35] add function definiton --- fence/blueprints/data/blueprint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index 35a33b38b..acd0db8a2 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -12,6 +12,7 @@ EmbeddingIndex, IndexedFile, get_signed_url_for_file, + bulk_get_signed_url_for_file, verify_data_upload_bucket_configuration, ) from fence.config import config From f5bec5927fac30b6b3e88de376f543d37f600477 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 14:05:52 -0600 Subject: [PATCH 23/35] fix class call --- fence/blueprints/data/blueprint.py | 1 + fence/blueprints/data/indexd.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index acd0db8a2..0e531218b 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -11,6 +11,7 @@ BlankIndex, EmbeddingIndex, IndexedFile, + BulkIndexedFile, get_signed_url_for_file, bulk_get_signed_url_for_file, verify_data_upload_bucket_configuration, diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index a8da684f6..7a3ac834d 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -215,7 +215,7 @@ def bulk_get_signed_url_for_file( "additional_data": x_forwarded_headers, } - indexed_files = BulkIndexedFile(file_ids) + indexed_files = BulkIndexedFiles(file_ids) default_expires_in = config.get("MAX_PRESIGNED_URL_TTL", 3600) expires_in = get_valid_expiration_from_request( From 4af5368653426f7fb7f9e5faf778309ce8b50688 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 14:10:37 -0600 Subject: [PATCH 24/35] clean up imports --- fence/blueprints/data/blueprint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index 0e531218b..acd0db8a2 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -11,7 +11,6 @@ BlankIndex, EmbeddingIndex, IndexedFile, - BulkIndexedFile, get_signed_url_for_file, bulk_get_signed_url_for_file, verify_data_upload_bucket_configuration, From 7220fe51904f659ba4e00f7fe11faa1318531876 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 14:22:44 -0600 Subject: [PATCH 25/35] update function arguments --- fence/blueprints/data/indexd.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 7a3ac834d..fd68d1f58 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -229,7 +229,6 @@ def bulk_get_signed_url_for_file( expires_in, force_signed_url=force_signed_url, r_pays_project=r_pays_project, - bucket=bucket, ) return signed_urls From 14526018845a64425d1fcc4635c89ba8ac6fafbc Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 15:01:01 -0600 Subject: [PATCH 26/35] fix loop --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index fd68d1f58..fff580c60 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -715,7 +715,7 @@ def get_signed_urls( ): users_from_passports = users_from_passports or {} signed_urls = [] - for file_id in file_ids: + for file_id in self.file_ids: authorized_user = None file_authz = self.index_document.get(file_id).get("authz") From c34c07473d75758d35519e7ddf0d967fc6a8d9a0 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 15:13:04 -0600 Subject: [PATCH 27/35] actions of bulk to only support download --- fence/blueprints/data/indexd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index fff580c60..15f3c7dff 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -727,7 +727,7 @@ def get_signed_urls( } is_authorized, authorized_username = ( self.get_authorized_with_username( - action_to_permission[action], + action_to_permission["read-storage"], file_id, # keys are usernames usernames_from_passports=list(users_from_passports.keys()), @@ -736,7 +736,7 @@ def get_signed_urls( if not is_authorized: msg = ( f"Either you weren't authenticated successfully or you don't have " - f"{action_to_permission[action]} permission " + f"{action_to_permission["read-storage"]} permission " f"on authorization resource: {self.index_document['authz']}." ) logger.debug( From c132e406faf76cbff40c4ed0ee8e93d4bbc8b20c Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 15:13:43 -0600 Subject: [PATCH 28/35] actions of bulk to only support download --- fence/blueprints/data/indexd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 15f3c7dff..0e248a4d9 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -727,7 +727,7 @@ def get_signed_urls( } is_authorized, authorized_username = ( self.get_authorized_with_username( - action_to_permission["read-storage"], + action_to_permission["download"], file_id, # keys are usernames usernames_from_passports=list(users_from_passports.keys()), @@ -736,7 +736,7 @@ def get_signed_urls( if not is_authorized: msg = ( f"Either you weren't authenticated successfully or you don't have " - f"{action_to_permission["read-storage"]} permission " + f"{action_to_permission["download"]} permission " f"on authorization resource: {self.index_document['authz']}." ) logger.debug( From 8a548c53a60a6a7fa5d4d5b312b6ae1c47e699b9 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 15:22:37 -0600 Subject: [PATCH 29/35] fix escape quotes --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 0e248a4d9..c80c8c050 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -736,7 +736,7 @@ def get_signed_urls( if not is_authorized: msg = ( f"Either you weren't authenticated successfully or you don't have " - f"{action_to_permission["download"]} permission " + f"read-storage permission " f"on authorization resource: {self.index_document['authz']}." ) logger.debug( From f403b3650678468cc8832c9d963b43cedf6cf822 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 15:29:33 -0600 Subject: [PATCH 30/35] no actions for bulk downloads --- fence/blueprints/data/indexd.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index c80c8c050..93e466d2a 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -748,8 +748,6 @@ def get_signed_urls( ) authorized_user = users_from_passports.get(authorized_username) - if action is not None and action not in SUPPORTED_ACTIONS: - raise NotSupported("action {} is not supported".format(action)) signed_url_tuple = ( self._get_signed_urls( protocol, From 34d2afeacf3b02f08b1a4aeb0b3b1575e83849db Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 15:37:23 -0600 Subject: [PATCH 31/35] fix bulk actions --- fence/blueprints/data/indexd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 93e466d2a..2e0cb2a22 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -778,7 +778,7 @@ def _get_signed_urls( # no protocol specified, return first location as signed url try: return file_locations[0].get_signed_url( - action, + "download", expires_in, force_signed_url=force_signed_url, r_pays_project=r_pays_project, @@ -793,7 +793,7 @@ def _get_signed_urls( protocol == "http" and file_location.protocol == "https" ): return file_location.get_signed_url( - action, + "download", expires_in, force_signed_url=force_signed_url, r_pays_project=r_pays_project, From a352217f8d0826326231e1e1a021118b76cd307d Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 15:54:49 -0600 Subject: [PATCH 32/35] fix bulk download return --- fence/blueprints/data/blueprint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py index acd0db8a2..ad292ddf0 100755 --- a/fence/blueprints/data/blueprint.py +++ b/fence/blueprints/data/blueprint.py @@ -461,7 +461,7 @@ def download_bulk_files(): raise UserError("missing required argument `guids`") guids = params["guids"] - result = bulk_get_signed_url_for_file(guids) + results = bulk_get_signed_url_for_file(guids) return flask.jsonify(results) # results = {} From 21f44bdc85edc8d8e215268ba438d0a8af131949 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 15 Dec 2025 16:21:27 -0600 Subject: [PATCH 33/35] reformat bulk response --- fence/blueprints/data/indexd.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 2e0cb2a22..735650dbe 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -224,14 +224,14 @@ def bulk_get_signed_url_for_file( ) # prepare_presigned_url_audit_log(requested_protocol, indexed_file) - signed_urls = indexed_files.get_signed_urls( + signed_urls, users = indexed_files.get_signed_urls( requested_protocol, expires_in, force_signed_url=force_signed_url, r_pays_project=r_pays_project, ) - return signed_urls + return {"urls": signed_urls} def get_bucket_from_urls(urls, protocol): @@ -715,6 +715,7 @@ def get_signed_urls( ): users_from_passports = users_from_passports or {} signed_urls = [] + users = [] for file_id in self.file_ids: authorized_user = None file_authz = self.index_document.get(file_id).get("authz") @@ -748,7 +749,7 @@ def get_signed_urls( ) authorized_user = users_from_passports.get(authorized_username) - signed_url_tuple = ( + signed_url, user = ( self._get_signed_urls( protocol, file_id, @@ -759,8 +760,10 @@ def get_signed_urls( ), authorized_user, ) - signed_urls.append(signed_url_tuple) - return signed_urls + signed_urls.append(signed_url) + users.append(user) + + return signed_urls, users def _get_signed_urls( self, From 256f199eabacdd913fe6aee1b04c85770e03286b Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Jan 2026 15:22:03 -0600 Subject: [PATCH 34/35] change ems service name --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 735650dbe..5638525d5 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -349,7 +349,7 @@ def create_embedding_record(self): create a record in the embedding management service """ - embedding_management_service_url = f"http://gen3-embedding-management-service/vector/indexes/{self.model}/embeddings" + embedding_management_service_url = f"http://embedding-management-service/vector/indexes/{self.model}/embeddings" params = { "file_id": self.file_id, "vector": self.embedding, From 9eeb4b0bda5b4a35705b5867a56faa2c3b24df13 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 12 Jan 2026 15:14:11 -0600 Subject: [PATCH 35/35] change service networking for embedding management service --- fence/blueprints/data/indexd.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 5638525d5..92840621d 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1268,17 +1268,6 @@ def get_signed_url( ): return self.url - # def get_vector(self, action, expires_in): - # # need to return the vector from the embedding management service - # model, emsID = str(self.url).replace("vec://", "").split("/") - # emsUrl = f"gen3-embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" - # try: - # req = requests.get(emsUrl) - # return req.json() - # except Exception as e: - # print(f"error {e}") - # raise NotFound(f"No embedding found with id {emsID}") - class VecIndexdFileLocation(IndexedFileLocation): def get_signed_url( @@ -1291,7 +1280,7 @@ def get_signed_url( ): # need to return the vector from the embedding management service model, emsID = str(self.url).replace("vec://", "").split("/") - emsUrl = f"http://gen3-embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" + emsUrl = f"http://embedding-management-service/vector/indexes/{model}/embeddings/{emsID}" try: req = requests.get(emsUrl) return req.json()