diff --git a/Configuration.md b/Configuration.md index 13b4f9c..51a91b8 100644 --- a/Configuration.md +++ b/Configuration.md @@ -198,16 +198,15 @@ target_template=gs://ecmwf-downloads/hres-single-level/{}.nc partition_keys= date [parameters.deepmind] -api_key=KKKKK1 -api_url=UUUUU1 +secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 [parameters.research] -api_key=KKKKK2 -api_url=UUUUU2 +secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 [parameters.cloud] -api_key=KKKKK3 -api_url=UUUUU3 +secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 ``` +Note: Here, secret_key is the resource value of [secret-manager](https://cloud.google.com/secret-manager) like: `"projects/my-project/secrets/my-secret/versions/latest"` and it's actual value which store there is looks like this dict: {"api_url": "URL", "api_key": "KEY"}. + ## `selection` Section _Parameters used to select desired data_ diff --git a/setup.py b/setup.py index af79888..53af6e9 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ "google-cloud-spanner==1.19.3", "google-cloud-videointelligence==1.16.3", "google-cloud-vision==1.0.2", + "google-cloud-secret-manager==2.16.2", "apache-beam[gcp]==2.40.0", ] diff --git a/weather_dl/download_pipeline/parsers.py b/weather_dl/download_pipeline/parsers.py index 5ad978b..a68d3f9 100644 --- a/weather_dl/download_pipeline/parsers.py +++ b/weather_dl/download_pipeline/parsers.py @@ -23,6 +23,7 @@ import typing as t import numpy as np from collections import OrderedDict +from google.cloud import secretmanager from urllib.parse import urlparse from .clients import CLIENTS @@ -460,6 +461,23 @@ def prepare_target_name(config: Config) -> str: return target +def get_secret(secret_key: str) -> t.Dict: + """Retrieve the secret value from the Google Cloud Secret Manager. + + Parameters: + secret_key (str): The name or identifier of the secret in the Google + Cloud Secret Manager. + + Returns: + dict: A dictionary containing the retrieved secret data. + """ + client = secretmanager.SecretManagerServiceClient() + response = client.access_secret_version(request={"name": secret_key}) + payload = response.payload.data.decode("UTF-8") + secret_dict = json.loads(payload) + return secret_dict + + def get_subsections(config: Config) -> t.List[t.Tuple[str, t.Dict]]: """Collect parameter subsections from main configuration. @@ -471,17 +489,14 @@ def get_subsections(config: Config) -> t.List[t.Tuple[str, t.Dict]]: For example: ``` [parameters.alice] - api_key=KKKKK1 - api_url=UUUUU1 + secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 [parameters.bob] - api_key=KKKKK2 - api_url=UUUUU2 + secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 [parameters.eve] - api_key=KKKKK3 - api_url=UUUUU3 + secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 ``` """ - return [(name, params) for name, params in config.kwargs.items() + return [(name, get_secret(params.get('secret_key'))) for name, params in config.kwargs.items() if isinstance(params, dict)] or [('default', {})] diff --git a/weather_dl/download_pipeline/parsers_test.py b/weather_dl/download_pipeline/parsers_test.py index 2d5dd64..91c7b70 100644 --- a/weather_dl/download_pipeline/parsers_test.py +++ b/weather_dl/download_pipeline/parsers_test.py @@ -13,11 +13,14 @@ # limitations under the License. import datetime import io +import json import unittest +from unittest.mock import patch, MagicMock from .manifest import MockManifest, Location from .parsers import ( date, + get_secret, parse_config, process_config, _number_of_replacements, @@ -538,6 +541,29 @@ def test_cfg_parses_parameter_subsections(self): }, }) + @patch("weather_dl.download_pipeline.parsers.secretmanager.SecretManagerServiceClient") + def test_get_secret_success(self, mock_secretmanager): + secret_data = { + "api_url": "https://example.com/api", + "api_key": "my_secret_api_key" + } + mock_response = MagicMock() + mock_response.payload.data.decode.return_value = json.dumps(secret_data) + mock_secretmanager.return_value.access_secret_version.return_value = ( + mock_response) + + secret_key = "projects/my-project/secrets/my-secret/versions/latest" + result = get_secret(secret_key) + self.assertEqual(result, secret_data) + + @patch("weather_dl.download_pipeline.parsers.secretmanager.SecretManagerServiceClient") + def test_get_secret_failure(self, mock_secretmanager): + mock_secretmanager.return_value.access_secret_version.side_effect = ( + Exception("Error retrieving secret")) + secret_key = "projects/my-project/secrets/my-secret/versions/latest" + with self.assertRaises(Exception): + get_secret(secret_key) + class HelpersTest(unittest.TestCase): diff --git a/weather_dl/setup.py b/weather_dl/setup.py index b96466f..a54f2a6 100644 --- a/weather_dl/setup.py +++ b/weather_dl/setup.py @@ -28,6 +28,7 @@ "google-cloud-spanner==1.19.3", "google-cloud-videointelligence==1.16.3", "google-cloud-vision==1.0.2", + "google-cloud-secret-manager==2.16.2", "apache-beam[gcp]==2.40.0", ]