From c66b36698117125a8c48ee69e45121e3105805c2 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Mon, 31 Jul 2023 11:20:06 +0000 Subject: [PATCH 1/4] Secret manager added in weather-dl. --- Configuration.md | 11 ++++---- setup.py | 1 + weather_dl/download_pipeline/parsers.py | 34 ++++++++++++++++++++----- weather_dl/setup.py | 1 + 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/Configuration.md b/Configuration.md index 13b4f9c..693b57b 100644 --- a/Configuration.md +++ b/Configuration.md @@ -198,16 +198,15 @@ target_template=gs://ecmwf-downloads/hres-single-level/{}.nc partition_keys= date [parameters.deepmind] -api_key=KKKKK1 -api_url=UUUUU1 +secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 [parameters.research] -api_key=KKKKK2 -api_url=UUUUU2 +secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 [parameters.cloud] -api_key=KKKKK3 -api_url=UUUUU3 +secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 ``` +Note: here, secret_key is the [secret-manager](https://cloud.google.com/secret-manager) key with value likes this: {"api_url": "URL", "api_key": "KEY"} + ## `selection` Section _Parameters used to select desired data_ diff --git a/setup.py b/setup.py index af79888..53af6e9 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ "google-cloud-spanner==1.19.3", "google-cloud-videointelligence==1.16.3", "google-cloud-vision==1.0.2", + "google-cloud-secret-manager==2.16.2", "apache-beam[gcp]==2.40.0", ] diff --git a/weather_dl/download_pipeline/parsers.py b/weather_dl/download_pipeline/parsers.py index 5ad978b..461bc7b 100644 --- a/weather_dl/download_pipeline/parsers.py +++ b/weather_dl/download_pipeline/parsers.py @@ -23,6 +23,7 @@ import typing as t import numpy as np from collections import OrderedDict +from google.cloud import secretmanager from urllib.parse import urlparse from .clients import CLIENTS @@ -460,6 +461,23 @@ def prepare_target_name(config: Config) -> str: return target +def get_secret(api_key: str) -> dict: + """Retrieve the secret value from the Google Cloud Secret Manager. + + Parameters: + api_key (str): The name or identifier of the secret in the Google + Cloud Secret Manager. + + Returns: + dict: A dictionary containing the retrieved secret data. + """ + client = secretmanager.SecretManagerServiceClient() + response = client.access_secret_version(request={"name": api_key}) + payload = response.payload.data.decode("UTF-8") + secret_dict = json.loads(payload) + return secret_dict + + def get_subsections(config: Config) -> t.List[t.Tuple[str, t.Dict]]: """Collect parameter subsections from main configuration. @@ -471,19 +489,21 @@ def get_subsections(config: Config) -> t.List[t.Tuple[str, t.Dict]]: For example: ``` [parameters.alice] - api_key=KKKKK1 - api_url=UUUUU1 + secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 [parameters.bob] - api_key=KKKKK2 - api_url=UUUUU2 + secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 [parameters.eve] - api_key=KKKKK3 - api_url=UUUUU3 + secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 ``` """ - return [(name, params) for name, params in config.kwargs.items() + config_licences = [(name, params) for name, params in config.kwargs.items() if isinstance(params, dict)] or [('default', {})] + actual_licences = [] + for d in config_licences: + actual_licences.append( (d[0], get_secret(d[1]['secret_key'])) if d[1] else d ) + return actual_licences + def all_equal(iterator): iterator = iter(iterator) diff --git a/weather_dl/setup.py b/weather_dl/setup.py index b96466f..a54f2a6 100644 --- a/weather_dl/setup.py +++ b/weather_dl/setup.py @@ -28,6 +28,7 @@ "google-cloud-spanner==1.19.3", "google-cloud-videointelligence==1.16.3", "google-cloud-vision==1.0.2", + "google-cloud-secret-manager==2.16.2", "apache-beam[gcp]==2.40.0", ] From 01dccab6348bf5049a2f014afa548a010d70c367 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Mon, 31 Jul 2023 12:23:34 +0000 Subject: [PATCH 2/4] Nit changes done. --- Configuration.md | 2 +- weather_dl/download_pipeline/parsers.py | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Configuration.md b/Configuration.md index 693b57b..b9a9799 100644 --- a/Configuration.md +++ b/Configuration.md @@ -205,7 +205,7 @@ secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 ``` -Note: here, secret_key is the [secret-manager](https://cloud.google.com/secret-manager) key with value likes this: {"api_url": "URL", "api_key": "KEY"} +Note: Here, secret_key is the [secret-manager](https://cloud.google.com/secret-manager) key with value like this: {"api_url": "URL", "api_key": "KEY"} ## `selection` Section diff --git a/weather_dl/download_pipeline/parsers.py b/weather_dl/download_pipeline/parsers.py index 461bc7b..a68d3f9 100644 --- a/weather_dl/download_pipeline/parsers.py +++ b/weather_dl/download_pipeline/parsers.py @@ -461,18 +461,18 @@ def prepare_target_name(config: Config) -> str: return target -def get_secret(api_key: str) -> dict: +def get_secret(secret_key: str) -> t.Dict: """Retrieve the secret value from the Google Cloud Secret Manager. Parameters: - api_key (str): The name or identifier of the secret in the Google + secret_key (str): The name or identifier of the secret in the Google Cloud Secret Manager. Returns: dict: A dictionary containing the retrieved secret data. """ client = secretmanager.SecretManagerServiceClient() - response = client.access_secret_version(request={"name": api_key}) + response = client.access_secret_version(request={"name": secret_key}) payload = response.payload.data.decode("UTF-8") secret_dict = json.loads(payload) return secret_dict @@ -496,14 +496,9 @@ def get_subsections(config: Config) -> t.List[t.Tuple[str, t.Dict]]: secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 ``` """ - config_licences = [(name, params) for name, params in config.kwargs.items() + return [(name, get_secret(params.get('secret_key'))) for name, params in config.kwargs.items() if isinstance(params, dict)] or [('default', {})] - actual_licences = [] - for d in config_licences: - actual_licences.append( (d[0], get_secret(d[1]['secret_key'])) if d[1] else d ) - return actual_licences - def all_equal(iterator): iterator = iter(iterator) From 926bf30882a0c6b3c9294631c8028434e6d2a9fc Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Mon, 31 Jul 2023 12:53:31 +0000 Subject: [PATCH 3/4] Test cases added. --- weather_dl/download_pipeline/parsers_test.py | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/weather_dl/download_pipeline/parsers_test.py b/weather_dl/download_pipeline/parsers_test.py index 2d5dd64..d74f5cd 100644 --- a/weather_dl/download_pipeline/parsers_test.py +++ b/weather_dl/download_pipeline/parsers_test.py @@ -13,11 +13,14 @@ # limitations under the License. import datetime import io +import json import unittest +from unittest.mock import patch, MagicMock from .manifest import MockManifest, Location from .parsers import ( date, + get_secret, parse_config, process_config, _number_of_replacements, @@ -538,6 +541,29 @@ def test_cfg_parses_parameter_subsections(self): }, }) + @patch("weather_dl.download_pipeline.parsers.secretmanager.SecretManagerServiceClient") + def test_get_secret_success(self, mock_secretmanager): + secret_data = { + "api_url": "https://example.com/api", + "api_key": "my_secret_api_key" + } + mock_response = MagicMock() + mock_response.payload.data.decode.return_value = json.dumps(secret_data) + mock_secretmanager.return_value.access_secret_version.return_value = ( + mock_response) + + api_key = "projects/my-project/secrets/my-secret/versions/latest" + result = get_secret(api_key) + self.assertEqual(result, secret_data) + + @patch("weather_dl.download_pipeline.parsers.secretmanager.SecretManagerServiceClient") + def test_get_secret_failure(self, mock_secretmanager): + mock_secretmanager.return_value.access_secret_version.side_effect = ( + Exception("Error retrieving secret")) + api_key = "projects/my-project/secrets/my-secret/versions/latest" + with self.assertRaises(Exception): + get_secret(api_key) + class HelpersTest(unittest.TestCase): From 7795b6664275ffa0c0c6db3a12769e2733868eae Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Mon, 31 Jul 2023 13:02:14 +0000 Subject: [PATCH 4/4] Nit changes done. --- Configuration.md | 2 +- weather_dl/download_pipeline/parsers_test.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Configuration.md b/Configuration.md index b9a9799..51a91b8 100644 --- a/Configuration.md +++ b/Configuration.md @@ -205,7 +205,7 @@ secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1 ``` -Note: Here, secret_key is the [secret-manager](https://cloud.google.com/secret-manager) key with value like this: {"api_url": "URL", "api_key": "KEY"} +Note: Here, secret_key is the resource value of [secret-manager](https://cloud.google.com/secret-manager) like: `"projects/my-project/secrets/my-secret/versions/latest"` and it's actual value which store there is looks like this dict: {"api_url": "URL", "api_key": "KEY"}. ## `selection` Section diff --git a/weather_dl/download_pipeline/parsers_test.py b/weather_dl/download_pipeline/parsers_test.py index d74f5cd..91c7b70 100644 --- a/weather_dl/download_pipeline/parsers_test.py +++ b/weather_dl/download_pipeline/parsers_test.py @@ -552,17 +552,17 @@ def test_get_secret_success(self, mock_secretmanager): mock_secretmanager.return_value.access_secret_version.return_value = ( mock_response) - api_key = "projects/my-project/secrets/my-secret/versions/latest" - result = get_secret(api_key) + secret_key = "projects/my-project/secrets/my-secret/versions/latest" + result = get_secret(secret_key) self.assertEqual(result, secret_data) @patch("weather_dl.download_pipeline.parsers.secretmanager.SecretManagerServiceClient") def test_get_secret_failure(self, mock_secretmanager): mock_secretmanager.return_value.access_secret_version.side_effect = ( Exception("Error retrieving secret")) - api_key = "projects/my-project/secrets/my-secret/versions/latest" + secret_key = "projects/my-project/secrets/my-secret/versions/latest" with self.assertRaises(Exception): - get_secret(api_key) + get_secret(secret_key) class HelpersTest(unittest.TestCase):