diff --git a/README.md b/README.md index 15d3e2a..056d774 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,61 @@ GET /api/resources/get-dependent-workloads?id=x86-ubuntu-18.04-img ] ``` +### 5. List All Resources by gem5 Version + +**Endpoint**: `GET /api/resources/list-all-resources` + +Retrieve all resources that are compatible with a specific gem5 version. The endpoint performs prefix matching, so a version like `25.0.0.1` will match resources that have `25.0` or `25.0.0` in their `gem5_versions` field. + +**Parameters**: + +- `gem5-version` (required): The gem5 version to match against (e.g., "23.0", "25.0.0.1") +- `latest-version` (optional): If set to `"true"`, returns only the latest compatible version of each resource instead of all compatible versions. Default is `"false"` (returns all versions). + +**Examples**: + +```bash +# Get all resources compatible with gem5 version 23.0 +GET /api/resources/list-all-resources?gem5-version=23.0 + +# Get all resources compatible with gem5 version 25.0.0.1 +# This will match resources with gem5_versions containing "25", "25.0", "25.0.0", or "25.0.0.1" +GET /api/resources/list-all-resources?gem5-version=25.0.0.1 + +# Get only the latest version of each resource compatible with gem5 version 23.0 +GET /api/resources/list-all-resources?gem5-version=23.0&latest-version=true +``` + +**Response Format**: + +```json +[ + { + "id": "riscv-ubuntu-20.04-boot", + "resource_version": "3.0.0", + "category": "workload", + "architecture": "RISCV", + "gem5_versions": ["23.0", "22.1", "22.0"], + // ... other resource fields + }, + { + "id": "arm-hello64-static", + "resource_version": "1.0.0", + "category": "binary", + "architecture": "ARM", + "gem5_versions": ["23.0", "22.0"], + // ... other resource fields + } +] +``` + +**Notes**: + +- Uses prefix matching: a parameter like `25.0.0.1` matches resources with `25.0`, `25.0.0`, or `25.0.0.1` in their `gem5_versions` field +- Version parameter must have at least `major` version (e.g., `23.0`), gem5 major versions follow the format `release-year.release_num` (eg, "25.1" would be the second release of year 2025). +- Returns all versions of resources that match +- Returns an empty list if no resources match the specified gem5 version + ## Development Setup ### Prerequisites @@ -216,7 +271,8 @@ gem5-resources-api/ │ ├── get_resources_by_batch.py │ ├── search_resources.py │ ├── get_filters.py -│ └── get_dependent_workloads.py +│ ├── get_dependent_workloads.py +│ └── list_all_resources.py ├── shared/ # Shared utilities │ ├── database.py # Database connection & config │ └── utils.py # Common utilities & validation @@ -244,6 +300,7 @@ Functions: search_resources: [GET] http://localhost:7071/api/resources/search get_filters: [GET] http://localhost:7071/api/resources/filters get_dependent_workloads: [GET] http://localhost:7071/api/resources/get-dependent-workloads + list_all_resources: [GET] http://localhost:7071/api/resources/list-all-resources ``` ### Testing diff --git a/function_app.py b/function_app.py index 3ff8459..681cada 100644 --- a/function_app.py +++ b/function_app.py @@ -8,6 +8,7 @@ get_dependent_workloads, get_filters, get_resources_by_batch, + list_all_resources, search_resources, ) from shared.azure_search_client import get_search_client @@ -27,3 +28,4 @@ search_resources.register_function(app, search_client) get_filters.register_function(app, collection, db["filter_values"]) get_dependent_workloads.register_function(app, collection) +list_all_resources.register_function(app, collection) diff --git a/functions/list_all_resources.py b/functions/list_all_resources.py new file mode 100644 index 0000000..dee2b90 --- /dev/null +++ b/functions/list_all_resources.py @@ -0,0 +1,96 @@ +# Copyright (c) 2025 The Regents of the University of California +# SPDX-License-Identifier: BSD-3-Clause + +import json +import logging + +import azure.functions as func + +from shared.database import RESOURCE_FIELDS +from shared.utils import ( + create_error_response, + get_latest_versions, + sanitize_version, +) + + +def register_function(app, collection): + """Register the function with the app.""" + + @app.function_name(name="list_all_resources") + @app.route( + route="resources/list-all-resources", + auth_level=func.AuthLevel.ANONYMOUS, + ) + def list_all_resources(req: func.HttpRequest) -> func.HttpResponse: + """ + Get all resources where a gem5 version in the resource is a prefix + of the specified version parameter. + + Route: /resources/list-all-resources + + Query Parameters: + - gem5-version: Required, the gem5 version to match against + (e.g., "25.0.0.1" will match resources with "25.0") + - latest-version: Optional, if set to "true", returns only the latest + compatible version of each resource instead of all + compatible versions. Default is false (return all). + """ + logging.info( + "Processing request to list all resources by gem5 version" + ) + try: + gem5_version = req.params.get("gem5-version") + latest_version_only = ( + req.params.get("latest-version", "false").lower() == "true" + ) + + if not gem5_version: + return create_error_response( + 400, "'gem5-version' parameter is required" + ) + + gem5_version = sanitize_version(gem5_version) + + if not gem5_version: + return create_error_response( + 400, "Invalid 'gem5-version' parameter format" + ) + + # Build a list of all possible prefixes from the version + # Starting from major.minor since gem5 versions always have at + # least one dot (e.g., "25.0", "23.1") + # e.g., "25.0.0.1" -> ["25.0", "25.0.0", "25.0.0.1"] + version_parts = gem5_version.split(".") + prefixes = [] + for i in range(2, len(version_parts) + 1): + prefixes.append(".".join(version_parts[:i])) + + # If only one part provided (e.g., "25"), return error + if not prefixes: + return create_error_response( + 400, + "Invalid 'gem5-version' parameter: must have at least " + "major version format (e.g., '23.0', '25.1')", + ) + + # Query for resources where any gem5_version matches one of + # the prefixes + query = {"gem5_versions": {"$in": prefixes}} + + resources = list(collection.find(query, RESOURCE_FIELDS)) + + # If latest-version is set, return only the latest version of + # each resource + if latest_version_only: + resources = get_latest_versions(resources) + + return func.HttpResponse( + body=json.dumps(resources), + status_code=200, + headers={"Content-Type": "application/json"}, + ) + + except Exception as e: + logging.error(f"Error listing resources by gem5 version: {str(e)}") + return create_error_response(500, "Internal server error") diff --git a/shared/utils.py b/shared/utils.py index b0dbecd..5edb269 100644 --- a/shared/utils.py +++ b/shared/utils.py @@ -64,3 +64,37 @@ def create_json_response(data, status_code=200): headers={"Content-Type": "application/json"}, status_code=status_code, ) + + +def parse_version(version_str): + """Parse a version string into a tuple of integers for comparison.""" + try: + return tuple(int(x) for x in version_str.split(".")) + except (ValueError, AttributeError): + return (0,) + + +def get_latest_versions(resources): + """ + Filter resources to return only the latest version of each resource. + Groups resources by 'id' and returns the one with the highest + 'resource_version'. + """ + latest_by_id = {} + for resource in resources: + resource_id = resource.get("id") + if not resource_id: + continue + + current_version = parse_version(resource.get("resource_version", "0")) + + if resource_id not in latest_by_id: + latest_by_id[resource_id] = resource + else: + existing_version = parse_version( + latest_by_id[resource_id].get("resource_version", "0") + ) + if current_version > existing_version: + latest_by_id[resource_id] = resource + + return list(latest_by_id.values()) diff --git a/tests/resources_api_unit_tests.py b/tests/resources_api_unit_tests.py index be44fd5..00bae03 100644 --- a/tests/resources_api_unit_tests.py +++ b/tests/resources_api_unit_tests.py @@ -568,6 +568,217 @@ def test_search_combined_filters_sort_pagination(self): ids = [r["id"].lower() for r in resources] self.assertEqual(ids, sorted(ids)) + def test_list_all_resources_valid_gem5_version(self): + """Test listing all resources with a valid gem5 version.""" + response = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0"}, + ) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertIsInstance(data, list) + + # Verify all returned resources have a gem5 version that is a prefix + # of or equal to the requested version + for resource in data: + self.assertIn("gem5_versions", resource) + # Should have "23.0" in gem5_versions + self.assertIn("23.0", resource["gem5_versions"]) + + def test_list_all_resources_missing_gem5_version(self): + """Test listing resources without gem5-version parameter.""" + response = requests.get( + f"{self.base_url}/resources/list-all-resources" + ) + self.assertEqual(response.status_code, 400) + data = response.json() + self.assertIn("error", data) + self.assertIn("gem5-version", data["error"]) + + def test_list_all_resources_invalid_gem5_version(self): + """Test listing resources with invalid gem5-version format.""" + response = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "invalid-version!@#"}, + ) + self.assertEqual(response.status_code, 400) + data = response.json() + self.assertIn("error", data) + + def test_list_all_resources_nonexistent_gem5_version(self): + """Test listing resources with a gem5 version that doesn't exist.""" + response = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "99.99.99"}, + ) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertIsInstance(data, list) + self.assertEqual(len(data), 0) + + def test_list_all_resources_returns_multiple_versions(self): + """Test that list-all-resources returns all versions of resources.""" + response = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0"}, + ) + self.assertEqual(response.status_code, 200) + data = response.json() + + # Check if any resource ID appears multiple times (different versions) + ids = [r["id"] for r in data] + # Just verify it returns valid resources + self.assertIsInstance(data, list) + + def test_list_all_resources_different_versions(self): + """Test listing resources with different gem5 versions returns + different results.""" + response_v23 = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0"}, + ) + response_v22 = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "22.0"}, + ) + + self.assertEqual(response_v23.status_code, 200) + self.assertEqual(response_v22.status_code, 200) + + data_v23 = response_v23.json() + data_v22 = response_v22.json() + + # Verify correct gem5 version prefix in each response + for resource in data_v23: + self.assertIn("23.0", resource["gem5_versions"]) + for resource in data_v22: + self.assertIn("22.0", resource["gem5_versions"]) + + def test_list_all_resources_prefix_matching(self): + """Test that a longer version like 23.0.0.1 matches resources + with shorter prefixes like 23.0.""" + # First get resources with 23.0 + response_short = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0"}, + ) + # Then get resources with 23.0.0.1 (should match same resources) + response_long = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0.0.1"}, + ) + + self.assertEqual(response_short.status_code, 200) + self.assertEqual(response_long.status_code, 200) + + data_short = response_short.json() + data_long = response_long.json() + + # Resources from 23.0.0.1 should include all resources from 23.0 + # (since 23.0 is a prefix of 23.0.0.1) + short_ids = {(r["id"], r["resource_version"]) for r in data_short} + long_ids = {(r["id"], r["resource_version"]) for r in data_long} + self.assertTrue(short_ids.issubset(long_ids)) + + def test_list_all_resources_latest_version_only(self): + """Test that latest-version=true returns only one version per + resource.""" + response = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0", "latest-version": "true"}, + ) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertIsInstance(data, list) + + # Check no duplicate IDs (each resource appears only once) + ids = [r["id"] for r in data] + self.assertEqual(len(ids), len(set(ids))) + + def test_list_all_resources_latest_version_false(self): + """Test that latest-version=false returns all versions (default + behavior).""" + response_default = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0"}, + ) + response_false = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0", "latest-version": "false"}, + ) + + self.assertEqual(response_default.status_code, 200) + self.assertEqual(response_false.status_code, 200) + + data_default = response_default.json() + data_false = response_false.json() + + # Both should return the same results + self.assertEqual(len(data_default), len(data_false)) + + def test_list_all_resources_latest_version_reduces_results(self): + """Test that latest-version=true returns fewer or equal resources + than default.""" + response_all = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0"}, + ) + response_latest = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0", "latest-version": "true"}, + ) + + self.assertEqual(response_all.status_code, 200) + self.assertEqual(response_latest.status_code, 200) + + data_all = response_all.json() + data_latest = response_latest.json() + + # Latest should have <= resources than all versions + self.assertLessEqual(len(data_latest), len(data_all)) + + # All IDs in latest should exist in all versions + latest_ids = {r["id"] for r in data_latest} + all_ids = {r["id"] for r in data_all} + self.assertTrue(latest_ids.issubset(all_ids)) + + def test_list_all_resources_latest_version_returns_highest(self): + """Test that latest-version=true returns the highest version of each + resource.""" + response_all = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0"}, + ) + response_latest = requests.get( + f"{self.base_url}/resources/list-all-resources", + params={"gem5-version": "23.0", "latest-version": "true"}, + ) + + self.assertEqual(response_all.status_code, 200) + self.assertEqual(response_latest.status_code, 200) + + data_all = response_all.json() + data_latest = response_latest.json() + + # Build a dict of max versions from all resources + max_versions = {} + for r in data_all: + rid = r["id"] + ver = r.get("resource_version", "0") + if rid not in max_versions: + max_versions[rid] = ver + else: + # Compare versions + current = tuple(int(x) for x in ver.split(".")) + existing = tuple(int(x) for x in max_versions[rid].split(".")) + if current > existing: + max_versions[rid] = ver + + # Verify latest returns the max version for each resource + for r in data_latest: + rid = r["id"] + self.assertEqual(r["resource_version"], max_versions[rid]) + if __name__ == "__main__": unittest.main()