Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 54 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,57 @@ GET /api/resources/get-dependent-workloads?id=x86-ubuntu-18.04-img
]
```

### 5. List All Resources by gem5 Version

**Endpoint**: `GET /api/resources/list-all-resources`

Retrieve all resources that are compatible with a specific gem5 version. The endpoint performs prefix matching, so a version like `25.0.0.1` will match resources that have `25.0` or `25.0.0` in their `gem5_versions` field.

**Parameters**:

- `gem5-version` (required): The gem5 version to match against (e.g., "23.0", "25.0.0.1")

**Examples**:

```bash
# Get all resources compatible with gem5 version 23.0
GET /api/resources/list-all-resources?gem5-version=23.0

# Get all resources compatible with gem5 version 25.0.0.1
# This will match resources with gem5_versions containing "25", "25.0", "25.0.0", or "25.0.0.1"
GET /api/resources/list-all-resources?gem5-version=25.0.0.1
```

**Response Format**:

```json
[
{
"id": "riscv-ubuntu-20.04-boot",
"resource_version": "3.0.0",
"category": "workload",
"architecture": "RISCV",
"gem5_versions": ["23.0", "22.1", "22.0"],
// ... other resource fields
},
{
"id": "arm-hello64-static",
"resource_version": "1.0.0",
"category": "binary",
"architecture": "ARM",
"gem5_versions": ["23.0", "22.0"],
// ... other resource fields
}
]
```

**Notes**:

- Uses prefix matching: a parameter like `25.0.0.1` matches resources with `25.0`, `25.0.0`, or `25.0.0.1` in their `gem5_versions` field
- Version parameter must have at least `major.minor` format (e.g., `23.0`, not just `23`)
- Returns all versions of resources that match
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it? Is that what we want? Shouldn't this return the more current compatible version of each resource. E.g., if Resource A version 1 and Resource A version 2 both state compatibility with the specified gem5 version then this function would return Resource A version 2 only. Version 1 has been replaced with something, presumably, better. I can't see a case you'd need an older version over the most current.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, currently we return all versions.
I feel it is better to return every version and then post process to the latest one.
I like the idea of adding an optional parameter that will return only the latest compatible version.
By default, we return all compatible version.

- Returns an empty list if no resources match the specified gem5 version

## Development Setup

### Prerequisites
Expand Down Expand Up @@ -216,7 +267,8 @@ gem5-resources-api/
│ ├── get_resources_by_batch.py
│ ├── search_resources.py
│ ├── get_filters.py
│ └── get_dependent_workloads.py
│ ├── get_dependent_workloads.py
│ └── list_all_resources.py
├── shared/ # Shared utilities
│ ├── database.py # Database connection & config
│ └── utils.py # Common utilities & validation
Expand Down Expand Up @@ -244,6 +296,7 @@ Functions:
search_resources: [GET] http://localhost:7071/api/resources/search
get_filters: [GET] http://localhost:7071/api/resources/filters
get_dependent_workloads: [GET] http://localhost:7071/api/resources/get-dependent-workloads
list_all_resources: [GET] http://localhost:7071/api/resources/list-all-resources
```

### Testing
Expand Down
2 changes: 2 additions & 0 deletions function_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
get_dependent_workloads,
get_filters,
get_resources_by_batch,
list_all_resources,
search_resources,
)
from shared.azure_search_client import get_search_client
Expand All @@ -27,3 +28,4 @@
search_resources.register_function(app, search_client)
get_filters.register_function(app, collection, db["filter_values"])
get_dependent_workloads.register_function(app, collection)
list_all_resources.register_function(app, collection)
84 changes: 84 additions & 0 deletions functions/list_all_resources.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
# Copyright (c) 2025 The Regents of the University of California
# SPDX-License-Identifier: BSD-3-Clause

import json
import logging

import azure.functions as func

from shared.database import RESOURCE_FIELDS
from shared.utils import (
create_error_response,
sanitize_version,
)


def register_function(app, collection):
"""Register the function with the app."""

@app.function_name(name="list_all_resources")
@app.route(
route="resources/list-all-resources",
auth_level=func.AuthLevel.ANONYMOUS,
)
def list_all_resources(req: func.HttpRequest) -> func.HttpResponse:
"""
Get all resources where a gem5 version in the resource is a prefix
of the specified version parameter.
Route: /resources/list-all-resources
Query Parameters:
- gem5-version: Required, the gem5 version to match against
(e.g., "25.0.0.1" will match resources with "25.0")
"""
logging.info(
"Processing request to list all resources by gem5 version"
)
try:
gem5_version = req.params.get("gem5-version")

if not gem5_version:
return create_error_response(
400, "'gem5-version' parameter is required"
)

gem5_version = sanitize_version(gem5_version)

if not gem5_version:
return create_error_response(
400, "Invalid 'gem5-version' parameter format"
)

# Build a list of all possible prefixes from the version
# Starting from major.minor since gem5 versions always have at
# least one dot (e.g., "25.0", "23.1")
# e.g., "25.0.0.1" -> ["25.0", "25.0.0", "25.0.0.1"]
version_parts = gem5_version.split(".")
prefixes = []
for i in range(2, len(version_parts) + 1):
prefixes.append(".".join(version_parts[:i]))
Comment on lines +60 to +67
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we have any real reason to search for compatibility with minor and hotfix values? It kind of makes things more complex. Are there any resources that express compatibility beyond the gem5 major versions? This is more me trying to understand. I assume we do this same procedure elsewhere and it'd be too annoying to change at this stage.

Just to check my assumptions on how this works: What would it mean for a resource of a particular version to have a stated compatibility of say, 'v24.0.1', with a gem5 version of 'v24.0.2'? I take it this gem5 resource would be seen as incompatible? Also, am I correct in assuming when a resource has one stated compatibility at the granularity of the a major release, and another stated compatibility at the granularity of am minor/hotfix release of that same major (.e.g., ['v24.1' 'v24.1.1'], or ['v24.0', 'v24.0.0.1']), then the major/hotfix compatibility is effectively redundant as the major version compatibility assumes compatibility with all releases within that major release (e.g., ['v24.1' 'v24.1.1'] is equal to ['v24.1'] in practice).

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We dont currently use the functionality but the get_resources function in gem5 does this for post processing so I am doing this to keep the behavior consistent. I also chose this approach and the gem5_version that is passed by default is always the full version, and I feel it is better to search in the current way than truncating the version to the major release as this approach will limit us in case we want this flexibility in the future.

For your second question, if a particular version to have a stated compatibility of say, 'v24.0.1', then the resource would be incompatible with gem5 version of 'v24.0.2', this would potentially be helpful in case we need to update checkpoint in hot fixes, etc.

Yes, if a resource has one stated compatibility at the granularity of the a major release, and another stated compatibility at the granularity of am minor/hotfix release of that same major (.e.g., ['v24.1' 'v24.1.1'], or ['v24.0', 'v24.0.0.1']), then the major/hotfix compatibility is effectively redundant as the major version compatibility assumes compatibility with all releases within that major release (e.g., ['v24.1' 'v24.1.1'] is equal to ['v24.1'] in practice).


# If only one part provided (e.g., "25"), return error
if not prefixes:
return create_error_response(
400,
"Invalid 'gem5-version' parameter: must have at least "
"major.minor format (e.g., '23.0')",
)

# Query for resources where any gem5_version matches one of
# the prefixes
query = {"gem5_versions": {"$in": prefixes}}

resources = list(collection.find(query, RESOURCE_FIELDS))

return func.HttpResponse(
body=json.dumps(resources),
status_code=200,
headers={"Content-Type": "application/json"},
)

except Exception as e:
logging.error(f"Error listing resources by gem5 version: {str(e)}")
return create_error_response(500, "Internal server error")
112 changes: 112 additions & 0 deletions tests/resources_api_unit_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,118 @@ def test_search_combined_filters_sort_pagination(self):
ids = [r["id"].lower() for r in resources]
self.assertEqual(ids, sorted(ids))

def test_list_all_resources_valid_gem5_version(self):
"""Test listing all resources with a valid gem5 version."""
response = requests.get(
f"{self.base_url}/resources/list-all-resources",
params={"gem5-version": "23.0"},
)
self.assertEqual(response.status_code, 200)
data = response.json()
self.assertIsInstance(data, list)

# Verify all returned resources have a gem5 version that is a prefix
# of or equal to the requested version
for resource in data:
self.assertIn("gem5_versions", resource)
# Should have "23.0" in gem5_versions
self.assertIn("23.0", resource["gem5_versions"])

def test_list_all_resources_missing_gem5_version(self):
"""Test listing resources without gem5-version parameter."""
response = requests.get(
f"{self.base_url}/resources/list-all-resources"
)
self.assertEqual(response.status_code, 400)
data = response.json()
self.assertIn("error", data)
self.assertIn("gem5-version", data["error"])

def test_list_all_resources_invalid_gem5_version(self):
"""Test listing resources with invalid gem5-version format."""
response = requests.get(
f"{self.base_url}/resources/list-all-resources",
params={"gem5-version": "invalid-version!@#"},
)
self.assertEqual(response.status_code, 400)
data = response.json()
self.assertIn("error", data)

def test_list_all_resources_nonexistent_gem5_version(self):
"""Test listing resources with a gem5 version that doesn't exist."""
response = requests.get(
f"{self.base_url}/resources/list-all-resources",
params={"gem5-version": "99.99.99"},
)
self.assertEqual(response.status_code, 200)
data = response.json()
self.assertIsInstance(data, list)
self.assertEqual(len(data), 0)

def test_list_all_resources_returns_multiple_versions(self):
"""Test that list-all-resources returns all versions of resources."""
response = requests.get(
f"{self.base_url}/resources/list-all-resources",
params={"gem5-version": "23.0"},
)
self.assertEqual(response.status_code, 200)
data = response.json()

# Check if any resource ID appears multiple times (different versions)
ids = [r["id"] for r in data]
# Just verify it returns valid resources
self.assertIsInstance(data, list)

def test_list_all_resources_different_versions(self):
"""Test listing resources with different gem5 versions returns
different results."""
response_v23 = requests.get(
f"{self.base_url}/resources/list-all-resources",
params={"gem5-version": "23.0"},
)
response_v22 = requests.get(
f"{self.base_url}/resources/list-all-resources",
params={"gem5-version": "22.0"},
)

self.assertEqual(response_v23.status_code, 200)
self.assertEqual(response_v22.status_code, 200)

data_v23 = response_v23.json()
data_v22 = response_v22.json()

# Verify correct gem5 version prefix in each response
for resource in data_v23:
self.assertIn("23.0", resource["gem5_versions"])
for resource in data_v22:
self.assertIn("22.0", resource["gem5_versions"])

def test_list_all_resources_prefix_matching(self):
"""Test that a longer version like 23.0.0.1 matches resources
with shorter prefixes like 23.0."""
# First get resources with 23.0
response_short = requests.get(
f"{self.base_url}/resources/list-all-resources",
params={"gem5-version": "23.0"},
)
# Then get resources with 23.0.0.1 (should match same resources)
response_long = requests.get(
f"{self.base_url}/resources/list-all-resources",
params={"gem5-version": "23.0.0.1"},
)

self.assertEqual(response_short.status_code, 200)
self.assertEqual(response_long.status_code, 200)

data_short = response_short.json()
data_long = response_long.json()

# Resources from 23.0.0.1 should include all resources from 23.0
# (since 23.0 is a prefix of 23.0.0.1)
short_ids = {(r["id"], r["resource_version"]) for r in data_short}
long_ids = {(r["id"], r["resource_version"]) for r in data_long}
self.assertTrue(short_ids.issubset(long_ids))


if __name__ == "__main__":
unittest.main()
Loading