From 8a0bdfa25566d017c1a880d3a0ee26e84eaf97ff Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 31 Jul 2026 10:58:47 -0700 Subject: [PATCH] Make version generation not overwrite anything --- tests/BUILD.bazel | 6 ++ tests/generate_versions_test.py | 182 ++++++++++++++++++++++++++++++++ tools/BUILD.bazel | 10 ++ tools/generate_ndk_versions.py | 42 ++++++-- tools/generate_sdk_versions.py | 34 +++++- 5 files changed, 262 insertions(+), 12 deletions(-) create mode 100644 tests/generate_versions_test.py create mode 100644 tools/BUILD.bazel diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index 14665d6..98704fb 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -12,6 +12,12 @@ py_test( tags = ["manual"], ) +py_test( + name = "generate_versions_test", + srcs = ["generate_versions_test.py"], + deps = ["//tools:generate_versions"], +) + action_command_line_test( name = "cc_compile_uses_hermetic_ndk_test", expected_argv = [ diff --git a/tests/generate_versions_test.py b/tests/generate_versions_test.py new file mode 100644 index 0000000..3aefdff --- /dev/null +++ b/tests/generate_versions_test.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Tests that version metadata updates are additive.""" + +import copy +import unittest +from unittest import mock + +from tools import generate_ndk_versions +from tools import generate_sdk_versions + + +def _archives(prefix, *, sha1=None): + return [ + { + "file": "{}-{}.zip".format(prefix, platform), + "platform": platform, + "sha1": sha1, + "url": "https://example.com/{}-{}.zip".format(prefix, platform), + } + for platform in ("darwin", "linux", "windows") + ] + + +class GenerateSdkVersionsTest(unittest.TestCase): + def test_generate_preserves_existing_values_and_adds_new_values(self): + existing = { + "components": { + "build_tools": {"38.0.0": {"existing": "build-tools"}}, + "emulator": {"2.0.0": {"existing": "emulator"}}, + "platform_tools": {"1.0.0": {"existing": "platform-tools"}}, + "system_images": {"old/image": {"existing": "system-image"}}, + }, + "versions": { + "24": { + "platform_tools_version": "1.0.0", + "platform": {"existing": "platform"}, + } + }, + } + repo_packages = [ + { + "path": "platforms;android-24", + "revision": (24, 0, 1, 0), + "archives": [ + { + "file": "changed-platform-24.zip", + "platform": None, + "sha1": None, + "url": "https://example.com/changed-platform-24.zip", + } + ], + }, + { + "path": "platforms;android-25", + "revision": (25, 0, 0, 0), + "archives": [ + { + "file": "platform-25.zip", + "platform": None, + "sha1": None, + "url": "https://example.com/platform-25.zip", + } + ], + }, + { + "path": "build-tools;38.0.0", + "revision": (38, 0, 0, 0), + "archives": _archives("changed-build-tools"), + }, + { + "path": "platform-tools", + "revision": (2, 0, 0, 0), + "archives": _archives("platform-tools-2"), + }, + { + "path": "emulator", + "revision": (2, 0, 0, 0), + "archives": _archives("changed-emulator-2"), + }, + { + "path": "emulator", + "revision": (3, 0, 0, 0), + "archives": _archives("emulator-3"), + }, + ] + + def archive_json(archive, _metadata, infer_prefix): + result = {"file": archive["file"], "sha256": "new"} + if infer_prefix: + result["strip_prefix"] = "new" + return result + + with ( + mock.patch.object( + generate_sdk_versions, + "_load_existing", + return_value=({}, copy.deepcopy(existing)), + ), + mock.patch.object(generate_sdk_versions, "_fetch_xml", return_value=None), + mock.patch.object( + generate_sdk_versions, + "_parse_packages", + side_effect=[repo_packages, []], + ), + mock.patch.object( + generate_sdk_versions, + "_archive_json", + side_effect=archive_json, + ), + ): + actual = generate_sdk_versions._generate() + + self.assertEqual( + actual["components"]["build_tools"]["38.0.0"], + existing["components"]["build_tools"]["38.0.0"], + ) + self.assertEqual( + actual["components"]["emulator"]["2.0.0"], + existing["components"]["emulator"]["2.0.0"], + ) + self.assertEqual( + actual["components"]["platform_tools"]["1.0.0"], + existing["components"]["platform_tools"]["1.0.0"], + ) + self.assertEqual( + actual["components"]["system_images"], + existing["components"]["system_images"], + ) + self.assertEqual(actual["versions"]["24"], existing["versions"]["24"]) + self.assertIn("3.0.0", actual["components"]["emulator"]) + self.assertIn("2.0.0", actual["components"]["platform_tools"]) + self.assertIn("25", actual["versions"]) + + +class GenerateNdkVersionsTest(unittest.TestCase): + def test_generate_preserves_retired_versions_and_changed_aliases(self): + existing = { + "aliases": {"r25": "25.0.1"}, + "versions": { + "25.0.1": { + "strip_prefix": "existing", + "archives": {"linux": {"existing": "archive"}}, + } + }, + } + sha1 = "00" * 20 + manifest_versions = { + "25.0.9": { + "alias": "r25", + "archives": _archives("changed-r25", sha1=sha1), + }, + "26.0.2": { + "alias": "r26", + "archives": _archives("r26", sha1=sha1), + }, + } + aliases = {"r25": "25.0.9", "r26": "26.0.2"} + + with ( + mock.patch.object( + generate_ndk_versions, + "_load_existing", + return_value=copy.deepcopy(existing), + ), + mock.patch.object(generate_ndk_versions, "_fetch_xml", return_value=None), + mock.patch.object( + generate_ndk_versions, + "_parse_packages", + return_value=(manifest_versions, aliases), + ), + ): + actual = generate_ndk_versions._generate() + + self.assertEqual(actual["aliases"]["r25"], "25.0.1") + self.assertEqual(actual["versions"]["25.0.1"], existing["versions"]["25.0.1"]) + self.assertEqual(actual["aliases"]["r26"], "26.0.2") + self.assertIn("25.0.9", actual["versions"]) + self.assertIn("26.0.2", actual["versions"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel new file mode 100644 index 0000000..6e8852c --- /dev/null +++ b/tools/BUILD.bazel @@ -0,0 +1,10 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "generate_versions", + srcs = [ + "generate_ndk_versions.py", + "generate_sdk_versions.py", + ], + visibility = ["//tests:__pkg__"], +) diff --git a/tools/generate_ndk_versions.py b/tools/generate_ndk_versions.py index 8179d15..d91ddd9 100755 --- a/tools/generate_ndk_versions.py +++ b/tools/generate_ndk_versions.py @@ -182,20 +182,33 @@ def _archives_json(pkg): } -def _selected_versions(existing_versions, manifest_versions): - return { - version - for version in set(existing_versions) | set(manifest_versions) - if _NDK_PATH_RE.fullmatch("ndk;{}".format(version)) - and _package_version_key(version) >= _MIN_PACKAGE_VERSION - } +def _load_existing(path): + if not path.exists(): + return {} + + with path.open() as f: + return json.load(f) + + +def _merge_preserving_existing(existing, generated): + result = dict(existing) + for name in ("aliases", "versions"): + values = dict(existing.get(name, {})) + for key, value in generated.get(name, {}).items(): + values.setdefault(key, value) + result[name] = values + + for key, value in generated.items(): + result.setdefault(key, value) + return result def _generate(): + existing = _load_existing(_OUTPUT) manifest_versions, aliases = _parse_packages(_fetch_xml(_REPOSITORY_URL)) versions = {} - for version in _selected_versions({}, manifest_versions): + for version in manifest_versions: alias = manifest_versions[version]["alias"] versions[version] = { "strip_prefix": "android-ndk-{}".format(alias), @@ -207,7 +220,7 @@ def _generate(): for alias, version in aliases.items() if version in versions and _NDK_VERSION_RE.fullmatch(alias) } - return { + generated = { "aliases": dict( sorted(aliases.items(), key=lambda item: _version_key(item[0])) ), @@ -215,6 +228,17 @@ def _generate(): sorted(versions.items(), key=lambda item: _package_version_key(item[0])) ), } + merged = _merge_preserving_existing(existing, generated) + merged["aliases"] = dict( + sorted(merged["aliases"].items(), key=lambda item: _version_key(item[0])) + ) + merged["versions"] = dict( + sorted( + merged["versions"].items(), + key=lambda item: _package_version_key(item[0]), + ) + ) + return merged def _main(): diff --git a/tools/generate_sdk_versions.py b/tools/generate_sdk_versions.py index 07bad86..9949a8f 100755 --- a/tools/generate_sdk_versions.py +++ b/tools/generate_sdk_versions.py @@ -184,7 +184,31 @@ def _load_existing(path): metadata = {} _collect_archive_metadata(data, metadata) - return metadata, data.get("components", {}) + return metadata, data + + +def _merge_registry(existing, generated): + result = dict(existing) + for key, value in generated.items(): + result.setdefault(key, value) + return result + + +def _merge_preserving_existing(existing, generated): + result = dict(existing) + + components = dict(existing.get("components", {})) + for name, values in generated.get("components", {}).items(): + components[name] = _merge_registry(components.get(name, {}), values) + result["components"] = components + result["versions"] = _merge_registry( + existing.get("versions", {}), + generated.get("versions", {}), + ) + + for key, value in generated.items(): + result.setdefault(key, value) + return result def _hashes(path): @@ -301,7 +325,8 @@ def _generate_system_images(existing_components, system_image_packages, metadata def _generate(): - metadata, existing_components = _load_existing(_OUTPUT) + metadata, existing = _load_existing(_OUTPUT) + existing_components = existing.get("components", {}) repo_packages = _parse_packages(_fetch_xml(_REPOSITORY_URL)) repo = _latest_by_path(repo_packages) repo_by_path = _by_path(repo_packages) @@ -352,7 +377,10 @@ def _generate(): ), } - return {"components": components, "versions": versions} + return _merge_preserving_existing( + existing, + {"components": components, "versions": versions}, + ) def _main():