Make version generation not overwrite anything

tests/BUILD.bazel

@@ -12,6 +12,12 @@ py_test(
     tags = ["manual"],
 )
 
+py_test(
+    name = "generate_versions_test",
+    srcs = ["generate_versions_test.py"],
+    deps = ["//tools:generate_versions"],
+)
+
 action_command_line_test(
     name = "cc_compile_uses_hermetic_ndk_test",
     expected_argv = [

tests/generate_versions_test.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Tests that version metadata updates are additive."""
+
+import copy
+import unittest
+from unittest import mock
+
+from tools import generate_ndk_versions
+from tools import generate_sdk_versions
+
+
+def _archives(prefix, *, sha1=None):
+    return [
+        {
+            "file": "{}-{}.zip".format(prefix, platform),
+            "platform": platform,
+            "sha1": sha1,
+            "url": "https://example.com/{}-{}.zip".format(prefix, platform),
+        }
+        for platform in ("darwin", "linux", "windows")
+    ]
+
+
+class GenerateSdkVersionsTest(unittest.TestCase):
+    def test_generate_preserves_existing_values_and_adds_new_values(self):
+        existing = {
+            "components": {
+                "build_tools": {"38.0.0": {"existing": "build-tools"}},
+                "emulator": {"2.0.0": {"existing": "emulator"}},
+                "platform_tools": {"1.0.0": {"existing": "platform-tools"}},
+                "system_images": {"old/image": {"existing": "system-image"}},
+            },
+            "versions": {
+                "24": {
+                    "platform_tools_version": "1.0.0",
+                    "platform": {"existing": "platform"},
+                }
+            },
+        }
+        repo_packages = [
+            {
+                "path": "platforms;android-24",
+                "revision": (24, 0, 1, 0),
+                "archives": [
+                    {
+                        "file": "changed-platform-24.zip",
+                        "platform": None,
+                        "sha1": None,
+                        "url": "https://example.com/changed-platform-24.zip",
+                    }
+                ],
+            },
+            {
+                "path": "platforms;android-25",
+                "revision": (25, 0, 0, 0),
+                "archives": [
+                    {
+                        "file": "platform-25.zip",
+                        "platform": None,
+                        "sha1": None,
+                        "url": "https://example.com/platform-25.zip",
+                    }
+                ],
+            },
+            {
+                "path": "build-tools;38.0.0",
+                "revision": (38, 0, 0, 0),
+                "archives": _archives("changed-build-tools"),
+            },
+            {
+                "path": "platform-tools",
+                "revision": (2, 0, 0, 0),
+                "archives": _archives("platform-tools-2"),
+            },
+            {
+                "path": "emulator",
+                "revision": (2, 0, 0, 0),
+                "archives": _archives("changed-emulator-2"),
+            },
+            {
+                "path": "emulator",
+                "revision": (3, 0, 0, 0),
+                "archives": _archives("emulator-3"),
+            },
+        ]
+
+        def archive_json(archive, _metadata, infer_prefix):
+            result = {"file": archive["file"], "sha256": "new"}
+            if infer_prefix:
+                result["strip_prefix"] = "new"
+            return result
+
+        with (
+            mock.patch.object(
+                generate_sdk_versions,
+                "_load_existing",
+                return_value=({}, copy.deepcopy(existing)),
+            ),
+            mock.patch.object(generate_sdk_versions, "_fetch_xml", return_value=None),
+            mock.patch.object(
+                generate_sdk_versions,
+                "_parse_packages",
+                side_effect=[repo_packages, []],
+            ),
+            mock.patch.object(
+                generate_sdk_versions,
+                "_archive_json",
+                side_effect=archive_json,
+            ),
+        ):
+            actual = generate_sdk_versions._generate()
+
+        self.assertEqual(
+            actual["components"]["build_tools"]["38.0.0"],
+            existing["components"]["build_tools"]["38.0.0"],
+        )
+        self.assertEqual(
+            actual["components"]["emulator"]["2.0.0"],
+            existing["components"]["emulator"]["2.0.0"],
+        )
+        self.assertEqual(
+            actual["components"]["platform_tools"]["1.0.0"],
+            existing["components"]["platform_tools"]["1.0.0"],
+        )
+        self.assertEqual(
+            actual["components"]["system_images"],
+            existing["components"]["system_images"],
+        )
+        self.assertEqual(actual["versions"]["24"], existing["versions"]["24"])
+        self.assertIn("3.0.0", actual["components"]["emulator"])
+        self.assertIn("2.0.0", actual["components"]["platform_tools"])
+        self.assertIn("25", actual["versions"])
+
+
+class GenerateNdkVersionsTest(unittest.TestCase):
+    def test_generate_preserves_retired_versions_and_changed_aliases(self):
+        existing = {
+            "aliases": {"r25": "25.0.1"},
+            "versions": {
+                "25.0.1": {
+                    "strip_prefix": "existing",
+                    "archives": {"linux": {"existing": "archive"}},
+                }
+            },
+        }
+        sha1 = "00" * 20
+        manifest_versions = {
+            "25.0.9": {
+                "alias": "r25",
+                "archives": _archives("changed-r25", sha1=sha1),
+            },
+            "26.0.2": {
+                "alias": "r26",
+                "archives": _archives("r26", sha1=sha1),
+            },
+        }
+        aliases = {"r25": "25.0.9", "r26": "26.0.2"}
+
+        with (
+            mock.patch.object(
+                generate_ndk_versions,
+                "_load_existing",
+                return_value=copy.deepcopy(existing),
+            ),
+            mock.patch.object(generate_ndk_versions, "_fetch_xml", return_value=None),
+            mock.patch.object(
+                generate_ndk_versions,
+                "_parse_packages",
+                return_value=(manifest_versions, aliases),
+            ),
+        ):
+            actual = generate_ndk_versions._generate()
+
+        self.assertEqual(actual["aliases"]["r25"], "25.0.1")
+        self.assertEqual(actual["versions"]["25.0.1"], existing["versions"]["25.0.1"])
+        self.assertEqual(actual["aliases"]["r26"], "26.0.2")
+        self.assertIn("25.0.9", actual["versions"])
+        self.assertIn("26.0.2", actual["versions"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/BUILD.bazel

@@ -0,0 +1,10 @@
+load("@rules_python//python:defs.bzl", "py_library")
+
+py_library(
+    name = "generate_versions",
+    srcs = [
+        "generate_ndk_versions.py",
+        "generate_sdk_versions.py",
+    ],
+    visibility = ["//tests:__pkg__"],
+)

tools/generate_ndk_versions.py

@@ -182,20 +182,33 @@ def _archives_json(pkg):
     }
 
 
-def _selected_versions(existing_versions, manifest_versions):
-    return {
-        version
-        for version in set(existing_versions) | set(manifest_versions)
-        if _NDK_PATH_RE.fullmatch("ndk;{}".format(version))
-        and _package_version_key(version) >= _MIN_PACKAGE_VERSION
-    }
+def _load_existing(path):
+    if not path.exists():
+        return {}
+
+    with path.open() as f:
+        return json.load(f)
+
+
+def _merge_preserving_existing(existing, generated):
+    result = dict(existing)
+    for name in ("aliases", "versions"):
+        values = dict(existing.get(name, {}))
+        for key, value in generated.get(name, {}).items():
+            values.setdefault(key, value)
+        result[name] = values
+
+    for key, value in generated.items():
+        result.setdefault(key, value)
+    return result
 
 
 def _generate():
+    existing = _load_existing(_OUTPUT)
     manifest_versions, aliases = _parse_packages(_fetch_xml(_REPOSITORY_URL))
     versions = {}
 
-    for version in _selected_versions({}, manifest_versions):
+    for version in manifest_versions:
         alias = manifest_versions[version]["alias"]
         versions[version] = {
             "strip_prefix": "android-ndk-{}".format(alias),
@@ -207,14 +220,25 @@ def _generate():
         for alias, version in aliases.items()
         if version in versions and _NDK_VERSION_RE.fullmatch(alias)
     }
-    return {
+    generated = {
         "aliases": dict(
             sorted(aliases.items(), key=lambda item: _version_key(item[0]))
         ),
         "versions": dict(
             sorted(versions.items(), key=lambda item: _package_version_key(item[0]))
         ),
     }
+    merged = _merge_preserving_existing(existing, generated)
+    merged["aliases"] = dict(
+        sorted(merged["aliases"].items(), key=lambda item: _version_key(item[0]))
+    )
+    merged["versions"] = dict(
+        sorted(
+            merged["versions"].items(),
+            key=lambda item: _package_version_key(item[0]),
+        )
+    )
+    return merged
 
 
 def _main():

tools/generate_sdk_versions.py

@@ -184,7 +184,31 @@ def _load_existing(path):
 
     metadata = {}
     _collect_archive_metadata(data, metadata)
-    return metadata, data.get("components", {})
+    return metadata, data
+
+
+def _merge_registry(existing, generated):
+    result = dict(existing)
+    for key, value in generated.items():
+        result.setdefault(key, value)
+    return result
+
+
+def _merge_preserving_existing(existing, generated):
+    result = dict(existing)
+
+    components = dict(existing.get("components", {}))
+    for name, values in generated.get("components", {}).items():
+        components[name] = _merge_registry(components.get(name, {}), values)
+    result["components"] = components
+    result["versions"] = _merge_registry(
+        existing.get("versions", {}),
+        generated.get("versions", {}),
+    )
+
+    for key, value in generated.items():
+        result.setdefault(key, value)
+    return result
 
 
 def _hashes(path):
@@ -301,7 +325,8 @@ def _generate_system_images(existing_components, system_image_packages, metadata
 
 
 def _generate():
-    metadata, existing_components = _load_existing(_OUTPUT)
+    metadata, existing = _load_existing(_OUTPUT)
+    existing_components = existing.get("components", {})
     repo_packages = _parse_packages(_fetch_xml(_REPOSITORY_URL))
     repo = _latest_by_path(repo_packages)
     repo_by_path = _by_path(repo_packages)
@@ -352,7 +377,10 @@ def _generate():
             ),
         }
 
-    return {"components": components, "versions": versions}
+    return _merge_preserving_existing(
+        existing,
+        {"components": components, "versions": versions},
+    )
 
 
 def _main():