Upgrade jsonschema to 4.25 and additional fixes to pass pre-commit (#1111)

Co-authored-by: Rory Xu <[email protected]>

Author: rory-xu

.pre-commit-config.yaml

@@ -55,6 +55,7 @@ repos:
   hooks:
   - id: bandit
     files: "^src/"
+    additional_dependencies: [pbr]
     # have to skip B101, contract tests use it and there's no way to skip for specific files
     args: ["--skip", "B101"]
 - repo: local

setup.cfg

@@ -29,11 +29,12 @@ include_trailing_comma = true
 combine_as_imports = True
 force_grid_wrap = 0
 known_first_party = rpdk
-known_third_party = boto3,botocore,cfn_tools,cfnlint,colorama,docker,hypothesis,jinja2,jsonpatch,jsonschema,nested_lookup,ordered_set,pkg_resources,pytest,pytest_localserver,requests,setuptools,yaml
+known_third_party = boto3,botocore,cfn_tools,cfnlint,colorama,docker,hypothesis,jinja2,jsonpatch,jsonschema,nested_lookup,ordered_set,pkg_resources,pytest,pytest_localserver,referencing,requests,setuptools,yaml
 
 [tool:pytest]
 # can't do anything about 3rd part modules, so don't spam us
 filterwarnings =
     ignore::DeprecationWarning:botocore
     ignore::DeprecationWarning:werkzeug
     ignore::DeprecationWarning:yaml
+addopts = --doctest-modules

setup.py

@@ -42,7 +42,7 @@ def find_version(*file_paths):
         "Jinja2>=3.1.2",
         "markupsafe>=2.1.0",
         "jsonpatch",
-        "jsonschema>=3.0.0,<=4.17.3",
+        "jsonschema<=4.25",
         "pytest>=4.5.0",
         "pytest-random-order>=1.0.4",
         "pytest-localserver>=0.5.0",

src/rpdk/core/contract/resource_generator.py

@@ -17,7 +17,7 @@
     text,
     tuples,
 )
-from jsonschema import RefResolver
+from jsonschema import RefResolver  # pylint: disable=no-name-in-module
 
 from ..jsonutils.utils import schema_merge
 

src/rpdk/core/data_loaders.py

@@ -7,9 +7,14 @@
 from pathlib import Path
 
 import pkg_resources
+import referencing
+import referencing.exceptions
 import yaml
-from jsonschema import Draft7Validator, RefResolver
-from jsonschema.exceptions import RefResolutionError, ValidationError
+from jsonschema import Draft7Validator
+from jsonschema.exceptions import (  # pylint: disable=no-name-in-module
+    RefResolutionError,
+    ValidationError,
+)
 from nested_lookup import nested_lookup
 
 from .exceptions import InternalError, SpecValidationError
@@ -56,30 +61,52 @@ def copy_resource(package_name, resource_name, out_path):
         shutil.copyfileobj(fsrc, fdst)
 
 
-def get_schema_store(schema_search_path):
-    """Load all the schemas in schema_search_path and return a dict"""
-    schema_store = {}
+def get_schema_registry(schema_search_path):
+    """Load all the schemas in schema_search_path and return a registry"""
+    schemas = {}
     schema_fnames = os.listdir(schema_search_path)
     for schema_fname in schema_fnames:
         schema_path = os.path.join(schema_search_path, schema_fname)
         if schema_path.endswith(".json"):
             with open(schema_path, "r", encoding="utf-8") as schema_f:
                 schema = json.load(schema_f)
                 if "$id" in schema:
-                    schema_store[schema["$id"]] = schema
-    return schema_store
+                    schemas[schema["$id"]] = schema
+
+    # Add HTTPS version of JSON Schema for compatibility
+    if "http://json-schema.org/draft-07/schema#" in schemas:
+        schemas["https://json-schema.org/draft-07/schema#"] = schemas[
+            "http://json-schema.org/draft-07/schema#"
+        ]
+
+    # Create resources with proper specification
+    resources = []
+    for uri, schema in schemas.items():
+        try:
+            resource = referencing.Resource.from_contents(
+                schema, default_specification=referencing.jsonschema.DRAFT7
+            )
+            resources.append((uri, resource))
+        except (
+            ValueError,
+            TypeError,
+            KeyError,
+        ):  # pylint: disable=broad-exception-caught
+            # Fallback for schemas without proper $schema
+            resource = referencing.Resource.from_contents(
+                schema, default_specification=referencing.jsonschema.DRAFT7
+            )
+            resources.append((uri, resource))
+
+    return referencing.Registry().with_resources(resources)
 
 
 def make_validator(schema):
     schema_search_path = Path(os.path.dirname(os.path.realpath(__file__))).joinpath(
         "data/schema/"
     )
-    resolver = RefResolver(
-        base_uri=Draft7Validator.ID_OF(schema),
-        store=get_schema_store(schema_search_path),
-        referrer=schema,
-    )
-    return Draft7Validator(schema, resolver=resolver)
+    registry = get_schema_registry(schema_search_path)
+    return Draft7Validator(schema, registry=registry)
 
 
 def make_resource_validator():
@@ -379,7 +406,7 @@ def load_resource_spec(resource_spec_file):  # pylint: disable=R # noqa: C901
     inliner = RefInliner(base_uri, resource_spec)
     try:
         inlined = inliner.inline()
-    except RefResolutionError as e:
+    except (RefResolutionError, referencing.exceptions.Unresolvable) as e:
         LOG.debug("Resource spec validation failed", exc_info=True)
         raise SpecValidationError(str(e)) from e
 
@@ -444,7 +471,7 @@ def load_hook_spec(hook_spec_file):  # pylint: disable=R # noqa: C901
     inliner = RefInliner(base_uri, hook_spec)
     try:
         inlined = inliner.inline()
-    except RefResolutionError as e:
+    except (RefResolutionError, referencing.exceptions.Unresolvable) as e:
         LOG.debug("Hook spec validation failed", exc_info=True)
         raise SpecValidationError(str(e)) from e
 

src/rpdk/core/jsonutils/inliner.py

@@ -1,7 +1,10 @@
 import logging
 from collections.abc import Iterable, Mapping
 
-from jsonschema import RefResolutionError, RefResolver
+from jsonschema import RefResolver  # pylint: disable=no-name-in-module
+from jsonschema.exceptions import (  # pylint: disable=no-name-in-module
+    RefResolutionError,
+)
 
 from .renamer import RefRenamer
 from .utils import BASE, rewrite_ref, traverse

tests/test_data_loaders.py

@@ -10,13 +10,16 @@
 
 import pytest
 import yaml
-from jsonschema.exceptions import RefResolutionError, ValidationError
+from jsonschema.exceptions import (  # pylint: disable=no-name-in-module
+    RefResolutionError,
+    ValidationError,
+)
 from pytest_localserver.http import Request, Response, WSGIServer
 
 from rpdk.core.data_loaders import (
     STDIN_NAME,
     get_file_base_uri,
-    get_schema_store,
+    get_schema_registry,
     load_hook_spec,
     load_resource_spec,
     resource_json,
@@ -669,36 +672,39 @@ def test_resource_yaml():
     assert result == obj
 
 
-def test_get_schema_store_schemas_with_id():
-    schema_store = get_schema_store(
+def test_get_schema_registry_schemas_with_id():
+    schema_registry = get_schema_registry(
         BASEDIR.parent / "src" / "rpdk" / "core" / "data" / "schema"
     )
-    assert len(schema_store) == 7
-    assert "http://json-schema.org/draft-07/schema#" in schema_store
+    assert len(schema_registry) == 8  # 7 original + HTTPS version of JSON Schema
+    assert "http://json-schema.org/draft-07/schema#" in schema_registry
+    assert (
+        "https://json-schema.org/draft-07/schema#" in schema_registry
+    )  # HTTPS version
     assert (
         "https://schema.cloudformation.us-east-1.amazonaws.com/base.definition.schema.v1.json"
-        in schema_store
+        in schema_registry
     )
     assert (
         "https://schema.cloudformation.us-east-1.amazonaws.com/provider.configuration.definition.schema.v1.json"
-        in schema_store
+        in schema_registry
     )
     assert (
         "https://schema.cloudformation.us-east-1.amazonaws.com/provider.definition.schema.v1.json"
-        in schema_store
+        in schema_registry
     )
     assert (
         "https://schema.cloudformation.us-east-1.amazonaws.com/provider.definition.schema.hooks.v1.json"
-        in schema_store
+        in schema_registry
     )
     assert (
         "https://schema.cloudformation.us-east-1.amazonaws.com/provider.configuration.definition.schema.hooks.v1.json"
-        in schema_store
+        in schema_registry
     )
 
 
-def test_get_schema_store_schemas_with_out_id():
-    schema_store = get_schema_store(
+def test_get_schema_registry_schemas_with_out_id():
+    schema_registry = get_schema_registry(
         BASEDIR.parent / "src" / "rpdk" / "core" / "data" / "examples" / "resource"
     )
-    assert len(schema_store) == 0
+    assert len(schema_registry) == 0