diff --git a/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala b/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala index b0d1cd7b..c7208df6 100644 --- a/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala +++ b/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala @@ -37,17 +37,13 @@ class CachedResolvedFile( s"Resolved import path $resolvedImportPath is too large: ${jFile.length()} bytes > $memoryLimitBytes bytes" ) - private val resolvedImportContent: ResolvedFile = { - // TODO: Support caching binary data - if (jFile.length() > cacheThresholdBytes) { - // If the file is too large, then we will just read it from disk - null - } else if (binaryData) { - StaticBinaryResolvedFile(readRawBytes(jFile)) - } else { - StaticResolvedFile(readString(jFile)) - } - } + private val cachedBytes: Array[Byte] = + if (jFile.length() > cacheThresholdBytes) null + else readRawBytes(jFile) + + private val cachedBinaryContent: ResolvedFile = + if (cachedBytes != null && binaryData) StaticBinaryResolvedFile(cachedBytes) + else null private def readString(jFile: File): String = { new String(Files.readAllBytes(jFile.toPath), StandardCharsets.UTF_8) @@ -55,45 +51,54 @@ class CachedResolvedFile( private def readRawBytes(jFile: File): Array[Byte] = Files.readAllBytes(jFile.toPath) + private lazy val resolvedTextContent: ResolvedFile = + StaticResolvedFile(new String(cachedBytes, StandardCharsets.UTF_8)) + + private lazy val cachedBytesHash: String = Platform.hashBytes(cachedBytes) + /** * A method that will return a reader for the resolved import. If the import is too large, then * this will return a reader that will read the file from disk. Otherwise, it will return a reader * that reads from memory. */ def getParserInput(): ParserInput = { - if (resolvedImportContent == null) { + if (cachedBytes == null) { FileParserInput(jFile) + } else if (binaryData) { + cachedBinaryContent.getParserInput() } else { - resolvedImportContent.getParserInput() + resolvedTextContent.getParserInput() } } override def readString(): String = { - if (resolvedImportContent == null) { + if (cachedBytes == null) { // If the file is too large, then we will just read it from disk readString(jFile) + } else if (binaryData) { + cachedBinaryContent.readString() } else { // Otherwise, we will read it from memory - resolvedImportContent.readString() + resolvedTextContent.readString() } } override def contentHash(): String = { - if (resolvedImportContent == null) { + if (cachedBytes == null) { // If the file is too large, then we will just read it from disk Platform.hashFile(jFile) } else { - resolvedImportContent.contentHash() + cachedBytesHash } } override def readRawBytes(): Array[Byte] = { - if (resolvedImportContent == null) { + if (cachedBytes == null) { // If the file is too large, then we will just read it from disk readRawBytes(jFile) } else { // Otherwise, we will read it from memory - resolvedImportContent.readRawBytes() + cachedBytes } } } diff --git a/sjsonnet/src-jvm/sjsonnet/Platform.scala b/sjsonnet/src-jvm/sjsonnet/Platform.scala index 5bf307c5..23f329ac 100644 --- a/sjsonnet/src-jvm/sjsonnet/Platform.scala +++ b/sjsonnet/src-jvm/sjsonnet/Platform.scala @@ -140,6 +140,9 @@ object Platform { private val xxHashFactory = XXHashFactory.fastestInstance() + def hashBytes(bytes: Array[Byte]): String = + xxHashFactory.hash64().hash(bytes, 0, bytes.length, 0).toString + def hashFile(file: File): String = { val buffer = new Array[Byte](8192) val hash: StreamingXXHash64 = xxHashFactory.newStreamingHash64(0) diff --git a/sjsonnet/src-native/sjsonnet/Platform.scala b/sjsonnet/src-native/sjsonnet/Platform.scala index fa9b3aa7..c9570722 100644 --- a/sjsonnet/src-native/sjsonnet/Platform.scala +++ b/sjsonnet/src-native/sjsonnet/Platform.scala @@ -197,6 +197,9 @@ object Platform { // Same as go-jsonnet https://github.com/google/go-jsonnet/blob/2b4d7535f540f128e38830492e509a550eb86d57/builtins.go#L959 def sha3(s: String): String = computeHash("SHA3-512", s) + def hashBytes(bytes: Array[Byte]): String = + scala.util.hashing.MurmurHash3.bytesHash(bytes).toHexString + def hashFile(file: File): String = { scala.util.hashing.MurmurHash3 .orderedHash( diff --git a/sjsonnet/src/sjsonnet/Importer.scala b/sjsonnet/src/sjsonnet/Importer.scala index ca823389..0ddc7c78 100644 --- a/sjsonnet/src/sjsonnet/Importer.scala +++ b/sjsonnet/src/sjsonnet/Importer.scala @@ -302,7 +302,7 @@ object CachedResolver { try { val visitor = new JsonImportVisitor(fileScope, internedStrings, settings) - Some((ujson.StringParser.transform(content.readString(), visitor), fileScope)) + Some((ujson.ByteArrayParser.transform(content.readRawBytes(), visitor), fileScope)) } catch { case _: ujson.ParsingFailedException | _: DuplicateJsonKey | _: InvalidJsonNumber | _: JsonParseDepthExceeded | _: NumberFormatException => diff --git a/sjsonnet/test/src/sjsonnet/PreloaderTests.scala b/sjsonnet/test/src/sjsonnet/PreloaderTests.scala index 9d3bc985..f8b9e081 100644 --- a/sjsonnet/test/src/sjsonnet/PreloaderTests.scala +++ b/sjsonnet/test/src/sjsonnet/PreloaderTests.scala @@ -173,7 +173,8 @@ object PreloaderTests extends TestSuite { class JsonOnlyResolvedFile(content: String) extends ResolvedFile { def getParserInput(): fastparse.ParserInput = throw new RuntimeException("strict JSON should not be parsed with fastparse") - def readString(): String = content + def readString(): String = + throw new RuntimeException("strict JSON should not be decoded as text") def contentHash(): String = content def readRawBytes(): Array[Byte] = content.getBytes(java.nio.charset.StandardCharsets.UTF_8)