From 4001da1ac3ecffa4eeaf9459cc50200cef85a97f Mon Sep 17 00:00:00 2001 From: Hugh Kaznowski Date: Tue, 9 Jun 2026 13:04:11 +0100 Subject: [PATCH 1/3] Add forge-scryfall-uuid-map CLI tool for generating cdn_uuid JSON files Reads a Scryfall bulk data export and writes one JSON file per card print: cdn_uuid/{setCode}/{collectorNumber}.json -> {"en":"uuid","ja":"uuid",...} Defaults to all_cards (~2.5 GB) so every language print is included in each file. Pass --default-cards for a faster English-only run (~100 MB). This is the root cause of single-language files: using default_cards only yields one entry per card. all_cards is required for multi-language UUID maps. Usage: java -jar forge-scryfall-uuid-map.jar --output-dir path/to/res/cdn_uuid Co-Authored-By: Claude Sonnet 4.6 --- forge-scryfall-uuid-map/pom.xml | 75 ++++++++ .../scryfall/uuidmap/BulkDataFetcher.java | 169 ++++++++++++++++++ .../forge/scryfall/uuidmap/CardRecord.java | 9 + .../scryfall/uuidmap/CardStreamParser.java | 156 ++++++++++++++++ .../scryfall/uuidmap/CdnUuidJsonWriter.java | 93 ++++++++++ .../java/forge/scryfall/uuidmap/Main.java | 136 ++++++++++++++ pom.xml | 1 + 7 files changed, 639 insertions(+) create mode 100644 forge-scryfall-uuid-map/pom.xml create mode 100644 forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/BulkDataFetcher.java create mode 100644 forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardRecord.java create mode 100644 forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardStreamParser.java create mode 100644 forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java create mode 100644 forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/Main.java diff --git a/forge-scryfall-uuid-map/pom.xml b/forge-scryfall-uuid-map/pom.xml new file mode 100644 index 000000000000..25a575a1d33e --- /dev/null +++ b/forge-scryfall-uuid-map/pom.xml @@ -0,0 +1,75 @@ + + 4.0.0 + + + forge + forge + ${revision} + + + forge-scryfall-uuid-map + jar + Forge Scryfall UUID Map Builder + + Standalone tool that parses a Scryfall all_cards bulk JSON file and writes a + (set_code, collector_number, lang) to (front_uuid, back_uuid) lookup table + in Apache Parquet format. Covers all cards, all languages, and all art variants. + + + + + + blue.strategic.parquet + parquet-floor + 2.1 + + + + + com.google.code.gson + gson + 2.11.0 + + + + + forge-scryfall-uuid-map-${revision} + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + package + + shade + + + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + forge.scryfall.uuidmap.Main + + + + + + + + + + diff --git a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/BulkDataFetcher.java b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/BulkDataFetcher.java new file mode 100644 index 000000000000..30b2bc762b20 --- /dev/null +++ b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/BulkDataFetcher.java @@ -0,0 +1,169 @@ +package forge.scryfall.uuidmap; + +import com.google.gson.stream.JsonReader; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +/** + * Fetches the Scryfall bulk-data index and downloads the {@code all_cards} dataset. + * + *

The bulk-data index at {@code api.scryfall.com/bulk-data} is a small JSON file + * (~3 KB) listing available datasets and their CDN download URIs. Once we have the + * download URI, the actual data file is served from {@code data.scryfall.io} (CDN, + * no rate limit). + */ +public final class BulkDataFetcher { + + private static final String BULK_INDEX_URL = "https://api.scryfall.com/bulk-data"; + private static final int CONNECT_TIMEOUT = 10_000; + private static final int READ_TIMEOUT = 300_000; // 5 min for large downloads + + private BulkDataFetcher() {} + + /** + * Fetches the bulk-data index and returns the download URI for the + * {@code default_cards} dataset (one English entry per print, ~100 MB). + * Sufficient for patching edition files; prefer this over {@link #fetchAllCardsUri} + * for faster downloads. + */ + public static String fetchDefaultCardsUri() throws IOException { + System.err.println("Fetching Scryfall bulk-data index..."); + String json = fetchText(BULK_INDEX_URL); + String uri = parseDownloadUri(json, "default_cards"); + if (uri == null) { + throw new IOException("'default_cards' entry not found in Scryfall bulk-data index"); + } + System.err.println(" Found: " + uri); + return uri; + } + + /** + * Fetches the bulk-data index and returns the download URI for the + * {@code all_cards} dataset (every language, every art variant, ~2.5 GB). + */ + public static String fetchAllCardsUri() throws IOException { + System.err.println("Fetching Scryfall bulk-data index..."); + String json = fetchText(BULK_INDEX_URL); + String uri = parseDownloadUri(json, "all_cards"); + if (uri == null) { + throw new IOException("'all_cards' entry not found in Scryfall bulk-data index"); + } + System.err.println(" Found: " + uri); + return uri; + } + + /** + * Downloads {@code sourceUrl} to {@code dest}, printing progress every 50 MB. + */ + public static void downloadToFile(String sourceUrl, Path dest) throws IOException { + System.err.println("Downloading: " + sourceUrl); + System.err.println(" to: " + dest.toAbsolutePath()); + + URL url = new URL(sourceUrl); + HttpURLConnection conn = openConnection(url); + long total = conn.getContentLengthLong(); + long bytesRead = 0L; + long lastReport = 0L; + byte[] buf = new byte[65_536]; + + try (InputStream in = conn.getInputStream(); + OutputStream out = new BufferedOutputStream(new FileOutputStream(dest.toFile()))) { + int n; + while ((n = in.read(buf)) > 0) { + out.write(buf, 0, n); + bytesRead += n; + if (bytesRead - lastReport >= 50L << 20) { + lastReport = bytesRead; + String progress = total > 0 + ? String.format("%.0f%%", 100.0 * bytesRead / total) + : String.format("%.0f MB received", bytesRead / 1e6); + System.err.printf(" %.1f MB [%s]%n", bytesRead / 1e6, progress); + } + } + } finally { + conn.disconnect(); + } + System.err.printf(" Download complete: %.1f MB%n", bytesRead / 1e6); + } + + /** + * Parses the bulk-data index JSON and returns the {@code download_uri} for + * the entry whose {@code type} matches {@code targetType}. + */ + static String parseDownloadUri(String json, String targetType) throws IOException { + try (JsonReader reader = new JsonReader(new StringReader(json))) { + reader.beginObject(); + while (reader.hasNext()) { + if ("data".equals(reader.nextName())) { + reader.beginArray(); + while (reader.hasNext()) { + String uri = readEntry(reader, targetType); + if (uri != null) { + return uri; + } + } + reader.endArray(); + } else { + reader.skipValue(); + } + } + reader.endObject(); + } catch (IOException e) { + return null; + } + return null; + } + + private static String readEntry(JsonReader reader, String targetType) throws IOException { + String type = null; + String downloadUri = null; + reader.beginObject(); + while (reader.hasNext()) { + String name = reader.nextName(); + if ("type".equals(name)) { + type = reader.nextString(); + } else if ("download_uri".equals(name)) { + downloadUri = reader.nextString(); + } else { + reader.skipValue(); + } + } + reader.endObject(); + return targetType.equals(type) ? downloadUri : null; + } + + private static String fetchText(String urlStr) throws IOException { + URL url = new URL(urlStr); + HttpURLConnection conn = openConnection(url); + try { + byte[] data = conn.getInputStream().readAllBytes(); + return new String(data, StandardCharsets.UTF_8); + } finally { + conn.disconnect(); + } + } + + private static HttpURLConnection openConnection(URL url) throws IOException { + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty("User-Agent", "forge-scryfall-uuid-map/1.0"); + conn.setConnectTimeout(CONNECT_TIMEOUT); + conn.setReadTimeout(READ_TIMEOUT); + conn.setInstanceFollowRedirects(true); + conn.connect(); + int code = conn.getResponseCode(); + if (code != HttpURLConnection.HTTP_OK) { + conn.disconnect(); + throw new IOException("HTTP " + code + " fetching " + url); + } + return conn; + } +} diff --git a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardRecord.java b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardRecord.java new file mode 100644 index 000000000000..58f2befc13ee --- /dev/null +++ b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardRecord.java @@ -0,0 +1,9 @@ +package forge.scryfall.uuidmap; + +public record CardRecord( + String setCode, + String collectorNumber, + String lang, + String frontUuid, + String backUuid +) {} diff --git a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardStreamParser.java b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardStreamParser.java new file mode 100644 index 000000000000..611998f9a80f --- /dev/null +++ b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardStreamParser.java @@ -0,0 +1,156 @@ +package forge.scryfall.uuidmap; + +import com.google.gson.stream.JsonReader; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.function.Consumer; + +/** + * Streams a Scryfall bulk JSON file (all_cards or default_cards) and emits + * a {@link CardRecord} for every card entry that carries image data. + * + *

Uses Gson's streaming {@link JsonReader} so the 2.5 GB file is never + * fully loaded into memory — only one object at a time is in heap. + */ +public final class CardStreamParser { + + private CardStreamParser() {} + + /** + * Parses {@code bulkFile} and calls {@code consumer} for every record with image data. + * + * @return number of records emitted to the consumer + */ + public static long parse(Path bulkFile, Consumer consumer) throws IOException { + long written = 0L; + long skipped = 0L; + try (JsonReader reader = new JsonReader(new BufferedReader( + new InputStreamReader(new FileInputStream(bulkFile.toFile()), StandardCharsets.UTF_8), + 1 << 20 /* 1 MB read buffer */))) { + reader.beginArray(); + while (reader.hasNext()) { + CardRecord record = readCard(reader); + if (record != null) { + consumer.accept(record); + written++; + } else { + skipped++; + } + long total = written + skipped; + if (total % 50_000 == 0) { + System.err.printf(" %,d processed (%,d written, %,d skipped)%n", + total, written, skipped); + } + } + reader.endArray(); + } + System.err.printf(" Done: %,d written, %,d skipped (no image)%n", written, skipped); + return written; + } + + private static CardRecord readCard(JsonReader reader) throws IOException { + String id = null; + String set = null; + String cn = null; + String lang = null; + String frontUrl = null; + String backUrl = null; + + reader.beginObject(); + while (reader.hasNext()) { + String field = reader.nextName(); + switch (field) { + case "id": id = reader.nextString(); break; + case "set": set = reader.nextString(); break; + case "collector_number": cn = reader.nextString(); break; + case "lang": lang = reader.nextString(); break; + case "image_uris": + frontUrl = readNormalUrl(reader); + break; + case "card_faces": { + String[] urls = readFaceUrls(reader); + frontUrl = urls[0]; + backUrl = urls[1]; + break; + } + default: reader.skipValue(); break; + } + } + reader.endObject(); + + if (id == null || set == null || cn == null || lang == null || frontUrl == null) { + return null; + } + + return new CardRecord( + set, cn, lang, + uuidFromUrl(frontUrl, id), + backUrl != null ? uuidFromUrl(backUrl, id) : null); + } + + /** Reads an {@code image_uris} object and returns the value of the {@code normal} key. */ + private static String readNormalUrl(JsonReader reader) throws IOException { + String normal = null; + reader.beginObject(); + while (reader.hasNext()) { + if ("normal".equals(reader.nextName())) { + normal = reader.nextString(); + } else { + reader.skipValue(); + } + } + reader.endObject(); + return normal; + } + + /** Reads a {@code card_faces} array and returns {@code [front_normal_url, back_normal_url]}. */ + private static String[] readFaceUrls(JsonReader reader) throws IOException { + String[] urls = new String[2]; + int idx = 0; + reader.beginArray(); + while (reader.hasNext()) { + reader.beginObject(); + while (reader.hasNext()) { + String field = reader.nextName(); + if ("image_uris".equals(field) && idx < 2) { + urls[idx] = readNormalUrl(reader); + } else { + reader.skipValue(); + } + } + reader.endObject(); + idx++; + } + reader.endArray(); + return urls; + } + + /** + * Extracts the UUID segment from a Scryfall CDN image URL. + * + *

URL format: {@code https://cards.scryfall.io/normal/front/4/e/{uuid}.jpg?timestamp} + * + *

Parsing the UUID from the URL (rather than using the card's {@code id} field directly) + * correctly handles the two Secret Lair DFC cards where both faces share an artwork UUID + * that differs from the card's own {@code id}. + * + *

Falls back to {@code cardId} for non-CDN URLs such as + * {@code errors.scryfall.com/soon.jpg} (placeholder for missing images). + */ + static String uuidFromUrl(String url, String cardId) { + if (url == null || !url.contains("cards.scryfall.io")) { + return cardId; + } + int qmark = url.indexOf('?'); + String path = qmark >= 0 ? url.substring(0, qmark) : url; + int slash = path.lastIndexOf('/'); + String filename = slash >= 0 ? path.substring(slash + 1) : path; + int dot = filename.lastIndexOf('.'); + return dot >= 0 ? filename.substring(0, dot) : filename; + } +} diff --git a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java new file mode 100644 index 000000000000..87dc34fbb29d --- /dev/null +++ b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java @@ -0,0 +1,93 @@ +package forge.scryfall.uuidmap; + +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Reads a Scryfall bulk JSON export and writes one UUID JSON file per card print to + * {@code outputDir/{setCode}/{collectorNumber}.json}. + * + *

Each file maps language code(s) to the CDN UUID for that card: + * 

+ *   {"en": "uuid"}                           — single language
+ *   {"en": "uuid", "ja": "ja-uuid"}          — multiple languages (same collector number)
+ *   {"en": ["front-uuid", "back-uuid"]}      — DFC with distinct face UUIDs (rare)
+ *
+ * + *

All languages present in the bulk data are written for each card, so callers + * can prefer a specific language and fall back to English when no language-specific + * entry exists. + */ +public final class CdnUuidJsonWriter { + + private CdnUuidJsonWriter() {} + + /** + * Parses {@code bulkFile} and writes UUID JSON files under {@code outputDir}. + * + * @return total number of per-collector-number files written + */ + public static long write(Path bulkFile, Path outputDir) throws IOException { + System.err.println("Parsing UUIDs from " + bulkFile.toAbsolutePath()); + + // setCode/cn -> lang -> [frontUuid, backUuidOrNull] + Map> byCard = new LinkedHashMap<>(700_000); + + CardStreamParser.parse(bulkFile, record -> { + String key = record.setCode().toLowerCase() + "/" + record.collectorNumber(); + byCard.computeIfAbsent(key, k -> new LinkedHashMap<>()) + .put(record.lang(), new String[]{record.frontUuid(), record.backUuid()}); + }); + + System.err.printf(" Collected %,d unique set/collector-number entries.%n", byCard.size()); + + long filesWritten = 0; + for (Map.Entry> e : byCard.entrySet()) { + String[] parts = e.getKey().split("/", 2); + String setCode = parts[0]; + String cn = parts[1]; + Map langs = e.getValue(); + + Path dir = outputDir.resolve(setCode); + Files.createDirectories(dir); + Path out = dir.resolve(cn + ".json"); + writeJson(out, langs); + filesWritten++; + } + + System.err.printf("Done: %,d files written under %s%n", filesWritten, outputDir.toAbsolutePath()); + return filesWritten; + } + + // ------------------------------------------------------------------------- + + private static void writeJson(Path out, Map langs) throws IOException { + StringBuilder sb = new StringBuilder(langs.size() * 60); + sb.append('{'); + boolean first = true; + for (Map.Entry e : langs.entrySet()) { + if (!first) sb.append(','); + first = false; + String lang = e.getKey(); + String front = e.getValue()[0]; + String back = e.getValue()[1]; + sb.append('"').append(lang).append('"').append(':'); + if (back != null && !back.equals(front)) { + sb.append('[').append('"').append(front).append('"') + .append(',').append('"').append(back).append('"').append(']'); + } else { + sb.append('"').append(front).append('"'); + } + } + sb.append('}'); + try (PrintWriter pw = new PrintWriter(Files.newBufferedWriter(out, StandardCharsets.UTF_8))) { + pw.print(sb); + pw.println(); + } + } +} diff --git a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/Main.java b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/Main.java new file mode 100644 index 000000000000..97110bc688ce --- /dev/null +++ b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/Main.java @@ -0,0 +1,136 @@ +package forge.scryfall.uuidmap; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +/** + * Entry point for the Scryfall CDN UUID map generator. + * + *

Reads a Scryfall bulk JSON file and writes one JSON file per card print: + * 

+ *   {outputDir}/{setCode}/{collectorNumber}.json  →  {"en":"uuid","ja":"uuid",...}
+ *
+ * + *

Uses the {@code all_cards} Scryfall dataset by default (every language, every + * art variant, ~2.5 GB) so that per-language UUID entries are fully populated. + * Pass {@code --default-cards} to instead fetch {@code default_cards} (~100 MB, + * one entry per card in English or nearest language only). + * + *

CDN URL formula (for reference): + * {@code https://cards.scryfall.io/{size}/{front|back}/{uuid[0]}/{uuid[1]}/{uuid}.jpg} + * + *

Usage: + * 

+ *   # Download all_cards from Scryfall and write to cdn_uuid (full multi-language)
+ *   java -jar forge-scryfall-uuid-map.jar --output-dir path/to/res/cdn_uuid
+ *
+ *   # Provide a pre-downloaded bulk file
+ *   java -jar forge-scryfall-uuid-map.jar --bulk-file all-cards-20260608.json \
+ *                                          --output-dir path/to/res/cdn_uuid
+ *
+ *   # English-only, smaller download (~100 MB)
+ *   java -jar forge-scryfall-uuid-map.jar --output-dir path/to/res/cdn_uuid --default-cards
+ *
+ */ +public final class Main { + + public static void main(String[] args) throws Exception { + Path bulkFile = null; + Path outputDir = null; + boolean defaultCards = false; + + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "--bulk-file": bulkFile = Path.of(args[++i]); break; + case "--output-dir": outputDir = Path.of(args[++i]); break; + case "--default-cards": defaultCards = true; break; + case "--help": printUsage(); return; + default: + System.err.println("Unknown argument: " + args[i]); + printUsage(); + System.exit(1); + } + } + + if (outputDir == null) { + System.err.println("Error: --output-dir is required."); + printUsage(); + System.exit(1); + } + Files.createDirectories(outputDir); + + if (bulkFile == null) { + bulkFile = findLocalBulkFile(); + if (bulkFile != null) { + System.err.println("Auto-detected: " + bulkFile); + } + } + + if (bulkFile == null || !Files.exists(bulkFile)) { + if (defaultCards) { + System.err.println("Fetching 'default_cards' from Scryfall (~100 MB, English only)."); + String uri = BulkDataFetcher.fetchDefaultCardsUri(); + bulkFile = Path.of(uriFilename(uri)); + BulkDataFetcher.downloadToFile(uri, bulkFile); + } else { + System.err.println("Fetching 'all_cards' from Scryfall (~2.5 GB, all languages)."); + System.err.println("Use --default-cards for a smaller English-only download."); + String uri = BulkDataFetcher.fetchAllCardsUri(); + bulkFile = Path.of(uriFilename(uri)); + BulkDataFetcher.downloadToFile(uri, bulkFile); + } + } + + System.err.println("Input: " + bulkFile.toAbsolutePath()); + System.err.println("Output dir: " + outputDir.toAbsolutePath()); + + long startMs = System.currentTimeMillis(); + long written = CdnUuidJsonWriter.write(bulkFile, outputDir); + long elapsedMs = System.currentTimeMillis() - startMs; + + System.err.printf("%nDone: %,d card entries written in %.1f s%n", written, elapsedMs / 1000.0); + } + + private static String uriFilename(String uri) { + String path = uri.contains("?") ? uri.substring(0, uri.indexOf('?')) : uri; + return path.substring(path.lastIndexOf('/') + 1); + } + + private static Path findLocalBulkFile() { + try (Stream entries = Files.list(Path.of("."))) { + return entries + .filter(p -> { + String name = p.getFileName().toString(); + return (name.startsWith("all-cards-") || name.startsWith("default-cards-")) + && name.endsWith(".json"); + }) + .findFirst() + .orElse(null); + } catch (IOException e) { + return null; + } + } + + private static void printUsage() { + System.err.println("Usage: java -jar forge-scryfall-uuid-map.jar --output-dir DIR [options]"); + System.err.println(); + System.err.println("Options:"); + System.err.println(" --output-dir DIR Output directory for JSON files (required)"); + System.err.println(" e.g. forge-gui/res/cdn_uuid"); + System.err.println(" --bulk-file FILE Pre-downloaded Scryfall all_cards or default_cards JSON."); + System.err.println(" Auto-detected if a matching file exists locally."); + System.err.println(" If absent, all_cards is downloaded from Scryfall."); + System.err.println(" --default-cards Download default_cards (~100 MB) instead of all_cards"); + System.err.println(" (~2.5 GB). Produces English-only output."); + System.err.println(); + System.err.println("Output: {outputDir}/{setCode}/{collectorNumber}.json"); + System.err.println(" Each file maps language code -> UUID string, or [front, back] for DFCs"); + System.err.println(" with distinct face UUIDs (rare). Example:"); + System.err.println(" {\"en\":\"4e7a547f-...\",\"ja\":\"9b2c1234-...\"}"); + System.err.println(); + System.err.println("CDN URL formula:"); + System.err.println(" https://cards.scryfall.io/{size}/{front|back}/{uuid[0]}/{uuid[1]}/{uuid}.jpg"); + } +} diff --git a/pom.xml b/pom.xml index 27ae425425d7..0e891ed8a3d8 100644 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ adventure-editor forge-gui-android forge-installer + forge-scryfall-uuid-map From 0a97123e5686b614aee8c1de41373f02b141f550 Mon Sep 17 00:00:00 2001 From: Hugh Kaznowski Date: Wed, 10 Jun 2026 01:31:09 +0100 Subject: [PATCH 2/3] =?UTF-8?q?Fix=20mvn=20exec:java=20invocation=20?= =?UTF-8?q?=E2=80=94=20remove=20unused=20parquet=20dep,=20wire=20mainClass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the unused parquet-floor dependency (the tool writes JSON, not Parquet) and pre-configure exec-maven-plugin with the mainClass so callers only need -Dexec.args instead of -Dexec.mainClass and the broken -- separator. Co-Authored-By: Claude Sonnet 4.6 --- forge-scryfall-uuid-map/pom.xml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/forge-scryfall-uuid-map/pom.xml b/forge-scryfall-uuid-map/pom.xml index 25a575a1d33e..3728303468a2 100644 --- a/forge-scryfall-uuid-map/pom.xml +++ b/forge-scryfall-uuid-map/pom.xml @@ -13,19 +13,12 @@ jar Forge Scryfall UUID Map Builder - Standalone tool that parses a Scryfall all_cards bulk JSON file and writes a - (set_code, collector_number, lang) to (front_uuid, back_uuid) lookup table - in Apache Parquet format. Covers all cards, all languages, and all art variants. + Standalone tool that parses a Scryfall bulk JSON file and writes one JSON file per + card print: {outputDir}/{setCode}/{collectorNumber}.json mapping language codes to + Scryfall UUIDs. Used to generate the res/cdn_uuid assets shipped with Forge. - - - blue.strategic.parquet - parquet-floor - 2.1 - - com.google.code.gson @@ -37,6 +30,16 @@ forge-scryfall-uuid-map-${revision} + + + org.codehaus.mojo + exec-maven-plugin + 3.5.0 + + forge.scryfall.uuidmap.Main + + + org.apache.maven.plugins From 1fbab2ffba666adaee537d0f95440f786e6ffa9c Mon Sep 17 00:00:00 2001 From: Hugh Kaznowski Date: Wed, 10 Jun 2026 12:24:45 +0100 Subject: [PATCH 3/3] Change output to one JSON per set instead of one per card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runtime now fetches a single set file on demand and caches it locally, so there is no need to ship ~115k individual files with the game. Output: {outputDir}/{setCode}.json → {"cn": {"lang": "uuid", ...}, ...} Co-Authored-By: Claude Sonnet 4.6 --- .../scryfall/uuidmap/CdnUuidJsonWriter.java | 105 ++++++++++-------- .../java/forge/scryfall/uuidmap/Main.java | 4 +- 2 files changed, 62 insertions(+), 47 deletions(-) diff --git a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java index 87dc34fbb29d..be21e24327ab 100644 --- a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java +++ b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java @@ -9,85 +9,100 @@ import java.util.Map; /** - * Reads a Scryfall bulk JSON export and writes one UUID JSON file per card print to - * {@code outputDir/{setCode}/{collectorNumber}.json}. + * Reads a Scryfall bulk JSON export and writes one UUID JSON file per set to + * {@code outputDir/{setCode}.json}. * - *

Each file maps language code(s) to the CDN UUID for that card: + *

Each file maps collector number to a per-language UUID map: * 

- *   {"en": "uuid"}                           — single language
- *   {"en": "uuid", "ja": "ja-uuid"}          — multiple languages (same collector number)
- *   {"en": ["front-uuid", "back-uuid"]}      — DFC with distinct face UUIDs (rare)
+ *   {
+ *     "1":   {"en": "uuid"}
+ *     "2":   {"en": "uuid", "ja": "ja-uuid"}
+ *     "A-40":{"en": ["front-uuid", "back-uuid"]}
+ *   }
  *
* - *

All languages present in the bulk data are written for each card, so callers - * can prefer a specific language and fall back to English when no language-specific - * entry exists. + *

This set-per-file layout lets the runtime fetch exactly one file per set on + * demand and cache it locally, rather than shipping ~115k individual files with + * the game distribution. */ public final class CdnUuidJsonWriter { private CdnUuidJsonWriter() {} /** - * Parses {@code bulkFile} and writes UUID JSON files under {@code outputDir}. + * Parses {@code bulkFile} and writes per-set UUID JSON files under {@code outputDir}. * - * @return total number of per-collector-number files written + * @return total number of set files written */ public static long write(Path bulkFile, Path outputDir) throws IOException { System.err.println("Parsing UUIDs from " + bulkFile.toAbsolutePath()); - // setCode/cn -> lang -> [frontUuid, backUuidOrNull] - Map> byCard = new LinkedHashMap<>(700_000); + // setCode -> cn -> lang -> [frontUuid, backUuidOrNull] + Map>> bySet = new LinkedHashMap<>(1500); CardStreamParser.parse(bulkFile, record -> { - String key = record.setCode().toLowerCase() + "/" + record.collectorNumber(); - byCard.computeIfAbsent(key, k -> new LinkedHashMap<>()) - .put(record.lang(), new String[]{record.frontUuid(), record.backUuid()}); + String setCode = record.setCode().toLowerCase(); + bySet.computeIfAbsent(setCode, k -> new LinkedHashMap<>()) + .computeIfAbsent(record.collectorNumber(), k -> new LinkedHashMap<>()) + .put(record.lang(), new String[]{record.frontUuid(), record.backUuid()}); }); - System.err.printf(" Collected %,d unique set/collector-number entries.%n", byCard.size()); + System.err.printf(" Collected %,d unique sets.%n", bySet.size()); + Files.createDirectories(outputDir); long filesWritten = 0; - for (Map.Entry> e : byCard.entrySet()) { - String[] parts = e.getKey().split("/", 2); - String setCode = parts[0]; - String cn = parts[1]; - Map langs = e.getValue(); - - Path dir = outputDir.resolve(setCode); - Files.createDirectories(dir); - Path out = dir.resolve(cn + ".json"); - writeJson(out, langs); + for (Map.Entry>> setEntry : bySet.entrySet()) { + Path out = outputDir.resolve(setEntry.getKey() + ".json"); + writeSetJson(out, setEntry.getValue()); filesWritten++; } - System.err.printf("Done: %,d files written under %s%n", filesWritten, outputDir.toAbsolutePath()); + System.err.printf("Done: %,d set files written under %s%n", filesWritten, + outputDir.toAbsolutePath()); return filesWritten; } // ------------------------------------------------------------------------- - private static void writeJson(Path out, Map langs) throws IOException { - StringBuilder sb = new StringBuilder(langs.size() * 60); + /** Writes {@code {cn: {lang: uuid|[front,back]}, ...}} to {@code out}. */ + private static void writeSetJson(Path out, + Map> cards) throws IOException { + StringBuilder sb = new StringBuilder(cards.size() * 80); sb.append('{'); - boolean first = true; - for (Map.Entry e : langs.entrySet()) { - if (!first) sb.append(','); - first = false; - String lang = e.getKey(); - String front = e.getValue()[0]; - String back = e.getValue()[1]; - sb.append('"').append(lang).append('"').append(':'); - if (back != null && !back.equals(front)) { - sb.append('[').append('"').append(front).append('"') - .append(',').append('"').append(back).append('"').append(']'); - } else { - sb.append('"').append(front).append('"'); + boolean firstCn = true; + for (Map.Entry> cnEntry : cards.entrySet()) { + if (!firstCn) sb.append(','); + firstCn = false; + appendQuoted(sb, cnEntry.getKey()); + sb.append(":{"); + boolean firstLang = true; + for (Map.Entry langEntry : cnEntry.getValue().entrySet()) { + if (!firstLang) sb.append(','); + firstLang = false; + String lang = langEntry.getKey(); + String front = langEntry.getValue()[0]; + String back = langEntry.getValue()[1]; + appendQuoted(sb, lang); + sb.append(':'); + if (back != null && !back.equals(front)) { + sb.append('['); + appendQuoted(sb, front); + sb.append(','); + appendQuoted(sb, back); + sb.append(']'); + } else { + appendQuoted(sb, front); + } } + sb.append('}'); } sb.append('}'); try (PrintWriter pw = new PrintWriter(Files.newBufferedWriter(out, StandardCharsets.UTF_8))) { - pw.print(sb); - pw.println(); + pw.println(sb); } } + + private static void appendQuoted(StringBuilder sb, String s) { + sb.append('"').append(s).append('"'); + } } diff --git a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/Main.java b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/Main.java index 97110bc688ce..04ed0687abda 100644 --- a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/Main.java +++ b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/Main.java @@ -8,9 +8,9 @@ /** * Entry point for the Scryfall CDN UUID map generator. * - *

Reads a Scryfall bulk JSON file and writes one JSON file per card print: + *

Reads a Scryfall bulk JSON file and writes one JSON file per set: * 

- *   {outputDir}/{setCode}/{collectorNumber}.json  →  {"en":"uuid","ja":"uuid",...}
+ *   {outputDir}/{setCode}.json  →  {"cn":{"en":"uuid","ja":"uuid",...}, ...}
  *
* *

Uses the {@code all_cards} Scryfall dataset by default (every language, every