- {engines.map((engine) => (
-
handleEngineClick(engine.id)}
- >
-
-
-
-
- {engine.name}
-
- {selectionMode && (
-
- {selectedEngines.includes(engine.id) && (
-
-
- )}
-
- )}
-
-
-
- {!selectionMode && (
-
- )}
-
+
+
+
+
+ {selectionMode && Select }
+ Query Engine
+ {featureKeys.map((f) => (
+
+ {FEATURE_SHORT_NAMES[f as keyof typeof engines[0]['features']]}
+
+ ))}
+ Score
+
+
+
+ {engines.map((engine, rowIndex) => (
+
+ {selectionMode && (
+
+
+ )}
+
+ window.open(buildEngineDetailUrl(engine.id, engine.version), '_self')}>
+ {engine.baseName} ({engine.version.toUpperCase()})
+
+ {engine.description || 'N/A'}
+
+
+ {featureKeys.map((f, colIndex) => {
+ const feature = engine.features[f as keyof typeof engine.features];
+ const details = feature?.details;
+ const featureName = FEATURE_NAMES[f as keyof typeof engines[0]['features']];
+ const isHovered = hoveredCell?.row === rowIndex && hoveredCell?.col === f;
-
- {engine.description}
-
+ const isLastCols = colIndex >= featureKeys.length - 2;
+ const isFirstCols = colIndex <= 1;
+ const isFirstRow = rowIndex === 0;
- {/* Feature Summary for Mobile */}
-
- {featureKeys.slice(0, 6).map((feature) => (
-
-
- {FEATURE_SHORT_NAMES[feature as keyof QueryEngine['features']]}
-
-
- ))}
-
-
Support Score
-
-
- {calculateSupportScore(engine)}/32
-
-
-
-
-
-
-
- {selectionMode && (
-
-
- Select
-
-
- )}
-
-
-
- Query Engine
-
-
-
- {featureKeys.map((feature) => (
-
-
-
- {FEATURE_SHORT_NAMES[feature as keyof QueryEngine['features']]}
-
-
- {FEATURE_NAMES[feature as keyof QueryEngine['features']]}
-
-
-
- ))}
-
-
- Score
-
-
-
-
-
- {engines.map((engine, index) => (
- handleEngineClick(engine.id)}
- >
- {selectionMode && (
-
-
- {selectedEngines.includes(engine.id) && (
-
-
- )}
-
- )}
-
-
-
-
-
- {engine.name}
-
- {!selectionMode && (
-
- )}
-
-
-
-
- {engine.description}
-
-
-
+ return (
+ details && setHoveredCell({ row: rowIndex, col: f })}
+ onMouseLeave={() => setHoveredCell(null)}
+ >
+
+
- {featureKeys.map((feature) => (
-
-
-
-
-
- {FEATURE_NAMES[feature as keyof QueryEngine['features']]}
-
-
- {engine.features[feature as keyof QueryEngine['features']].details}
+ {isHovered && details && (
+
+
+
+
{featureName}
+
{details}
+
-
-
+ )}
- ))}
-
-
-
-
- {calculateSupportScore(engine)}
-
-
- / 32
-
-
-
-
- ))}
-
-
-
{score(engine)}/32
+
+ ))}
+
+
);
};
-export default TableView;
\ No newline at end of file
+export default TableView;
diff --git a/src/components/Iceberg/versionState.ts b/src/components/Iceberg/versionState.ts
new file mode 100644
index 00000000..2c2e8011
--- /dev/null
+++ b/src/components/Iceberg/versionState.ts
@@ -0,0 +1,25 @@
+import { VersionMode } from '../../types/iceberg';
+
+const STORAGE_KEY = 'iceberg_query_engine_version';
+
+export const normalizeVersion = (value?: string | null): VersionMode => (value === 'v2' ? 'v2' : 'v3');
+
+export const getVersionFromUrl = (search?: string): VersionMode | null => {
+ if (!search) return null;
+ const params = new URLSearchParams(search);
+ const version = params.get('version');
+ return version === 'v2' || version === 'v3' ? version : null;
+};
+
+export const getPersistedVersion = (): VersionMode | null => {
+ if (typeof window === 'undefined') return null;
+ return normalizeVersion(window.localStorage.getItem(STORAGE_KEY));
+};
+
+export const persistVersion = (version: VersionMode): void => {
+ if (typeof window === 'undefined') return;
+ window.localStorage.setItem(STORAGE_KEY, version);
+};
+
+export const buildEngineDetailUrl = (engineId: string, version: VersionMode): string =>
+ `/iceberg/query-engine/${engineId}?version=${version}`;
diff --git a/src/components/Iceberg/versionedTypes.ts b/src/components/Iceberg/versionedTypes.ts
new file mode 100644
index 00000000..719d3d45
--- /dev/null
+++ b/src/components/Iceberg/versionedTypes.ts
@@ -0,0 +1,17 @@
+import { EngineVersion, QueryEngine } from '../../types/iceberg';
+
+export interface ResolvedEngineView {
+ key: string;
+ id: string;
+ name: string;
+ baseName: string;
+ category: QueryEngine['category'];
+ website: string;
+ documentation: string;
+ quickStart: string;
+ bestPractices: string[];
+ version: EngineVersion;
+ features: QueryEngine['features'];
+ description: string;
+ score: number | null;
+}
diff --git a/src/data/query-engines/athena.ts b/src/data/query-engines/athena.ts
index 7992c19f..186d1841 100644
--- a/src/data/query-engines/athena.ts
+++ b/src/data/query-engines/athena.ts
@@ -1,7 +1,8 @@
// data/query-engines/athena.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const athena: QueryEngine = {
+export const athena: QueryEngine = createVersionedEngine({
id: 'athena',
name: 'Amazon Athena (Engine v3)',
description: 'Serverless AWS-native query engine with complete DML operations, Lake Formation governance, time travel, and deep AWS ecosystem integration for Iceberg tables',
@@ -73,7 +74,7 @@ export const athena: QueryEngine = {
},
formatV3: {
support: 'none',
- details: 'Not yet supported; Athena uses Iceberg 1.2.x libraries, spec v3 features (DV, lineage) not available. Creates/writes only spec v2 tables',
+ details: 'Not yet supported; Athena uses Iceberg 1.4.2 libraries; spec v3 features (deletion vectors, row lineage) not available. Creates/writes only spec v2 tables',
externalLinks: [
{
label: 'Query Apache Iceberg Tables',
@@ -151,5 +152,65 @@ OPTIMIZE iceberg_sales_data REWRITE DATA;`,
'Consider millisecond timestamp precision limitations when designing schemas',
'Use schema evolution (ADD/DROP/RENAME COLUMNS) for metadata-only table changes',
'Plan external ingestion strategy as Athena provides no streaming or CDC capabilities'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v2: {
+ features: {
+ catalogs: {
+ support: 'partial',
+ details: 'Only AWS Glue Data Catalog supported for Iceberg. Hive, REST, Nessie, or JDBC catalogs not recognized. Polaris and Unity Catalog accessible via Glue Catalog Federation (read-only)',
+ externalLinks: [{ label: 'AWS Glue Data Catalog', url: 'https://docs.aws.amazon.com/athena/latest/ug/data-sources-glue.html' }]
+ },
+ readWrite: {
+ support: 'full',
+ details: 'SELECT, CREATE TABLE STORED AS ICEBERG, CTAS, INSERT INTO. All writes create new snapshots and become immediately queryable',
+ externalLinks: [{ label: 'Create Iceberg Tables', url: 'https://docs.aws.amazon.com/athena/latest/ug/querying-iceberg-creating-tables.html' }]
+ },
+ dml: {
+ support: 'full',
+ details: 'INSERT INTO, UPDATE, DELETE, and MERGE INTO supported via Athena Engine v3. UPDATE/DELETE/MERGE write position-delete files (MoR) for row-level changes',
+ externalLinks: [{ label: 'MERGE INTO Operations', url: 'https://docs.aws.amazon.com/athena/latest/ug/querying-iceberg-merge-into.html' }]
+ },
+ morCow: {
+ support: 'full',
+ details: 'Supports merge-on-read for both position and equality deletes. Copy-on-write is the default write mode; however MERGE and DELETE always use MoR regardless of table properties',
+ externalLinks: [{ label: 'Apache Iceberg on AWS Guide', url: 'https://docs.aws.amazon.com/prescriptive-guidance/latest/apache-iceberg-on-aws/iceberg-athena.html' }]
+ },
+ streaming: {
+ support: 'none',
+ details: 'No built-in streaming ingestion or CDC APIs. External tools (Glue ETL, Flink) must land data in Iceberg; Athena queries latest committed snapshot'
+ },
+ formatV3: {
+ support: 'none',
+ details: 'Iceberg Format V3 features are not applicable to V2 tables.'
+ },
+ timeTravel: {
+ support: 'full',
+ details: 'FOR TIMESTAMP AS OF and FOR VERSION AS OF clauses let you query historical snapshots with millisecond precision',
+ externalLinks: [{ label: 'Athena Iceberg Tutorial', url: 'https://aws-sdk-pandas.readthedocs.io/en/3.3.0/tutorials/039%20-%20Athena%20Iceberg.html' }]
+ },
+ security: {
+ support: 'full',
+ details: 'Access enforced through IAM plus AWS Lake Formation policies (column-, row-, and cell-level). Lake Formation filters govern metadata table visibility',
+ externalLinks: [{ label: 'Lake Formation Fine-grained Access', url: 'https://docs.aws.amazon.com/athena/latest/ug/querying-iceberg-table-data.html' }]
+ }
+ },
+ score: 22,
+ description: 'Athena Engine v3 provides full Iceberg V2 support with complete DML operations, time travel, and deep AWS ecosystem integration — V3 format is not supported'
+ },
+ v3: {
+ features: {
+ catalogs: { support: 'none', details: 'Athena does not support Iceberg V3 format tables' },
+ readWrite: { support: 'none', details: 'Athena does not support Iceberg V3 format tables' },
+ dml: { support: 'none', details: 'Athena does not support Iceberg V3 format tables' },
+ morCow: { support: 'none', details: 'Athena does not support Iceberg V3 format tables' },
+ streaming: { support: 'none', details: 'Athena does not support Iceberg V3 format tables' },
+ formatV3: { support: 'none', details: 'Athena uses Iceberg 1.2.x libraries; spec V3 features (deletion vectors, row lineage) are not available. Tables are created as spec V2 only' },
+ timeTravel: { support: 'none', details: 'Athena does not support Iceberg V3 format tables' },
+ security: { support: 'none', details: 'Athena does not support Iceberg V3 format tables' }
+ },
+ score: 0,
+ description: 'Athena does not support Iceberg V3 format tables. All Athena Iceberg operations are limited to spec V2'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/bigquery.ts b/src/data/query-engines/bigquery.ts
index 648a9da9..85ed1995 100644
--- a/src/data/query-engines/bigquery.ts
+++ b/src/data/query-engines/bigquery.ts
@@ -1,7 +1,8 @@
// data/query-engines/bigquery.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const bigquery: QueryEngine = {
+export const bigquery: QueryEngine = createVersionedEngine({
id: 'bigquery',
name: 'Google BigQuery',
description: 'Serverless Google Cloud data warehouse with managed Iceberg tables, automatic optimization, Storage Write API streaming, and deep GCP ecosystem integration',
@@ -10,7 +11,7 @@ export const bigquery: QueryEngine = {
documentation: 'https://cloud.google.com/bigquery/docs/iceberg-tables',
features: {
catalogs: {
- support: 'partial',
+ support: 'none',
details: 'BigQuery-managed Iceberg (internal catalog) and BigLake external Iceberg (Dataplex, HMS, AWS Glue via GCS). No direct REST/Nessie support',
externalLinks: [
{
@@ -63,7 +64,7 @@ export const bigquery: QueryEngine = {
},
streaming: {
support: 'partial',
- details: 'High-throughput streaming via Storage Write API (Preview) - Dataflow, Beam, Spark. No built-in CDC apply; use Datastream + Dataflow patterns',
+ details: 'High-throughput streaming via Storage Write API; CDC ingestion available (preview) via Datastream BigLake Iceberg destination and Storage Write API UPSERT mode (_CHANGE_SEQUENCE_NUMBER)',
externalLinks: [
{
label: 'Storage Write API Streaming',
@@ -137,6 +138,67 @@ WHEN NOT MATCHED THEN
-- Time travel query
SELECT * FROM iceberg_dataset.sales_data
FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR);`,
+ versions: {
+ v2: {
+ features: {
+ catalogs: {
+ support: 'none',
+ details: 'No native support for external Iceberg catalogs like Hive Metastore, AWS Glue, REST, Nessie, Polaris, Unity Catalog, Hadoop, or JDBC. BigQuery relies on BigLake Metastore and managed catalog abstractions instead',
+ externalLinks: [{ label: 'BigQuery Managed Iceberg Tables', url: 'https://cloud.google.com/blog/products/data-analytics/announcing-bigquery-tables-for-apache-iceberg' }]
+ },
+ readWrite: {
+ support: 'full',
+ details: 'Full read and write support for BigLake Iceberg tables including INSERT, MERGE, UPDATE, and DELETE via GoogleSQL',
+ externalLinks: [{ label: 'BigQuery Iceberg DML Operations', url: 'https://cloud.google.com/bigquery/docs/iceberg-tables#dml' }]
+ },
+ dml: {
+ support: 'full',
+ details: 'MERGE, UPDATE, DELETE fully supported through GoogleSQL with transactional guarantees',
+ externalLinks: [{ label: 'Data Manipulation Language DML', url: 'https://cloud.google.com/bigquery/docs/iceberg-tables#dml' }]
+ },
+ morCow: {
+ support: 'partial',
+ details: 'Copy-on-write is fully supported, while merge-on-read, position deletes, and equality deletes are only partially supported and abstracted from the user',
+ externalLinks: [{ label: 'Automatic Storage Optimization', url: 'https://cloud.google.com/blog/products/data-analytics/announcing-bigquery-tables-for-apache-iceberg' }]
+ },
+ streaming: {
+ support: 'partial',
+ details: 'Streaming ingestion via Storage Write API (GA); CDC available in preview via Datastream BigLake Iceberg destination and Storage Write API UPSERT (_CHANGE_SEQUENCE_NUMBER)',
+ externalLinks: [{ label: 'Storage Write API Streaming', url: 'https://cloud.google.com/blog/products/data-analytics/announcing-bigquery-tables-for-apache-iceberg' }]
+ },
+ formatV3: {
+ support: 'none',
+ details: 'Streaming ingestion supported via Storage Write API, but CDC-style updates are limited.'
+ },
+ timeTravel: {
+ support: 'full',
+ details: 'Full snapshot-based time travel support for querying historical versions of data.',
+ externalLinks: [{ label: 'Time Travel for Historical Data', url: 'https://cloud.google.com/bigquery/docs/iceberg-tables#time_travel' }]
+ },
+ security: {
+ support: 'full',
+ details: 'IAM permissions like native BigQuery tables. Column-level security & masking on managed Iceberg. External via BigLake/Dataplex policy tags',
+ externalLinks: [{ label: 'Column-level Security and Data Masking', url: 'https://cloud.google.com/bigquery/docs/iceberg-tables#security' }]
+ }
+ },
+ score: 20,
+ description: 'BigQuery supports full Iceberg V2 operations with managed tables, automatic optimization, and deep GCP ecosystem integration — V3 format is not supported'
+ },
+ v3: {
+ features: {
+ catalogs: { support: 'none', details: 'BigQuery only supports Iceberg V2 snapshot export; V3 is not supported' },
+ readWrite: { support: 'none', details: 'BigQuery only exports Iceberg V2 snapshots; V3 read/write support is not available' },
+ dml: { support: 'none', details: 'BigQuery only supports Iceberg V2 snapshot export; V3 is not supported' },
+ morCow: { support: 'none', details: 'BigQuery only supports Iceberg V2 snapshot export; V3 is not supported' },
+ streaming: { support: 'none', details: 'BigQuery only supports Iceberg V2; V3 streaming not supported' },
+ formatV3: { support: 'none', details: 'BigQuery only supports Iceberg V2 snapshot export; V3 format is not supported. Only Iceberg V2 format is supported for snapshot export' },
+ timeTravel: { support: 'none', details: 'BigQuery only supports Iceberg V2 snapshot export; V3 time travel is not available' },
+ security: { support: 'none', details: 'BigQuery only supports Iceberg V2 snapshot export; V3 is not supported' }
+ },
+ score: 0,
+ description: 'BigQuery does not support Iceberg V3 format tables. All BigQuery Iceberg operations are limited to spec V2'
+ }
+ },
bestPractices: [
'Enable Iceberg Tables Preview or BigLake Iceberg in your GCP project for access',
'Use managed Iceberg tables for full DML capabilities and automatic optimization',
@@ -159,4 +221,4 @@ FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR);`,
'Leverage end-to-end lineage through Dataplex integration',
'Use external table writes via Dataflow/Spark when BigQuery-native DML is insufficient'
]
-};
\ No newline at end of file
+});
\ No newline at end of file
diff --git a/src/data/query-engines/clickhouse.ts b/src/data/query-engines/clickhouse.ts
index a59317a7..50927e9c 100644
--- a/src/data/query-engines/clickhouse.ts
+++ b/src/data/query-engines/clickhouse.ts
@@ -1,10 +1,11 @@
// data/query-engines/clickhouse.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const clickhouse: QueryEngine = {
+export const clickhouse: QueryEngine = createVersionedEngine({
id: 'clickhouse',
- name: 'ClickHouse v25.4',
- description: 'Rapidly evolving OLAP database with experimental Iceberg read support, time travel, REST catalogs, and comprehensive write capabilities planned for 2025',
+ name: 'ClickHouse v25.9',
+ description: 'Rapidly evolving OLAP database with Iceberg read+write support (v25.7+), time travel (v25.4+), REST catalogs, and basic DML (INSERT/ALTER DELETE/ALTER UPDATE) via LTS v25.8',
category: 'general-purpose',
website: 'https://clickhouse.com/',
documentation: 'https://clickhouse.com/docs/en/engines/table-engines/integrations/iceberg',
@@ -29,25 +30,25 @@ export const clickhouse: QueryEngine = {
},
readWrite: {
support: 'partial',
- details: 'Read-only: ENGINE=Iceberg tables and icebergS3()/icebergCluster() functions; full SQL on Parquet files. Writes/compaction scheduled Q3 2025',
+ details: 'Full reads via ENGINE=Iceberg and icebergS3()/icebergCluster() functions. Write support added v25.7 (INSERT INTO existing tables), CREATE TABLE and DROP TABLE added v25.8 (LTS). No compaction yet',
externalLinks: [
{
- label: 'ClickHouse Release 25.04',
- url: 'https://clickhouse.com/blog/clickhouse-release-25-04'
+ label: 'ClickHouse Release 25.08 (LTS)',
+ url: 'https://clickhouse.com/blog/clickhouse-release-25-08'
},
{
- label: 'Iceberg Write Support Tracking',
- url: 'https://github.com/ClickHouse/ClickHouse/issues/71407'
+ label: 'ClickHouse 2025 Year in Review',
+ url: 'https://clickhouse.com/blog/clickhouse-2025-roundup'
}
]
},
dml: {
- support: 'none',
- details: 'Reading of position & equality deletes supported since 24.12; queries merge delete files on-the-fly (MoR). No DELETE/UPDATE/MERGE writers until write support lands',
+ support: 'partial',
+ details: 'INSERT INTO existing tables (v25.7+), CREATE TABLE (v25.8+), ALTER TABLE DELETE (positional+equality deletes, v25.8+), ALTER TABLE UPDATE (v25.9+). MERGE not yet supported',
externalLinks: [
{
- label: 'Delete Files Support',
- url: 'https://clickhouse.com/blog/clickhouse-release-24-12'
+ label: 'ClickHouse Release 25.08',
+ url: 'https://clickhouse.com/blog/clickhouse-release-25-08'
},
{
label: 'DML Write Support Tracking',
@@ -133,6 +134,65 @@ SETTINGS iceberg_timestamp_ms = 1640995200000;
-- Use cluster function for distributed reads
SELECT * FROM icebergCluster('cluster', 's3://bucket/warehouse/table/');`,
+ versions: {
+ v2: {
+ features: {
+ catalogs: {
+ support: 'full',
+ details: 'REST catalog (icebergS3 function with storage_catalog_type), AWS Glue, Polaris (partial), Unity Catalog (partial) supported via icebergS3/icebergLocal table functions and experimental DataLakeCatalog engine',
+ externalLinks: [{ label: 'Iceberg Engine Documentation', url: 'https://clickhouse.com/docs/en/engines/table-engines/integrations/iceberg' }]
+ },
+ readWrite: {
+ support: 'partial',
+ details: 'Full reads via icebergS3/icebergAzure/icebergLocal table functions and ENGINE=Iceberg. INSERT INTO existing tables (v25.7+), CREATE TABLE and DROP TABLE (v25.8+, LTS). No compaction',
+ externalLinks: [{ label: 'ClickHouse Release 25.08', url: 'https://clickhouse.com/blog/clickhouse-release-25-08' }]
+ },
+ dml: {
+ support: 'partial',
+ details: 'INSERT INTO (v25.7+), ALTER TABLE DELETE with positional and equality deletes (v25.8+), ALTER TABLE UPDATE (v25.9+). No MERGE support. Applies to V2 tables',
+ externalLinks: [{ label: 'ClickHouse Release 25.08', url: 'https://clickhouse.com/blog/clickhouse-release-25-08' }]
+ },
+ morCow: {
+ support: 'partial',
+ details: 'Reads CoW and MoR (position+equality deletes) since v24.12; writes position/equality delete files for ALTER TABLE DELETE (v25.8+); no compaction',
+ externalLinks: [{ label: 'ClickHouse Release 25.08', url: 'https://clickhouse.com/blog/clickhouse-release-25-08' }]
+ },
+ streaming: {
+ support: 'none',
+ details: 'No native streaming ingestion; users poll Iceberg or ingest with ClickHouse Kafka engine'
+ },
+ formatV3: {
+ support: 'none',
+ details: 'Iceberg Format V3 features are not applicable to V2 tables.'
+ },
+ timeTravel: {
+ support: 'full',
+ details: 'Time travel via SET iceberg_timestamp_ms=
or iceberg_snapshot_id since v25.4; partition pruning via use_iceberg_partition_pruning=1',
+ externalLinks: [{ label: 'Time Travel in 25.4', url: 'https://clickhouse.com/blog/clickhouse-release-25-04' }]
+ },
+ security: {
+ support: 'partial',
+ details: 'Relies on object-store credentials (AWS_ACCESS_KEY_ID, S3 V4 tokens) or catalog credential vending; ClickHouse RBAC controls database/table access; no column-masking yet'
+ }
+ },
+ score: 16,
+ description: 'ClickHouse v25.8+ (LTS) supports Iceberg V2 read+write with INSERT INTO, CREATE/DROP TABLE, ALTER TABLE DELETE/UPDATE, time travel, and REST catalog support; MERGE and V3 not yet available'
+ },
+ v3: {
+ features: {
+ catalogs: { support: 'none', details: 'ClickHouse only supports reading Iceberg V1 and V2 via table functions; V3 is not yet supported' },
+ readWrite: { support: 'none', details: 'ClickHouse currently supports reading Iceberg V1 and V2 only; V3 support is not yet available' },
+ dml: { support: 'none', details: 'Read-only integration for V2 only; V3 not supported' },
+ morCow: { support: 'none', details: 'V3 deletion vectors not supported; only V1 and V2 tables readable' },
+ streaming: { support: 'none', details: 'No streaming support for any version' },
+ formatV3: { support: 'none', details: 'Iceberg Format V3 is not yet supported in ClickHouse; V3 reader/writer planned for a future release' },
+ timeTravel: { support: 'none', details: 'Time travel only available for V1 and V2 tables; V3 not supported' },
+ security: { support: 'none', details: 'V3 not supported; security features are V2 only' }
+ },
+ score: 0,
+ description: 'ClickHouse does not yet support Iceberg V3 format tables. V3 reader/writer planned for a future release'
+ }
+ },
bestPractices: [
'Use ClickHouse v25.4+ for time travel and metadata caching capabilities',
'Leverage REST catalog support (24.12+) for integration with Nessie, Polaris/Unity, and Glue',
@@ -155,4 +215,4 @@ SELECT * FROM icebergCluster('cluster', 's3://bucket/warehouse/table/');`,
'Use materialized views for query acceleration on frequently accessed data',
'Monitor GitHub issues for rapid feature development and breaking changes'
]
-};
\ No newline at end of file
+});
\ No newline at end of file
diff --git a/src/data/query-engines/databricks.ts b/src/data/query-engines/databricks.ts
index d02d6b54..5ce010ad 100644
--- a/src/data/query-engines/databricks.ts
+++ b/src/data/query-engines/databricks.ts
@@ -1,36 +1,33 @@
// data/query-engines/databricks.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const databricks: QueryEngine = {
+export const databricks: QueryEngine = createVersionedEngine({
id: 'databricks',
- name: 'Databricks Runtime 14.3 LTS+',
- description: 'UniForm technology enables multi-format lakehouse with read-only Iceberg views of Delta tables via Unity Catalog REST endpoint',
+ name: 'Databricks (DBR 16.4+)',
+ description: 'Native Iceberg support in Databricks Runtime 16.4+ with Unity Catalog. Full DML on managed Iceberg tables, deletion vectors (V3 Beta in DBR 17.3), and UniForm for multi-format lakehouse interoperability',
category: 'lakehouse',
website: 'https://databricks.com/',
documentation: 'https://docs.databricks.com/aws/en/delta/uniform.html',
features: {
catalogs: {
- support: 'partial',
- details: 'Unity Catalog exposes Iceberg REST catalog at /api/2.1/unity-catalog/iceberg for external engines; UniForm tables generate Iceberg metadata on Delta commits',
+ support: 'full',
+ details: 'Unity Catalog implements the Iceberg REST Catalog API for both read and write on managed Iceberg tables. Hive Metastore and AWS Glue accessible via Lakehouse Federation (read-only). Nessie, Hadoop, JDBC not supported',
externalLinks: [
{
label: 'Unity Catalog Iceberg Endpoint',
url: 'https://docs.databricks.com/aws/en/external-access/iceberg.html'
},
{
- label: 'Read Delta Tables with Iceberg Clients',
- url: 'https://docs.databricks.com/aws/en/delta/uniform.html'
+ label: 'Full Apache Iceberg Support Announcement',
+ url: 'https://www.databricks.com/blog/announcing-full-apache-iceberg-support-databricks'
}
]
},
readWrite: {
- support: 'partial',
- details: 'Full reads via REST catalog or direct metadata paths; writes supported for Managed Iceberg Tables via external engines, but UniForm Delta tables are read-only for Iceberg clients',
+ support: 'full',
+ details: 'Full read/write for native managed Iceberg V3 tables via Unity Catalog (DBR 18.0+ Public Preview). INSERT INTO, CREATE TABLE, and full DDL supported. External engines can read and write via REST catalog',
externalLinks: [
- {
- label: 'Read Delta Tables with Iceberg Clients',
- url: 'https://docs.databricks.com/aws/en/delta/uniform.html'
- },
{
label: 'Full Apache Iceberg Support Announcement',
url: 'https://www.databricks.com/blog/announcing-full-apache-iceberg-support-databricks'
@@ -38,52 +35,56 @@ export const databricks: QueryEngine = {
]
},
dml: {
- support: 'limited',
- details: 'Full DML (INSERT, MERGE, UPDATE, DELETE) available for Delta users inside Databricks; Iceberg clients can only read through REST catalog',
+ support: 'full',
+ details: 'MERGE, UPDATE, DELETE fully supported for native managed Iceberg V3 tables with deletion vectors for efficient row-level changes (DBR 18.0+ Public Preview)',
externalLinks: [
{
- label: 'Read Delta Tables with Iceberg Clients - Limitations',
- url: 'https://docs.databricks.com/aws/en/delta/uniform.html'
+ label: 'Unity Catalog Iceberg Endpoint',
+ url: 'https://docs.databricks.com/aws/en/external-access/iceberg.html'
}
]
},
morCow: {
- support: 'limited',
- details: 'Copy-on-Write (CoW) semantics for Delta commits; no Iceberg delete files produced; external readers see fully merged snapshots',
+ support: 'partial',
+ details: 'V3 uses deletion vectors (MoR-like) for efficient row-level deletes without rewriting data files. Copy-on-write also fully supported. V2 position/equality deletes not used — DVs replace them in V3',
externalLinks: [
{
- label: 'Unity Catalog Iceberg Endpoint - Notes',
- url: 'https://docs.databricks.com/aws/en/external-access/iceberg.html'
+ label: 'Full Apache Iceberg Support Announcement',
+ url: 'https://www.databricks.com/blog/announcing-full-apache-iceberg-support-databricks'
}
]
},
streaming: {
- support: 'internal',
- details: 'Spark Structured Streaming and Delta Change Data Feed available inside Databricks; Iceberg REST interface does not expose streaming ingestion or CDC endpoints',
+ support: 'none',
+ details: 'Change Data Feed (CDF) not supported on Iceberg tables — CDF is Delta Lake-only. Iceberg V3 row lineage provides CDC building blocks but CDC streaming is not directly exposed',
externalLinks: [
{
- label: 'Read Delta Tables - Streaming Limitations',
+ label: 'Read Delta Tables with Iceberg Clients',
url: 'https://docs.databricks.com/aws/en/delta/uniform.html'
}
]
},
formatV3: {
- support: 'none',
- details: 'UniForm currently targets Iceberg spec v2 only; no public roadmap for v3 support announced yet',
+ support: 'partial',
+ details: 'Iceberg V3 Public Preview in DBR 18.0+. Supports deletion vectors, VARIANT columns, V3 table creation. Existing V2 tables can be upgraded. V2 position/equality deletes not supported — Databricks uses DVs (V3) exclusively',
externalLinks: [
{
- label: 'Full Apache Iceberg Support - V2 Preview',
+ label: 'Iceberg V3 in Databricks',
+ url: 'https://docs.databricks.com/aws/en/iceberg/iceberg-v3'
+ },
+ {
+ label: 'Full Apache Iceberg Support Announcement',
url: 'https://www.databricks.com/blog/announcing-full-apache-iceberg-support-databricks'
}
]
},
timeTravel: {
support: 'full',
- details: 'External engines can time-travel by snapshot-ID or timestamp using standard Iceberg syntax; includes converted_delta_version and converted_delta_timestamp properties',
+ details: 'Time travel and RESTORE TABLE supported for managed Iceberg V3 tables (DBR 18.0+ Public Preview). For foreign Iceberg tables, time travel is limited',
externalLinks: [
{
- label: 'Iceberg REST API Specification',
- url: 'https://github.com/apache/iceberg/blob/master/api/src/main/java/org/apache/iceberg/rest/RestCatalog.java'
+ label: 'Unity Catalog Iceberg Endpoint',
+ url: 'https://docs.databricks.com/aws/en/external-access/iceberg.html'
}
]
},
@@ -116,6 +117,52 @@ TBLPROPERTIES (
-- External Iceberg client connection
-- REST Catalog: https:///api/2.1/unity-catalog/iceberg
-- OAuth Token: `,
+ versions: {
+ v2: {
+ features: {
+ catalogs: {
+ support: 'full',
+ details: 'Unity Catalog provides native Iceberg REST Catalog API for both read and write on managed Iceberg V2 tables (Public Preview, DBR 16.4 LTS). Hive Metastore and AWS Glue accessible via Lakehouse Federation (read-only)',
+ externalLinks: [{ label: 'Unity Catalog Iceberg Endpoint', url: 'https://docs.databricks.com/aws/en/external-access/iceberg.html' }]
+ },
+ readWrite: {
+ support: 'full',
+ details: 'Full read/write for native managed Iceberg V2 tables via Unity Catalog (Public Preview, DBR 16.4 LTS). INSERT INTO, CREATE TABLE, and full DDL supported',
+ externalLinks: [{ label: 'Full Apache Iceberg Support Announcement', url: 'https://www.databricks.com/blog/announcing-full-apache-iceberg-support-databricks' }]
+ },
+ dml: {
+ support: 'full',
+ details: 'MERGE, UPDATE, DELETE fully supported for native managed Iceberg V2 tables via copy-on-write. Managed Iceberg tables use CoW exclusively in V2',
+ externalLinks: [{ label: 'Unity Catalog Iceberg Endpoint', url: 'https://docs.databricks.com/aws/en/external-access/iceberg.html' }]
+ },
+ morCow: {
+ support: 'partial',
+ details: 'Copy-on-write is the default and only write mode for managed Iceberg V2 tables. Iceberg V2 position and equality deletes are explicitly not supported — Databricks uses deletion vectors (V3 feature) instead',
+ externalLinks: [{ label: 'Full Apache Iceberg Support Announcement', url: 'https://www.databricks.com/blog/announcing-full-apache-iceberg-support-databricks' }]
+ },
+ streaming: {
+ support: 'none',
+ details: 'Change Data Feed not supported on Iceberg tables. CDC and streaming ingestion must use external engines (Flink, Spark Structured Streaming) writing to Iceberg'
+ },
+ formatV3: {
+ support: 'none',
+ details: 'Iceberg Format V3 features are not applicable to V2 tables.'
+ },
+ timeTravel: {
+ support: 'full',
+ details: 'Time travel fully supported for managed Iceberg V2 tables by snapshot ID and timestamp. For foreign Iceberg tables, time travel is limited',
+ externalLinks: [{ label: 'Unity Catalog Iceberg Endpoint', url: 'https://docs.databricks.com/aws/en/external-access/iceberg.html' }]
+ },
+ security: {
+ support: 'full',
+ details: 'Unity Catalog RBAC governs access; Iceberg REST clients receive temporary, scoped cloud-storage credentials via credential vending during handshake',
+ externalLinks: [{ label: 'Access Databricks Tables from Iceberg Clients', url: 'https://docs.databricks.com/aws/en/external-access/iceberg.html' }]
+ }
+ },
+ score: 22,
+ description: 'Databricks DBR 16.4+ provides full native Iceberg V2 support with Unity Catalog, complete DML operations, and copy-on-write semantics for managed tables'
+ }
+ },
bestPractices: [
'Use Databricks Runtime 14.3 LTS or newer for IcebergCompatV2 feature support',
'Enable UniForm via delta.universalFormat.enabledFormats=iceberg for new tables',
@@ -134,4 +181,4 @@ TBLPROPERTIES (
'Use Unity Catalog REST API v2.1 for proper Iceberg catalog endpoint access',
'Test external engine compatibility before production deployment of UniForm tables'
]
-};
\ No newline at end of file
+});
\ No newline at end of file
diff --git a/src/data/query-engines/doris.ts b/src/data/query-engines/doris.ts
index 5acd8c1e..438c5d9e 100644
--- a/src/data/query-engines/doris.ts
+++ b/src/data/query-engines/doris.ts
@@ -1,7 +1,8 @@
// data/query-engines/doris.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const doris: QueryEngine = {
+export const doris: QueryEngine = createVersionedEngine({
id: 'doris',
name: 'Apache Doris v2.1+',
description: 'MPP analytical database with comprehensive Iceberg read/write capabilities, vectorized execution, materialized view acceleration, and multi-catalog support for lake ingestion and analytics',
@@ -162,5 +163,21 @@ SELECT * FROM iceberg_meta("table"="sales_data", "query_type"="snapshots");`,
'Be aware that Avro data files are not supported in current versions',
'Configure appropriate catalog credentials and metastore URIs for secure access',
'Monitor Iceberg client version (currently 1.6.1) for compatibility with other engines'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v3: {
+ features: {
+ catalogs: { support: 'none', details: 'Apache Doris supports only Iceberg spec v1 & v2; V3 catalog operations not supported' },
+ readWrite: { support: 'none', details: 'Cannot read or write Iceberg V3 format tables; V3 spec work follows upstream Iceberg roadmap' },
+ dml: { support: 'none', details: 'DML operations produce V2 format outputs only; V3 table format not supported' },
+ morCow: { support: 'none', details: 'V3 deletion vectors not supported; only V2 position/equality delete files supported' },
+ streaming: { support: 'none', details: 'No native streaming for any format version' },
+ formatV3: { support: 'none', details: 'Supports spec v1 & v2 only; spec v3 work follows upstream Iceberg roadmap — no GA support yet' },
+ timeTravel: { support: 'none', details: 'Time travel via FOR TIMESTAMP/VERSION AS OF only for V1/V2 format tables' },
+ security: { support: 'none', details: 'V3 format not supported; Doris RBAC and catalog IAM apply to V1/V2 tables only' }
+ },
+ score: 0,
+ description: 'Apache Doris v2.1+ supports Iceberg spec V1/V2 only; Format V3 (deletion vectors, row lineage, new data types) not yet supported'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/dremio.ts b/src/data/query-engines/dremio.ts
index 6c12c5de..5aeced2a 100644
--- a/src/data/query-engines/dremio.ts
+++ b/src/data/query-engines/dremio.ts
@@ -1,7 +1,8 @@
// data/query-engines/dremio.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const dremio: QueryEngine = {
+export const dremio: QueryEngine = createVersionedEngine({
id: 'dremio',
name: 'Dremio v26',
description: 'Full Iceberg authoring engine with built-in Polaris catalog, complete DML including MERGE, Arctic git-like branching, and Data Reflections acceleration',
@@ -88,9 +89,13 @@ export const dremio: QueryEngine = {
]
},
formatV3: {
- support: 'none',
- details: 'Planned (2025) - roadmap calls for reading Deletion Vectors & row-lineage columns first; writer support (DV emission) to follow once Iceberg 1.8+ library adopted',
+ support: 'full',
+ details: 'GA for Dremio Cloud (announced April 6, 2026): full V3 read+write including Deletion Vectors, VARIANT columns for JSON, and row-level lineage. Self-managed deployment roadmap follows',
externalLinks: [
+ {
+ label: 'Dremio V3 GA Announcement (April 2026)',
+ url: 'https://www.globenewswire.com/news-release/2026/04/06/3268593/0/en/Dremio-Deepens-Apache-Iceberg-Leadership-with-V3-Support-New-Community-Appointments-and-Polaris-Momentum.html'
+ },
{
label: 'What\'s New in Iceberg v3?',
url: 'https://www.dremio.com/blog/apache-iceberg-v3/'
@@ -176,5 +181,21 @@ SELECT * FROM iceberg_catalog.sales.orders@main;`,
'Use Arctic commit logs for comprehensive audit trails and data lineage',
'Leverage Dremio\'s catalog credential vending for secure multi-tenant access',
'Plan for Format V3 support arriving in 2025 with deletion vectors and row lineage'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v3: {
+ features: {
+ catalogs: { support: 'full', details: 'Polaris/Dremio Catalog, Generic REST, Arctic/Nessie, HMS, AWS Glue, Hadoop fully support V3 table registration and queries (Dremio Cloud, April 2026)' },
+ readWrite: { support: 'full', details: 'Full V3 read+write GA for Dremio Cloud (April 2026): reads and writes tables with Deletion Vectors, VARIANT columns, and row lineage' },
+ dml: { support: 'full', details: 'MERGE, UPDATE, DELETE, INSERT with V3 Deletion Vectors for efficient MoR-style row-level changes (Dremio Cloud GA, April 2026)' },
+ morCow: { support: 'full', details: 'V3 Deletion Vectors supported for MoR-style writes; Copy-on-Write also available (Dremio Cloud, April 2026); self-managed roadmap follows' },
+ streaming: { support: 'none', details: 'No native streaming for any format version; external engines ingest into V3 tables' },
+ formatV3: { support: 'full', details: 'GA for Dremio Cloud (April 6, 2026): Deletion Vectors, VARIANT type for JSON, row-level lineage. Self-managed deployment V3 roadmap follows' },
+ timeTravel: { support: 'full', details: 'Arctic/Nessie branches, tags, and snapshot-based time travel fully supported for V3 tables in Dremio Cloud' },
+ security: { support: 'full', details: 'Dremio RBAC, column masking, and credential vending apply to V3 tables; Arctic commit log provides full audit trail' }
+ },
+ score: 28,
+ description: 'Dremio Cloud (April 2026) provides full Iceberg V3 GA support including Deletion Vectors, VARIANT type, and row lineage; self-managed Dremio v26 V3 support on roadmap'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/duckdb.ts b/src/data/query-engines/duckdb.ts
index 18aae3d6..23c0237a 100644
--- a/src/data/query-engines/duckdb.ts
+++ b/src/data/query-engines/duckdb.ts
@@ -1,7 +1,8 @@
// data/query-engines/duckdb.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const duckdb: QueryEngine = {
+export const duckdb: QueryEngine = createVersionedEngine({
id: 'duckdb',
name: 'DuckDB v1.3+',
description: 'A light-weight, read-only analytics engine for Iceberg with SQL time travel, external file caching, and REST catalog support',
@@ -11,12 +12,8 @@ export const duckdb: QueryEngine = {
features: {
catalogs: {
support: 'partial',
- details: 'Hadoop (file-system) and Iceberg REST catalogs supported via rest option with bearer/OAuth tokens; no native Hive/Glue catalog yet',
+ details: 'REST catalog, AWS Glue (via ENDPOINT_TYPE=glue), and Polaris supported for V3 tables with V2-compatible data types. Hive Metastore, Nessie, Hadoop, JDBC not supported',
externalLinks: [
- {
- label: 'Iceberg Extension Overview',
- url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/overview.html'
- },
{
label: 'Iceberg REST Catalogs',
url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/iceberg_rest_catalogs.html'
@@ -24,36 +21,28 @@ export const duckdb: QueryEngine = {
]
},
readWrite: {
- support: 'readonly',
- details: 'Full SELECT support with predicate evaluation, manifest pruning and external file-cache to avoid re-downloading S3/GCS objects; write operations not available',
+ support: 'partial',
+ details: 'DuckDB can read V3 tables that use only V2-compatible data types. INSERT into V3 tables supported; UPDATE/DELETE on V3 not yet documented',
externalLinks: [
{
label: 'Iceberg Extension Overview',
url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/overview.html'
- },
- {
- label: 'Troubleshooting - Write Limitations',
- url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/troubleshooting.html'
}
]
},
dml: {
support: 'none',
- details: 'iceberg_scan() and CREATE VIEW for reads; metadata helper functions available; no INSERT/UPDATE/DELETE/MERGE operations',
+ details: 'MERGE INTO not supported on any version. UPDATE/DELETE on V3 tables not yet documented. INSERT into V3 supported per extension limitations page',
externalLinks: [
{
- label: 'GitHub - duckdb-iceberg README',
- url: 'https://github.com/duckdb/duckdb-iceberg'
- },
- {
- label: 'Troubleshooting - Writing Not Supported',
+ label: 'Troubleshooting - Current Limitations',
url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/troubleshooting.html'
}
]
},
morCow: {
- support: 'limited',
- details: 'Reading tables with deletes is not yet supported; only Copy-on-Write tables without delete files can be read',
+ support: 'partial',
+ details: 'V3 tables with position deletes can be read if only V2-compatible data types are used. Full MoR/CoW behavior on V3 not yet documented',
externalLinks: [
{
label: 'Troubleshooting - Delete Limitations',
@@ -72,13 +61,9 @@ export const duckdb: QueryEngine = {
]
},
formatV3: {
- support: 'none',
- details: 'DuckDB 1.3 only reads v1 & v2 tables; v3 metadata changes will be evaluated post-GA of the spec',
+ support: 'partial',
+ details: 'DuckDB can read V3 tables that use only V2-compatible data types. V3-only data types (nanosecond timestamps, geometry, vector, shredded variant) are not supported',
externalLinks: [
- {
- label: 'Iceberg Extension Overview',
- url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/overview.html'
- },
{
label: 'Troubleshooting - Current Limitations',
url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/troubleshooting.html'
@@ -86,8 +71,8 @@ export const duckdb: QueryEngine = {
]
},
timeTravel: {
- support: 'full',
- details: 'Convenient SQL syntax: SELECT * FROM tbl AT (VERSION => 314159) or AT (TIMESTAMP => \'2025-05-01 10:15:00\'); older function-style still works',
+ support: 'partial',
+ details: 'Time travel on V3 tables likely works for V2-compatible data types via AT (VERSION => snapshot_id) and AT (TIMESTAMP => ts) syntax, but not explicitly documented for V3',
externalLinks: [
{
label: 'Iceberg Extension Overview',
@@ -96,13 +81,9 @@ export const duckdb: QueryEngine = {
]
},
security: {
- support: 'basic',
- details: 'Uses DuckDB\'s standard S3/Azure creds in httpfs extension; REST-catalog tokens may be supplied per-session; no built-in RBAC/row-masking',
+ support: 'partial',
+ details: 'Uses DuckDB\'s standard S3/Azure credentials in httpfs extension; REST-catalog OAuth2 tokens supported since v1.3+; no built-in RBAC or row-level masking',
externalLinks: [
- {
- label: 'S3 Iceberg Import',
- url: 'https://duckdb.org/docs/stable/guides/network_cloud_storage/s3_iceberg_import.html'
- },
{
label: 'Iceberg REST Catalogs Authentication',
url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/iceberg_rest_catalogs.html'
@@ -128,6 +109,64 @@ CREATE SECRET iceberg_rest (
-- Time travel query
SELECT * FROM iceberg_scan('/bucket/table/')
AT (TIMESTAMP => '2025-05-01 10:15:00');`,
+ versions: {
+ v2: {
+ features: {
+ catalogs: {
+ support: 'full',
+ details: 'Supports attaching to Iceberg REST Catalogs (OAuth2 since v1.3+), AWS Glue via ENDPOINT_TYPE=glue, Polaris, and S3 Tables. Only REST-based catalogs supported — no Hive Metastore, Hadoop, Nessie, or JDBC',
+ externalLinks: [
+ { label: 'Iceberg REST Catalogs', url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/iceberg_rest_catalogs.html' }
+ ]
+ },
+ readWrite: {
+ support: 'full',
+ details: 'Full read support for V1 and V2 tables via the iceberg extension with predicate push-down, manifest pruning, and external file cache. INSERT INTO supported since v1.4.0 via REST catalog attachment',
+ externalLinks: [
+ { label: 'Iceberg Extension Overview', url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/overview.html' }
+ ]
+ },
+ dml: {
+ support: 'partial',
+ details: 'UPDATE and DELETE supported since v1.4.2; MERGE INTO and ALTER TABLE are not supported. Requires REST catalog attachment for write operations',
+ externalLinks: [
+ { label: 'Troubleshooting - Current Limitations', url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/troubleshooting.html' }
+ ]
+ },
+ morCow: {
+ support: 'full',
+ details: 'Full MoR semantics: UPDATE/DELETE use positional deletes (MoR). INSERT uses COW semantics. Supports reading tables with position deletes and equality deletes',
+ externalLinks: [
+ { label: 'Iceberg Extension Overview', url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/overview.html' }
+ ]
+ },
+ streaming: {
+ support: 'none',
+ details: 'Batch-only analytics engine; no built-in streaming ingestion or CDC subscribe APIs'
+ },
+ formatV3: {
+ support: 'none',
+ details: 'Iceberg Format V3 features are not applicable to V2 tables.'
+ },
+ timeTravel: {
+ support: 'full',
+ details: 'SQL time travel via AT (VERSION => snapshot_id) and AT (TIMESTAMP => ts) syntax. Older iceberg_scan() function-style parameters still work',
+ externalLinks: [
+ { label: 'Iceberg Extension Overview', url: 'https://duckdb.org/docs/stable/core_extensions/iceberg/overview.html' }
+ ]
+ },
+ security: {
+ support: 'partial',
+ details: 'Standard S3/Azure credentials via httpfs extension; REST-catalog OAuth2 tokens supported since v1.3+; no built-in RBAC or row-level masking',
+ externalLinks: [
+ { label: 'S3 Iceberg Import', url: 'https://duckdb.org/docs/stable/guides/network_cloud_storage/s3_iceberg_import.html' }
+ ]
+ }
+ },
+ score: 20,
+ description: 'DuckDB v1.4+ supports full Iceberg V2 read/write operations with REST catalog, time travel, MoR/CoW semantics, and partial DML (UPDATE/DELETE but no MERGE)'
+ }
+ },
bestPractices: [
'Use DuckDB v1.3.0 or later for the built-in Iceberg extension',
'Configure external file-cache via SET s3_cache_size=\'4GB\'; to halve cold-scan latency',
@@ -146,4 +185,4 @@ AT (TIMESTAMP => '2025-05-01 10:15:00');`,
'Use object-store IAM plus catalog ACLs for security and governance',
'Rely on cost-based optimization improvements in 1.3 for better query planning'
]
-};
\ No newline at end of file
+});
\ No newline at end of file
diff --git a/src/data/query-engines/flink.ts b/src/data/query-engines/flink.ts
index 9106d46f..11a94583 100644
--- a/src/data/query-engines/flink.ts
+++ b/src/data/query-engines/flink.ts
@@ -1,7 +1,8 @@
// data/query-engines/flink.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const flink: QueryEngine = {
+export const flink: QueryEngine = createVersionedEngine({
id: 'flink',
name: 'Apache Flink 1.18+',
description: 'The reference implementation for CDC to Iceberg with comprehensive streaming support, exactly-once semantics, and advanced FLIP-27 incremental reads',
@@ -184,5 +185,21 @@ CREATE TABLE iceberg_catalog.db.events (
'Run maintenance actions (rewrite_data_files) as separate Flink batch jobs',
'Monitor Flink job IDs in Iceberg snapshot summaries for troubleshooting',
'Use external tools for schema changes (ADD/RENAME columns) due to Flink DDL limitations'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v2: {
+ features: {
+ catalogs: { support: 'full', details: 'Hive Metastore, Hadoop catalog, REST catalog (incl. Nessie), AWS Glue, JDBC, plus any custom implementation via catalog-impl' },
+ readWrite: { support: 'full', details: 'Batch and streaming jobs read V2 snapshots or incremental DataStreams; Iceberg Sink commits on each Flink checkpoint with exactly-once semantics' },
+ dml: { support: 'partial', details: 'INSERT append always available; row-level upserts via write.upsert.enabled=true on V2 tables, emitting V2 equality-delete files; MERGE INTO not supported in Flink SQL' },
+ morCow: { support: 'full', details: 'Copy-on-Write for batch rewrites; Merge-on-Read for streaming/upsert with V2 position/equality delete files instead of partition rewrites' },
+ streaming: { support: 'full', details: 'Reference engine for CDC → Iceberg V2: consume Debezium/Kafka changelogs, upsert with exactly-once semantics, FLIP-27 incremental reads' },
+ formatV3: { support: 'none', details: 'Iceberg Format V3 features are not applicable to V2 tables.' },
+ timeTravel: { support: 'full', details: 'Point-in-time reads via source options: start-snapshot-id, start-snapshot-timestamp, branch, tag; filter push-down and partition pruning automatic' },
+ security: { support: 'full', details: 'Inherits ACLs from underlying catalog (Hive Ranger, AWS IAM, Nessie authorization); REST catalog secured with credential/token properties' }
+ },
+ score: 26,
+ description: 'Flink provides the reference CDC→Iceberg V2 implementation with exactly-once streaming semantics, full MoR/CoW via delete files, and comprehensive catalog support'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/hive.ts b/src/data/query-engines/hive.ts
index 271f0ea2..37469840 100644
--- a/src/data/query-engines/hive.ts
+++ b/src/data/query-engines/hive.ts
@@ -1,7 +1,8 @@
// data/query-engines/hive.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const hive: QueryEngine = {
+export const hive: QueryEngine = createVersionedEngine({
id: 'hive',
name: 'Apache Hive 4.0+',
description: 'Traditional data warehouse with first-class Iceberg support, full SQL DML, hidden partitioning, and Ranger-based governance for batch analytics',
@@ -98,12 +99,16 @@ export const hive: QueryEngine = {
]
},
timeTravel: {
- support: 'partial',
- details: 'Hidden partitioning supported (PARTITIONED BY SPEC); time-travel via snapshot/branch properties, not SQL clauses',
+ support: 'full',
+ details: 'FOR SYSTEM_TIME AS OF and FOR SYSTEM_VERSION AS OF SQL clauses supported in Hive 4.0+; hidden partitioning (PARTITIONED BY SPEC) also available',
externalLinks: [
{
- label: 'Branching and Tagging',
- url: 'https://iceberg.apache.org/docs/1.8.1/branching/'
+ label: 'Hive Time Travel',
+ url: 'https://iceberg.apache.org/docs/latest/hive/'
+ },
+ {
+ label: 'Iceberg Time Travel in CDW',
+ url: 'https://docs.cloudera.com/cdw-runtime/cloud/iceberg-how-to/topics/iceberg-hive-time-travel.html'
}
]
},
@@ -151,5 +156,21 @@ SELECT * FROM iceberg_table WHERE created_date >= '2024-01-01';`,
'Use branch/tag properties for time travel rather than expecting SQL time travel syntax',
'Configure appropriate storage handlers and catalog properties for different deployment scenarios',
'Consider micro-batch processing patterns for near-real-time data ingestion requirements'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v3: {
+ features: {
+ catalogs: { support: 'none', details: 'Hive 4 bundles Iceberg 1.4.3, predating V3 spec; V3 catalog operations not supported' },
+ readWrite: { support: 'none', details: 'Hive cannot read or write Iceberg V3 format tables; requires Iceberg ≥ 1.8.0 for V3 support' },
+ dml: { support: 'none', details: 'DML operations only produce V2 format (CoW); V3 table format not supported' },
+ morCow: { support: 'none', details: 'V3 deletion vectors not readable or writable; Hive uses CoW rewrites only' },
+ streaming: { support: 'none', details: 'No native streaming for any format version' },
+ formatV3: { support: 'none', details: 'Not supported; Hive 4 bundles Iceberg 1.4.3, predating spec v3. Cannot write or reliably read v3 tables until upgrade to Iceberg ≥ 1.8.0' },
+ timeTravel: { support: 'none', details: 'FOR SYSTEM_TIME/VERSION AS OF works for V1/V2 tables; V3-format tables not supported in Hive 4.0.x (Iceberg 1.4.3)' },
+ security: { support: 'none', details: 'V3 format not supported; Ranger/SQL-standard policies apply to V1/V2 tables only' }
+ },
+ score: 0,
+ description: 'Apache Hive 4.0 (Iceberg 1.4.3) does not support Iceberg V3 format tables; upgrade to Iceberg ≥ 1.8.0 required for V3 support'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/impala.ts b/src/data/query-engines/impala.ts
index 1e21a734..39d556c1 100644
--- a/src/data/query-engines/impala.ts
+++ b/src/data/query-engines/impala.ts
@@ -1,7 +1,8 @@
// data/query-engines/impala.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const impala: QueryEngine = {
+export const impala: QueryEngine = createVersionedEngine({
id: 'impala',
name: 'Apache Impala v4.4+',
description: 'High-performance analytics engine with Iceberg v2 support, row-level operations via position deletes, and deep HMS integration for enterprise environments',
@@ -31,7 +32,7 @@ export const impala: QueryEngine = {
},
dml: {
support: 'partial',
- details: 'INSERT INTO & INSERT OVERWRITE, DELETE (v2 position-delete files), UPDATE (v2 position deletes); MERGE planned/preview in CDW 1.5.5',
+ details: 'INSERT INTO & INSERT OVERWRITE, DELETE (v2 position-delete files), UPDATE (v2 position deletes); MERGE added in Impala 4.5 (previously CDW-only preview); equality-delete MERGE planned for 5.0.0+',
externalLinks: [
{
label: 'Iceberg V2 Tables - Impala',
@@ -138,5 +139,21 @@ WHERE id = 123;`,
'Consider schema evolution limitations on complex types when designing table schemas',
'Monitor manifest cache effectiveness and tune cache settings appropriately',
'Use snapshot isolation guarantees for consistent read operations'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v3: {
+ features: {
+ catalogs: { support: 'none', details: 'Impala supports only HiveCatalog and HadoopCatalog with V1/V2 tables; V3 format not supported' },
+ readWrite: { support: 'none', details: 'Cannot read or write Iceberg V3 format tables; spec v1/v2 only' },
+ dml: { support: 'none', details: 'DML (DELETE, UPDATE, MERGE preview) produces V2 position-delete files only; V3 not supported' },
+ morCow: { support: 'none', details: 'V3 deletion vectors not supported; only V2 position-delete files via MoR; equality deletes not supported in any version' },
+ streaming: { support: 'none', details: 'No built-in streaming ingestion for any format version' },
+ formatV3: { support: 'none', details: 'Supports spec v1 and v2 only; spec v3 features like deletion vectors, row lineage, and new catalog RPCs not supported' },
+ timeTravel: { support: 'none', details: 'Time travel (FOR SYSTEM_TIME/VERSION AS OF) only for V1/V2 format tables' },
+ security: { support: 'none', details: 'V3 format not supported; Ranger/HMS security applies to V1/V2 tables only' }
+ },
+ score: 0,
+ description: 'Apache Impala v4.4+ supports Iceberg spec V1/V2 only; Format V3 (deletion vectors, row lineage, new data types) not yet supported'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/index.ts b/src/data/query-engines/index.ts
index 8b759618..979218f4 100644
--- a/src/data/query-engines/index.ts
+++ b/src/data/query-engines/index.ts
@@ -124,7 +124,7 @@ function validateAllEngines(engines: QueryEngine[]): void {
* All available query engines
* Add new engines here after creating their data files
*/
-export const QUERY_ENGINES: QueryEngine[] = [
+export const engines: QueryEngine[] = [
spark,
flink,
hive,
@@ -145,9 +145,12 @@ export const QUERY_ENGINES: QueryEngine[] = [
// Validate all engines in development
if (process.env.NODE_ENV === 'development') {
- validateAllEngines(QUERY_ENGINES);
+ validateAllEngines(engines);
}
+// Backward-compatible alias
+export const QUERY_ENGINES = engines;
+
// Export individual engines for direct access
export {
spark,
@@ -173,18 +176,18 @@ export {
*/
export const getEngineById = (id: string): QueryEngine | undefined => {
- return QUERY_ENGINES.find(engine => engine.id === id);
+ return engines.find(engine => engine.id === id);
};
export const getEnginesByCategory = (category: QueryEngine['category']): QueryEngine[] => {
- return QUERY_ENGINES.filter(engine => engine.category === category);
+ return engines.filter(engine => engine.category === category);
};
export const getEnginesBySupportLevel = (
feature: keyof QueryEngine['features'],
level: QueryEngine['features'][keyof QueryEngine['features']]['support']
): QueryEngine[] => {
- return QUERY_ENGINES.filter(engine => engine.features[feature].support === level);
+ return engines.filter(engine => engine.features[feature].support === level);
};
export const getEnginesByFeatureSupport = (
@@ -200,7 +203,7 @@ export const getEnginesByFeatureSupport = (
export const searchEngines = (query: string): QueryEngine[] => {
const searchTerm = query.toLowerCase();
- return QUERY_ENGINES.filter(engine =>
+ return engines.filter(engine =>
engine.name.toLowerCase().includes(searchTerm) ||
engine.description.toLowerCase().includes(searchTerm) ||
engine.id.toLowerCase().includes(searchTerm)
@@ -209,20 +212,20 @@ export const searchEngines = (query: string): QueryEngine[] => {
export const getEngineStats = () => {
const stats = {
- total: QUERY_ENGINES.length,
+ total: engines.length,
byCategory: {} as Record,
bySupport: {} as Record>
};
// Count by category
- QUERY_ENGINES.forEach(engine => {
+ engines.forEach(engine => {
stats.byCategory[engine.category] = (stats.byCategory[engine.category] || 0) + 1;
});
// Count by feature support
- Object.keys(QUERY_ENGINES[0]?.features || {}).forEach(feature => {
+ Object.keys(engines[0]?.features || {}).forEach(feature => {
stats.bySupport[feature] = { full: 0, partial: 0, preview: 0, none: 0 };
- QUERY_ENGINES.forEach(engine => {
+ engines.forEach(engine => {
const support = engine.features[feature as keyof QueryEngine['features']].support;
stats.bySupport[feature][support]++;
});
diff --git a/src/data/query-engines/presto.ts b/src/data/query-engines/presto.ts
index d2ae92b1..1ae2725c 100644
--- a/src/data/query-engines/presto.ts
+++ b/src/data/query-engines/presto.ts
@@ -1,7 +1,8 @@
// data/query-engines/presto.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const presto: QueryEngine = {
+export const presto: QueryEngine = createVersionedEngine({
id: 'presto',
name: 'Presto 0.288+',
description: 'Distributed SQL query engine with REST/Nessie catalogs, row-level DELETE, time travel, and configurable MoR/CoW modes for interactive analytics',
@@ -77,7 +78,7 @@ export const presto: QueryEngine = {
},
formatV3: {
support: 'none',
- details: 'Roadmap: read Deletion Vectors & Row Lineage after Iceberg 1.8 libraries land; write DV planned post-0.295. Currently supports v1/v2 only',
+ details: 'Presto 0.296 now bundles Iceberg 1.8.1 (library landed), but V3 format features (DV read/write, row lineage) have not yet shipped. V3 support planned in a future 0.29x release',
externalLinks: [
{
label: 'Format Version Support',
@@ -157,5 +158,21 @@ FOR VERSION AS OF 1234567890;`,
'Monitor Presto logs for audit and security compliance requirements',
'Test experimental features thoroughly before production deployment',
'Plan migration strategy for when MERGE operations become available'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v3: {
+ features: {
+ catalogs: { support: 'none', details: 'Presto only supports Iceberg V1/V2 format tables; V3 format not yet supported' },
+ readWrite: { support: 'none', details: 'Presto cannot read or write V3-format tables; deletion vectors and row lineage not supported' },
+ dml: { support: 'none', details: 'DML outputs V2 format only; MERGE not yet supported in any version; V3 not supported' },
+ morCow: { support: 'none', details: 'V3 deletion vectors not supported; only V2 position/equality delete files via table properties' },
+ streaming: { support: 'none', details: 'Batch-only; no streaming support for any format version' },
+ formatV3: { support: 'none', details: 'Presto 0.296 bundles Iceberg 1.8.1 but V3 format features (deletion vectors, row lineage) have not yet shipped; planned for a future release' },
+ timeTravel: { support: 'none', details: 'Time travel only for V1/V2 format tables; V3-format tables not supported' },
+ security: { support: 'none', details: 'V3 format not supported; security features apply to V1/V2 tables only' }
+ },
+ score: 0,
+ description: 'Presto supports Iceberg V1/V2 tables only; Format V3 (deletion vectors, row lineage) planned post-0.295 after Iceberg 1.8 library adoption'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/snowflake.ts b/src/data/query-engines/snowflake.ts
index cc463f9d..7b4966a5 100644
--- a/src/data/query-engines/snowflake.ts
+++ b/src/data/query-engines/snowflake.ts
@@ -1,7 +1,8 @@
// data/query-engines/snowflake.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const snowflake: QueryEngine = {
+export const snowflake: QueryEngine = createVersionedEngine({
id: 'snowflake',
name: 'Snowflake',
description: 'Enterprise cloud data warehouse with native Iceberg catalog, automatic optimization, Snowpipe Streaming, UniForm interoperability, and full integration with Snowflake features',
@@ -11,7 +12,7 @@ export const snowflake: QueryEngine = {
features: {
catalogs: {
support: 'partial',
- details: 'Snowflake catalog (native) with full read/write. External catalogs (Glue, Open Table Catalog) read-only via catalog integration objects',
+ details: 'Snowflake Horizon Catalog (Polaris) native, REST Catalog, and AWS Glue supported for V3 (Public Preview). Hive Metastore, Nessie, Hadoop, JDBC not supported',
externalLinks: [
{
label: 'CREATE ICEBERG TABLE Documentation',
@@ -24,8 +25,8 @@ export const snowflake: QueryEngine = {
]
},
readWrite: {
- support: 'partial',
- details: 'Full SELECT, DML & DDL on Snowflake-catalog tables including COPY INTO, CTAS, multi-statement transactions. External catalogs read-only',
+ support: 'full',
+ details: 'Full read and write support for Snowflake-managed Iceberg tables including Parquet, COPY INTO, CTAS, and multi-statement transactions. V3 support in Public Preview (March 2026)',
externalLinks: [
{
label: 'Iceberg Table Tutorial',
@@ -38,22 +39,18 @@ export const snowflake: QueryEngine = {
]
},
dml: {
- support: 'partial',
- details: 'INSERT, UPDATE, DELETE, MERGE INTO fully ACID on Snowflake-catalog tables. Position-delete files, equality-delete in preview. External tables read-only',
+ support: 'full',
+ details: 'INSERT, UPDATE, DELETE, MERGE INTO fully ACID on Snowflake-managed Iceberg tables. V3 uses deletion vectors for row-level changes (Public Preview March 2026)',
externalLinks: [
{
label: 'DML Commands with Iceberg',
url: 'https://docs.snowflake.com/en/user-guide/tables-iceberg-manage'
- },
- {
- label: 'Row-level Deletes',
- url: 'https://docs.snowflake.com/en/user-guide/tables-iceberg-manage'
}
]
},
morCow: {
support: 'full',
- details: 'DML writes merge-on-read delete files. Automatic Storage Optimization compacts files & merges delete files, switching to copy-on-write during clustering',
+ details: 'V3 supports full MoR via deletion vectors and CoW. V3 deletion vectors replace position deletes for improved write performance (Public Preview March 2026)',
externalLinks: [
{
label: 'Iceberg Storage Management',
@@ -62,22 +59,18 @@ export const snowflake: QueryEngine = {
]
},
streaming: {
- support: 'partial',
- details: 'Snowpipe Streaming & Storage Write API for real-time ingestion (GA). Streams & Tasks supported on Snowflake-catalog tables. No built-in CDC ingestion',
+ support: 'full',
+ details: 'Snowflake V3 row lineage enables CDC capabilities for data governance and auditing. Snowpipe Streaming for real-time ingestion (Public Preview March 2026)',
externalLinks: [
{
label: 'Snowpipe Streaming with Iceberg',
url: 'https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-iceberg'
- },
- {
- label: 'Introduction to Streams',
- url: 'https://docs.snowflake.com/en/user-guide/streams-intro'
}
]
},
formatV3: {
- support: 'none',
- details: 'Not yet supported. Snowflake-catalog tables use spec v2; external v3 tables readable if future reader upgrades land. Roadmap evaluation ongoing',
+ support: 'partial',
+ details: 'Iceberg V3 support in Public Preview (March 2026). Supports deletion vectors, nanosecond timestamps, geometry type, variant type, and row lineage',
externalLinks: [
{
label: 'Apache Iceberg v3 Table Spec Blog',
@@ -148,6 +141,53 @@ AT(TIME => '2024-01-15 10:00:00');
-- Create zero-copy clone
CREATE ICEBERG TABLE sales_data_backup
CLONE sales_data;`,
+ versions: {
+ v2: {
+ features: {
+ catalogs: {
+ support: 'partial',
+ details: 'Snowflake Horizon Catalog (Polaris) native, REST Catalog, and AWS Glue supported. Hive Metastore, Nessie, Hadoop, JDBC, Unity Catalog (read-only via REST) not fully supported',
+ externalLinks: [{ label: 'Apache Iceberg Tables Overview', url: 'https://docs.snowflake.com/en/user-guide/tables-iceberg' }]
+ },
+ readWrite: {
+ support: 'full',
+ details: 'Full read and write support for Snowflake-managed Iceberg tables. Parquet-only format. COPY INTO, CTAS, multi-statement transactions all supported',
+ externalLinks: [{ label: 'Manage Iceberg Tables', url: 'https://docs.snowflake.com/en/user-guide/tables-iceberg-manage' }]
+ },
+ dml: {
+ support: 'full',
+ details: 'INSERT, UPDATE, DELETE, MERGE INTO fully ACID on Snowflake-managed Iceberg tables. All DML operations use copy-on-write mode',
+ externalLinks: [{ label: 'DML Commands with Iceberg', url: 'https://docs.snowflake.com/en/user-guide/tables-iceberg-manage' }]
+ },
+ morCow: {
+ support: 'partial',
+ details: 'Copy-on-write is the exclusive write strategy for Snowflake-managed tables. Merge-on-read supported only for externally managed Iceberg tables via ENABLE_ICEBERG_MERGE_ON_READ session parameter',
+ externalLinks: [{ label: 'Iceberg Storage Management', url: 'https://docs.snowflake.com/en/user-guide/tables-iceberg-storage' }]
+ },
+ streaming: {
+ support: 'partial',
+ details: 'Snowpipe Streaming & Storage Write API for real-time ingestion (GA). Streams & Tasks supported on Snowflake-catalog tables. No built-in CDC ingestion',
+ externalLinks: [{ label: 'Snowpipe Streaming with Iceberg', url: 'https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-iceberg' }]
+ },
+ formatV3: {
+ support: 'none',
+ details: 'Iceberg Format V3 features are not applicable to V2 tables.'
+ },
+ timeTravel: {
+ support: 'full',
+ details: 'Query snapshots with AT(SNAPSHOT => id) or AT(TIME => ts). Zero-Copy Clones work on Iceberg tables. Full time travel for Snowflake-managed tables',
+ externalLinks: [{ label: 'Understanding Time Travel', url: 'https://docs.snowflake.com/en/user-guide/data-time-travel' }]
+ },
+ security: {
+ support: 'full',
+ details: 'Full Snowflake RBAC, column masking, row-access policies, tag-based masking. Query activity in ACCOUNT_USAGE & ACCESS_HISTORY. Customer-managed IAM roles',
+ externalLinks: [{ label: 'Access Control Privileges', url: 'https://docs.snowflake.com/en/user-guide/security-access-control-privileges' }]
+ }
+ },
+ score: 22,
+ description: 'Snowflake provides complete Iceberg V2 support with full DML operations, automatic optimization, and enterprise-grade security for managed tables'
+ }
+ },
bestPractices: [
'Use Snowflake 8.20+ for GA Iceberg support and latest features',
'Leverage native Snowflake catalog for full DML capabilities and Snowflake feature integration',
@@ -170,4 +210,4 @@ CLONE sales_data;`,
'Use customer-managed IAM roles for secure access to external storage',
'Monitor cross-region egress charges when compute and storage are in different regions'
]
-};
\ No newline at end of file
+});
\ No newline at end of file
diff --git a/src/data/query-engines/spark.ts b/src/data/query-engines/spark.ts
index 56d67315..252a0c3e 100644
--- a/src/data/query-engines/spark.ts
+++ b/src/data/query-engines/spark.ts
@@ -1,7 +1,8 @@
// data/query-engines/spark.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const spark: QueryEngine = {
+export const spark: QueryEngine = createVersionedEngine({
id: 'spark',
name: 'Apache Spark 3.3+',
description: 'The reference implementation for Apache Iceberg with comprehensive read/write support',
@@ -51,5 +52,21 @@ df.writeTo("prod.db.table").append()`,
'Enable adaptive query execution for better performance',
'Use write.distribution.mode for optimized writes',
'Regularly run maintenance procedures (rewrite_data_files, expire_snapshots)'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v2: {
+ features: {
+ catalogs: { support: 'full', details: 'Hive Metastore, Hadoop warehouse, REST, AWS Glue, JDBC, Nessie, plus custom plug-ins via spark.sql.catalog.* settings' },
+ readWrite: { support: 'full', details: 'Full table scans, metadata-table reads, INSERT INTO, atomic INSERT OVERWRITE, DataFrame writeTo, and stored procedures' },
+ dml: { support: 'full', details: 'MERGE INTO, UPDATE, DELETE via Spark Session Extensions; emits V2 position/equality delete files (Iceberg 0.14+)' },
+ morCow: { support: 'full', details: 'Copy-on-Write default; Merge-on-Read enabled via write.delete.mode=merge-on-read, emitting V2 position/equality delete files' },
+ streaming: { support: 'partial', details: 'Incremental reads with stream-from-timestamp; append/complete output modes; delete and overwrite snapshots skipped by streaming readers by default' },
+ formatV3: { support: 'none', details: 'Iceberg Format V3 features are not applicable to V2 tables.' },
+ timeTravel: { support: 'full', details: 'SQL VERSION AS OF / TIMESTAMP AS OF supported since Spark 3.3+; DataFrame as-of-timestamp option' },
+ security: { support: 'full', details: 'Delegates ACLs to underlying catalog (Hive Ranger, AWS IAM, Nessie policies); snapshot isolation; audit metadata' }
+ },
+ score: 26,
+ description: 'Spark provides comprehensive Iceberg V2 support with full DML, MoR/CoW via position/equality delete files, streaming reads, and native SQL time travel'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/starburst.ts b/src/data/query-engines/starburst.ts
index 13390f5f..69c16df6 100644
--- a/src/data/query-engines/starburst.ts
+++ b/src/data/query-engines/starburst.ts
@@ -1,7 +1,8 @@
// data/query-engines/starburst.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const starburst: QueryEngine = {
+export const starburst: QueryEngine = createVersionedEngine({
id: 'starburst',
name: 'Starburst Enterprise SEP 414-E+',
description: 'End-to-end Iceberg analytics platform with comprehensive catalog support, full DML operations, enterprise governance, and advanced optimization features',
@@ -49,7 +50,7 @@ export const starburst: QueryEngine = {
},
morCow: {
support: 'full',
- details: 'Default copy-on-write for large rewrites; fine-grained updates create separate delete files (MoR) merged at query time; handles both position & equality deletes',
+ details: 'Default copy-on-write for large rewrites; V2 fine-grained updates use position/equality delete files (MoR); SEP 476-e+ writes deletion vectors (V3 MoR) when format-version=3',
externalLinks: [
{
label: 'Data Management - Deletion Strategies',
@@ -68,12 +69,16 @@ export const starburst: QueryEngine = {
]
},
formatV3: {
- support: 'preview',
- details: 'Supports Iceberg spec v1 & v2; can read v3 preview metadata under feature flag but no v3 writes; production v3 GA on roadmap for 2025',
+ support: 'full',
+ details: 'Full Iceberg V3 read+write GA in SEP 476-e (September 2025); deletion vectors (MoR writes), VARIANT type, nanosecond timestamps, and row lineage all supported',
externalLinks: [
{
- label: 'Iceberg Connector - Spec Support',
- url: 'https://docs.starburst.io/latest/connector/iceberg.html'
+ label: 'SEP 476-e Release Notes',
+ url: 'https://docs.starburst.io/latest/release/release-476-e.html'
+ },
+ {
+ label: 'Iceberg V3 in Starburst Blog',
+ url: 'https://www.starburst.io/blog/iceberg-v3/'
}
]
},
@@ -137,5 +142,21 @@ FOR TIMESTAMP AS OF TIMESTAMP '2025-01-01 00:00:00';`,
'Leverage $snapshots, $history, $manifests metadata tables for table introspection',
'Configure appropriate data file formats (Parquet default, ORC, Avro) and codecs',
'Use rollback_to_snapshot, expire_snapshots, remove_orphan_files procedures for maintenance'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v3: {
+ features: {
+ catalogs: { support: 'full', details: 'All catalog types (HMS, Glue, REST, Nessie, Snowflake, Galaxy) accessible; V3 table creation fully supported in SEP 476-e+' },
+ readWrite: { support: 'full', details: 'Full V3 read+write GA in SEP 476-e (September 2025); creates and queries V3-format Iceberg tables natively' },
+ dml: { support: 'full', details: 'INSERT, UPDATE, DELETE, MERGE with V3 deletion vectors for MoR writes; partition-aligned predicates become partition deletes' },
+ morCow: { support: 'full', details: 'Deletion vectors (V3 MoR) written for fine-grained updates; copy-on-write still available for large partition rewrites' },
+ streaming: { support: 'none', details: 'No built-in streaming ingestion for any format version' },
+ formatV3: { support: 'full', details: 'Full Iceberg V3 GA in SEP 476-e (September 2025); deletion vectors, VARIANT type, nanosecond timestamps, row lineage all supported' },
+ timeTravel: { support: 'full', details: 'FOR VERSION AS OF / FOR TIMESTAMP AS OF fully work on V3 tables; metadata tables expose row lineage columns' },
+ security: { support: 'full', details: 'Built-in ACL engine, LDAP/OAuth, Lake Formation, Ranger policies; row lineage (_row_id) visible for audit purposes' }
+ },
+ score: 28,
+ description: 'Starburst Enterprise SEP 476-e+ provides full Iceberg V3 read+write GA support including deletion vectors, VARIANT type, nanosecond timestamps, and row lineage (since September 2025)'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/starrocks.ts b/src/data/query-engines/starrocks.ts
index ab452d00..ff1b4144 100644
--- a/src/data/query-engines/starrocks.ts
+++ b/src/data/query-engines/starrocks.ts
@@ -1,7 +1,8 @@
// data/query-engines/starrocks.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const starrocks: QueryEngine = {
+export const starrocks: QueryEngine = createVersionedEngine({
id: 'starrocks',
name: 'StarRocks v3.2/3.3',
description: 'Vectorized OLAP engine with read-write Iceberg support, async materialized views, CBO optimization, and strong analytical performance for lakehouse analytics',
@@ -149,5 +150,21 @@ GROUP BY 1;`,
'Configure appropriate row-group size and page size for Parquet write performance',
'Plan migration path for when UPDATE/DELETE/MERGE operations become available',
'Consider StarRocks as primary query engine in lakehouse architecture with other engines for writes'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v3: {
+ features: {
+ catalogs: { support: 'none', details: 'StarRocks v3.2/3.3 supports only Iceberg V1/V2 format tables; V3 format not yet supported' },
+ readWrite: { support: 'none', details: 'Cannot read or write Iceberg V3 format tables; V1/V2 Parquet & ORC only' },
+ dml: { support: 'none', details: 'INSERT/INSERT OVERWRITE available for V2 tables only; V3 format not supported' },
+ morCow: { support: 'none', details: 'V3 deletion vectors not supported; reads V2 MoR (position & equality deletes) only' },
+ streaming: { support: 'none', details: 'No native streaming for any format version' },
+ formatV3: { support: 'none', details: 'Not yet GA; supports V1/V2 (Parquet & ORC) only; V3 support on roadmap' },
+ timeTravel: { support: 'none', details: 'Time travel (v3.4+) only for V1/V2 format tables; V3-format tables not supported' },
+ security: { support: 'none', details: 'V3 format not supported; StarRocks RBAC and catalog ACLs apply to V1/V2 tables only' }
+ },
+ score: 0,
+ description: 'StarRocks v3.2/3.3 supports Iceberg V1/V2 tables only; Format V3 (deletion vectors, row lineage, new data types) not yet supported'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/trino.ts b/src/data/query-engines/trino.ts
index 24e433b6..002db6a0 100644
--- a/src/data/query-engines/trino.ts
+++ b/src/data/query-engines/trino.ts
@@ -1,7 +1,8 @@
// data/query-engines/trino.ts
import { QueryEngine } from '../../types/iceberg';
+import { createVersionedEngine } from './versioning';
-export const trino: QueryEngine = {
+export const trino: QueryEngine = createVersionedEngine({
id: 'trino',
name: 'Trino 475+',
description: 'High-performance distributed SQL query engine with advanced DML, time travel, and native Iceberg optimization for interactive analytics',
@@ -152,5 +153,21 @@ FOR TIMESTAMP AS OF TIMESTAMP '2024-01-01 12:00:00';`,
'Use hidden partition transforms for automatic partition pruning without explicit WHERE clauses',
'Configure security delegation to underlying catalog systems (Ranger, IAM, Nessie policies)',
'Be aware that proliferation of small files can degrade performance - optimize regularly'
- ]
-};
\ No newline at end of file
+ ],
+ versions: {
+ v3: {
+ features: {
+ catalogs: { support: 'none', details: 'Trino only supports Iceberg V1/V2 format tables; V3 format not yet supported' },
+ readWrite: { support: 'none', details: 'Trino cannot read or write V3-format tables with deletion vectors or row lineage; V1/V2 only' },
+ dml: { support: 'none', details: 'All DML operations produce V2 format outputs only; V3 table format not supported' },
+ morCow: { support: 'none', details: 'V3 deletion vectors not supported; Trino only handles V2 position/equality delete files' },
+ streaming: { support: 'none', details: 'No streaming support for any format version' },
+ formatV3: { support: 'none', details: 'Not yet GA; connector supports spec v1/v2 only; deletion vectors & row lineage planned post-Iceberg 1.8 library update' },
+ timeTravel: { support: 'none', details: 'Time travel only available for V1/V2 format tables; V3-format tables not supported' },
+ security: { support: 'none', details: 'V3 format not supported; security features apply to V1/V2 tables only' }
+ },
+ score: 0,
+ description: 'Trino supports Iceberg V1/V2 tables only; Format V3 (deletion vectors, row lineage, new data types) not yet supported'
+ }
+ }
+});
\ No newline at end of file
diff --git a/src/data/query-engines/versioning.ts b/src/data/query-engines/versioning.ts
new file mode 100644
index 00000000..b5926bb2
--- /dev/null
+++ b/src/data/query-engines/versioning.ts
@@ -0,0 +1,45 @@
+import { EngineVersionData, QueryEngine } from '../../types/iceberg';
+import { SUPPORT_WEIGHTS } from '../constants/features';
+
+type LegacyEngine = Omit & {
+ versions?: QueryEngine['versions'];
+};
+
+const calculateScore = (features: QueryEngine['features']): number =>
+ Object.values(features).reduce((sum, feature) => sum + (SUPPORT_WEIGHTS[feature?.support] ?? 0), 0);
+
+const createV2FromFeatures = (features: QueryEngine['features'], engineDescription: string): EngineVersionData => {
+ const v2Features: QueryEngine['features'] = {
+ ...features,
+ formatV3: {
+ support: 'none',
+ details: 'Iceberg Format V3 features are not applicable to V2 tables.',
+ externalLinks: features.formatV3?.externalLinks ?? []
+ }
+ };
+
+ return {
+ features: v2Features,
+ score: calculateScore(v2Features),
+ description: engineDescription
+ };
+};
+
+export const createVersionedEngine = (engine: LegacyEngine): QueryEngine => {
+ const legacyV3: EngineVersionData = {
+ features: engine.features,
+ score: calculateScore(engine.features),
+ description: engine.description
+ };
+
+ const v2 = engine.versions?.v2 ?? createV2FromFeatures(engine.features, engine.description);
+ const v3 = engine.versions?.v3 ?? legacyV3;
+
+ return {
+ ...engine,
+ versions: {
+ v2,
+ v3
+ }
+ };
+};
diff --git a/src/types/iceberg.ts b/src/types/iceberg.ts
index 446cf727..463744ce 100644
--- a/src/types/iceberg.ts
+++ b/src/types/iceberg.ts
@@ -12,6 +12,14 @@ export interface Feature {
externalLinks?: ExternalLink[];
}
+export type EngineVersion = 'v2' | 'v3';
+
+export interface EngineVersionData {
+ features: QueryEngine['features'];
+ score?: number | null;
+ description?: string | null;
+}
+
export interface QueryEngine {
id: string;
name: string;
@@ -29,6 +37,10 @@ export interface QueryEngine {
timeTravel: Feature;
security: Feature;
};
+ versions: {
+ v2?: EngineVersionData | null;
+ v3?: EngineVersionData | null;
+ };
quickStart: string;
bestPractices: string[];
}
@@ -38,4 +50,11 @@ export interface FilterOptions {
category: QueryEngine['category'] | 'all';
}
-export type ViewType = 'table' | 'cards' | 'features';
\ No newline at end of file
+export type ViewType = 'table' | 'cards' | 'features';
+
+export interface EngineVersionSelection {
+ engine: string;
+ version: EngineVersion;
+}
+
+export type VersionMode = 'v2' | 'v3';
\ No newline at end of file