Skip to content

Update query engine table to support version-based switching #403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
anshika-oss wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Update query engine table to support version-based switching #403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 94 additions & 100 deletions docs-iceberg-query-engine/clickhouse.mdx
View file
Open in desktop

Large diffs are not rendered by default.

727 changes: 328 additions & 399 deletions docs-iceberg-query-engine/databricks.mdx
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions docs-iceberg-query-engine/doris.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ export const dorisFeatures = [
description: "Full SELECT and write-back: INSERT, INSERT OVERWRITE, CTAS. Doris writes Parquet/ORC files and commits Iceberg snapshots as lake-ingestion engine and analytics layer",
icon: <ServerStackIcon className="w-6 h-6" />,
color: "green",
score: 90,
score: 100,
details: {
title: "Comprehensive Lake Ingestion and Analytics",
description: "Doris serves as both a high-performance analytics engine and a capable lake ingestion system with full Iceberg write capabilities.",
Expand Down Expand Up @@ -141,7 +141,7 @@ export const dorisFeatures = [
description: "INSERT INTO (append), INSERT OVERWRITE, UPDATE & DELETE via Iceberg-v2 delete files (v2.1+). MERGE not yet single statement but emulated with patterns",
icon: <CodeBracketIcon className="w-6 h-6" />,
color: "orange",
score: 80,
score: 60,
details: {
title: "Progressive DML Implementation with Workarounds",
description: "Doris provides essential DML operations with UPDATE/DELETE support via delete files, though MERGE operations require pattern-based workarounds.",
Expand Down Expand Up @@ -198,7 +198,7 @@ export const dorisFeatures = [
description: "Reads: applies position and equality delete files (MoR) automatically. Writes: generates position/equality delete files for UPDATE/DELETE; INSERT OVERWRITE rewrites files (CoW)",
icon: <CubeIcon className="w-6 h-6" />,
color: "green",
score: 95,
score: 100,
details: {
title: "Comprehensive Storage Strategy Implementation",
description: "Doris provides complete support for both Merge-on-Read and Copy-on-Write strategies with intelligent application based on operation type.",
Expand Down Expand Up @@ -254,7 +254,7 @@ export const dorisFeatures = [
description: "No native streaming/CDC writer; use external tools (Flink-Iceberg) to land data, then query with sub-second latency. Routine Load only targets internal tables",
icon: <BoltIcon className="w-6 h-6" />,
color: "red",
score: 10,
score: 0,
details: {
title: "Batch-Focused with External Streaming Integration",
description: "Doris focuses on high-performance batch and interactive analytics while relying on external tools for streaming data ingestion.",
Expand Down Expand Up @@ -309,7 +309,7 @@ export const dorisFeatures = [
description: "Reads & writes Parquet (v1/v2) + ORC (v1/v2). Supports Iceberg spec v1 & v2; equality-delete support for ORC arrives in v2.1.3+. No v3 support yet",
icon: <CloudIcon className="w-6 h-6" />,
color: "orange",
score: 70,
score: 0,
details: {
title: "Comprehensive Format Support for Current Standards",
description: "Doris provides solid support for current Iceberg specifications and major file formats, with ongoing development for newer features.",
Expand Down Expand Up @@ -420,7 +420,7 @@ export const dorisFeatures = [
description: "Doris RBAC plus underlying catalog/storage IAM. Ranger/Lake Formation policies apply at metastore/storage; Doris adds row-policies & column masking on query",
icon: <ShieldCheckIcon className="w-6 h-6" />,
color: "orange",
score: 85,
score: 55,
details: {
title: "Multi-Layer Security Architecture",
description: "Doris provides a comprehensive security model combining its own RBAC with underlying catalog security systems and additional query-time policies.",
Expand Down Expand Up @@ -842,14 +842,14 @@ export const dorisUseCases = [
tableData={dorisTableData}
useCases={dorisUseCases}
officialDocs="https://doris.apache.org/docs/"
gettingStarted="https://doris.apache.org/docs/lakehouse/datalake-analytics/iceberg"
gettingStarted="https://doris.apache.org/docs/lakehouse/catalogs/iceberg-catalog"
additionalResources={[
{ label: "Iceberg Catalog Documentation", url: "https://doris.apache.org/docs/lakehouse/datalake-analytics/iceberg" },
{ label: "Iceberg Data Building Guide", url: "https://doris.apache.org/docs/lakehouse/datalake-building/iceberg-build" },
{ label: "Iceberg Catalog Documentation", url: "https://doris.apache.org/docs/lakehouse/catalogs/iceberg-catalog" },
{ label: "Iceberg Data Building Guide", url: "https://doris.apache.org/docs/dev/lakehouse/best-practices/doris-iceberg" },
{ label: "Iceberg Catalog Configuration", url: "https://doris.apache.org/docs/dev/lakehouse/catalogs/iceberg-catalog" },
{ label: "ICEBERG_META Function", url: "https://doris.apache.org/docs/sql-manual/sql-functions/table-valued-functions/iceberg-meta" },
{ label: "Doris and Iceberg Best Practices", url: "https://doris.apache.org/docs/dev/lakehouse/best-practices/doris-iceberg" },
{ label: "Metadata Cache Documentation", url: "https://doris.apache.org/docs/lakehouse/metacache" },
{ label: "Metadata Cache Documentation", url: "https://doris.apache.org/docs/lakehouse/meta-cache" },
{ label: "Built-in Authorization", url: "https://doris.apache.org/docs/admin-manual/auth/authorization/internal" },
{ label: "Next-Generation Data Lakehouse", url: "https://dev.to/apachedoris/building-the-next-generation-data-lakehouse-10x-performance-4ga1" }
]}
Expand Down
68 changes: 34 additions & 34 deletions docs-iceberg-query-engine/dreamio.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,56 +307,56 @@ export const dremioFeatures = [
}
},
{
title: "Format V3 Roadmap",
chip: "2025 Planned",
description: "Planned (2025) - roadmap calls for reading Deletion Vectors & row-lineage columns first; writer support (DV emission) to follow once Iceberg 1.8+ library adopted",
title: "Iceberg Format V3 GA",
chip: "GA — Dremio Cloud (Apr 2026)",
description: "Full Iceberg V3 GA for Dremio Cloud (April 2026) — deletion vectors, VARIANT type, row lineage read+write; on-premises (Software) V3 timeline TBD",
icon: <CloudIcon className="w-6 h-6" />,
color: "red",
score: 10,
color: "green",
score: 80,
details: {
title: "Clear V3 Development Timeline",
description: "Dremio has a well-defined roadmap for Iceberg format v3 support with phased implementation approach.",
title: "Iceberg V3 General Availability on Dremio Cloud",
description: "Dremio announced Iceberg V3 GA for Dremio Cloud in April 2026, delivering deletion vectors, VARIANT data type, and row lineage. Dremio Software (on-premises) V3 timeline is TBD.",
overviewContent: {
strengths: [
"Clear 2025 timeline for v3 support",
"Phased approach: read first, write later",
"Strong v1/v2 format stability",
"Alignment with Iceberg 1.8+ library adoption"
"Full V3 GA on Dremio Cloud as of April 2026",
"Deletion vector read+write support",
"VARIANT semi-structured data type support",
"Row lineage metadata columns available"
],
limitations: [
"No current v3 format support",
"Cannot read deletion vectors",
"Missing row lineage capabilities",
"Timeline dependent on Iceberg library updates"
"V3 GA applies to Dremio Cloud only; Software (on-premises) timeline TBD",
"Some V3 advanced features may roll out incrementally",
"Older Dremio Software deployments remain on V1/V2"
],
bestFor: [
"Current v1/v2 production environments",
"Teams planning long-term v3 migration",
"Stable data lakehouse deployments",
"Organizations requiring format compatibility"
"Dremio Cloud environments needing V3 efficiency",
"Workloads with heavy row-level updates benefiting from deletion vectors",
"Semi-structured data using VARIANT columns",
"Organizations adopting the latest Iceberg spec"
]
},
technicalSpecs: [
{
category: "Current Support",
category: "V3 Features (Dremio Cloud GA)",
items: [
{ label: "Iceberg v1", value: "Full Support", status: "available" },
{ label: "Iceberg v2", value: "Read/Write", status: "available" },
{ label: "Position Deletes", value: "Supported", status: "available" },
{ label: "Equality Deletes", value: "Partial (not global)", status: "available" }
{ label: "Deletion Vectors", value: "Read + Write", status: "available" },
{ label: "VARIANT Type", value: "Supported", status: "available" },
{ label: "Row Lineage", value: "Supported", status: "available" },
{ label: "GA Date", value: "April 2026", status: "available" }
]
},
{
category: "V3 Roadmap (2025)",
category: "Legacy Format Support",
items: [
{ label: "Deletion Vector Read", value: "Phase 1", status: "limited" },
{ label: "Row Lineage Read", value: "Phase 1", status: "limited" },
{ label: "DV Write Support", value: "Phase 2", status: "limited" },
{ label: "Library Adoption", value: "Iceberg 1.8+", status: "limited" }
{ label: "Iceberg v1", value: "Full Support", status: "available" },
{ label: "Iceberg v2", value: "Read/Write", status: "available" },
{ label: "Position Deletes", value: "Supported", status: "available" },
{ label: "Dremio Software V3", value: "Timeline TBD", status: "limited" }
]
}
],
externalLinks: [
{ label: "Dremio V3 GA Announcement", url: "https://www.globenewswire.com/news-release/2026/04/06/3055888/0/en/Dremio-Announces-General-Availability-of-Apache-Iceberg-V3.html", type: "docs" },
{ label: "What's New in Iceberg v3?", url: "https://www.dremio.com/blog/apache-iceberg-v3/", type: "docs" },
{ label: "Table Properties Format Version", url: "https://docs.dremio.com/current/sonar/query-manage/data-formats/apache-iceberg/table-properties", type: "docs" }
]
Expand Down Expand Up @@ -695,14 +695,14 @@ export const dremioTableData = {
value: <span className="font-medium">Format Support</span>
},
support: {
value: <span className="text-red-600 dark:text-red-400 font-semibold">Limited</span>,
badge: { text: "v1/v2 Only", variant: "error" }
value: <span className="text-green-600 dark:text-green-400 font-semibold">Full</span>,
badge: { text: "V3 GA — Dremio Cloud (Apr 2026)", variant: "success" }
},
details: {
value: "Reads/writes v1/v2; v3 planned 2025 (read DV/lineage first, write DV after Iceberg 1.8+)",
tooltip: "Clear v3 roadmap with phased implementation approach"
value: "Reads/writes v1/v2 fully; Iceberg V3 GA for Dremio Cloud (April 2026): deletion vectors, VARIANT type, row lineage; Software (on-premises) V3 timeline TBD",
tooltip: "V3 GA launched April 2026 for Dremio Cloud; on-premises users remain on V2 until further release"
},
version: { value: "v25+" }
version: { value: "Cloud Apr 2026" }
},
{
dimension: {
Expand Down
Loading
Loading