Add format argument to write_file and add file format types to DuckDB library (#14427)

- Fix for `Table.format` on `Char` and `Mixed` columns.
- Fix for `Postgres_Connection` table viz to only show chosen schema.
- Added `DuckDB_Format` for setting parameters on CSV, JSON, Parquet.
- Updated `read_file` and `write_file` to use the `DuckDB_Format`.
- Added `DuckDB_FIle_Format` to allow connection to DuckDB via `Data.read` (like SQLite).
- Added `Parquet_Format` to read and write data to parquet files via DuckDB.
- Added `DuckDB_Table_Format` to allow reading a format using DuckDB in as a table either in memory or in duckdb.
- Alter case of `Snowflake.File_Format` to be less capitalised.

(cherry picked from commit 64c3d05)

Author: jdunkerley

CHANGELOG.md

@@ -74,10 +74,11 @@
 - [Initial Spatial support within DuckDB][14331]
 - [Add email SMTP support][14350]
 - [Read files into DuckDB both spatial and not.][14367]
-- [Implement Text_Column to_case for DB backends][14386]
-- [Implement bulk loading to DuckDB][14402]
+- [Implement Text_Column to_case for DB backends.][14386]
+- [Implement bulk loading to DuckDB.][14402]
 - [Implement `Text_Column.text_mid` for in-memory and database backends.][14420]
-- [Initial file writing from DuckDB][14421]
+- [Initial file writing from DuckDB.][14421]
+- [Parquet file reading and writing, DuckDB formats.][14427]
 
 [13769]: https://github.com/enso-org/enso/pull/13769
 [14026]: https://github.com/enso-org/enso/pull/14026
@@ -104,6 +105,7 @@
 [14402]: https://github.com/enso-org/enso/pull/14402
 [14420]: https://github.com/enso-org/enso/pull/14420
 [14421]: https://github.com/enso-org/enso/pull/14421
+[14427]: https://github.com/enso-org/enso/pull/14427
 
 #### Enso Language & Runtime
 

distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Postgres_Connection.enso

@@ -430,5 +430,7 @@ type Postgres_Connection
 ## ---
    private: true
    ---
-Table_Viz_Data.from (that:Postgres_Connection) = 
-    Connections_Helpers.get_table_viz_data (that.connection.tables.at "Name" . to_vector)
+Table_Viz_Data.from (that:Postgres_Connection) =
+    tables = that.tables . read . filter "Schema" (..Equal that.schema)
+    names = tables.at "Name" . to_vector
+    Connections_Helpers.get_table_viz_data (names)

distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB.md

@@ -3,7 +3,7 @@
 - type DuckDB
     - From_File location:Standard.Base.System.File.File= schema:Standard.Base.Data.Text.Text= read_only:Standard.Base.Data.Boolean.Boolean=
     - In_Memory schema:Standard.Base.Data.Text.Text= read_only:Standard.Base.Data.Boolean.Boolean=
-    - connect self options:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - connect self options:Standard.Base.Any.Any= -> Standard.Base.Any.Any
     - jdbc_properties self -> Standard.Base.Any.Any
     - jdbc_url self -> Standard.Base.Any.Any
     - resolve constructor:Standard.Base.Any.Any -> Standard.Base.Any.Any

distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md

@@ -21,7 +21,7 @@
     - jdbc_connection self -> Standard.Base.Any.Any
     - query self query:(Standard.Database.SQL_Query.SQL_Query_With_Schema|Standard.Database.SQL_Query.SQL_Query) alias:Standard.Base.Any.Any= -> Standard.Base.Any.Any
     - read self query:(Standard.Database.SQL_Query.SQL_Query_With_Schema|Standard.Database.SQL_Query.SQL_Query) limit:Standard.Table.Rows_To_Read.Rows_To_Read= -> Standard.Base.Any.Any
-    - read_file self path:Standard.Base.Any.Any= as:Standard.Base.Data.Text.Text= key_columns:Standard.Base.Data.Vector.Vector= replace_if_present:Standard.Base.Data.Boolean.Boolean= -> Standard.Base.Any.Any
+    - read_file self path:Standard.Base.Any.Any= as:Standard.Base.Data.Text.Text= format:Standard.DuckDB.DuckDB_Format.DuckDB_Format= key_columns:Standard.Base.Data.Vector.Vector= replace_if_present:Standard.Base.Data.Boolean.Boolean= -> Standard.Base.Any.Any
     - read_spatial_file self path:Standard.Base.Any.Any= as:Standard.Base.Data.Text.Text= key_columns:Standard.Base.Data.Vector.Vector= replace_if_present:Standard.Base.Data.Boolean.Boolean= -> Standard.Base.Any.Any
     - schema self -> Standard.Base.Any.Any
     - schemas self -> Standard.Base.Any.Any
@@ -35,6 +35,6 @@
     - type_mapping self -> Standard.Base.Any.Any
     - version self -> Standard.Base.Data.Text.Text
     - write_database self path:Standard.Base.System.File.Generic.Writable_File.Writable_File= schema_only:Standard.Base.Data.Boolean.Boolean= on_existing_file:Standard.Base.System.File.Existing_File_Behavior.Existing_File_Behavior= -> Standard.Base.Any.Any
-    - write_file self query:(Standard.Database.SQL_Query.SQL_Query_With_Schema|Standard.Database.SQL_Query.SQL_Query|Standard.Table.Table.Table)= path:Standard.Base.System.File.Generic.Writable_File.Writable_File= on_existing_file:Standard.Base.System.File.Existing_File_Behavior.Existing_File_Behavior= -> Standard.Base.Any.Any
+    - write_file self query:(Standard.Database.SQL_Query.SQL_Query_With_Schema|Standard.Database.SQL_Query.SQL_Query|Standard.Table.Table.Table)= path:Standard.Base.System.File.Generic.Writable_File.Writable_File= format:Standard.DuckDB.DuckDB_Format.DuckDB_Format= on_existing_file:Standard.Base.System.File.Existing_File_Behavior.Existing_File_Behavior= -> Standard.Base.Any.Any
 - schema_black_list -> Standard.Base.Any.Any
 - Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data.from that:Standard.DuckDB.DuckDB_Connection.DuckDB_Connection -> Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data

distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Format.md

@@ -0,0 +1,24 @@
+## Enso Signatures 1.0
+## module Standard.DuckDB.DuckDB_Format
+- type DuckDB_Compression
+    - Auto
+    - GZip
+    - None
+    - Zstandard
+- type DuckDB_Format
+    - Auto
+    - CSV delimiter:Standard.Base.Data.Text.Text= header:Standard.Base.Data.Boolean.Boolean= date_format:Standard.Base.Data.Text.Text= timestamp_format:Standard.Base.Data.Text.Text= quote_char:Standard.Base.Data.Text.Text= escape_char:Standard.Base.Data.Text.Text= compression:Standard.DuckDB.DuckDB_Format.DuckDB_Compression=
+    - JSON date_format:Standard.Base.Data.Text.Text= timestamp_format:Standard.Base.Data.Text.Text= compression:Standard.DuckDB.DuckDB_Format.DuckDB_Compression=
+    - JSON_Lines date_format:Standard.Base.Data.Text.Text= timestamp_format:Standard.Base.Data.Text.Text= compression:Standard.DuckDB.DuckDB_Format.DuckDB_Compression=
+    - Parquet compression:Standard.DuckDB.DuckDB_Format.DuckDB_Parquet_Compression= field_ids:Standard.Base.Data.Text.Text= version:Standard.DuckDB.DuckDB_Format.DuckDB_Parquet_Version=
+- type DuckDB_Parquet_Compression
+    - Brotli
+    - GZip
+    - LZ4
+    - LZ4_Raw
+    - None
+    - Snappy
+    - Zstandard compression_level:Standard.Base.Data.Numbers.Integer=
+- type DuckDB_Parquet_Version
+    - V1
+    - V2

distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/File_Formats/DuckDB_File_Format.md

@@ -0,0 +1,11 @@
+## Enso Signatures 1.0
+## module Standard.DuckDB.File_Formats.DuckDB_File_Format
+- type DuckDB_File_Format
+    - DuckDB
+    - for_file_write file:Standard.Base.System.File.Generic.Writable_File.Writable_File -> Standard.Base.Any.Any
+    - for_read file:Standard.Base.System.File_Format_Metadata.File_Format_Metadata -> Standard.Base.Any.Any
+    - get_dropdown_options -> Standard.Base.Any.Any
+    - get_name_patterns -> (Standard.Base.Data.Vector.Vector Standard.Base.System.File_Format.File_Name_Pattern)
+    - read self file:Standard.Base.Any.Any on_problems:Standard.Base.Errors.Problem_Behavior.Problem_Behavior -> Standard.Base.Any.Any
+    - read_stream self stream:Standard.Base.Any.Any metadata:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - resolve constructor:Standard.Base.Any.Any -> Standard.Base.Any.Any

distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/File_Formats/DuckDB_Table_Format.md

@@ -0,0 +1,12 @@
+## Enso Signatures 1.0
+## module Standard.DuckDB.File_Formats.DuckDB_Table_Format
+- type DuckDB_Table_Format
+    - DuckDB_Table format:Standard.DuckDB.DuckDB_Format.DuckDB_Format= in_memory:Standard.Base.Data.Boolean.Boolean=
+    - for_file_write file:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - for_read file:Standard.Base.System.File_Format_Metadata.File_Format_Metadata -> Standard.Base.Any.Any
+    - get_dropdown_options -> Standard.Base.Any.Any
+    - get_name_patterns -> (Standard.Base.Data.Vector.Vector Standard.Base.System.File_Format.File_Name_Pattern)
+    - read self file:Standard.Base.Any.Any on_problems:Standard.Base.Errors.Problem_Behavior.Problem_Behavior -> Standard.Base.Any.Any
+    - read_stream self stream:Standard.Base.System.Input_Stream.Input_Stream metadata:Standard.Base.System.File_Format_Metadata.File_Format_Metadata= -> Standard.Base.Any.Any
+    - resolve constructor:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - write_table self file:Standard.Base.Any.Any table:Standard.Base.Any.Any on_existing_file:Standard.Base.Any.Any match_columns:Standard.Base.Any.Any on_problems:Standard.Base.Errors.Problem_Behavior.Problem_Behavior -> Standard.Base.Any.Any

distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/File_Formats/Parquet_Format.md

@@ -0,0 +1,12 @@
+## Enso Signatures 1.0
+## module Standard.DuckDB.File_Formats.Parquet_Format
+- type Parquet_Format
+    - Parquet compression:Standard.DuckDB.DuckDB_Format.DuckDB_Parquet_Compression= field_ids:Standard.Base.Data.Text.Text= version:Standard.DuckDB.DuckDB_Format.DuckDB_Parquet_Version=
+    - for_file_write file:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - for_read file:Standard.Base.System.File_Format_Metadata.File_Format_Metadata -> Standard.Base.Any.Any
+    - get_dropdown_options -> Standard.Base.Any.Any
+    - get_name_patterns -> (Standard.Base.Data.Vector.Vector Standard.Base.System.File_Format.File_Name_Pattern)
+    - read self file:Standard.Base.Any.Any on_problems:Standard.Base.Errors.Problem_Behavior.Problem_Behavior -> Standard.Base.Any.Any
+    - read_stream self stream:Standard.Base.System.Input_Stream.Input_Stream metadata:Standard.Base.System.File_Format_Metadata.File_Format_Metadata= -> Standard.Base.Any.Any
+    - resolve constructor:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - write_table self file:Standard.Base.Any.Any table:Standard.Base.Any.Any on_existing_file:Standard.Base.Any.Any match_columns:Standard.Base.Any.Any on_problems:Standard.Base.Errors.Problem_Behavior.Problem_Behavior -> Standard.Base.Any.Any

distribution/lib/Standard/DuckDB/0.0.0-dev/package.yaml

@@ -12,3 +12,6 @@ maintainers:
 services:
   - provides: Standard.Database.Connection.Database.Database_Connection_Details_SPI
     with: Standard.DuckDB.Internal.DuckDB_Services.Impl
+
+  - provides: Standard.Base.System.File_Format.File_Format_SPI
+    with: Standard.DuckDB.Internal.DuckDB_Services.Impl

distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB.enso

@@ -39,7 +39,7 @@ type DuckDB
        ## Arguments
         - `options`: Overrides for the connection properties.
     connect : Connection_Options -> DuckDB_Connection
-    connect self options =
+    connect self options=Connection_Options.Value =
         properties = options.merge self.jdbc_properties
 
         make_new schema =