feat(connectors): add S3 sink connector

.github/workflows/_build_rust_artifacts.yml

@@ -46,7 +46,7 @@ on:
       connector_plugins:
         type: string
         required: false
-        default: "iggy_connector_elasticsearch_sink,iggy_connector_elasticsearch_source,iggy_connector_iceberg_sink,iggy_connector_postgres_sink,iggy_connector_postgres_source,iggy_connector_quickwit_sink,iggy_connector_random_source,iggy_connector_stdout_sink"
+        default: "iggy_connector_elasticsearch_sink,iggy_connector_elasticsearch_source,iggy_connector_iceberg_sink,iggy_connector_postgres_sink,iggy_connector_postgres_source,iggy_connector_quickwit_sink,iggy_connector_random_source,iggy_connector_s3_sink,iggy_connector_stdout_sink"
         description: "Comma-separated list of connector plugin crates to build as shared libraries"
     outputs:
       artifact_name:

.github/workflows/edge-release.yml

@@ -102,6 +102,7 @@ jobs:
             - `iggy_connector_postgres_source`
             - `iggy_connector_quickwit_sink`
             - `iggy_connector_random_source`
+            - `iggy_connector_s3_sink`
             - `iggy_connector_stdout_sink`
 
             ## Downloads

Cargo.toml

@@ -40,6 +40,7 @@ members = [
     "core/connectors/sinks/mongodb_sink",
     "core/connectors/sinks/postgres_sink",
     "core/connectors/sinks/quickwit_sink",
+    "core/connectors/sinks/s3_sink",
     "core/connectors/sinks/stdout_sink",
     "core/connectors/sources/elasticsearch_source",
     "core/connectors/sources/influxdb_source",

DEPENDENCIES.md

@@ -75,13 +75,16 @@ atoi: 2.0.0, "MIT",
 atomic: 0.6.1, "Apache-2.0 OR MIT",
 atomic-polyfill: 1.0.3, "Apache-2.0 OR MIT",
 atomic-waker: 1.1.2, "Apache-2.0 OR MIT",
+attohttpc: 0.30.1, "MPL-2.0",
 autocfg: 1.5.0, "Apache-2.0 OR MIT",
 autotools: 0.2.7, "MIT",
 av-scenechange: 0.14.1, "MIT",
 av1-grain: 0.2.5, "BSD-2-Clause",
 avif-serialize: 0.8.8, "BSD-3-Clause",
+aws-creds: 0.39.1, "MIT",
 aws-lc-rs: 1.16.2, "(Apache-2.0 OR ISC) AND ISC",
 aws-lc-sys: 0.39.1, "(Apache-2.0 OR ISC OR MIT) AND (Apache-2.0 OR ISC OR MIT-0) AND (Apache-2.0 OR ISC) AND Apache-2.0 AND BSD-3-Clause AND ISC AND MIT",
+aws-region: 0.28.1, "MIT",
 axum: 0.8.8, "MIT",
 axum-core: 0.5.6, "MIT",
 axum-macros: 0.5.0, "MIT",
@@ -147,6 +150,7 @@ capacity_builder: 0.5.0, "MIT",
 capacity_builder_macros: 0.3.0, "MIT",
 cargo-platform: 0.3.2, "Apache-2.0 OR MIT",
 cargo_metadata: 0.23.1, "MIT",
+castaway: 0.2.4, "MIT",
 cc: 1.2.59, "Apache-2.0 OR MIT",
 cesu8: 1.1.0, "Apache-2.0 OR MIT",
 cexpr: 0.6.0, "Apache-2.0 OR MIT",
@@ -171,6 +175,7 @@ colorchoice: 1.0.5, "Apache-2.0 OR MIT",
 colored: 3.1.1, "MPL-2.0",
 combine: 4.6.7, "MIT",
 comfy-table: 7.2.2, "MIT",
+compact_str: 0.7.1, "MIT",
 compio: 0.18.0, "MIT",
 compio-buf: 0.8.1, "MIT",
 compio-driver: 0.11.4, "MIT",
@@ -494,6 +499,7 @@ iggy_connector_postgres_sink: 0.4.0, "Apache-2.0",
 iggy_connector_postgres_source: 0.4.0, "Apache-2.0",
 iggy_connector_quickwit_sink: 0.4.0, "Apache-2.0",
 iggy_connector_random_source: 0.4.0, "Apache-2.0",
+iggy_connector_s3_sink: 0.4.0, "Apache-2.0",
 iggy_connector_sdk: 0.3.0, "Apache-2.0",
 iggy_connector_stdout_sink: 0.4.0, "Apache-2.0",
 iggy_examples: 0.0.6, "Apache-2.0",
@@ -595,8 +601,10 @@ macro_rules_attribute: 0.1.3, "MIT",
 macro_rules_attribute-proc_macro: 0.1.3, "MIT",
 matchers: 0.2.0, "MIT",
 matchit: 0.8.4, "BSD-3-Clause AND MIT",
+maybe-async: 0.2.10, "MIT",
 maybe-rayon: 0.1.1, "MIT",
 md-5: 0.10.6, "Apache-2.0 OR MIT",
+md5: 0.8.0, "Apache-2.0 OR MIT",
 memchr: 2.8.0, "MIT OR Unlicense",
 memmap2: 0.9.10, "Apache-2.0 OR MIT",
 message_bus: 0.1.0, "Apache-2.0",
@@ -606,6 +614,7 @@ miette-derive: 7.6.0, "Apache-2.0",
 mimalloc: 0.1.48, "MIT",
 mime: 0.3.17, "Apache-2.0 OR MIT",
 mime_guess: 2.0.5, "MIT",
+minidom: 0.16.0, "MPL-2.0",
 minimal-lexical: 0.2.1, "Apache-2.0 OR MIT",
 miniz_oxide: 0.8.9, "Apache-2.0 OR MIT OR Zlib",
 mio: 1.2.0, "MIT",
@@ -817,6 +826,7 @@ rust-embed: 8.11.0, "MIT",
 rust-embed-impl: 8.11.0, "MIT",
 rust-embed-utils: 8.11.0, "MIT",
 rust-ini: 0.21.3, "MIT",
+rust-s3: 0.37.1, "MIT",
 rust_decimal: 1.41.0, "MIT",
 rustc-hash: 2.1.2, "Apache-2.0 OR MIT",
 rustc_version: 0.4.1, "Apache-2.0 OR MIT",
@@ -833,6 +843,8 @@ rustls-platform-verifier-android: 0.1.1, "Apache-2.0 OR MIT",
 rustls-webpki: 0.103.10, "ISC",
 rustversion: 1.0.22, "Apache-2.0 OR MIT",
 rustybuzz: 0.20.1, "MIT",
+rxml: 0.11.1, "MIT",
+rxml_validation: 0.11.0, "MIT",
 ryu: 1.0.23, "Apache-2.0 OR BSL-1.0",
 same-file: 1.0.6, "MIT OR Unlicense",
 scc: 2.4.0, "Apache-2.0",

core/connectors/README.md

@@ -82,6 +82,7 @@ Each sink should have its own, custom configuration, which is passed along with
 - **Iceberg Sink** - writes data to Apache Iceberg tables via REST catalog
 - **PostgreSQL Sink** - stores messages in PostgreSQL database tables
 - **Quickwit Sink** - indexes messages in Quickwit search engine
+- **S3 Sink** - writes messages to Amazon S3 and S3-compatible stores (MinIO, R2, B2, DO Spaces)
 - **Stdout Sink** - prints messages to standard output (useful for debugging/development)
 
 ## Source

core/connectors/sinks/README.md

@@ -12,6 +12,7 @@ Sink connectors are responsible for writing data from Iggy streams to external s
 | **iceberg_sink** | Writes data to Apache Iceberg tables via REST catalog with S3/GCS/Azure storage |
 | **postgres_sink** | Stores messages in PostgreSQL database tables with configurable schemas |
 | **quickwit_sink** | Indexes messages in Quickwit search engine for log analytics |
+| **s3_sink** | Writes messages to Amazon S3 and S3-compatible stores (MinIO, R2, B2, DO Spaces) |
 | **stdout_sink** | Prints messages to standard output (useful for debugging and development) |
 
 The sink is represented by the single `Sink` trait, which defines the basic interface for all sink connectors. It provides methods for initializing the sink, writing data to external destination, and closing the sink.