diff --git a/minio-alternatives/apache-ozone/arch.excalidraw.png b/minio-alternatives/apache-ozone/arch.excalidraw.png new file mode 100644 index 0000000..6ef4bbb Binary files /dev/null and b/minio-alternatives/apache-ozone/arch.excalidraw.png differ diff --git a/minio-alternatives/apache-ozone/docker-compose.yml b/minio-alternatives/apache-ozone/docker-compose.yml new file mode 100644 index 0000000..8ff2558 --- /dev/null +++ b/minio-alternatives/apache-ozone/docker-compose.yml @@ -0,0 +1,111 @@ +services: + scm: + image: apache/ozone:2.1.0 + ports: + - "9876:9876" + environment: + OZONE-SITE.XML_ozone.scm.names: scm + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata + OZONE-SITE.XML_ozone.scm.block.client.address: scm + OZONE-SITE.XML_ozone.scm.client.address: scm + OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata + OZONE-SITE.XML_ozone.replication: "1" + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "1" + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["ozone", "scm"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9876/"] + interval: 5s + timeout: 3s + retries: 10 + + om: + image: apache/ozone:2.1.0 + ports: + - "9874:9874" + environment: + OZONE-SITE.XML_ozone.om.address: om + OZONE-SITE.XML_ozone.om.http-address: om:9874 + OZONE-SITE.XML_ozone.scm.names: scm + OZONE-SITE.XML_ozone.scm.block.client.address: scm + OZONE-SITE.XML_ozone.scm.client.address: scm + OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata + OZONE-SITE.XML_ozone.replication: "1" + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + WAITFOR: scm:9876 + command: ["ozone", "om"] + depends_on: + scm: + condition: service_healthy + + datanode: + image: apache/ozone:2.1.0 + ports: + - "9864:9864" + environment: + OZONE-SITE.XML_hdds.datanode.dir: /data/hdds + OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata + OZONE-SITE.XML_ozone.scm.names: scm + OZONE-SITE.XML_ozone.scm.block.client.address: scm + OZONE-SITE.XML_ozone.scm.client.address: scm + OZONE-SITE.XML_ozone.replication: "1" + command: ["ozone", "datanode"] + depends_on: + scm: + condition: service_healthy + + s3g: + image: apache/ozone:2.1.0 + ports: + - "9878:9878" + environment: + OZONE-SITE.XML_ozone.om.address: om + OZONE-SITE.XML_ozone.scm.names: scm + OZONE-SITE.XML_ozone.scm.block.client.address: scm + OZONE-SITE.XML_ozone.scm.client.address: scm + OZONE-SITE.XML_ozone.replication: "1" + command: ["ozone", "s3g"] + depends_on: + scm: + condition: service_healthy + om: + condition: service_started + datanode: + condition: service_started + + mc: + image: minio/mc:latest + depends_on: + - s3g + entrypoint: > + /bin/sh -c " + sleep 10; + until (mc alias set ozone http://s3g:9878 admin password) do echo '...waiting...' && sleep 1; done; + mc mb -p ozone/warehouse; + tail -f /dev/null + " + + iceberg-rest: + image: tabulario/iceberg-rest:latest + depends_on: + - mc + ports: + - "8181:8181" + environment: + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password + CATALOG_WAREHOUSE: s3://warehouse/ + CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO + CATALOG_S3_ENDPOINT: http://s3g:9878 + CATALOG_S3_ACCESS__KEY__ID: admin + CATALOG_S3_SECRET__ACCESS__KEY: password + CATALOG_S3_PATH__STYLE__ACCESS: "true" + CATALOG_S3_REGION: us-east-1 + + duckdb: + image: duckdb/duckdb:latest + depends_on: + - iceberg-rest + stdin_open: true + tty: true diff --git a/minio-alternatives/apache-ozone/test.sh b/minio-alternatives/apache-ozone/test.sh new file mode 100755 index 0000000..dabfd20 --- /dev/null +++ b/minio-alternatives/apache-ozone/test.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -e + +echo "" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo " ▒ ▒" +echo " ▓ ___ ▓▓▓▓▓ ▓" +echo " █ ___( o)> ┌───────────┐ ▓▓▓▓▓▓▓ ▒▒▒▒▒▒▒ █" +echo " █ \\ <_. ) │ DUCKDB │ ▓░Local S3 ░▓ ▒▒▒▒ █" +echo " █ \`---' └───────────┘ ▓▓ Ozone ▒▒▒ █" +echo " ▓ ┏━━━━━━━━━━━━━┓ ░░░ ▓" +echo " ▒ ≋≋≋≋≋≋≋ ┃ ICEBERG ┃ ≋≋≋≋≋≋≋≋ Smoke test ▒" +echo " ░ ≋≋≋≋≋≋≋ ┗━━━━━━━━━━━━━┛ ≋≋≋≋≋≋≋≋ ░" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo "" + +echo "1. Checking Ozone buckets (before)..." +docker compose exec mc mc ls ozone +echo "" + +echo "2. Creating Iceberg table and inserting data..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; + +-- Configure S3 +CREATE SECRET s3_secret ( + TYPE S3, + KEY_ID 'admin', + SECRET 'password', + REGION 'us-east-1', + ENDPOINT 's3g:9878', + USE_SSL false, + URL_STYLE 'path' +); + +-- Configure Iceberg catalog +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); + +-- Create schema and table +CREATE SCHEMA IF NOT EXISTS cat.test; +DROP TABLE IF EXISTS cat.test.products; +CREATE TABLE cat.test.products (id INTEGER, name VARCHAR, price DECIMAL(10,2)); + +-- Insert data +INSERT INTO cat.test.products VALUES + (1, 'Widget', 9.99), + (2, 'Gadget', 19.99), + (3, 'Doohickey', 14.99); + +-- Query data +SELECT 'Row count:' as metric, COUNT(*)::VARCHAR as value FROM cat.test.products +UNION ALL +SELECT 'Total value:', CAST(SUM(price) AS VARCHAR) FROM cat.test.products; +" +echo "" + +echo "3. Verifying data in DuckDB..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; +CREATE SECRET s3_secret (TYPE S3, KEY_ID 'admin', SECRET 'password', REGION 'us-east-1', ENDPOINT 's3g:9878', USE_SSL false, URL_STYLE 'path'); +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); +SELECT * FROM cat.test.products ORDER BY id; +" +echo "" + +echo "4. Checking Ozone bucket contents (after)..." +docker compose exec mc mc ls --recursive ozone/warehouse/test/products/ | head -10 +echo "" diff --git a/minio-alternatives/cloudserver/arch.excalidraw.png b/minio-alternatives/cloudserver/arch.excalidraw.png new file mode 100644 index 0000000..7c8b23d Binary files /dev/null and b/minio-alternatives/cloudserver/arch.excalidraw.png differ diff --git a/minio-alternatives/cloudserver/docker-compose.yml b/minio-alternatives/cloudserver/docker-compose.yml new file mode 100644 index 0000000..8b7300b --- /dev/null +++ b/minio-alternatives/cloudserver/docker-compose.yml @@ -0,0 +1,49 @@ +services: + cloudserver: + image: ghcr.io/scality/cloudserver:9.2.8 + platform: linux/amd64 + ports: + - "8000:8000" + environment: + S3BACKEND: mem + REMOTE_MANAGEMENT_DISABLE: "1" + LOG_LEVEL: info + ENDPOINT: "cloudserver,s3.docker.internal,localhost,127.0.0.1" + SCALITY_ACCESS_KEY_ID: admin + SCALITY_SECRET_ACCESS_KEY: password + + mc: + image: minio/mc:latest + depends_on: + - cloudserver + entrypoint: > + /bin/sh -c " + until (mc alias set cloudserver http://cloudserver:8000 admin password) do echo '...waiting...' && sleep 1; done; + mc mb -p cloudserver/warehouse; + tail -f /dev/null + " + + iceberg-rest: + image: tabulario/iceberg-rest:latest + depends_on: + - mc + ports: + - "8181:8181" + environment: + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password + CATALOG_WAREHOUSE: s3://warehouse/ + CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO + CATALOG_S3_ENDPOINT: http://cloudserver:8000 + CATALOG_S3_ACCESS__KEY__ID: admin + CATALOG_S3_SECRET__ACCESS__KEY: password + CATALOG_S3_PATH__STYLE__ACCESS: "true" + CATALOG_S3_REGION: us-east-1 + + duckdb: + image: duckdb/duckdb:latest + depends_on: + - iceberg-rest + stdin_open: true + tty: true diff --git a/minio-alternatives/cloudserver/test.sh b/minio-alternatives/cloudserver/test.sh new file mode 100755 index 0000000..bd16bef --- /dev/null +++ b/minio-alternatives/cloudserver/test.sh @@ -0,0 +1,87 @@ +#!/bin/bash +set -e + +echo "" +echo " ░▒▓██████████████████████████████████████████████▓▒░" +echo " ▒ ▒" +echo " ▓ ___ ▓▓▓▓▓ ▓" +echo " █ ___( o)> ┌───────────┐ ▓▓▓▓▓▓▓ ▒▒▒▒▒▒▒ █" +echo " █ \\ <_. ) │ DUCKDB │ ▓░Local S3 ░▓ ▒▒▒▒ █" +echo " █ \`---' └───────────┘ ▓CloudServer▒ █" +echo " ▓ ┏━━━━━━━━━━━━━┓ ░░░ ▓" +echo " ▒ ≋≋≋≋≋≋≋ ┃ ICEBERG ┃ ≋≋≋≋≋≋≋≋ Smoke test ▒" +echo " ░ ≋≋≋≋≋≋≋ ┗━━━━━━━━━━━━━┛ ≋≋≋≋≋≋≋≋ ░" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo "" + +echo "Waiting for all services to be ready..." +until docker compose exec mc mc ls cloudserver >/dev/null 2>&1; do + echo "...waiting for CloudServer..." + sleep 2 +done +until curl -sf http://localhost:8181/v1/config >/dev/null 2>&1; do + echo "...waiting for Iceberg REST..." + sleep 2 +done +echo "" + +echo "1. Checking CloudServer buckets (before)..." +docker compose exec mc mc ls cloudserver +echo "" + +echo "2. Creating Iceberg table and inserting data..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; + +-- Configure S3 +CREATE SECRET s3_secret ( + TYPE S3, + KEY_ID 'admin', + SECRET 'password', + REGION 'us-east-1', + ENDPOINT 'cloudserver:8000', + USE_SSL false, + URL_STYLE 'path' +); + +-- Configure Iceberg catalog +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); + +-- Create schema and table +CREATE SCHEMA IF NOT EXISTS cat.test; +DROP TABLE IF EXISTS cat.test.products; +CREATE TABLE cat.test.products (id INTEGER, name VARCHAR, price DECIMAL(10,2)); + +-- Insert data +INSERT INTO cat.test.products VALUES + (1, 'Widget', 9.99), + (2, 'Gadget', 19.99), + (3, 'Doohickey', 14.99); + +-- Query data +SELECT 'Row count:' as metric, COUNT(*)::VARCHAR as value FROM cat.test.products +UNION ALL +SELECT 'Total value:', CAST(SUM(price) AS VARCHAR) FROM cat.test.products; +" +echo "" + +echo "3. Verifying data in DuckDB..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; +CREATE SECRET s3_secret (TYPE S3, KEY_ID 'admin', SECRET 'password', REGION 'us-east-1', ENDPOINT 'cloudserver:8000', USE_SSL false, URL_STYLE 'path'); +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); +SELECT * FROM cat.test.products ORDER BY id; +" +echo "" + +echo "4. Checking CloudServer bucket contents (after)..." +docker compose exec mc mc ls --recursive cloudserver/warehouse/test/products/ | head -10 +echo "" diff --git a/minio-alternatives/garage/arch.excalidraw.png b/minio-alternatives/garage/arch.excalidraw.png new file mode 100644 index 0000000..f3a480b Binary files /dev/null and b/minio-alternatives/garage/arch.excalidraw.png differ diff --git a/minio-alternatives/garage/docker-compose.yml b/minio-alternatives/garage/docker-compose.yml new file mode 100644 index 0000000..1f86b3b --- /dev/null +++ b/minio-alternatives/garage/docker-compose.yml @@ -0,0 +1,102 @@ +services: + garage: + image: dxflrs/garage:v1.0.0 + ports: + - "9000:3900" + - "9001:3902" + environment: + GARAGE_RPC_SECRET: "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + command: /garage -c /etc/garage.toml server + configs: + - source: garage-config + target: /etc/garage.toml + healthcheck: + test: ["CMD", "/garage", "-c", "/etc/garage.toml", "status"] + interval: 5s + timeout: 3s + retries: 10 + + garage-init: + image: alpine:latest + depends_on: + garage: + condition: service_healthy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + entrypoint: > + /bin/sh -c " + apk add --no-cache docker-cli; + CONTAINER=$$(docker ps --filter name=garage-garage-1 --format '{{.Names}}'); + NODE_ID=$$(docker exec $$CONTAINER /garage -c /etc/garage.toml status 2>/dev/null | grep 'NO ROLE ASSIGNED' | awk '{print \$$1}'); + if [ ! -z \"$$NODE_ID\" ]; then + docker exec $$CONTAINER /garage -c /etc/garage.toml layout assign -z dc1 -c 10G $$NODE_ID; + docker exec $$CONTAINER /garage -c /etc/garage.toml layout apply --version 1; + sleep 2; + fi; + docker exec $$CONTAINER /garage -c /etc/garage.toml key import -n admin --yes GKb69252a3b0643e8bd08d4cd4 9115be9c9e4994306a4176543b0db461f4eb04c5c8a388676d1b57392d0f4e93 2>/dev/null || true; + docker exec $$CONTAINER /garage -c /etc/garage.toml bucket create warehouse 2>/dev/null || true; + docker exec $$CONTAINER /garage -c /etc/garage.toml bucket allow --read --write warehouse --key GKb69252a3b0643e8bd08d4cd4; + echo 'Garage initialization complete'; + " + + mc: + image: minio/mc:latest + depends_on: + garage-init: + condition: service_completed_successfully + entrypoint: > + /bin/sh -c " + mc alias set garage http://garage:3900 GKb69252a3b0643e8bd08d4cd4 9115be9c9e4994306a4176543b0db461f4eb04c5c8a388676d1b57392d0f4e93 --insecure; + tail -f /dev/null + " + + iceberg-rest: + image: tabulario/iceberg-rest:latest + depends_on: + garage-init: + condition: service_completed_successfully + ports: + - "8181:8181" + environment: + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: GKb69252a3b0643e8bd08d4cd4 + AWS_SECRET_ACCESS_KEY: 9115be9c9e4994306a4176543b0db461f4eb04c5c8a388676d1b57392d0f4e93 + CATALOG_WAREHOUSE: s3://warehouse/ + CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO + CATALOG_S3_ENDPOINT: http://garage:3900 + CATALOG_S3_ACCESS__KEY__ID: GKb69252a3b0643e8bd08d4cd4 + CATALOG_S3_SECRET__ACCESS__KEY: 9115be9c9e4994306a4176543b0db461f4eb04c5c8a388676d1b57392d0f4e93 + CATALOG_S3_PATH__STYLE__ACCESS: "true" + CATALOG_S3_REGION: us-east-1 + + duckdb: + image: duckdb/duckdb:latest + depends_on: + - iceberg-rest + stdin_open: true + tty: true + +configs: + garage-config: + content: | + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + db_engine = "sqlite" + replication_mode = "none" + rpc_bind_addr = "[::]:3901" + rpc_secret = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + + [s3_api] + s3_region = "us-east-1" + api_bind_addr = "[::]:3900" + root_domain = ".s3.garage" + + [s3_web] + bind_addr = "[::]:3902" + root_domain = ".web.garage" + + [admin] + api_bind_addr = "[::]:3903" + + [k2v_api] + api_bind_addr = "0.0.0.0:3904" diff --git a/minio-alternatives/garage/test.sh b/minio-alternatives/garage/test.sh new file mode 100755 index 0000000..2e2ddd0 --- /dev/null +++ b/minio-alternatives/garage/test.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -e + +echo "" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo " ▒ ▒" +echo " ▓ ___ ▓▓▓▓▓ ▓" +echo " █ ___( o)> ┌───────────┐ ▓▓▓▓▓▓▓ ▒▒▒▒▒▒▒ █" +echo " █ \\ <_. ) │ DUCKDB │ ▓░Local S3 ░▓ ▒▒▒▒ █" +echo " █ \`---' └───────────┘ ▓▓ Garage ▒▒▒ █" +echo " ▓ ┏━━━━━━━━━━━━━┓ ░░░ ▓" +echo " ▒ ≋≋≋≋≋≋≋ ┃ ICEBERG ┃ ≋≋≋≋≋≋≋≋ Smoke test ▒" +echo " ░ ≋≋≋≋≋≋≋ ┗━━━━━━━━━━━━━┛ ≋≋≋≋≋≋≋≋ ░" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo "" + +echo "1. Checking Garage buckets (before)..." +docker compose exec garage /garage -c /etc/garage.toml bucket list +echo "" + +echo "2. Creating Iceberg table and inserting data..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; + +-- Configure S3 +CREATE SECRET s3_secret ( + TYPE S3, + KEY_ID 'GKb69252a3b0643e8bd08d4cd4', + SECRET '9115be9c9e4994306a4176543b0db461f4eb04c5c8a388676d1b57392d0f4e93', + REGION 'us-east-1', + ENDPOINT 'garage:3900', + USE_SSL false, + URL_STYLE 'path' +); + +-- Configure Iceberg catalog +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); + +-- Create schema and table +CREATE SCHEMA IF NOT EXISTS cat.test; +DROP TABLE IF EXISTS cat.test.products; +CREATE TABLE cat.test.products (id INTEGER, name VARCHAR, price DECIMAL(10,2)); + +-- Insert data +INSERT INTO cat.test.products VALUES + (1, 'Widget', 9.99), + (2, 'Gadget', 19.99), + (3, 'Doohickey', 14.99); + +-- Query data +SELECT 'Row count:' as metric, COUNT(*)::VARCHAR as value FROM cat.test.products +UNION ALL +SELECT 'Total value:', CAST(SUM(price) AS VARCHAR) FROM cat.test.products; +" +echo "" + +echo "3. Verifying data in DuckDB..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; +CREATE SECRET s3_secret (TYPE S3, KEY_ID 'GKb69252a3b0643e8bd08d4cd4', SECRET '9115be9c9e4994306a4176543b0db461f4eb04c5c8a388676d1b57392d0f4e93', REGION 'us-east-1', ENDPOINT 'garage:3900', USE_SSL false, URL_STYLE 'path'); +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); +SELECT * FROM cat.test.products ORDER BY id; +" +echo "" + +echo "4. Listing bucket objects with mc..." +docker compose exec mc mc ls --recursive garage/warehouse/ +echo "" diff --git a/minio-alternatives/minio/arch.excalidraw.png b/minio-alternatives/minio/arch.excalidraw.png new file mode 100644 index 0000000..f3ad11c Binary files /dev/null and b/minio-alternatives/minio/arch.excalidraw.png differ diff --git a/minio-alternatives/minio/docker-compose.yml b/minio-alternatives/minio/docker-compose.yml new file mode 100644 index 0000000..bb95791 --- /dev/null +++ b/minio-alternatives/minio/docker-compose.yml @@ -0,0 +1,46 @@ +services: + minio: + image: minio/minio:latest + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: admin + MINIO_ROOT_PASSWORD: password + command: ["server", "/data", "--console-address", ":9001"] + + mc: + image: minio/mc:latest + depends_on: + - minio + entrypoint: > + /bin/sh -c " + until (mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; + mc mb -p minio/warehouse; + tail -f /dev/null + " + + iceberg-rest: + image: tabulario/iceberg-rest:latest + depends_on: + - mc + ports: + - "8181:8181" + environment: + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password + CATALOG_WAREHOUSE: s3://warehouse/ + CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO + CATALOG_S3_ENDPOINT: http://minio:9000 + CATALOG_S3_ACCESS__KEY__ID: admin + CATALOG_S3_SECRET__ACCESS__KEY: password + CATALOG_S3_PATH__STYLE__ACCESS: "true" + CATALOG_S3_REGION: us-east-1 + + duckdb: + image: duckdb/duckdb:latest + depends_on: + - iceberg-rest + stdin_open: true + tty: true diff --git a/minio-alternatives/minio/test.sh b/minio-alternatives/minio/test.sh new file mode 100755 index 0000000..e750952 --- /dev/null +++ b/minio-alternatives/minio/test.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -e + +echo "" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo " ▒ ▒" +echo " ▓ ___ ▓▓▓▓▓ ▓" +echo " █ ___( o)> ┌───────────┐ ▓▓▓▓▓▓▓ ▒▒▒▒▒▒▒ █" +echo " █ \\ <_. ) │ DUCKDB │ ▓░Local S3 ░▓ ▒▒▒▒ █" +echo " █ \`---' └───────────┘ ▓▓ MinIO ▒▒▒ █" +echo " ▓ ┏━━━━━━━━━━━━━┓ ░░░ ▓" +echo " ▒ ≋≋≋≋≋≋≋ ┃ ICEBERG ┃ ≋≋≋≋≋≋≋≋ Smoke test ▒" +echo " ░ ≋≋≋≋≋≋≋ ┗━━━━━━━━━━━━━┛ ≋≋≋≋≋≋≋≋ ░" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo "" + +echo "1. Checking MinIO buckets (before)..." +docker compose exec mc mc ls minio +echo "" + +echo "2. Creating Iceberg table and inserting data..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; + +-- Configure S3 +CREATE SECRET s3_secret ( + TYPE S3, + KEY_ID 'admin', + SECRET 'password', + REGION 'us-east-1', + ENDPOINT 'minio:9000', + USE_SSL false, + URL_STYLE 'path' +); + +-- Configure Iceberg catalog +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); + +-- Create schema and table +CREATE SCHEMA IF NOT EXISTS cat.test; +DROP TABLE IF EXISTS cat.test.products; +CREATE TABLE cat.test.products (id INTEGER, name VARCHAR, price DECIMAL(10,2)); + +-- Insert data +INSERT INTO cat.test.products VALUES + (1, 'Widget', 9.99), + (2, 'Gadget', 19.99), + (3, 'Doohickey', 14.99); + +-- Query data +SELECT 'Row count:' as metric, COUNT(*)::VARCHAR as value FROM cat.test.products +UNION ALL +SELECT 'Total value:', CAST(SUM(price) AS VARCHAR) FROM cat.test.products; +" +echo "" + +echo "3. Verifying data in DuckDB..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; +CREATE SECRET s3_secret (TYPE S3, KEY_ID 'admin', SECRET 'password', REGION 'us-east-1', ENDPOINT 'minio:9000', USE_SSL false, URL_STYLE 'path'); +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); +SELECT * FROM cat.test.products ORDER BY id; +" +echo "" + +echo "4. Checking MinIO bucket contents (after)..." +docker compose exec mc mc ls --recursive minio/warehouse/test/products/ | head -10 +echo "" diff --git a/minio-alternatives/rustfs/arch.excalidraw.png b/minio-alternatives/rustfs/arch.excalidraw.png new file mode 100644 index 0000000..f12482e Binary files /dev/null and b/minio-alternatives/rustfs/arch.excalidraw.png differ diff --git a/minio-alternatives/rustfs/docker-compose.yml b/minio-alternatives/rustfs/docker-compose.yml new file mode 100644 index 0000000..52f8910 --- /dev/null +++ b/minio-alternatives/rustfs/docker-compose.yml @@ -0,0 +1,46 @@ +services: + rustfs: + image: rustfs/rustfs:1.0.0-alpha.79 + ports: + - "9000:9000" + - "9001:9001" + environment: + RUSTFS_CONSOLE_ENABLE: true + RUSTFS_ACCESS_KEY: admin + RUSTFS_SECRET_KEY: password + + mc: + image: minio/mc:latest + depends_on: + - rustfs + entrypoint: > + /bin/sh -c " + until (mc alias set rustfs http://rustfs:9000 admin password) do echo '...waiting...' && sleep 1; done; + mc mb -p rustfs/warehouse; + tail -f /dev/null + " + + iceberg-rest: + image: tabulario/iceberg-rest:latest + depends_on: + - mc + ports: + - "8181:8181" + environment: + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password + CATALOG_WAREHOUSE: s3://warehouse/ + CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO + CATALOG_S3_ENDPOINT: http://rustfs:9000 + CATALOG_S3_ACCESS__KEY__ID: admin + CATALOG_S3_SECRET__ACCESS__KEY: password + CATALOG_S3_PATH__STYLE__ACCESS: "true" + CATALOG_S3_REGION: us-east-1 + + duckdb: + image: duckdb/duckdb:latest + depends_on: + - iceberg-rest + stdin_open: true + tty: true diff --git a/minio-alternatives/rustfs/test.sh b/minio-alternatives/rustfs/test.sh new file mode 100755 index 0000000..40436bc --- /dev/null +++ b/minio-alternatives/rustfs/test.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -e + +echo "" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo " ▒ ▒" +echo " ▓ ___ ▓▓▓▓▓ ▓" +echo " █ ___( o)> ┌───────────┐ ▓▓▓▓▓▓▓ ▒▒▒▒▒▒▒ █" +echo " █ \\ <_. ) │ DUCKDB │ ▓░Local S3 ░▓ ▒▒▒▒ █" +echo " █ \`---' └───────────┘ ▓▓ RustFS ▒▒▒ █" +echo " ▓ ┏━━━━━━━━━━━━━┓ ░░░ ▓" +echo " ▒ ≋≋≋≋≋≋≋ ┃ ICEBERG ┃ ≋≋≋≋≋≋≋≋ Smoke test ▒" +echo " ░ ≋≋≋≋≋≋≋ ┗━━━━━━━━━━━━━┛ ≋≋≋≋≋≋≋≋ ░" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo "" + +echo "1. Checking MinIO buckets (before)..." +docker compose exec mc mc ls minio +echo "" + +echo "2. Creating Iceberg table and inserting data..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; + +-- Configure S3 +CREATE SECRET s3_secret ( + TYPE S3, + KEY_ID 'admin', + SECRET 'password', + REGION 'us-east-1', + ENDPOINT 'rustfs:9000', + USE_SSL false, + URL_STYLE 'path' +); + +-- Configure Iceberg catalog +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); + +-- Create schema and table +CREATE SCHEMA IF NOT EXISTS cat.test; +DROP TABLE IF EXISTS cat.test.products; +CREATE TABLE cat.test.products (id INTEGER, name VARCHAR, price DECIMAL(10,2)); + +-- Insert data +INSERT INTO cat.test.products VALUES + (1, 'Widget', 9.99), + (2, 'Gadget', 19.99), + (3, 'Doohickey', 14.99); + +-- Query data +SELECT 'Row count:' as metric, COUNT(*)::VARCHAR as value FROM cat.test.products +UNION ALL +SELECT 'Total value:', CAST(SUM(price) AS VARCHAR) FROM cat.test.products; +" +echo "" + +echo "3. Verifying data in DuckDB..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; +CREATE SECRET s3_secret (TYPE S3, KEY_ID 'admin', SECRET 'password', REGION 'us-east-1', ENDPOINT 'rustfs:9000', USE_SSL false, URL_STYLE 'path'); +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); +SELECT * FROM cat.test.products ORDER BY id; +" +echo "" + +echo "4. Checking RustFS bucket contents (after)..." +docker compose exec mc mc ls --recursive rustfs/warehouse/test/products/ | head -10 +echo "" diff --git a/minio-alternatives/s3proxy/arch.excalidraw.png b/minio-alternatives/s3proxy/arch.excalidraw.png new file mode 100644 index 0000000..e7f9751 Binary files /dev/null and b/minio-alternatives/s3proxy/arch.excalidraw.png differ diff --git a/minio-alternatives/s3proxy/docker-compose.yml b/minio-alternatives/s3proxy/docker-compose.yml new file mode 100644 index 0000000..83043af --- /dev/null +++ b/minio-alternatives/s3proxy/docker-compose.yml @@ -0,0 +1,46 @@ +services: + s3proxy: + image: andrewgaul/s3proxy:3.0.0 + ports: + - "8000:8000" + environment: + S3PROXY_AUTHORIZATION: aws-v2-or-v4 + S3PROXY_IDENTITY: admin + S3PROXY_CREDENTIAL: password + S3PROXY_ENDPOINT: http://0.0.0.0:8000 + + mc: + image: minio/mc:latest + depends_on: + - s3proxy + entrypoint: > + /bin/sh -c " + until (mc alias set s3proxy http://s3proxy:8000 admin password) do echo '...waiting...' && sleep 1; done; + mc mb -p s3proxy/warehouse; + tail -f /dev/null + " + + iceberg-rest: + image: tabulario/iceberg-rest:latest + depends_on: + - mc + ports: + - "8181:8181" + environment: + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password + CATALOG_WAREHOUSE: s3://warehouse/ + CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO + CATALOG_S3_ENDPOINT: http://s3proxy:8000 + CATALOG_S3_ACCESS__KEY__ID: admin + CATALOG_S3_SECRET__ACCESS__KEY: password + CATALOG_S3_PATH__STYLE__ACCESS: "true" + CATALOG_S3_REGION: us-east-1 + + duckdb: + image: duckdb/duckdb:latest + depends_on: + - iceberg-rest + stdin_open: true + tty: true diff --git a/minio-alternatives/s3proxy/test.sh b/minio-alternatives/s3proxy/test.sh new file mode 100755 index 0000000..5a1ddfc --- /dev/null +++ b/minio-alternatives/s3proxy/test.sh @@ -0,0 +1,87 @@ +#!/bin/bash +set -e + +echo "" +echo " ░▒▓██████████████████████████████████████████████▓▒░" +echo " ▒ ▒" +echo " ▓ ___ ▓▓▓▓▓ ▓" +echo " █ ___( o)> ┌───────────┐ ▓▓▓▓▓▓▓ ▒▒▒▒▒▒▒ █" +echo " █ \\ <_. ) │ DUCKDB │ ▓░Local S3 ░▓ ▒▒▒▒ █" +echo " █ \`---' └───────────┘ ▓S3Proxy▒ █" +echo " ▓ ┏━━━━━━━━━━━━━┓ ░░░ ▓" +echo " ▒ ≋≋≋≋≋≋≋ ┃ ICEBERG ┃ ≋≋≋≋≋≋≋≋ Smoke test ▒" +echo " ░ ≋≋≋≋≋≋≋ ┗━━━━━━━━━━━━━┛ ≋≋≋≋≋≋≋≋ ░" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo "" + +echo "Waiting for all services to be ready..." +until docker compose exec mc mc ls s3proxy >/dev/null 2>&1; do + echo "...waiting for s3proxy..." + sleep 2 +done +until curl -sf http://localhost:8181/v1/config >/dev/null 2>&1; do + echo "...waiting for Iceberg REST..." + sleep 2 +done +echo "" + +echo "1. Checking s3proxy buckets (before)..." +docker compose exec mc mc ls s3proxy +echo "" + +echo "2. Creating Iceberg table and inserting data..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; + +-- Configure S3 +CREATE SECRET s3_secret ( + TYPE S3, + KEY_ID 'admin', + SECRET 'password', + REGION 'us-east-1', + ENDPOINT 's3proxy:8000', + USE_SSL false, + URL_STYLE 'path' +); + +-- Configure Iceberg catalog +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); + +-- Create schema and table +CREATE SCHEMA IF NOT EXISTS cat.test; +DROP TABLE IF EXISTS cat.test.products; +CREATE TABLE cat.test.products (id INTEGER, name VARCHAR, price DECIMAL(10,2)); + +-- Insert data +INSERT INTO cat.test.products VALUES + (1, 'Widget', 9.99), + (2, 'Gadget', 19.99), + (3, 'Doohickey', 14.99); + +-- Query data +SELECT 'Row count:' as metric, COUNT(*)::VARCHAR as value FROM cat.test.products +UNION ALL +SELECT 'Total value:', CAST(SUM(price) AS VARCHAR) FROM cat.test.products; +" +echo "" + +echo "3. Verifying data in DuckDB..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; +CREATE SECRET s3_secret (TYPE S3, KEY_ID 'admin', SECRET 'password', REGION 'us-east-1', ENDPOINT 's3proxy:8000', USE_SSL false, URL_STYLE 'path'); +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); +SELECT * FROM cat.test.products ORDER BY id; +" +echo "" + +echo "4. Checking s3proxy bucket contents (after)..." +docker compose exec mc mc ls --recursive s3proxy/warehouse/test/products/ | head -10 +echo "" diff --git a/minio-alternatives/seaweedfs/arch.excalidraw.png b/minio-alternatives/seaweedfs/arch.excalidraw.png new file mode 100644 index 0000000..3d696bf Binary files /dev/null and b/minio-alternatives/seaweedfs/arch.excalidraw.png differ diff --git a/minio-alternatives/seaweedfs/docker-compose.yml b/minio-alternatives/seaweedfs/docker-compose.yml new file mode 100644 index 0000000..ce05aa0 --- /dev/null +++ b/minio-alternatives/seaweedfs/docker-compose.yml @@ -0,0 +1,52 @@ +services: + seaweedfs: + image: chrislusf/seaweedfs:4.06 + ports: + - "9333:9333" + - "8888:8888" + command: server -s3 -s3.port=9000 -s3.config=/etc/seaweedfs/s3.json -master.port=9333 -volume.port=8080 -filer -filer.port=8888 + configs: + - source: s3-config + target: /etc/seaweedfs/s3.json + + mc: + image: minio/mc:latest + depends_on: + - seaweedfs + entrypoint: > + /bin/sh -c " + until (mc alias set seaweedfs http://seaweedfs:9000 admin password) do echo '...waiting...' && sleep 1; done; + mc mb -p seaweedfs/warehouse; + tail -f /dev/null + " + + iceberg-rest: + image: tabulario/iceberg-rest:latest + depends_on: + - mc + ports: + - "8181:8181" + restart: on-failure + environment: + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password + CATALOG_WAREHOUSE: s3://warehouse/ + CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO + CATALOG_S3_ENDPOINT: http://seaweedfs:9000 + CATALOG_S3_ACCESS__KEY__ID: admin + CATALOG_S3_SECRET__ACCESS__KEY: password + CATALOG_S3_PATH__STYLE__ACCESS: "true" + CATALOG_S3_REGION: us-east-1 + + duckdb: + image: duckdb/duckdb:latest + depends_on: + - iceberg-rest + stdin_open: true + tty: true + +configs: + s3-config: + content: | + { "identities": [ { "name": "anonymous", "credentials": [ { "accessKey": "admin", "secretKey": "password" } ], "actions": [ "Admin", "Read", "Write" ] } ] } diff --git a/minio-alternatives/seaweedfs/test.sh b/minio-alternatives/seaweedfs/test.sh new file mode 100755 index 0000000..d368f9a --- /dev/null +++ b/minio-alternatives/seaweedfs/test.sh @@ -0,0 +1,87 @@ +#!/bin/bash +set -e + +echo "" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo " ▒ ▒" +echo " ▓ ___ ▓▓▓▓▓ ▓" +echo " █ ___( o)> ┌───────────┐ ▓▓▓▓▓▓▓ ▒▒▒▒▒▒▒ █" +echo " █ \\ <_. ) │ DUCKDB │ ▓░Local S3 ░▓ ▒▒▒▒ █" +echo " █ \`---' └───────────┘ ▓SeaweedFS▒▒▒ █" +echo " ▓ ┏━━━━━━━━━━━━━┓ ░░░ ▓" +echo " ▒ ≋≋≋≋≋≋≋ ┃ ICEBERG ┃ ≋≋≋≋≋≋≋≋ Smoke test ▒" +echo " ░ ≋≋≋≋≋≋≋ ┗━━━━━━━━━━━━━┛ ≋≋≋≋≋≋≋≋ ░" +echo " ░▒▓██████████████████████████████████████████████████▓▒░" +echo "" + +echo "Waiting for all services to be ready..." +until docker compose exec mc mc ls seaweedfs >/dev/null 2>&1; do + echo "...waiting for SeaweedFS..." + sleep 2 +done +until curl -sf http://localhost:8181/v1/config >/dev/null 2>&1; do + echo "...waiting for Iceberg REST..." + sleep 2 +done +echo "" + +echo "1. Checking SeaweedFS buckets (before)..." +docker compose exec mc mc ls seaweedfs +echo "" + +echo "2. Creating Iceberg table and inserting data..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; + +-- Configure S3 +CREATE SECRET s3_secret ( + TYPE S3, + KEY_ID 'admin', + SECRET 'password', + REGION 'us-east-1', + ENDPOINT 'seaweedfs:9000', + USE_SSL false, + URL_STYLE 'path' +); + +-- Configure Iceberg catalog +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); + +-- Create schema and table +CREATE SCHEMA IF NOT EXISTS cat.test; +DROP TABLE IF EXISTS cat.test.products; +CREATE TABLE cat.test.products (id INTEGER, name VARCHAR, price DECIMAL(10,2)); + +-- Insert data +INSERT INTO cat.test.products VALUES + (1, 'Widget', 9.99), + (2, 'Gadget', 19.99), + (3, 'Doohickey', 14.99); + +-- Query data +SELECT 'Row count:' as metric, COUNT(*)::VARCHAR as value FROM cat.test.products +UNION ALL +SELECT 'Total value:', CAST(SUM(price) AS VARCHAR) FROM cat.test.products; +" +echo "" + +echo "3. Verifying data in DuckDB..." +docker compose exec duckdb duckdb -c " +INSTALL iceberg; +INSTALL httpfs; +LOAD iceberg; +LOAD httpfs; +CREATE SECRET s3_secret (TYPE S3, KEY_ID 'admin', SECRET 'password', REGION 'us-east-1', ENDPOINT 'seaweedfs:9000', USE_SSL false, URL_STYLE 'path'); +CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy'); +ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret); +SELECT * FROM cat.test.products ORDER BY id; +" +echo "" + +echo "4. Checking SeaweedFS bucket contents (after)..." +docker compose exec mc mc ls --recursive seaweedfs/warehouse/test/products/ | head -10 +echo ""