ClickBench Playground

clickhouse-web/create.sql

@@ -108,5 +108,5 @@ ATTACH TABLE hits UUID 'c449dfbf-ba06-4d13-abec-8396559eb955'
     PRIMARY KEY (CounterID, EventDate, UserID, EventTime, WatchID)
 )
 ENGINE = MergeTree
-SETTINGS disk = disk(type = cache, path = '/dev/shm/clickhouse/', max_size_ratio_to_total_space = 0.9,
+SETTINGS disk = disk(type = cache, path = '/var/lib/clickhouse/caches/web/', max_size_ratio_to_total_space = 0.9,
          disk = disk(type = web, endpoint = 'https://clickhouse-public-datasets.s3.amazonaws.com/web/'));

clickhouse-web/install

@@ -10,6 +10,17 @@ if [ ! -x /usr/bin/clickhouse ]; then
     sudo ./clickhouse install --noninteractive
 fi
 
-# Cache directory used by the web disk.
-sudo mkdir -p /dev/shm/clickhouse
+# Cache directory used by the web disk. ClickHouse rejects any
+# filesystem-cache path outside /var/lib/clickhouse/caches/ with
+# BAD_ARGUMENTS at CREATE TABLE time, but we still want the actual
+# bytes to live in tmpfs (/dev/shm) for the speed: cold queries
+# pull ~1 GB on first run and tmpfs avoids touching the host SSD.
+#
+# Trick: hand ClickHouse a path that satisfies its policy check
+# (.../caches/web) but is itself a symlink into /dev/shm. CH only
+# verifies the lexical prefix of the configured path; it doesn't
+# canonicalise the target.
+sudo mkdir -p /dev/shm/clickhouse /var/lib/clickhouse/caches
 sudo chown clickhouse:clickhouse /dev/shm/clickhouse
+# Replace any prior real dir / stale symlink atomically.
+sudo ln -sfn /dev/shm/clickhouse /var/lib/clickhouse/caches/web

firebolt-parquet-partitioned/benchmark.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+# Playground reads this line to pre-stage the dataset on the per-VM
+# system disk; the rest of benchmark.sh is unchanged.
+export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
+
 # Download the partitioned hits parquet files
 echo "Downloading dataset..."
 rm -rf data

firebolt-parquet-partitioned/check

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+# Firebolt-core's HTTP port answers immediately but may return a
+# cluster-not-ready JSON error at HTTP 200. Test for an actual result.
+curl -sSf --max-time 5 'http://localhost:3473/' \
+    --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1'

firebolt-parquet-partitioned/install

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -eu
+
+sudo apt-get update -y
+sudo apt-get install -y docker.io jq
+sudo docker pull ghcr.io/firebolt-db/firebolt-core:preview-rc

firebolt-parquet-partitioned/load

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eu
+
+# Partitioned-parquet variant: stage hits_*.parquet under ./data so
+# the container sees them at /firebolt-core/clickbench/*.parquet;
+# create.sql declares an external table with FROM PATTERN that
+# matches the glob.
+mkdir -p data
+shopt -s nullglob
+for f in hits_*.parquet; do
+    mv -f "$f" "data/$f"
+done
+shopt -u nullglob
+
+curl -sSf 'http://localhost:3473/?enable_multi_query_requests=true' \
+    --data-binary 'DROP DATABASE IF EXISTS clickbench;CREATE DATABASE clickbench;'
+curl -sSf 'http://localhost:3473/?database=clickbench&enable_multi_query_requests=true' \
+    --data-binary @create.sql
+
+sync

firebolt-parquet-partitioned/query

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Reads a SQL query from stdin, runs it against the firebolt-core
+# container via /?database=clickbench.
+# Stdout: query result (firebolt's JSON_Compact format).
+# Stderr: query runtime in fractional seconds on the last line,
+#         pulled from the response's `.statistics.elapsed`.
+# Exit non-zero on error.
+set -e
+
+query=$(cat)
+
+# Result + sub-result caches off so timings are real; output_format
+# matches what firebolt's run.sh uses for the public benchmark.
+PARAMS='database=clickbench&enable_result_cache=false&enable_subresult_cache=false&enable_scan_cache=false&output_format=JSON_Compact'
+
+resp=$(curl -sS --max-time 600 "http://localhost:3473/?${PARAMS}" \
+            --data-binary "$query")
+
+# Firebolt returns a JSON object whether the query succeeded or not.
+# A failed query has an "errors" key; a successful one carries
+# "data" + "statistics".
+if printf '%s' "$resp" | jq -e '.errors' >/dev/null 2>&1; then
+    printf '%s\n' "$resp" >&2
+    exit 1
+fi
+
+printf '%s\n' "$resp"
+printf '%s\n' "$resp" | jq -r '.statistics.elapsed' >&2

firebolt-parquet-partitioned/start

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -eu
+
+# Idempotent: if firebolt-core already answers SELECT 1, do nothing.
+if curl -sS --max-time 5 'http://localhost:3473/' \
+        --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1'; then
+    exit 0
+fi
+
+mkdir -p data fb-volume
+
+# If the container exists (stopped from a prior agent pre-snapshot
+# cycle), just start it back — the data lives on the bind-mounted
+# fb-volume below, so the previously-created `clickbench` database
+# is still there. Otherwise create the container fresh.
+if sudo docker ps -a --format '{{.Names}}' | grep -qx firebolt-core; then
+    sudo docker start firebolt-core >/dev/null
+else
+    # `firebolt-core` is the public self-hosted image. Container needs
+    # memlock 8 GiB and seccomp unconfined per upstream's run docs.
+    # /firebolt-core/clickbench: parquet source (read at load time).
+    # /firebolt-core/volume:     engine data directory (must persist
+    #                            across the agent's pre-snapshot
+    #                            stop+start cycle or the snapshot
+    #                            ships an empty DB).
+    sudo docker run -dit --name firebolt-core \
+        --ulimit memlock=8589934592:8589934592 \
+        --security-opt seccomp=unconfined \
+        -p 127.0.0.1:3473:3473 \
+        -v "$(pwd)/fb-volume:/firebolt-core/volume" \
+        -v "$(pwd)/data:/firebolt-core/clickbench" \
+        ghcr.io/firebolt-db/firebolt-core:preview-rc >/dev/null
+fi
+
+# Wait for the cluster to be "actually" ready. firebolt-core's HTTP
+# port comes up immediately but returns
+#   {"errors":[{"description":"Cluster not yet healthy: ..."}]}
+# at HTTP 200 until the engine threads have warmed; bench against a
+# sentinel string instead of HTTP status to avoid that trap.
+for _ in $(seq 1 600); do
+    if curl -sS --max-time 5 'http://localhost:3473/' \
+            --data-binary "SELECT 'firebolt-ready';" 2>/dev/null \
+            | grep -q 'firebolt-ready'; then
+        exit 0
+    fi
+    sleep 1
+done
+echo "firebolt-core did not become healthy in 10 min" >&2
+exit 1

firebolt-parquet-partitioned/stop

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+# Plain stop — leave the container in place so its bind-mounted
+# fb-volume keeps the loaded database for the next ./start. The
+# container is removed and the volume re-initialised only on
+# explicit re-provision.
+sudo docker container stop firebolt-core >/dev/null 2>&1 || true

firebolt-parquet/benchmark.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+# Playground reads this line to pre-stage the dataset on the per-VM
+# system disk; the rest of benchmark.sh is unchanged.
+export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
+
 # Download the hits.parquet file
 echo "Downloading dataset..."
 rm -rf data

firebolt-parquet/check

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+# Firebolt-core's HTTP port answers immediately but may return a
+# cluster-not-ready JSON error at HTTP 200. Test for an actual result.
+curl -sSf --max-time 5 'http://localhost:3473/' \
+    --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1'

firebolt-parquet/install

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -eu
+
+sudo apt-get update -y
+sudo apt-get install -y docker.io jq
+sudo docker pull ghcr.io/firebolt-db/firebolt-core:preview-rc

firebolt-parquet/load

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -eu
+
+# Parquet variant: data stays in ./data (mounted as
+# /firebolt-core/clickbench in the container), create.sql declares
+# an external table that reads it on every query.
+mkdir -p data
+if [ -f hits.parquet ]; then
+    mv -f hits.parquet data/hits.parquet
+fi
+
+curl -sSf 'http://localhost:3473/?enable_multi_query_requests=true' \
+    --data-binary 'DROP DATABASE IF EXISTS clickbench;CREATE DATABASE clickbench;'
+curl -sSf 'http://localhost:3473/?database=clickbench&enable_multi_query_requests=true' \
+    --data-binary @create.sql
+
+sync

firebolt-parquet/query

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Reads a SQL query from stdin, runs it against the firebolt-core
+# container via /?database=clickbench.
+# Stdout: query result (firebolt's JSON_Compact format).
+# Stderr: query runtime in fractional seconds on the last line,
+#         pulled from the response's `.statistics.elapsed`.
+# Exit non-zero on error.
+set -e
+
+query=$(cat)
+
+# Result + sub-result caches off so timings are real; output_format
+# matches what firebolt's run.sh uses for the public benchmark.
+PARAMS='database=clickbench&enable_result_cache=false&enable_subresult_cache=false&enable_scan_cache=false&output_format=JSON_Compact'
+
+resp=$(curl -sS --max-time 600 "http://localhost:3473/?${PARAMS}" \
+            --data-binary "$query")
+
+# Firebolt returns a JSON object whether the query succeeded or not.
+# A failed query has an "errors" key; a successful one carries
+# "data" + "statistics".
+if printf '%s' "$resp" | jq -e '.errors' >/dev/null 2>&1; then
+    printf '%s\n' "$resp" >&2
+    exit 1
+fi
+
+printf '%s\n' "$resp"
+printf '%s\n' "$resp" | jq -r '.statistics.elapsed' >&2

firebolt-parquet/start

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -eu
+
+# Idempotent: if firebolt-core already answers SELECT 1, do nothing.
+if curl -sS --max-time 5 'http://localhost:3473/' \
+        --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1'; then
+    exit 0
+fi
+
+mkdir -p data fb-volume
+
+# If the container exists (stopped from a prior agent pre-snapshot
+# cycle), just start it back — the data lives on the bind-mounted
+# fb-volume below, so the previously-created `clickbench` database
+# is still there. Otherwise create the container fresh.
+if sudo docker ps -a --format '{{.Names}}' | grep -qx firebolt-core; then
+    sudo docker start firebolt-core >/dev/null
+else
+    # `firebolt-core` is the public self-hosted image. Container needs
+    # memlock 8 GiB and seccomp unconfined per upstream's run docs.
+    # /firebolt-core/clickbench: parquet source (read at load time).
+    # /firebolt-core/volume:     engine data directory (must persist
+    #                            across the agent's pre-snapshot
+    #                            stop+start cycle or the snapshot
+    #                            ships an empty DB).
+    sudo docker run -dit --name firebolt-core \
+        --ulimit memlock=8589934592:8589934592 \
+        --security-opt seccomp=unconfined \
+        -p 127.0.0.1:3473:3473 \
+        -v "$(pwd)/fb-volume:/firebolt-core/volume" \
+        -v "$(pwd)/data:/firebolt-core/clickbench" \
+        ghcr.io/firebolt-db/firebolt-core:preview-rc >/dev/null
+fi
+
+# Wait for the cluster to be "actually" ready. firebolt-core's HTTP
+# port comes up immediately but returns
+#   {"errors":[{"description":"Cluster not yet healthy: ..."}]}
+# at HTTP 200 until the engine threads have warmed; bench against a
+# sentinel string instead of HTTP status to avoid that trap.
+for _ in $(seq 1 600); do
+    if curl -sS --max-time 5 'http://localhost:3473/' \
+            --data-binary "SELECT 'firebolt-ready';" 2>/dev/null \
+            | grep -q 'firebolt-ready'; then
+        exit 0
+    fi
+    sleep 1
+done
+echo "firebolt-core did not become healthy in 10 min" >&2
+exit 1

firebolt-parquet/stop

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+# Plain stop — leave the container in place so its bind-mounted
+# fb-volume keeps the loaded database for the next ./start. The
+# container is removed and the volume re-initialised only on
+# explicit re-provision.
+sudo docker container stop firebolt-core >/dev/null 2>&1 || true

firebolt/benchmark.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+# Playground reads this line to pre-stage the dataset on the per-VM
+# system disk; the rest of benchmark.sh is unchanged.
+export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
+
 # Download the hits.parquet file
 echo "Downloading dataset..."
 rm -rf data

firebolt/check

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+# Firebolt-core's HTTP port answers immediately but may return a
+# cluster-not-ready JSON error at HTTP 200. Test for an actual result.
+curl -sSf --max-time 5 'http://localhost:3473/' \
+    --data-binary 'SELECT 1;' 2>/dev/null | grep -q '^1'

firebolt/install

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -eu
+
+sudo apt-get update -y
+sudo apt-get install -y docker.io jq
+sudo docker pull ghcr.io/firebolt-db/firebolt-core:preview-rc

firebolt/load

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eu
+
+# Stage hits.parquet where the container can see it (./data is
+# bind-mounted as /firebolt-core/clickbench).
+mkdir -p data
+if [ -f hits.parquet ]; then
+    mv -f hits.parquet data/hits.parquet
+fi
+
+# create.sql CREATEs hits_external pointing at the parquet file, then
+# INSERTs into the managed `hits` table — the ingested-to-Firebolt
+# variant of the benchmark.
+curl -sSf 'http://localhost:3473/?enable_multi_query_requests=true' \
+    --data-binary 'DROP DATABASE IF EXISTS clickbench;CREATE DATABASE clickbench;'
+curl -sSf 'http://localhost:3473/?database=clickbench&enable_multi_query_requests=true' \
+    --data-binary @create.sql
+
+rm -f data/hits.parquet
+sync

firebolt/query

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Reads a SQL query from stdin, runs it against the firebolt-core
+# container via /?database=clickbench.
+# Stdout: query result (firebolt's JSON_Compact format).
+# Stderr: query runtime in fractional seconds on the last line,
+#         pulled from the response's `.statistics.elapsed`.
+# Exit non-zero on error.
+set -e
+
+query=$(cat)
+
+# Result + sub-result caches off so timings are real; output_format
+# matches what firebolt's run.sh uses for the public benchmark.
+PARAMS='database=clickbench&enable_result_cache=false&enable_subresult_cache=false&enable_scan_cache=false&output_format=JSON_Compact'
+
+resp=$(curl -sS --max-time 600 "http://localhost:3473/?${PARAMS}" \
+            --data-binary "$query")
+
+# Firebolt returns a JSON object whether the query succeeded or not.
+# A failed query has an "errors" key; a successful one carries
+# "data" + "statistics".
+if printf '%s' "$resp" | jq -e '.errors' >/dev/null 2>&1; then
+    printf '%s\n' "$resp" >&2
+    exit 1
+fi
+
+printf '%s\n' "$resp"
+printf '%s\n' "$resp" | jq -r '.statistics.elapsed' >&2