diff --git a/clickhouse-datalake-partitioned/benchmark.sh b/clickhouse-datalake-partitioned/benchmark.sh index 015f8d7fb..c4fec4d80 100755 --- a/clickhouse-datalake-partitioned/benchmark.sh +++ b/clickhouse-datalake-partitioned/benchmark.sh @@ -6,11 +6,9 @@ curl https://clickhouse.com/ | sh # Configure +RAM=$(awk '/MemTotal/ {print int($2 * 0.8 * 1024)}' /proc/meminfo) > clickhouse-local.yaml echo " -filesystem_caches: - cache: - path: '/dev/shm/clickhouse/' - max_size_ratio_to_total_space: 0.9 +page_cache_max_size: ${RAM} " # Run the queries diff --git a/clickhouse-datalake-partitioned/results/c6a.4xlarge.json b/clickhouse-datalake-partitioned/results/c6a.4xlarge.json index fed6f66d1..f6c589ef5 100644 --- a/clickhouse-datalake-partitioned/results/c6a.4xlarge.json +++ b/clickhouse-datalake-partitioned/results/c6a.4xlarge.json @@ -10,49 +10,49 @@ "load_time": 0, "data_size": 14737666736, "result": [ - [0.491, 0.071, 0.054], - [1.445, 0.07, 0.077], - [1.759, 0.122, 0.123], - [1.429, 0.137, 0.127], - [1.665, 0.523, 0.495], - [1.698, 0.749, 0.771], - [1.271, 0.085, 0.075], - [1.194, 0.069, 0.068], - [1.718, 0.602, 0.558], - [2.909, 0.74, 0.706], - [2.373, 0.283, 0.253], - [2.626, 0.246, 0.245], - [1.752, 0.815, 0.779], - [2.837, 1.141, 1.087], - [1.833, 0.851, 0.88], - [1.753, 0.554, 0.606], - [3.427, 2.075, 1.998], - [2.64, 1.397, 1.327], - [5.687, 4.094, 4.26], - [1.196, 0.118, 0.122], - [3.464, 1.384, 1.23], - [4.86, 1.48, 1.563], - [6.593, 3.248, 1.799], - [11.428, 3.734, 3.393], - [2.655, 0.477, 0.443], - [1.467, 0.287, 0.296], - [2.668, 0.403, 0.439], - [4.053, 1.289, 1.365], - [10.85, 9.879, 9.658], - [1.288, 0.092, 0.094], - [2.923, 0.712, 0.76], - [4.339, 1.332, 1.29], - [6.806, 5.284, 5.073], - [5.274, 3.503, 3.371], - [5.07, 3.499, 3.541], - [1.522, 0.433, 0.419], - [0.67, 0.131, 0.128], - [0.602, 0.089, 0.104], - [0.684, 0.095, 0.095], - [0.846, 0.241, 0.232], - [0.55, 0.075, 0.074], - [0.601, 0.085, 0.075], - [0.506, 0.077, 0.078] + [0.735, 0.026, 0.03], + [2.043, 0.05, 0.041], + [2.561, 0.086, 0.087], + [2.318, 0.112, 0.114], + [1.694, 0.454, 0.442], + [1.833, 0.704, 0.674], + [0.927, 0.05, 0.062], + [0.777, 0.048, 0.046], + [1.781, 0.526, 0.539], + [3.018, 0.648, 0.692], + [2.008, 0.192, 0.194], + [2.053, 0.185, 0.201], + [1.896, 0.675, 0.655], + [3.176, 1.002, 1.05], + [1.898, 0.8, 0.76], + [1.65, 0.5, 0.497], + [3.617, 1.9, 1.875], + [2.9, 1.27, 1.164], + [5.745, 3.611, 3.485], + [1.114, 0.099, 0.103], + [5.908, 1.241, 1.198], + [7.771, 1.348, 1.295], + [13.155, 1.738, 1.733], + [25.514, 3.222, 3.279], + [3.001, 0.374, 0.376], + [1.626, 0.279, 0.278], + [2.946, 0.394, 0.377], + [7.483, 1.15, 1.166], + [10.901, 9.671, 9.802], + [0.899, 0.09, 0.065], + [3.316, 0.682, 0.665], + [5.545, 1.116, 1.209], + [6.813, 4.668, 4.679], + [9.027, 3.308, 3.271], + [7.841, 3.198, 3.235], + [1.272, 0.347, 0.352], + [0.89, 0.103, 0.094], + [0.635, 0.07, 0.065], + [0.852, 0.066, 0.055], + [1.225, 0.176, 0.16], + [0.524, 0.053, 0.051], + [0.519, 0.05, 0.049], + [0.409, 0.046, 0.045] ] } diff --git a/clickhouse-datalake-partitioned/results/c6a.metal.json b/clickhouse-datalake-partitioned/results/c6a.metal.json index 554d47cb4..3b3589120 100644 --- a/clickhouse-datalake-partitioned/results/c6a.metal.json +++ b/clickhouse-datalake-partitioned/results/c6a.metal.json @@ -10,49 +10,49 @@ "load_time": 0, "data_size": 14737666736, "result": [ - [0.254, 0.09, 0.109], - [0.503, 0.132, 0.142], - [0.69, 0.17, 0.174], - [0.881, 0.179, 0.173], - [0.74, 0.315, 0.324], - [0.776, 0.349, 0.343], - [0.527, 0.124, 0.134], - [0.567, 0.126, 0.132], - [1.181, 0.444, 0.491], - [1.291, 0.486, 0.491], - [0.867, 0.253, 0.244], - [0.902, 0.257, 0.255], - [1.095, 0.331, 0.34], - [1.159, 0.404, 0.395], - [0.861, 0.331, 0.337], - [1.055, 0.256, 0.242], - [1.267, 0.581, 0.539], - [1.157, 0.492, 0.466], - [1.77, 0.978, 0.943], - [0.517, 0.158, 0.172], - [1.907, 0.568, 0.559], - [2.403, 0.568, 0.525], - [2.871, 0.654, 0.731], - [4.332, 1.304, 1.232], - [0.99, 0.269, 0.254], - [0.668, 0.222, 0.23], - [1.02, 0.256, 0.256], - [2.078, 0.49, 0.534], - [3.355, 2.335, 2.517], - [0.574, 0.168, 0.177], - [1.151, 0.31, 0.32], - [1.529, 0.444, 0.466], - [2.261, 1.362, 1.59], - [2.012, 0.973, 0.982], - [2.522, 1.002, 0.953], - [0.642, 0.203, 0.246], - [0.671, 0.177, 0.167], - [0.477, 0.146, 0.156], - [0.605, 0.14, 0.141], - [0.768, 0.255, 0.22], - [0.524, 0.121, 0.135], - [0.518, 0.132, 0.139], - [0.463, 0.116, 0.105] + [0.206, 0.032, 0.033], + [0.354, 0.052, 0.057], + [0.739, 0.069, 0.077], + [0.519, 0.077, 0.091], + [0.779, 0.245, 0.249], + [0.916, 0.242, 0.242], + [0.404, 0.056, 0.061], + [0.363, 0.06, 0.056], + [0.903, 0.363, 0.342], + [1.343, 0.43, 0.417], + [0.896, 0.15, 0.152], + [0.816, 0.147, 0.147], + [1, 0.233, 0.222], + [1.358, 0.339, 0.315], + [0.9, 0.247, 0.232], + [0.688, 0.171, 0.155], + [1.549, 0.557, 0.523], + [1.47, 0.404, 0.37], + [2.127, 1.004, 0.892], + [0.475, 0.069, 0.08], + [4.918, 0.391, 0.43], + [4.121, 0.468, 0.45], + [5.851, 0.596, 0.515], + [10.18, 1, 1.049], + [1.14, 0.152, 0.149], + [0.882, 0.13, 0.128], + [1.148, 0.144, 0.157], + [3.152, 0.463, 0.445], + [4.234, 2.44, 2.224], + [0.429, 0.085, 0.085], + [1.544, 0.305, 0.287], + [2.054, 0.472, 0.417], + [2.403, 1.446, 1.188], + [4.371, 1.006, 0.939], + [3.481, 1.039, 0.863], + [0.581, 0.128, 0.12], + [0.931, 0.11, 0.126], + [0.567, 0.076, 0.087], + [0.918, 0.078, 0.069], + [1.224, 0.16, 0.139], + [0.555, 0.063, 0.06], + [0.633, 0.057, 0.06], + [0.48, 0.051, 0.049] ] } diff --git a/clickhouse-datalake-partitioned/results/c7a.metal-48xl.json b/clickhouse-datalake-partitioned/results/c7a.metal-48xl.json index 0de2895b0..1b7463c91 100644 --- a/clickhouse-datalake-partitioned/results/c7a.metal-48xl.json +++ b/clickhouse-datalake-partitioned/results/c7a.metal-48xl.json @@ -10,49 +10,49 @@ "load_time": 0, "data_size": 14737666736, "result": [ - [0.252, 0.13, 0.107], - [0.404, 0.133, 0.145], - [0.889, 0.161, 0.163], - [0.854, 0.16, 0.156], - [1.021, 0.506, 0.515], - [0.829, 0.521, 0.511], - [0.806, 0.153, 0.124], - [0.495, 0.159, 0.134], - [1.126, 0.452, 0.453], - [1.144, 0.469, 0.493], - [1.202, 0.27, 0.261], - [0.83, 0.277, 0.252], - [0.962, 0.306, 0.306], - [0.952, 0.361, 0.366], - [0.662, 0.314, 0.281], - [0.914, 0.246, 0.246], - [1.18, 0.435, 0.452], - [0.994, 0.42, 0.44], - [1.199, 0.814, 0.772], - [0.939, 0.164, 0.151], - [1.437, 0.554, 0.561], - [1.953, 0.551, 0.568], - [2.506, 0.78, 0.771], - [4.011, 1.564, 1.529], - [1.058, 0.266, 0.268], - [0.607, 0.233, 0.225], - [1.097, 0.263, 0.274], - [2.005, 0.572, 0.556], - [2.917, 1.779, 1.757], - [0.523, 0.193, 0.198], - [1.043, 0.342, 0.335], - [1.384, 0.429, 0.428], - [1.555, 1.018, 0.897], - [2.017, 0.902, 0.949], - [2.041, 0.823, 0.795], - [0.554, 0.219, 0.21], - [0.614, 0.184, 0.177], - [0.545, 0.146, 0.144], - [0.627, 0.137, 0.138], - [0.767, 0.243, 0.243], - [0.541, 0.134, 0.147], - [0.567, 0.139, 0.131], - [0.485, 0.123, 0.113] + [0.22, 0.034, 0.036], + [0.343, 0.061, 0.059], + [0.628, 0.067, 0.069], + [0.497, 0.069, 0.069], + [0.861, 0.365, 0.346], + [1.02, 0.406, 0.408], + [0.309, 0.063, 0.066], + [0.295, 0.07, 0.069], + [0.911, 0.332, 0.336], + [1.204, 0.485, 0.33], + [0.762, 0.178, 0.155], + [0.838, 0.151, 0.148], + [0.799, 0.155, 0.159], + [1.007, 0.2, 0.189], + [0.837, 0.174, 0.185], + [0.592, 0.111, 0.133], + [1.18, 0.272, 0.244], + [1.054, 0.265, 0.191], + [1.755, 0.502, 0.444], + [0.402, 0.057, 0.064], + [3.648, 0.393, 0.369], + [3.666, 0.304, 0.295], + [4.671, 0.384, 0.357], + [9.166, 0.885, 0.849], + [0.981, 0.13, 0.123], + [0.783, 0.128, 0.133], + [0.876, 0.134, 0.128], + [3.012, 0.348, 0.299], + [3.302, 1.924, 1.988], + [0.411, 0.106, 0.106], + [1.204, 0.176, 0.169], + [1.647, 0.178, 0.227], + [1.824, 0.714, 0.58], + [3.161, 0.736, 0.499], + [3.468, 0.679, 0.456], + [0.524, 0.097, 0.093], + [0.97, 0.12, 0.119], + [0.579, 0.099, 0.093], + [0.948, 0.083, 0.085], + [1.234, 0.174, 0.163], + [0.613, 0.072, 0.074], + [0.519, 0.066, 0.066], + [0.508, 0.054, 0.054] ] } diff --git a/clickhouse-datalake-partitioned/results/c8g.4xlarge.json b/clickhouse-datalake-partitioned/results/c8g.4xlarge.json index 625fd3619..a57c170bb 100644 --- a/clickhouse-datalake-partitioned/results/c8g.4xlarge.json +++ b/clickhouse-datalake-partitioned/results/c8g.4xlarge.json @@ -10,49 +10,49 @@ "load_time": 0, "data_size": 14737666736, "result": [ - [0.302, 0.045, 0.043], - [0.948, 0.059, 0.063], - [1.365, 0.091, 0.093], - [1.309, 0.077, 0.077], - [1.291, 0.223, 0.231], - [1.352, 0.365, 0.352], - [1.034, 0.062, 0.06], - [0.948, 0.072, 0.065], - [1.339, 0.299, 0.286], - [2.378, 0.359, 0.351], - [1.92, 0.173, 0.164], - [1.917, 0.161, 0.163], - [1.323, 0.369, 0.332], - [2.251, 0.485, 0.51], - [1.391, 0.384, 0.398], - [1.271, 0.269, 0.268], - [2.364, 0.758, 0.85], - [2.058, 0.574, 0.654], - [2.985, 1.418, 1.425], - [0.997, 0.081, 0.099], - [2.89, 0.537, 0.531], - [4.081, 0.573, 0.571], - [6.185, 3.026, 0.798], - [10.702, 2.978, 1.503], - [2.166, 0.229, 0.229], - [1.242, 0.168, 0.175], - [2.226, 0.224, 0.229], - [3.308, 0.46, 0.451], - [6.299, 5.375, 5.449], - [1.065, 0.086, 0.081], - [2.394, 0.339, 0.325], - [3.412, 0.457, 0.455], - [3.672, 1.632, 1.61], - [3.916, 1.247, 1.264], - [3.519, 1.281, 1.262], - [1.231, 0.218, 0.215], - [0.575, 0.101, 0.1], - [0.509, 0.076, 0.076], - [0.542, 0.077, 0.08], - [0.703, 0.155, 0.148], - [0.43, 0.074, 0.066], - [0.456, 0.063, 0.065], - [0.385, 0.053, 0.053] + [0.257, 0.022, 0.025], + [0.73, 0.039, 0.039], + [1.718, 0.055, 0.053], + [1.201, 0.049, 0.049], + [1.439, 0.185, 0.182], + [1.789, 0.325, 0.324], + [0.795, 0.039, 0.04], + [0.757, 0.043, 0.039], + [1.587, 0.261, 0.252], + [2.94, 0.306, 0.299], + [1.961, 0.131, 0.132], + [1.99, 0.155, 0.133], + [1.711, 0.3, 0.32], + [2.696, 0.417, 0.423], + [1.667, 0.321, 0.316], + [1.394, 0.174, 0.177], + [2.948, 0.64, 0.712], + [2.629, 0.563, 0.535], + [3.966, 1.187, 1.136], + [1.083, 0.047, 0.048], + [6.195, 0.419, 0.423], + [8.158, 0.514, 0.473], + [12.932, 0.697, 0.676], + [24.369, 1.298, 2.884], + [2.955, 0.184, 0.184], + [1.527, 0.132, 0.132], + [2.981, 0.188, 0.189], + [6.828, 0.375, 0.347], + [7.734, 5.315, 5.258], + [0.84, 0.059, 0.057], + [3.185, 0.267, 0.257], + [5.219, 0.366, 0.37], + [4.634, 1.426, 1.383], + [6.808, 1.147, 1.098], + [7.537, 1.14, 1.092], + [1.272, 0.151, 0.172], + [0.89, 0.071, 0.074], + [0.569, 0.059, 0.051], + [0.845, 0.054, 0.053], + [1.213, 0.132, 0.119], + [0.477, 0.048, 0.044], + [0.452, 0.044, 0.043], + [0.411, 0.036, 0.038] ] } diff --git a/clickhouse-datalake-partitioned/results/c8g.metal-48xl.json b/clickhouse-datalake-partitioned/results/c8g.metal-48xl.json index b02709be0..b74b965c1 100644 --- a/clickhouse-datalake-partitioned/results/c8g.metal-48xl.json +++ b/clickhouse-datalake-partitioned/results/c8g.metal-48xl.json @@ -10,49 +10,49 @@ "load_time": 0, "data_size": 14737666736, "result": [ - [0.279, 0.107, 0.094], - [0.819, 0.128, 0.126], - [0.834, 0.127, 0.128], - [0.593, 0.136, 0.124], - [1.085, 0.281, 0.297], - [0.995, 0.321, 0.315], - [0.858, 0.107, 0.132], - [0.439, 0.127, 0.132], - [1.037, 0.34, 0.363], - [1.31, 0.401, 0.393], - [0.745, 0.191, 0.21], - [0.765, 0.185, 0.202], - [0.591, 0.24, 0.235], - [0.957, 0.295, 0.284], - [0.626, 0.252, 0.245], - [0.587, 0.214, 0.22], - [0.979, 0.407, 0.395], - [1.072, 0.378, 0.344], - [1.131, 0.686, 0.733], - [0.485, 0.128, 0.131], - [1.746, 0.407, 0.391], - [2.055, 0.466, 0.486], - [2.473, 0.534, 0.567], - [4.682, 1.232, 1.229], - [1.137, 0.211, 0.199], - [0.902, 0.182, 0.173], - [0.89, 0.208, 0.213], - [1.793, 0.397, 0.44], - [2.815, 2.145, 2.044], - [0.591, 0.201, 0.189], - [1.101, 0.252, 0.27], - [1.293, 0.326, 0.304], - [1.506, 0.914, 0.884], - [2.137, 0.658, 0.659], - [1.927, 0.711, 0.663], - [0.956, 0.176, 0.178], - [0.608, 0.163, 0.181], - [0.432, 0.131, 0.133], - [0.537, 0.146, 0.122], - [0.698, 0.169, 0.189], - [0.453, 0.11, 0.121], - [0.479, 0.125, 0.135], - [0.414, 0.112, 0.124] + [0.166, 0.024, 0.025], + [0.306, 0.055, 0.044], + [0.619, 0.049, 0.055], + [0.479, 0.04, 0.048], + [0.638, 0.19, 0.261], + [0.93, 0.333, 0.349], + [0.406, 0.06, 0.056], + [0.355, 0.049, 0.05], + [0.714, 0.25, 0.254], + [1.135, 0.251, 0.267], + [0.792, 0.117, 0.182], + [0.716, 0.108, 0.123], + [0.766, 0.116, 0.108], + [1.019, 0.154, 0.15], + [0.77, 0.129, 0.125], + [0.603, 0.092, 0.093], + [1.16, 0.247, 0.225], + [1.076, 0.225, 0.18], + [1.545, 0.48, 0.482], + [0.478, 0.05, 0.054], + [3.732, 0.256, 0.247], + [3.642, 0.294, 0.278], + [4.271, 0.287, 0.329], + [9.578, 0.943, 0.837], + [1.098, 0.093, 0.095], + [0.75, 0.072, 0.082], + [0.994, 0.095, 0.092], + [3.653, 0.244, 0.239], + [3.664, 1.989, 1.874], + [0.412, 0.113, 0.118], + [1.234, 0.12, 0.119], + [1.671, 0.149, 0.143], + [1.746, 0.505, 0.458], + [3.741, 0.392, 0.359], + [3.597, 0.444, 0.393], + [0.504, 0.09, 0.093], + [0.834, 0.085, 0.088], + [0.557, 0.064, 0.066], + [0.937, 0.059, 0.06], + [1.155, 0.146, 0.144], + [0.575, 0.056, 0.057], + [0.573, 0.052, 0.054], + [0.458, 0.043, 0.048] ] } diff --git a/clickhouse-datalake-partitioned/run.sh b/clickhouse-datalake-partitioned/run.sh index 6f98bf537..5e0d4ac11 100755 --- a/clickhouse-datalake-partitioned/run.sh +++ b/clickhouse-datalake-partitioned/run.sh @@ -5,21 +5,16 @@ QUERY_NUM=1 ./clickhouse local --path . --query="$(cat create.sql)" cat queries.sql | while read -r query; do - rm -rf /dev/shm/clickhouse/* - sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null - echo -n "[" - for i in $(seq 1 $TRIES); do - RES=$(./clickhouse local --path . --time --format Null --filesystem_cache_name cache --query="$query" 2>&1) # (*) - [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null" - [[ "$i" != $TRIES ]] && echo -n ", " - - echo "${QUERY_NUM},${i},${RES}" >> result.csv - done + i=0 + while read -r RES; do + [[ "$i" -gt 0 ]] && echo -n ", " + [[ "$RES" =~ ^[0-9] ]] && echo -n "${RES}" || echo -n "null" + echo "${QUERY_NUM},$((i+1)),${RES}" >> result.csv + i=$((i+1)) + done <<< "$(./clickhouse local --path . --time --format Null --use_page_cache_for_object_storage 1 --query "$query; $query; $query" 2>&1)" echo "]," - # (*) --format=Null is client-side formatting. The query result is still sent back to the client. - QUERY_NUM=$((QUERY_NUM + 1)) done ./clickhouse local --path . --query="DROP TABLE hits" diff --git a/clickhouse-datalake/benchmark.sh b/clickhouse-datalake/benchmark.sh index 0d8cf4905..8b4d0718d 100755 --- a/clickhouse-datalake/benchmark.sh +++ b/clickhouse-datalake/benchmark.sh @@ -6,11 +6,9 @@ curl https://clickhouse.com/ | sh # Configure +RAM=$(awk '/MemTotal/ {print int($2 * 0.8 * 1024)}' /proc/meminfo) > clickhouse-local.yaml echo " -filesystem_caches: - cache: - path: '/dev/shm/clickhouse/' - max_size_ratio_to_total_space: 0.9 +page_cache_max_size: ${RAM} " # Run the queries diff --git a/clickhouse-datalake/results/c6a.4xlarge.json b/clickhouse-datalake/results/c6a.4xlarge.json index b331aaf64..1e67968ac 100644 --- a/clickhouse-datalake/results/c6a.4xlarge.json +++ b/clickhouse-datalake/results/c6a.4xlarge.json @@ -10,49 +10,49 @@ "load_time": 0, "data_size": 14779976446, "result": [ - [0.308, 0.158, 0.157], - [0.978, 0.12, 0.132], - [1.886, 0.199, 0.2], - [1.22, 0.172, 0.161], - [1.499, 0.504, 0.516], - [1.761, 0.762, 0.76], - [1.064, 0.108, 0.143], - [1.062, 0.121, 0.115], - [1.849, 0.651, 0.604], - [5.319, 0.711, 0.716], - [2.014, 0.314, 0.284], - [2.04, 0.322, 0.322], - [1.791, 0.723, 0.74], - [2.757, 1.064, 1.045], - [1.801, 0.821, 0.806], - [1.493, 0.577, 0.606], - [3.597, 1.936, 1.99], - [2.736, 1.275, 1.269], - [5.728, 3.882, 3.959], - [1.093, 0.157, 0.131], - [4.292, 1.248, 1.205], - [5.086, 1.436, 1.397], - [7.325, 4.085, 2.06], - [19.384, 8.33, 4.275], - [2.376, 0.534, 0.522], - [1.39, 0.318, 0.306], - [2.391, 0.544, 0.533], - [5.92, 1.248, 1.271], - [10.461, 9.505, 9.479], - [1.056, 0.128, 0.132], - [6.651, 0.727, 0.727], - [8.124, 1.132, 1.131], - [8.829, 4.827, 4.788], - [5.806, 3.292, 3.302], - [5.884, 3.245, 3.282], - [1.305, 0.433, 0.436], - [0.671, 0.157, 0.173], - [0.564, 0.131, 0.126], - [0.652, 0.115, 0.131], - [0.788, 0.184, 0.197], - [0.545, 0.11, 0.11], - [0.548, 0.099, 0.121], - [0.489, 0.091, 0.099] + [0.29, 0.031, 0.036], + [0.732, 0.052, 0.043], + [3.224, 0.109, 0.098], + [1.209, 0.087, 0.08], + [1.452, 0.415, 0.412], + [2.076, 0.66, 0.654], + [0.643, 0.055, 0.051], + [0.591, 0.053, 0.05], + [2.215, 0.506, 0.503], + [8.281, 0.6, 0.6], + [1.94, 0.219, 0.213], + [2.088, 0.233, 0.211], + [2.06, 0.626, 0.637], + [3.11, 0.929, 0.928], + [2.151, 0.709, 0.698], + [1.466, 0.484, 0.474], + [4.205, 1.789, 1.766], + [3.518, 1.119, 1.109], + [7.405, 3.24, 3.192], + [1.043, 0.079, 0.08], + [8.895, 1.164, 1.14], + [9.657, 1.346, 1.278], + [15.3, 2.013, 1.968], + [39.549, 4.17, 4.08], + [2.961, 0.462, 0.459], + [1.811, 0.25, 0.248], + [2.871, 0.461, 0.458], + [13.158, 1.196, 1.156], + [10.657, 9.335, 9.329], + [0.881, 0.08, 0.076], + [12.057, 0.629, 0.596], + [15.104, 0.926, 0.909], + [12.291, 4.576, 4.524], + [10.187, 3.182, 3.172], + [9.904, 3.174, 3.162], + [1.191, 0.303, 0.298], + [0.853, 0.095, 0.093], + [0.653, 0.079, 0.069], + [0.817, 0.071, 0.057], + [1.324, 0.115, 0.11], + [0.526, 0.059, 0.062], + [0.484, 0.051, 0.051], + [0.36, 0.041, 0.041] ] } diff --git a/clickhouse-datalake/results/c6a.metal.json b/clickhouse-datalake/results/c6a.metal.json index d5751d6ac..4491168c8 100644 --- a/clickhouse-datalake/results/c6a.metal.json +++ b/clickhouse-datalake/results/c6a.metal.json @@ -10,49 +10,49 @@ "load_time": 0, "data_size": 14779976446, "result": [ - [0.323, 0.161, 0.139], - [0.532, 0.136, 0.176], - [1.001, 0.208, 0.182], - [0.654, 0.179, 0.17], - [1.166, 0.324, 0.347], - [0.887, 0.35, 0.333], - [0.688, 0.165, 0.147], - [0.515, 0.147, 0.148], - [1.048, 0.493, 0.478], - [1.411, 0.523, 0.53], - [0.857, 0.273, 0.281], - [1.118, 0.276, 0.271], - [1.118, 0.323, 0.309], - [1.069, 0.37, 0.363], - [0.759, 0.311, 0.299], - [0.808, 0.244, 0.267], - [1.204, 0.521, 0.504], - [1.059, 0.415, 0.406], - [1.657, 1.036, 1.008], - [0.664, 0.153, 0.202], - [2.095, 0.586, 0.813], - [2.63, 1.314, 1.238], - [5.049, 1.411, 1.611], - [14.009, 6.419, 6.464], - [0.957, 0.241, 0.228], - [0.991, 0.191, 0.224], - [0.896, 0.228, 0.239], - [2.356, 0.573, 0.584], - [1.896, 1.412, 1.436], - [0.55, 0.158, 0.178], - [1.255, 0.428, 0.466], - [1.644, 0.58, 0.509], - [2.034, 1.126, 1.127], - [2.437, 0.973, 0.977], - [2.398, 0.983, 0.936], - [0.985, 0.223, 0.223], - [0.65, 0.196, 0.189], - [0.571, 0.18, 0.159], - [0.643, 0.168, 0.159], - [0.782, 0.206, 0.232], - [0.554, 0.114, 0.112], - [0.552, 0.115, 0.13], - [0.484, 0.124, 0.102] + [0.262, 0.028, 0.028], + [0.35, 0.068, 0.066], + [0.883, 0.091, 0.085], + [0.502, 0.091, 0.083], + [0.698, 0.264, 0.256], + [0.864, 0.247, 0.223], + [0.38, 0.074, 0.068], + [0.358, 0.071, 0.07], + [0.836, 0.397, 0.382], + [1.522, 0.446, 0.415], + [0.662, 0.176, 0.163], + [0.659, 0.16, 0.161], + [0.721, 0.235, 0.224], + [1.028, 0.29, 0.277], + [0.768, 0.219, 0.214], + [0.595, 0.179, 0.164], + [1.198, 0.492, 0.436], + [1.04, 0.345, 0.306], + [1.915, 0.77, 0.687], + [0.49, 0.087, 0.09], + [3.862, 0.701, 0.494], + [4.964, 0.963, 0.903], + [10.243, 1.384, 1.325], + [32.511, 6.553, 6.646], + [0.918, 0.177, 0.164], + [0.661, 0.143, 0.141], + [0.878, 0.179, 0.163], + [4.264, 0.563, 0.558], + [2.414, 1.269, 1.249], + [0.42, 0.098, 0.093], + [2.201, 0.37, 0.363], + [2.249, 0.501, 0.444], + [2.118, 1.271, 1.029], + [4.223, 1.069, 0.932], + [4.071, 0.982, 1.065], + [0.513, 0.135, 0.135], + [0.918, 0.118, 0.109], + [0.67, 0.087, 0.083], + [0.849, 0.095, 0.082], + [1.315, 0.126, 0.118], + [0.5, 0.077, 0.068], + [0.491, 0.053, 0.061], + [0.368, 0.047, 0.051] ] } diff --git a/clickhouse-datalake/results/c7a.metal-48xl.json b/clickhouse-datalake/results/c7a.metal-48xl.json index 3b1817b4e..1ab89f41e 100644 --- a/clickhouse-datalake/results/c7a.metal-48xl.json +++ b/clickhouse-datalake/results/c7a.metal-48xl.json @@ -10,49 +10,49 @@ "load_time": 0, "data_size": 14779976446, "result": [ - [0.323, 0.12, 0.144], - [0.551, 0.149, 0.13], - [0.631, 0.171, 0.165], - [0.56, 0.142, 0.149], - [0.864, 0.495, 0.505], - [0.893, 0.528, 0.53], - [0.596, 0.162, 0.127], - [1.013, 0.122, 0.144], - [0.821, 0.417, 0.459], - [1.18, 0.5, 0.492], - [1.239, 0.296, 0.328], - [1.207, 0.238, 0.25], - [0.755, 0.314, 0.335], - [1.258, 0.363, 0.372], - [1.033, 0.29, 0.287], - [1.014, 0.26, 0.258], - [1.364, 0.424, 0.386], - [1.133, 0.348, 0.362], - [1.286, 0.559, 0.575], - [0.674, 0.141, 0.141], - [1.893, 0.548, 0.516], - [2.625, 1.348, 1.504], - [4.723, 1.889, 1.872], - [17.403, 9.746, 9.799], - [1.135, 0.351, 0.254], - [0.989, 0.206, 0.206], - [0.801, 0.277, 0.284], - [2.186, 0.782, 0.811], - [1.427, 0.968, 3.786], - [0.544, 0.169, 0.173], - [1.086, 0.614, 0.683], - [1.297, 0.682, 0.69], - [1.51, 0.853, 0.861], - [2.114, 0.874, 0.835], - [2.102, 0.858, 0.895], - [0.554, 0.22, 0.207], - [0.68, 0.202, 0.177], - [0.555, 0.174, 0.172], - [0.646, 0.159, 0.14], - [0.786, 0.209, 0.226], - [0.584, 0.134, 0.126], - [0.549, 0.145, 0.148], - [0.475, 0.104, 0.112] + [0.225, 0.056, 0.03], + [0.384, 0.064, 0.064], + [0.59, 0.069, 0.069], + [0.436, 0.064, 0.068], + [0.814, 0.35, 0.347], + [0.862, 0.375, 0.394], + [0.366, 0.064, 0.064], + [0.335, 0.074, 0.069], + [0.707, 0.334, 0.323], + [1.175, 0.358, 0.341], + [0.654, 0.175, 0.173], + [0.729, 0.166, 0.175], + [0.695, 0.153, 0.16], + [0.836, 0.248, 0.322], + [0.761, 0.191, 0.123], + [0.502, 0.113, 0.106], + [0.842, 0.191, 0.186], + [0.933, 0.183, 0.17], + [1.388, 1.053, 1.085], + [0.469, 0.06, 0.06], + [3.465, 0.37, 0.358], + [4.897, 1.579, 1.409], + [9.748, 1.816, 1.866], + [34.754, 8.215, 8.473], + [0.895, 0.262, 0.19], + [0.59, 0.099, 0.092], + [0.785, 0.259, 0.327], + [3.89, 0.639, 0.627], + [2.067, 0.728, 1.423], + [0.407, 0.106, 0.102], + [1.384, 0.541, 0.602], + [1.586, 0.605, 0.866], + [1.529, 0.526, 0.487], + [3.515, 0.614, 0.513], + [3.798, 2.647, 0.529], + [0.456, 0.109, 0.097], + [0.865, 0.128, 0.13], + [0.664, 0.104, 0.114], + [0.849, 0.084, 0.08], + [1.424, 0.142, 0.152], + [0.582, 0.082, 0.095], + [0.519, 0.063, 0.062], + [0.416, 0.055, 0.051] ] } diff --git a/clickhouse-datalake/results/c8g.4xlarge.json b/clickhouse-datalake/results/c8g.4xlarge.json index 40786d552..ba0bd280d 100644 --- a/clickhouse-datalake/results/c8g.4xlarge.json +++ b/clickhouse-datalake/results/c8g.4xlarge.json @@ -10,49 +10,49 @@ "load_time": 0, "data_size": 14779976446, "result": [ - [0.243, 0.111, 0.104], - [1.069, 0.103, 0.108], - [1.905, 0.161, 0.15], - [1.269, 0.13, 0.131], - [1.393, 0.253, 0.268], - [1.445, 0.385, 0.369], - [1.015, 0.084, 0.09], - [1.051, 0.103, 0.081], - [1.633, 0.324, 0.345], - [5.038, 0.41, 0.416], - [2.141, 0.239, 0.24], - [2.274, 0.257, 0.24], - [1.505, 0.376, 0.375], - [2.498, 0.572, 0.557], - [1.628, 0.441, 0.448], - [1.313, 0.279, 0.298], - [2.712, 0.79, 0.785], - [2.561, 0.595, 0.598], - [3.672, 1.475, 1.478], - [1.134, 0.118, 0.12], - [4.536, 0.698, 0.716], - [5.234, 0.976, 0.992], - [7.79, 4.054, 1.531], - [18.505, 7.871, 3.616], - [2.575, 0.387, 0.402], - [1.498, 0.239, 0.228], - [2.642, 0.373, 0.405], - [5.878, 0.872, 0.827], - [5.736, 5.008, 5.017], - [1.061, 0.112, 0.115], - [6.924, 0.515, 0.505], - [8.598, 0.604, 0.615], - [6.876, 1.664, 1.682], - [4.747, 1.371, 1.355], - [4.622, 1.369, 1.384], - [1.318, 0.235, 0.23], - [0.606, 0.138, 0.119], - [0.455, 0.104, 0.098], - [0.529, 0.1, 0.092], - [0.699, 0.139, 0.128], - [0.494, 0.114, 0.094], - [0.58, 0.083, 0.079], - [0.392, 0.079, 0.069] + [0.206, 0.029, 0.033], + [0.63, 0.044, 0.043], + [2.946, 0.079, 0.075], + [1.138, 0.059, 0.056], + [1.189, 0.174, 0.174], + [1.766, 0.288, 0.283], + [0.649, 0.045, 0.048], + [0.661, 0.046, 0.084], + [2.244, 0.255, 0.222], + [8.139, 0.322, 0.297], + [1.679, 0.167, 0.162], + [1.869, 0.179, 0.177], + [1.93, 0.292, 0.282], + [2.861, 0.43, 0.424], + [1.957, 0.356, 0.347], + [1.226, 0.168, 0.169], + [3.556, 0.611, 0.601], + [3.546, 0.438, 0.415], + [5.834, 1.075, 1.059], + [1.023, 0.051, 0.053], + [8.472, 0.61, 0.572], + [9.307, 0.884, 0.841], + [14.858, 1.371, 1.325], + [39.015, 2.797, 2.748], + [2.814, 0.31, 0.32], + [1.899, 0.18, 0.178], + [2.825, 0.312, 0.312], + [11.423, 0.737, 0.71], + [7.456, 4.876, 4.876], + [0.852, 0.069, 0.067], + [13.299, 0.398, 0.382], + [16.5, 0.451, 0.439], + [11.08, 1.363, 1.348], + [9.143, 1.161, 1.121], + [8.99, 1.144, 1.118], + [1.127, 0.145, 0.148], + [0.856, 0.072, 0.066], + [0.596, 0.057, 0.051], + [0.833, 0.053, 0.053], + [1.202, 0.079, 0.067], + [0.561, 0.049, 0.045], + [0.463, 0.044, 0.041], + [0.303, 0.035, 0.035] ] } diff --git a/clickhouse-datalake/results/c8g.metal-48xl.json b/clickhouse-datalake/results/c8g.metal-48xl.json index dbf19f412..0032e8fdb 100644 --- a/clickhouse-datalake/results/c8g.metal-48xl.json +++ b/clickhouse-datalake/results/c8g.metal-48xl.json @@ -10,49 +10,49 @@ "load_time": 0, "data_size": 14779976446, "result": [ - [0.224, 0.115, 0.111], - [0.474, 0.107, 0.122], - [0.904, 0.148, 0.163], - [0.928, 0.142, 0.13], - [1.071, 0.385, 0.409], - [0.696, 0.425, 0.432], - [0.418, 0.117, 0.114], - [0.953, 0.125, 0.156], - [0.706, 0.366, 0.374], - [1.009, 0.355, 0.361], - [1.141, 0.227, 0.233], - [0.807, 0.204, 0.191], - [1.01, 0.244, 0.24], - [1.239, 0.268, 0.312], - [1.041, 0.226, 0.251], - [0.563, 0.214, 0.215], - [1.23, 0.361, 0.372], - [0.997, 0.365, 0.34], - [1.021, 0.497, 0.519], - [0.908, 0.155, 0.114], - [1.636, 0.335, 0.381], - [2.658, 1.276, 1.446], - [4.934, 1.319, 1.423], - [15.502, 4.805, 4.755], - [1.134, 0.23, 0.253], - [0.657, 0.139, 0.176], - [1.101, 0.197, 0.257], - [2.021, 0.364, 0.415], - [1.158, 0.737, 0.731], - [0.955, 0.18, 0.177], - [0.912, 0.241, 0.254], - [1.183, 0.272, 0.304], - [1.428, 0.815, 0.917], - [1.952, 0.642, 0.661], - [1.876, 2.155, 1.642], - [0.97, 0.184, 0.189], - [0.579, 0.15, 0.151], - [0.452, 0.132, 0.139], - [0.552, 0.131, 0.126], - [0.684, 0.165, 0.149], - [0.439, 0.112, 0.123], - [0.451, 0.115, 0.127], - [0.413, 0.103, 0.094] + [0.24, 0.045, 0.027], + [0.312, 0.049, 0.048], + [0.66, 0.051, 0.051], + [0.465, 0.049, 0.047], + [0.614, 0.229, 0.323], + [0.703, 0.318, 0.3], + [0.286, 0.05, 0.051], + [0.302, 0.052, 0.066], + [0.693, 0.247, 0.25], + [1.117, 0.256, 0.258], + [0.658, 0.143, 0.133], + [0.603, 0.107, 0.108], + [0.6, 0.112, 0.1], + [0.844, 0.158, 0.141], + [0.684, 0.19, 0.123], + [0.53, 0.093, 0.083], + [0.794, 0.182, 0.172], + [0.812, 0.183, 0.166], + [1.361, 0.289, 0.361], + [0.475, 0.05, 0.05], + [3.188, 0.237, 0.215], + [4.911, 0.996, 1.11], + [9.871, 1.303, 1.366], + [32.249, 4.586, 4.806], + [0.786, 0.186, 0.224], + [0.569, 0.079, 0.065], + [0.843, 0.178, 0.18], + [3.925, 0.323, 0.264], + [1.89, 0.581, 0.575], + [0.433, 0.109, 0.107], + [1.334, 0.166, 0.155], + [1.547, 0.183, 0.261], + [1.562, 0.574, 0.517], + [3.488, 0.437, 0.392], + [3.537, 0.432, 0.501], + [0.437, 0.082, 0.077], + [0.86, 0.08, 0.085], + [0.608, 0.064, 0.064], + [0.835, 0.062, 0.063], + [1.234, 0.093, 0.082], + [0.529, 0.056, 0.056], + [0.478, 0.051, 0.052], + [0.355, 0.057, 0.044] ] } diff --git a/clickhouse-datalake/run.sh b/clickhouse-datalake/run.sh index 6f98bf537..5e0d4ac11 100755 --- a/clickhouse-datalake/run.sh +++ b/clickhouse-datalake/run.sh @@ -5,21 +5,16 @@ QUERY_NUM=1 ./clickhouse local --path . --query="$(cat create.sql)" cat queries.sql | while read -r query; do - rm -rf /dev/shm/clickhouse/* - sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null - echo -n "[" - for i in $(seq 1 $TRIES); do - RES=$(./clickhouse local --path . --time --format Null --filesystem_cache_name cache --query="$query" 2>&1) # (*) - [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null" - [[ "$i" != $TRIES ]] && echo -n ", " - - echo "${QUERY_NUM},${i},${RES}" >> result.csv - done + i=0 + while read -r RES; do + [[ "$i" -gt 0 ]] && echo -n ", " + [[ "$RES" =~ ^[0-9] ]] && echo -n "${RES}" || echo -n "null" + echo "${QUERY_NUM},$((i+1)),${RES}" >> result.csv + i=$((i+1)) + done <<< "$(./clickhouse local --path . --time --format Null --use_page_cache_for_object_storage 1 --query "$query; $query; $query" 2>&1)" echo "]," - # (*) --format=Null is client-side formatting. The query result is still sent back to the client. - QUERY_NUM=$((QUERY_NUM + 1)) done ./clickhouse local --path . --query="DROP TABLE hits"