diff --git a/sail-partitioned/install b/sail-partitioned/install index 78db67204..318bfadb9 100755 --- a/sail-partitioned/install +++ b/sail-partitioned/install @@ -49,5 +49,5 @@ source myenv/bin/activate # RECORD file, so `--upgrade` fails to uninstall it. `--ignore-installed` # avoids the uninstall and just lays a fresh copy in the venv. pip install --upgrade --ignore-installed setuptools wheel -pip install --no-cache-dir "pysail==0.6.5" +pip install --no-cache-dir "pysail==0.6.6" pip install "pyspark-client==4.1.1" pandas psutil diff --git a/sail-partitioned/queries.sql b/sail-partitioned/queries.sql index 47114dee7..7eabbd211 100644 --- a/sail-partitioned/queries.sql +++ b/sail-partitioned/queries.sql @@ -26,7 +26,7 @@ SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; -SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '$1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/sail-partitioned/query b/sail-partitioned/query index 06afad5d9..11a37f8de 100755 --- a/sail-partitioned/query +++ b/sail-partitioned/query @@ -24,7 +24,6 @@ import timeit os.environ["SAIL_PARQUET__BINARY_AS_STRING"] = "true" os.environ["SAIL_PARQUET__REORDER_FILTERS"] = "true" -os.environ["SAIL_OPTIMIZER__ENABLE_JOIN_REORDER"] = "true" from pysail.spark import SparkConnectServer from pyspark.sql import SparkSession @@ -40,7 +39,6 @@ _, port = server.listening_address spark = SparkSession.builder.remote(f"sc://localhost:{port}").getOrCreate() df = spark.read.parquet("partitioned") -df = df.withColumn("EventTime", F.col("EventTime").cast("timestamp")) df = df.withColumn("EventDate", F.col("EventDate").cast("int").cast("date")) df.createOrReplaceTempView("hits") diff --git a/sail-partitioned/results/20260720/c6a.2xlarge.json b/sail-partitioned/results/20260720/c6a.2xlarge.json new file mode 100644 index 000000000..cc4d5d8c2 --- /dev/null +++ b/sail-partitioned/results/20260720/c6a.2xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet, partitioned)", + "date": "2026-07-20", + "machine": "c6a.2xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 9, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.246, 0.063, 0.062], + [0.484, 0.095, 0.096], + [0.435, 0.167, 0.164], + [0.954, 0.166, 0.169], + [1.47, 1.173, 1.193], + [1.878, 1.236, 1.241], + [0.241, 0.066, 0.064], + [0.336, 0.099, 0.099], + [1.947, 1.509, 1.539], + [2.284, 1.623, 1.616], + [1.054, 0.371, 0.373], + [1.155, 0.412, 0.41], + [2.01, 1.227, 1.263], + [2.976, 1.61, 1.636], + [1.572, 1.196, 1.199], + [1.725, 1.389, 1.422], + [3.619, 2.809, 2.826], + [3.612, 2.813, 2.79], + [6.784, 5.478, 5.311], + [0.794, 0.153, 0.152], + [9.84, 1.484, 1.49], + [11.595, 1.794, 1.777], + [22.334, 3.26, 3.25], + [55.143, 44.328, 43.562], + [0.742, 0.177, 0.176], + [1.092, 0.453, 0.457], + [0.526, 0.183, 0.18], + [9.894, 1.936, 1.939], + [16.547, 15.73, 15.725], + [1.643, 1.423, 1.436], + [2.672, 1.139, 1.184], + [6.224, 1.141, 1.111], + [4.977, 3.956, 3.949], + [12.498, 6.562, 6.524], + [12.537, 6.537, 6.52], + [1.412, 1.134, 1.141], + [0.603, 0.204, 0.203], + [0.365, 0.117, 0.118], + [0.407, 0.132, 0.135], + [0.637, 0.347, 0.342], + [0.347, 0.09, 0.089], + [0.338, 0.086, 0.086], + [0.356, 0.084, 0.088] +] + } + \ No newline at end of file diff --git a/sail-partitioned/results/20260720/c6a.4xlarge.json b/sail-partitioned/results/20260720/c6a.4xlarge.json new file mode 100644 index 000000000..e7d75ce7a --- /dev/null +++ b/sail-partitioned/results/20260720/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet, partitioned)", + "date": "2026-07-20", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 20, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.231, 0.067, 0.063], + [0.29, 0.081, 0.082], + [0.33, 0.121, 0.121], + [0.851, 0.127, 0.126], + [1.134, 0.729, 0.728], + [1.53, 0.809, 0.801], + [0.248, 0.063, 0.065], + [0.312, 0.086, 0.087], + [1.533, 0.928, 0.973], + [1.798, 1.004, 0.94], + [0.935, 0.239, 0.247], + [1.027, 0.259, 0.26], + [1.612, 0.862, 0.857], + [3.081, 1.245, 1.196], + [1.579, 0.814, 0.822], + [1.441, 0.912, 0.892], + [3.449, 1.804, 1.825], + [3.567, 1.779, 1.807], + [6.124, 3.621, 3.662], + [0.592, 0.12, 0.121], + [10.439, 1.367, 1.356], + [11.591, 1.637, 1.633], + [22.37, 3.49, 3.523], + [54.802, 10.014, 9.958], + [0.45, 0.151, 0.153], + [1.093, 0.287, 0.29], + [0.409, 0.152, 0.154], + [9.929, 1.884, 1.928], + [9.272, 8.268, 8.258], + [0.978, 0.8, 0.786], + [2.521, 0.783, 0.728], + [6.236, 0.823, 0.809], + [4.924, 3.073, 3.05], + [11.713, 5.245, 5.264], + [11.685, 5.305, 5.278], + [0.927, 0.704, 0.691], + [0.467, 0.203, 0.2], + [0.374, 0.122, 0.117], + [0.433, 0.131, 0.129], + [0.651, 0.35, 0.349], + [0.35, 0.093, 0.091], + [0.337, 0.088, 0.088], + [0.359, 0.086, 0.087] +] + } + \ No newline at end of file diff --git a/sail-partitioned/results/20260720/c6a.metal.json b/sail-partitioned/results/20260720/c6a.metal.json new file mode 100644 index 000000000..99a08a995 --- /dev/null +++ b/sail-partitioned/results/20260720/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet, partitioned)", + "date": "2026-07-20", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 64, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.264, 0.063, 0.062], + [0.341, 0.11, 0.111], + [0.387, 0.09, 0.105], + [0.739, 0.113, 0.106], + [0.79, 0.269, 0.273], + [1.23, 0.356, 0.35], + [0.27, 0.063, 0.063], + [0.376, 0.132, 0.127], + [1.062, 0.337, 0.335], + [1.569, 0.478, 0.479], + [0.905, 0.251, 0.267], + [1.412, 0.269, 0.259], + [1.145, 0.38, 0.389], + [2.566, 0.711, 0.688], + [1.223, 0.373, 0.404], + [0.762, 0.327, 0.332], + [2.388, 0.68, 0.734], + [2.381, 0.669, 0.666], + [4.375, 1.327, 1.378], + [0.44, 0.11, 0.12], + [9.746, 0.435, 0.366], + [11.378, 0.471, 0.488], + [21.893, 0.723, 0.752], + [51.102, 2.053, 1.662], + [2.18, 0.126, 0.133], + [1.014, 0.176, 0.156], + [2.191, 0.119, 0.125], + [9.79, 0.471, 0.51], + [9.039, 1.611, 1.636], + [0.475, 0.267, 0.261], + [2.334, 0.315, 0.298], + [5.865, 0.427, 0.439], + [4.849, 1.57, 1.698], + [10.502, 1.769, 1.679], + [10.415, 1.665, 1.724], + [0.527, 0.267, 0.276], + [0.529, 0.248, 0.266], + [0.487, 0.165, 0.167], + [0.502, 0.178, 0.17], + [0.745, 0.382, 0.402], + [0.414, 0.136, 0.132], + [0.389, 0.141, 0.146], + [0.414, 0.123, 0.138] +] + } + \ No newline at end of file diff --git a/sail-partitioned/results/20260720/c6a.xlarge.json b/sail-partitioned/results/20260720/c6a.xlarge.json new file mode 100644 index 000000000..b70718f6d --- /dev/null +++ b/sail-partitioned/results/20260720/c6a.xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet, partitioned)", + "date": "2026-07-20", + "machine": "c6a.xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 4, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.23, 0.068, 0.067], + [0.369, 0.126, 0.13], + [0.577, 0.27, 0.272], + [0.989, 0.252, 0.254], + [2.671, 2.27, 2.265], + [2.739, 2.266, 2.266], + [0.249, 0.069, 0.067], + [0.406, 0.132, 0.133], + [3.392, 2.857, 2.879], + [3.657, 2.97, 2.926], + [1.194, 0.665, 0.666], + [1.322, 0.736, 0.737], + [2.711, 2.157, 2.202], + [3.486, 2.863, 3.195], + [2.794, 2.182, 2.18], + [2.958, 2.486, 2.552], + [5.773, 5.022, 5.054], + [5.932, 5.198, 5.116], + [10.794, 10.586, 10.654], + [0.667, 0.232, 0.232], + [9.854, 2.6, 2.618], + [11.589, 3.128, 3.111], + [22.34, 5.526, 5.463], + [55.439, 45.888, 45.573], + [0.586, 0.268, 0.255], + [1.276, 0.798, 0.793], + [0.706, 0.267, 0.266], + [9.882, 3.323, 3.347], + [31.866, 30.66, 30.748], + [2.95, 2.774, 2.742], + [2.915, 2.028, 2], + [6.25, 2.034, 2.007], + [7.142, 6.956, 7.013], + [60.528, 83.668, 74.888], + [30.824, 77.991, 70.608], + [2.417, 1.968, 1.983], + [0.499, 0.223, 0.235], + [0.403, 0.127, 0.121], + [0.457, 0.142, 0.137], + [0.699, 0.383, 0.382], + [0.372, 0.104, 0.101], + [0.348, 0.098, 0.093], + [0.364, 0.091, 0.095] +] + } + \ No newline at end of file diff --git a/sail-partitioned/results/20260720/c7a.metal-48xl.json b/sail-partitioned/results/20260720/c7a.metal-48xl.json new file mode 100644 index 000000000..8d2fe3165 --- /dev/null +++ b/sail-partitioned/results/20260720/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet, partitioned)", + "date": "2026-07-20", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 65, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.234, 0.058, 0.057], + [0.293, 0.114, 0.112], + [0.339, 0.103, 0.098], + [0.709, 0.103, 0.099], + [0.753, 0.237, 0.242], + [1.24, 0.359, 0.327], + [0.237, 0.06, 0.059], + [0.333, 0.131, 0.127], + [0.887, 0.284, 0.352], + [1.386, 0.418, 0.458], + [0.712, 0.29, 0.293], + [0.78, 0.317, 0.314], + [1.124, 0.41, 0.409], + [2.545, 0.666, 0.642], + [1.204, 0.351, 0.345], + [0.649, 0.304, 0.308], + [2.368, 0.609, 0.572], + [2.384, 0.6, 0.568], + [4.3, 1.215, 1.164], + [0.398, 0.126, 0.112], + [9.716, 0.289, 0.311], + [11.321, 0.334, 0.34], + [21.866, 0.498, 0.5], + [51.255, 1.651, 1.113], + [2.124, 0.162, 0.152], + [0.995, 0.198, 0.201], + [2.174, 0.153, 0.157], + [9.736, 0.346, 0.349], + [9.171, 1.464, 1.526], + [0.388, 0.204, 0.209], + [2.308, 0.284, 0.289], + [5.832, 0.351, 0.373], + [4.673, 1.341, 1.346], + [10.357, 1.425, 1.457], + [10.42, 1.387, 1.452], + [0.519, 0.225, 0.226], + [0.483, 0.242, 0.236], + [0.391, 0.172, 0.168], + [0.462, 0.167, 0.174], + [0.808, 0.385, 0.394], + [0.36, 0.142, 0.145], + [0.351, 0.134, 0.138], + [0.375, 0.134, 0.138] +] + } + \ No newline at end of file diff --git a/sail-partitioned/results/20260720/c8g.4xlarge.json b/sail-partitioned/results/20260720/c8g.4xlarge.json new file mode 100644 index 000000000..d30f78f6d --- /dev/null +++ b/sail-partitioned/results/20260720/c8g.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet, partitioned)", + "date": "2026-07-20", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 21, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.162, 0.052, 0.053], + [0.169, 0.066, 0.066], + [0.215, 0.091, 0.087], + [0.779, 0.083, 0.085], + [0.888, 0.348, 0.367], + [1.249, 0.435, 0.447], + [0.141, 0.056, 0.053], + [0.2, 0.069, 0.069], + [1.177, 0.511, 0.466], + [1.551, 0.523, 0.525], + [0.828, 0.162, 0.164], + [0.906, 0.175, 0.172], + [1.267, 0.453, 0.45], + [2.82, 0.62, 0.596], + [1.368, 0.44, 0.446], + [0.989, 0.439, 0.444], + [2.92, 0.917, 0.94], + [2.94, 0.969, 0.995], + [5.27, 1.909, 1.919], + [0.521, 0.083, 0.08], + [10.351, 0.594, 0.592], + [12.165, 0.689, 0.696], + [23.487, 1.193, 1.203], + [55.607, 3.463, 3.498], + [0.432, 0.113, 0.108], + [1.01, 0.204, 0.201], + [0.326, 0.108, 0.11], + [9.856, 0.732, 0.719], + [8.935, 6.067, 6.097], + [0.6, 0.484, 0.485], + [2.425, 0.413, 0.411], + [6.129, 0.415, 0.407], + [4.725, 1.426, 1.487], + [10.562, 2.507, 2.472], + [10.519, 2.416, 2.483], + [0.498, 0.37, 0.378], + [0.344, 0.152, 0.151], + [0.241, 0.098, 0.098], + [0.287, 0.108, 0.108], + [0.464, 0.27, 0.266], + [0.218, 0.074, 0.074], + [0.213, 0.072, 0.072], + [0.226, 0.073, 0.072] +] + } + \ No newline at end of file diff --git a/sail-partitioned/results/20260720/c8g.metal-48xl.json b/sail-partitioned/results/20260720/c8g.metal-48xl.json new file mode 100644 index 000000000..42ae25d53 --- /dev/null +++ b/sail-partitioned/results/20260720/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet, partitioned)", + "date": "2026-07-20", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 65, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.15, 0.053, 0.053], + [0.188, 0.077, 0.077], + [0.205, 0.081, 0.075], + [0.635, 0.089, 0.088], + [0.712, 0.17, 0.177], + [1.136, 0.246, 0.242], + [0.149, 0.054, 0.054], + [0.208, 0.092, 0.092], + [0.932, 0.211, 0.22], + [1.218, 0.327, 0.343], + [0.599, 0.163, 0.162], + [0.665, 0.172, 0.177], + [1.005, 0.267, 0.261], + [2.312, 0.415, 0.472], + [1.089, 0.247, 0.249], + [0.563, 0.195, 0.195], + [2.259, 0.461, 0.421], + [2.28, 0.471, 0.477], + [4.198, 0.941, 0.922], + [0.316, 0.088, 0.087], + [9.632, 0.311, 0.319], + [11.272, 0.351, 0.349], + [21.809, 0.561, 0.556], + [51.781, 2.239, 1.236], + [1.543, 0.096, 0.097], + [0.921, 0.123, 0.125], + [1.574, 0.1, 0.098], + [9.682, 0.363, 0.356], + [9.01, 1.518, 1.478], + [0.301, 0.21, 0.194], + [2.222, 0.195, 0.195], + [5.755, 0.267, 0.267], + [4.322, 1.161, 1.039], + [10.215, 1.336, 1.324], + [10.227, 1.336, 1.266], + [0.361, 0.176, 0.175], + [0.369, 0.182, 0.183], + [0.277, 0.127, 0.129], + [0.328, 0.132, 0.134], + [0.542, 0.313, 0.312], + [0.25, 0.099, 0.101], + [0.236, 0.1, 0.099], + [0.253, 0.101, 0.103] +] + } + \ No newline at end of file diff --git a/sail/install b/sail/install index 24a9df609..1c3c36b39 100755 --- a/sail/install +++ b/sail/install @@ -55,5 +55,5 @@ source myenv/bin/activate # `--ignore-installed` skips the uninstall and just lays the new copy on # top, which is what we want inside this venv anyway. pip install --upgrade --ignore-installed setuptools wheel -pip install --no-cache-dir "pysail==0.6.5" +pip install --no-cache-dir "pysail==0.6.6" pip install "pyspark-client==4.1.1" pandas psutil diff --git a/sail/queries.sql b/sail/queries.sql index 47114dee7..7eabbd211 100644 --- a/sail/queries.sql +++ b/sail/queries.sql @@ -26,7 +26,7 @@ SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; -SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '$1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/sail/query b/sail/query index 297c07e9e..d15c05f66 100755 --- a/sail/query +++ b/sail/query @@ -24,7 +24,6 @@ import timeit os.environ["SAIL_PARQUET__BINARY_AS_STRING"] = "true" os.environ["SAIL_PARQUET__REORDER_FILTERS"] = "true" -os.environ["SAIL_OPTIMIZER__ENABLE_JOIN_REORDER"] = "true" from pysail.spark import SparkConnectServer from pyspark.sql import SparkSession @@ -41,7 +40,6 @@ _, port = server.listening_address spark = SparkSession.builder.remote(f"sc://localhost:{port}").getOrCreate() df = spark.read.parquet("hits.parquet") -df = df.withColumn("EventTime", F.col("EventTime").cast("timestamp")) df = df.withColumn("EventDate", F.col("EventDate").cast("int").cast("date")) df.createOrReplaceTempView("hits") diff --git a/sail/results/20260720/c6a.2xlarge.json b/sail/results/20260720/c6a.2xlarge.json new file mode 100644 index 000000000..35bf21731 --- /dev/null +++ b/sail/results/20260720/c6a.2xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet)", + "date": "2026-07-20", + "machine": "c6a.2xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 5, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.211, 0.014, 0.015], + [0.297, 0.056, 0.055], + [0.401, 0.129, 0.13], + [0.535, 0.132, 0.123], + [1.587, 1.263, 1.321], + [1.575, 1.245, 1.251], + [0.217, 0.015, 0.015], + [0.312, 0.058, 0.062], + [2.033, 1.571, 1.642], + [2.362, 1.847, 1.883], + [0.808, 0.372, 0.375], + [0.869, 0.416, 0.414], + [1.62, 1.258, 1.25], + [3.238, 2.02, 2.128], + [1.61, 1.209, 1.208], + [1.831, 1.43, 1.473], + [3.399, 2.913, 2.901], + [3.367, 2.967, 2.927], + [5.734, 5.029, 5.109], + [0.454, 0.116, 0.114], + [9.708, 1.621, 1.597], + [11.454, 1.904, 1.93], + [22.32, 3.684, 3.601], + [55.895, 20.157, 23.219], + [2.851, 0.587, 0.601], + [0.944, 0.453, 0.451], + [2.86, 0.604, 0.598], + [9.782, 1.982, 1.983], + [19.067, 18.048, 18.148], + [1.703, 1.47, 1.477], + [2.467, 1.14, 1.124], + [5.869, 1.174, 1.161], + [4.68, 4.18, 4.262], + [12.087, 6.899, 6.97], + [11.96, 6.929, 6.943], + [1.462, 1.216, 1.193], + [0.603, 0.244, 0.238], + [0.379, 0.095, 0.092], + [0.507, 0.148, 0.141], + [0.926, 0.48, 0.497], + [0.349, 0.06, 0.063], + [0.339, 0.054, 0.057], + [0.359, 0.055, 0.053] +] + } + \ No newline at end of file diff --git a/sail/results/20260720/c6a.4xlarge.json b/sail/results/20260720/c6a.4xlarge.json new file mode 100644 index 000000000..9ff590e57 --- /dev/null +++ b/sail/results/20260720/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet)", + "date": "2026-07-20", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 3, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.204, 0.014, 0.015], + [0.289, 0.046, 0.045], + [0.304, 0.077, 0.085], + [0.638, 0.084, 0.084], + [1.011, 0.725, 0.798], + [1.254, 0.799, 0.792], + [0.203, 0.014, 0.015], + [0.29, 0.048, 0.049], + [1.197, 0.937, 0.948], + [1.481, 1.072, 1.03], + [0.665, 0.22, 0.229], + [0.707, 0.231, 0.242], + [1.146, 0.845, 0.85], + [2.724, 1.232, 1.26], + [1.245, 0.838, 0.855], + [1.164, 0.896, 0.9], + [2.813, 1.741, 1.689], + [2.794, 1.758, 1.736], + [5.339, 3.273, 3.345], + [0.471, 0.089, 0.09], + [9.748, 1.37, 1.373], + [11.537, 1.662, 1.656], + [22.349, 3.643, 3.703], + [55.986, 10.627, 10.591], + [2.847, 0.377, 0.39], + [0.986, 0.277, 0.281], + [2.833, 0.389, 0.376], + [9.797, 1.731, 1.729], + [10.343, 9.366, 9.338], + [1.012, 0.776, 0.778], + [2.362, 0.72, 0.693], + [5.883, 0.822, 0.829], + [4.613, 3.004, 3.118], + [10.979, 5.281, 5.204], + [10.82, 5.144, 5.203], + [0.958, 0.7, 0.712], + [0.557, 0.237, 0.238], + [0.384, 0.097, 0.097], + [0.504, 0.139, 0.137], + [0.907, 0.47, 0.469], + [0.363, 0.066, 0.065], + [0.343, 0.059, 0.058], + [0.353, 0.054, 0.056] +] + } + \ No newline at end of file diff --git a/sail/results/20260720/c6a.metal.json b/sail/results/20260720/c6a.metal.json new file mode 100644 index 000000000..6146716b1 --- /dev/null +++ b/sail/results/20260720/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet)", + "date": "2026-07-20", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 4, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.166, 0.017, 0.016], + [0.556, 0.327, 0.313], + [0.27, 0.064, 0.065], + [0.401, 0.067, 0.059], + [0.42, 0.25, 0.252], + [0.89, 0.339, 0.381], + [0.162, 0.017, 0.017], + [0.561, 0.303, 0.28], + [0.774, 0.314, 0.27], + [1.161, 0.42, 0.422], + [0.81, 0.414, 0.45], + [0.867, 0.428, 0.428], + [1.203, 0.542, 0.544], + [2.558, 0.886, 0.844], + [1.234, 0.516, 0.502], + [0.502, 0.305, 0.285], + [2.137, 0.596, 0.588], + [2.163, 0.633, 0.605], + [4.081, 1.344, 1.25], + [0.73, 0.294, 0.304], + [9.61, 0.383, 0.467], + [11.358, 0.671, 0.648], + [22.083, 0.943, 1.028], + [55.931, 2.089, 2.109], + [2.727, 0.296, 0.328], + [1.011, 0.297, 0.308], + [2.725, 0.253, 0.307], + [9.86, 0.72, 0.634], + [9.125, 2.225, 2.138], + [0.432, 0.255, 0.272], + [2.334, 0.474, 0.48], + [5.717, 0.586, 0.586], + [4.553, 1.686, 1.687], + [10.375, 1.67, 1.707], + [10.349, 1.716, 1.683], + [0.47, 0.207, 0.225], + [0.707, 0.326, 0.35], + [0.617, 0.241, 0.257], + [0.514, 0.235, 0.24], + [0.876, 0.474, 0.49], + [0.474, 0.191, 0.199], + [0.565, 0.186, 0.177], + [0.535, 0.192, 0.222] +] + } + \ No newline at end of file diff --git a/sail/results/20260720/c6a.xlarge.json b/sail/results/20260720/c6a.xlarge.json new file mode 100644 index 000000000..baaabe620 --- /dev/null +++ b/sail/results/20260720/c6a.xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet)", + "date": "2026-07-20", + "machine": "c6a.xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 1, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.207, 0.015, 0.015], + [0.345, 0.078, 0.08], + [0.533, 0.212, 0.212], + [0.675, 0.199, 0.195], + [2.509, 2.163, 2.189], + [2.603, 2.149, 2.166], + [0.213, 0.016, 0.016], + [0.367, 0.083, 0.082], + [3.147, 2.658, 2.643], + [3.603, 2.951, 2.968], + [1.139, 0.609, 0.612], + [1.245, 0.677, 0.677], + [2.634, 2.122, 2.115], + [3.942, 3.305, 3.315], + [2.632, 2.072, 2.078], + [2.843, 2.413, 2.409], + [5.514, 4.755, 4.814], + [5.45, 4.822, 4.864], + [9.986, 9.464, 9.542], + [0.632, 0.183, 0.191], + [9.712, 2.659, 2.682], + [11.403, 3.193, 3.195], + [22.314, 5.976, 5.871], + [55.842, 46.802, 46.522], + [2.78, 0.997, 1.016], + [1.316, 0.776, 0.779], + [2.833, 1.002, 0.992], + [9.815, 3.336, 3.342], + [33.191, 31.295, 31.628], + [2.939, 2.721, 2.717], + [2.772, 1.961, 1.939], + [5.909, 2.085, 2.073], + [6.978, 6.891, 6.86], + [63.855, 29.224, 63.496], + [17.525, 53.002, 36.559], + [2.273, 1.862, 1.872], + [0.616, 0.243, 0.246], + [0.393, 0.094, 0.09], + [0.506, 0.141, 0.134], + [0.94, 0.475, 0.496], + [0.359, 0.06, 0.063], + [0.343, 0.053, 0.054], + [0.36, 0.054, 0.054] +] + } + \ No newline at end of file diff --git a/sail/results/20260720/c7a.metal-48xl.json b/sail/results/20260720/c7a.metal-48xl.json new file mode 100644 index 000000000..98e95a759 --- /dev/null +++ b/sail/results/20260720/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet)", + "date": "2026-07-20", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 5, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.198, 0.016, 0.014], + [0.513, 0.295, 0.27], + [0.265, 0.069, 0.052], + [0.405, 0.076, 0.073], + [0.403, 0.235, 0.23], + [0.896, 0.375, 0.339], + [0.152, 0.015, 0.014], + [0.525, 0.249, 0.276], + [0.757, 0.28, 0.246], + [1.185, 0.377, 0.394], + [0.739, 0.456, 0.467], + [0.835, 0.476, 0.449], + [1.158, 0.519, 0.585], + [2.535, 0.777, 0.754], + [1.202, 0.463, 0.477], + [0.564, 0.261, 0.293], + [2.148, 0.514, 0.499], + [2.133, 0.525, 0.517], + [3.969, 1.183, 1.126], + [0.509, 0.263, 0.288], + [9.576, 0.294, 0.316], + [11.39, 0.483, 0.448], + [21.955, 0.625, 0.601], + [55.863, 1.275, 1.386], + [2.668, 0.251, 0.36], + [0.981, 0.35, 0.326], + [2.662, 0.342, 0.329], + [9.806, 0.459, 0.519], + [9.031, 2.275, 2.083], + [0.369, 0.194, 0.192], + [2.288, 0.422, 0.437], + [5.717, 0.488, 0.498], + [4.298, 1.441, 1.357], + [10.354, 1.453, 1.421], + [10.323, 1.455, 1.402], + [0.451, 0.188, 0.207], + [0.628, 0.32, 0.294], + [0.579, 0.23, 0.225], + [0.558, 0.216, 0.238], + [0.916, 0.44, 0.48], + [0.522, 0.197, 0.184], + [0.516, 0.197, 0.2], + [0.493, 0.181, 0.177] +] + } + \ No newline at end of file diff --git a/sail/results/20260720/c8g.4xlarge.json b/sail/results/20260720/c8g.4xlarge.json new file mode 100644 index 000000000..28ca587e4 --- /dev/null +++ b/sail/results/20260720/c8g.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet)", + "date": "2026-07-20", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 5, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.095, 0.013, 0.013], + [0.165, 0.032, 0.031], + [0.207, 0.059, 0.056], + [0.536, 0.051, 0.053], + [0.846, 0.355, 0.373], + [1.136, 0.468, 0.46], + [0.105, 0.013, 0.013], + [0.151, 0.035, 0.036], + [0.996, 0.481, 0.466], + [1.198, 0.613, 0.617], + [0.51, 0.151, 0.144], + [0.572, 0.187, 0.167], + [0.94, 0.479, 0.486], + [2.415, 0.659, 0.638], + [1.046, 0.479, 0.494], + [0.607, 0.434, 0.419], + [2.46, 0.92, 0.919], + [2.498, 0.945, 0.94], + [4.578, 1.712, 1.748], + [0.278, 0.05, 0.051], + [9.607, 0.741, 0.741], + [11.378, null, 0.819], + [22.227, 1.331, 1.318], + [55.8, 3.773, 3.748], + [2.722, 0.252, 0.256], + [0.844, 0.202, 0.209], + [2.716, 0.251, 0.247], + [9.672, 0.871, 0.871], + [10.197, 7.61, 7.571], + [0.626, 0.501, 0.503], + [2.241, 0.395, 0.396], + [5.754, 0.418, 0.417], + [4.396, 1.458, 1.547], + [10.47, 2.476, 2.458], + [10.444, 2.445, 2.444], + [0.512, 0.347, 0.364], + [0.436, 0.191, 0.186], + [0.26, 0.082, 0.083], + [0.366, 0.119, 0.118], + [0.69, 0.368, 0.37], + [0.221, 0.047, 0.045], + [0.207, 0.04, 0.041], + [0.221, 0.043, 0.043] +] + } + \ No newline at end of file diff --git a/sail/results/20260720/c8g.metal-48xl.json b/sail/results/20260720/c8g.metal-48xl.json new file mode 100644 index 000000000..81df83629 --- /dev/null +++ b/sail/results/20260720/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Sail (Parquet)", + "date": "2026-07-20", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["column-oriented","stateless"], + "load_time": 5, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.106, 0.011, 0.011], + [0.217, 0.109, 0.11], + [0.163, 0.04, 0.041], + [0.315, 0.044, 0.045], + [0.356, 0.132, 0.124], + [0.793, 0.197, 0.198], + [0.11, 0.011, 0.011], + [0.234, 0.109, 0.111], + [0.632, 0.165, 0.194], + [1.037, 0.266, 0.313], + [0.65, 0.172, 0.164], + [0.558, 0.184, 0.189], + [0.895, 0.281, 0.252], + [2.158, 0.409, 0.382], + [0.959, 0.255, 0.243], + [0.378, 0.141, 0.145], + [2.065, 0.354, 0.372], + [2.047, 0.326, 0.387], + [3.893, 0.883, 0.875], + [0.285, 0.115, 0.124], + [9.542, 0.345, 0.341], + [11.22, 0.43, 0.422], + [21.842, 0.623, 0.629], + [55.846, 1.574, 1.441], + [2.504, 0.153, 0.149], + [0.798, 0.153, 0.145], + [2.504, 0.157, 0.151], + [9.652, 0.437, 0.43], + [8.782, 1.903, 2.282], + [0.305, 0.214, 0.197], + [2.082, 0.223, 0.223], + [5.466, 0.302, 0.285], + [4.18, 1.276, 1.271], + [10.131, 1.362, 1.305], + [10.167, 1.317, 1.277], + [0.303, 0.12, 0.116], + [0.448, 0.214, 0.206], + [0.331, 0.164, 0.16], + [0.357, 0.132, 0.147], + [0.61, 0.346, 0.335], + [0.268, 0.112, 0.091], + [0.278, 0.106, 0.098], + [0.286, 0.095, 0.106] +] + } + \ No newline at end of file