From e41255fdd2cec64001b39eea39cda449bca23f38 Mon Sep 17 00:00:00 2001 From: Alex Bulankou Date: Wed, 12 Aug 2026 08:45:15 +0000 Subject: [PATCH] benchmarking: add SLO-knee throughput post-processor Compute throughput-at-SLO-knee from a concurrency sweep of locust runs. For each op (Type_Name) in a build tag's sweep, the knee is the maximum sustained RPS among sweep points whose chosen latency percentile stays under a ceiling AND whose failure ratio is within tolerance. This maps directly onto the spec-doc substrate-row throughput axes (@<1s / @<5s == --ceiling-ms 1000 / 5000) with p50/p95 read straight off each point. - Pure stdlib over runner.py's stats.jsonl schema (no locust/pandas/numpy), so it runs offline over saved run artifacts. - Offered concurrency (-u user count) is absent from stats.jsonl; the knee is computed from (rps, latency) pairs and does not need it. Supplying a name->users map via --users-by-name / --tests-yaml labels each point's concurrency so the knee's location is interpretable. - Failure-ratio guard (default <=1%) prevents a fast-failing high-RPS run from being selected as usable capacity. - 18 unit tests + synthetic stats.jsonl fixtures (clean ceiling-crossing sweep, fast-failing exclusion, SLO-never-met, percentile normalization, multi-metric grouping, CLI e2e). Runnable via pytest or a built-in stdlib runner (no pytest dependency required). Signed-off-by: Alex Bulankou --- benchmarking/analysis/slo_knee.py | 443 ++++++++++++++++++ .../analysis/tests/fixtures/sweep_clean.jsonl | 4 + .../tests/fixtures/sweep_failing.jsonl | 3 + .../tests/fixtures/sweep_never_met.jsonl | 2 + benchmarking/analysis/tests/test_slo_knee.py | 396 ++++++++++++++++ hack/util/verify-boilerplate.py | 2 +- 6 files changed, 849 insertions(+), 1 deletion(-) create mode 100644 benchmarking/analysis/slo_knee.py create mode 100644 benchmarking/analysis/tests/fixtures/sweep_clean.jsonl create mode 100644 benchmarking/analysis/tests/fixtures/sweep_failing.jsonl create mode 100644 benchmarking/analysis/tests/fixtures/sweep_never_met.jsonl create mode 100644 benchmarking/analysis/tests/test_slo_knee.py diff --git a/benchmarking/analysis/slo_knee.py b/benchmarking/analysis/slo_knee.py new file mode 100644 index 000000000..9193bf478 --- /dev/null +++ b/benchmarking/analysis/slo_knee.py @@ -0,0 +1,443 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compute throughput-at-SLO-knee from a concurrency sweep of locust runs. + +Each run in a sweep is a `stats.jsonl` file produced by runner.py: one JSON +object per gRPC op (Type_Name), carrying `requests_per_s` throughput and the +`p50..p100` response-time percentiles (milliseconds). A sweep is the set of +runs sharing a build `tag`, taken at increasing offered load (distinct +`test_name`s, e.g. glutton_baseline_1_user .. glutton_oversubscribe_15_users). + +The SLO-knee throughput for an op is the *maximum sustained RPS achieved while +a chosen latency percentile stays under a ceiling*. Concretely, among the +sweep points whose p95 (or p99, ...) response time is <= the ceiling AND whose +failure ratio is within tolerance, the knee is the point with the highest RPS. +Past the knee, offered load keeps climbing but latency has blown past the SLO, +so that extra "throughput" is not usable capacity. + +This maps directly onto the spec-doc substrate-row axes: throughput @<1s and +@<5s are `--ceiling-ms 1000` and `--ceiling-ms 5000`; p50/p95 are read straight +off each point. + +Offered concurrency (the `-u` user count) is NOT present in stats.jsonl -- it +lives in tests.yaml / the runner argv. The knee metric does not need it (it is +computed from (rps, latency) pairs), but supplying a name->users map via +--users-by-name or --tests-yaml labels each point with its concurrency, which +is what makes the knee's *location* interpretable. + +Pure stdlib: no locust, pandas, or numpy dependency, so it runs offline over +saved run artifacts and is unit-testable with synthetic fixtures. +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +import sys +from dataclasses import dataclass, field +from typing import Iterable + +# Default latency ceilings (ms) == the spec-doc substrate-row throughput axes +# (@<1s and @<5s). +DEFAULT_CEILINGS_MS = (1000.0, 5000.0) + +# A sweep point whose failure ratio exceeds this is not a valid throughput +# measurement -- a run that "achieves" high RPS by fast-failing requests is +# not delivering usable capacity, so it must not be eligible to be the knee. +DEFAULT_MAX_FAILURE_RATIO = 0.01 + + +def normalize_percentile_key(p: str) -> str: + """Map a user-facing percentile spec onto the measurements dict key. + + runner.py encodes locust's CSV percentile columns as p with dots + turned into underscores: "50%"->"p50", "95%"->"p95", "99.9%"->"p99_9". + Accept any of "p95", "95", "95%", "p99.9", "99.9" and return "p95"/"p99_9". + """ + s = p.strip().lower().lstrip("p").rstrip("%") + return "p" + s.replace(".", "_") + + +def _to_float(v: object) -> float | None: + """Coerce a measurements value (always a string from csv.DictReader) to + float, treating locust's empty / "N/A" placeholders as missing.""" + if v is None: + return None + s = str(v).strip() + if not s or s.upper() == "N/A": + return None + try: + return float(s) + except ValueError: + return None + + +@dataclass +class Point: + """One sweep point: one op (metric) from one run (test_name/tag).""" + + tag: str + test_name: str + metric: str + rps: float | None + request_count: float | None + failure_count: float | None + latencies_ms: dict[str, float] # normalized key -> ms + users: int | None = None + + @property + def failure_ratio(self) -> float | None: + if self.request_count is None or self.request_count <= 0: + return None + fc = self.failure_count or 0.0 + return fc / self.request_count + + def latency(self, pct_key: str) -> float | None: + return self.latencies_ms.get(pct_key) + + +def record_to_point(record: dict, users_map: dict[str, int] | None = None) -> Point: + m = record.get("measurements", {}) or {} + latencies = {} + for k, v in m.items(): + if k.startswith("p"): + fv = _to_float(v) + if fv is not None: + latencies[k] = fv + test_name = record.get("test_name", "") or "" + users = None + if users_map: + users = users_map.get(test_name) + return Point( + tag=record.get("tag", "") or "", + test_name=test_name, + metric=record.get("metric", "") or "", + rps=_to_float(m.get("requests_per_s")), + request_count=_to_float(m.get("request_count")), + failure_count=_to_float(m.get("failure_count")), + latencies_ms=latencies, + users=users, + ) + + +def iter_jsonl_records(paths: Iterable[str]) -> Iterable[dict]: + for path in paths: + with open(path) as f: + for line in f: + line = line.strip() + if not line: + continue + yield json.loads(line) + + +def expand_inputs(inputs: Iterable[str]) -> list[str]: + """Resolve each input (a stats.jsonl file, a glob, or a directory that is + walked for stats.jsonl / *.jsonl) into a flat list of jsonl file paths.""" + paths: list[str] = [] + for item in inputs: + if os.path.isdir(item): + found = sorted(glob.glob(os.path.join(item, "**", "*.jsonl"), recursive=True)) + paths.extend(found) + elif any(ch in item for ch in "*?["): + paths.extend(sorted(glob.glob(item, recursive=True))) + else: + paths.append(item) + # De-dup while preserving order. + seen = set() + out = [] + for p in paths: + if p not in seen: + seen.add(p) + out.append(p) + return out + + +@dataclass +class KneeResult: + tag: str + metric: str + percentile: str # normalized key, e.g. "p95" + ceiling_ms: float + knee_rps: float | None + knee_users: int | None + knee_test_name: str | None + knee_latency_ms: float | None + slo_ever_met: bool # any point under ceiling (before failure filter) + n_points: int + n_under_ceiling: int + n_valid: int # under ceiling AND within failure tolerance + points: list[dict] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "tag": self.tag, + "metric": self.metric, + "percentile": self.percentile, + "ceiling_ms": self.ceiling_ms, + "knee_rps": self.knee_rps, + "knee_users": self.knee_users, + "knee_test_name": self.knee_test_name, + "knee_latency_ms": self.knee_latency_ms, + "slo_ever_met": self.slo_ever_met, + "n_points": self.n_points, + "n_under_ceiling": self.n_under_ceiling, + "n_valid": self.n_valid, + "points": self.points, + } + + +def compute_knee( + points: list[Point], + pct_key: str, + ceiling_ms: float, + max_failure_ratio: float, +) -> KneeResult: + """Knee = max-RPS point among those with latency<=ceiling and an + acceptable failure ratio. `points` must already be a single (tag, metric) + group.""" + tag = points[0].tag if points else "" + metric = points[0].metric if points else "" + + under_ceiling = 0 + valid: list[Point] = [] + point_rows = [] + for p in points: + lat = p.latency(pct_key) + fr = p.failure_ratio + latency_ok = lat is not None and lat <= ceiling_ms + failure_ok = fr is None or fr <= max_failure_ratio + if latency_ok: + under_ceiling += 1 + # A point is knee-eligible only if it has an RPS reading, is under the + # latency ceiling, and did not fast-fail its way there. + if latency_ok and failure_ok and p.rps is not None: + valid.append(p) + point_rows.append( + { + "test_name": p.test_name, + "users": p.users, + "rps": p.rps, + pct_key: lat, + "failure_ratio": fr, + "latency_ok": latency_ok, + "failure_ok": failure_ok, + "knee_eligible": latency_ok and failure_ok and p.rps is not None, + } + ) + + knee = max(valid, key=lambda p: p.rps) if valid else None + return KneeResult( + tag=tag, + metric=metric, + percentile=pct_key, + ceiling_ms=ceiling_ms, + knee_rps=(knee.rps if knee else None), + knee_users=(knee.users if knee else None), + knee_test_name=(knee.test_name if knee else None), + knee_latency_ms=(knee.latency(pct_key) if knee else None), + slo_ever_met=under_ceiling > 0, + n_points=len(points), + n_under_ceiling=under_ceiling, + n_valid=len(valid), + points=point_rows, + ) + + +def group_points(points: list[Point]) -> dict[tuple[str, str], list[Point]]: + groups: dict[tuple[str, str], list[Point]] = {} + for p in points: + groups.setdefault((p.tag, p.metric), []).append(p) + return groups + + +def analyze( + records: Iterable[dict], + percentiles: list[str], + ceilings_ms: list[float], + users_map: dict[str, int] | None = None, + max_failure_ratio: float = DEFAULT_MAX_FAILURE_RATIO, + metric_filter: str | None = None, +) -> list[KneeResult]: + points = [record_to_point(r, users_map) for r in records] + if metric_filter: + points = [p for p in points if metric_filter in p.metric] + groups = group_points(points) + results: list[KneeResult] = [] + for (_tag, _metric), grp in sorted(groups.items()): + # Order a group's points by offered load when known, else by RPS, so + # the emitted `points` table reads as a load ladder. + grp_sorted = sorted( + grp, + key=lambda p: (p.users if p.users is not None else -1, p.rps or 0.0), + ) + for pct in percentiles: + pct_key = normalize_percentile_key(pct) + for ceiling in ceilings_ms: + results.append( + compute_knee(grp_sorted, pct_key, ceiling, max_failure_ratio) + ) + return results + + +def parse_users_by_name(spec: str | None) -> dict[str, int]: + """Parse "name1=1,name2=5,name3=10" into {name: users}.""" + out: dict[str, int] = {} + if not spec: + return out + for chunk in spec.split(","): + chunk = chunk.strip() + if not chunk: + continue + name, _, val = chunk.partition("=") + if not _: + raise ValueError(f"bad --users-by-name entry (want name=users): {chunk!r}") + out[name.strip()] = int(val.strip()) + return out + + +def users_map_from_tests_yaml(path: str) -> dict[str, int]: + """Read tests.yaml -> {test.name: test.users}. Lazy-imports PyYAML so the + module has no hard YAML dependency for the common (jsonl-only) path.""" + import yaml # lazy: only needed when --tests-yaml is used + + with open(path) as f: + doc = yaml.safe_load(f) or {} + out: dict[str, int] = {} + for t in doc.get("tests", []) or []: + name = t.get("name") + users = t.get("users") + if name is not None and users is not None: + out[name] = int(users) + return out + + +def format_table(results: list[KneeResult]) -> str: + lines = [] + for r in results: + header = ( + f"[{r.metric}] {r.percentile} <= {r.ceiling_ms:.0f}ms tag={r.tag or '(none)'}" + ) + lines.append(header) + if r.knee_rps is None: + if not r.slo_ever_met: + lines.append( + f" SLO NEVER MET: no point kept {r.percentile} <= {r.ceiling_ms:.0f}ms " + f"across {r.n_points} point(s)" + ) + else: + lines.append( + f" no valid knee: {r.n_under_ceiling} point(s) under ceiling but all " + f"exceeded the failure tolerance" + ) + else: + loc = ( + f"@ {r.knee_users} users" + if r.knee_users is not None + else f"@ {r.knee_test_name}" + ) + lines.append( + f" knee = {r.knee_rps:.2f} rps {loc} " + f"({r.percentile}={r.knee_latency_ms:.0f}ms; " + f"{r.n_valid}/{r.n_points} points valid)" + ) + lines.append("") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "inputs", + nargs="+", + help="stats.jsonl file(s), glob(s), or run director(y/ies) to walk for *.jsonl", + ) + p.add_argument( + "--percentile", + "-p", + action="append", + default=None, + help="latency percentile to gate on (repeatable). Default: p95. " + "Accepts p95 / 95 / 95%% / p99_9 / 99.9", + ) + p.add_argument( + "--ceiling-ms", + type=float, + action="append", + default=None, + help=f"latency ceiling in ms (repeatable). Default: {list(DEFAULT_CEILINGS_MS)}", + ) + p.add_argument( + "--max-failure-ratio", + type=float, + default=DEFAULT_MAX_FAILURE_RATIO, + help=f"exclude sweep points whose failure ratio exceeds this " + f"(default {DEFAULT_MAX_FAILURE_RATIO})", + ) + p.add_argument( + "--users-by-name", + default=None, + help='label points with offered concurrency, e.g. "t1=1,t5=5,t10=10"', + ) + p.add_argument( + "--tests-yaml", + default=None, + help="tests.yaml to read test.name->test.users from (needs PyYAML)", + ) + p.add_argument( + "--metric-filter", + default=None, + help="only analyze metrics containing this substring (e.g. ResumeActor)", + ) + p.add_argument( + "--json", + action="store_true", + help="emit JSON instead of a human-readable table", + ) + args = p.parse_args(argv) + + percentiles = args.percentile or ["p95"] + ceilings = args.ceiling_ms or list(DEFAULT_CEILINGS_MS) + + users_map: dict[str, int] = {} + if args.tests_yaml: + users_map.update(users_map_from_tests_yaml(args.tests_yaml)) + if args.users_by_name: + users_map.update(parse_users_by_name(args.users_by_name)) + + paths = expand_inputs(args.inputs) + if not paths: + print("no input jsonl files resolved", file=sys.stderr) + return 2 + records = list(iter_jsonl_records(paths)) + results = analyze( + records, + percentiles=percentiles, + ceilings_ms=ceilings, + users_map=users_map or None, + max_failure_ratio=args.max_failure_ratio, + metric_filter=args.metric_filter, + ) + + if args.json: + print(json.dumps([r.to_dict() for r in results], indent=2)) + else: + print(format_table(results)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarking/analysis/tests/fixtures/sweep_clean.jsonl b/benchmarking/analysis/tests/fixtures/sweep_clean.jsonl new file mode 100644 index 000000000..109293775 --- /dev/null +++ b/benchmarking/analysis/tests/fixtures/sweep_clean.jsonl @@ -0,0 +1,4 @@ +{"timestamp": "2026-08-12T00:00:01Z", "tag": "abc1234", "test_name": "glutton_baseline_1_user", "metric": "grpc_ResumeActor", "measurements": {"request_count": "600", "failure_count": "0", "median_response_time": "120", "average_response_time": "130", "min_response_time": "80", "max_response_time": "400", "p50": "120", "p66": "140", "p75": "160", "p80": "180", "p90": "220", "p95": "300", "p98": "360", "p99": "390", "p99_9": "398", "p99_99": "400", "p100": "400", "requests_per_s": "10.0", "failures_per_s": "0.0"}} +{"timestamp": "2026-08-12T00:05:01Z", "tag": "abc1234", "test_name": "glutton_baseline_5_users", "metric": "grpc_ResumeActor", "measurements": {"request_count": "3000", "failure_count": "0", "median_response_time": "300", "average_response_time": "340", "min_response_time": "100", "max_response_time": "900", "p50": "300", "p66": "380", "p75": "440", "p80": "500", "p90": "650", "p95": "800", "p98": "870", "p99": "890", "p99_9": "899", "p99_99": "900", "p100": "900", "requests_per_s": "48.0", "failures_per_s": "0.0"}} +{"timestamp": "2026-08-12T00:10:01Z", "tag": "abc1234", "test_name": "glutton_baseline_10_users", "metric": "grpc_ResumeActor", "measurements": {"request_count": "6000", "failure_count": "0", "median_response_time": "700", "average_response_time": "820", "min_response_time": "150", "max_response_time": "2500", "p50": "700", "p66": "950", "p75": "1100", "p80": "1300", "p90": "1800", "p95": "2200", "p98": "2400", "p99": "2480", "p99_9": "2498", "p99_99": "2500", "p100": "2500", "requests_per_s": "92.0", "failures_per_s": "0.0"}} +{"timestamp": "2026-08-12T00:15:01Z", "tag": "abc1234", "test_name": "glutton_oversubscribe_15_users", "metric": "grpc_ResumeActor", "measurements": {"request_count": "9000", "failure_count": "0", "median_response_time": "3200", "average_response_time": "3800", "min_response_time": "200", "max_response_time": "12000", "p50": "3200", "p66": "4500", "p75": "5500", "p80": "6500", "p90": "8500", "p95": "10500", "p98": "11500", "p99": "11800", "p99_9": "11980", "p99_99": "12000", "p100": "12000", "requests_per_s": "120.0", "failures_per_s": "0.0"}} diff --git a/benchmarking/analysis/tests/fixtures/sweep_failing.jsonl b/benchmarking/analysis/tests/fixtures/sweep_failing.jsonl new file mode 100644 index 000000000..3491c111d --- /dev/null +++ b/benchmarking/analysis/tests/fixtures/sweep_failing.jsonl @@ -0,0 +1,3 @@ +{"timestamp": "2026-08-12T01:00:01Z", "tag": "def5678", "test_name": "load_low", "metric": "grpc_GetActor", "measurements": {"request_count": "1000", "failure_count": "0", "p50": "50", "p90": "90", "p95": "120", "p99": "180", "p100": "220", "requests_per_s": "20.0", "failures_per_s": "0.0"}} +{"timestamp": "2026-08-12T01:05:01Z", "tag": "def5678", "test_name": "load_mid", "metric": "grpc_GetActor", "measurements": {"request_count": "2000", "failure_count": "5", "p50": "120", "p90": "300", "p95": "450", "p99": "700", "p100": "950", "requests_per_s": "60.0", "failures_per_s": "0.15"}} +{"timestamp": "2026-08-12T01:10:01Z", "tag": "def5678", "test_name": "load_high_failing", "metric": "grpc_GetActor", "measurements": {"request_count": "4000", "failure_count": "1200", "p50": "80", "p90": "200", "p95": "300", "p99": "500", "p100": "800", "requests_per_s": "200.0", "failures_per_s": "60.0"}} diff --git a/benchmarking/analysis/tests/fixtures/sweep_never_met.jsonl b/benchmarking/analysis/tests/fixtures/sweep_never_met.jsonl new file mode 100644 index 000000000..a26c3061e --- /dev/null +++ b/benchmarking/analysis/tests/fixtures/sweep_never_met.jsonl @@ -0,0 +1,2 @@ +{"timestamp": "2026-08-12T02:00:01Z", "tag": "cafe999", "test_name": "cold_1", "metric": "grpc_ResumeActor", "measurements": {"request_count": "100", "failure_count": "0", "p50": "6000", "p90": "7000", "p95": "7500", "p99": "8000", "p100": "9000", "requests_per_s": "2.0", "failures_per_s": "0.0"}} +{"timestamp": "2026-08-12T02:05:01Z", "tag": "cafe999", "test_name": "cold_2", "metric": "grpc_ResumeActor", "measurements": {"request_count": "200", "failure_count": "0", "p50": "6500", "p90": "8000", "p95": "9000", "p99": "9500", "p100": "10000", "requests_per_s": "4.0", "failures_per_s": "0.0"}} diff --git a/benchmarking/analysis/tests/test_slo_knee.py b/benchmarking/analysis/tests/test_slo_knee.py new file mode 100644 index 000000000..91c99accd --- /dev/null +++ b/benchmarking/analysis/tests/test_slo_knee.py @@ -0,0 +1,396 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for slo_knee -- pure stdlib, runnable offline with `pytest` or +directly via `python3 test_slo_knee.py` (a tiny built-in runner is provided so +the suite does not depend on pytest being installed in the analysis venv).""" + +from __future__ import annotations + +import io +import json +import os +import sys + +# Make the analysis package importable whether run from the repo root, the +# analysis/ dir, or the tests/ dir. +_HERE = os.path.dirname(os.path.abspath(__file__)) +_ANALYSIS = os.path.dirname(_HERE) +_BENCH = os.path.dirname(_ANALYSIS) +for _p in (_BENCH, _ANALYSIS): + if _p not in sys.path: + sys.path.insert(0, _p) + +import slo_knee # noqa: E402 + +FIXTURES = os.path.join(_HERE, "fixtures") + + +def _fx(name: str) -> str: + return os.path.join(FIXTURES, name) + + +# --------------------------------------------------------------------------- +# normalize_percentile_key +# --------------------------------------------------------------------------- + + +def test_normalize_percentile_variants(): + assert slo_knee.normalize_percentile_key("p95") == "p95" + assert slo_knee.normalize_percentile_key("95") == "p95" + assert slo_knee.normalize_percentile_key("95%") == "p95" + assert slo_knee.normalize_percentile_key("P95") == "p95" + assert slo_knee.normalize_percentile_key("p99.9") == "p99_9" + assert slo_knee.normalize_percentile_key("99.9") == "p99_9" + assert slo_knee.normalize_percentile_key("99.99%") == "p99_99" + assert slo_knee.normalize_percentile_key(" p50 ") == "p50" + + +# --------------------------------------------------------------------------- +# _to_float / failure_ratio +# --------------------------------------------------------------------------- + + +def test_to_float_placeholders(): + assert slo_knee._to_float("12.5") == 12.5 + assert slo_knee._to_float("") is None + assert slo_knee._to_float("N/A") is None + assert slo_knee._to_float(None) is None + assert slo_knee._to_float("garbage") is None + + +def test_failure_ratio_guards_zero_and_missing(): + p = slo_knee.Point( + tag="t", test_name="n", metric="m", rps=1.0, + request_count=0.0, failure_count=0.0, latencies_ms={}, + ) + assert p.failure_ratio is None # request_count == 0 -> undefined, not div0 + p2 = slo_knee.Point( + tag="t", test_name="n", metric="m", rps=1.0, + request_count=100.0, failure_count=3.0, latencies_ms={}, + ) + assert abs(p2.failure_ratio - 0.03) < 1e-9 + + +# --------------------------------------------------------------------------- +# record_to_point: only p* keys become latencies +# --------------------------------------------------------------------------- + + +def test_record_to_point_extracts_latencies_and_rps(): + rec = { + "tag": "abc", + "test_name": "load_x", + "metric": "grpc_ResumeActor", + "measurements": { + "request_count": "1000", + "failure_count": "2", + "requests_per_s": "42.5", + "p50": "100", + "p95": "900", + "p99_9": "1500", + "median_response_time": "100", # NOT a p* key -> excluded + "average_response_time": "150", + }, + } + pt = slo_knee.record_to_point(rec, users_map={"load_x": 5}) + assert pt.rps == 42.5 + assert pt.users == 5 + assert pt.latency("p50") == 100 + assert pt.latency("p95") == 900 + assert pt.latency("p99_9") == 1500 + # non-percentile measurements must not leak into latencies_ms + assert "median_response_time" not in pt.latencies_ms + assert "average_response_time" not in pt.latencies_ms + assert abs(pt.failure_ratio - 0.002) < 1e-9 + + +def test_record_to_point_na_latency_dropped(): + rec = { + "tag": "t", "test_name": "n", "metric": "m", + "measurements": {"p95": "N/A", "requests_per_s": "5", "request_count": "10", "failure_count": "0"}, + } + pt = slo_knee.record_to_point(rec) + assert pt.latency("p95") is None + + +# --------------------------------------------------------------------------- +# compute_knee: clean sweep, ceiling crossing mid-ladder +# --------------------------------------------------------------------------- + + +def _load(name: str, users_map=None) -> list[slo_knee.Point]: + recs = list(slo_knee.iter_jsonl_records([_fx(name)])) + return [slo_knee.record_to_point(r, users_map) for r in recs] + + +def test_knee_clean_sweep_p95_1s_and_5s(): + users = { + "glutton_baseline_1_user": 1, + "glutton_baseline_5_users": 5, + "glutton_baseline_10_users": 10, + "glutton_oversubscribe_15_users": 15, + } + pts = _load("sweep_clean.jsonl", users) + + # ceiling 1000ms @ p95: only 1_user(300) & 5_users(800) qualify; + # knee = higher-rps = 5_users @ 48 rps. + r1 = slo_knee.compute_knee(pts, "p95", 1000.0, slo_knee.DEFAULT_MAX_FAILURE_RATIO) + assert r1.knee_rps == 48.0 + assert r1.knee_users == 5 + assert r1.knee_test_name == "glutton_baseline_5_users" + assert r1.knee_latency_ms == 800 + assert r1.slo_ever_met is True + assert r1.n_under_ceiling == 2 + assert r1.n_valid == 2 + assert r1.n_points == 4 + + # ceiling 5000ms @ p95: 1,5,10_users qualify (2200<=5000); 15_users(10500) no. + # knee = 10_users @ 92 rps. + r5 = slo_knee.compute_knee(pts, "p95", 5000.0, slo_knee.DEFAULT_MAX_FAILURE_RATIO) + assert r5.knee_rps == 92.0 + assert r5.knee_users == 10 + assert r5.n_under_ceiling == 3 + + +def test_knee_clean_sweep_p50(): + users = { + "glutton_baseline_1_user": 1, + "glutton_baseline_5_users": 5, + "glutton_baseline_10_users": 10, + "glutton_oversubscribe_15_users": 15, + } + pts = _load("sweep_clean.jsonl", users) + # p50 @ 1000ms: 120,300,700 under; 3200 over. knee = 10_users @ 92 rps. + r = slo_knee.compute_knee(pts, "p50", 1000.0, slo_knee.DEFAULT_MAX_FAILURE_RATIO) + assert r.knee_rps == 92.0 + assert r.knee_users == 10 + + +# --------------------------------------------------------------------------- +# compute_knee: failure-ratio guard excludes a fast-failing high-RPS point +# --------------------------------------------------------------------------- + + +def test_knee_failure_guard_excludes_fast_failing_point(): + pts = _load("sweep_failing.jsonl") + # All three p95 (120,450,300) are <= 1000ms, but load_high_failing has + # failure_ratio 0.30 >> 0.01, so despite its 200 rps it must NOT be the knee. + r = slo_knee.compute_knee(pts, "p95", 1000.0, slo_knee.DEFAULT_MAX_FAILURE_RATIO) + assert r.n_under_ceiling == 3 + assert r.n_valid == 2 # load_high_failing filtered out + assert r.knee_rps == 60.0 # load_mid, not the 200-rps fast-failer + assert r.knee_test_name == "load_mid" + + +def test_knee_failure_guard_relaxed_admits_failing_point(): + pts = _load("sweep_failing.jsonl") + # With an absurdly loose tolerance the 200-rps point becomes eligible again. + r = slo_knee.compute_knee(pts, "p95", 1000.0, 0.99) + assert r.n_valid == 3 + assert r.knee_rps == 200.0 + assert r.knee_test_name == "load_high_failing" + + +# --------------------------------------------------------------------------- +# compute_knee: SLO never met +# --------------------------------------------------------------------------- + + +def test_knee_slo_never_met(): + pts = _load("sweep_never_met.jsonl") + r = slo_knee.compute_knee(pts, "p95", 5000.0, slo_knee.DEFAULT_MAX_FAILURE_RATIO) + assert r.knee_rps is None + assert r.slo_ever_met is False + assert r.n_under_ceiling == 0 + assert r.n_valid == 0 + # table rendering should say so explicitly + txt = slo_knee.format_table([r]) + assert "SLO NEVER MET" in txt + + +def test_knee_no_valid_but_slo_met_message(): + # A point under the ceiling but over the failure tolerance -> "no valid knee" + # (distinct from SLO-never-met). + pts = [ + slo_knee.Point( + tag="t", test_name="n", metric="m", rps=10.0, + request_count=100.0, failure_count=50.0, latencies_ms={"p95": 200.0}, + ) + ] + r = slo_knee.compute_knee(pts, "p95", 1000.0, slo_knee.DEFAULT_MAX_FAILURE_RATIO) + assert r.knee_rps is None + assert r.slo_ever_met is True # it WAS under the ceiling + assert r.n_under_ceiling == 1 + assert r.n_valid == 0 + txt = slo_knee.format_table([r]) + assert "no valid knee" in txt + + +# --------------------------------------------------------------------------- +# analyze: multi-metric grouping +# --------------------------------------------------------------------------- + + +def test_analyze_groups_by_tag_and_metric(): + records = [ + {"tag": "T", "test_name": "n1", "metric": "grpc_GetActor", + "measurements": {"request_count": "100", "failure_count": "0", "requests_per_s": "10", "p95": "200"}}, + {"tag": "T", "test_name": "n2", "metric": "grpc_GetActor", + "measurements": {"request_count": "200", "failure_count": "0", "requests_per_s": "20", "p95": "400"}}, + {"tag": "T", "test_name": "n1", "metric": "grpc_ResumeActor", + "measurements": {"request_count": "100", "failure_count": "0", "requests_per_s": "5", "p95": "900"}}, + {"tag": "T", "test_name": "n2", "metric": "grpc_ResumeActor", + "measurements": {"request_count": "200", "failure_count": "0", "requests_per_s": "8", "p95": "3000"}}, + ] + results = slo_knee.analyze(records, percentiles=["p95"], ceilings_ms=[1000.0]) + # one KneeResult per (tag, metric) group at this single percentile/ceiling + by_metric = {r.metric: r for r in results} + assert set(by_metric) == {"grpc_GetActor", "grpc_ResumeActor"} + # GetActor: both under 1000 -> knee = higher rps (n2 @ 20) + assert by_metric["grpc_GetActor"].knee_rps == 20.0 + # ResumeActor: only n1 (900) under 1000; n2 (3000) over -> knee = n1 @ 5 + assert by_metric["grpc_ResumeActor"].knee_rps == 5.0 + + +def test_analyze_metric_filter(): + records = list(slo_knee.iter_jsonl_records([_fx("sweep_clean.jsonl")])) + results = slo_knee.analyze( + records, percentiles=["p95"], ceilings_ms=[1000.0], + metric_filter="ResumeActor", + ) + assert results and all("ResumeActor" in r.metric for r in results) + results_none = slo_knee.analyze( + records, percentiles=["p95"], ceilings_ms=[1000.0], + metric_filter="NoSuchOp", + ) + assert results_none == [] + + +# --------------------------------------------------------------------------- +# expand_inputs / iter_jsonl_records +# --------------------------------------------------------------------------- + + +def test_expand_inputs_dir_walk_and_dedup(): + # directory walk finds the fixtures; passing the dir AND a file de-dups. + one = _fx("sweep_clean.jsonl") + got = slo_knee.expand_inputs([FIXTURES, one]) + assert one in got + assert len(got) == len(set(got)) # no dupes + + +def test_iter_jsonl_skips_blank_lines(tmp_path=None): + import tempfile + + with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False) as f: + f.write('{"tag":"t","test_name":"n","metric":"m","measurements":{}}\n') + f.write("\n") + f.write(" \n") + f.write('{"tag":"t","test_name":"n2","metric":"m","measurements":{}}\n') + path = f.name + try: + recs = list(slo_knee.iter_jsonl_records([path])) + assert len(recs) == 2 + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# parse_users_by_name +# --------------------------------------------------------------------------- + + +def test_parse_users_by_name(): + assert slo_knee.parse_users_by_name("a=1,b=5,c=10") == {"a": 1, "b": 5, "c": 10} + assert slo_knee.parse_users_by_name("") == {} + assert slo_knee.parse_users_by_name(None) == {} + try: + slo_knee.parse_users_by_name("bad_entry_without_equals") + except ValueError: + pass + else: + raise AssertionError("expected ValueError on entry without '='") + + +# --------------------------------------------------------------------------- +# main() CLI end-to-end (JSON output) +# --------------------------------------------------------------------------- + + +def test_main_json_output_end_to_end(capsys=None): + argv = [ + _fx("sweep_clean.jsonl"), + "-p", "p95", + "--ceiling-ms", "1000", + "--ceiling-ms", "5000", + "--users-by-name", + "glutton_baseline_1_user=1,glutton_baseline_5_users=5," + "glutton_baseline_10_users=10,glutton_oversubscribe_15_users=15", + "--json", + ] + buf = io.StringIO() + old = sys.stdout + sys.stdout = buf + try: + rc = slo_knee.main(argv) + finally: + sys.stdout = old + assert rc == 0 + out = json.loads(buf.getvalue()) + # two ceilings -> two results for the single (tag, metric) group + assert len(out) == 2 + by_ceiling = {r["ceiling_ms"]: r for r in out} + assert by_ceiling[1000.0]["knee_rps"] == 48.0 + assert by_ceiling[1000.0]["knee_users"] == 5 + assert by_ceiling[5000.0]["knee_rps"] == 92.0 + + +def test_main_no_inputs_resolved_returns_2(): + buf = io.StringIO() + old = sys.stderr + sys.stderr = buf + try: + rc = slo_knee.main([_fx("does_not_exist_*.jsonl")]) + finally: + sys.stderr = old + assert rc == 2 + + +# --------------------------------------------------------------------------- +# Minimal stdlib runner (so the suite works without pytest installed). +# --------------------------------------------------------------------------- + + +def _run_all() -> int: + fns = [ + (name, obj) + for name, obj in sorted(globals().items()) + if name.startswith("test_") and callable(obj) + ] + failed = 0 + for name, fn in fns: + try: + fn() + except Exception as e: # noqa: BLE001 + failed += 1 + print(f"FAIL {name}: {type(e).__name__}: {e}") + else: + print(f"ok {name}") + print(f"\n{len(fns) - failed}/{len(fns)} passed") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(_run_all()) diff --git a/hack/util/verify-boilerplate.py b/hack/util/verify-boilerplate.py index c315150f9..6881055c7 100755 --- a/hack/util/verify-boilerplate.py +++ b/hack/util/verify-boilerplate.py @@ -106,7 +106,7 @@ def main(): filename = os.path.basename(filepath) # Skip non-source-code files - if ext in ['.md', '.txt', '.png', '.jpg', '.jpeg', '.gif', '.mp4', '.json', '.pdf', '.ico', '.woff', '.woff2', '.ttf', '.otf', '.svg']: + if ext in ['.md', '.txt', '.png', '.jpg', '.jpeg', '.gif', '.mp4', '.json', '.jsonl', '.pdf', '.ico', '.woff', '.woff2', '.ttf', '.otf', '.svg']: continue if filename in ['LICENSE', 'NOTICE', 'CODEOWNERS', '.gitignore', 'go.mod', 'go.sum']: continue