Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
54.17% |
117 / 216 |
|
46.15% |
6 / 13 |
CRAP | |
0.00% |
0 / 1 |
| TimingsAggregator | |
54.17% |
117 / 216 |
|
46.15% |
6 / 13 |
323.46 | |
0.00% |
0 / 1 |
| percentiles | |
100.00% |
18 / 18 |
|
100.00% |
1 / 1 |
2 | |||
| slowest | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
1 | |||
| byEndpoint | |
100.00% |
22 / 22 |
|
100.00% |
1 / 1 |
3 | |||
| statusCodes | |
100.00% |
9 / 9 |
|
100.00% |
1 / 1 |
2 | |||
| timeseries | |
88.46% |
23 / 26 |
|
0.00% |
0 / 1 |
8.10 | |||
| consecutiveWindows | |
0.00% |
0 / 52 |
|
0.00% |
0 / 1 |
132 | |||
| windowContext | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
2 | |||
| alertImpact | |
0.00% |
0 / 23 |
|
0.00% |
0 / 1 |
20 | |||
| computeContext | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
12 | |||
| readWindow | |
93.75% |
30 / 32 |
|
0.00% |
0 / 1 |
11.03 | |||
| parseWindow | |
85.71% |
6 / 7 |
|
0.00% |
0 / 1 |
5.07 | |||
| normalizeEndpoint | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| pickPercentile | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace App\Services; |
| 4 | |
| 5 | use Illuminate\Support\Facades\Cache; |
| 6 | |
| 7 | /** |
| 8 | * FIRE-1197: aggregate the per-request log written by FIRE-1143's |
| 9 | * RequestTiming middleware (storage/logs/request_timings-YYYY-MM-DD.log, |
| 10 | * 14-day rotation) into the shapes the observability dashboard needs. |
| 11 | * |
| 12 | * Cached for 15 s per (method, args) combination via the default cache store |
| 13 | * (Redis on prod since 2026-06-08). File parsing is the expensive part; the |
| 14 | * cache keeps it bounded at ~once per 15 s per endpoint regardless of QPS. |
| 15 | * |
| 16 | * Window strings are accepted as `<N><m|h|d>`: '5m' = 5 min, '24h' = 24 h, |
| 17 | * '7d' = 7 days. The aggregator reads as many daily log files as the window |
| 18 | * spans (capped by the 14-day log retention). |
| 19 | * |
| 20 | * Endpoint normalisation: parameterised paths (e.g. `POST api/orders/174204`) |
| 21 | * are collapsed to `POST api/orders/{id}` via the `\b\d+\b` regex so per- |
| 22 | * endpoint group-by doesn't get one row per ID. Imperfect (numeric segments |
| 23 | * that aren't IDs collapse too) but adequate for v1 — see findings doc. |
| 24 | */ |
| 25 | class TimingsAggregator |
| 26 | { |
| 27 | private const CACHE_TTL_SECONDS = 15; |
| 28 | |
| 29 | /** |
| 30 | * p50/p95/p99 + count for the window. |
| 31 | * |
| 32 | * @return array{p50_ms:?int,p95_ms:?int,p99_ms:?int,count:int,window:string} |
| 33 | */ |
| 34 | public function percentiles(string $window = '5m'): array |
| 35 | { |
| 36 | return Cache::remember("observability:percentiles:{$window}", self::CACHE_TTL_SECONDS, function () use ($window) { |
| 37 | $rows = $this->readWindow($window); |
| 38 | $times = array_column($rows, 'wall_time_ms'); |
| 39 | |
| 40 | if (empty($times)) { |
| 41 | return [ |
| 42 | 'p50_ms' => null, 'p95_ms' => null, 'p99_ms' => null, |
| 43 | 'count' => 0, 'window' => $window, |
| 44 | ]; |
| 45 | } |
| 46 | |
| 47 | sort($times); |
| 48 | $count = count($times); |
| 49 | |
| 50 | return [ |
| 51 | 'p50_ms' => $this->pickPercentile($times, 0.50), |
| 52 | 'p95_ms' => $this->pickPercentile($times, 0.95), |
| 53 | 'p99_ms' => $this->pickPercentile($times, 0.99), |
| 54 | 'count' => $count, |
| 55 | 'window' => $window, |
| 56 | ]; |
| 57 | }); |
| 58 | } |
| 59 | |
| 60 | /** |
| 61 | * Top N slowest individual requests in the window. |
| 62 | * |
| 63 | * @return list<array{endpoint:string,wall_time_ms:int,db_query_count:int,db_time_ms:int,status:int,datetime:string,request_id:string}> |
| 64 | */ |
| 65 | public function slowest(string $window = '24h', int $limit = 10): array |
| 66 | { |
| 67 | $limit = max(1, min($limit, 100)); |
| 68 | |
| 69 | return Cache::remember("observability:slowest:{$window}:{$limit}", self::CACHE_TTL_SECONDS, function () use ($window, $limit) { |
| 70 | $rows = $this->readWindow($window); |
| 71 | usort($rows, fn (array $a, array $b): int => $b['wall_time_ms'] <=> $a['wall_time_ms']); |
| 72 | |
| 73 | return array_slice($rows, 0, $limit); |
| 74 | }); |
| 75 | } |
| 76 | |
| 77 | /** |
| 78 | * Per-endpoint aggregation. Sorted by p95 desc. |
| 79 | * |
| 80 | * @return list<array{endpoint:string,count:int,avg_ms:int,p95_ms:int,avg_queries:float}> |
| 81 | */ |
| 82 | public function byEndpoint(string $window = '24h'): array |
| 83 | { |
| 84 | return Cache::remember("observability:by-endpoint:{$window}", self::CACHE_TTL_SECONDS, function () use ($window) { |
| 85 | $rows = $this->readWindow($window); |
| 86 | $groups = []; |
| 87 | |
| 88 | foreach ($rows as $row) { |
| 89 | $endpoint = $this->normalizeEndpoint($row['endpoint']); |
| 90 | $groups[$endpoint] ??= ['times' => [], 'queries' => []]; |
| 91 | $groups[$endpoint]['times'][] = $row['wall_time_ms']; |
| 92 | $groups[$endpoint]['queries'][] = $row['db_query_count']; |
| 93 | } |
| 94 | |
| 95 | $result = []; |
| 96 | foreach ($groups as $endpoint => $g) { |
| 97 | sort($g['times']); |
| 98 | $count = count($g['times']); |
| 99 | $result[] = [ |
| 100 | 'endpoint' => $endpoint, |
| 101 | 'count' => $count, |
| 102 | 'avg_ms' => (int) round(array_sum($g['times']) / $count), |
| 103 | 'p95_ms' => $this->pickPercentile($g['times'], 0.95), |
| 104 | 'avg_queries' => round(array_sum($g['queries']) / $count, 1), |
| 105 | ]; |
| 106 | } |
| 107 | |
| 108 | usort($result, fn (array $a, array $b): int => $b['p95_ms'] <=> $a['p95_ms']); |
| 109 | |
| 110 | return $result; |
| 111 | }); |
| 112 | } |
| 113 | |
| 114 | /** |
| 115 | * Count of requests grouped by HTTP status code. Returned sorted by code asc. |
| 116 | * |
| 117 | * @return array<string,int> |
| 118 | */ |
| 119 | public function statusCodes(string $window = '1h'): array |
| 120 | { |
| 121 | return Cache::remember("observability:status-codes:{$window}", self::CACHE_TTL_SECONDS, function () use ($window) { |
| 122 | $rows = $this->readWindow($window); |
| 123 | $counts = []; |
| 124 | |
| 125 | foreach ($rows as $row) { |
| 126 | $status = (string) $row['status']; |
| 127 | $counts[$status] = ($counts[$status] ?? 0) + 1; |
| 128 | } |
| 129 | |
| 130 | ksort($counts); |
| 131 | |
| 132 | return $counts; |
| 133 | }); |
| 134 | } |
| 135 | |
| 136 | /** |
| 137 | * Hourly time-series of the chosen metric across the window. Suitable for |
| 138 | * an apexcharts series — `{x: 'YYYY-MM-DD HH', y: ms}`. |
| 139 | * |
| 140 | * Supported metrics: 'p50', 'p95' (default), 'p99', 'avg'. |
| 141 | * |
| 142 | * @return list<array{hour:string,value:int,count:int}> |
| 143 | */ |
| 144 | public function timeseries(string $metric = 'p95', string $window = '24h'): array |
| 145 | { |
| 146 | $metric = in_array($metric, ['p50', 'p95', 'p99', 'avg'], true) ? $metric : 'p95'; |
| 147 | |
| 148 | return Cache::remember("observability:timeseries:{$metric}:{$window}", self::CACHE_TTL_SECONDS, function () use ($metric, $window) { |
| 149 | $rows = $this->readWindow($window); |
| 150 | $buckets = []; |
| 151 | |
| 152 | foreach ($rows as $row) { |
| 153 | // 'YYYY-MM-DDTHH:MM:SS...' → 'YYYY-MM-DD HH' |
| 154 | $hour = substr($row['datetime'], 0, 13); |
| 155 | $hour = str_replace('T', ' ', $hour); |
| 156 | $buckets[$hour][] = $row['wall_time_ms']; |
| 157 | } |
| 158 | |
| 159 | ksort($buckets); |
| 160 | |
| 161 | $result = []; |
| 162 | foreach ($buckets as $hour => $times) { |
| 163 | sort($times); |
| 164 | $count = count($times); |
| 165 | $value = match ($metric) { |
| 166 | 'p50' => $this->pickPercentile($times, 0.50), |
| 167 | 'p99' => $this->pickPercentile($times, 0.99), |
| 168 | 'avg' => (int) round(array_sum($times) / $count), |
| 169 | default => $this->pickPercentile($times, 0.95), |
| 170 | }; |
| 171 | $result[] = [ |
| 172 | 'hour' => $hour, |
| 173 | 'value' => $value, |
| 174 | 'count' => $count, |
| 175 | ]; |
| 176 | } |
| 177 | |
| 178 | return $result; |
| 179 | }); |
| 180 | } |
| 181 | |
| 182 | /** |
| 183 | * FIRE-1275 (SOP align): split the last N minutes into equal sub-windows |
| 184 | * and return the per-bucket p95, error rate, count and distinct-users |
| 185 | * affected. Used by the alerter cron to enforce "sostenido durante 15 min" |
| 186 | * (3 consecutive 5-min buckets must all breach) instead of firing on |
| 187 | * single-window spikes — the SOP's core anti-flap rule. |
| 188 | * |
| 189 | * @return list<array{ |
| 190 | * from_ts:int, to_ts:int, |
| 191 | * count:int, users_active:int, |
| 192 | * p95_ms:int, p99_ms:int, |
| 193 | * error_count:int, error_users:int, |
| 194 | * users_affected_5xx:int, users_affected_slow:int |
| 195 | * }> |
| 196 | */ |
| 197 | public function consecutiveWindows(int $buckets, int $minutesPerBucket): array |
| 198 | { |
| 199 | $windowStr = ($buckets * $minutesPerBucket).'m'; |
| 200 | $cacheKey = "observability:consecutive:{$buckets}x{$minutesPerBucket}m"; |
| 201 | |
| 202 | return Cache::remember($cacheKey, self::CACHE_TTL_SECONDS, function () use ($buckets, $minutesPerBucket, $windowStr) { |
| 203 | $rows = $this->readWindow($windowStr); |
| 204 | $now = time(); |
| 205 | $secondsPerBucket = $minutesPerBucket * 60; |
| 206 | |
| 207 | $result = []; |
| 208 | for ($i = $buckets - 1; $i >= 0; $i--) { |
| 209 | // Bucket i covers [now - (i+1)*Δ, now - i*Δ). Oldest → newest. |
| 210 | $fromTs = $now - ($i + 1) * $secondsPerBucket; |
| 211 | $toTs = $now - $i * $secondsPerBucket; |
| 212 | |
| 213 | $bucketRows = array_filter($rows, static function (array $r) use ($fromTs, $toTs): bool { |
| 214 | $ts = strtotime($r['datetime']); |
| 215 | |
| 216 | return $ts !== false && $ts >= $fromTs && $ts < $toTs; |
| 217 | }); |
| 218 | $bucketRows = array_values($bucketRows); |
| 219 | $count = count($bucketRows); |
| 220 | if ($count === 0) { |
| 221 | $result[] = [ |
| 222 | 'from_ts' => $fromTs, 'to_ts' => $toTs, |
| 223 | 'count' => 0, 'users_active' => 0, |
| 224 | 'p95_ms' => 0, 'p99_ms' => 0, |
| 225 | 'error_count' => 0, 'error_users' => 0, |
| 226 | 'users_affected_5xx' => 0, 'users_affected_slow' => 0, |
| 227 | ]; |
| 228 | |
| 229 | continue; |
| 230 | } |
| 231 | |
| 232 | $times = array_column($bucketRows, 'wall_time_ms'); |
| 233 | sort($times); |
| 234 | $users = []; |
| 235 | $errorUsers = []; |
| 236 | $slowUsers = []; |
| 237 | $errorCount = 0; |
| 238 | foreach ($bucketRows as $r) { |
| 239 | if ($r['user_id'] !== null) { |
| 240 | $users[$r['user_id']] = true; |
| 241 | } |
| 242 | if ($r['status'] >= 500) { |
| 243 | $errorCount++; |
| 244 | if ($r['user_id'] !== null) { |
| 245 | $errorUsers[$r['user_id']] = true; |
| 246 | } |
| 247 | } |
| 248 | if ($r['wall_time_ms'] >= 3000) { // A2 slow-request predicate |
| 249 | if ($r['user_id'] !== null) { |
| 250 | $slowUsers[$r['user_id']] = true; |
| 251 | } |
| 252 | } |
| 253 | } |
| 254 | |
| 255 | $result[] = [ |
| 256 | 'from_ts' => $fromTs, 'to_ts' => $toTs, |
| 257 | 'count' => $count, 'users_active' => count($users), |
| 258 | 'p95_ms' => $this->pickPercentile($times, 0.95), |
| 259 | 'p99_ms' => $this->pickPercentile($times, 0.99), |
| 260 | 'error_count' => $errorCount, 'error_users' => count($errorUsers), |
| 261 | 'users_affected_5xx' => count($errorUsers), |
| 262 | 'users_affected_slow' => count($slowUsers), |
| 263 | ]; |
| 264 | } |
| 265 | |
| 266 | return $result; |
| 267 | }); |
| 268 | } |
| 269 | |
| 270 | /** |
| 271 | * FIRE-1275: per-window denominators used by the observability alert |
| 272 | * banner to render "N of M requests" and "A of B users" ratios. |
| 273 | * |
| 274 | * Anonymous requests (user_id = null — pre-login, webhooks, health) are |
| 275 | * excluded from `total_users_active` so the ratio isn't skewed by |
| 276 | * fire-and-forget traffic that has no real "user" behind it. They are |
| 277 | * still counted in `total_requests` because the request-scale ratio |
| 278 | * needs the full denominator. |
| 279 | * |
| 280 | * @return array{window:string,total_requests:int,total_users_active:int,anonymous_requests:int} |
| 281 | */ |
| 282 | public function windowContext(string $window = '5m'): array |
| 283 | { |
| 284 | return Cache::remember("observability:context:{$window}", self::CACHE_TTL_SECONDS, function () use ($window) { |
| 285 | $rows = $this->readWindow($window); |
| 286 | |
| 287 | return $this->computeContext($rows, $window); |
| 288 | }); |
| 289 | } |
| 290 | |
| 291 | /** |
| 292 | * FIRE-1275: per-alert impact — how many requests / users are in the |
| 293 | * red bucket for each alert type. Same window is used for numerator |
| 294 | * and denominator, so the ratios the frontend renders are honest. |
| 295 | * |
| 296 | * Alert-to-predicate mapping (matches the "watch" threshold in the FE |
| 297 | * component — the value that colours the tile red): |
| 298 | * p50 → wall_time_ms >= 500 |
| 299 | * p95 → wall_time_ms >= 3000 |
| 300 | * p99 → wall_time_ms >= 8000 |
| 301 | * 5xx → status >= 500 |
| 302 | * |
| 303 | * Users are counted distinct, non-null. Requests are counted raw. |
| 304 | * |
| 305 | * @return array{window:string,context:array,alerts:array<string,array{affected_requests:int,affected_users:int}>} |
| 306 | */ |
| 307 | public function alertImpact(string $window = '5m'): array |
| 308 | { |
| 309 | return Cache::remember("observability:alert-impact:{$window}", self::CACHE_TTL_SECONDS, function () use ($window) { |
| 310 | $rows = $this->readWindow($window); |
| 311 | $context = $this->computeContext($rows, $window); |
| 312 | |
| 313 | $countFor = function (callable $predicate) use ($rows): array { |
| 314 | $reqs = 0; |
| 315 | $users = []; |
| 316 | foreach ($rows as $r) { |
| 317 | if (! $predicate($r)) { |
| 318 | continue; |
| 319 | } |
| 320 | $reqs++; |
| 321 | if ($r['user_id'] !== null) { |
| 322 | $users[$r['user_id']] = true; |
| 323 | } |
| 324 | } |
| 325 | |
| 326 | return ['affected_requests' => $reqs, 'affected_users' => count($users)]; |
| 327 | }; |
| 328 | |
| 329 | return [ |
| 330 | 'window' => $window, |
| 331 | 'context' => $context, |
| 332 | 'alerts' => [ |
| 333 | 'p50' => $countFor(static fn (array $r): bool => $r['wall_time_ms'] >= 500), |
| 334 | 'p95' => $countFor(static fn (array $r): bool => $r['wall_time_ms'] >= 3000), |
| 335 | 'p99' => $countFor(static fn (array $r): bool => $r['wall_time_ms'] >= 8000), |
| 336 | '5xx' => $countFor(static fn (array $r): bool => $r['status'] >= 500), |
| 337 | ], |
| 338 | ]; |
| 339 | }); |
| 340 | } |
| 341 | |
| 342 | /** |
| 343 | * Shared helper for {@see windowContext} and {@see alertImpact}. Kept |
| 344 | * private because the caller must already be inside a Cache::remember |
| 345 | * block — we don't want an extra cache layer. |
| 346 | * |
| 347 | * @param list<array{user_id:?int}> $rows |
| 348 | * @return array{window:string,total_requests:int,total_users_active:int,anonymous_requests:int} |
| 349 | */ |
| 350 | private function computeContext(array $rows, string $window): array |
| 351 | { |
| 352 | $totalRequests = count($rows); |
| 353 | $users = []; |
| 354 | $anonymous = 0; |
| 355 | foreach ($rows as $r) { |
| 356 | if ($r['user_id'] === null) { |
| 357 | $anonymous++; |
| 358 | |
| 359 | continue; |
| 360 | } |
| 361 | $users[$r['user_id']] = true; |
| 362 | } |
| 363 | |
| 364 | return [ |
| 365 | 'window' => $window, |
| 366 | 'total_requests' => $totalRequests, |
| 367 | 'total_users_active' => count($users), |
| 368 | 'anonymous_requests' => $anonymous, |
| 369 | ]; |
| 370 | } |
| 371 | |
| 372 | /** |
| 373 | * Read all log entries within the given window (relative to now). Spans |
| 374 | * multiple daily files when the window crosses midnight. |
| 375 | * |
| 376 | * @return list<array{datetime:string,endpoint:string,wall_time_ms:int,db_query_count:int,db_time_ms:int,status:int,request_id:string}> |
| 377 | */ |
| 378 | private function readWindow(string $window): array |
| 379 | { |
| 380 | $seconds = $this->parseWindow($window); |
| 381 | $cutoff = time() - $seconds; |
| 382 | |
| 383 | // One extra day of slack to cover the case where the window starts a |
| 384 | // few seconds before midnight. Capped at 14 days (the log retention). |
| 385 | $daysToRead = min(14, (int) ceil($seconds / 86400) + 1); |
| 386 | $rows = []; |
| 387 | |
| 388 | for ($i = 0; $i < $daysToRead; $i++) { |
| 389 | $date = date('Y-m-d', time() - $i * 86400); |
| 390 | $file = storage_path("logs/request_timings-{$date}.log"); |
| 391 | if (! file_exists($file)) { |
| 392 | continue; |
| 393 | } |
| 394 | |
| 395 | $fp = @fopen($file, 'r'); |
| 396 | if (! $fp) { |
| 397 | continue; |
| 398 | } |
| 399 | |
| 400 | while (($line = fgets($fp)) !== false) { |
| 401 | $row = json_decode($line, true); |
| 402 | if (! is_array($row) || empty($row['datetime']) || empty($row['context'])) { |
| 403 | continue; |
| 404 | } |
| 405 | |
| 406 | $ts = strtotime($row['datetime']); |
| 407 | if ($ts === false || $ts < $cutoff) { |
| 408 | continue; |
| 409 | } |
| 410 | |
| 411 | $ctx = $row['context']; |
| 412 | $rows[] = [ |
| 413 | 'datetime' => $row['datetime'], |
| 414 | 'endpoint' => (string) ($ctx['endpoint'] ?? ''), |
| 415 | 'wall_time_ms' => (int) ($ctx['wall_time_ms'] ?? 0), |
| 416 | 'db_query_count' => (int) ($ctx['db_query_count'] ?? 0), |
| 417 | 'db_time_ms' => (int) ($ctx['db_time_ms'] ?? 0), |
| 418 | 'status' => (int) ($ctx['status'] ?? 0), |
| 419 | 'request_id' => (string) ($ctx['request_id'] ?? ''), |
| 420 | // FIRE-1275: null for anonymous routes (pre-login, webhooks, health). |
| 421 | // Pre-deploy log lines never had this key at all — coerce to null. |
| 422 | 'user_id' => isset($ctx['user_id']) ? (int) $ctx['user_id'] : null, |
| 423 | ]; |
| 424 | } |
| 425 | fclose($fp); |
| 426 | } |
| 427 | |
| 428 | return $rows; |
| 429 | } |
| 430 | |
| 431 | /** |
| 432 | * Parse a window string like '5m', '1h', '24h', '7d' into seconds. |
| 433 | * Falls back to 300 (5 min) on malformed input. |
| 434 | */ |
| 435 | private function parseWindow(string $window): int |
| 436 | { |
| 437 | if (! preg_match('/^(\d+)([mhd])$/', $window, $m)) { |
| 438 | return 300; |
| 439 | } |
| 440 | |
| 441 | return (int) $m[1] * match ($m[2]) { |
| 442 | 'm' => 60, |
| 443 | 'h' => 3600, |
| 444 | 'd' => 86400, |
| 445 | }; |
| 446 | } |
| 447 | |
| 448 | /** |
| 449 | * "POST api/orders/174204" → "POST api/orders/{id}". |
| 450 | * Collapses any standalone numeric segment so per-endpoint group-by |
| 451 | * doesn't fragment into per-ID rows. |
| 452 | */ |
| 453 | private function normalizeEndpoint(string $endpoint): string |
| 454 | { |
| 455 | return (string) preg_replace('/\b\d+\b/', '{id}', $endpoint); |
| 456 | } |
| 457 | |
| 458 | /** |
| 459 | * Pick the p-th percentile from a pre-sorted ascending array. |
| 460 | */ |
| 461 | private function pickPercentile(array $sortedTimes, float $p): int |
| 462 | { |
| 463 | $count = count($sortedTimes); |
| 464 | |
| 465 | return (int) $sortedTimes[min($count - 1, (int) floor($count * $p))]; |
| 466 | } |
| 467 | } |