Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 165 |
|
0.00% |
0 / 11 |
CRAP | |
0.00% |
0 / 1 |
| CheckPerformanceThresholds | |
0.00% |
0 / 165 |
|
0.00% |
0 / 11 |
3080 | |
0.00% |
0 / 1 |
| handle | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
2 | |||
| checkA1FiveXX | |
0.00% |
0 / 33 |
|
0.00% |
0 / 1 |
90 | |||
| checkA2Latency | |
0.00% |
0 / 36 |
|
0.00% |
0 / 1 |
90 | |||
| checkA3Downtime | |
0.00% |
0 / 22 |
|
0.00% |
0 / 1 |
30 | |||
| checkA4NPlusOne | |
0.00% |
0 / 26 |
|
0.00% |
0 / 1 |
30 | |||
| allBucketsBreach | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
20 | |||
| isBusinessHours | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
12 | |||
| emitAlert | |
0.00% |
0 / 19 |
|
0.00% |
0 / 1 |
156 | |||
| formatSlackContext | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
20 | |||
| humanLatency | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
6 | |||
| fmtPct | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace App\Console\Commands; |
| 4 | |
| 5 | use App\Services\TimingsAggregator; |
| 6 | use Carbon\Carbon; |
| 7 | use Illuminate\Console\Command; |
| 8 | use Illuminate\Support\Facades\Cache; |
| 9 | use Illuminate\Support\Facades\Log; |
| 10 | use Sentry\Severity; |
| 11 | use Sentry\State\Hub; |
| 12 | |
| 13 | /** |
| 14 | * FIRE-1197 Phase 3 / FIRE-1275 SOP alignment: rolling threshold check that |
| 15 | * publishes Slack + Sentry alerts when TITAN is in a real problem — as |
| 16 | * defined by docs/info/SOP_Alertas_Slack_TITAN.docx. |
| 17 | * |
| 18 | * Runs every minute via `Schedule::command(...)->everyMinute()` in |
| 19 | * routes/console.php with `withoutOverlapping(5)` so a slow POST never |
| 20 | * tramples the next tick. |
| 21 | * |
| 22 | * ---- Alerts (per SOP) ------------------------------------------------ |
| 23 | * A1 · Errores de servidor (5xx) URGENTE @here Slack + Sentry |
| 24 | * Trigger: (5xx rate > 5 % sostenido 15 min) OR (≥3 usuarios afectados en 5 min) |
| 25 | * A2 · Lentitud sostenida (p95/p99) ALTO — Sentry only |
| 26 | * Trigger: (p95 > 3 s sostenido 15 min) OR (p99 > 8 s sostenido 15 min) |
| 27 | * A3 · TITAN no responde / posible caída URGENTE @here Slack + Sentry |
| 28 | * Trigger: peticiones = 0 durante 10 min en horario laboral (L-V, 8-20 Madrid) |
| 29 | * A4 · Endpoint con demasiadas consultas BD BAJO — Sentry only |
| 30 | * Trigger: cualquier endpoint con avg_queries > 50 sobre la última hora |
| 31 | * |
| 32 | * ---- Anti-flap ------------------------------------------------------- |
| 33 | * "Sostenido" = las últimas 3 ventanas de 5 min consecutivas TODAS |
| 34 | * incumplen el umbral. Se lee vía TimingsAggregator::consecutiveWindows(). |
| 35 | * Un pico de un solo minuto que se recupera no dispara alerta — es |
| 36 | * exactamente lo que la SOP quiere evitar. |
| 37 | * |
| 38 | * Alert dedup uses `Cache::add($key, 1, 900)` — atomic on Redis (default |
| 39 | * store on prod since 2026-06-08). If CACHE_DRIVER flips back to `file`, |
| 40 | * dedup silently breaks and Slack will spam. Keep CACHE_DRIVER=redis. |
| 41 | * |
| 42 | * ---- Routing --------------------------------------------------------- |
| 43 | * URGENTE (A1, A3) → Sentry ERROR + Slack (@here). Business channel pages. |
| 44 | * ALTO (A2) → Sentry WARNING. Slack skipped (engineers only). |
| 45 | * BAJO (A4) → Sentry INFO. Slack skipped (engineers only). |
| 46 | * |
| 47 | * ---- Message copy ---------------------------------------------------- |
| 48 | * Spanish, business-friendly. Mirrors the templates in the SOP §6 |
| 49 | * ("Detalle de las alertas"). Includes users-affected/active ratios |
| 50 | * introduced by FIRE-1275. |
| 51 | */ |
| 52 | class CheckPerformanceThresholds extends Command |
| 53 | { |
| 54 | protected $signature = 'observability:check-thresholds'; |
| 55 | |
| 56 | protected $description = 'FIRE-1275: emit Sentry / Slack alerts per SOP thresholds (5-min buckets, 15-min sostenido).'; |
| 57 | |
| 58 | // --- A1 5xx --- |
| 59 | private const A1_ERROR_RATE_THRESHOLD = 0.05; // 5% |
| 60 | |
| 61 | private const A1_MIN_REQUESTS_FOR_RATE = 20; |
| 62 | |
| 63 | private const A1_MIN_USERS_INSTANT = 3; // burst trigger |
| 64 | |
| 65 | // --- A2 latency --- |
| 66 | private const A2_P95_THRESHOLD_MS = 3000; // SOP: 3 s |
| 67 | |
| 68 | private const A2_P99_THRESHOLD_MS = 8000; |
| 69 | |
| 70 | // --- A3 no-traffic (business hours) --- |
| 71 | private const A3_NO_TRAFFIC_BUCKETS = 2; // 2 × 5 min = 10 min |
| 72 | |
| 73 | private const A3_BUSINESS_TZ = 'Europe/Madrid'; |
| 74 | |
| 75 | private const A3_BUSINESS_START_HOUR = 8; |
| 76 | |
| 77 | private const A3_BUSINESS_END_HOUR = 20; // exclusive |
| 78 | |
| 79 | // --- A4 N+1 --- |
| 80 | private const A4_QUERIES_PER_REQ_THRESHOLD = 50; |
| 81 | |
| 82 | private const A4_WINDOW = '1h'; |
| 83 | |
| 84 | // --- shared --- |
| 85 | private const SOSTENIDO_BUCKETS = 3; // 3 × 5 min = 15 min |
| 86 | |
| 87 | private const BUCKET_MINUTES = 5; |
| 88 | |
| 89 | private const WINDOW_5M = '5m'; |
| 90 | |
| 91 | private const DEDUP_TTL_SECONDS = 3600; // SOP §8: cooldown 1h |
| 92 | |
| 93 | public function handle(TimingsAggregator $aggregator): int |
| 94 | { |
| 95 | // Read once — same set of rows drives A1 (users trigger), A2 (sostenido), |
| 96 | // and A3 (no-traffic). A4 uses a different window (1h), fetched below. |
| 97 | $buckets = $aggregator->consecutiveWindows(self::SOSTENIDO_BUCKETS, self::BUCKET_MINUTES); |
| 98 | $context5m = $aggregator->windowContext(self::WINDOW_5M); |
| 99 | $impact5m = $aggregator->alertImpact(self::WINDOW_5M); |
| 100 | |
| 101 | $this->checkA1FiveXX($buckets, $context5m, $impact5m); |
| 102 | $this->checkA2Latency($buckets); |
| 103 | $this->checkA3Downtime($buckets, $context5m); |
| 104 | $this->checkA4NPlusOne($aggregator->byEndpoint(self::A4_WINDOW)); |
| 105 | |
| 106 | return self::SUCCESS; |
| 107 | } |
| 108 | |
| 109 | // ── A1 · errores 5xx ──────────────────────────────────────────────────── |
| 110 | private function checkA1FiveXX(array $buckets, array $context5m, array $impact5m): void |
| 111 | { |
| 112 | $latest = end($buckets) ?: null; |
| 113 | if ($latest === null) { |
| 114 | return; |
| 115 | } |
| 116 | |
| 117 | $sostenidoBreach = $this->allBucketsBreach( |
| 118 | $buckets, |
| 119 | static fn (array $b): bool => $b['count'] >= self::A1_MIN_REQUESTS_FOR_RATE |
| 120 | && ($b['error_count'] / max($b['count'], 1)) > self::A1_ERROR_RATE_THRESHOLD, |
| 121 | ); |
| 122 | |
| 123 | $usersBurst = $latest['users_affected_5xx'] >= self::A1_MIN_USERS_INSTANT; |
| 124 | |
| 125 | if (! $sostenidoBreach && ! $usersBurst) { |
| 126 | return; |
| 127 | } |
| 128 | |
| 129 | if (! Cache::add('observability:alert:a1:fired', 1, self::DEDUP_TTL_SECONDS)) { |
| 130 | return; |
| 131 | } |
| 132 | |
| 133 | $errUsers = (int) ($impact5m['alerts']['5xx']['affected_users'] ?? 0); |
| 134 | $errRequests = (int) ($impact5m['alerts']['5xx']['affected_requests'] ?? 0); |
| 135 | $totalReq = (int) ($context5m['total_requests'] ?? 0); |
| 136 | $totalUsers = (int) ($context5m['total_users_active'] ?? 0); |
| 137 | $pctReq = $totalReq > 0 ? ($errRequests / $totalReq) * 100 : 0; |
| 138 | $pctUsers = $totalUsers > 0 ? ($errUsers / $totalUsers) * 100 : 0; |
| 139 | |
| 140 | $message = sprintf( |
| 141 | "🚨 *TITAN: errores de servidor (5xx)* <!here>\n" |
| 142 | .'Algo se ha roto en el servidor (bug real). Los usuarios verán acciones que fallan.'."\n" |
| 143 | .'*Impacto (últimos 5 min):*'."\n" |
| 144 | .'• Peticiones: %d de %d con error (%s %%)'."\n" |
| 145 | .'• Usuarios: %d de %d afectados (%s %%)'."\n" |
| 146 | .'• Hora: %s', |
| 147 | $errRequests, $totalReq, $this->fmtPct($pctReq), |
| 148 | $errUsers, $totalUsers, $this->fmtPct($pctUsers), |
| 149 | Carbon::now(self::A3_BUSINESS_TZ)->format('H:i'), |
| 150 | ); |
| 151 | |
| 152 | // Berta feedback (FIRE-1275): the Slack fields block should only show |
| 153 | // the alert flags — counts/percentages are already in the message body, |
| 154 | // so echoing them below is noise. Keep the numbers for the Log::error |
| 155 | // + Sentry paths by passing them separately? No — we prefer symmetry. |
| 156 | // If Sentry needs numeric filtering later, add tags there directly. |
| 157 | $this->emitAlert($message, [ |
| 158 | 'alert_id' => 'A1', 'sustained' => $sostenidoBreach, 'users_burst' => $usersBurst, |
| 159 | ], sendSlack: true, sentrySeverity: Severity::error()); |
| 160 | } |
| 161 | |
| 162 | // ── A2 · lentitud sostenida ───────────────────────────────────────────── |
| 163 | private function checkA2Latency(array $buckets): void |
| 164 | { |
| 165 | $p95Breach = $this->allBucketsBreach( |
| 166 | $buckets, |
| 167 | static fn (array $b): bool => $b['count'] > 0 && $b['p95_ms'] > self::A2_P95_THRESHOLD_MS, |
| 168 | ); |
| 169 | $p99Breach = $this->allBucketsBreach( |
| 170 | $buckets, |
| 171 | static fn (array $b): bool => $b['count'] > 0 && $b['p99_ms'] > self::A2_P99_THRESHOLD_MS, |
| 172 | ); |
| 173 | |
| 174 | if (! $p95Breach && ! $p99Breach) { |
| 175 | return; |
| 176 | } |
| 177 | |
| 178 | if (! Cache::add('observability:alert:a2:fired', 1, self::DEDUP_TTL_SECONDS)) { |
| 179 | return; |
| 180 | } |
| 181 | |
| 182 | // Aggregate the last 15 min for the message body. |
| 183 | $totalReq = array_sum(array_column($buckets, 'count')); |
| 184 | $slowReq = 0; |
| 185 | $slowUsersSet = []; |
| 186 | foreach ($buckets as $b) { |
| 187 | // Slow requests = count × (1 - 0.95) roughly, but we have the p95 value not the raw predicate. |
| 188 | // Estimate: count of requests > 3s ≈ 5 % of bucket count when p95 > 3s. |
| 189 | // The exact count is in impact-5m for the LAST 5m, but we want the full 15m — approximation is fine for the copy. |
| 190 | $slowReq += (int) round($b['count'] * 0.05); |
| 191 | } |
| 192 | $activeUsers = array_sum(array_column($buckets, 'users_active')); // sum-of-bucket approximation |
| 193 | $affectedUsers = array_sum(array_column($buckets, 'users_affected_slow')); |
| 194 | |
| 195 | $worstP95 = max(array_column($buckets, 'p95_ms')); |
| 196 | $worstP99 = max(array_column($buckets, 'p99_ms')); |
| 197 | |
| 198 | $message = sprintf( |
| 199 | "⚠️ *TITAN: lentitud sostenida*\n" |
| 200 | .'Degradación real, no un pico puntual.'."\n" |
| 201 | .'*Impacto (últimos 15 min):*'."\n" |
| 202 | .'• Peticiones lentas (>3 s): ~%d de %d (%s %%)'."\n" |
| 203 | .'• Usuarios notando lentitud: ~%d de %d (%s %%)'."\n" |
| 204 | .'• p95 peor: %s · p99 peor: %s'."\n" |
| 205 | .'• Revisar `p95 por endpoint` en el dashboard para localizar el culpable.', |
| 206 | $slowReq, $totalReq, $this->fmtPct($totalReq > 0 ? ($slowReq / $totalReq) * 100 : 0), |
| 207 | $affectedUsers, max($activeUsers, 1), $this->fmtPct($activeUsers > 0 ? ($affectedUsers / $activeUsers) * 100 : 0), |
| 208 | $this->humanLatency($worstP95), $this->humanLatency($worstP99), |
| 209 | ); |
| 210 | |
| 211 | $this->emitAlert($message, [ |
| 212 | 'alert_id' => 'A2', 'p95_breach' => $p95Breach, 'p99_breach' => $p99Breach, |
| 213 | ], sendSlack: false, sentrySeverity: Severity::warning()); |
| 214 | } |
| 215 | |
| 216 | // ── A3 · TITAN no responde ────────────────────────────────────────────── |
| 217 | private function checkA3Downtime(array $buckets, array $context5m): void |
| 218 | { |
| 219 | if (! $this->isBusinessHours()) { |
| 220 | return; |
| 221 | } |
| 222 | // A3 fires when count = 0 across the last N buckets (10 min = 2 buckets). |
| 223 | $recent = array_slice($buckets, -self::A3_NO_TRAFFIC_BUCKETS); |
| 224 | $allEmpty = ! empty($recent) && array_sum(array_column($recent, 'count')) === 0; |
| 225 | if (! $allEmpty) { |
| 226 | return; |
| 227 | } |
| 228 | |
| 229 | if (! Cache::add('observability:alert:a3:fired', 1, self::DEDUP_TTL_SECONDS)) { |
| 230 | return; |
| 231 | } |
| 232 | |
| 233 | $message = sprintf( |
| 234 | "🚨 *TITAN: sin actividad / posible caída* <!here>\n" |
| 235 | ."Sin actividad en los últimos %d min en horario laboral: o TITAN está caído o la medición se ha roto.\n" |
| 236 | .'*Impacto:*'."\n" |
| 237 | .'• Peticiones (últimos %d min): 0'."\n" |
| 238 | .'• Acción: comprobad abriendo https://fireservicetitan.com'."\n" |
| 239 | .'• Hora: %s', |
| 240 | self::A3_NO_TRAFFIC_BUCKETS * self::BUCKET_MINUTES, |
| 241 | self::A3_NO_TRAFFIC_BUCKETS * self::BUCKET_MINUTES, |
| 242 | Carbon::now(self::A3_BUSINESS_TZ)->format('H:i'), |
| 243 | ); |
| 244 | |
| 245 | $this->emitAlert($message, [ |
| 246 | 'alert_id' => 'A3', |
| 247 | ], sendSlack: true, sentrySeverity: Severity::error()); |
| 248 | } |
| 249 | |
| 250 | // ── A4 · N+1 en un endpoint ───────────────────────────────────────────── |
| 251 | private function checkA4NPlusOne(array $byEndpoint): void |
| 252 | { |
| 253 | $offenders = array_values(array_filter( |
| 254 | $byEndpoint, |
| 255 | static fn (array $row): bool => ($row['avg_queries'] ?? 0) > self::A4_QUERIES_PER_REQ_THRESHOLD |
| 256 | && ($row['count'] ?? 0) >= 20, // ignore tiny sample sizes |
| 257 | )); |
| 258 | if (empty($offenders)) { |
| 259 | return; |
| 260 | } |
| 261 | |
| 262 | // Dedup PER endpoint so re-fires only when a NEW endpoint enters the red zone. |
| 263 | foreach ($offenders as $row) { |
| 264 | $endpoint = (string) $row['endpoint']; |
| 265 | $key = 'observability:alert:a4:'.md5($endpoint); |
| 266 | if (! Cache::add($key, 1, self::DEDUP_TTL_SECONDS)) { |
| 267 | continue; |
| 268 | } |
| 269 | |
| 270 | $message = sprintf( |
| 271 | "ℹ️ *TITAN: posible N+1 en un endpoint*\n" |
| 272 | .'Problema de código, no urgente.'."\n" |
| 273 | .'• Endpoint: `%s`'."\n" |
| 274 | .'• Consultas/petición (media): %.1f'."\n" |
| 275 | .'• Alcance (última hora): %d peticiones · p95 = %s', |
| 276 | $endpoint, |
| 277 | (float) ($row['avg_queries'] ?? 0), |
| 278 | (int) ($row['count'] ?? 0), |
| 279 | $this->humanLatency((int) ($row['p95_ms'] ?? 0)), |
| 280 | ); |
| 281 | |
| 282 | $this->emitAlert($message, [ |
| 283 | 'alert_id' => 'A4', |
| 284 | ], sendSlack: false, sentrySeverity: Severity::info()); |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | // ── helpers ───────────────────────────────────────────────────────────── |
| 289 | |
| 290 | /** |
| 291 | * True if every bucket in the list satisfies the predicate. Empty list |
| 292 | * returns false to avoid firing when we haven't read any data. |
| 293 | */ |
| 294 | private function allBucketsBreach(array $buckets, callable $predicate): bool |
| 295 | { |
| 296 | if (empty($buckets)) { |
| 297 | return false; |
| 298 | } |
| 299 | foreach ($buckets as $b) { |
| 300 | if (! $predicate($b)) { |
| 301 | return false; |
| 302 | } |
| 303 | } |
| 304 | |
| 305 | return true; |
| 306 | } |
| 307 | |
| 308 | /** |
| 309 | * L–V, 8:00–20:00 in Madrid time. Weekends and off-hours skip A3 because |
| 310 | * low traffic then is normal, not an outage. |
| 311 | */ |
| 312 | private function isBusinessHours(): bool |
| 313 | { |
| 314 | $now = Carbon::now(self::A3_BUSINESS_TZ); |
| 315 | if ($now->isWeekend()) { |
| 316 | return false; |
| 317 | } |
| 318 | $h = (int) $now->format('H'); |
| 319 | |
| 320 | return $h >= self::A3_BUSINESS_START_HOUR && $h < self::A3_BUSINESS_END_HOUR; |
| 321 | } |
| 322 | |
| 323 | private function emitAlert(string $message, array $context, bool $sendSlack, Severity $sentrySeverity): void |
| 324 | { |
| 325 | // Local log — always, at the same severity we're escalating to. |
| 326 | // Severity has __toString → 'fatal' / 'error' / 'warning' / 'info' / 'debug'. |
| 327 | $sev = (string) $sentrySeverity; |
| 328 | $isError = $sev === 'error' || $sev === 'fatal'; |
| 329 | $isInfo = $sev === 'info' || $sev === 'debug'; |
| 330 | if ($isError) { |
| 331 | Log::error($message, $context); |
| 332 | } elseif ($isInfo) { |
| 333 | Log::info($message, $context); |
| 334 | } else { |
| 335 | Log::warning($message, $context); |
| 336 | } |
| 337 | |
| 338 | // Sentry path — works only if sentry-laravel is installed and DSN set. |
| 339 | if (class_exists(Hub::class) && ! empty(config('sentry.dsn'))) { |
| 340 | \Sentry\captureMessage($message, $sentrySeverity); |
| 341 | } |
| 342 | |
| 343 | // Slack path — only routed alerts go here (URGENTE only). |
| 344 | // |
| 345 | // Berta feedback (FIRE-1275): the engineering context (alert_id / |
| 346 | // sustained / flags) should live INSIDE the message body — not in a |
| 347 | // separate attachment fields block. Slack's "Show more" truncation |
| 348 | // only folds the trailing text of the body, so appending the context |
| 349 | // as body lines makes it collapse cleanly under Show more/less along |
| 350 | // with the Hora line, keeping the collapsed view business-clean. |
| 351 | if ($sendSlack && ! empty(config('logging.channels.slack.url'))) { |
| 352 | $slackMessage = $message; |
| 353 | if (! empty($context)) { |
| 354 | $slackMessage .= "\n\n_Detalle técnico:_ ".$this->formatSlackContext($context); |
| 355 | } |
| 356 | if ($isError) { |
| 357 | Log::channel('slack')->error($slackMessage); |
| 358 | } elseif ($isInfo) { |
| 359 | Log::channel('slack')->info($slackMessage); |
| 360 | } else { |
| 361 | Log::channel('slack')->warning($slackMessage); |
| 362 | } |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | /** |
| 367 | * Render the engineering context as a single inline string appended to |
| 368 | * the Slack message body (so it participates in Slack's Show more/less |
| 369 | * fold rather than sitting as an always-visible fields block below). |
| 370 | * Booleans render as 1/0 to match Laravel's slack-driver formatting. |
| 371 | */ |
| 372 | private function formatSlackContext(array $context): string |
| 373 | { |
| 374 | $parts = []; |
| 375 | foreach ($context as $k => $v) { |
| 376 | if (is_bool($v)) { |
| 377 | $v = $v ? '1' : '0'; |
| 378 | } |
| 379 | $parts[] = "`{$k}={$v}`"; |
| 380 | } |
| 381 | |
| 382 | return implode(' · ', $parts); |
| 383 | } |
| 384 | |
| 385 | private function humanLatency(int $ms): string |
| 386 | { |
| 387 | if ($ms >= 1000) { |
| 388 | return number_format($ms / 1000, 1, ',', '.').' s'; |
| 389 | } |
| 390 | |
| 391 | return $ms.' ms'; |
| 392 | } |
| 393 | |
| 394 | private function fmtPct(float $pct): string |
| 395 | { |
| 396 | return number_format($pct, 1, ',', '.'); |
| 397 | } |
| 398 | } |