Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
52.58% |
51 / 97 |
|
20.00% |
1 / 5 |
CRAP | |
0.00% |
0 / 1 |
| IfexHealthCheck | |
52.58% |
51 / 97 |
|
20.00% |
1 / 5 |
68.03 | |
0.00% |
0 / 1 |
| handle | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
1 | |||
| checkFullStall | |
76.74% |
33 / 43 |
|
0.00% |
0 / 1 |
7.62 | |||
| checkPartialStall | |
35.00% |
14 / 40 |
|
0.00% |
0 / 1 |
20.46 | |||
| emitAlert | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
20 | |||
| formatSlackContext | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
6 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace App\Console\Commands; |
| 4 | |
| 5 | use App\Models\TblIfexLastUpdate; |
| 6 | use Carbon\Carbon; |
| 7 | use Illuminate\Console\Command; |
| 8 | use Illuminate\Support\Facades\Cache; |
| 9 | use Illuminate\Support\Facades\DB; |
| 10 | use Illuminate\Support\Facades\Log; |
| 11 | use Sentry\Severity; |
| 12 | use Sentry\State\Hub; |
| 13 | |
| 14 | /** |
| 15 | * FIRE-1673 stall detector for the IFEX sync. |
| 16 | * |
| 17 | * The DoD asks for two independent failure modes to be surfaced within 24h: |
| 18 | * |
| 19 | * H1 · full stall |
| 20 | * `tbl_ifex_last_update.last_date` is older than `services.ifex.stall_alert_hours`. |
| 21 | * This is what would have caught the 09-21 June 2026 outage — the |
| 22 | * checkpoint stopped advancing while the cron kept "succeeding" on |
| 23 | * empty windows. |
| 24 | * |
| 25 | * H2 · partial stall |
| 26 | * One of series C / O / P has had NO new row created in the last |
| 27 | * three days while at least one of the others did. This is the |
| 28 | * "one series disappearing while the others keep flowing" case |
| 29 | * Berta specifically called out — a plain last_date check misses it |
| 30 | * because the checkpoint keeps moving on the healthy series. |
| 31 | * |
| 32 | * Alerts follow the CheckPerformanceThresholds pattern: |
| 33 | * Sentry ERROR (paged) + Slack (@here) for both. IFEX is the source of |
| 34 | * ~half the quotation traffic; a silent stall is a real business incident. |
| 35 | * |
| 36 | * Alerts are deduped for one hour via `Cache::add(..., 3600)` — same knob as |
| 37 | * the observability alerts, keyed per-alert so H1 firing doesn't suppress H2. |
| 38 | */ |
| 39 | class IfexHealthCheck extends Command |
| 40 | { |
| 41 | protected $signature = 'ifex:health-check |
| 42 | {--dry-run : Report the current state without firing an alert or writing to the dedup cache}'; |
| 43 | |
| 44 | protected $description = 'FIRE-1673: alert on IFEX sync stall (full or per-series).'; |
| 45 | |
| 46 | private const DEDUP_TTL_SECONDS = 3600; |
| 47 | |
| 48 | private const PARTIAL_STALL_DAYS = 3; |
| 49 | |
| 50 | /** IFEX series that must all be flowing. G is deliberately excluded (FIRE-1671). */ |
| 51 | private const SERIES = ['C', 'O', 'P']; |
| 52 | |
| 53 | public function handle(): int |
| 54 | { |
| 55 | $dryRun = (bool) $this->option('dry-run'); |
| 56 | |
| 57 | $this->checkFullStall($dryRun); |
| 58 | $this->checkPartialStall($dryRun); |
| 59 | |
| 60 | return self::SUCCESS; |
| 61 | } |
| 62 | |
| 63 | /** |
| 64 | * H1: last_date older than the configured threshold. |
| 65 | */ |
| 66 | private function checkFullStall(bool $dryRun): void |
| 67 | { |
| 68 | $thresholdHours = max((int) config('services.ifex.stall_alert_hours', 24), 1); |
| 69 | |
| 70 | $lastDate = TblIfexLastUpdate::where('id', 1)->value('last_date'); |
| 71 | if (empty($lastDate)) { |
| 72 | // Fresh install / never run — a separate deploy problem, not a |
| 73 | // stall. Reporting it as a stall would page on every new env. |
| 74 | $this->line('H1 skipped: last_date is empty (fresh install?).'); |
| 75 | |
| 76 | return; |
| 77 | } |
| 78 | |
| 79 | try { |
| 80 | $lastCovered = Carbon::parse($lastDate); |
| 81 | } catch (\Throwable) { |
| 82 | $this->warn("H1 skipped: last_date '{$lastDate}' is unparseable."); |
| 83 | |
| 84 | return; |
| 85 | } |
| 86 | |
| 87 | $hoursBehind = Carbon::now()->diffInHours($lastCovered, false) * -1; |
| 88 | $this->line(sprintf( |
| 89 | 'H1: last_date = %s (%.1f h ago, threshold %d h)', |
| 90 | $lastCovered->format('Y-m-d'), |
| 91 | $hoursBehind, |
| 92 | $thresholdHours, |
| 93 | )); |
| 94 | |
| 95 | if ($hoursBehind < $thresholdHours) { |
| 96 | return; |
| 97 | } |
| 98 | |
| 99 | if (! $dryRun && ! Cache::add('ifex:alert:h1:fired', 1, self::DEDUP_TTL_SECONDS)) { |
| 100 | $this->line('H1 breach, but a recent alert is still in the dedup window.'); |
| 101 | |
| 102 | return; |
| 103 | } |
| 104 | |
| 105 | $message = sprintf( |
| 106 | "🚨 *TITAN · IFEX sync detenida* <!here>\n" |
| 107 | .'La sincronización no ha avanzado en las últimas %d h. ' |
| 108 | .'Riesgo directo: presupuestos IFEX no llegan al informe.'."\n" |
| 109 | .'*Estado:*'."\n" |
| 110 | .'• Último día cubierto: %s'."\n" |
| 111 | .'• Hace: %.1f h'."\n" |
| 112 | .'• Umbral: %d h'."\n" |
| 113 | .'• Acción: revisar el cron `ifex-quotations:sync` y el canal de log `ifex`.', |
| 114 | (int) round($hoursBehind), |
| 115 | $lastCovered->format('Y-m-d'), |
| 116 | $hoursBehind, |
| 117 | $thresholdHours, |
| 118 | ); |
| 119 | |
| 120 | if ($dryRun) { |
| 121 | $this->warn("[dry-run] would fire H1:\n{$message}"); |
| 122 | |
| 123 | return; |
| 124 | } |
| 125 | |
| 126 | $this->emitAlert($message, [ |
| 127 | 'alert_id' => 'ifex-h1', |
| 128 | 'hours_behind' => (int) round($hoursBehind), |
| 129 | 'last_covered' => $lastCovered->format('Y-m-d'), |
| 130 | ]); |
| 131 | } |
| 132 | |
| 133 | /** |
| 134 | * H2: one series has no new rows in the last N days while the others do. |
| 135 | * |
| 136 | * Reads `created_at` because that's the sync-insert timestamp (FIRE-1258 |
| 137 | * cleanup made it a real datetime rather than the document date). A series |
| 138 | * that produces documents but where the sync stopped pulling them will |
| 139 | * still show a stale max(created_at) even if the checkpoint is fresh. |
| 140 | */ |
| 141 | private function checkPartialStall(bool $dryRun): void |
| 142 | { |
| 143 | $since = Carbon::now()->subDays(self::PARTIAL_STALL_DAYS)->format('Y-m-d H:i:s'); |
| 144 | |
| 145 | $activity = []; |
| 146 | foreach (self::SERIES as $series) { |
| 147 | $activity[$series] = (int) DB::table('tbl_quotations') |
| 148 | ->where('commercial', 'Ifex') |
| 149 | ->where('internal_quote_id', 'LIKE', "202626{$series}%") |
| 150 | ->where('created_at', '>=', $since) |
| 151 | ->count(); |
| 152 | } |
| 153 | |
| 154 | $this->line('H2: rows created in the last '.self::PARTIAL_STALL_DAYS.' days per series → ' |
| 155 | .json_encode($activity)); |
| 156 | |
| 157 | $silentSeries = array_keys(array_filter($activity, static fn (int $n): bool => $n === 0)); |
| 158 | $activeSeries = array_diff(array_keys($activity), $silentSeries); |
| 159 | |
| 160 | // Fire only when at least one series is silent AND at least one is |
| 161 | // active. If ALL are silent that's a full stall — H1 handles it and |
| 162 | // paging twice is noise. If none are silent, obviously nothing to do. |
| 163 | if (empty($silentSeries) || empty($activeSeries)) { |
| 164 | return; |
| 165 | } |
| 166 | |
| 167 | // Dedup key includes the silent-series list so a NEW series going |
| 168 | // silent re-fires even inside the 1h window. |
| 169 | $dedupKey = 'ifex:alert:h2:'.implode('-', $silentSeries); |
| 170 | |
| 171 | if (! $dryRun && ! Cache::add($dedupKey, 1, self::DEDUP_TTL_SECONDS)) { |
| 172 | $this->line('H2 breach on '.implode(',', $silentSeries).', still inside dedup window.'); |
| 173 | |
| 174 | return; |
| 175 | } |
| 176 | |
| 177 | $activityLine = implode(' · ', array_map( |
| 178 | static fn (string $s): string => "{$s}: {$activity[$s]}", |
| 179 | array_keys($activity), |
| 180 | )); |
| 181 | |
| 182 | $message = sprintf( |
| 183 | "🚨 *TITAN · IFEX: una serie ha dejado de sincronizarse* <!here>\n" |
| 184 | .'Serie(s) sin nuevas filas en los últimos %d días: `%s`. ' |
| 185 | ."Las demás siguen flujo, así que el checkpoint no lo detecta solo.\n" |
| 186 | .'*Actividad últimos %d días:* %s', |
| 187 | self::PARTIAL_STALL_DAYS, |
| 188 | implode(', ', $silentSeries), |
| 189 | self::PARTIAL_STALL_DAYS, |
| 190 | $activityLine, |
| 191 | ); |
| 192 | |
| 193 | if ($dryRun) { |
| 194 | $this->warn("[dry-run] would fire H2:\n{$message}"); |
| 195 | |
| 196 | return; |
| 197 | } |
| 198 | |
| 199 | $this->emitAlert($message, [ |
| 200 | 'alert_id' => 'ifex-h2', |
| 201 | 'silent_series' => implode(',', $silentSeries), |
| 202 | 'active_series' => implode(',', $activeSeries), |
| 203 | ]); |
| 204 | } |
| 205 | |
| 206 | /** |
| 207 | * Route to Log::error + Sentry ERROR + Slack, matching |
| 208 | * CheckPerformanceThresholds::emitAlert for URGENTE alerts. |
| 209 | */ |
| 210 | private function emitAlert(string $message, array $context): void |
| 211 | { |
| 212 | Log::error($message, $context); |
| 213 | |
| 214 | if (class_exists(Hub::class) && ! empty(config('sentry.dsn'))) { |
| 215 | \Sentry\captureMessage($message, Severity::error()); |
| 216 | } |
| 217 | |
| 218 | if (! empty(config('logging.channels.slack.url'))) { |
| 219 | $slackMessage = $message."\n\n_Detalle técnico:_ ".$this->formatSlackContext($context); |
| 220 | Log::channel('slack')->error($slackMessage); |
| 221 | } |
| 222 | } |
| 223 | |
| 224 | private function formatSlackContext(array $context): string |
| 225 | { |
| 226 | $parts = []; |
| 227 | foreach ($context as $k => $v) { |
| 228 | $parts[] = "`{$k}={$v}`"; |
| 229 | } |
| 230 | |
| 231 | return implode(' · ', $parts); |
| 232 | } |
| 233 | } |