Latency spread by region

import { BoxPlot, summarize } from "@/components/ui/box-plot"
import { ChartCard } from "@/components/ui/chart-card"

import { latencySpread, SPREAD_SAMPLES } from "../data"

const ms = (value: number) => `${Math.round(value)}ms`

/**
 * A server component: the samples are drawn here and the box plot lays its
 * boxes out here too, so neither reaches the browser — the formatter included,
 * which a client chart could not have been handed.
 */
export function LatencySpread() {
  const spread = latencySpread()
  const slowest = spread
    .map((region) => ({ label: region.label, median: summarize(region.values)?.median ?? 0 }))
    .reduce((worst, region) => (region.median > worst.median ? region : worst), { label: "", median: -1 })

  return (
    <ChartCard
      data-widget="widget-engineering-monitoring-latency-spread"
      title="Latency spread by region"
      description={`A sample of ${SPREAD_SAMPLES} requests a region, in milliseconds`}
      height={176}
      className="h-full"
      footer={slowest.label ? `${slowest.label} runs slowest, at a median of ${ms(slowest.median)}.` : undefined}
    >
      <BoxPlot data={spread} label="Latency spread by region" valueFormatter={ms} />
    </ChartCard>
  )
}

/**
 * What this page reads. Services, incidents and the gateway log come straight
 * from `db`; latency and the error grid are counted out of the same log lines,
 * bucketed by hour. Two things no entity records are drawn: cluster load comes
 * off `seeded("dashboard-03")`, and each region's requests off a stream of
 * their own around a median that is a rule over the region's services. "Now"
 * is `REFERENCE_DATE`.
 */
import { getInitials } from "@/lib/format"
import { db, REFERENCE_DATE, seeded, type Incident, type Member, type Service } from "@/lib/sample-data"
import { type IncidentSeverity } from "@/components/ui/incident-list"
import { type ServiceStatus } from "@/components/ui/system-status"

const SERVICES = db.services.all()
const LOGS = db.logEntries.all()

// Five status words in the data, four on a status page: both kinds of outage
// read as "outage" there.
const STATUS: Record<Service["status"], ServiceStatus> = {
  operational: "operational",
  degraded: "degraded",
  partial_outage: "outage",
  major_outage: "outage",
  maintenance: "maintenance",
}

/** Every service, with the region it is served from. */
export function services() {
  return SERVICES.map((service) => ({
    name: service.name,
    status: STATUS[service.status],
    description: service.region,
  }))
}

/** How many regions the services run across, for the page's meta line. */
export function fleetSummary(): string {
  const regions = new Set(SERVICES.map((service) => service.region))
  return `${SERVICES.length} services across ${regions.size} regions`
}

const DAY_MS = 86_400_000

export type UptimeStatus = "up" | "degraded" | "down"

/** A day is up above 99.5%, degraded down to 97%, and down below that. */
function dayStatus(uptime: number): { status: UptimeStatus; label: string } {
  if (uptime >= 99.5) return { status: "up", label: "No incidents" }
  if (uptime >= 97) return { status: "degraded", label: "Degraded for part of the day" }
  return { status: "down", label: "Down" }
}

const UPTIME_COUNT = 3

/** Ninety days of daily uptime for the services that had the worst of it. */
export function uptime() {
  const average = (service: Service) =>
    service.uptime90d.reduce((sum, day) => sum + day, 0) / service.uptime90d.length

  return [...SERVICES]
    .sort((a, b) => average(a) - average(b))
    .slice(0, UPTIME_COUNT)
    .map((service) => {
      const start = REFERENCE_DATE.getTime() - (service.uptime90d.length - 1) * DAY_MS
      return {
        label: service.name,
        // Stated rather than left to UptimeBar's own count: the strip marks
        // whole days, and this is the minutes the probes actually missed.
        uptime: Math.round(average(service) * 100) / 100,
        days: service.uptime90d.map((value, index) => ({
          date: new Date(start + index * DAY_MS).toISOString().slice(0, 10),
          ...dayStatus(value),
        })),
      }
    })
}

/** The `p`th percentile of `values`, nearest-rank; 0 for an empty list. */
function percentile(values: number[], p: number): number {
  if (values.length === 0) return 0
  const sorted = [...values].sort((a, b) => a - b)
  return sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))]
}

const HOURS = Array.from({ length: 24 }, (_, hour) => hour)

// Only a line that carries a request id was timed, so those are the ones the
// percentiles are taken over.
const TIMED = LOGS.filter((line) => line.durationMs !== undefined)

export type LatencyPoint = { hour: string; p50: number; p95: number; p99: number }

/** Request latency by hour of day, over every timed line in the log. */
export function latency(): LatencyPoint[] {
  return HOURS.map((hour) => {
    const durations = TIMED.filter((line) => line.at.getUTCHours() === hour).map(
      (line) => line.durationMs as number
    )
    return {
      hour: `${String(hour).padStart(2, "0")}:00`,
      p50: percentile(durations, 0.5),
      p95: percentile(durations, 0.95),
      p99: percentile(durations, 0.99),
    }
  })
}

/** How many days of log the percentiles and the grid below are taken over. */
export function logWindowDays(): number {
  const oldest = Math.min(...LOGS.map((line) => line.at.getTime()))
  return Math.round((REFERENCE_DATE.getTime() - oldest) / DAY_MS)
}

/** The band p95 stayed inside, for the chart's footer. */
export function p95Band(): { low: number; high: number } {
  const values = latency().map((point) => point.p95)
  return { low: Math.min(...values), high: Math.max(...values) }
}

export const ERROR_HOURS = HOURS.map((hour) => String(hour).padStart(2, "0"))
export const ERROR_DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]

/** Errors the log recorded, by weekday and hour. */
export function errorsByHour(): number[][] {
  const grid = ERROR_DAYS.map(() => HOURS.map(() => 0))
  for (const line of LOGS) {
    if (line.level !== "error") continue
    // getUTCDay is Sunday-first; the grid reads Monday-first.
    grid[(line.at.getUTCDay() + 6) % 7][line.at.getUTCHours()] += 1
  }
  return grid
}

/** The worst block in the grid, named the way the footer says it. */
export function worstErrorBlock(): { label: string; count: number } {
  const grid = errorsByHour()
  let best = { label: `${ERROR_DAYS[0]} 00:00`, count: -1 }
  grid.forEach((row, day) =>
    row.forEach((count, hour) => {
      if (count > best.count) best = { label: `${ERROR_DAYS[day]} ${ERROR_HOURS[hour]}:00`, count }
    })
  )
  return best
}

export type ErrorGrid = {
  /** The rows, Monday first. */
  days: string[]
  /** The columns, one an hour of the UTC day. */
  hours: string[]
  /** Errors a cell, `values[day][hour]`. */
  values: number[][]
  /** How many days of log the grid is counted over. */
  windowDays: number
  worst: { label: string; count: number }
}

/** The error grid with what its caption and its footer say, counted here for the heatmap to draw. */
export function errorGrid(): ErrorGrid {
  return {
    days: ERROR_DAYS,
    hours: ERROR_HOURS,
    values: errorsByHour(),
    windowDays: logWindowDays(),
    worst: worstErrorBlock(),
  }
}

/** Cluster load right now. No entity records it, so it comes off the block's seed. */
export function resources() {
  const rand = seeded("dashboard-03")
  return [
    { label: "CPU", value: Math.round(52 + rand() * 18), thresholds: { warning: 70, danger: 90 } },
    { label: "Memory", value: Math.round(70 + rand() * 14), thresholds: { warning: 75, danger: 92 } },
    { label: "Disk", value: Math.round(34 + rand() * 14), thresholds: { warning: 80, danger: 92 } },
  ]
}

/**
 * A region's latency as `BoxPlot` reads it: the label it is drawn under and
 * its samples in milliseconds. Written out rather than imported from the
 * plot: every card of the page installs this file, and an import of the
 * plot's types would install the plot with each.
 */
export type LatencySample = { label: string; values: number[] }

/** How many requests each region's box is drawn from. */
export const SPREAD_SAMPLES = 240

/**
 * How far up the fleet's band — from its p50 towards the lowest its p95 went
 * — a region's median sits for each service in it that is not operational.
 * A region slides up the band with its trouble, never past 0.6 of it, so a
 * median drawn from the samples stays inside the band.
 */
const SLOWDOWN: Record<Service["status"], number> = {
  operational: 0,
  maintenance: 0.15,
  degraded: 0.25,
  partial_outage: 0.4,
  major_outage: 0.6,
}
const MOST_SLOWDOWN = 0.6

// A log-normal's 95th percentile sits 1.645 standard deviations above its median.
const Z95 = 1.645

/** A standard normal draw, from two of the stream's uniforms (Box–Muller). */
function normal(rand: () => number): number {
  return Math.sqrt(-2 * Math.log(1 - rand())) * Math.cos(2 * Math.PI * rand())
}

/**
 * Each region's latency, in the region list's order. A log line names the
 * worker that wrote it, not the region it served, so no row says which
 * region a request was slow in: each region's requests are drawn from a
 * stream of their own (`engineering-monitoring:latency-spread:<region>`),
 * log-normal like the fleet's own — the shape that runs through its p50 and
 * its p95 — so the slow tail is long. The median is a rule over the
 * region's services: the fleet's p50, moved up its band for each service in
 * the region that is not operational.
 */
export function latencySpread(): LatencySample[] {
  const durations = TIMED.map((line) => line.durationMs as number)
  const p50 = percentile(durations, 0.5)
  const p95 = percentile(durations, 0.95)
  const shape = p50 > 0 && p95 > p50 ? Math.log(p95 / p50) / Z95 : 0
  const band = Math.max(0, p95Band().low - p50)

  return regions().map((region) => {
    const trouble = SERVICES.filter((service) => service.region === region.name).reduce(
      (sum, service) => sum + SLOWDOWN[service.status],
      0
    )
    const median = p50 + band * Math.min(trouble, MOST_SLOWDOWN)
    const rand = seeded(`engineering-monitoring:latency-spread:${region.name}`)
    return {
      label: region.name,
      values: Array.from({ length: SPREAD_SAMPLES }, () => Math.max(1, Math.round(median * Math.exp(shape * normal(rand))))),
    }
  })
}

// StatusIndicator's vocabulary: "online" is healthy, "warning" is anything
// degraded or worse.
type RegionStatus = "online" | "warning"

/** Where traffic is served from, worst region first. */
export function regions() {
  const names = [...new Set(SERVICES.map((service) => service.region))].sort()

  return names
    .map((name) => {
      const here = SERVICES.filter((service) => service.region === name)
      const healthy = here.every((service) => service.status === "operational")
      const worst = Math.min(
        ...here.map((service) => service.uptime90d[service.uptime90d.length - 1])
      )
      return {
        name,
        status: (healthy ? "online" : "warning") as RegionStatus,
        note: `${here.length} ${here.length === 1 ? "service" : "services"} · ${worst.toFixed(2)}% today`,
      }
    })
    .sort((a, b) => Number(a.status === "online") - Number(b.status === "online"))
}

const SEVERITY: Record<Incident["severity"], IncidentSeverity> = {
  sev1: "critical",
  sev2: "major",
  sev3: "minor",
}

const SERVICE_NAME_BY_ID = new Map(SERVICES.map((service) => [service.id, service.name]))
const INCIDENT_COUNT = 6

/** Open incidents first, then the ones that closed most recently. */
export function incidents() {
  const open = (incident: Incident) => incident.status !== "resolved"

  return [...db.incidents.all()]
    .sort((a, b) => {
      if (open(a) !== open(b)) return open(a) ? -1 : 1
      return b.startedAt.getTime() - a.startedAt.getTime()
    })
    .slice(0, INCIDENT_COUNT)
    .map((incident) => ({
      id: incident.id,
      title: incident.title,
      severity: SEVERITY[incident.severity],
      status: incident.status,
      startedAt: incident.startedAt,
      resolvedAt: incident.resolvedAt,
      affected: incident.serviceIds.map((id) => SERVICE_NAME_BY_ID.get(id) ?? id),
    }))
}

const LOG_LINES = 60

/** The tail of the gateway log, oldest of the tail first. */
export function logLines() {
  return [...LOGS]
    .sort((a, b) => a.at.getTime() - b.at.getTime())
    .slice(-LOG_LINES)
    .map((line) => ({
      id: line.id,
      timestamp: line.at,
      level: line.level,
      source: line.service,
      message: line.requestId ? `${line.message} request=${line.requestId}` : line.message,
    }))
}

function ownerRow(): Member {
  return db.members.all().find((member) => member.role === "owner") ?? db.members.all()[0]
}

/** The person looking at the page: whoever owns this workspace. */
export function currentUser() {
  const owner = ownerRow()
  return { name: owner.name, email: owner.email, initials: getInitials(owner.name), avatarUrl: owner.avatarUrl }
}

/** The bell's contents: the newest notifications, unread first in the panel. */
export function shellNotifications() {
  return db.notifications
    .all()
    .sort((a, b) => b.at.getTime() - a.at.getTime())
    .slice(0, 6)
    .map(({ id, title, description, at, read, href }) => ({ id, title, description, at, read, href }))
}