<#
.SYNOPSIS
    Read-only readiness probe for a Coolify service. Explains a 503
    "no available server" instead of leaving you guessing.

.DESCRIPTION
    Coolify's UI reports a service as green when its container is *running*.
    Traefik, however, only puts a container in the load balancer once Docker
    reports it *healthy*. On this host a first boot can take minutes (HDD-backed
    loopback storage, ~39 ms/write), so a brand-new service is Running but not
    yet healthy — Traefik has no route for it, the request falls through to
    Coolify's catch-all router (priority -1000, service `noop`, empty server
    list) and Traefik answers 503 "no available server".

    This script reports both signals side by side so the gap is visible, and
    tells you whether you should simply wait.

    Read-only: it never restarts, redeploys or mutates anything.

.PARAMETER Uuid
    Coolify resource UUID (the last path segment of the service URL in the UI).
    Container names carry this as a suffix and change on every redeploy, so the
    real name is resolved here rather than typed by hand.

.PARAMETER Fqdn
    Public URL to probe. Defaults to whatever COOLIFY_FQDN the container carries.

.PARAMETER WaitSeconds
    Poll until every container is healthy, up to this many seconds. Default 0
    (report once and exit). Use 600 for a first boot on this host.

.EXAMPLE
    .\coolify_skill\scripts\Test-CoolifyServiceReady.ps1 -Uuid znpmxv2o6ggooi6qxksiagke

.EXAMPLE
    # First boot of a service from the Coolify library: wait it out.
    .\coolify_skill\scripts\Test-CoolifyServiceReady.ps1 -Uuid znpmxv2o6ggooi6qxksiagke -WaitSeconds 600
#>
[CmdletBinding()]
param(
    [Parameter(Mandatory = $true)]
    [string]$Uuid,

    [string]$Fqdn,

    [int]$WaitSeconds = 0
)

$ErrorActionPreference = "Stop"

$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..")
$invokeSsh = Join-Path $repoRoot "scripts\Invoke-ProxmoxSsh.ps1"
$agentScript = Join-Path $repoRoot "scripts\ProxmoxAgent.ps1"

foreach ($required in @($invokeSsh, $agentScript)) {
    if (-not (Test-Path -LiteralPath $required)) {
        throw "Missing dependency: $required"
    }
}

. $agentScript
$config = Get-ProxmoxConfig
$lxc = $config.CoolifyLxc

# Nested quoting is corrupted by the SSH wrapper (TOOL-INDEX.md 1.2), so every
# non-trivial remote command is base64-encoded.
function Invoke-InLxc {
    param([Parameter(Mandatory = $true)][string]$Script)

    $bytes = [Text.Encoding]::UTF8.GetBytes($Script)
    $b64 = [Convert]::ToBase64String($bytes)
    return @(& $invokeSsh -Command "pct exec $lxc -- bash -c 'echo $b64 | base64 -d | bash'")
}

function Get-ServiceContainers {
    param([string]$ResourceUuid)

    # Container names carry the uuid as a suffix and change on every redeploy.
    $lines = Invoke-InLxc -Script @"
docker ps -a --filter "label=coolify.resourceName" --format '{{.Names}}' 2>/dev/null | grep -- '$ResourceUuid' || true
docker ps -a --format '{{.Names}}' 2>/dev/null | grep -- '$ResourceUuid' || true
"@
    return @($lines | Where-Object { $_ -and $_.Trim() } | ForEach-Object { $_.Trim() } | Sort-Object -Unique)
}

function Get-ContainerReport {
    param([string]$Name)

    $raw = Invoke-InLxc -Script @"
docker inspect '$Name' --format '{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}|{{.State.Health.FailingStreak}}{{else}}none|0{{end}}|{{.State.ExitCode}}|{{.State.StartedAt}}|{{.RestartCount}}|{{index .Config.Labels "coolify.serviceName"}}'
docker inspect '$Name' --format 'HC|{{if .Config.Healthcheck}}{{.Config.Healthcheck.Interval}}|{{.Config.Healthcheck.Retries}}|{{.Config.Healthcheck.StartPeriod}}{{else}}absent|0|0{{end}}'
echo "ENVFQDN|`$(docker inspect '$Name' --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | sed -n 's/^COOLIFY_FQDN=//p' | head -1)"
"@

    $stateLine = @($raw | Where-Object { $_ -and $_ -notmatch '^(HC|ENVFQDN)\|' })[0]
    $hcLine = @($raw | Where-Object { $_ -match '^HC\|' })[0]
    $envLine = @($raw | Where-Object { $_ -match '^ENVFQDN\|' })[0]

    if (-not $stateLine) { return $null }
    $f = $stateLine.Split('|')

    $interval = $null; $retries = $null; $startPeriod = $null
    if ($hcLine) {
        $h = $hcLine.Split('|')
        $interval = $h[1]; $retries = $h[2]; $startPeriod = $h[3]
    }

    $containerFqdn = $null
    if ($envLine) { $containerFqdn = $envLine.Split('|', 2)[1] }

    # Docker prints healthcheck durations either as raw nanoseconds or as a Go
    # duration string ("2s", "1m30s"), depending on the daemon version.
    $toSeconds = {
        param($value)
        if (-not $value) { return $null }
        if ($value -match '^\d+$') { return [math]::Round([double]$value / 1e9, 1) }
        $total = 0.0; $matched = $false
        foreach ($m in [regex]::Matches($value, '([\d.]+)(h|ms|m|s)')) {
            $n = [double]$m.Groups[1].Value
            switch ($m.Groups[2].Value) {
                'h'  { $total += $n * 3600 }
                'm'  { $total += $n * 60 }
                's'  { $total += $n }
                'ms' { $total += $n / 1000 }
            }
            $matched = $true
        }
        if ($matched) { return [math]::Round($total, 1) }
        return $null
    }

    return [pscustomobject]@{
        Name              = $Name
        State             = $f[0]
        Health            = $f[1]
        FailingStreak     = [int]$f[2]
        ExitCode          = $f[3]
        StartedAt         = $f[4]
        RestartCount      = $f[5]
        ServiceName       = $f[6]
        HealthInterval    = & $toSeconds $interval
        HealthRetries     = $retries
        HealthStartPeriod = & $toSeconds $startPeriod
        Fqdn              = $containerFqdn
        # A container that exited 0 and has no healthcheck is a one-shot init
        # step (migrations, bucket creation). It is done, not broken.
        IsOneShot         = ($f[0] -eq 'exited') -and ($f[3] -eq '0') -and ($f[1] -eq 'none')
        RoutableByTraefik = ($f[0] -eq 'running') -and ($f[1] -in @('healthy', 'none'))
    }
}

function Get-StartupWork {
    param([string]$Name)

    # A container stuck in its entrypoint (apt/dpkg/chown) is starting, not broken.
    $lines = Invoke-InLxc -Script @"
docker top '$Name' -o pid,stat,etime,cmd 2>/dev/null | tail -n +2 || true
"@
    return @($lines | Where-Object { $_ -match '\b(chown|apt|apt-get|dpkg|unzip|tar|cp)\b' })
}

Write-Host ""
Write-Host "Coolify service readiness - $Uuid" -ForegroundColor Cyan
Write-Host ("-" * 72)

$containers = @(Get-ServiceContainers -ResourceUuid $Uuid)
if (-not $containers -or $containers.Count -eq 0) {
    throw "No container found carrying uuid '$Uuid' in LXC $lxc. Has the service been deployed at all?"
}

$deadline = (Get-Date).AddSeconds($WaitSeconds)
$reports = @()

while ($true) {
    $reports = @($containers | ForEach-Object { Get-ContainerReport -Name $_ } | Where-Object { $_ })

    $notReady = @($reports | Where-Object { -not $_.RoutableByTraefik -and -not $_.IsOneShot })
    if ($notReady.Count -eq 0 -or (Get-Date) -ge $deadline) { break }

    $names = ($notReady | ForEach-Object { "$($_.Name)=$($_.Health)" }) -join ', '
    $left = [int]($deadline - (Get-Date)).TotalSeconds
    Write-Host "  waiting ($left s left): $names" -ForegroundColor DarkGray
    Start-Sleep -Seconds 10
}

$reports |
    Select-Object Name, State, Health, FailingStreak,
        @{ n = 'Routed'; e = { if ($_.IsOneShot) { 'n/a (one-shot)' } else { $_.RoutableByTraefik } } } |
    Format-Table -AutoSize

# Healthcheck tuning is the amplifier that turns "slow boot" into "stuck 503".
# A short grace window is the amplifier that turns "slow boot" into "stuck 503":
# once flagged unhealthy, the container loses its Traefik route entirely.
$graceFloorSeconds = 180
foreach ($r in $reports) {
    if ($r.Health -eq 'none' -or $r.HealthStartPeriod) { continue }
    if (-not $r.HealthInterval -or -not $r.HealthRetries) { continue }

    $grace = $r.HealthInterval * [int]$r.HealthRetries
    if ($grace -ge $graceFloorSeconds) { continue }

    Write-Host "  ! $($r.Name): no start_period; flagged unhealthy after ~$grace s" -ForegroundColor Yellow
    Write-Host "    (interval=$($r.HealthInterval)s x retries=$($r.HealthRetries)). A first boot on this host" -ForegroundColor Yellow
    Write-Host "    can exceed that, and an unhealthy container has no Traefik route -> 503." -ForegroundColor Yellow
}

foreach ($r in $reports) {
    if ($r.RoutableByTraefik -or $r.IsOneShot) { continue }
    $work = @(Get-StartupWork -Name $r.Name)
    if ($work.Count -gt 0) {
        Write-Host "  i $($r.Name) is still running setup work in its entrypoint:" -ForegroundColor DarkCyan
        $work | ForEach-Object { Write-Host "      $_" -ForegroundColor DarkCyan }
        Write-Host "    This is slow-but-progressing, not a failure. Wait, do not redeploy." -ForegroundColor DarkCyan
    }
}

$target = $Fqdn
if (-not $target) {
    $withFqdn = @($reports | Where-Object { $_.Fqdn })
    if ($withFqdn.Count -gt 0) { $target = $withFqdn[0].Fqdn }
}

if ($target) {
    if ($target -notmatch '^https?://') { $target = "https://$target" }
    Write-Host ""
    Write-Host "Probing $target" -ForegroundColor Cyan

    $status = $null
    $body = ''
    try {
        $resp = Invoke-WebRequest -Uri $target -UseBasicParsing -TimeoutSec 25
        $status = [int]$resp.StatusCode
        $body = [string]$resp.Content
    }
    catch {
        if ($_.Exception.Response) {
            $status = [int]$_.Exception.Response.StatusCode
            try {
                $reader = New-Object IO.StreamReader($_.Exception.Response.GetResponseStream())
                $body = $reader.ReadToEnd()
            }
            catch { $body = '' }
        }
        else {
            Write-Host "  transport error: $($_.Exception.Message)" -ForegroundColor Red
        }
    }

    if ($status) { Write-Host "  HTTP $status" }

    if ($status -eq 503 -and $body -match 'no available server') {
        Write-Host ""
        Write-Host "  DIAGNOSIS: Traefik has no route for this host." -ForegroundColor Yellow
        Write-Host "  The request fell through to Coolify's catch-all router (priority -1000," -ForegroundColor Yellow
        Write-Host "  service 'noop', empty server list), which is what emits this exact string." -ForegroundColor Yellow
        Write-Host "  Traefik's docker provider only registers containers Docker reports healthy," -ForegroundColor Yellow
        Write-Host "  so an unhealthy/starting container has no route at all." -ForegroundColor Yellow
        Write-Host "  Note: a route that exists but whose backend refuses would return 502, not 503." -ForegroundColor Yellow
        Write-Host "  -> Re-run with -WaitSeconds 600 before changing any configuration." -ForegroundColor Yellow
    }
    elseif ($status -ge 200 -and $status -lt 400) {
        Write-Host "  Service is reachable and routed." -ForegroundColor Green
    }
}
else {
    Write-Host "  (no FQDN found on the containers; pass -Fqdn to probe)" -ForegroundColor DarkGray
}

Write-Host ""
$reports