271 lines
11 KiB
PowerShell
271 lines
11 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Read-only readiness probe for a Coolify service. Explains a 503
|
|
"no available server" instead of leaving you guessing.
|
|
|
|
.DESCRIPTION
|
|
Coolify's UI reports a service as green when its container is *running*.
|
|
Traefik, however, only puts a container in the load balancer once Docker
|
|
reports it *healthy*. On this host a first boot can take minutes (HDD-backed
|
|
loopback storage, ~39 ms/write), so a brand-new service is Running but not
|
|
yet healthy — Traefik has no route for it, the request falls through to
|
|
Coolify's catch-all router (priority -1000, service `noop`, empty server
|
|
list) and Traefik answers 503 "no available server".
|
|
|
|
This script reports both signals side by side so the gap is visible, and
|
|
tells you whether you should simply wait.
|
|
|
|
Read-only: it never restarts, redeploys or mutates anything.
|
|
|
|
.PARAMETER Uuid
|
|
Coolify resource UUID (the last path segment of the service URL in the UI).
|
|
Container names carry this as a suffix and change on every redeploy, so the
|
|
real name is resolved here rather than typed by hand.
|
|
|
|
.PARAMETER Fqdn
|
|
Public URL to probe. Defaults to whatever COOLIFY_FQDN the container carries.
|
|
|
|
.PARAMETER WaitSeconds
|
|
Poll until every container is healthy, up to this many seconds. Default 0
|
|
(report once and exit). Use 600 for a first boot on this host.
|
|
|
|
.EXAMPLE
|
|
.\coolify_skill\scripts\Test-CoolifyServiceReady.ps1 -Uuid znpmxv2o6ggooi6qxksiagke
|
|
|
|
.EXAMPLE
|
|
# First boot of a service from the Coolify library: wait it out.
|
|
.\coolify_skill\scripts\Test-CoolifyServiceReady.ps1 -Uuid znpmxv2o6ggooi6qxksiagke -WaitSeconds 600
|
|
#>
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$Uuid,
|
|
|
|
[string]$Fqdn,
|
|
|
|
[int]$WaitSeconds = 0
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..")
|
|
$invokeSsh = Join-Path $repoRoot "scripts\Invoke-ProxmoxSsh.ps1"
|
|
$agentScript = Join-Path $repoRoot "scripts\ProxmoxAgent.ps1"
|
|
|
|
foreach ($required in @($invokeSsh, $agentScript)) {
|
|
if (-not (Test-Path -LiteralPath $required)) {
|
|
throw "Missing dependency: $required"
|
|
}
|
|
}
|
|
|
|
. $agentScript
|
|
$config = Get-ProxmoxConfig
|
|
$lxc = $config.CoolifyLxc
|
|
|
|
# Nested quoting is corrupted by the SSH wrapper (TOOL-INDEX.md 1.2), so every
|
|
# non-trivial remote command is base64-encoded.
|
|
function Invoke-InLxc {
|
|
param([Parameter(Mandatory = $true)][string]$Script)
|
|
|
|
$bytes = [Text.Encoding]::UTF8.GetBytes($Script)
|
|
$b64 = [Convert]::ToBase64String($bytes)
|
|
return @(& $invokeSsh -Command "pct exec $lxc -- bash -c 'echo $b64 | base64 -d | bash'")
|
|
}
|
|
|
|
function Get-ServiceContainers {
|
|
param([string]$ResourceUuid)
|
|
|
|
# Container names carry the uuid as a suffix and change on every redeploy.
|
|
$lines = Invoke-InLxc -Script @"
|
|
docker ps -a --filter "label=coolify.resourceName" --format '{{.Names}}' 2>/dev/null | grep -- '$ResourceUuid' || true
|
|
docker ps -a --format '{{.Names}}' 2>/dev/null | grep -- '$ResourceUuid' || true
|
|
"@
|
|
return @($lines | Where-Object { $_ -and $_.Trim() } | ForEach-Object { $_.Trim() } | Sort-Object -Unique)
|
|
}
|
|
|
|
function Get-ContainerReport {
|
|
param([string]$Name)
|
|
|
|
$raw = Invoke-InLxc -Script @"
|
|
docker inspect '$Name' --format '{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}|{{.State.Health.FailingStreak}}{{else}}none|0{{end}}|{{.State.ExitCode}}|{{.State.StartedAt}}|{{.RestartCount}}|{{index .Config.Labels "coolify.serviceName"}}'
|
|
docker inspect '$Name' --format 'HC|{{if .Config.Healthcheck}}{{.Config.Healthcheck.Interval}}|{{.Config.Healthcheck.Retries}}|{{.Config.Healthcheck.StartPeriod}}{{else}}absent|0|0{{end}}'
|
|
echo "ENVFQDN|`$(docker inspect '$Name' --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | sed -n 's/^COOLIFY_FQDN=//p' | head -1)"
|
|
"@
|
|
|
|
$stateLine = @($raw | Where-Object { $_ -and $_ -notmatch '^(HC|ENVFQDN)\|' })[0]
|
|
$hcLine = @($raw | Where-Object { $_ -match '^HC\|' })[0]
|
|
$envLine = @($raw | Where-Object { $_ -match '^ENVFQDN\|' })[0]
|
|
|
|
if (-not $stateLine) { return $null }
|
|
$f = $stateLine.Split('|')
|
|
|
|
$interval = $null; $retries = $null; $startPeriod = $null
|
|
if ($hcLine) {
|
|
$h = $hcLine.Split('|')
|
|
$interval = $h[1]; $retries = $h[2]; $startPeriod = $h[3]
|
|
}
|
|
|
|
$containerFqdn = $null
|
|
if ($envLine) { $containerFqdn = $envLine.Split('|', 2)[1] }
|
|
|
|
# Docker prints healthcheck durations either as raw nanoseconds or as a Go
|
|
# duration string ("2s", "1m30s"), depending on the daemon version.
|
|
$toSeconds = {
|
|
param($value)
|
|
if (-not $value) { return $null }
|
|
if ($value -match '^\d+$') { return [math]::Round([double]$value / 1e9, 1) }
|
|
$total = 0.0; $matched = $false
|
|
foreach ($m in [regex]::Matches($value, '([\d.]+)(h|ms|m|s)')) {
|
|
$n = [double]$m.Groups[1].Value
|
|
switch ($m.Groups[2].Value) {
|
|
'h' { $total += $n * 3600 }
|
|
'm' { $total += $n * 60 }
|
|
's' { $total += $n }
|
|
'ms' { $total += $n / 1000 }
|
|
}
|
|
$matched = $true
|
|
}
|
|
if ($matched) { return [math]::Round($total, 1) }
|
|
return $null
|
|
}
|
|
|
|
return [pscustomobject]@{
|
|
Name = $Name
|
|
State = $f[0]
|
|
Health = $f[1]
|
|
FailingStreak = [int]$f[2]
|
|
ExitCode = $f[3]
|
|
StartedAt = $f[4]
|
|
RestartCount = $f[5]
|
|
ServiceName = $f[6]
|
|
HealthInterval = & $toSeconds $interval
|
|
HealthRetries = $retries
|
|
HealthStartPeriod = & $toSeconds $startPeriod
|
|
Fqdn = $containerFqdn
|
|
# A container that exited 0 and has no healthcheck is a one-shot init
|
|
# step (migrations, bucket creation). It is done, not broken.
|
|
IsOneShot = ($f[0] -eq 'exited') -and ($f[3] -eq '0') -and ($f[1] -eq 'none')
|
|
RoutableByTraefik = ($f[0] -eq 'running') -and ($f[1] -in @('healthy', 'none'))
|
|
}
|
|
}
|
|
|
|
function Get-StartupWork {
|
|
param([string]$Name)
|
|
|
|
# A container stuck in its entrypoint (apt/dpkg/chown) is starting, not broken.
|
|
$lines = Invoke-InLxc -Script @"
|
|
docker top '$Name' -o pid,stat,etime,cmd 2>/dev/null | tail -n +2 || true
|
|
"@
|
|
return @($lines | Where-Object { $_ -match '\b(chown|apt|apt-get|dpkg|unzip|tar|cp)\b' })
|
|
}
|
|
|
|
Write-Host ""
|
|
Write-Host "Coolify service readiness - $Uuid" -ForegroundColor Cyan
|
|
Write-Host ("-" * 72)
|
|
|
|
$containers = @(Get-ServiceContainers -ResourceUuid $Uuid)
|
|
if (-not $containers -or $containers.Count -eq 0) {
|
|
throw "No container found carrying uuid '$Uuid' in LXC $lxc. Has the service been deployed at all?"
|
|
}
|
|
|
|
$deadline = (Get-Date).AddSeconds($WaitSeconds)
|
|
$reports = @()
|
|
|
|
while ($true) {
|
|
$reports = @($containers | ForEach-Object { Get-ContainerReport -Name $_ } | Where-Object { $_ })
|
|
|
|
$notReady = @($reports | Where-Object { -not $_.RoutableByTraefik -and -not $_.IsOneShot })
|
|
if ($notReady.Count -eq 0 -or (Get-Date) -ge $deadline) { break }
|
|
|
|
$names = ($notReady | ForEach-Object { "$($_.Name)=$($_.Health)" }) -join ', '
|
|
$left = [int]($deadline - (Get-Date)).TotalSeconds
|
|
Write-Host " waiting ($left s left): $names" -ForegroundColor DarkGray
|
|
Start-Sleep -Seconds 10
|
|
}
|
|
|
|
$reports |
|
|
Select-Object Name, State, Health, FailingStreak,
|
|
@{ n = 'Routed'; e = { if ($_.IsOneShot) { 'n/a (one-shot)' } else { $_.RoutableByTraefik } } } |
|
|
Format-Table -AutoSize
|
|
|
|
# Healthcheck tuning is the amplifier that turns "slow boot" into "stuck 503".
|
|
# A short grace window is the amplifier that turns "slow boot" into "stuck 503":
|
|
# once flagged unhealthy, the container loses its Traefik route entirely.
|
|
$graceFloorSeconds = 180
|
|
foreach ($r in $reports) {
|
|
if ($r.Health -eq 'none' -or $r.HealthStartPeriod) { continue }
|
|
if (-not $r.HealthInterval -or -not $r.HealthRetries) { continue }
|
|
|
|
$grace = $r.HealthInterval * [int]$r.HealthRetries
|
|
if ($grace -ge $graceFloorSeconds) { continue }
|
|
|
|
Write-Host " ! $($r.Name): no start_period; flagged unhealthy after ~$grace s" -ForegroundColor Yellow
|
|
Write-Host " (interval=$($r.HealthInterval)s x retries=$($r.HealthRetries)). A first boot on this host" -ForegroundColor Yellow
|
|
Write-Host " can exceed that, and an unhealthy container has no Traefik route -> 503." -ForegroundColor Yellow
|
|
}
|
|
|
|
foreach ($r in $reports) {
|
|
if ($r.RoutableByTraefik -or $r.IsOneShot) { continue }
|
|
$work = @(Get-StartupWork -Name $r.Name)
|
|
if ($work.Count -gt 0) {
|
|
Write-Host " i $($r.Name) is still running setup work in its entrypoint:" -ForegroundColor DarkCyan
|
|
$work | ForEach-Object { Write-Host " $_" -ForegroundColor DarkCyan }
|
|
Write-Host " This is slow-but-progressing, not a failure. Wait, do not redeploy." -ForegroundColor DarkCyan
|
|
}
|
|
}
|
|
|
|
$target = $Fqdn
|
|
if (-not $target) {
|
|
$withFqdn = @($reports | Where-Object { $_.Fqdn })
|
|
if ($withFqdn.Count -gt 0) { $target = $withFqdn[0].Fqdn }
|
|
}
|
|
|
|
if ($target) {
|
|
if ($target -notmatch '^https?://') { $target = "https://$target" }
|
|
Write-Host ""
|
|
Write-Host "Probing $target" -ForegroundColor Cyan
|
|
|
|
$status = $null
|
|
$body = ''
|
|
try {
|
|
$resp = Invoke-WebRequest -Uri $target -UseBasicParsing -TimeoutSec 25
|
|
$status = [int]$resp.StatusCode
|
|
$body = [string]$resp.Content
|
|
}
|
|
catch {
|
|
if ($_.Exception.Response) {
|
|
$status = [int]$_.Exception.Response.StatusCode
|
|
try {
|
|
$reader = New-Object IO.StreamReader($_.Exception.Response.GetResponseStream())
|
|
$body = $reader.ReadToEnd()
|
|
}
|
|
catch { $body = '' }
|
|
}
|
|
else {
|
|
Write-Host " transport error: $($_.Exception.Message)" -ForegroundColor Red
|
|
}
|
|
}
|
|
|
|
if ($status) { Write-Host " HTTP $status" }
|
|
|
|
if ($status -eq 503 -and $body -match 'no available server') {
|
|
Write-Host ""
|
|
Write-Host " DIAGNOSIS: Traefik has no route for this host." -ForegroundColor Yellow
|
|
Write-Host " The request fell through to Coolify's catch-all router (priority -1000," -ForegroundColor Yellow
|
|
Write-Host " service 'noop', empty server list), which is what emits this exact string." -ForegroundColor Yellow
|
|
Write-Host " Traefik's docker provider only registers containers Docker reports healthy," -ForegroundColor Yellow
|
|
Write-Host " so an unhealthy/starting container has no route at all." -ForegroundColor Yellow
|
|
Write-Host " Note: a route that exists but whose backend refuses would return 502, not 503." -ForegroundColor Yellow
|
|
Write-Host " -> Re-run with -WaitSeconds 600 before changing any configuration." -ForegroundColor Yellow
|
|
}
|
|
elseif ($status -ge 200 -and $status -lt 400) {
|
|
Write-Host " Service is reachable and routed." -ForegroundColor Green
|
|
}
|
|
}
|
|
else {
|
|
Write-Host " (no FQDN found on the containers; pass -Fqdn to probe)" -ForegroundColor DarkGray
|
|
}
|
|
|
|
Write-Host ""
|
|
$reports
|