Actualiza toolkit operativo y documentación
This commit is contained in:
urieljareth
@@ -47,8 +47,12 @@ if ($PSBoundParameters.ContainsKey("BodyJson")) {
|
||||
throw "BodyJson is not valid JSON: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
$request.Body = $BodyJson
|
||||
$request.ContentType = "application/json"
|
||||
# Send bytes, not a string. PowerShell 5.1 encodes a string body using the
|
||||
# default codepage, so any non-ASCII character (an accented comment inside a
|
||||
# compose file, for instance) reaches Coolify mangled and it answers
|
||||
# 400 {"message":"Invalid request.","error":"Invalid JSON."}.
|
||||
$request.Body = [Text.Encoding]::UTF8.GetBytes($BodyJson)
|
||||
$request.ContentType = "application/json; charset=utf-8"
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Give a Coolify service's healthchecks a start_period long enough for a first
|
||||
boot on this host. Dry-run by default.
|
||||
|
||||
.DESCRIPTION
|
||||
Coolify's library templates ship healthchecks tuned for SSD hosts: a short
|
||||
interval, a handful of retries and no start_period at all. On this host a
|
||||
first boot takes minutes (rootfs is ext4 over loopback over an HDD, ~39 ms
|
||||
per write), so the container is flagged `unhealthy` long before the app
|
||||
listens. Traefik only routes containers Docker reports `healthy`, so an
|
||||
unhealthy container has NO route and the request falls through to Coolify's
|
||||
catch-all (`noop`, empty server list) -> 503 "no available server".
|
||||
|
||||
Worse, once flagged unhealthy the container is a candidate for recreation,
|
||||
and recreating restarts the slow entrypoint from zero. That is what turns a
|
||||
transient 503 into a permanent one.
|
||||
|
||||
This script edits `services.docker_compose_raw` (the editable template
|
||||
Coolify regenerates the deployed compose from) and inserts a `start_period`
|
||||
into every healthcheck that lacks one, optionally raising a very short
|
||||
`interval`.
|
||||
|
||||
It does NOT redeploy. The new healthcheck only takes effect when the
|
||||
container is recreated, which is a separate, explicit step.
|
||||
|
||||
Honest scope: start_period does NOT make the site answer sooner. During
|
||||
startup Docker reports `starting`, which Traefik does not route either, so
|
||||
the startup 503 window still exists. What it prevents is the container being
|
||||
*marked failed* and entering the recreation loop.
|
||||
|
||||
.PARAMETER Uuid
|
||||
Coolify service uuid (last path segment of the service URL in the UI).
|
||||
|
||||
.PARAMETER StartPeriodSeconds
|
||||
Grace window to insert. Default 300 (measured first boots here ran into the
|
||||
low minutes).
|
||||
|
||||
.PARAMETER MinIntervalSeconds
|
||||
Raise any `interval` below this. A 2 s interval spawns a health exec every
|
||||
two seconds against an already saturated disk. Default 10. Pass 0 to leave
|
||||
every interval untouched.
|
||||
|
||||
.PARAMETER Apply
|
||||
Actually write. Without it the script only prints the diff and changes
|
||||
nothing.
|
||||
|
||||
.PARAMETER ShowResult
|
||||
Also print the resulting healthcheck blocks so the exact YAML can be
|
||||
reviewed before writing.
|
||||
|
||||
.EXAMPLE
|
||||
# Inspect what would change. Safe, read-only.
|
||||
.\coolify_skill\scripts\Set-CoolifyHealthcheckGrace.ps1 -Uuid uyn0js6pqbwo8mubw5edy95f
|
||||
|
||||
.EXAMPLE
|
||||
# Write it, then redeploy that service yourself from the Coolify UI.
|
||||
.\coolify_skill\scripts\Set-CoolifyHealthcheckGrace.ps1 -Uuid uyn0js6pqbwo8mubw5edy95f -Apply
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Uuid,
|
||||
|
||||
[int]$StartPeriodSeconds = 300,
|
||||
|
||||
[int]$MinIntervalSeconds = 10,
|
||||
|
||||
[switch]$Apply,
|
||||
|
||||
[switch]$ShowResult
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..")
|
||||
$invokeSsh = Join-Path $repoRoot "scripts\Invoke-ProxmoxSsh.ps1"
|
||||
$agentScript = Join-Path $repoRoot "scripts\ProxmoxAgent.ps1"
|
||||
|
||||
foreach ($required in @($invokeSsh, $agentScript)) {
|
||||
if (-not (Test-Path -LiteralPath $required)) { throw "Missing dependency: $required" }
|
||||
}
|
||||
|
||||
. $agentScript
|
||||
$config = Get-ProxmoxConfig
|
||||
$lxc = $config.CoolifyLxc
|
||||
|
||||
# Nested quoting is corrupted by the SSH wrapper (TOOL-INDEX.md 1.2); base64
|
||||
# every remote command. Compose bodies also travel base64 so that newlines,
|
||||
# quotes and Coolify's ${...} magic survive intact.
|
||||
function Invoke-InLxc {
|
||||
param([Parameter(Mandatory = $true)][string]$Script)
|
||||
|
||||
$b64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Script))
|
||||
return @(& $invokeSsh -Command "pct exec $lxc -- bash -c 'echo $b64 | base64 -d | bash'")
|
||||
}
|
||||
|
||||
function Invoke-CoolifyDb {
|
||||
param([Parameter(Mandatory = $true)][string]$Sql)
|
||||
|
||||
# psql -At: unaligned, no header. Quotes are safe inside the base64 payload.
|
||||
return Invoke-InLxc -Script "docker exec coolify-db psql -U coolify -At -c ""$Sql"""
|
||||
}
|
||||
|
||||
function Get-ComposeRaw {
|
||||
param([string]$ServiceUuid)
|
||||
|
||||
$sql = "select encode(convert_to(docker_compose_raw,'UTF8'),'base64') from services where uuid='$ServiceUuid'"
|
||||
$lines = Invoke-CoolifyDb -Sql $sql
|
||||
$payload = ($lines -join '').Trim()
|
||||
if (-not $payload) { return $null }
|
||||
return [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($payload))
|
||||
}
|
||||
|
||||
function Set-ComposeRaw {
|
||||
param([string]$ServiceUuid, [string]$Content)
|
||||
|
||||
$b64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Content))
|
||||
$sql = "update services set docker_compose_raw = convert_from(decode('$b64','base64'),'UTF8'), updated_at = now() where uuid='$ServiceUuid'"
|
||||
$out = Invoke-CoolifyDb -Sql $sql
|
||||
return ($out -join ' ').Trim()
|
||||
}
|
||||
|
||||
function Get-Indent {
|
||||
param([string]$Line)
|
||||
if ($Line -match '^(\s*)') { return $Matches[1].Length }
|
||||
return 0
|
||||
}
|
||||
|
||||
<#
|
||||
Insert start_period into every healthcheck block that lacks one, and raise a
|
||||
too-short interval. Deliberately line-based: re-serialising the YAML would
|
||||
reformat Coolify's magic placeholders and its `- SERVICE_URL_X` shorthand.
|
||||
#>
|
||||
function Update-Healthchecks {
|
||||
param(
|
||||
[string[]]$Lines,
|
||||
[int]$StartPeriod,
|
||||
[int]$MinInterval
|
||||
)
|
||||
|
||||
$out = New-Object 'System.Collections.Generic.List[string]'
|
||||
$changes = New-Object 'System.Collections.Generic.List[object]'
|
||||
|
||||
$i = 0
|
||||
while ($i -lt $Lines.Count) {
|
||||
$line = $Lines[$i]
|
||||
|
||||
if ($line -notmatch '^\s*healthcheck:\s*$') {
|
||||
$out.Add($line)
|
||||
$i++
|
||||
continue
|
||||
}
|
||||
|
||||
$hcIndent = Get-Indent -Line $line
|
||||
$out.Add($line)
|
||||
$hcLineNumber = $i + 1
|
||||
$i++
|
||||
|
||||
# Collect the block: every following line indented deeper than
|
||||
# `healthcheck:` itself. Blank lines inside the block are kept.
|
||||
$block = New-Object 'System.Collections.Generic.List[string]'
|
||||
while ($i -lt $Lines.Count) {
|
||||
$candidate = $Lines[$i]
|
||||
if ($candidate.Trim() -eq '') { $block.Add($candidate); $i++; continue }
|
||||
if ((Get-Indent -Line $candidate) -le $hcIndent) { break }
|
||||
$block.Add($candidate)
|
||||
$i++
|
||||
}
|
||||
|
||||
# Trailing blank lines belong after the block, not inside it.
|
||||
while ($block.Count -gt 0 -and $block[$block.Count - 1].Trim() -eq '') {
|
||||
$block.RemoveAt($block.Count - 1)
|
||||
$i--
|
||||
}
|
||||
|
||||
$childIndent = ' ' * ($hcIndent + 2)
|
||||
foreach ($b in $block) {
|
||||
if ($b.Trim() -ne '') { $childIndent = ' ' * (Get-Indent -Line $b); break }
|
||||
}
|
||||
|
||||
$hasStartPeriod = @($block | Where-Object { $_ -match '^\s*start_period\s*:' }).Count -gt 0
|
||||
|
||||
# Raise a too-short interval.
|
||||
for ($j = 0; $j -lt $block.Count; $j++) {
|
||||
if ($MinInterval -le 0) { break }
|
||||
if ($block[$j] -notmatch '^(\s*)interval\s*:\s*(\S+)\s*$') { continue }
|
||||
|
||||
$indent = $Matches[1]
|
||||
$current = $Matches[2]
|
||||
$seconds = $null
|
||||
if ($current -match '^(\d+(?:\.\d+)?)s$') { $seconds = [double]$Matches[1] }
|
||||
elseif ($current -match '^(\d+)$') { $seconds = [double]$Matches[1] }
|
||||
|
||||
if ($null -ne $seconds -and $seconds -lt $MinInterval) {
|
||||
$block[$j] = "${indent}interval: ${MinInterval}s"
|
||||
$changes.Add([pscustomobject]@{
|
||||
Line = $hcLineNumber
|
||||
Kind = 'interval'
|
||||
From = "interval: $current"
|
||||
To = "interval: ${MinInterval}s"
|
||||
})
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if (-not $hasStartPeriod) {
|
||||
$block.Add("${childIndent}start_period: ${StartPeriod}s")
|
||||
$changes.Add([pscustomobject]@{
|
||||
Line = $hcLineNumber
|
||||
Kind = 'start_period'
|
||||
From = '(absent)'
|
||||
To = "start_period: ${StartPeriod}s"
|
||||
})
|
||||
}
|
||||
|
||||
foreach ($b in $block) { $out.Add($b) }
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
Lines = $out.ToArray()
|
||||
Changes = $changes.ToArray()
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Healthcheck grace - service $Uuid" -ForegroundColor Cyan
|
||||
Write-Host ("-" * 72)
|
||||
|
||||
$original = Get-ComposeRaw -ServiceUuid $Uuid
|
||||
if ($null -eq $original) {
|
||||
throw "No service with uuid '$Uuid' (or its docker_compose_raw is empty). Has it been deleted? Check: Invoke-CoolifyApi.ps1 -Path /services -Raw"
|
||||
}
|
||||
|
||||
$originalLines = $original -split "`r?`n"
|
||||
$result = Update-Healthchecks -Lines $originalLines -StartPeriod $StartPeriodSeconds -MinInterval $MinIntervalSeconds
|
||||
|
||||
$hcCount = @($originalLines | Where-Object { $_ -match '^\s*healthcheck:\s*$' }).Count
|
||||
Write-Host "healthcheck blocks found: $hcCount"
|
||||
|
||||
if ($result.Changes.Count -eq 0) {
|
||||
Write-Host "Nothing to change: every healthcheck already has a start_period and an acceptable interval." -ForegroundColor Green
|
||||
return
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Proposed changes:" -ForegroundColor Yellow
|
||||
$result.Changes | Format-Table Line, Kind, From, To -AutoSize
|
||||
|
||||
$updated = ($result.Lines -join "`n")
|
||||
|
||||
# Guard: the edit must only ever add/modify healthcheck lines. If the line count
|
||||
# moved by more than the number of inserted lines, something went wrong.
|
||||
$inserted = @($result.Changes | Where-Object { $_.Kind -eq 'start_period' }).Count
|
||||
$delta = $result.Lines.Count - $originalLines.Count
|
||||
if ($delta -ne $inserted) {
|
||||
throw "Refusing to write: line count moved by $delta but only $inserted lines should have been inserted. The block parser mis-scoped a healthcheck."
|
||||
}
|
||||
|
||||
if ($ShowResult) {
|
||||
Write-Host ""
|
||||
Write-Host "Resulting healthcheck blocks:" -ForegroundColor Cyan
|
||||
$lines = $result.Lines
|
||||
for ($k = 0; $k -lt $lines.Count; $k++) {
|
||||
if ($lines[$k] -notmatch '^\s*healthcheck:\s*$') { continue }
|
||||
$indent = ($lines[$k] -replace '\S.*$', '').Length
|
||||
Write-Host (" {0,4}: {1}" -f ($k + 1), $lines[$k]) -ForegroundColor DarkGray
|
||||
for ($m = $k + 1; $m -lt $lines.Count; $m++) {
|
||||
if ($lines[$m].Trim() -ne '' -and (($lines[$m] -replace '\S.*$', '').Length -le $indent)) { break }
|
||||
$colour = if ($lines[$m] -match 'start_period|interval') { 'Green' } else { 'DarkGray' }
|
||||
Write-Host (" {0,4}: {1}" -f ($m + 1), $lines[$m]) -ForegroundColor $colour
|
||||
}
|
||||
Write-Host ""
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $Apply) {
|
||||
Write-Host "DRY RUN - nothing was written. Re-run with -Apply to persist." -ForegroundColor Cyan
|
||||
Write-Host "After applying you must redeploy the service for it to take effect." -ForegroundColor Cyan
|
||||
return
|
||||
}
|
||||
|
||||
$backupDir = Join-Path $repoRoot "backups"
|
||||
if (-not (Test-Path -LiteralPath $backupDir)) {
|
||||
New-Item -ItemType Directory -Path $backupDir | Out-Null
|
||||
}
|
||||
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$backupFile = Join-Path $backupDir "compose-raw_${Uuid}_$stamp.yml"
|
||||
[IO.File]::WriteAllText($backupFile, $original, (New-Object Text.UTF8Encoding($false)))
|
||||
Write-Host "Rollback copy: $backupFile" -ForegroundColor DarkGray
|
||||
|
||||
$status = Set-ComposeRaw -ServiceUuid $Uuid -Content $updated
|
||||
Write-Host "psql: $status"
|
||||
|
||||
# Read back and compare, rather than trusting the UPDATE.
|
||||
$verify = Get-ComposeRaw -ServiceUuid $Uuid
|
||||
if ($verify -ne $updated) {
|
||||
Write-Host "VERIFY FAILED - stored content does not match what was sent." -ForegroundColor Red
|
||||
Write-Host "Restore with the rollback copy above before doing anything else." -ForegroundColor Red
|
||||
throw "Write-back verification failed for service $Uuid."
|
||||
}
|
||||
|
||||
Write-Host "Verified: stored docker_compose_raw matches the intended content." -ForegroundColor Green
|
||||
Write-Host ""
|
||||
Write-Host "NOT redeployed. The healthcheck changes only apply once the container" -ForegroundColor Yellow
|
||||
Write-Host "is recreated. Redeploy the service from the Coolify UI, then confirm:" -ForegroundColor Yellow
|
||||
Write-Host " .\coolify_skill\scripts\Test-CoolifyServiceReady.ps1 -Uuid $Uuid -WaitSeconds 600" -ForegroundColor Yellow
|
||||
@@ -0,0 +1,270 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Read-only readiness probe for a Coolify service. Explains a 503
|
||||
"no available server" instead of leaving you guessing.
|
||||
|
||||
.DESCRIPTION
|
||||
Coolify's UI reports a service as green when its container is *running*.
|
||||
Traefik, however, only puts a container in the load balancer once Docker
|
||||
reports it *healthy*. On this host a first boot can take minutes (HDD-backed
|
||||
loopback storage, ~39 ms/write), so a brand-new service is Running but not
|
||||
yet healthy — Traefik has no route for it, the request falls through to
|
||||
Coolify's catch-all router (priority -1000, service `noop`, empty server
|
||||
list) and Traefik answers 503 "no available server".
|
||||
|
||||
This script reports both signals side by side so the gap is visible, and
|
||||
tells you whether you should simply wait.
|
||||
|
||||
Read-only: it never restarts, redeploys or mutates anything.
|
||||
|
||||
.PARAMETER Uuid
|
||||
Coolify resource UUID (the last path segment of the service URL in the UI).
|
||||
Container names carry this as a suffix and change on every redeploy, so the
|
||||
real name is resolved here rather than typed by hand.
|
||||
|
||||
.PARAMETER Fqdn
|
||||
Public URL to probe. Defaults to whatever COOLIFY_FQDN the container carries.
|
||||
|
||||
.PARAMETER WaitSeconds
|
||||
Poll until every container is healthy, up to this many seconds. Default 0
|
||||
(report once and exit). Use 600 for a first boot on this host.
|
||||
|
||||
.EXAMPLE
|
||||
.\coolify_skill\scripts\Test-CoolifyServiceReady.ps1 -Uuid znpmxv2o6ggooi6qxksiagke
|
||||
|
||||
.EXAMPLE
|
||||
# First boot of a service from the Coolify library: wait it out.
|
||||
.\coolify_skill\scripts\Test-CoolifyServiceReady.ps1 -Uuid znpmxv2o6ggooi6qxksiagke -WaitSeconds 600
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Uuid,
|
||||
|
||||
[string]$Fqdn,
|
||||
|
||||
[int]$WaitSeconds = 0
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..")
|
||||
$invokeSsh = Join-Path $repoRoot "scripts\Invoke-ProxmoxSsh.ps1"
|
||||
$agentScript = Join-Path $repoRoot "scripts\ProxmoxAgent.ps1"
|
||||
|
||||
foreach ($required in @($invokeSsh, $agentScript)) {
|
||||
if (-not (Test-Path -LiteralPath $required)) {
|
||||
throw "Missing dependency: $required"
|
||||
}
|
||||
}
|
||||
|
||||
. $agentScript
|
||||
$config = Get-ProxmoxConfig
|
||||
$lxc = $config.CoolifyLxc
|
||||
|
||||
# Nested quoting is corrupted by the SSH wrapper (TOOL-INDEX.md 1.2), so every
|
||||
# non-trivial remote command is base64-encoded.
|
||||
function Invoke-InLxc {
|
||||
param([Parameter(Mandatory = $true)][string]$Script)
|
||||
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes($Script)
|
||||
$b64 = [Convert]::ToBase64String($bytes)
|
||||
return @(& $invokeSsh -Command "pct exec $lxc -- bash -c 'echo $b64 | base64 -d | bash'")
|
||||
}
|
||||
|
||||
function Get-ServiceContainers {
|
||||
param([string]$ResourceUuid)
|
||||
|
||||
# Container names carry the uuid as a suffix and change on every redeploy.
|
||||
$lines = Invoke-InLxc -Script @"
|
||||
docker ps -a --filter "label=coolify.resourceName" --format '{{.Names}}' 2>/dev/null | grep -- '$ResourceUuid' || true
|
||||
docker ps -a --format '{{.Names}}' 2>/dev/null | grep -- '$ResourceUuid' || true
|
||||
"@
|
||||
return @($lines | Where-Object { $_ -and $_.Trim() } | ForEach-Object { $_.Trim() } | Sort-Object -Unique)
|
||||
}
|
||||
|
||||
function Get-ContainerReport {
|
||||
param([string]$Name)
|
||||
|
||||
$raw = Invoke-InLxc -Script @"
|
||||
docker inspect '$Name' --format '{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}|{{.State.Health.FailingStreak}}{{else}}none|0{{end}}|{{.State.ExitCode}}|{{.State.StartedAt}}|{{.RestartCount}}|{{index .Config.Labels "coolify.serviceName"}}'
|
||||
docker inspect '$Name' --format 'HC|{{if .Config.Healthcheck}}{{.Config.Healthcheck.Interval}}|{{.Config.Healthcheck.Retries}}|{{.Config.Healthcheck.StartPeriod}}{{else}}absent|0|0{{end}}'
|
||||
echo "ENVFQDN|`$(docker inspect '$Name' --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | sed -n 's/^COOLIFY_FQDN=//p' | head -1)"
|
||||
"@
|
||||
|
||||
$stateLine = @($raw | Where-Object { $_ -and $_ -notmatch '^(HC|ENVFQDN)\|' })[0]
|
||||
$hcLine = @($raw | Where-Object { $_ -match '^HC\|' })[0]
|
||||
$envLine = @($raw | Where-Object { $_ -match '^ENVFQDN\|' })[0]
|
||||
|
||||
if (-not $stateLine) { return $null }
|
||||
$f = $stateLine.Split('|')
|
||||
|
||||
$interval = $null; $retries = $null; $startPeriod = $null
|
||||
if ($hcLine) {
|
||||
$h = $hcLine.Split('|')
|
||||
$interval = $h[1]; $retries = $h[2]; $startPeriod = $h[3]
|
||||
}
|
||||
|
||||
$containerFqdn = $null
|
||||
if ($envLine) { $containerFqdn = $envLine.Split('|', 2)[1] }
|
||||
|
||||
# Docker prints healthcheck durations either as raw nanoseconds or as a Go
|
||||
# duration string ("2s", "1m30s"), depending on the daemon version.
|
||||
$toSeconds = {
|
||||
param($value)
|
||||
if (-not $value) { return $null }
|
||||
if ($value -match '^\d+$') { return [math]::Round([double]$value / 1e9, 1) }
|
||||
$total = 0.0; $matched = $false
|
||||
foreach ($m in [regex]::Matches($value, '([\d.]+)(h|ms|m|s)')) {
|
||||
$n = [double]$m.Groups[1].Value
|
||||
switch ($m.Groups[2].Value) {
|
||||
'h' { $total += $n * 3600 }
|
||||
'm' { $total += $n * 60 }
|
||||
's' { $total += $n }
|
||||
'ms' { $total += $n / 1000 }
|
||||
}
|
||||
$matched = $true
|
||||
}
|
||||
if ($matched) { return [math]::Round($total, 1) }
|
||||
return $null
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
Name = $Name
|
||||
State = $f[0]
|
||||
Health = $f[1]
|
||||
FailingStreak = [int]$f[2]
|
||||
ExitCode = $f[3]
|
||||
StartedAt = $f[4]
|
||||
RestartCount = $f[5]
|
||||
ServiceName = $f[6]
|
||||
HealthInterval = & $toSeconds $interval
|
||||
HealthRetries = $retries
|
||||
HealthStartPeriod = & $toSeconds $startPeriod
|
||||
Fqdn = $containerFqdn
|
||||
# A container that exited 0 and has no healthcheck is a one-shot init
|
||||
# step (migrations, bucket creation). It is done, not broken.
|
||||
IsOneShot = ($f[0] -eq 'exited') -and ($f[3] -eq '0') -and ($f[1] -eq 'none')
|
||||
RoutableByTraefik = ($f[0] -eq 'running') -and ($f[1] -in @('healthy', 'none'))
|
||||
}
|
||||
}
|
||||
|
||||
function Get-StartupWork {
|
||||
param([string]$Name)
|
||||
|
||||
# A container stuck in its entrypoint (apt/dpkg/chown) is starting, not broken.
|
||||
$lines = Invoke-InLxc -Script @"
|
||||
docker top '$Name' -o pid,stat,etime,cmd 2>/dev/null | tail -n +2 || true
|
||||
"@
|
||||
return @($lines | Where-Object { $_ -match '\b(chown|apt|apt-get|dpkg|unzip|tar|cp)\b' })
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Coolify service readiness - $Uuid" -ForegroundColor Cyan
|
||||
Write-Host ("-" * 72)
|
||||
|
||||
$containers = @(Get-ServiceContainers -ResourceUuid $Uuid)
|
||||
if (-not $containers -or $containers.Count -eq 0) {
|
||||
throw "No container found carrying uuid '$Uuid' in LXC $lxc. Has the service been deployed at all?"
|
||||
}
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($WaitSeconds)
|
||||
$reports = @()
|
||||
|
||||
while ($true) {
|
||||
$reports = @($containers | ForEach-Object { Get-ContainerReport -Name $_ } | Where-Object { $_ })
|
||||
|
||||
$notReady = @($reports | Where-Object { -not $_.RoutableByTraefik -and -not $_.IsOneShot })
|
||||
if ($notReady.Count -eq 0 -or (Get-Date) -ge $deadline) { break }
|
||||
|
||||
$names = ($notReady | ForEach-Object { "$($_.Name)=$($_.Health)" }) -join ', '
|
||||
$left = [int]($deadline - (Get-Date)).TotalSeconds
|
||||
Write-Host " waiting ($left s left): $names" -ForegroundColor DarkGray
|
||||
Start-Sleep -Seconds 10
|
||||
}
|
||||
|
||||
$reports |
|
||||
Select-Object Name, State, Health, FailingStreak,
|
||||
@{ n = 'Routed'; e = { if ($_.IsOneShot) { 'n/a (one-shot)' } else { $_.RoutableByTraefik } } } |
|
||||
Format-Table -AutoSize
|
||||
|
||||
# Healthcheck tuning is the amplifier that turns "slow boot" into "stuck 503".
|
||||
# A short grace window is the amplifier that turns "slow boot" into "stuck 503":
|
||||
# once flagged unhealthy, the container loses its Traefik route entirely.
|
||||
$graceFloorSeconds = 180
|
||||
foreach ($r in $reports) {
|
||||
if ($r.Health -eq 'none' -or $r.HealthStartPeriod) { continue }
|
||||
if (-not $r.HealthInterval -or -not $r.HealthRetries) { continue }
|
||||
|
||||
$grace = $r.HealthInterval * [int]$r.HealthRetries
|
||||
if ($grace -ge $graceFloorSeconds) { continue }
|
||||
|
||||
Write-Host " ! $($r.Name): no start_period; flagged unhealthy after ~$grace s" -ForegroundColor Yellow
|
||||
Write-Host " (interval=$($r.HealthInterval)s x retries=$($r.HealthRetries)). A first boot on this host" -ForegroundColor Yellow
|
||||
Write-Host " can exceed that, and an unhealthy container has no Traefik route -> 503." -ForegroundColor Yellow
|
||||
}
|
||||
|
||||
foreach ($r in $reports) {
|
||||
if ($r.RoutableByTraefik -or $r.IsOneShot) { continue }
|
||||
$work = @(Get-StartupWork -Name $r.Name)
|
||||
if ($work.Count -gt 0) {
|
||||
Write-Host " i $($r.Name) is still running setup work in its entrypoint:" -ForegroundColor DarkCyan
|
||||
$work | ForEach-Object { Write-Host " $_" -ForegroundColor DarkCyan }
|
||||
Write-Host " This is slow-but-progressing, not a failure. Wait, do not redeploy." -ForegroundColor DarkCyan
|
||||
}
|
||||
}
|
||||
|
||||
$target = $Fqdn
|
||||
if (-not $target) {
|
||||
$withFqdn = @($reports | Where-Object { $_.Fqdn })
|
||||
if ($withFqdn.Count -gt 0) { $target = $withFqdn[0].Fqdn }
|
||||
}
|
||||
|
||||
if ($target) {
|
||||
if ($target -notmatch '^https?://') { $target = "https://$target" }
|
||||
Write-Host ""
|
||||
Write-Host "Probing $target" -ForegroundColor Cyan
|
||||
|
||||
$status = $null
|
||||
$body = ''
|
||||
try {
|
||||
$resp = Invoke-WebRequest -Uri $target -UseBasicParsing -TimeoutSec 25
|
||||
$status = [int]$resp.StatusCode
|
||||
$body = [string]$resp.Content
|
||||
}
|
||||
catch {
|
||||
if ($_.Exception.Response) {
|
||||
$status = [int]$_.Exception.Response.StatusCode
|
||||
try {
|
||||
$reader = New-Object IO.StreamReader($_.Exception.Response.GetResponseStream())
|
||||
$body = $reader.ReadToEnd()
|
||||
}
|
||||
catch { $body = '' }
|
||||
}
|
||||
else {
|
||||
Write-Host " transport error: $($_.Exception.Message)" -ForegroundColor Red
|
||||
}
|
||||
}
|
||||
|
||||
if ($status) { Write-Host " HTTP $status" }
|
||||
|
||||
if ($status -eq 503 -and $body -match 'no available server') {
|
||||
Write-Host ""
|
||||
Write-Host " DIAGNOSIS: Traefik has no route for this host." -ForegroundColor Yellow
|
||||
Write-Host " The request fell through to Coolify's catch-all router (priority -1000," -ForegroundColor Yellow
|
||||
Write-Host " service 'noop', empty server list), which is what emits this exact string." -ForegroundColor Yellow
|
||||
Write-Host " Traefik's docker provider only registers containers Docker reports healthy," -ForegroundColor Yellow
|
||||
Write-Host " so an unhealthy/starting container has no route at all." -ForegroundColor Yellow
|
||||
Write-Host " Note: a route that exists but whose backend refuses would return 502, not 503." -ForegroundColor Yellow
|
||||
Write-Host " -> Re-run with -WaitSeconds 600 before changing any configuration." -ForegroundColor Yellow
|
||||
}
|
||||
elseif ($status -ge 200 -and $status -lt 400) {
|
||||
Write-Host " Service is reachable and routed." -ForegroundColor Green
|
||||
}
|
||||
}
|
||||
else {
|
||||
Write-Host " (no FQDN found on the containers; pass -Fqdn to probe)" -ForegroundColor DarkGray
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
$reports
|
||||
Reference in New Issue
Block a user