<# .SYNOPSIS Give a Coolify service's healthchecks a start_period long enough for a first boot on this host. Dry-run by default. .DESCRIPTION Coolify's library templates ship healthchecks tuned for SSD hosts: a short interval, a handful of retries and no start_period at all. On this host a first boot takes minutes (rootfs is ext4 over loopback over an HDD, ~39 ms per write), so the container is flagged `unhealthy` long before the app listens. Traefik only routes containers Docker reports `healthy`, so an unhealthy container has NO route and the request falls through to Coolify's catch-all (`noop`, empty server list) -> 503 "no available server". Worse, once flagged unhealthy the container is a candidate for recreation, and recreating restarts the slow entrypoint from zero. That is what turns a transient 503 into a permanent one. This script edits `services.docker_compose_raw` (the editable template Coolify regenerates the deployed compose from) and inserts a `start_period` into every healthcheck that lacks one, optionally raising a very short `interval`. It does NOT redeploy. The new healthcheck only takes effect when the container is recreated, which is a separate, explicit step. Honest scope: start_period does NOT make the site answer sooner. During startup Docker reports `starting`, which Traefik does not route either, so the startup 503 window still exists. What it prevents is the container being *marked failed* and entering the recreation loop. .PARAMETER Uuid Coolify service uuid (last path segment of the service URL in the UI). .PARAMETER StartPeriodSeconds Grace window to insert. Default 300 (measured first boots here ran into the low minutes). .PARAMETER MinIntervalSeconds Raise any `interval` below this. A 2 s interval spawns a health exec every two seconds against an already saturated disk. Default 10. Pass 0 to leave every interval untouched. .PARAMETER Apply Actually write. Without it the script only prints the diff and changes nothing. .PARAMETER ShowResult Also print the resulting healthcheck blocks so the exact YAML can be reviewed before writing. .EXAMPLE # Inspect what would change. Safe, read-only. .\coolify_skill\scripts\Set-CoolifyHealthcheckGrace.ps1 -Uuid uyn0js6pqbwo8mubw5edy95f .EXAMPLE # Write it, then redeploy that service yourself from the Coolify UI. .\coolify_skill\scripts\Set-CoolifyHealthcheckGrace.ps1 -Uuid uyn0js6pqbwo8mubw5edy95f -Apply #> [CmdletBinding()] param( [Parameter(Mandatory = $true)] [string]$Uuid, [int]$StartPeriodSeconds = 300, [int]$MinIntervalSeconds = 10, [switch]$Apply, [switch]$ShowResult ) $ErrorActionPreference = "Stop" $repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") $invokeSsh = Join-Path $repoRoot "scripts\Invoke-ProxmoxSsh.ps1" $agentScript = Join-Path $repoRoot "scripts\ProxmoxAgent.ps1" foreach ($required in @($invokeSsh, $agentScript)) { if (-not (Test-Path -LiteralPath $required)) { throw "Missing dependency: $required" } } . $agentScript $config = Get-ProxmoxConfig $lxc = $config.CoolifyLxc # Nested quoting is corrupted by the SSH wrapper (TOOL-INDEX.md 1.2); base64 # every remote command. Compose bodies also travel base64 so that newlines, # quotes and Coolify's ${...} magic survive intact. function Invoke-InLxc { param([Parameter(Mandatory = $true)][string]$Script) $b64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Script)) return @(& $invokeSsh -Command "pct exec $lxc -- bash -c 'echo $b64 | base64 -d | bash'") } function Invoke-CoolifyDb { param([Parameter(Mandatory = $true)][string]$Sql) # psql -At: unaligned, no header. Quotes are safe inside the base64 payload. return Invoke-InLxc -Script "docker exec coolify-db psql -U coolify -At -c ""$Sql""" } function Get-ComposeRaw { param([string]$ServiceUuid) $sql = "select encode(convert_to(docker_compose_raw,'UTF8'),'base64') from services where uuid='$ServiceUuid'" $lines = Invoke-CoolifyDb -Sql $sql $payload = ($lines -join '').Trim() if (-not $payload) { return $null } return [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($payload)) } function Set-ComposeRaw { param([string]$ServiceUuid, [string]$Content) $b64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Content)) $sql = "update services set docker_compose_raw = convert_from(decode('$b64','base64'),'UTF8'), updated_at = now() where uuid='$ServiceUuid'" $out = Invoke-CoolifyDb -Sql $sql return ($out -join ' ').Trim() } function Get-Indent { param([string]$Line) if ($Line -match '^(\s*)') { return $Matches[1].Length } return 0 } <# Insert start_period into every healthcheck block that lacks one, and raise a too-short interval. Deliberately line-based: re-serialising the YAML would reformat Coolify's magic placeholders and its `- SERVICE_URL_X` shorthand. #> function Update-Healthchecks { param( [string[]]$Lines, [int]$StartPeriod, [int]$MinInterval ) $out = New-Object 'System.Collections.Generic.List[string]' $changes = New-Object 'System.Collections.Generic.List[object]' $i = 0 while ($i -lt $Lines.Count) { $line = $Lines[$i] if ($line -notmatch '^\s*healthcheck:\s*$') { $out.Add($line) $i++ continue } $hcIndent = Get-Indent -Line $line $out.Add($line) $hcLineNumber = $i + 1 $i++ # Collect the block: every following line indented deeper than # `healthcheck:` itself. Blank lines inside the block are kept. $block = New-Object 'System.Collections.Generic.List[string]' while ($i -lt $Lines.Count) { $candidate = $Lines[$i] if ($candidate.Trim() -eq '') { $block.Add($candidate); $i++; continue } if ((Get-Indent -Line $candidate) -le $hcIndent) { break } $block.Add($candidate) $i++ } # Trailing blank lines belong after the block, not inside it. while ($block.Count -gt 0 -and $block[$block.Count - 1].Trim() -eq '') { $block.RemoveAt($block.Count - 1) $i-- } $childIndent = ' ' * ($hcIndent + 2) foreach ($b in $block) { if ($b.Trim() -ne '') { $childIndent = ' ' * (Get-Indent -Line $b); break } } $hasStartPeriod = @($block | Where-Object { $_ -match '^\s*start_period\s*:' }).Count -gt 0 # Raise a too-short interval. for ($j = 0; $j -lt $block.Count; $j++) { if ($MinInterval -le 0) { break } if ($block[$j] -notmatch '^(\s*)interval\s*:\s*(\S+)\s*$') { continue } $indent = $Matches[1] $current = $Matches[2] $seconds = $null if ($current -match '^(\d+(?:\.\d+)?)s$') { $seconds = [double]$Matches[1] } elseif ($current -match '^(\d+)$') { $seconds = [double]$Matches[1] } if ($null -ne $seconds -and $seconds -lt $MinInterval) { $block[$j] = "${indent}interval: ${MinInterval}s" $changes.Add([pscustomobject]@{ Line = $hcLineNumber Kind = 'interval' From = "interval: $current" To = "interval: ${MinInterval}s" }) } break } if (-not $hasStartPeriod) { $block.Add("${childIndent}start_period: ${StartPeriod}s") $changes.Add([pscustomobject]@{ Line = $hcLineNumber Kind = 'start_period' From = '(absent)' To = "start_period: ${StartPeriod}s" }) } foreach ($b in $block) { $out.Add($b) } } return [pscustomobject]@{ Lines = $out.ToArray() Changes = $changes.ToArray() } } Write-Host "" Write-Host "Healthcheck grace - service $Uuid" -ForegroundColor Cyan Write-Host ("-" * 72) $original = Get-ComposeRaw -ServiceUuid $Uuid if ($null -eq $original) { throw "No service with uuid '$Uuid' (or its docker_compose_raw is empty). Has it been deleted? Check: Invoke-CoolifyApi.ps1 -Path /services -Raw" } $originalLines = $original -split "`r?`n" $result = Update-Healthchecks -Lines $originalLines -StartPeriod $StartPeriodSeconds -MinInterval $MinIntervalSeconds $hcCount = @($originalLines | Where-Object { $_ -match '^\s*healthcheck:\s*$' }).Count Write-Host "healthcheck blocks found: $hcCount" if ($result.Changes.Count -eq 0) { Write-Host "Nothing to change: every healthcheck already has a start_period and an acceptable interval." -ForegroundColor Green return } Write-Host "" Write-Host "Proposed changes:" -ForegroundColor Yellow $result.Changes | Format-Table Line, Kind, From, To -AutoSize $updated = ($result.Lines -join "`n") # Guard: the edit must only ever add/modify healthcheck lines. If the line count # moved by more than the number of inserted lines, something went wrong. $inserted = @($result.Changes | Where-Object { $_.Kind -eq 'start_period' }).Count $delta = $result.Lines.Count - $originalLines.Count if ($delta -ne $inserted) { throw "Refusing to write: line count moved by $delta but only $inserted lines should have been inserted. The block parser mis-scoped a healthcheck." } if ($ShowResult) { Write-Host "" Write-Host "Resulting healthcheck blocks:" -ForegroundColor Cyan $lines = $result.Lines for ($k = 0; $k -lt $lines.Count; $k++) { if ($lines[$k] -notmatch '^\s*healthcheck:\s*$') { continue } $indent = ($lines[$k] -replace '\S.*$', '').Length Write-Host (" {0,4}: {1}" -f ($k + 1), $lines[$k]) -ForegroundColor DarkGray for ($m = $k + 1; $m -lt $lines.Count; $m++) { if ($lines[$m].Trim() -ne '' -and (($lines[$m] -replace '\S.*$', '').Length -le $indent)) { break } $colour = if ($lines[$m] -match 'start_period|interval') { 'Green' } else { 'DarkGray' } Write-Host (" {0,4}: {1}" -f ($m + 1), $lines[$m]) -ForegroundColor $colour } Write-Host "" } } if (-not $Apply) { Write-Host "DRY RUN - nothing was written. Re-run with -Apply to persist." -ForegroundColor Cyan Write-Host "After applying you must redeploy the service for it to take effect." -ForegroundColor Cyan return } $backupDir = Join-Path $repoRoot "backups" if (-not (Test-Path -LiteralPath $backupDir)) { New-Item -ItemType Directory -Path $backupDir | Out-Null } $stamp = Get-Date -Format 'yyyyMMdd-HHmmss' $backupFile = Join-Path $backupDir "compose-raw_${Uuid}_$stamp.yml" [IO.File]::WriteAllText($backupFile, $original, (New-Object Text.UTF8Encoding($false))) Write-Host "Rollback copy: $backupFile" -ForegroundColor DarkGray $status = Set-ComposeRaw -ServiceUuid $Uuid -Content $updated Write-Host "psql: $status" # Read back and compare, rather than trusting the UPDATE. $verify = Get-ComposeRaw -ServiceUuid $Uuid if ($verify -ne $updated) { Write-Host "VERIFY FAILED - stored content does not match what was sent." -ForegroundColor Red Write-Host "Restore with the rollback copy above before doing anything else." -ForegroundColor Red throw "Write-back verification failed for service $Uuid." } Write-Host "Verified: stored docker_compose_raw matches the intended content." -ForegroundColor Green Write-Host "" Write-Host "NOT redeployed. The healthcheck changes only apply once the container" -ForegroundColor Yellow Write-Host "is recreated. Redeploy the service from the Coolify UI, then confirm:" -ForegroundColor Yellow Write-Host " .\coolify_skill\scripts\Test-CoolifyServiceReady.ps1 -Uuid $Uuid -WaitSeconds 600" -ForegroundColor Yellow