poprhythm
50a5a48ed4
find -maxdepth 3 couldn't reach immich's thumbs/<uuid>/XX/YY/file (4 levels deep), so it exhausted the 256x256 hash-bucket fan-out with no match and timed out on every run -- 3 spurious remounts/restarts of immich_server within 15 minutes of deploy, all falsely alerted as staleness. Drop -maxdepth entirely; -quit already stops at the first match via depth-first search, so it's fast regardless of tree depth.
62 lines
2.4 KiB
Bash
62 lines
2.4 KiB
Bash
#!/bin/bash
|
|
# Deep health check + self-heal for permanent NFS mounts backing
|
|
# immich, calibre, and ocis.
|
|
#
|
|
# A plain `ls` on a mount root can succeed even when nested file
|
|
# handles inside it are stale (ESTALE / errno -116) -- that's the
|
|
# failure mode that broke immich on 2026-08-25 and is documented as
|
|
# a recurring issue for calibre-import in calibre/MOUNT-HISTORY.md.
|
|
# This does an actual nested read; on failure it forces a full
|
|
# remount of the NFS mount unit, which triggers the matching
|
|
# <container>-mount-ready.service (BindsTo) to restart the container
|
|
# automatically.
|
|
|
|
set -uo pipefail
|
|
|
|
source /home/poprhythm/docker-infrastructure/.credentials
|
|
|
|
# mount:unit:testdir:container
|
|
CHECKS=(
|
|
"/mnt/nas_family:mnt-nas_family.mount:/mnt/nas_family/immich-library/thumbs:immich_server"
|
|
"/mnt/nas_books:mnt-nas_books.mount:/mnt/nas_books/calibre-import:calibre"
|
|
"/mnt/nas_owncloud:mnt-nas_owncloud.mount:/mnt/nas_owncloud:ocis"
|
|
)
|
|
|
|
alert() {
|
|
curl -fsS -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
|
-d chat_id="${TELEGRAM_CHAT_ID}" -d text="$1" > /dev/null 2>&1
|
|
}
|
|
|
|
healthy() {
|
|
local testdir="$1"
|
|
local f
|
|
# No -maxdepth: -quit stops at the first match via depth-first search,
|
|
# so this is fast regardless of tree depth. A maxdepth that's too
|
|
# shallow for the actual file depth (e.g. immich's thumbs/<uuid>/XX/YY/
|
|
# file is 4 levels deep) makes find exhaust a huge fan-out with no
|
|
# match and time out -- a false positive, not real staleness (hit in
|
|
# production 2026-08-25, 3 spurious remounts in 15 min).
|
|
f=$(timeout 10 find "$testdir" -type f -print -quit 2>/dev/null) || return 1
|
|
[ -n "$f" ] || return 1
|
|
timeout 10 cat "$f" > /dev/null 2>&1
|
|
}
|
|
|
|
for entry in "${CHECKS[@]}"; do
|
|
IFS=':' read -r mount unit testdir container <<< "$entry"
|
|
|
|
healthy "$testdir" && continue
|
|
|
|
logger -t nfs-mount-heal "${mount} unhealthy (stale/unreachable), remounting ${unit}"
|
|
|
|
if timeout 30 systemctl restart "$unit"; then
|
|
sleep 3
|
|
if healthy "$testdir"; then
|
|
alert "🔧 NFS self-heal: ${mount} went stale, remounted ${unit} and restarted ${container} automatically. All clear."
|
|
else
|
|
alert "⚠️ NFS self-heal: ${mount} was stale, remounted ${unit} and restarted ${container}, but it's still failing the health check. Needs a look."
|
|
fi
|
|
else
|
|
alert "❗ NFS self-heal FAILED: ${mount} is unhealthy and systemctl restart ${unit} failed. Manual intervention needed."
|
|
fi
|
|
done
|