#!/bin/bash
# Deep health check + self-heal for permanent NFS mounts backing
# immich, calibre, ocis, and audiobookshelf.
#
# A plain `ls` on a mount root can succeed even when nested file
# handles inside it are stale (ESTALE / errno -116) -- that's the
# failure mode that broke immich on 2026-08-25 and is documented as
# a recurring issue for calibre-import in calibre/MOUNT-HISTORY.md.
# This does an actual nested read; on failure it forces a full
# remount of the NFS mount unit, which triggers the matching
# <container>-mount-ready.service (BindsTo) to restart the container
# automatically.

set -uo pipefail

source /home/poprhythm/docker-infrastructure/.credentials

# mount:unit:testdir:container
CHECKS=(
  "/mnt/nas_family:mnt-nas_family.mount:/mnt/nas_family/immich-library/thumbs:immich_server"
  "/mnt/nas_books:mnt-nas_books.mount:/mnt/nas_books/calibre-import:calibre"
  "/mnt/nas_owncloud:mnt-nas_owncloud.mount:/mnt/nas_owncloud:ocis"
  "/mnt/nas_audiobooks:mnt-nas_audiobooks.mount:/mnt/nas_audiobooks:audiobookshelf"
)

alert() {
  curl -fsS -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
    -d chat_id="${TELEGRAM_CHAT_ID}" -d text="$1" > /dev/null 2>&1
}

LAST_ERROR=""

healthy() {
  local testdir="$1"
  local f rc errfile
  errfile=$(mktemp)

  # No -maxdepth: -quit stops at the first match via depth-first search,
  # so this is fast regardless of tree depth. A maxdepth that's too
  # shallow for the actual file depth (e.g. immich's thumbs/<uuid>/XX/YY/
  # file is 4 levels deep) makes find exhaust a huge fan-out with no
  # match and time out -- a false positive, not real staleness (hit in
  # production 2026-08-25, 3 spurious remounts in 15 min).
  f=$(timeout 10 find "$testdir" -type f -print -quit 2>"$errfile")
  rc=$?
  if [ $rc -ne 0 ] || [ -z "$f" ]; then
    LAST_ERROR="find exit=${rc}: $(cat "$errfile")"
    rm -f "$errfile"
    return 1
  fi

  # head -c, not cat: this only needs to prove the file handle/NFS path is
  # alive, not fully transfer the file. Reading the whole thing was fine for
  # small marker files (immich's 13-byte .immich, owncloud's 0-byte
  # .migrations.lock) but nas_audiobooks has no such marker, so `find`
  # legitimately picks a real audiobook - discovered 2026-09-18 to be 2.9GB,
  # which a full `cat` can never finish inside a 10s timeout regardless of
  # mount health. That was generating a "stale" verdict (cat exit=124,
  # timeout) on a mount that was actually fine post the automount fix -
  # this bug, not a real NFS issue, was the second flapping source.
  timeout 10 head -c 65536 "$f" > /dev/null 2>"$errfile"
  rc=$?
  if [ $rc -ne 0 ]; then
    LAST_ERROR="head exit=${rc} on '${f}': $(cat "$errfile")"
    rm -f "$errfile"
    return 1
  fi

  rm -f "$errfile"
  return 0
}

for entry in "${CHECKS[@]}"; do
  IFS=':' read -r mount unit testdir container <<< "$entry"

  healthy "$testdir" && continue

  # Captured so we log/alert the actual syscall error (stale handle vs
  # timeout vs something else) instead of guessing - added 2026-09-18
  # after nas_audiobooks kept flapping with no corroborating error on
  # the NAS side even after fixing its automount-vs-permanent-mount bug.
  first_error="$LAST_ERROR"
  logger -t nfs-mount-heal "${mount} unhealthy, remounting ${unit}: ${first_error}"

  if timeout 30 systemctl restart "$unit"; then
    sleep 3
    if healthy "$testdir"; then
      alert "🔧 NFS self-heal: ${mount} went stale (${first_error}), remounted ${unit} and restarted ${container} automatically. All clear."
    else
      alert "⚠️ NFS self-heal: ${mount} was stale (${first_error}), remounted ${unit} and restarted ${container}, but it's still failing (${LAST_ERROR}). Needs a look."
    fi
  else
    alert "❗ NFS self-heal FAILED: ${mount} is unhealthy (${first_error}) and systemctl restart ${unit} failed. Manual intervention needed."
  fi
done