#!/bin/bash
# Deep health check + self-heal for permanent NFS mounts backing
# immich, calibre, and ocis.
#
# A plain `ls` on a mount root can succeed even when nested file
# handles inside it are stale (ESTALE / errno -116) -- that's the
# failure mode that broke immich on 2026-08-25 and is documented as
# a recurring issue for calibre-import in calibre/MOUNT-HISTORY.md.
# This does an actual nested read; on failure it forces a full
# remount of the NFS mount unit, which triggers the matching
# <container>-mount-ready.service (BindsTo) to restart the container
# automatically.

set -uo pipefail

source /home/poprhythm/docker-infrastructure/.credentials

# mount:unit:testdir:container
CHECKS=(
  "/mnt/nas_family:mnt-nas_family.mount:/mnt/nas_family/immich-library/thumbs:immich_server"
  "/mnt/nas_books:mnt-nas_books.mount:/mnt/nas_books/calibre-import:calibre"
  "/mnt/nas_owncloud:mnt-nas_owncloud.mount:/mnt/nas_owncloud:ocis"
)

alert() {
  curl -fsS -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
    -d chat_id="${TELEGRAM_CHAT_ID}" -d text="$1" > /dev/null 2>&1
}

healthy() {
  local testdir="$1"
  local f
  f=$(timeout 10 find "$testdir" -maxdepth 3 -type f -print -quit 2>/dev/null) || return 1
  [ -n "$f" ] || return 1
  timeout 10 cat "$f" > /dev/null 2>&1
}

for entry in "${CHECKS[@]}"; do
  IFS=':' read -r mount unit testdir container <<< "$entry"

  healthy "$testdir" && continue

  logger -t nfs-mount-heal "${mount} unhealthy (stale/unreachable), remounting ${unit}"

  if timeout 30 systemctl restart "$unit"; then
    sleep 3
    if healthy "$testdir"; then
      alert "🔧 NFS self-heal: ${mount} went stale, remounted ${unit} and restarted ${container} automatically. All clear."
    else
      alert "⚠️ NFS self-heal: ${mount} was stale, remounted ${unit} and restarted ${container}, but it's still failing the health check. Needs a look."
    fi
  else
    alert "❗ NFS self-heal FAILED: ${mount} is unhealthy and systemctl restart ${unit} failed. Manual intervention needed."
  fi
done