From c95fdef0e59b3aa7d70ad400a30480534b103c90 Mon Sep 17 00:00:00 2001
From: poprhythm <james.kolpack@gmail.com>
Date: Fri, 18 Sep 2026 03:26:09 +0000
Subject: [PATCH] nfs-mount-heal: read a small chunk, not the whole file, in
 health checks

Root cause of nas_audiobooks still flapping after the automount fix: the
health check finds the first file in each share via `find -quit` and
`cat`s the whole thing within a 10s timeout. That's fine for immich's
13-byte marker and owncloud's 0-byte one, but audiobooks has no such
marker - find picked a real 2.9GB audiobook file, which can never fully
transfer in 10s regardless of mount health. Switched to `head -c 64k`,
which only needs to prove the file handle/NFS path is alive. Confirmed:
the mount was never actually unhealthy post the automount fix, this test
methodology was generating the "stale" verdicts (cat exit=124) itself.

Also captures and logs/alerts the actual find/head error text now instead
of a generic "stale/unreachable" guess, so any future real failure is
diagnosable without SSH archaeology.
---
 system-config/nfs-self-heal/nfs-mount-heal.sh | 49 ++++++++++++++++---
 1 file changed, 41 insertions(+), 8 deletions(-)

diff --git a/system-config/nfs-self-heal/nfs-mount-heal.sh b/system-config/nfs-self-heal/nfs-mount-heal.sh
index cd8c6bf..89c9dae 100644
--- a/system-config/nfs-self-heal/nfs-mount-heal.sh
+++ b/system-config/nfs-self-heal/nfs-mount-heal.sh
@@ -28,18 +28,46 @@ alert() {
     -d chat_id="${TELEGRAM_CHAT_ID}" -d text="$1" > /dev/null 2>&1
 }
 
+LAST_ERROR=""
+
 healthy() {
   local testdir="$1"
-  local f
+  local f rc errfile
+  errfile=$(mktemp)
+
   # No -maxdepth: -quit stops at the first match via depth-first search,
   # so this is fast regardless of tree depth. A maxdepth that's too
   # shallow for the actual file depth (e.g. immich's thumbs/<uuid>/XX/YY/
   # file is 4 levels deep) makes find exhaust a huge fan-out with no
   # match and time out -- a false positive, not real staleness (hit in
   # production 2026-08-25, 3 spurious remounts in 15 min).
-  f=$(timeout 10 find "$testdir" -type f -print -quit 2>/dev/null) || return 1
-  [ -n "$f" ] || return 1
-  timeout 10 cat "$f" > /dev/null 2>&1
+  f=$(timeout 10 find "$testdir" -type f -print -quit 2>"$errfile")
+  rc=$?
+  if [ $rc -ne 0 ] || [ -z "$f" ]; then
+    LAST_ERROR="find exit=${rc}: $(cat "$errfile")"
+    rm -f "$errfile"
+    return 1
+  fi
+
+  # head -c, not cat: this only needs to prove the file handle/NFS path is
+  # alive, not fully transfer the file. Reading the whole thing was fine for
+  # small marker files (immich's 13-byte .immich, owncloud's 0-byte
+  # .migrations.lock) but nas_audiobooks has no such marker, so `find`
+  # legitimately picks a real audiobook - discovered 2026-09-18 to be 2.9GB,
+  # which a full `cat` can never finish inside a 10s timeout regardless of
+  # mount health. That was generating a "stale" verdict (cat exit=124,
+  # timeout) on a mount that was actually fine post the automount fix -
+  # this bug, not a real NFS issue, was the second flapping source.
+  timeout 10 head -c 65536 "$f" > /dev/null 2>"$errfile"
+  rc=$?
+  if [ $rc -ne 0 ]; then
+    LAST_ERROR="head exit=${rc} on '${f}': $(cat "$errfile")"
+    rm -f "$errfile"
+    return 1
+  fi
+
+  rm -f "$errfile"
+  return 0
 }
 
 for entry in "${CHECKS[@]}"; do
@@ -47,16 +75,21 @@ for entry in "${CHECKS[@]}"; do
 
   healthy "$testdir" && continue
 
-  logger -t nfs-mount-heal "${mount} unhealthy (stale/unreachable), remounting ${unit}"
+  # Captured so we log/alert the actual syscall error (stale handle vs
+  # timeout vs something else) instead of guessing - added 2026-09-18
+  # after nas_audiobooks kept flapping with no corroborating error on
+  # the NAS side even after fixing its automount-vs-permanent-mount bug.
+  first_error="$LAST_ERROR"
+  logger -t nfs-mount-heal "${mount} unhealthy, remounting ${unit}: ${first_error}"
 
   if timeout 30 systemctl restart "$unit"; then
     sleep 3
     if healthy "$testdir"; then
-      alert "🔧 NFS self-heal: ${mount} went stale, remounted ${unit} and restarted ${container} automatically. All clear."
+      alert "🔧 NFS self-heal: ${mount} went stale (${first_error}), remounted ${unit} and restarted ${container} automatically. All clear."
     else
-      alert "⚠️ NFS self-heal: ${mount} was stale, remounted ${unit} and restarted ${container}, but it's still failing the health check. Needs a look."
+      alert "⚠️ NFS self-heal: ${mount} was stale (${first_error}), remounted ${unit} and restarted ${container}, but it's still failing (${LAST_ERROR}). Needs a look."
     fi
   else
-    alert "❗ NFS self-heal FAILED: ${mount} is unhealthy and systemctl restart ${unit} failed. Manual intervention needed."
+    alert "❗ NFS self-heal FAILED: ${mount} is unhealthy (${first_error}) and systemctl restart ${unit} failed. Manual intervention needed."
   fi
 done