#!/usr/bin/env bash # tclock-system-health — system-health status block for a tclock clock widget. # # Renders a compact ANSI dashboard designed for a full-width bottom widget # (position = "bottom", title = ""): a one-line title+verdict header, paired # two-column status rows, and a full-width btrfs maintenance row. # # left column scheduled protection jobs, in the order given on the # command line (--timer / --snapshots) # right column live state: system (zombies/load/mem), jobs (user timers + # failed units), storage (per-filesystem usage) # bottom row btrfs: scrub age per filesystem, fstrim age, allocation # pressure (balance indicator), device I/O error counters # # Everything is read without root: systemd unit properties via systemctl, # btrfs device stats / filesystem usage (both work unprivileged), df, ps, # /proc, and grub-btrfs.cfg (world-readable, rewritten on snapshot events). # # Host specifics are flags, so the script itself stays machine-agnostic. # No btrfs? The bottom row disappears by itself (or force it off with # --no-btrfs). No restic/backup timer? Just don't pass a --timer for it. # No timeshift? Skip --snapshots. Every section degrades independently. # # Zero-config: with no --timer/--snapshots flags, the left column is # auto-detected — user timers whose names look like backup tools (restic, # borg, kopia, btrbk, *backup*, ...) are shown with a staleness window # derived from each timer's own period, and the snapshot line appears when # grub-btrfs.cfg contains timeshift snapshots. Removable media under # /run/media and /media are excluded from storage unless you pass your own # --exclude-mount flags. # # Example (wrap host flags in a small script on your PATH): # tclock-system-health \ # --timer backups:restic-backup:26:restic \ # --snapshots \ # --timer cleanup:dev-cache-clean:384:dev-cache \ # --exclude-mount /run/media --exclude-mount /mnt/nas # # Options: # --timer LABEL:UNIT[:STALE_HOURS[:NAME]] # report a systemd *user* timer (repeatable; # order defines the left column). UNIT is the # base unit name; .timer/.service suffixes are # accepted and stripped. STALE_HOURS defaults # to auto (timer period + 50%); NAME is an # optional short display name # --snapshots insert the timeshift snapshot line at this # point of the left column # --grub-btrfs-cfg PATH snapshot source (default # /boot/grub/grub-btrfs.cfg) # --snapshot-max N warn when more than N snapshots are kept # (pruning likely broken; default 26) # --exclude-mount PREFIX storage: ignore mounts under PREFIX # (repeatable; e.g. USB media, NAS) # --benign-zombie COMM treat zombies with this comm as benign # (repeatable; replaces the default list: # uwsm-app systemctl; zombies inside docker # cgroups are always benign — containers # without an init never reap) # --disk-warn N / --disk-crit N storage usage thresholds (default 85 / 95) # --left-col N left column visible width (default 44) # --single-column stack all rows instead of pairing # --no-btrfs omit the btrfs row even if btrfs is mounted # --details open with the list of problems behind the # dashboard verdict, then explain units and # timer jobs, zombies/load/memory, timeshift # snapshots, scheduled jobs, storage pressure, # and btrfs allocation/I/O state (intended # for a tclock popup action) # --title TEXT header title (default "System Health") # --theme NAME color theme: default, evangelion, nerv # (default: default; # or $TCLOCK_SYSTEM_HEALTH_THEME, falling back # to generic $TCLOCK_WIDGET_THEME from tclock) # --list-themes print available themes and exit set -u export LC_ALL=C ME=${0##*/} die() { printf '%s: %s\n' "$ME" "$*" >&2; exit 2; } is_uint() { case "${1-}" in '' | *[!0-9]*) return 1 ;; *) return 0 ;; esac; } # ---------- themes ---------- # Theme contract for contributors: # - Set semantic colors G/Y/R for ok/warn/error values. # - Set D/B/N for dim, title emphasis, and reset. # - Set LBL for section labels. # - Rebuild OK/WA/ER glyphs from those colors. sgr() { printf '\033[%sm' "$1"; } theme_default() { G=$'\033[32m'; Y=$'\033[33m'; R=$'\033[31m'; D=$'\033[2m'; B=$'\033[1m'; N=$'\033[0m' LBL=$'\033[1;36m' # bold cyan: section labels, contrasts with values OK="${G}✔${N}"; WA="${Y}▲${N}"; ER="${R}✖${N}" } theme_evangelion() { # Original Evangelion-inspired terminal palette: deep purple labels, # tactical orange warnings, EVA green success, and alarm red failures. G=$(sgr '38;5;118') # EVA green Y=$(sgr '38;5;208') # NERV orange R=$(sgr '38;5;196') # alarm red D=$(sgr '38;5;103') # muted lavender B=$(sgr '1;38;5;171') # bold purple title N=$'\033[0m' LBL=$(sgr '1;38;5;99') # purple section labels OK="${G}◆${N}"; WA="${Y}△${N}"; ER="${R}▣${N}" } theme_nerv() { # Screenshot-matched NERV terminal palette: amber/orange UI language, # neon green status accents, and emergency red failures. G=$(sgr '38;5;48') # neon green Y=$(sgr '38;5;214') # amber warning R=$(sgr '38;5;196') # emergency red D=$(sgr '38;5;242') # dim neutral gray B=$(sgr '1;38;5;214') # bold amber title N=$'\033[0m' LBL=$(sgr '1;38;5;208') # bold orange section labels OK="${G}◆${N}"; WA="${Y}▲${N}"; ER="${R}▣${N}" } list_themes() { printf 'default\n' printf 'evangelion\n' printf 'nerv\n' } available_themes() { list_themes | paste -sd, - | sed 's/,/, /g'; } apply_theme() { case "$1" in default) theme_default ;; evangelion) theme_evangelion ;; nerv) theme_nerv ;; *) die "unknown theme: $1 (available: $(available_themes))" ;; esac } # ---------- configuration (defaults + flags) ---------- GRUB_BTRFS_CFG=/boot/grub/grub-btrfs.cfg SNAPSHOT_MAX=26 DISK_WARN=85 DISK_CRIT=95 LEFT_COL=44 SINGLE_COLUMN=0 NO_BTRFS=0 DETAILS=0 TITLE="System Health" THEME=${TCLOCK_SYSTEM_HEALTH_THEME:-${TCLOCK_WIDGET_THEME:-default}} LEFT_SPEC=() # ordered left-column entries: "timer:LABEL:UNIT:STALE" | "snapshots" EXCLUDE_MOUNTS=() BENIGN_ZOMBIES=(uwsm-app systemctl) BENIGN_ZOMBIES_SET=0 need_val() { [ -n "${2-}" ] || die "option $1 requires a value (see --help)"; } while [ $# -gt 0 ]; do case "$1" in --timer) need_val "$1" "${2-}" IFS=: read -r t_label t_unit t_stale t_name t_extra <<<"$2" [ -n "$t_label" ] && [ -n "$t_unit" ] && [ -z "${t_extra-}" ] \ || die "--timer expects LABEL:UNIT[:STALE_HOURS[:NAME]], got '$2'" t_unit=${t_unit%.timer} t_unit=${t_unit%.service} t_stale=${t_stale:-auto} [ "$t_stale" = auto ] || is_uint "$t_stale" \ || die "--timer STALE_HOURS must be a number or 'auto', got '$t_stale'" LEFT_SPEC+=("timer:$t_label:$t_unit:$t_stale:${t_name-}"); shift 2 ;; --snapshots) LEFT_SPEC+=("snapshots"); shift ;; --grub-btrfs-cfg) need_val "$1" "${2-}"; GRUB_BTRFS_CFG=$2; shift 2 ;; --snapshot-max) need_val "$1" "${2-}"; is_uint "$2" || die "--snapshot-max must be a number"; SNAPSHOT_MAX=$2; shift 2 ;; --exclude-mount) need_val "$1" "${2-}"; EXCLUDE_MOUNTS+=("$2"); shift 2 ;; --benign-zombie) need_val "$1" "${2-}" [ "$BENIGN_ZOMBIES_SET" = 1 ] || { BENIGN_ZOMBIES=(); BENIGN_ZOMBIES_SET=1; } BENIGN_ZOMBIES+=("$2"); shift 2 ;; --disk-warn) need_val "$1" "${2-}"; is_uint "$2" || die "--disk-warn must be a number"; DISK_WARN=$2; shift 2 ;; --disk-crit) need_val "$1" "${2-}"; is_uint "$2" || die "--disk-crit must be a number"; DISK_CRIT=$2; shift 2 ;; --left-col) need_val "$1" "${2-}"; is_uint "$2" || die "--left-col must be a number"; LEFT_COL=$2; shift 2 ;; --single-column) SINGLE_COLUMN=1; shift ;; --no-btrfs) NO_BTRFS=1; shift ;; --details) DETAILS=1; shift ;; --title) need_val "$1" "${2-}"; TITLE=$2; shift 2 ;; --theme) need_val "$1" "${2-}"; THEME=$2; shift 2 ;; --list-themes) list_themes; exit 0 ;; -h|--help) awk 'NR >= 2 && /^set -u$/ { exit } NR >= 2 { sub(/^# ?/, ""); print }' "$0"; exit 0 ;; *) die "unknown option: $1 (see --help)" ;; esac done [ "$DISK_WARN" -lt "$DISK_CRIT" ] || die "--disk-warn ($DISK_WARN) must be below --disk-crit ($DISK_CRIT)" [ "$LEFT_COL" -ge 20 ] || die "--left-col must be at least 20" command -v systemctl >/dev/null 2>&1 || die "systemctl is required" apply_theme "$THEME" # default exclusions: removable media (override by passing any --exclude-mount) [ ${#EXCLUDE_MOUNTS[@]} -eq 0 ] && EXCLUDE_MOUNTS=(/run/media /media) # zero-config: detect common setups when no left-column flags were given if [ ${#LEFT_SPEC[@]} -eq 0 ]; then while read -r t_unit; do t_unit=${t_unit%.timer} [ -n "$t_unit" ] || continue case "$t_unit" in *restic*|*borg*|*kopia*|*rsnapshot*|*duplic*|*btrbk*|*vorta*|*backup*) LEFT_SPEC+=("timer:${t_unit:0:8}:$t_unit:auto:") ;; esac done < <(systemctl --user list-timers --no-legend 2>/dev/null | awk '{print $(NF-1)}') grep -qs 'timeshift-btrfs/snapshots/' "$GRUB_BTRFS_CFG" && LEFT_SPEC+=("snapshots") fi # ---------- shared state + helpers ---------- worst=0 # 0 ok, 1 warn, 2 crit bump() { [ "$1" -gt "$worst" ] && worst=$1; } # Every check that raises the verdict also records *why*, so the --details # popup can open with the same list of problems the dashboard summarised. # FINDINGS entries: "LEVELSECTIONMESSAGE" (message may carry ANSI). FINDINGS=() flag() { local lvl=$1 section=$2 shift 2 FINDINGS+=("$lvl"$'\t'"$section"$'\t'"$*") bump "$lvl" } # Per-section facts gathered by the dashboard functions, replayed in details. SEP=$'\x1f' # field separator that survives empty fields (tabs collapse in read) TIMER_INFO=() # "LABEL SEP UNIT SEP STATUS SEP LAST SEP NEXT SEP NOTE SEP GLYPH" ZOMBIES=() # "PIDCOMMPPIDPARENT_COMMbenign|stray" SNAP_STATUS="" # missing-cfg | none | ok | late | stale SNAP_NEWEST=""; SNAP_COUNT=0; SNAP_AGE_H=0 SYS_LOAD="?"; SYS_CORES="?"; SYS_MEM="?" now_s=$(date +%s) age_hours() { echo $(( (now_s - $(date -d "$1" +%s 2>/dev/null || echo "$now_s")) / 3600 )); } age_days() { echo $(( (now_s - $(date -d "$1" +%s 2>/dev/null || echo "$now_s")) / 86400 )); } human_bytes() { awk -v bytes="$1" 'BEGIN { split("KiB MiB GiB TiB PiB", units, " ") if (bytes < 1024) { printf "%d B", bytes; exit } value = bytes for (i = 1; i <= 5 && value >= 1024; i++) value /= 1024 if (value >= 10) printf "%.0f %s", value, units[i - 1] else printf "%.1f %s", value, units[i - 1] }' } # visible length: strip ANSI, count characters (not bytes) vlen() { local s s=$(sed -E $'s/\x1b\\[[0-9;]*m//g' <<<"$1") LC_ALL=C.UTF-8 bash -c 'echo ${#1}' _ "$s" } # pair LEFT RIGHT: left padded to LEFT_COL visible chars pair() { local l=$1 r=$2 pad [ -z "$r" ] && { printf '%s\n' "$l"; return; } [ -z "$l" ] && { printf '%*s%s\n' "$LEFT_COL" '' "$r"; return; } pad=$(( LEFT_COL - $(vlen "$l") )) [ "$pad" -lt 1 ] && pad=1 printf '%s%*s%s\n' "$l" "$pad" '' "$r" } # mounted btrfs filesystems, one line per backing device: "mount:label:esc". # A device can be mounted at several subvol paths but has at most one scrub # timer, keyed to one of those paths — prefer the mount that actually has an # armed btrfs-scrub@ timer, falling back to the first mount seen. btrfs_filesystems() { local src tgt dev label esc next local -A first_mnt timer_mnt stats_mnt local order=() command -v findmnt >/dev/null 2>&1 || return 0 while read -r tgt src; do [ -n "$tgt" ] || continue dev=${src%%\[*} if [ -z "${first_mnt[$dev]:-}" ]; then first_mnt[$dev]=$tgt order+=("$dev") fi if [ -z "${timer_mnt[$dev]:-}" ]; then esc=$(systemd-escape -p "$tgt" 2>/dev/null) || continue next=$(systemctl show "btrfs-scrub@$esc.timer" -p NextElapseUSecRealtime --value 2>/dev/null) [ -n "$next" ] && [ "$next" != "n/a" ] && timer_mnt[$dev]=$tgt fi # device stats work from ANY mountpoint of the filesystem, but only if the # mountpoint itself is accessible (e.g. /var/lib/docker is root-only while # a sibling subvol mount of the same device is world-readable) if [ -z "${stats_mnt[$dev]:-}" ] && btrfs device stats "$tgt" >/dev/null 2>&1; then stats_mnt[$dev]=$tgt fi done < <(findmnt -rn -t btrfs -o TARGET,SOURCE 2>/dev/null) for dev in ${order[@]+"${order[@]}"}; do tgt=${timer_mnt[$dev]:-${first_mnt[$dev]}} label=$(basename "$tgt"); [ "$tgt" = "/" ] && label=root esc=$(systemd-escape -p "$tgt" 2>/dev/null) || continue printf '%s:%s:%s:%s\n' "$tgt" "$label" "$esc" "${stats_mnt[$dev]:-}" done } # ---------- left column: systemd user timer status ---------- # Timer next/last come from `list-timers -o json` (usec epochs) because the # NextElapseUSecRealtime property is n/a for monotonic (OnUnitActiveSec-style) # timers; falls back to unit properties on systems without JSON output / jq. # timer_line LABEL UNIT STALE_HOURS|auto [NAME] timer_line() { local label=$1 unit=$2 stale_h=$3 name=${4-} local result last_s=0 next_s=0 tj last_age age_str next_str glyph=$OK note="" rword="ok" local disp=""; [ -n "$name" ] && disp="$name " result=$(systemctl --user show "$unit.service" -p Result --value 2>/dev/null) if command -v jq >/dev/null 2>&1; then tj=$(systemctl --user list-timers --all "$unit.timer" -o json 2>/dev/null) if [ -n "$tj" ] && [ "$tj" != "[]" ]; then next_s=$(( $(jq -r '.[0].next // 0' <<<"$tj") / 1000000 )) last_s=$(( $(jq -r '.[0].last // 0' <<<"$tj") / 1000000 )) fi fi if [ "$next_s" -eq 0 ] && [ "$last_s" -eq 0 ]; then # fallback: realtime properties (systems without list-timers JSON / jq) local last_us next_us last_us=$(systemctl --user show "$unit.timer" -p LastTriggerUSec --value 2>/dev/null) next_us=$(systemctl --user show "$unit.timer" -p NextElapseUSecRealtime --value 2>/dev/null) [ -n "$next_us" ] && [ "$next_us" != "n/a" ] && next_s=$(date -d "$next_us" +%s 2>/dev/null || echo 0) [ -n "$last_us" ] && [ "$last_us" != "n/a" ] && last_s=$(date -d "$last_us" +%s 2>/dev/null || echo 0) fi # timer armed but never fired (fresh enable / manual runs only): the # service's own last completion is the honest "last ran" value if [ "$last_s" -eq 0 ]; then local exit_us exit_us=$(systemctl --user show "$unit.service" -p ExecMainExitTimestamp --value 2>/dev/null) [ -n "$exit_us" ] && [ "$exit_us" != "n/a" ] && last_s=$(date -d "$exit_us" +%s 2>/dev/null || echo 0) fi if [ "$next_s" -eq 0 ] && [ "$last_s" -eq 0 ]; then flag 2 "$label" "$unit.timer is not scheduled — enable the timer or check that the unit exists" TIMER_INFO+=("$label$SEP$unit$SEP${R}not scheduled${N}$SEP—$SEP—$SEP$SEP$ER") printf '%s %s%-8s%s %s timer not scheduled\n' "$ER" "$LBL" "$label" "$N" "$unit"; return fi if [ "$last_s" -eq 0 ]; then next_str=$(date -d "@$next_s" +'%b %-d' 2>/dev/null || echo '?') TIMER_INFO+=("$label$SEP$unit${SEP}armed, never fired$SEP—$SEP$next_str$SEP$SEP$OK") printf '%s %s%-8s%s %sarmed %s· next %s%s\n' "$OK" "$LBL" "$label" "$N" "${disp:-$unit }" \ "$D" "$next_str" "$N" return fi last_age=$(( (now_s - last_s) / 3600 )) if [ "$last_age" -ge 48 ]; then age_str="$(( last_age / 24 ))d"; else age_str="${last_age}h"; fi if [ "$next_s" -eq 0 ]; then next_str="…" # e.g. monotonic timer whose service is running right now elif [ $(( next_s - now_s )) -lt 86400 ]; then next_str=$(date -d "@$next_s" +%H:%M) else next_str=$(date -d "@$next_s" +'%b %-d' 2>/dev/null || echo "?") fi # auto staleness: the timer's own period (next - last) + 50% margin if [ "$stale_h" = auto ]; then local period_h if [ "$next_s" -gt 0 ] && [ "$last_s" -gt 0 ] && [ "$next_s" -gt "$last_s" ]; then period_h=$(( (next_s - last_s) / 3600 )) stale_h=$(( period_h + period_h / 2 )) [ "$stale_h" -lt 2 ] && stale_h=2 else stale_h=48 fi fi local status_str="${G}ok${N}" note_plain="" if [ "$result" != "success" ]; then glyph=$ER; rword="${R}FAILED${N}"; note=" ${R}($result)${N}" status_str="${R}FAILED${N} ${D}($result)${N}" flag 2 "$label" "last run of $unit.service failed ${D}(result: $result, $age_str ago)${N}" elif [ "$last_age" -gt "$stale_h" ]; then glyph=$WA; note=" ${Y}stale${N}" status_str="${Y}stale${N}"; note_plain="allowed ${stale_h}h between runs" flag 1 "$label" "last successful run of $unit was $age_str ago ${D}(stale after ${stale_h}h)${N}" fi TIMER_INFO+=("$label$SEP$unit$SEP$status_str$SEP$age_str ago$SEP$next_str$SEP$note_plain$SEP$glyph") printf '%s %s%-8s%s %s%b %s%s %s· next %s%s\n' \ "$glyph" "$LBL" "$label" "$N" "$disp" "$rword" "$age_str" "$note" "$D" "$next_str" "$N" } # ---------- left column: timeshift snapshots via grub-btrfs.cfg ---------- # grub-btrfsd rewrites the cfg on every snapshot event, so a fresh entry # proves both timeshift and grub-btrfsd in one check. snapshot_line() { local newest count age_h glyph=$OK note="" cnote="" if [ ! -r "$GRUB_BTRFS_CFG" ]; then SNAP_STATUS=missing-cfg printf '%s %s%-8s%s %s(no grub-btrfs.cfg at %s)%s\n' "$WA" "$LBL" "snapshot" "$N" "$D" "$GRUB_BTRFS_CFG" "$N" flag 1 snapshot "grub-btrfs.cfg is missing or unreadable at $GRUB_BTRFS_CFG — timeshift snapshots cannot be verified" return fi newest=$(grep -oE 'timeshift-btrfs/snapshots/[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}' \ "$GRUB_BTRFS_CFG" 2>/dev/null | sed 's#.*/##' | sort -u | tail -1) count=$(grep -oE 'timeshift-btrfs/snapshots/[0-9_-]+' "$GRUB_BTRFS_CFG" 2>/dev/null | sort -u | wc -l) SNAP_NEWEST=$newest; SNAP_COUNT=$count if [ -z "$newest" ]; then SNAP_STATUS=none flag 2 snapshot "no timeshift snapshots are listed in $GRUB_BTRFS_CFG" printf '%s %s%-8s%s no snapshots in grub-btrfs.cfg\n' "$ER" "$LBL" "snapshot" "$N"; return fi local snap_date=${newest%%_*} snap_time=${newest#*_} snap_s snap_time=${snap_time//-/:} snap_s=$(date -d "$snap_date $snap_time" +%s 2>/dev/null) [ -n "$snap_s" ] || snap_s=$now_s age_h=$(( (now_s - snap_s) / 3600 )) SNAP_AGE_H=$age_h; SNAP_STATUS=ok if [ "$age_h" -gt 24 ]; then glyph=$ER; note=" ${R}STALE${N}"; SNAP_STATUS=stale flag 2 snapshot "newest timeshift snapshot is ${age_h}h old ${D}(expected within 24h)${N}" elif [ "$age_h" -gt 3 ]; then glyph=$WA; note=" ${Y}late${N}"; SNAP_STATUS=late flag 1 snapshot "newest timeshift snapshot is ${age_h}h old ${D}(expected within 3h)${N}" fi if [ "$count" -gt "$SNAPSHOT_MAX" ]; then cnote=" ${Y}(pruning?)${N}"; [ "$glyph" = "$OK" ] && glyph=$WA flag 1 snapshot "$count snapshots are kept, above the $SNAPSHOT_MAX limit — check timeshift pruning" fi printf '%s %s%-8s%s timeshift %sh%s %s· %s kept%s%b\n' \ "$glyph" "$LBL" "snapshot" "$N" "$age_h" "$note" "$D" "$count" "$N" "$cnote" } # ---------- right column: user jobs (timers + failed units) ---------- list_failed_units() { if [ "$1" = user ]; then systemctl --user --failed --no-legend --plain 2>/dev/null | awk '{print $1}' else systemctl --failed --no-legend --plain 2>/dev/null | awk '{print $1}' fi } # A failed automount can be a retained boot-race result even though its paired # mount later recovered. Only soften it when the mount is currently mounted and # reports success; an unavailable mount remains a real failure. recovered_automount() { local scope=$1 unit=$2 mount_unit props active sub result local -a ctl case "$unit" in *.automount) ;; *) return 1 ;; esac mount_unit=${unit%.automount}.mount if [ "$scope" = user ]; then ctl=(systemctl --user); else ctl=(systemctl); fi props=$("${ctl[@]}" show "$mount_unit" -p ActiveState -p SubState -p Result 2>/dev/null) active=$(awk -F= '$1=="ActiveState"{print $2; exit}' <<<"$props") sub=$(awk -F= '$1=="SubState"{print $2; exit}' <<<"$props") result=$(awk -F= '$1=="Result"{print $2; exit}' <<<"$props") [ "$active" = active ] && [ "$sub" = mounted ] && [ "$result" = success ] || return 1 printf '%s\t%s\t%s\t%s\n' "$mount_unit" "$active" "$sub" "$result" } jobs_line() { local timers failed_user=0 failed_sys=0 retained_user=0 retained_sys=0 local bad="" svc res glyph=$OK scope unit recovery failed_total retained_total fword timers=$(systemctl --user list-timers --no-legend 2>/dev/null | grep -c .) for scope in user system; do while read -r unit; do [ -n "$unit" ] || continue if recovery=$(recovered_automount "$scope" "$unit"); then if [ "$scope" = user ]; then retained_user=$((retained_user + 1)) else retained_sys=$((retained_sys + 1)) fi elif [ "$scope" = user ]; then failed_user=$((failed_user + 1)) else failed_sys=$((failed_sys + 1)) fi done < <(list_failed_units "$scope") done while read -r svc; do [ -n "$svc" ] || continue res=$(systemctl --user show "$svc" -p Result --value 2>/dev/null) [ "$res" = "success" ] || [ -z "$res" ] || bad="$bad ${svc%.service}" done < <(systemctl --user list-timers --no-legend 2>/dev/null | awk '{print $NF}') failed_total=$((failed_user + failed_sys)) retained_total=$((retained_user + retained_sys)) if [ "$failed_total" -gt 0 ]; then glyph=$ER flag 2 jobs "$failed_total failed systemd unit(s) ${D}($failed_user user, $failed_sys system)${N}" fword="${R}$failed_total failed${N} (${failed_user}u/${failed_sys}s)" [ "$retained_total" -gt 0 ] \ && fword="$fword ${D}·${N} ${Y}$retained_total retained${N} (${retained_user}u/${retained_sys}s)" elif [ "$retained_total" -gt 0 ]; then glyph=$WA fword="${Y}$retained_total retained${N} (${retained_user}u/${retained_sys}s)" else fword="0 failed units" fi [ "$retained_total" -gt 0 ] && flag 1 jobs \ "$retained_total automount unit(s) still show an old failure although the mount recovered ${D}($retained_user user, $retained_sys system)${N}" if [ -n "$bad" ]; then [ "$glyph" = "$OK" ] && glyph=$WA flag 1 jobs "timer job(s) whose last run did not succeed:${bad}" fi printf '%s %s%-8s%s %s timers %s·%s %b%s\n' \ "$glyph" "$LBL" "jobs" "$N" "$timers" "$D" "$N" "$fword" \ "$([ -n "$bad" ] && printf ' %s· fail:%s%s' "$Y" "$bad" "$N")" } # ---------- popup details: failed units and timer jobs ---------- declare -A DETAIL_SEEN=() detail_count=0 detail_failed_count=0 detail_retained_count=0 detail_unit() { local scope=$1 unit=$2 source=$3 kind=${4:-failed} local props description result state since code status line logs header_color=$R recovery local recovered_unit recovered_active recovered_sub recovered_result local -a ctl journal [ -n "$unit" ] || return [ -z "${DETAIL_SEEN[$scope:$unit]:-}" ] || return DETAIL_SEEN[$scope:$unit]=1 detail_count=$((detail_count + 1)) if [ "$kind" = retained ]; then detail_retained_count=$((detail_retained_count + 1)) header_color=$Y else detail_failed_count=$((detail_failed_count + 1)) fi if [ "$scope" = user ]; then ctl=(systemctl --user) journal=(journalctl --user -u "$unit") else ctl=(systemctl) journal=(journalctl -u "$unit") fi props=$("${ctl[@]}" show "$unit" \ -p Description -p Result -p ActiveState -p SubState \ -p StateChangeTimestamp -p ExecMainCode -p ExecMainStatus 2>/dev/null) description=$(awk -F= '$1=="Description"{sub(/^[^=]*=/, ""); print; exit}' <<<"$props") result=$(awk -F= '$1=="Result"{print $2; exit}' <<<"$props") state=$(awk -F= '$1=="ActiveState"{a=$2} $1=="SubState"{s=$2} END{print a "/" s}' <<<"$props") since=$(awk -F= '$1=="StateChangeTimestamp"{sub(/^[^=]*=/, ""); print; exit}' <<<"$props") code=$(awk -F= '$1=="ExecMainCode"{print $2; exit}' <<<"$props") status=$(awk -F= '$1=="ExecMainStatus"{print $2; exit}' <<<"$props") local glyph=$ER [ "$kind" = retained ] && glyph=$WA printf '%s%s %s%s%s %s[%s · %s]%s\n' "$M" "$glyph" "$header_color" "$unit" "$N" "$D" "$scope" "$source" "$N" [ -n "$description" ] && printf '%s%s\n' "$IND" "$description" printf '%sstate %s%s%s · result %s%s%s' \ "$IND" "$Y" "${state:-unknown}" "$N" "$R" "${result:-unknown}" "$N" [ -n "$code$status" ] && printf ' · exit %s/%s' "${code:-?}" "${status:-?}" [ -n "$since" ] && printf ' · since %s' "$since" printf '\n' if [ "$kind" = retained ] && recovery=$(recovered_automount "$scope" "$unit"); then IFS=$'\t' read -r recovered_unit recovered_active recovered_sub recovered_result <<<"$recovery" printf '%s%sRecovered:%s %s is %s/%s with result %s.\n' \ "$IND" "$G" "$N" "$recovered_unit" "$recovered_active" "$recovered_sub" "$recovered_result" printf '%sThe resource is available; systemd is retaining the earlier automount failure.\n' "$IND" fi logs=$({ # Preserve high-signal resource errors even when later shutdown noise would # otherwise push the original cause out of the short excerpt. "${journal[@]}" -n 160 --no-pager -o cat 2>/dev/null \ | grep -Eai 'quota|no space|read-only file system|out of memory' | tail -n 2 "${journal[@]}" -n 120 --no-pager -o cat 2>/dev/null \ | grep -Eai 'error|fail|quota|assert|core|abrt|segmentation|killed|status=' | tail -n 3 } | awk 'NF && !seen[$0]++' | cut -c1-240) if [ -z "$logs" ]; then logs=$("${journal[@]}" -n 3 --no-pager -o cat 2>/dev/null \ | awk 'NF' | cut -c1-240) fi while IFS= read -r line; do [ -n "$line" ] && printf '%s%s%s%s\n' "$IND" "$D" "$line" "$N" done <<<"$logs" printf '\n' } failed_unit_details() { local scope=$1 unit recovery while read -r unit; do if recovery=$(recovered_automount "$scope" "$unit"); then detail_unit "$scope" "$unit" "retained failed state" retained else detail_unit "$scope" "$unit" "failed unit" fi done < <(list_failed_units "$scope") } timer_failure_details() { local service result while read -r service; do [ -n "$service" ] || continue result=$(systemctl --user show "$service" -p Result --value 2>/dev/null) [ -z "$result" ] || [ "$result" = success ] || detail_unit user "$service" "timer job" done < <(systemctl --user list-timers --all --no-legend 2>/dev/null | awk '{print $NF}') } # ---------- popup details: layout helpers ---------- M=' ' # left margin: keeps text off the popup border IND="$M " # indented continuation lines under an item RULE_W=76 # section header rule width (popup is at most 110 cols) section() { local title=$1 fill fill=$(( RULE_W - ${#title} - 1 )) [ "$fill" -lt 4 ] && fill=4 printf '\n%s%s%s%s %s' "$M" "$B" "$title" "$N" "$D" printf '─%.0s' $(seq 1 "$fill") printf '%s\n\n' "$N" } # item GLYPH TEXT: one bullet line at the margin item() { printf '%s%s %s\n' "$M" "$1" "$2"; } # sub TEXT: continuation line under an item sub() { printf '%s%s\n' "$IND" "$1"; } # hint TEXT: dim explanatory continuation line hint() { printf '%s%s%s%s\n' "$IND" "$D" "$1" "$N"; } glyph_for() { case "$1" in 2) printf '%s' "$ER" ;; 1) printf '%s' "$WA" ;; *) printf '%s' "$OK" ;; esac; } # The summary at the top of the popup: exactly the problems that produced the # dashboard verdict, worst first, tagged with the row they came from. flagged_summary() { local crit=0 warn=0 entry lvl sect msg verdict for entry in ${FINDINGS[@]+"${FINDINGS[@]}"}; do IFS=$'\t' read -r lvl _ _ <<<"$entry" [ "$lvl" -eq 2 ] && crit=$((crit + 1)) || warn=$((warn + 1)) done case $worst in 0) verdict="${G}●${N} ${G}all systems healthy${N}" ;; 1) verdict="${Y}●${N} ${Y}attention needed${N}" ;; *) verdict="${R}●${N} ${R}problems detected${N}" ;; esac printf '\n%s%s%s%s %s·%s %s' "$M" "$B" "$TITLE" "$N" "$D" "$N" "$verdict" if [ "$worst" -gt 0 ]; then printf ' %s·%s ' "$D" "$N" [ "$crit" -gt 0 ] && printf '%s%s critical%s' "$R" "$crit" "$N" [ "$crit" -gt 0 ] && [ "$warn" -gt 0 ] && printf ', ' [ "$warn" -gt 0 ] && printf '%s%s warning%s%s' "$Y" "$warn" "$([ "$warn" -eq 1 ] || printf s)" "$N" fi printf '\n' section "Flagged" if [ ${#FINDINGS[@]} -eq 0 ]; then item "$OK" "Nothing is flagged — every check passed." return fi for lvl in 2 1; do for entry in ${FINDINGS[@]+"${FINDINGS[@]}"}; do IFS=$'\t' read -r l sect msg <<<"$entry" [ "$l" -eq "$lvl" ] || continue printf '%s%s %s%-9s%s %s\n' "$M" "$(glyph_for "$lvl")" "$LBL" "$sect" "$N" "$msg" done done printf '\n' hint "$ER critical raises the verdict to \"problems detected\"; $WA warning raises it to \"attention needed\"." hint "Each entry names the dashboard row it came from. Sections below give the underlying facts." } unit_details() { section "Units and timer jobs" failed_unit_details user failed_unit_details system timer_failure_details if [ "$detail_count" -eq 0 ]; then item "$OK" "No failed user/system units or unsuccessful timer jobs." else [ "$detail_retained_count" -gt 0 ] && hint "Retained states are read-only here; reset them only after confirming recovery." [ "$detail_failed_count" -gt 0 ] && hint "Failed states stay visible until the unit succeeds or is reset." fi } system_details() { local entry pid comm ppid pcomm kind k stray=0 benign=0 g section "System" for entry in ${ZOMBIES[@]+"${ZOMBIES[@]}"}; do IFS=$'\t' read -r _ _ _ _ kind <<<"$entry" [ "$kind" = stray ] && stray=$((stray + 1)) || benign=$((benign + 1)) done if [ "$stray" -gt 10 ]; then g=$ER; elif [ "$stray" -gt 0 ]; then g=$WA; else g=$OK; fi local zword="zombie processes"; [ "$stray" -eq 1 ] && zword="zombie process" item "$g" "load ${SYS_LOAD} on ${SYS_CORES} cores ${D}·${N} memory ${SYS_MEM}% in use ${D}·${N} $stray $zword ${D}(+$benign ignored as benign)${N}" if [ ${#ZOMBIES[@]} -gt 0 ]; then printf '\n' for kind in stray benign; do for entry in ${ZOMBIES[@]+"${ZOMBIES[@]}"}; do IFS=$'\t' read -r pid comm ppid pcomm k <<<"$entry" [ "$k" = "$kind" ] || continue if [ "$kind" = stray ]; then item "$WA" "$comm ${D}(pid $pid)${N} — parent $pcomm ${D}(pid $ppid)${N}" else item "${D}·${N}" "${D}$comm (pid $pid) — parent $pcomm (pid $ppid) — benign, ignored${N}" fi done done fi if [ "$stray" -gt 0 ]; then printf '\n' hint "A zombie has already exited; it lingers only because its parent has not reaped it." hint "Restart or stop the parent process to clear it, or ignore it if the parent is short-lived." hint "Use --benign-zombie NAME to silence a process name that always leaves harmless zombies." fi } snapshot_details() { local spec want=0 for spec in ${LEFT_SPEC[@]+"${LEFT_SPEC[@]}"}; do [ "$spec" = snapshots ] && want=1; done [ "$want" -eq 1 ] || return 0 section "Timeshift snapshots" case "$SNAP_STATUS" in missing-cfg) item "$WA" "$GRUB_BTRFS_CFG is missing or not readable by this user." hint "grub-btrfsd rewrites this file after every timeshift snapshot, so it doubles as proof that" hint "both timeshift and grub-btrfsd are working. Without it, snapshot age cannot be verified." hint "If it never existed: install/enable grub-btrfs (grub-btrfsd.service). If it exists: check read" hint "permission on /boot/grub, or point --grub-btrfs-cfg at the right path." ;; none) item "$ER" "No timeshift snapshots are listed in $GRUB_BTRFS_CFG." hint "Timeshift has not created a btrfs snapshot that grub-btrfsd picked up. Check timeshift's schedule." ;; stale) item "$ER" "Newest snapshot ${SNAP_NEWEST} is ${SNAP_AGE_H}h old ${D}(expected within 24h)${N} · ${SNAP_COUNT} kept ${D}(limit ${SNAPSHOT_MAX})${N}" ;; late) item "$WA" "Newest snapshot ${SNAP_NEWEST} is ${SNAP_AGE_H}h old ${D}(expected within 3h)${N} · ${SNAP_COUNT} kept ${D}(limit ${SNAPSHOT_MAX})${N}" ;; ok) item "$OK" "Newest snapshot ${SNAP_NEWEST} is ${SNAP_AGE_H}h old · ${SNAP_COUNT} kept ${D}(limit ${SNAPSHOT_MAX})${N}" ;; *) item "$WA" "Snapshot state was not evaluated." ;; esac [ "$SNAP_COUNT" -gt "$SNAPSHOT_MAX" ] \ && hint "More snapshots than the configured limit are kept — timeshift pruning may not be running." } timer_details() { [ ${#TIMER_INFO[@]} -gt 0 ] || return 0 local entry label unit status last next note g section "Scheduled jobs" for entry in ${TIMER_INFO[@]+"${TIMER_INFO[@]}"}; do IFS=$SEP read -r label unit status last next note g <<<"$entry" printf '%s%s %s%-9s%s %s %s·%s %s %s·%s last %s %s·%s next %s%s\n' \ "$M" "${g:-$OK}" "$LBL" "$label" "$N" "$unit" "$D" "$N" "$status" "$D" "$N" "$last" \ "$D" "$N" "$next" "${note:+ ${D}($note)${N}}" done } details_main() { # Replay the dashboard checks silently so FINDINGS and the per-section # facts are populated exactly as the verdict saw them. local spec t_label t_unit t_stale t_name for spec in ${LEFT_SPEC[@]+"${LEFT_SPEC[@]}"}; do case "$spec" in timer:*) IFS=: read -r _ t_label t_unit t_stale t_name <<<"$spec" timer_line "$t_label" "$t_unit" "$t_stale" "${t_name-}" >/dev/null ;; snapshots) snapshot_line >/dev/null ;; esac done system_line >/dev/null jobs_line >/dev/null storage_line >/dev/null btrfs_line >/dev/null flagged_summary unit_details system_details snapshot_details timer_details storage_details btrfs_details printf '\n' } # ---------- right column: storage (real filesystems, exclusions applied) ---------- storage_records() { local dev size used avail pct mnt seen="" skip ex while read -r dev size used avail pct mnt; do case "$dev" in /dev/*) ;; *) continue ;; esac skip=0 for ex in ${EXCLUDE_MOUNTS[@]+"${EXCLUDE_MOUNTS[@]}"}; do case "$mnt" in "$ex"*) skip=1; break ;; esac done [ "$skip" = 1 ] && continue case " $seen " in *" $dev "*) continue ;; esac # dedupe subvol mounts seen="$seen $dev" pct=${pct%\%} is_uint "$pct" || continue printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$dev" "$size" "$used" "$avail" "$pct" "$mnt" done < <(df -B1 --output=source,size,used,avail,pcent,target 2>/dev/null | tail -n +2) } storage_line() { local out="" glyph=$OK dev size used avail pct mnt name lvl c while IFS=$'\t' read -r dev size used avail pct mnt; do name=$(basename "$mnt"); [ "$mnt" = "/" ] && name=root if [ "$pct" -ge "$DISK_CRIT" ]; then c=$R; lvl=2 elif [ "$pct" -ge "$DISK_WARN" ]; then c=$Y; lvl=1 else c=$G; lvl=0; fi [ $lvl -gt 0 ] && flag $lvl storage \ "$mnt is ${pct}% full ${D}($(human_bytes "$avail") free of $(human_bytes "$size"); thresholds ${DISK_WARN}% / ${DISK_CRIT}%)${N}" [ $lvl -eq 2 ] && glyph=$ER; [ $lvl -eq 1 ] && [ "$glyph" != "$ER" ] && glyph=$WA out="$out${out:+ ${D}·${N} }$name ${c}${pct}%${N}" done < <(storage_records) [ -n "$out" ] || out="${D}no local filesystems found${N}" printf '%s %s%-8s%s %b\n' "$glyph" "$LBL" "storage" "$N" "$out" } largest_directories() { local mnt=$1 scan scan_status bytes path shown=0 heading if ! command -v timeout >/dev/null 2>&1; then hint "Largest-directory scan unavailable (timeout is not installed)." return fi scan=$(timeout 3s du -x -B1 --max-depth=1 -- "$mnt" 2>/dev/null) scan_status=$? if [ "$scan_status" -eq 0 ]; then heading='Largest child directories:' else heading='Largest readable child directories found before the scan stopped:' fi [ -n "$scan" ] && hint "$heading" while IFS=$'\t' read -r bytes path; do [ -n "$bytes" ] && [ -n "$path" ] && [ "$path" != "$mnt" ] || continue sub "$path — $(human_bytes "$bytes")" shown=$((shown + 1)) done < <(sort -nr -k1,1 <<<"$scan" | head -n 5) [ "$shown" -gt 0 ] || hint "No readable child-directory sizes were returned." if [ "$scan_status" -eq 124 ]; then hint "Directory scan stopped after 3 seconds; the list is partial." elif [ "$scan_status" -ne 0 ]; then hint "Directory scan was incomplete (exit $scan_status); unreadable paths were skipped." fi } storage_details() { local dev size used avail pct mnt found=0 over=0 c g section "Storage" while IFS=$'\t' read -r dev size used avail pct mnt; do found=$((found + 1)) if [ "$pct" -ge "$DISK_CRIT" ]; then c=$R; g=$ER; over=$((over + 1)) elif [ "$pct" -ge "$DISK_WARN" ]; then c=$Y; g=$WA; over=$((over + 1)) else c=$G; g=$OK; fi printf '%s%s %s — %s%s%%%s used (%s / %s, %s free)\n' \ "$M" "$g" "$mnt" "$c" "$pct" "$N" \ "$(human_bytes "$used")" "$(human_bytes "$size")" "$(human_bytes "$avail")" if [ "$pct" -ge "$DISK_WARN" ]; then largest_directories "$mnt" printf '\n' fi done < <(storage_records) if [ "$found" -eq 0 ]; then item "$WA" "No local filesystems found." elif [ "$over" -eq 0 ]; then printf '\n' hint "All filesystems are below the ${DISK_WARN}% warning threshold (critical at ${DISK_CRIT}%)." fi } # ---------- right column: system (zombies, load, memory) ---------- system_line() { local z=0 benign=0 zpid zppid zcomm pcomm load cores memp glyph=$OK zc=$G note="" bz match kind stray="" while read -r zpid zppid zcomm; do match=0 for bz in ${BENIGN_ZOMBIES[@]+"${BENIGN_ZOMBIES[@]}"}; do [ "$zcomm" = "$bz" ] && { match=1; break; } done # zombies inside docker containers: PID 1 without an init never reaps if [ "$match" = 0 ] && grep -q ':/system.slice/docker-' "/proc/$zpid/cgroup" 2>/dev/null; then match=1 fi if [ "$match" = 1 ]; then benign=$((benign+1)); kind=benign else z=$((z+1)); kind=stray; stray="$stray${stray:+, }$zcomm ${D}(pid $zpid)${N}"; fi pcomm=$(ps -o comm= -p "$zppid" 2>/dev/null || true) ZOMBIES+=("$zpid"$'\t'"$zcomm"$'\t'"$zppid"$'\t'"${pcomm:-?}"$'\t'"$kind") done < <(ps -eo pid=,ppid=,stat=,comm= 2>/dev/null | awk '$3 ~ /^Z/ {print $1, $2, $4}') load=$(cut -d' ' -f1 /proc/loadavg 2>/dev/null || echo '?') cores=$(nproc 2>/dev/null || echo '?') memp=$(awk '/MemTotal/{t=$2} /MemAvailable/{a=$2} END{if (t>0) printf "%d", (t-a)*100/t}' /proc/meminfo 2>/dev/null) SYS_LOAD=$load; SYS_CORES=$cores; SYS_MEM=${memp:-?} if [ "$z" -gt 10 ]; then glyph=$ER; zc=$R; flag 2 system "$z zombie processes: $stray" elif [ "$z" -gt 1 ]; then glyph=$WA; zc=$Y; flag 1 system "$z zombie processes: $stray" elif [ "$z" -eq 1 ]; then glyph=$WA; zc=$Y; flag 1 system "1 zombie process: $stray"; fi [ "$benign" -gt 0 ] && note=" ${D}(+$benign)${N}" printf '%s %s%-8s%s %b%s zombies%b%b %s·%s load %s/%s %s·%s mem %s%%\n' \ "$glyph" "$LBL" "system" "$N" "$zc" "$z" "$N" "$note" "$D" "$N" "$load" "$cores" "$D" "$N" "${memp:-?}" } # ---------- bottom row: btrfs maintenance triad + error counters ---------- btrfs_allocation_info() { local mnt=$1 usage metrics size unallocated free pct used_pct state=unknown if ! usage=$(btrfs filesystem usage -b "$mnt" 2>/dev/null); then printf 'unknown\t0\t0\t0\t0\t0\n' return fi metrics=$(awk ' /Device size:/{size=$NF} /Device unallocated:/{unallocated=$NF} /Free \(estimated\):/{estimated=$NF} /Free \(statfs, df\):/{statfs=$NF} END { free = statfs != "" ? statfs : estimated if (size > 0 && unallocated != "") printf "%.0f\t%.0f\t%.0f\t%d", size, unallocated, free + 0, unallocated * 100 / size }' <<<"$usage") [ -n "$metrics" ] || { printf 'unknown\t0\t0\t0\t0\t0\n'; return; } IFS=$'\t' read -r size unallocated free pct <<<"$metrics" used_pct=$(df --output=pcent -- "$mnt" 2>/dev/null | tail -n 1) used_pct=${used_pct//[!0-9]/} if ! is_uint "$used_pct"; then used_pct=$(( size > 0 ? (size - free) * 100 / size : 0 )) fi if [ "$pct" -ge 15 ]; then state=ok elif [ "$used_pct" -ge "$DISK_WARN" ]; then state=capacity elif [ "$pct" -lt 8 ]; then state=critical else state=warn fi printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$state" "$pct" "$used_pct" "$size" "$unallocated" "$free" } btrfs_line() { [ "$NO_BTRFS" = 1 ] && return 0 command -v btrfs >/dev/null 2>&1 || return 0 local fs_list glyph=$OK errs=0 e scrub="" last_us days res c lvl=0 mnt label esc local stats io_unknown=0 allocation_seen=0 fs_list=$(btrfs_filesystems) [ -n "$fs_list" ] || return 0 # no btrfs on this host: omit the row entirely while IFS=: read -r mnt label esc smnt; do # stats via any accessible mountpoint of the device (smnt), not necessarily # the labeled one (which may be root-only, e.g. /var/lib/docker) if [ -n "$smnt" ] && stats=$(btrfs device stats "$smnt" 2>/dev/null); then e=$(awk '{s+=$NF} END{print s+0}' <<<"$stats") errs=$((errs + e)) else io_unknown=1 fi last_us=$(systemctl show "btrfs-scrub@$esc.timer" -p LastTriggerUSec --value 2>/dev/null) if [ -z "$last_us" ] || [ "$last_us" = "n/a" ]; then scrub="$scrub${scrub:+ }${D}$label —${N}" # no scrub timer: visible, not alarming continue fi days=$(age_days "$last_us") res=$(systemctl show "btrfs-scrub@$esc.service" -p Result --value 2>/dev/null) c=$G if [ "$res" != "success" ] && [ -n "$res" ]; then c=$R; lvl=2; flag 2 btrfs "scrub of $mnt failed ${D}(result: $res)${N}" elif [ "$days" -gt 40 ]; then c=$Y; lvl=1; flag 1 btrfs "scrub of $mnt last ran ${days}d ago ${D}(expected within 40d)${N}" fi scrub="$scrub${scrub:+ }$label ${c}${days}d${N}" done <<<"$fs_list" # trim: weekly fstrim.timer (discard=async may run inline too; this is the sweep) local trim="${D}—${N}" last_us=$(systemctl show fstrim.timer -p LastTriggerUSec --value 2>/dev/null) if [ -n "$last_us" ] && [ "$last_us" != "n/a" ]; then days=$(age_days "$last_us") res=$(systemctl show fstrim.service -p Result --value 2>/dev/null) c=$G if [ "$res" != "success" ] && [ -n "$res" ]; then c=$R; lvl=2; flag 2 btrfs "fstrim failed ${D}(result: $res)${N}" elif [ "$days" -gt 10 ]; then c=$Y; lvl=$((lvl > 1 ? lvl : 1)); flag 1 btrfs "fstrim last ran ${days}d ago ${D}(expected weekly)${N}" fi trim="${c}${days}d${N}" fi # Low unallocated device space only suggests balance while the filesystem # still has ordinary free space. On a nearly full filesystem it is a symptom # of capacity pressure already reported by the storage row. local alloc="${G}ok${N}" action_pct=101 action_fs="" action_lvl=0 local capacity_pct=101 capacity_fs="" unknown=0 info state pct used_pct size unallocated free while IFS=: read -r mnt label esc smnt; do [ -n "$smnt" ] || smnt=$mnt info=$(btrfs_allocation_info "$smnt") IFS=$'\t' read -r state pct used_pct size unallocated free <<<"$info" case "$state" in critical) allocation_seen=1 flag 2 btrfs "$mnt has only ${pct}% of its device unallocated ${D}($(human_bytes "$unallocated") of $(human_bytes "$size"), yet $(human_bytes "$free") free) — below the 8% critical mark${N}" if [ "$action_lvl" -lt 2 ] || [ "$pct" -lt "$action_pct" ]; then action_lvl=2; action_pct=$pct; action_fs=$label fi ;; warn) allocation_seen=1 flag 1 btrfs "$mnt has only ${pct}% of its device unallocated ${D}($(human_bytes "$unallocated") of $(human_bytes "$size"), yet $(human_bytes "$free") free) — below the 15% comfort mark${N}" if [ "$action_lvl" -eq 0 ] || { [ "$action_lvl" -eq 1 ] && [ "$pct" -lt "$action_pct" ]; }; then action_lvl=1; action_pct=$pct; action_fs=$label fi ;; capacity) allocation_seen=1 if [ "$pct" -lt "$capacity_pct" ]; then capacity_pct=$pct; capacity_fs=$label; fi ;; ok) allocation_seen=1 ;; *) unknown=1 ;; esac done <<<"$fs_list" if [ "$action_lvl" -eq 2 ]; then alloc="${R}$action_fs ${action_pct}% unalloc${N}"; lvl=2 elif [ "$action_lvl" -eq 1 ]; then alloc="${Y}$action_fs ${action_pct}% unalloc${N}"; lvl=$((lvl > 1 ? lvl : 1)) elif [ -n "$capacity_fs" ]; then alloc="${D}$capacity_fs capacity-bound${N}" elif [ "$allocation_seen" -eq 0 ] || [ "$unknown" -eq 1 ]; then alloc="${Y}unknown${N}"; lvl=$((lvl > 1 ? lvl : 1)) flag 1 btrfs "allocation state could not be read for at least one filesystem ${D}(btrfs filesystem usage failed)${N}" fi bump $lvl [ $lvl -eq 1 ] && glyph=$WA; [ $lvl -eq 2 ] && glyph=$ER if [ "$errs" -gt 0 ]; then glyph=$ER; flag 2 btrfs "$errs device I/O error(s) recorded across btrfs filesystems"; fi if [ "$io_unknown" -eq 1 ]; then [ "$glyph" = "$OK" ] && glyph=$WA flag 1 btrfs "device error counters could not be read for at least one filesystem ${D}(no accessible mountpoint)${N}" fi printf '%s %s%-8s%s scrub: %b %s·%s trim %b %s·%s alloc %b %s·%s %b io errors\n' \ "$glyph" "$LBL" "btrfs" "$N" "$scrub" "$D" "$N" "$trim" "$D" "$N" "$alloc" "$D" "$N" \ "$([ "$errs" -gt 0 ] && printf '%s%s%s' "$R" "$errs" "$N" || { [ "$io_unknown" -eq 1 ] && printf '%sunknown%s' "$Y" "$N" || printf '%s0%s' "$G" "$N"; })" } btrfs_details() { [ "$NO_BTRFS" = 1 ] && return 0 command -v btrfs >/dev/null 2>&1 || return 0 local fs_list mnt label esc smnt stats errors info state pct used_pct size unallocated free fs_list=$(btrfs_filesystems) [ -n "$fs_list" ] || return 0 local g pc ec explain=0 section "Btrfs allocation and I/O" while IFS=: read -r mnt label esc smnt; do [ -n "$smnt" ] || smnt=$mnt info=$(btrfs_allocation_info "$smnt") IFS=$'\t' read -r state pct used_pct size unallocated free <<<"$info" if stats=$(btrfs device stats "$smnt" 2>/dev/null); then errors=$(awk '{sum+=$NF} END{print sum+0}' <<<"$stats") else errors=unknown fi case "$state" in critical) g=$ER; pc=$R ;; warn|capacity|unknown) g=$WA; pc=$Y ;; *) g=$OK; pc=$G ;; esac if [ "$errors" = unknown ]; then ec=$Y; [ "$g" = "$OK" ] && g=$WA elif [ "$errors" != 0 ]; then ec=$R; g=$ER else ec=$G; fi printf '%s%s %s — %s%% used %s·%s %s%s%% unallocated%s %s(%s of %s)%s %s·%s %s free %s·%s I/O errors: %s%s%s\n' \ "$M" "$g" "$mnt" "$used_pct" "$D" "$N" "$pc" "$pct" "$N" \ "$D" "$(human_bytes "$unallocated")" "$(human_bytes "$size")" "$N" \ "$D" "$N" "$(human_bytes "$free")" "$D" "$N" "$ec" "$errors" "$N" case "$state" in capacity) sub "Low unallocated device space is a consequence of filesystem capacity pressure." sub "Delete or move data first; balancing does not create free space." ;; critical|warn) explain=1 sub "Low unallocated device space despite ordinary free space suggests sparse chunks." sub "Inspect chunk usage before considering a targeted balance." hint "Inspect: sudo btrfs filesystem usage $mnt" hint "Reclaim: sudo btrfs balance start -dusage=50 $mnt (compacts half-empty data chunks)" ;; unknown) sub "${Y}Allocation details could not be read.${N}" ;; esac [ "$errors" != 0 ] && [ "$errors" != unknown ] \ && sub "${R}Nonzero device error counters require investigation.${N}" [ "$errors" = unknown ] && sub "${Y}Device error counters could not be read.${N}" # healthy entries stay compact; only annotated ones get breathing room [ "$state" = ok ] && [ "$errors" = 0 ] || printf '\n' done <<<"$fs_list" if [ "$explain" -eq 1 ]; then hint "\"Unallocated\" is raw device space not yet assigned to any data/metadata chunk. Btrfs needs it" hint "to grow metadata or create new chunks; when it runs out, writes can fail with ENOSPC even" hint "though df still reports free space. Below 15% is a warning, below 8% is critical." fi } # ---------- assembly ---------- main() { local left=() right=() i l r spec t_label t_unit t_stale t_name rows tmp=$(mktemp -d) || die "mktemp failed" # global on purpose: EXIT trap outlives main trap 'rm -rf "$tmp"' EXIT # left column in flag order (per-line redirection: no subshells, $worst survives) i=0 for spec in ${LEFT_SPEC[@]+"${LEFT_SPEC[@]}"}; do case "$spec" in timer:*) IFS=: read -r _ t_label t_unit t_stale t_name <<<"$spec" timer_line "$t_label" "$t_unit" "$t_stale" "${t_name-}" > "$tmp/left.$i" ;; snapshots) snapshot_line > "$tmp/left.$i" ;; esac left+=("$(cat "$tmp/left.$i")"); i=$((i+1)) done # right column: live state, top to bottom system_line > "$tmp/system" jobs_line > "$tmp/jobs" storage_line > "$tmp/storage" right=("$(cat "$tmp/system")" "$(cat "$tmp/jobs")" "$(cat "$tmp/storage")") btrfs_line > "$tmp/btrfs" # header: title + verdict on one line (widget config: title = "") case $worst in 0) echo "${G}●${N} ${B}${TITLE}${N} ${D}·${N} ${G}all systems healthy${N}" ;; 1) echo "${Y}●${N} ${B}${TITLE}${N} ${D}·${N} ${Y}attention needed${N}" ;; *) echo "${R}●${N} ${B}${TITLE}${N} ${D}·${N} ${R}problems detected${N}" ;; esac echo if [ "$SINGLE_COLUMN" = 1 ] || [ ${#left[@]} -eq 0 ]; then for l in ${left[@]+"${left[@]}"} "${right[@]}"; do printf '%s\n' "$l"; done else rows=${#left[@]} [ ${#right[@]} -gt "$rows" ] && rows=${#right[@]} for (( i=0; i