シェルスクリプトでプロセスを監視して自動再起動する方法｜kill -0チェックとwatchdog設計の実践

この記事のポイント

・kill -0 $PID でプロセスの存在を確認し、停止時に自動再起動できる
・再起動回数をファイルで上限管理し、連続失敗時はアラートを送って自動収束させる
・cron実行時はflock -n でwatchdog自身の二重起動を防ぐ設計が必須
・systemd Restart設定で代替できる場合は必ずそちらを優先する

# PID 12345 のプロセスが存在するか確認
kill -0 12345 2>/dev/null && echo "running" || echo "not running"

# スクリプト内での使い方
is_running() {
    local pid
    pid=$(cat "${PIDFILE}" 2>/dev/null) || return 1
    kill -0 "${pid}" 2>/dev/null
}

# プロセス名 myapp が存在するか確認
pgrep -x myapp > /dev/null 2>&1 && echo "running" || echo "not running"

# -x はプロセス名の完全一致。部分一致だと誤検知する
pgrep -f "myapp --daemon" > /dev/null 2>&1

# サービスが active かどうか確認
systemctl is-active --quiet myapp.service && echo "active" || echo "inactive"

# 終了コード: 0=active, 1～3=inactive/failed/activating

#!/bin/bash
# /usr/local/sbin/watchdog-myapp.sh
# myapp プロセスを監視して自動再起動する watchdog スクリプト

set -euo pipefail

PROCESS_NAME="myapp"
START_CMD="/opt/myapp/bin/myapp --daemon --config /etc/myapp/myapp.conf"
PIDFILE="/var/run/myapp.pid"
LOGFILE="/var/log/watchdog-myapp.log"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "${LOGFILE}"
}

is_running() {
    local pid
    pid=$(cat "${PIDFILE}" 2>/dev/null) || return 1
    kill -0 "${pid}" 2>/dev/null
}

if is_running; then
    log "OK: ${PROCESS_NAME} is running (PID=$(cat "${PIDFILE}"))"
    exit 0
fi

log "WARN: ${PROCESS_NAME} is not running. Restarting..."
${START_CMD}
log "INFO: ${PROCESS_NAME} restart command executed."

#!/bin/bash
# /usr/local/sbin/watchdog-myapp.sh（上限・アラート付き版）

set -euo pipefail

PROCESS_NAME="myapp"
START_CMD="/opt/myapp/bin/myapp --daemon --config /etc/myapp/myapp.conf"
PIDFILE="/var/run/myapp.pid"
LOGFILE="/var/log/watchdog-myapp.log"
COUNTER_FILE="/tmp/watchdog-myapp-restarts"
MAX_RESTARTS=3
ALERT_TO="ops-team@example.com"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOGFILE}"
}

send_alert() {
    local subject="$1"
    local body="$2"
    echo "${body}" | mail -s "${subject}" "${ALERT_TO}" 2>/dev/null || \
        log "WARN: mail command failed. Install mailx or configure sendmail."
}

is_running() {
    local pid
    pid=$(cat "${PIDFILE}" 2>/dev/null) || return 1
    kill -0 "${pid}" 2>/dev/null
}

if is_running; then
    # 正常稼働中 → カウンタをリセット
    echo 0 > "${COUNTER_FILE}"
    log "OK: ${PROCESS_NAME} is running (PID=$(cat "${PIDFILE}"))"
    exit 0
fi

# 再起動カウンタを読み取り、インクリメント
count=$(cat "${COUNTER_FILE}" 2>/dev/null || echo 0)
count=$(( count + 1 ))
echo "${count}" > "${COUNTER_FILE}"

# 上限超過 → アラート送信して終了
if (( count > MAX_RESTARTS )); then
    log "ERROR: ${PROCESS_NAME} has failed ${count} times. Giving up. Manual check required."
    send_alert \
        "[ALERT] ${PROCESS_NAME} on $(hostname) repeated failures" \
        "${PROCESS_NAME} stopped and failed to recover ${count} times on $(hostname). Please check immediately."
    exit 1
fi

log "WARN: ${PROCESS_NAME} stopped. Restart attempt ${count}/${MAX_RESTARTS}..."
${START_CMD}
log "INFO: ${PROCESS_NAME} restart command executed (attempt ${count})."

# 実際のログ出力例（/var/log/watchdog-myapp.log）
[2026-09-01 03:00:01] WARN: myapp stopped. Restart attempt 1/3...
[2026-09-01 03:00:03] INFO: myapp restart command executed (attempt 1).
[2026-09-01 03:05:01] OK: myapp is running (PID=18234)
[2026-09-01 03:10:01] OK: myapp is running (PID=18234)
[2026-09-01 03:15:01] WARN: myapp stopped. Restart attempt 1/3...
[2026-09-01 03:15:02] INFO: myapp restart command executed (attempt 1).
[2026-09-01 03:20:01] WARN: myapp stopped. Restart attempt 2/3...
[2026-09-01 03:20:02] INFO: myapp restart command executed (attempt 2).
[2026-09-01 03:25:01] WARN: myapp stopped. Restart attempt 3/3...
[2026-09-01 03:25:02] INFO: myapp restart command executed (attempt 3).
[2026-09-01 03:30:01] ERROR: myapp has failed 4 times. Giving up. Manual check required.

# /etc/cron.d/watchdog-myapp
# 5分ごとに watchdog を実行する

MAILTO="ops-team@example.com"
PATH=/usr/local/sbin:/usr/sbin:/sbin:/usr/local/bin:/usr/bin:/bin

*/5 * * * * root /usr/bin/flock -n /tmp/watchdog-myapp.lock /usr/local/sbin/watchdog-myapp.sh

#!/bin/bash
# スクリプト冒頭に PATH を明示（cron 環境では /usr/local/sbin 等が通っていないことが多い）
export PATH=/usr/local/sbin:/usr/sbin:/sbin:/usr/local/bin:/usr/bin:/bin

set -euo pipefail
# ... 以降 watchdog 本体

is_running() {
    local pid
    pid=$(cat "${PIDFILE}" 2>/dev/null) || return 1
    # プロセスが存在するか
    kill -0 "${pid}" 2>/dev/null || return 1
    # プロセス名も確認（PID再利用対策）
    local proc_name
    proc_name=$(ps -p "${pid}" -o comm= 2>/dev/null) || return 1
    [[ "${proc_name}" == "${PROCESS_NAME}" ]]
}

# watchdog スクリプトを手動でデバッグ実行する
bash -x /usr/local/sbin/watchdog-myapp.sh 2>&1 | head -50

# cron 経由での動作確認（run-parts 相当）
/usr/bin/flock -n /tmp/watchdog-myapp.lock /usr/local/sbin/watchdog-myapp.sh
echo "Exit code: $?"

# プロセス名の完全一致（デフォルト）
pgrep -x myapp

# コマンドライン全体でのパターンマッチ（-f オプション）
pgrep -f "myapp --daemon --config /etc/myapp/myapp.conf"