diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index 9d3e573..31363ac 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -66,5 +66,5 @@ jobs: level: warning fail_level: error reporter: github-pr-review - shellcheck_flags: -x -e SC2001 -e SC2034 -e SC2064 -e SC2317 -e SC2153 -e SC2028 + shellcheck_flags: -x -e SC1091 -e SC2001 -e SC2034 -e SC2064 -e SC2317 -e SC2153 -e SC2028 github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/src/config.sh b/src/config.sh index 10d51b5..401e566 100644 --- a/src/config.sh +++ b/src/config.sh @@ -3,6 +3,7 @@ set -Eeuo pipefail DEF_OPTS="-nodefaults -boot strict=on" RAM_OPTS=$(echo "-m ${RAM_SIZE^^}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g') +MON_OPTS="-name $PROCESS,process=$PROCESS,debug-threads=on -pidfile $QEMU_PID" CPU_OPTS="-cpu $CPU_FLAGS -smp $CPU_CORES,sockets=1,dies=1,cores=$CPU_CORES,threads=1" MAC_OPTS="-machine type=q35,smm=off,usb=off,vmport=off,dump-guest-core=off,hpet=off${KVM_OPTS}" DEV_OPTS="-device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x4" diff --git a/src/entry.sh b/src/entry.sh index 2154ce5..5f27a1c 100755 --- a/src/entry.sh +++ b/src/entry.sh @@ -24,19 +24,23 @@ cd /run trap - ERR -version=$(qemu-system-x86_64 --version | head -n 1 | cut -d '(' -f 1 | awk '{ print $NF }') +cmd=(qemu-system-x86_64) +version=$("${cmd[@]}" --version | awk 'NR==1 { print $4 }') info "Booting $APP using QEMU v$version..." -if [[ "$CONSOLE" == [Yy]* ]]; then - exec qemu-system-x86_64 ${ARGS:+ $ARGS} +if [[ "$SHUTDOWN" != [Yy1]* ]]; then + exec "${cmd[@]}" ${ARGS:+ $ARGS} fi -{ qemu-system-x86_64 ${ARGS:+ $ARGS} >"$QEMU_OUT" 2>"$QEMU_LOG"; rc=$?; } || : -(( rc != 0 )) && error "$(<"$QEMU_LOG")" && exit 15 +if [ ! -t 1 ] || [ ! -c /dev/tty ]; then + "${cmd[@]}" ${ARGS:+ $ARGS} & +else + "${cmd[@]}" ${ARGS:+ $ARGS} /dev/tty & +fi -terminal -tail -fn +0 "$QEMU_LOG" --pid=$$ 2>/dev/null & -cat "$QEMU_TERM" 2>/dev/null & wait $! || : +rc=0 +wait $! || rc=$? +[ -f "$QEMU_END" ] && exit "$rc" sleep 1 & wait $! -[ ! -f "$QEMU_END" ] && finish 0 +finish "$rc" diff --git a/src/network.sh b/src/network.sh index 14f9cf8..2f1d293 100644 --- a/src/network.sh +++ b/src/network.sh @@ -198,9 +198,7 @@ configureDNS() { getHostPorts() { - local list="" - list+="$MON_PORT," - list+="${HOST_PORTS// /}," + local list="${HOST_PORTS// /}," # Remove duplicates list=$(echo "${list//,,/,}," | awk 'BEGIN{RS=ORS=","} !seen[$0]++' | sed 's/,*$//g') @@ -590,11 +588,8 @@ configureNAT() { closeBridge() { - [ -s "$PASST_PID" ] && pKill "$(<"$PASST_PID")" - rm -f "$PASST_PID" - - [ -s "$DNSMASQ_PID" ] && pKill "$(<"$DNSMASQ_PID")" - rm -f "$DNSMASQ_PID" + local pids=( "$PASST_PID" "$DNSMASQ_PID" ) + mKill "${pids[@]}" ip link set "$VM_NET_TAP" down promisc off &> /dev/null || : ip link delete "$VM_NET_TAP" &> /dev/null || : @@ -608,14 +603,8 @@ closeBridge() { closeWeb() { - # Shutdown nginx - nginx -s stop 2> /dev/null - fWait "nginx" - - # Shutdown websocket - local pid="/var/run/websocketd.pid" - [ -s "$pid" ] && pKill "$(<"$pid")" - rm -f "$pid" + local pids=( "$WEB_PID" "$WSD_PID" ) + mKill "${pids[@]}" return 0 } @@ -632,6 +621,7 @@ closeNetwork() { exec 40<&- || true closeBridge + return 0 } diff --git a/src/power.sh b/src/power.sh index 0906bef..39ac390 100644 --- a/src/power.sh +++ b/src/power.sh @@ -1,124 +1,89 @@ #!/usr/bin/env bash set -Eeuo pipefail -: "${API_TIMEOUT:="50"}" # API Call timeout -: "${QEMU_TIMEOUT:="50"}" # QEMU Termination timeout +: "${SHUTDOWN:="Y"}" # Graceful ACPI shutdown +: "${TIMEOUT:="115"}" # QEMU termination timeout +: "${API_TIMEOUT:="90"}" # External API call timeout # Configure QEMU for graceful shutdown API_CMD=6 API_HOST="127.0.0.1:$COM_PORT" -QEMU_TERM="" -QEMU_LOG="$QEMU_DIR/qemu.log" -QEMU_OUT="$QEMU_DIR/qemu.out" +# Configure QEMU for graceful shutdown + QEMU_END="$QEMU_DIR/qemu.end" -if [[ "$KVM" == [Nn]* ]]; then - API_TIMEOUT=$(( API_TIMEOUT*2 )) - QEMU_TIMEOUT=$(( QEMU_TIMEOUT*2 )) -fi - _trap() { - local func="$1" ; shift - for sig ; do + local func="$1"; shift + local sig + TRAP_PID=$BASHPID + + for sig; do trap "$func $sig" "$sig" done } +app() { + echo "$APP" && return 0 +} + finish() { - local pid - local cnt=0 + local i=0 + local pid="" local reason=$1 + local pids=( "${HOST_PID:-}" "${WSD_PID:-}" \ + "${WEB_PID:-}" "${PASST_PID:-}" "${DNSMASQ_PID:-}" ) touch "$QEMU_END" if [ -s "$QEMU_PID" ]; then - - pid=$(<"$QEMU_PID") - echo && error "Forcefully terminating Virtual DSM, reason: $reason..." - { kill -15 "$pid" || true; } 2>/dev/null - - while isAlive "$pid"; do - - sleep 1 - (( cnt++ )) - - # Workaround for zombie pid - [ ! -s "$QEMU_PID" ] && break - - if [ "$cnt" -eq 5 ]; then - echo && error "QEMU did not terminate itself, forcefully killing process..." - { kill -9 "$pid" || true; } 2>/dev/null + if read -r pid <"$QEMU_PID"; then + if [ -n "$pid" ] && isAlive "$pid"; then + local display="$reason" + case "$reason" in + 129 ) display="SIGHUP" ;; + 130 ) display="SIGINT" ;; + 131 ) display="SIGQUIT" ;; + 134 ) display="SIGABRT" ;; + 143 ) display="SIGTERM" ;; + esac + error "Forcefully terminating $(app), reason: $display..." + { disown "$pid" || :; kill -9 -- "$pid" || :; } 2>/dev/null fi - - done - - fi - - fKill "print.sh" - fKill "host.bin" - - closeNetwork - - sleep 1 - echo && echo "❯ Shutdown completed!" - - exit "$reason" -} - -terminal() { - - local dev="" - - if [ -s "$QEMU_OUT" ]; then - - local msg - msg=$(<"$QEMU_OUT") - - if [ -n "$msg" ]; then - - if [[ "${msg,,}" != "char"* || "$msg" != *"serial0)" ]]; then - echo "$msg" - fi - - dev="${msg#*/dev/p}" - dev="/dev/p${dev%% *}" - fi fi - if [ ! -c "$dev" ]; then - dev=$(echo 'info chardev' | nc -q 1 -w 1 localhost "$MON_PORT" | tr -d '\000') - dev="${dev#*serial0}" - dev="${dev#*pty:}" - dev="${dev%%$'\n'*}" - dev="${dev%%$'\r'*}" + mKill "${pids[@]}" + fKill "print.sh" + + closeNetwork + + if ! waitPidFile "$QEMU_PID" 10; then + warn "Timed out while waiting for $(app) to exit!" fi - if [ ! -c "$dev" ]; then - error "Device '$dev' not found!" - finish 34 && return 34 - fi - - QEMU_TERM="$dev" - return 0 + (( reason != 1 )) && echo && echo "❯ Shutdown completed!" + exit "$reason" } -_graceful_shutdown() { +graceful_shutdown() { local sig="$1" + local pid="" local code=0 - local pid url response + local start url response elapsed + + [[ $BASHPID != "$TRAP_PID" ]] && return case "$sig" in - SIGTERM) code=143 ;; - SIGINT) code=130 ;; SIGHUP) code=129 ;; - SIGABRT) code=134 ;; + SIGINT) code=130 ;; SIGQUIT) code=131 ;; - esac + SIGABRT) code=134 ;; + SIGTERM) code=143 ;; + esac if [ -f "$QEMU_END" ]; then echo && info "Received $1 signal while already shutting down..." @@ -126,23 +91,22 @@ _graceful_shutdown() { fi set +e + start=$SECONDS touch "$QEMU_END" echo && info "Received $1 signal, sending shutdown command..." - if [ ! -s "$QEMU_PID" ]; then - echo && error "QEMU PID file does not exist?" - finish "$code" && return "$code" + if [ ! -s "$QEMU_PID" ] || ! read -r pid <"$QEMU_PID"; then + warn "QEMU PID file ($QEMU_PID) does not exist?" + finish "$code" fi - pid=$(<"$QEMU_PID") - - if ! isAlive "$pid"; then - echo && error "QEMU process does not exist?" - finish "$code" && return "$code" + if [ -z "$pid" ] || ! isAlive "$pid"; then + warn "QEMU process with PID $pid does not exist?" + finish "$code" fi # Don't send the powerdown signal because vDSM ignores ACPI signals - # echo 'system_powerdown' | nc -q 1 -w 1 localhost "$MON_PORT" > /dev/null + # nc -q 1 -w 1 -U "$QEMU_DIR/monitor.sock" &> /dev/null <<<'system_powerdown' || : # Send shutdown command to guest agent via serial port url="http://$API_HOST/read?command=$API_CMD&timeout=$API_TIMEOUT" @@ -157,46 +121,52 @@ _graceful_shutdown() { response="${response#*message\"\: \"}" [ -z "$response" ] && response="second signal" echo && error "Forcefully terminating because of: ${response%%\"*}" - { kill -15 "$pid" || true; } 2>/dev/null + { kill -15 -- "$pid" || :; } 2>/dev/null fi - local cnt=0 + local cnt=0 abort=0 factor=3 offset=3 min max name - while [ "$cnt" -lt "$QEMU_TIMEOUT" ]; do + [[ "$TIMEOUT" =~ ^[0-9]+$ ]] || TIMEOUT=115 + [ "$TIMEOUT" -ge 15 ] && factor=4 && offset=4 + [ "$TIMEOUT" -ge 30 ] && factor=5 && offset=5 + min=$(( factor + offset + 1 )) + [ "$TIMEOUT" -lt "$min" ] && TIMEOUT="$min" + elapsed=$(( SECONDS - start )) + max=$(( TIMEOUT - offset - elapsed )) + [ "$max" -lt "$factor" ] && max=$(( factor + 1 )) + abort=$(( max - factor )) + name="$(app)" + + while [ "$cnt" -le "$max" ]; do + + sleep 1 & + local slp=$! ! isAlive "$pid" && break - - sleep 1 - (( cnt++ )) - - [[ "$DEBUG" == [Yy1]* ]] && info "Shutting down, waiting... ($cnt/$QEMU_TIMEOUT)" - # Workaround for zombie pid [ ! -s "$QEMU_PID" ] && break + if [ "$cnt" -ne "$abort" ]; then + if [ "$cnt" -gt 0 ] && [[ "$DEBUG" == [Yy1]* ]]; then + info "Waiting for $name to shut down... ($cnt/$max)" + fi + else + info "${name^} is still running, sending SIGTERM... ($cnt/$max)" + { kill -15 -- "$pid" || :; } 2>/dev/null + fi + + wait $slp + (( cnt++ )) + done - if [ "$cnt" -ge "$QEMU_TIMEOUT" ]; then - echo && error "Shutdown timeout reached, aborting..." - fi - - finish "$code" && return "$code" + finish "$code" } -touch "$QEMU_LOG" +[[ "$SHUTDOWN" != [Yy1]* ]] && return 0 +[ -n "${QEMU_TIMEOUT:-}" ] && TIMEOUT="$QEMU_TIMEOUT" -MON_OPTS="\ - -pidfile $QEMU_PID \ - -name $PROCESS,process=$PROCESS,debug-threads=on \ - -monitor telnet:localhost:$MON_PORT,server,nowait,nodelay" - -if [[ "$CONSOLE" != [Yy]* ]]; then - - MON_OPTS+=" -daemonize -D $QEMU_LOG" - - _trap _graceful_shutdown SIGTERM SIGHUP SIGINT SIGABRT SIGQUIT - -fi +_trap graceful_shutdown SIGTERM SIGHUP SIGABRT SIGQUIT return 0 diff --git a/src/reset.sh b/src/reset.sh index 29a58ef..e22670c 100644 --- a/src/reset.sh +++ b/src/reset.sh @@ -13,7 +13,6 @@ trap 'error "Status $? while: $BASH_COMMAND (line $LINENO/$BASH_LINENO)"' ERR : "${KVM:="Y"}" # KVM acceleration : "${DEBUG:="N"}" # Disable debugging mode : "${COUNTRY:=""}" # Country code for mirror -: "${CONSOLE:="N"}" # Disable console mode : "${ALLOCATE:=""}" # Preallocate diskspace : "${ARGUMENTS:=""}" # Extra QEMU parameters : "${CPU_CORES:="2"}" # Amount of CPU cores @@ -225,8 +224,8 @@ fi # Cleanup files rm -f "$QEMU_DIR"/qemu.* +rm -f "$QEMU_DIR"/*.pid rm -f "$QEMU_DIR"/dsm.url -rm -f "$QEMU_DIR"/remote.ip # Cleanup dirs rm -rf /tmp/dsm diff --git a/src/serial.sh b/src/serial.sh index 58975ce..d42f3d0 100644 --- a/src/serial.sh +++ b/src/serial.sh @@ -24,6 +24,8 @@ if [ -n "$HOST_MAC" ]; then fi +HOST_PID="$QEMU_DIR/host.pid" + HOST_ARGS=() HOST_ARGS+=("-cpu=$CPU_CORES") HOST_ARGS+=("-cpu_arch=$HOST_CPU") @@ -37,9 +39,11 @@ if [[ "$HOST_DEBUG" == [Yy1]* ]]; then set -x ./host.bin "${HOST_ARGS[@]}" & { set +x; } 2>/dev/null + echo "$!" > "$HOST_PID" echo else ./host.bin "${HOST_ARGS[@]}" >/dev/null & + echo "$!" > "$HOST_PID" fi cnt=0 @@ -60,14 +64,7 @@ while ! nc -z -w2 127.0.0.1 "$CHR_PORT" > /dev/null 2>&1; do done # Configure serial ports - -if [[ "$CONSOLE" != [Yy]* ]]; then - SERIAL_OPTS="-serial pty" -else - SERIAL_OPTS="-serial mon:stdio" -fi - -SERIAL_OPTS+=" \ +SERIAL_OPTS="-serial mon:stdio \ -device virtio-serial-pci,id=virtio-serial0,bus=pcie.0,addr=0x3 \ -chardev socket,id=charchannel0,host=127.0.0.1,port=$CHR_PORT,reconnect=10 \ -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=vchannel" diff --git a/src/server.sh b/src/server.sh index cea8c29..229a89d 100644 --- a/src/server.sh +++ b/src/server.sh @@ -2,13 +2,16 @@ set -Eeuo pipefail : "${COM_PORT:="2210"}" # Comm port -: "${MON_PORT:="7100"}" # Monitor port : "${WEB_PORT:="5000"}" # Webserver port : "${CHR_PORT:="12345"}" # Character port : "${WSD_PORT:="8004"}" # Websockets port -cp -r /var/www/* /run/shm -rm -f /var/run/websocketd.pid +WEB_PID="/run/nginx.pid" +WSD_PID="$QEMU_DIR/websocketd.pid" + +cp -r /var/www/* "$QEMU_DIR" +rm -f "$WSD_PID" +rm -f "$WEB_PID" html "Starting $APP for $ENGINE..." @@ -32,7 +35,7 @@ if [[ "${WEB:-}" != [Nn]* ]]; then # Start websocket server websocketd --address 127.0.0.1 --port="$WSD_PORT" /run/socket.sh >/var/log/websocketd.log & - echo "$!" > /var/run/websocketd.pid + echo "$!" > "$WSD_PID" fi diff --git a/src/utils.sh b/src/utils.sh index 342b462..b25d162 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -28,6 +28,7 @@ formatBytes() { isAlive() { local pid="$1" + [ -z "$pid" ] && return 1 if kill -0 "$pid" 2>/dev/null; then return 0 @@ -36,23 +37,67 @@ isAlive() { return 1 } -pKill() { +waitPid() { + local i=0 local pid="$1" + local timeout="${2:-10}" - { kill -15 "$pid" || true; } 2>/dev/null - - while isAlive "$pid"; do + while [ -n "$pid" ] && isAlive "$pid"; do sleep 0.2 + i=$((i + 1)) + (( i >= timeout * 5 )) && return 1 done return 0 } +waitPidFile() { + local i=0 + local pid="" + local file="$1" + local timeout="${2:-10}" + + [ ! -s "$file" ] && return 0 + ! read -r pid <"$file" && return 0 + [ -z "$pid" ] && return 0 + + while [ -s "$file" ] && isAlive "$pid"; do + sleep 0.2 + i=$((i + 1)) + (( i >= timeout * 5 )) && return 1 + done + + rm -f -- "$file" + return 0 +} + +pKill() { + local pid="$1" + local timeout="${2:-10}" + + { kill -15 -- "$pid" || :; } 2>/dev/null + + if ! waitPid "$pid" "$timeout"; then + warn "Timed out while waiting for PID $pid" + fi + + return 0 +} + fWait() { + local i=0 local name="$1" + local timeout="${2:-10}" + + [ -z "$name" ] && return 0 while pgrep -f -l "$name" >/dev/null; do sleep 0.2 + i=$((i + 1)) + if (( i >= timeout * 5 )); then + warn "Timed out while waiting for process: $name" + break + fi done return 0 @@ -60,9 +105,44 @@ fWait() { fKill() { local name="$1" + local timeout="${2:-10}" - { pkill -f "$name" || true; } 2>/dev/null - fWait "$name" + [ -z "$name" ] && return 0 + + { pkill -f "$name" || :; } 2>/dev/null + fWait "$name" "$timeout" + + return 0 +} + +sKill() { + local pid="" + local file="$1" + + [ ! -s "$file" ] && return 0 + ! read -r pid <"$file" && return 0 + [ -z "$pid" ] && return 0 + + if isAlive "$pid"; then + { kill -15 -- "$pid" || :; } 2>/dev/null + fi + + return 0 +} + +mKill() { + local timeout=10 + local files=("$@") + + for file in "${files[@]}"; do + sKill "$file" + done + + for file in "${files[@]}"; do + if ! waitPidFile "$file" "$timeout"; then + warn "Timed out while waiting for PID file: $file" + fi + done return 0 }