feat: add gitapplyfast.sh script for optimized deployment with animation and retry logic
This commit is contained in:
679
gitapplyfast.sh
Normal file
679
gitapplyfast.sh
Normal file
@@ -0,0 +1,679 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ============================================================================
|
||||
# Virtnet.bond - Paymenter Load Balancer Git Update Script (FAST MODE)
|
||||
# ============================================================================
|
||||
# This is a faster version that skips npm update/upgrade and composer upgrade
|
||||
|
||||
# 256-Color Gradient Palette (Blue to Teal)
|
||||
C1='\e[38;5;21m' # Deep Blue
|
||||
C2='\e[38;5;27m' # Blue
|
||||
C3='\e[38;5;33m' # Light Blue
|
||||
C4='\e[38;5;39m' # Cyan Blue
|
||||
C5='\e[38;5;44m' # Cyan
|
||||
C6='\e[38;5;43m' # Teal
|
||||
|
||||
# Standard colors
|
||||
WHITE='\e[97m'
|
||||
GRAY='\e[90m'
|
||||
RED='\e[91m'
|
||||
YELLOW='\e[93m'
|
||||
GREEN='\e[92m'
|
||||
BOLD='\e[1m'
|
||||
DIM='\e[2m'
|
||||
NC='\e[0m'
|
||||
|
||||
# Animation characters
|
||||
SPIN=('⣾' '⣽' '⣻' '⢿' '⡿' '⣟' '⣯' '⣷')
|
||||
DOTS=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏')
|
||||
BARS=('▁' '▂' '▃' '▄' '▅' '▆' '▇' '█' '▇' '▆' '▅' '▄' '▃' '▂')
|
||||
PULSE=('○' '◎' '●' '◉' '●' '◎')
|
||||
|
||||
# Node list
|
||||
NODES=(
|
||||
"10.0.254.100"
|
||||
"10.0.254.101"
|
||||
"10.0.254.102"
|
||||
"10.0.254.103"
|
||||
"10.0.254.104"
|
||||
"10.0.254.105"
|
||||
"10.0.254.106"
|
||||
"10.0.254.107"
|
||||
"10.0.254.108"
|
||||
"10.0.254.109"
|
||||
"10.0.254.110"
|
||||
"10.0.254.111"
|
||||
"10.0.254.112"
|
||||
"10.0.254.113"
|
||||
"10.0.254.114"
|
||||
"10.0.254.115"
|
||||
"10.0.254.116"
|
||||
"10.0.254.117"
|
||||
"10.0.254.118"
|
||||
"10.0.254.119"
|
||||
"10.0.254.120"
|
||||
"10.0.254.121"
|
||||
"10.0.254.122"
|
||||
"10.0.254.123"
|
||||
"10.0.254.124"
|
||||
"10.0.254.125"
|
||||
"10.0.254.126"
|
||||
"10.0.254.127"
|
||||
"10.0.254.128"
|
||||
"10.0.254.129"
|
||||
"10.0.254.130"
|
||||
"10.0.254.131"
|
||||
)
|
||||
|
||||
MAX_RETRIES=100
|
||||
|
||||
# Get current node IP
|
||||
CURRENT_IP=$(hostname -I | awk '{print $1}')
|
||||
|
||||
# Status tracking directory
|
||||
STATUS_DIR="/tmp/virtnet_deploy_$$"
|
||||
mkdir -p "$STATUS_DIR"
|
||||
|
||||
# SSH Key
|
||||
SSH_KEY="/root/.ssh/virtnet_deploy"
|
||||
|
||||
# ============================================================================
|
||||
# ANIMATION FUNCTIONS
|
||||
# ============================================================================
|
||||
|
||||
# Typewriter effect
|
||||
typewriter() {
|
||||
local text="$1"
|
||||
local delay="${2:-0.02}"
|
||||
for ((i=0; i<${#text}; i++)); do
|
||||
printf "%s" "${text:$i:1}"
|
||||
sleep "$delay"
|
||||
done
|
||||
}
|
||||
|
||||
# Animated line draw
|
||||
draw_line() {
|
||||
local width=$1
|
||||
local char="${2:-━}"
|
||||
local color="${3:-$C4}"
|
||||
printf " ${color}"
|
||||
for ((i=0; i<width; i++)); do
|
||||
printf "%s" "$char"
|
||||
sleep 0.008
|
||||
done
|
||||
printf "${NC}\n"
|
||||
}
|
||||
|
||||
# Progress bar
|
||||
progress_bar() {
|
||||
local current=$1
|
||||
local total=$2
|
||||
local width=40
|
||||
local percent=$((current * 100 / total))
|
||||
local filled=$((current * width / total))
|
||||
local empty=$((width - filled))
|
||||
|
||||
printf "\r ${C3}[${NC}"
|
||||
printf "${C5}"
|
||||
for ((i=0; i<filled; i++)); do printf "█"; done
|
||||
printf "${GRAY}"
|
||||
for ((i=0; i<empty; i++)); do printf "░"; done
|
||||
printf "${C3}]${NC} ${WHITE}%3d%%${NC}" "$percent"
|
||||
}
|
||||
|
||||
# Animated header reveal
|
||||
print_header() {
|
||||
clear
|
||||
echo ""
|
||||
|
||||
# Logo lines with animation delay
|
||||
local logo_lines=(
|
||||
" ${C1}██╗ ██╗${C2}██╗${C3}██████╗ ${C3}████████╗${C4}███╗ ██╗${C5}███████╗${C6}████████╗${NC}"
|
||||
" ${C1}██║ ██║${C2}██║${C3}██╔══██╗${C3}╚══██╔══╝${C4}████╗ ██║${C5}██╔════╝${C6}╚══██╔══╝${NC}"
|
||||
" ${C1}██║ ██║${C2}██║${C3}██████╔╝${C3} ██║ ${C4}██╔██╗ ██║${C5}█████╗ ${C6} ██║ ${NC}"
|
||||
" ${C1}╚██╗ ██╔╝${C2}██║${C3}██╔══██╗${C3} ██║ ${C4}██║╚██╗██║${C5}██╔══╝ ${C6} ██║ ${NC}"
|
||||
" ${C1} ╚████╔╝ ${C2}██║${C3}██║ ██║${C3} ██║ ${C4}██║ ╚████║${C5}███████╗${C6} ██║ ${NC}"
|
||||
" ${C1} ╚═══╝ ${C2}╚═╝${C3}╚═╝ ╚═╝${C3} ╚═╝ ${C4}╚═╝ ╚═══╝${C5}╚══════╝${C6} ╚═╝ ${NC}"
|
||||
)
|
||||
|
||||
for line in "${logo_lines[@]}"; do
|
||||
echo -e "$line"
|
||||
sleep 0.05
|
||||
done
|
||||
|
||||
# Animated .bond text
|
||||
printf " ${C6}${BOLD}"
|
||||
typewriter ".bond" 0.08
|
||||
printf "${NC}\n"
|
||||
|
||||
echo ""
|
||||
draw_line 57
|
||||
printf " ${WHITE}${BOLD}"
|
||||
typewriter " Paymenter Load Balancer - Git Update & Apply (FAST)" 0.015
|
||||
printf "${NC}\n"
|
||||
draw_line 57
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Spinner with message
|
||||
spinner() {
|
||||
local pid=$1
|
||||
local msg=$2
|
||||
local i=0
|
||||
while kill -0 "$pid" 2>/dev/null; do
|
||||
printf "\r ${C3}${SPIN[$i]}${NC} %s ${DIM}${DOTS[$((i % 10))]}${NC}" "$msg"
|
||||
i=$(( (i + 1) % 8 ))
|
||||
sleep 0.1
|
||||
done
|
||||
wait "$pid"
|
||||
return $?
|
||||
}
|
||||
|
||||
# Wave spinner for waiting
|
||||
wave_spinner() {
|
||||
local pid=$1
|
||||
local msg=$2
|
||||
local i=0
|
||||
local colors=("$C1" "$C2" "$C3" "$C4" "$C5" "$C6")
|
||||
while kill -0 "$pid" 2>/dev/null; do
|
||||
printf "\r "
|
||||
for ((j=0; j<6; j++)); do
|
||||
local idx=$(( (i + j) % 14 ))
|
||||
local cidx=$(( (i + j) % 6 ))
|
||||
printf "${colors[$cidx]}${BARS[$idx]}${NC}"
|
||||
done
|
||||
printf " %s" "$msg"
|
||||
i=$(( (i + 1) % 14 ))
|
||||
sleep 0.08
|
||||
done
|
||||
wait "$pid"
|
||||
return $?
|
||||
}
|
||||
|
||||
# Task section header with animation
|
||||
task_start() {
|
||||
echo ""
|
||||
printf "${C4}┌─${NC} ${C5}${BOLD}"
|
||||
typewriter "$1" 0.02
|
||||
printf "${NC}\n"
|
||||
}
|
||||
|
||||
# Task completion with checkmark animation
|
||||
task_done() {
|
||||
local msg=$1
|
||||
printf "\r "
|
||||
for p in "${PULSE[@]}"; do
|
||||
printf "\r ${C6}${p}${NC}"
|
||||
sleep 0.05
|
||||
done
|
||||
printf "\r ${C6}●${NC} %s ${GREEN}✓${NC}\n" "$msg"
|
||||
}
|
||||
|
||||
# Task failure
|
||||
task_fail() {
|
||||
printf "\r ${RED}●${NC} %s ${RED}✗${NC}\n" "$1"
|
||||
}
|
||||
|
||||
# Animated countdown
|
||||
countdown() {
|
||||
local secs=$1
|
||||
local circle=('◜' '◠' '◝' '◞' '◡' '◟')
|
||||
|
||||
for ((i=secs; i>0; i--)); do
|
||||
for c in "${circle[@]}"; do
|
||||
printf "\r ${C3}${c}${NC} Waiting ${C5}${BOLD}%d${NC}s " "$i"
|
||||
sleep 0.166
|
||||
done
|
||||
done
|
||||
|
||||
# Completion animation
|
||||
printf "\r "
|
||||
for p in "${PULSE[@]}"; do
|
||||
printf "\r ${C6}${p}${NC} Ready! "
|
||||
sleep 0.08
|
||||
done
|
||||
printf "\r ${C6}●${NC} Ready! ${GREEN}✓${NC}\n"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# NODE FUNCTIONS
|
||||
# ============================================================================
|
||||
|
||||
run_remote_update() {
|
||||
local node=$1
|
||||
local status_file="$STATUS_DIR/$node"
|
||||
echo "pending" > "$status_file"
|
||||
|
||||
# FAST MODE: Skip npm update, npm upgrade, and composer upgrade
|
||||
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes -i "$SSH_KEY" root@"$node" \
|
||||
'cd /var/www/paymenter && git pull && php artisan view:clear && php artisan cache:clear && php artisan optimize && while ! systemctl restart php8.3-fpm 2>/dev/null; do echo "PHP-FPM restart failed, running optimize and retrying..."; php artisan optimize; sleep 1; done' \
|
||||
> "$STATUS_DIR/${node}.log" 2>&1
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "done" > "$status_file"
|
||||
return 0
|
||||
else
|
||||
echo "failed" > "$status_file"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_remote_maintenance_down() {
|
||||
local node=$1
|
||||
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -o BatchMode=yes -i "$SSH_KEY" root@"$node" \
|
||||
'cd /var/www/paymenter && php artisan down' > /dev/null 2>&1
|
||||
}
|
||||
|
||||
run_remote_maintenance_up() {
|
||||
local node=$1
|
||||
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -o BatchMode=yes -i "$SSH_KEY" root@"$node" \
|
||||
'cd /var/www/paymenter && php artisan up' > /dev/null 2>&1
|
||||
}
|
||||
|
||||
get_remote_nodes() {
|
||||
local remotes=()
|
||||
for node in "${NODES[@]}"; do
|
||||
if [[ "$node" != "$CURRENT_IP" ]]; then
|
||||
remotes+=("$node")
|
||||
fi
|
||||
done
|
||||
echo "${remotes[@]}"
|
||||
}
|
||||
|
||||
# Get node status counts
|
||||
get_node_status_counts() {
|
||||
local done_count=0
|
||||
local pending_count=0
|
||||
local failed_count=0
|
||||
local waiting_count=0
|
||||
|
||||
for node in "${REMOTE_NODES[@]}"; do
|
||||
local status_file="$STATUS_DIR/$node"
|
||||
local status="waiting"
|
||||
[[ -f "$status_file" ]] && status=$(cat "$status_file")
|
||||
|
||||
case "$status" in
|
||||
"done") done_count=$((done_count + 1)) ;;
|
||||
"failed") failed_count=$((failed_count + 1)) ;;
|
||||
"pending") pending_count=$((pending_count + 1)) ;;
|
||||
*) waiting_count=$((waiting_count + 1)) ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "$done_count $pending_count $failed_count $waiting_count"
|
||||
}
|
||||
|
||||
# Animated node grid
|
||||
print_node_grid() {
|
||||
local title=$1
|
||||
shift
|
||||
local show_status=$1
|
||||
shift
|
||||
local nodes=("$@")
|
||||
local animate=${1:-false}
|
||||
|
||||
echo ""
|
||||
printf "${C4}┌─${NC} ${C5}${BOLD}"
|
||||
typewriter "$title" 0.02
|
||||
printf "${NC}\n"
|
||||
|
||||
# Animated border
|
||||
printf " ${GRAY}"
|
||||
typewriter "┌──────────┬──────────┬──────────┬──────────┬──────────┐" 0.005
|
||||
printf "${NC}\n"
|
||||
|
||||
local col=0
|
||||
local row_started=0
|
||||
|
||||
for node in "${nodes[@]}"; do
|
||||
if [ $row_started -eq 0 ]; then
|
||||
printf " ${GRAY}│${NC}"
|
||||
row_started=1
|
||||
fi
|
||||
|
||||
local short="${node##*.}"
|
||||
local icon="○"
|
||||
local color="$GRAY"
|
||||
|
||||
if [ "$show_status" = "true" ]; then
|
||||
local status_file="$STATUS_DIR/$node"
|
||||
local status="waiting"
|
||||
[[ -f "$status_file" ]] && status=$(cat "$status_file")
|
||||
|
||||
case "$status" in
|
||||
"done") icon="●"; color="$GREEN" ;;
|
||||
"failed") icon="●"; color="$RED" ;;
|
||||
"pending") icon="◐"; color="$C3" ;;
|
||||
*) icon="○"; color="$GRAY" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
printf " ${color}${icon}${NC} .%-6s${GRAY}│${NC}" "$short"
|
||||
col=$((col + 1))
|
||||
|
||||
if [ $((col % 5)) -eq 0 ]; then
|
||||
echo ""
|
||||
row_started=0
|
||||
sleep 0.03
|
||||
fi
|
||||
done
|
||||
|
||||
while [ $((col % 5)) -ne 0 ]; do
|
||||
printf " ${GRAY}│${NC}"
|
||||
col=$((col + 1))
|
||||
done
|
||||
if [ $row_started -eq 1 ]; then
|
||||
echo ""
|
||||
fi
|
||||
|
||||
printf " ${GRAY}"
|
||||
typewriter "└──────────┴──────────┴──────────┴──────────┴──────────┘" 0.005
|
||||
printf "${NC}\n"
|
||||
}
|
||||
|
||||
get_failed_nodes() {
|
||||
local failed=()
|
||||
for node in "${NODES[@]}"; do
|
||||
if [[ "$node" != "$CURRENT_IP" ]]; then
|
||||
local status_file="$STATUS_DIR/$node"
|
||||
if [[ -f "$status_file" ]] && [[ "$(cat "$status_file")" == "failed" ]]; then
|
||||
failed+=("$node")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
echo "${failed[@]}"
|
||||
}
|
||||
|
||||
count_results() {
|
||||
local success=0
|
||||
local failed=0
|
||||
|
||||
for node in "${NODES[@]}"; do
|
||||
if [[ "$node" == "$CURRENT_IP" ]]; then
|
||||
success=$((success + 1))
|
||||
continue
|
||||
fi
|
||||
local status_file="$STATUS_DIR/$node"
|
||||
if [[ -f "$status_file" ]] && [[ "$(cat "$status_file")" == "done" ]]; then
|
||||
success=$((success + 1))
|
||||
else
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo "$success $failed"
|
||||
}
|
||||
|
||||
# Animated completion message
|
||||
print_complete() {
|
||||
local results
|
||||
results=$(count_results)
|
||||
local success=$(echo $results | cut -d' ' -f1)
|
||||
local failed=$(echo $results | cut -d' ' -f2)
|
||||
|
||||
echo ""
|
||||
draw_line 57
|
||||
echo ""
|
||||
|
||||
# Rocket animation
|
||||
local rocket_frames=('🚀' '✨🚀' '✨ 🚀' '✨ 🚀' ' ✨🚀' ' 🚀')
|
||||
for frame in "${rocket_frames[@]}"; do
|
||||
printf "\r ${C6}${BOLD}%s UPDATE COMPLETE (FAST MODE)${NC} " "$frame"
|
||||
sleep 0.12
|
||||
done
|
||||
printf "\r ${C6}${BOLD}🚀 UPDATE COMPLETE (FAST MODE)${NC} \n"
|
||||
|
||||
echo ""
|
||||
printf " ${GRAY}"
|
||||
typewriter "Local node updated successfully." 0.02
|
||||
printf "${NC}\n"
|
||||
|
||||
# Animated stats
|
||||
printf " "
|
||||
for ((i=0; i<=success; i++)); do
|
||||
printf "\r ${GREEN}●${NC} ${GREEN}%d${NC} nodes updated" "$i"
|
||||
sleep 0.05
|
||||
done
|
||||
printf " ${RED}●${NC} "
|
||||
for ((i=0; i<=failed; i++)); do
|
||||
printf "\r ${GREEN}●${NC} ${success} nodes updated ${RED}●${NC} ${RED}%d${NC} nodes failed" "$i"
|
||||
sleep 0.05
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
draw_line 57
|
||||
echo ""
|
||||
printf " ${C5}${BOLD}All operations complete.${NC}\n"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$STATUS_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ============================================================================
|
||||
# MAIN SCRIPT
|
||||
# ============================================================================
|
||||
|
||||
print_header
|
||||
|
||||
# Animated info display
|
||||
printf " ${C3}→${NC} ${GRAY}Current Node:${NC} ${C5}"
|
||||
typewriter "$CURRENT_IP" 0.03
|
||||
printf "${NC}\n"
|
||||
|
||||
printf " ${C3}→${NC} ${GRAY}Target Nodes:${NC} ${C5}"
|
||||
typewriter "${#NODES[@]} nodes in cluster" 0.03
|
||||
printf "${NC}\n"
|
||||
|
||||
printf " ${YELLOW}→${NC} ${YELLOW}FAST MODE:${NC} ${WHITE}"
|
||||
typewriter "Skipping npm update/upgrade and composer upgrade" 0.02
|
||||
printf "${NC}\n"
|
||||
|
||||
REMOTE_NODES=($(get_remote_nodes))
|
||||
|
||||
sleep_time=$((RANDOM % 5 + 1))
|
||||
cd /var/www/paymenter
|
||||
|
||||
# Prepare
|
||||
task_start "Preparing Update"
|
||||
countdown "$sleep_time"
|
||||
|
||||
# Enable Maintenance Mode on All Nodes
|
||||
task_start "Cluster: Enabling Maintenance Mode"
|
||||
printf " ${C3}○${NC} Putting all nodes in maintenance mode..."
|
||||
|
||||
# Start maintenance mode on all remote nodes in parallel
|
||||
for node in "${REMOTE_NODES[@]}"; do
|
||||
run_remote_maintenance_down "$node" &
|
||||
done
|
||||
|
||||
# Also put local node in maintenance mode
|
||||
php artisan down > /dev/null 2>&1 &
|
||||
|
||||
# Wait for all maintenance commands to complete
|
||||
wait
|
||||
task_done "All nodes in maintenance mode"
|
||||
|
||||
# Deploy with progress
|
||||
task_start "Deploying to Remote Nodes"
|
||||
printf " ${GRAY}Initiating deployment to ${#REMOTE_NODES[@]} remote nodes"
|
||||
for ((i=0; i<3; i++)); do
|
||||
printf "."
|
||||
sleep 0.3
|
||||
done
|
||||
printf "${NC}\n"
|
||||
|
||||
for node in "${REMOTE_NODES[@]}"; do
|
||||
run_remote_update "$node" &
|
||||
done
|
||||
|
||||
print_node_grid "Initial Cluster Status" "false" "${REMOTE_NODES[@]}"
|
||||
|
||||
# Local Git Pull
|
||||
task_start "Local: Pulling Latest Changes"
|
||||
printf " ${C3}○${NC} Fetching from remote..."
|
||||
git pull > /tmp/git_output.txt 2>&1 &
|
||||
wave_spinner $! "Fetching from remote..."
|
||||
GIT_STATUS=$?
|
||||
|
||||
if [ $GIT_STATUS -ne 0 ]; then
|
||||
task_fail "Git pull failed"
|
||||
echo ""
|
||||
cat /tmp/git_output.txt
|
||||
echo ""
|
||||
printf " ${YELLOW}↻${NC} ${YELLOW}"
|
||||
typewriter "Restarting script in 2 seconds..." 0.03
|
||||
printf "${NC}\n"
|
||||
sleep 2
|
||||
exec "$0" "$@"
|
||||
fi
|
||||
|
||||
task_done "Git pull completed"
|
||||
head -3 /tmp/git_output.txt 2>/dev/null | while IFS= read -r line; do
|
||||
printf " ${DIM}"
|
||||
typewriter "$line" 0.01
|
||||
printf "${NC}\n"
|
||||
done
|
||||
|
||||
# Clear Caches (FAST MODE: Skip npm and composer updates)
|
||||
task_start "Local: Clearing Caches"
|
||||
|
||||
printf " ${C3}○${NC} View cache..."
|
||||
php artisan view:clear > /dev/null 2>&1 &
|
||||
spinner $! "View cache"
|
||||
task_done "View cache cleared"
|
||||
|
||||
printf " ${C3}○${NC} Application cache..."
|
||||
php artisan cache:clear > /dev/null 2>&1 &
|
||||
spinner $! "Application cache"
|
||||
task_done "Application cache cleared"
|
||||
|
||||
printf " ${C3}○${NC} Optimizing..."
|
||||
php artisan optimize > /dev/null 2>&1 &
|
||||
spinner $! "Optimizing"
|
||||
task_done "Optimization completed"
|
||||
|
||||
# Restart PHP-FPM
|
||||
task_start "Local: Restarting Services"
|
||||
printf " ${C3}○${NC} PHP 8.3 FPM..."
|
||||
|
||||
# Retry loop: if PHP-FPM fails to restart, run optimize and try again
|
||||
php_restart_success=false
|
||||
php_restart_attempts=0
|
||||
while [ "$php_restart_success" = false ]; do
|
||||
php_restart_attempts=$((php_restart_attempts + 1))
|
||||
if systemctl restart php8.3-fpm > /dev/null 2>&1; then
|
||||
php_restart_success=true
|
||||
else
|
||||
printf "\r ${YELLOW}○${NC} PHP 8.3 FPM restart failed (attempt ${php_restart_attempts}), running optimize...\n"
|
||||
php artisan optimize > /dev/null 2>&1
|
||||
printf " ${C3}○${NC} Retrying PHP 8.3 FPM..."
|
||||
sleep 1
|
||||
fi
|
||||
done
|
||||
task_done "PHP 8.3 FPM restarted (attempt ${php_restart_attempts})"
|
||||
|
||||
# Wait for remote nodes with live status updates
|
||||
task_start "Waiting for Remote Nodes"
|
||||
printf " ${C3}○${NC} Syncing cluster...\n"
|
||||
|
||||
# Live status update loop while waiting for remote nodes
|
||||
total_nodes=${#REMOTE_NODES[@]}
|
||||
while true; do
|
||||
# Check if any jobs are still running
|
||||
jobs_running=$(jobs -r | wc -l)
|
||||
|
||||
# Get current status counts
|
||||
status_counts=$(get_node_status_counts)
|
||||
done_count=$(echo $status_counts | cut -d' ' -f1)
|
||||
pending_count=$(echo $status_counts | cut -d' ' -f2)
|
||||
failed_count=$(echo $status_counts | cut -d' ' -f3)
|
||||
waiting_count=$(echo $status_counts | cut -d' ' -f4)
|
||||
|
||||
# Display live status
|
||||
printf "\r ${C3}○${NC} Syncing cluster... ${GREEN}●${NC} ${done_count} done ${C3}◐${NC} ${pending_count} syncing ${RED}●${NC} ${failed_count} failed ${GRAY}○${NC} ${waiting_count} waiting "
|
||||
|
||||
# Exit loop when all jobs complete
|
||||
if [ $jobs_running -eq 0 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
# Final wait to ensure all background processes complete
|
||||
wait
|
||||
|
||||
printf "\n"
|
||||
task_done "Initial sync complete"
|
||||
|
||||
print_node_grid "Cluster Status (Attempt 1/${MAX_RETRIES})" "true" "${REMOTE_NODES[@]}"
|
||||
|
||||
# Retry logic
|
||||
attempt=1
|
||||
while [ $attempt -lt $MAX_RETRIES ]; do
|
||||
FAILED_NODES=($(get_failed_nodes))
|
||||
|
||||
if [ ${#FAILED_NODES[@]} -eq 0 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
|
||||
task_start "Retrying Failed Nodes (Attempt ${attempt}/${MAX_RETRIES})"
|
||||
printf " ${YELLOW}↻${NC} ${YELLOW}"
|
||||
typewriter "${#FAILED_NODES[@]} node(s) failed, retrying..." 0.02
|
||||
printf "${NC}\n"
|
||||
|
||||
for node in "${FAILED_NODES[@]}"; do
|
||||
run_remote_update "$node" &
|
||||
done
|
||||
|
||||
# Live status update loop for retries
|
||||
printf " ${C3}○${NC} Waiting for retry...\n"
|
||||
while true; do
|
||||
jobs_running=$(jobs -r | wc -l)
|
||||
|
||||
status_counts=$(get_node_status_counts)
|
||||
done_count=$(echo $status_counts | cut -d' ' -f1)
|
||||
pending_count=$(echo $status_counts | cut -d' ' -f2)
|
||||
failed_count=$(echo $status_counts | cut -d' ' -f3)
|
||||
waiting_count=$(echo $status_counts | cut -d' ' -f4)
|
||||
|
||||
printf "\r ${C3}○${NC} Syncing cluster... ${GREEN}●${NC} ${done_count} done ${C3}◐${NC} ${pending_count} syncing ${RED}●${NC} ${failed_count} failed ${GRAY}○${NC} ${waiting_count} waiting "
|
||||
|
||||
if [ $jobs_running -eq 0 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
wait
|
||||
printf "\n"
|
||||
task_done "Retry attempt ${attempt} complete"
|
||||
|
||||
print_node_grid "Cluster Status (Attempt ${attempt}/${MAX_RETRIES})" "true" "${REMOTE_NODES[@]}"
|
||||
done
|
||||
|
||||
print_node_grid "Final Cluster Status" "true" "${REMOTE_NODES[@]}"
|
||||
|
||||
# Disable Maintenance Mode on All Nodes
|
||||
task_start "Cluster: Disabling Maintenance Mode"
|
||||
printf " ${C3}○${NC} Bringing all nodes back online..."
|
||||
|
||||
# Bring all remote nodes up in parallel
|
||||
for node in "${REMOTE_NODES[@]}"; do
|
||||
run_remote_maintenance_up "$node" &
|
||||
done
|
||||
|
||||
# Also bring local node up
|
||||
php artisan up > /dev/null 2>&1 &
|
||||
|
||||
# Wait for all nodes to come up
|
||||
wait
|
||||
task_done "All nodes are back online"
|
||||
|
||||
print_complete
|
||||
Reference in New Issue
Block a user