From f9f951cd2c0827f8c269989e9cd4f0318df6c214 Mon Sep 17 00:00:00 2001 From: "Joseph.Rawlings" Date: Tue, 23 Dec 2025 20:53:12 -0500 Subject: [PATCH] fix: clean up gitapply.sh and improve node status handling --- gitapply.sh | 305 +++++++++++++++++++++++++--------------------------- 1 file changed, 144 insertions(+), 161 deletions(-) diff --git a/gitapply.sh b/gitapply.sh index 4c0ac3e65..da1443ba7 100755 --- a/gitapply.sh +++ b/gitapply.sh @@ -5,7 +5,6 @@ # ============================================================================ # 256-Color Gradient Palette (Blue to Teal) -# #0c00ff -> #0066ff -> #008dff -> #00a6ff -> #00bbe7 -> #00cd9a C1='\e[38;5;21m' # Deep Blue C2='\e[38;5;27m' # Blue C3='\e[38;5;33m' # Light Blue @@ -41,6 +40,8 @@ NODES=( "10.0.254.110" ) +MAX_RETRIES=3 + # Get current node IP CURRENT_IP=$(hostname -I | awk '{print $1}') @@ -48,7 +49,7 @@ CURRENT_IP=$(hostname -I | awk '{print $1}') STATUS_DIR="/tmp/virtnet_deploy_$$" mkdir -p "$STATUS_DIR" -# SSH Key (stored securely) +# SSH Key SSH_KEY="/root/.ssh/virtnet_deploy" # ============================================================================ @@ -66,9 +67,9 @@ print_header() { echo -e " ${C1} ╚═══╝ ${C2}╚═╝${C3}╚═╝ ╚═╝${C3} ╚═╝ ${C4}╚═╝ ╚═══╝${C5}╚══════╝${C6} ╚═╝ ${NC}" echo -e " ${C6}${BOLD}.bond${NC}" echo "" - echo -e " ${C4}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - echo -e " ${WHITE}${BOLD} Paymenter Load Balancer - Git Update & Apply${NC}" - echo -e " ${C4}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e " ${C4}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e " ${WHITE}${BOLD} Paymenter Load Balancer - Git Update & Apply${NC}" + echo -e " ${C4}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" } @@ -77,7 +78,7 @@ spinner() { local msg=$2 local i=0 while kill -0 "$pid" 2>/dev/null; do - printf "\r ${C3}${SPIN[$i]}${NC} ${msg}" + printf "\r ${C3}${SPIN[$i]}${NC} %s" "$msg" i=$(( (i + 1) % 8 )) sleep 0.1 done @@ -89,31 +90,27 @@ task_start() { echo -e "\n${C4}┌─${NC} ${C5}${BOLD}$1${NC}" } -task_item() { - printf " ${C3}○${NC} $1" -} - task_done() { - printf "\r ${C6}●${NC} $1 ${GREEN}✓${NC}\n" + printf "\r ${C6}●${NC} %s ${GREEN}✓${NC}\n" "$1" } task_fail() { - printf "\r ${RED}●${NC} $1 ${RED}✗${NC}\n" + printf "\r ${RED}●${NC} %s ${RED}✗${NC}\n" "$1" } countdown() { local secs=$1 for ((i=secs; i>0; i--)); do - printf "\r ${C3}◐${NC} Waiting ${C5}${BOLD}$i${NC}s " + printf "\r ${C3}◐${NC} Waiting ${C5}${BOLD}%d${NC}s " "$i" sleep 0.25 - printf "\r ${C3}◓${NC} Waiting ${C5}${BOLD}$i${NC}s " + printf "\r ${C3}◓${NC} Waiting ${C5}${BOLD}%d${NC}s " "$i" sleep 0.25 - printf "\r ${C3}◑${NC} Waiting ${C5}${BOLD}$i${NC}s " + printf "\r ${C3}◑${NC} Waiting ${C5}${BOLD}%d${NC}s " "$i" sleep 0.25 - printf "\r ${C3}◒${NC} Waiting ${C5}${BOLD}$i${NC}s " + printf "\r ${C3}◒${NC} Waiting ${C5}${BOLD}%d${NC}s " "$i" sleep 0.25 done - printf "\r ${C6}●${NC} Ready! ${GREEN}✓${NC}\n" + printf "\r ${C6}●${NC} Ready! ${GREEN}✓${NC}\n" } # Run update on remote node @@ -122,116 +119,107 @@ run_remote_update() { local status_file="$STATUS_DIR/$node" echo "pending" > "$status_file" - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" root@"$node" \ + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes -i "$SSH_KEY" root@"$node" \ 'cd /var/www/paymenter && git pull && php artisan view:clear && php artisan cache:clear && systemctl restart php8.3-fpm' \ > "$STATUS_DIR/${node}.log" 2>&1 if [ $? -eq 0 ]; then echo "done" > "$status_file" + return 0 else echo "failed" > "$status_file" + return 1 fi } -# Display node status panel -print_node_status() { - local cols=3 - local count=0 - - echo -e "\n${C4}┌─${NC} ${C5}${BOLD}Cluster Node Status${NC}" - echo -e " ${GRAY}┌─────────────────┬─────────────────┬─────────────────┐${NC}" - - local row="" +# Get list of remote nodes (excluding current) +get_remote_nodes() { + local remotes=() for node in "${NODES[@]}"; do - if [[ "$node" == "$CURRENT_IP" ]]; then - continue - fi - - local status_file="$STATUS_DIR/$node" - local status="waiting" - [[ -f "$status_file" ]] && status=$(cat "$status_file") - - local icon="" - local color="" - case "$status" in - "pending") icon="◐"; color="$C3" ;; - "done") icon="●"; color="$GREEN" ;; - "failed") icon="●"; color="$RED" ;; - *) icon="○"; color="$GRAY" ;; - esac - - local short_ip="${node##*.}" - row+=" ${color}${icon}${NC} .${short_ip}" - count=$((count + 1)) - - if [ $((count % cols)) -eq 0 ]; then - printf " │%-17s│%-17s│%-17s│\n" $row - row="" + if [[ "$node" != "$CURRENT_IP" ]]; then + remotes+=("$node") fi done - - # Print remaining nodes - if [ -n "$row" ]; then - while [ $((count % cols)) -ne 0 ]; do - row+=" " - count=$((count + 1)) - done - printf " │%-17s│%-17s│%-17s│\n" $row - fi - - echo -e " ${GRAY}└─────────────────┴─────────────────┴─────────────────┘${NC}" + echo "${remotes[@]}" } -# Animated node status display -update_node_display() { - local all_done=false - while ! $all_done; do - all_done=true +# Print node grid +print_node_grid() { + local title=$1 + shift + local show_status=$1 + shift + local nodes=("$@") + + echo -e "\n${C4}┌─${NC} ${C5}${BOLD}${title}${NC}" + echo -e " ${GRAY}┌──────────┬──────────┬──────────┬──────────┬──────────┐${NC}" + + local col=0 + local row_started=0 + + for node in "${nodes[@]}"; do + if [ $row_started -eq 0 ]; then + printf " ${GRAY}│${NC}" + row_started=1 + fi - # Move cursor up and redraw - tput sc # Save cursor position + local short="${node##*.}" + local icon="○" + local color="$GRAY" - echo -e "\r ${GRAY}┌─────────────────────────────────────────────────────┐${NC}" - - for node in "${NODES[@]}"; do - if [[ "$node" == "$CURRENT_IP" ]]; then - continue - fi - + if [ "$show_status" = "true" ]; then local status_file="$STATUS_DIR/$node" local status="waiting" [[ -f "$status_file" ]] && status=$(cat "$status_file") - local icon="" color="" case "$status" in - "pending") icon="⣾"; color="$C3"; all_done=false ;; "done") icon="●"; color="$GREEN" ;; "failed") icon="●"; color="$RED" ;; - *) icon="○"; color="$GRAY"; all_done=false ;; + "pending") icon="◐"; color="$C3" ;; + *) icon="○"; color="$GRAY" ;; esac - - printf " ${GRAY}│${NC} ${color}${icon}${NC} %-15s" "$node" - done + fi - echo -e "\r ${GRAY}└─────────────────────────────────────────────────────┘${NC}" + printf " ${color}${icon}${NC} .%-6s${GRAY}│${NC}" "$short" + col=$((col + 1)) - sleep 0.3 - - if ! $all_done; then - tput rc # Restore cursor position + if [ $((col % 5)) -eq 0 ]; then + echo "" + row_started=0 fi done + + # Fill remaining cells if row not complete + while [ $((col % 5)) -ne 0 ]; do + printf " ${GRAY}│${NC}" + col=$((col + 1)) + done + if [ $row_started -eq 1 ]; then + echo "" + fi + + echo -e " ${GRAY}└──────────┴──────────┴──────────┴──────────┴──────────┘${NC}" } -print_complete() { - echo "" - echo -e " ${C4}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - echo "" - echo -e " ${C6}${BOLD}🚀 UPDATE COMPLETE${NC}" - echo "" +# Get failed nodes +get_failed_nodes() { + local failed=() + for node in "${NODES[@]}"; do + if [[ "$node" != "$CURRENT_IP" ]]; then + local status_file="$STATUS_DIR/$node" + if [[ -f "$status_file" ]] && [[ "$(cat "$status_file")" == "failed" ]]; then + failed+=("$node") + fi + fi + done + echo "${failed[@]}" +} + +# Count results +count_results() { + local success=0 + local failed=0 - # Count successes and failures - local success=0 failed=0 for node in "${NODES[@]}"; do if [[ "$node" == "$CURRENT_IP" ]]; then success=$((success + 1)) @@ -245,10 +233,24 @@ print_complete() { fi done - echo -e " ${GRAY}Local node updated successfully.${NC}" - echo -e " ${GREEN}●${NC} ${success} nodes updated ${RED}●${NC} ${failed} nodes failed" + echo "$success $failed" +} + +print_complete() { + local results + results=$(count_results) + local success=$(echo $results | cut -d' ' -f1) + local failed=$(echo $results | cut -d' ' -f2) + echo "" - echo -e " ${C4}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e " ${C4}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo "" + echo -e " ${C6}${BOLD}🚀 UPDATE COMPLETE${NC}" + echo "" + echo -e " ${GRAY}Local node updated successfully.${NC}" + echo -e " ${GREEN}●${NC} ${success} nodes updated ${RED}●${NC} ${failed} nodes failed" + echo "" + echo -e " ${C4}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" } @@ -266,6 +268,9 @@ print_header echo -e " ${C3}→${NC} ${GRAY}Current Node:${NC} ${C5}${CURRENT_IP}${NC}" echo -e " ${C3}→${NC} ${GRAY}Target Nodes:${NC} ${C5}${#NODES[@]} nodes in cluster${NC}" +# Get remote nodes +REMOTE_NODES=($(get_remote_nodes)) + # Random delay sleep_time=$((RANDOM % 5 + 1)) cd /var/www/paymenter @@ -276,37 +281,18 @@ countdown "$sleep_time" # Start remote updates in background task_start "Deploying to Remote Nodes" -for node in "${NODES[@]}"; do - if [[ "$node" != "$CURRENT_IP" ]]; then - run_remote_update "$node" & - fi +echo -e " ${GRAY}Starting deployment on ${#REMOTE_NODES[@]} remote nodes...${NC}" + +for node in "${REMOTE_NODES[@]}"; do + run_remote_update "$node" & done -# Show initial node status -echo -e " ${GRAY}Background deployment started on remote nodes...${NC}" -echo "" - -# Display node grid -echo -e " ${GRAY}┌───────────────┬───────────────┬───────────────┬───────────────┐${NC}" -printf " ${GRAY}│${NC}" -col=0 -for node in "${NODES[@]}"; do - if [[ "$node" == "$CURRENT_IP" ]]; then - continue - fi - short="${node##*.}" - printf " ${GRAY}○${NC} .%-11s${GRAY}│${NC}" "$short" - col=$((col + 1)) - if [ $((col % 4)) -eq 0 ]; then - echo "" - printf " ${GRAY}│${NC}" - fi -done -echo -e "\n ${GRAY}└───────────────┴───────────────┴───────────────┴───────────────┘${NC}" +# Show initial node grid +print_node_grid "Initial Cluster Status" "false" "${REMOTE_NODES[@]}" # Local Git Pull task_start "Local: Pulling Latest Changes" -task_item "Fetching from remote..." +printf " ${C3}○${NC} Fetching from remote..." git pull > /tmp/git_output.txt 2>&1 & spinner $! "Fetching from remote..." GIT_STATUS=$? @@ -322,69 +308,66 @@ if [ $GIT_STATUS -ne 0 ]; then fi task_done "Git pull completed" -head -3 /tmp/git_output.txt | while IFS= read -r line; do - echo -e " ${DIM}$line${NC}" +head -3 /tmp/git_output.txt 2>/dev/null | while IFS= read -r line; do + echo -e " ${DIM}${line}${NC}" done # Clear Caches task_start "Local: Clearing Caches" -task_item "View cache..." +printf " ${C3}○${NC} View cache..." php artisan view:clear > /dev/null 2>&1 & spinner $! "View cache..." task_done "View cache cleared" -task_item "Application cache..." +printf " ${C3}○${NC} Application cache..." php artisan cache:clear > /dev/null 2>&1 & spinner $! "Application cache..." task_done "Application cache cleared" # Restart PHP-FPM task_start "Local: Restarting Services" -task_item "PHP 8.3 FPM..." +printf " ${C3}○${NC} PHP 8.3 FPM..." systemctl restart php8.3-fpm & spinner $! "PHP 8.3 FPM..." task_done "PHP 8.3 FPM restarted" -# Wait for all remote nodes +# Wait for initial remote deployment task_start "Waiting for Remote Nodes" -task_item "Syncing cluster..." +printf " ${C3}○${NC} Syncing cluster..." wait -task_done "All nodes synchronized" +task_done "Initial sync complete" -# Final node status -echo "" -echo -e " ${C4}┌─${NC} ${C5}${BOLD}Final Cluster Status${NC}" -echo -e " ${GRAY}┌───────────────┬───────────────┬───────────────┬───────────────┐${NC}" -printf " ${GRAY}│${NC}" -col=0 -for node in "${NODES[@]}"; do - if [[ "$node" == "$CURRENT_IP" ]]; then - continue - fi - status_file="$STATUS_DIR/$node" - status="failed" - [[ -f "$status_file" ]] && status=$(cat "$status_file") +# Show status after first attempt +print_node_grid "Cluster Status (Attempt 1/${MAX_RETRIES})" "true" "${REMOTE_NODES[@]}" + +# Retry failed nodes +attempt=1 +while [ $attempt -lt $MAX_RETRIES ]; do + FAILED_NODES=($(get_failed_nodes)) - icon="" - color="" - if [[ "$status" == "done" ]]; then - icon="●"; color="$GREEN" - else - icon="●"; color="$RED" + if [ ${#FAILED_NODES[@]} -eq 0 ]; then + break fi - short="${node##*.}" - printf " ${color}${icon}${NC} .%-11s${GRAY}│${NC}" "$short" - col=$((col + 1)) - if [ $((col % 4)) -eq 0 ]; then - echo "" - if [ $col -lt 10 ]; then - printf " ${GRAY}│${NC}" - fi - fi + attempt=$((attempt + 1)) + + task_start "Retrying Failed Nodes (Attempt ${attempt}/${MAX_RETRIES})" + echo -e " ${YELLOW}↻${NC} ${YELLOW}${#FAILED_NODES[@]} node(s) failed, retrying...${NC}" + + for node in "${FAILED_NODES[@]}"; do + run_remote_update "$node" & + done + + printf " ${C3}○${NC} Waiting for retry..." + wait + task_done "Retry attempt ${attempt} complete" + + print_node_grid "Cluster Status (Attempt ${attempt}/${MAX_RETRIES})" "true" "${REMOTE_NODES[@]}" done -echo -e "\n ${GRAY}└───────────────┴───────────────┴───────────────┴───────────────┘${NC}" + +# Final status +print_node_grid "Final Cluster Status" "true" "${REMOTE_NODES[@]}" # Complete print_complete