info('Polling statistics for ' . $servers->count() . ' servers...'); $successCount = 0; $errorCount = 0; $maxAutomaticStatistics = 24; foreach ($servers as $server) { try { $proxmox = new Proxmox; $status = $proxmox->request( '/nodes/' . $server->node->name . '/qemu/' . $server->vm_id . '/status/current' )->json()['data'] ?? null; if ($status) { // Get disk I/O from rrddata (latest point) $diskRead = 0; $diskWrite = 0; $diskUsed = $status['disk'] ?? 0; $diskMax = $status['maxdisk'] ?? 0; try { $rrdData = $proxmox->request( '/nodes/' . $server->node->name . '/qemu/' . $server->vm_id . '/rrddata', 'get', ['timeframe' => 'hour', 'cf' => 'AVERAGE'] )->json()['data'] ?? null; if ($rrdData && is_array($rrdData) && count($rrdData) > 0) { $latest = end($rrdData); $diskRead = (int)($latest['diskread'] ?? 0); $diskWrite = (int)($latest['diskwrite'] ?? 0); } } catch (\Throwable $e) { // If rrddata fails, continue with 0 values } // Always try to get diskmax from VM config (more reliable than status endpoint) // The status endpoint may not return maxdisk, especially if guest agent is not enabled try { $config = $proxmox->request( '/nodes/' . $server->node->name . '/qemu/' . $server->vm_id . '/config' )->json()['data'] ?? null; if ($config) { // Try to find disk configuration (scsi0, virtio0, sata0, ide0, scsi1, etc.) $diskKeys = ['scsi0', 'scsi1', 'scsi2', 'virtio0', 'virtio1', 'virtio2', 'sata0', 'sata1', 'sata2', 'ide0', 'ide1', 'ide2']; $totalDiskSize = 0; foreach ($diskKeys as $diskKey) { if (isset($config[$diskKey])) { // Parse disk size from format like "local-lvm:vm-100-disk-0,size=32G" $diskConfig = $config[$diskKey]; if (preg_match('/size=(\d+)([TGMK]?)/i', $diskConfig, $matches)) { $size = (int)$matches[1]; $unit = strtoupper($matches[2] ?? 'G'); // Convert to bytes switch ($unit) { case 'T': $totalDiskSize += $size * 1024 * 1024 * 1024 * 1024; break; case 'G': $totalDiskSize += $size * 1024 * 1024 * 1024; break; case 'M': $totalDiskSize += $size * 1024 * 1024; break; case 'K': $totalDiskSize += $size * 1024; break; default: $totalDiskSize += $size * 1024 * 1024 * 1024; // Default to GB } } } } // Use config disk size if we found it, otherwise keep status value if ($totalDiskSize > 0) { $diskMax = $totalDiskSize; } } } catch (\Throwable $e) { // If config fetch fails, continue with status value } // Create the new automatic statistic ServerStatistics::create([ 'server_id' => $server->id, 'netin' => $status['netin'] ?? 0, 'netout' => $status['netout'] ?? 0, 'diskread' => $diskRead, 'diskwrite' => $diskWrite, 'diskused' => $diskUsed, 'diskmax' => $diskMax, 'memoryused' => $status['mem'] ?? 0, 'memorymax' => $status['maxmem'] ?? 0, 'cpuused' => $status['cpu'] ?? 0, 'cpumax' => 1.0, // CPU max is typically 1.0 (100%) 'uptime' => $status['uptime'] ?? 0, 'vmstatus' => $status['status'] ?? null, 'DateTime' => now(), 'is_automatic' => true, ]); // Maintain maximum of 24 automatic statistics per server // Remove oldest automatic statistics if limit is exceeded $automaticStatsCount = ServerStatistics::where('server_id', $server->id) ->where('is_automatic', true) ->count(); if ($automaticStatsCount > $maxAutomaticStatistics) { $excessCount = $automaticStatsCount - $maxAutomaticStatistics; $oldestStats = ServerStatistics::where('server_id', $server->id) ->where('is_automatic', true) ->orderBy('DateTime', 'asc') ->limit($excessCount) ->get(); foreach ($oldestStats as $oldStat) { $oldStat->delete(); } } $successCount++; } else { $this->warn('No status data for server ID: ' . $server->id); $errorCount++; } } catch (\Throwable $e) { $this->error('Error polling server ID ' . $server->id . ': ' . $e->getMessage()); $errorCount++; } } $this->info('Statistics polling completed. Success: ' . $successCount . ', Errors: ' . $errorCount); // Enforce bandwidth throttling for servers exceeding their limits $this->info('Checking bandwidth limits...'); $this->enforceBandwidthLimits(); } protected function enforceBandwidthLimits() { $threshold = $this->calculateBandwidthThreshold(); $this->info('Bandwidth threshold: ' . $threshold . ' GiB'); $throttledCount = 0; $servers = Server::all(); foreach ($servers as $server) { try { $monthlyTraffic = $this->calculateServerMonthlyTraffic($server); if ($monthlyTraffic > $threshold) { $this->warn('Server ID ' . $server->id . ' (' . $server->hostname . ') exceeded bandwidth: ' . round($monthlyTraffic, 2) . ' GiB / ' . $threshold . ' GiB'); $this->throttleServerBandwidth($server); $throttledCount++; } } catch (\Throwable $e) { $this->error('Error checking bandwidth for server ID ' . $server->id . ': ' . $e->getMessage()); } } $this->info('Bandwidth enforcement completed. Throttled: ' . $throttledCount . ' servers'); } protected function calculateBandwidthThreshold() { return Cache::remember('bandwidth_threshold_95_' . Carbon::now()->format('Y-m'), 15, function () { $startOfMonth = Carbon::now()->startOfMonth(); $allServers = Server::all(); if ($allServers->isEmpty()) { return 10; } $totalTraffic = 0; $serverCount = 0; foreach ($allServers as $server) { $firstStat = ServerStatistics::where('server_id', $server->id) ->where('DateTime', '>=', $startOfMonth) ->orderBy('DateTime', 'asc') ->first(); $latestStat = ServerStatistics::where('server_id', $server->id) ->where('DateTime', '>=', $startOfMonth) ->orderBy('DateTime', 'desc') ->first(); if ($firstStat && $latestStat) { $netinDiff = max(0, $latestStat->netin - $firstStat->netin); $netoutDiff = max(0, $latestStat->netout - $firstStat->netout); $totalBytes = $netinDiff + $netoutDiff; $totalTraffic += $totalBytes / (1024 * 1024 * 1024); $serverCount++; } } if ($serverCount === 0) { return 10; } $averageBandwidth = $totalTraffic / $serverCount; $threshold95Percent = $averageBandwidth * 0.95; return round(max($threshold95Percent, 10), 2); }); } protected function calculateServerMonthlyTraffic(Server $server) { $startOfMonth = Carbon::now()->startOfMonth(); $firstStat = ServerStatistics::where('server_id', $server->id) ->where('DateTime', '>=', $startOfMonth) ->orderBy('DateTime', 'asc') ->first(); $latestStat = ServerStatistics::where('server_id', $server->id) ->where('DateTime', '>=', $startOfMonth) ->orderBy('DateTime', 'desc') ->first(); if (!$firstStat || !$latestStat) { return 0; } $netinDiff = max(0, $latestStat->netin - $firstStat->netin); $netoutDiff = max(0, $latestStat->netout - $firstStat->netout); $totalBytes = $netinDiff + $netoutDiff; return round($totalBytes / (1024 * 1024 * 1024), 2); } protected function throttleServerBandwidth(Server $server) { try { $proxmox = new Proxmox; $config = $proxmox->request('/nodes/' . $server->node->name . '/qemu/' . $server->vm_id . '/config')->json()['data']; $updates = []; foreach ($config as $key => $value) { if (preg_match('/^net\d+$/', $key)) { $parts = explode(',', $value); $newParts = []; $hasRate = false; foreach ($parts as $part) { if (strpos($part, 'rate=') === 0) { $newParts[] = 'rate=1'; $hasRate = true; } else { $newParts[] = $part; } } if (!$hasRate) { $newParts[] = 'rate=1'; } $updates[$key] = implode(',', $newParts); } } if (!empty($updates)) { $proxmox->request('/nodes/' . $server->node->name . '/qemu/' . $server->vm_id . '/config', 'put', $updates); $this->info('Throttled network interfaces for server ID ' . $server->id . ' to 1 MiB/s'); } } catch (\Throwable $e) { $this->error('Failed to throttle bandwidth for server ID ' . $server->id . ': ' . $e->getMessage()); } } }