Joseph.Rawlings
5f2e591c6e
- Add is_lxc boolean column to ext_proxmox_nodes (default false, existing nodes unaffected) - Add LXC checkbox to admin node creation form (disabled on edit) - Dynamic API path selection via Node::vmType() (returns 'lxc' or 'qemu') - LXC deployment: syncLxcSettings() for cores, memory, swap, hostname, rootfs, net config - LXC network uses name=eth0 format with applyLxcPoolSettings() for VLAN/firewall/MTU - All lifecycle operations LXC-aware: install, suspend, unsuspend, terminate, upgrade, reinstall - VNC/xterm console uses correct type (lxc vs kvm) - All 20+ Livewire components use vmType() for Proxmox API paths - LXC user interface shows basic-only view (no advanced toggle, no KVM-specific tabs) - Basic tabs for LXC: Overview, Network Configuration, Backups, Console
317 lines
14 KiB
PHP
317 lines
14 KiB
PHP
<?php
|
|
|
|
namespace Paymenter\Extensions\Servers\Proxmox\Console\Commands;
|
|
|
|
use Carbon\Carbon;
|
|
use Illuminate\Console\Command;
|
|
use Illuminate\Support\Facades\Cache;
|
|
use Paymenter\Extensions\Servers\Proxmox\Models\Server;
|
|
use Paymenter\Extensions\Servers\Proxmox\Models\ServerStatistics;
|
|
use Paymenter\Extensions\Servers\Proxmox\Proxmox;
|
|
|
|
class PollServerStatistics extends Command
|
|
{
|
|
/**
|
|
* The name and signature of the console command.
|
|
*
|
|
* @var string
|
|
*/
|
|
protected $signature = 'proxmox:poll-statistics';
|
|
|
|
/**
|
|
* The console command description.
|
|
*
|
|
* @var string
|
|
*/
|
|
protected $description = 'Poll Proxmox server statistics hourly (runs in background)';
|
|
|
|
/**
|
|
* Execute the console command.
|
|
*/
|
|
public function handle(): void
|
|
{
|
|
// Poll all servers regardless of status or power condition
|
|
$serverCount = Server::count();
|
|
$this->info('Polling statistics for ' . $serverCount . ' servers...');
|
|
|
|
$successCount = 0;
|
|
$errorCount = 0;
|
|
$maxAutomaticStatistics = 24;
|
|
|
|
foreach (Server::with('node')->cursor() as $server) {
|
|
try {
|
|
if (!$server->node) {
|
|
$this->warn('Server ID ' . $server->id . ' has no node assigned, skipping.');
|
|
$errorCount++;
|
|
continue;
|
|
}
|
|
|
|
$proxmox = new Proxmox;
|
|
$status = $proxmox->request(
|
|
'/nodes/' . $server->node->name . '/' . $server->node->vmType() . '/' . $server->vm_id . '/status/current'
|
|
)->json()['data'] ?? null;
|
|
|
|
if ($status) {
|
|
// Get disk I/O from rrddata (latest point)
|
|
$diskRead = 0;
|
|
$diskWrite = 0;
|
|
$diskUsed = $status['disk'] ?? 0;
|
|
$diskMax = $status['maxdisk'] ?? 0;
|
|
|
|
try {
|
|
$rrdData = $proxmox->request(
|
|
'/nodes/' . $server->node->name . '/' . $server->node->vmType() . '/' . $server->vm_id . '/rrddata',
|
|
'get',
|
|
['timeframe' => 'hour', 'cf' => 'AVERAGE']
|
|
)->json()['data'] ?? null;
|
|
|
|
if ($rrdData && is_array($rrdData) && count($rrdData) > 0) {
|
|
$latest = end($rrdData);
|
|
$diskRead = (int)($latest['diskread'] ?? 0);
|
|
$diskWrite = (int)($latest['diskwrite'] ?? 0);
|
|
}
|
|
} catch (\Throwable $e) {
|
|
// If rrddata fails, continue with 0 values
|
|
}
|
|
|
|
// Always try to get diskmax from VM config (more reliable than status endpoint)
|
|
// The status endpoint may not return maxdisk, especially if guest agent is not enabled
|
|
try {
|
|
$config = $proxmox->request(
|
|
'/nodes/' . $server->node->name . '/' . $server->node->vmType() . '/' . $server->vm_id . '/config'
|
|
)->json()['data'] ?? null;
|
|
|
|
if ($config) {
|
|
// Try to find disk configuration (scsi0, virtio0, sata0, ide0, scsi1, etc.)
|
|
$diskKeys = ['scsi0', 'scsi1', 'scsi2', 'virtio0', 'virtio1', 'virtio2', 'sata0', 'sata1', 'sata2', 'ide0', 'ide1', 'ide2'];
|
|
$totalDiskSize = 0;
|
|
|
|
foreach ($diskKeys as $diskKey) {
|
|
if (isset($config[$diskKey])) {
|
|
// Parse disk size from format like "local-lvm:vm-100-disk-0,size=32G"
|
|
$diskConfig = $config[$diskKey];
|
|
if (preg_match('/size=(\d+)([TGMK]?)/i', $diskConfig, $matches)) {
|
|
$size = (int)$matches[1];
|
|
$unit = strtoupper($matches[2] ?? 'G');
|
|
|
|
// Convert to bytes
|
|
switch ($unit) {
|
|
case 'T':
|
|
$totalDiskSize += $size * 1024 * 1024 * 1024 * 1024;
|
|
break;
|
|
case 'G':
|
|
$totalDiskSize += $size * 1024 * 1024 * 1024;
|
|
break;
|
|
case 'M':
|
|
$totalDiskSize += $size * 1024 * 1024;
|
|
break;
|
|
case 'K':
|
|
$totalDiskSize += $size * 1024;
|
|
break;
|
|
default:
|
|
$totalDiskSize += $size * 1024 * 1024 * 1024; // Default to GB
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Use config disk size if we found it, otherwise keep status value
|
|
if ($totalDiskSize > 0) {
|
|
$diskMax = $totalDiskSize;
|
|
}
|
|
}
|
|
} catch (\Throwable $e) {
|
|
// If config fetch fails, continue with status value
|
|
}
|
|
|
|
// Create the new automatic statistic
|
|
ServerStatistics::create([
|
|
'server_id' => $server->id,
|
|
'netin' => $status['netin'] ?? 0,
|
|
'netout' => $status['netout'] ?? 0,
|
|
'diskread' => $diskRead,
|
|
'diskwrite' => $diskWrite,
|
|
'diskused' => $diskUsed,
|
|
'diskmax' => $diskMax,
|
|
'memoryused' => $status['mem'] ?? 0,
|
|
'memorymax' => $status['maxmem'] ?? 0,
|
|
'cpuused' => $status['cpu'] ?? 0,
|
|
'cpumax' => 1.0, // CPU max is typically 1.0 (100%)
|
|
'uptime' => $status['uptime'] ?? 0,
|
|
'vmstatus' => $status['status'] ?? null,
|
|
'DateTime' => now(),
|
|
'is_automatic' => true,
|
|
]);
|
|
|
|
// Maintain maximum of 24 automatic statistics per server
|
|
// Remove oldest automatic statistics if limit is exceeded
|
|
$automaticStatsCount = ServerStatistics::where('server_id', $server->id)
|
|
->where('is_automatic', true)
|
|
->count();
|
|
|
|
if ($automaticStatsCount > $maxAutomaticStatistics) {
|
|
$excessCount = $automaticStatsCount - $maxAutomaticStatistics;
|
|
$idsToDelete = ServerStatistics::where('server_id', $server->id)
|
|
->where('is_automatic', true)
|
|
->orderBy('DateTime', 'asc')
|
|
->limit($excessCount)
|
|
->pluck('id');
|
|
|
|
if ($idsToDelete->isNotEmpty()) {
|
|
ServerStatistics::whereIn('id', $idsToDelete)->delete();
|
|
}
|
|
}
|
|
|
|
$successCount++;
|
|
} else {
|
|
$this->warn('No status data for server ID: ' . $server->id);
|
|
$errorCount++;
|
|
}
|
|
} catch (\Throwable $e) {
|
|
$this->error('Error polling server ID ' . $server->id . ': ' . $e->getMessage());
|
|
$errorCount++;
|
|
}
|
|
}
|
|
|
|
$this->info('Statistics polling completed. Success: ' . $successCount . ', Errors: ' . $errorCount);
|
|
|
|
// Enforce bandwidth throttling for servers exceeding their limits
|
|
$this->info('Checking bandwidth limits...');
|
|
$this->enforceBandwidthLimits();
|
|
}
|
|
|
|
protected function enforceBandwidthLimits()
|
|
{
|
|
$threshold = $this->calculateBandwidthThreshold();
|
|
$this->info('Bandwidth threshold: ' . $threshold . ' GiB');
|
|
|
|
$throttledCount = 0;
|
|
foreach (Server::with('node')->cursor() as $server) {
|
|
try {
|
|
$monthlyTraffic = $this->calculateServerMonthlyTraffic($server);
|
|
|
|
if ($monthlyTraffic > $threshold) {
|
|
$this->warn('Server ID ' . $server->id . ' (' . $server->hostname . ') exceeded bandwidth: ' .
|
|
round($monthlyTraffic, 2) . ' GiB / ' . $threshold . ' GiB');
|
|
$this->throttleServerBandwidth($server);
|
|
$throttledCount++;
|
|
}
|
|
} catch (\Throwable $e) {
|
|
$this->error('Error checking bandwidth for server ID ' . $server->id . ': ' . $e->getMessage());
|
|
}
|
|
}
|
|
|
|
$this->info('Bandwidth enforcement completed. Throttled: ' . $throttledCount . ' servers');
|
|
}
|
|
|
|
protected function calculateBandwidthThreshold()
|
|
{
|
|
return Cache::remember('bandwidth_threshold_95_' . Carbon::now()->format('Y-m'), 15, function () {
|
|
$startOfMonth = Carbon::now()->startOfMonth();
|
|
$allServers = Server::all();
|
|
|
|
if ($allServers->isEmpty()) {
|
|
return 10;
|
|
}
|
|
|
|
$trafficValues = [];
|
|
|
|
foreach ($allServers as $server) {
|
|
$firstStat = ServerStatistics::where('server_id', $server->id)
|
|
->where('DateTime', '>=', $startOfMonth)
|
|
->orderBy('DateTime', 'asc')
|
|
->first();
|
|
|
|
$latestStat = ServerStatistics::where('server_id', $server->id)
|
|
->where('DateTime', '>=', $startOfMonth)
|
|
->orderBy('DateTime', 'desc')
|
|
->first();
|
|
|
|
if ($firstStat && $latestStat) {
|
|
$netinDiff = max(0, $latestStat->netin - $firstStat->netin);
|
|
$netoutDiff = max(0, $latestStat->netout - $firstStat->netout);
|
|
$totalBytes = $netinDiff + $netoutDiff;
|
|
$trafficValues[] = $totalBytes / (1024 * 1024 * 1024);
|
|
}
|
|
}
|
|
|
|
if (empty($trafficValues)) {
|
|
return 10;
|
|
}
|
|
|
|
// Calculate actual 95th percentile by sorting and taking the value at the 95th rank
|
|
sort($trafficValues);
|
|
$index = (int) ceil(0.95 * count($trafficValues)) - 1;
|
|
$threshold95Percent = $trafficValues[$index];
|
|
|
|
return round(max($threshold95Percent, 10), 2);
|
|
});
|
|
}
|
|
|
|
protected function calculateServerMonthlyTraffic(Server $server)
|
|
{
|
|
$startOfMonth = Carbon::now()->startOfMonth();
|
|
|
|
$firstStat = ServerStatistics::where('server_id', $server->id)
|
|
->where('DateTime', '>=', $startOfMonth)
|
|
->orderBy('DateTime', 'asc')
|
|
->first();
|
|
|
|
$latestStat = ServerStatistics::where('server_id', $server->id)
|
|
->where('DateTime', '>=', $startOfMonth)
|
|
->orderBy('DateTime', 'desc')
|
|
->first();
|
|
|
|
if (!$firstStat || !$latestStat) {
|
|
return 0;
|
|
}
|
|
|
|
$netinDiff = max(0, $latestStat->netin - $firstStat->netin);
|
|
$netoutDiff = max(0, $latestStat->netout - $firstStat->netout);
|
|
$totalBytes = $netinDiff + $netoutDiff;
|
|
|
|
return round($totalBytes / (1024 * 1024 * 1024), 2);
|
|
}
|
|
|
|
protected function throttleServerBandwidth(Server $server)
|
|
{
|
|
try {
|
|
$proxmox = new Proxmox;
|
|
$config = $proxmox->request('/nodes/' . $server->node->name . '/' . $server->node->vmType() . '/' . $server->vm_id . '/config')->json()['data'] ?? [];
|
|
|
|
$updates = [];
|
|
|
|
foreach ($config as $key => $value) {
|
|
if (preg_match('/^net\d+$/', $key)) {
|
|
$parts = explode(',', $value);
|
|
$newParts = [];
|
|
$hasRate = false;
|
|
|
|
foreach ($parts as $part) {
|
|
if (strpos($part, 'rate=') === 0) {
|
|
$newParts[] = 'rate=1';
|
|
$hasRate = true;
|
|
} else {
|
|
$newParts[] = $part;
|
|
}
|
|
}
|
|
|
|
if (!$hasRate) {
|
|
$newParts[] = 'rate=1';
|
|
}
|
|
|
|
$updates[$key] = implode(',', $newParts);
|
|
}
|
|
}
|
|
|
|
if (!empty($updates)) {
|
|
$proxmox->request('/nodes/' . $server->node->name . '/' . $server->node->vmType() . '/' . $server->vm_id . '/config', 'put', $updates);
|
|
$this->info('Throttled network interfaces for server ID ' . $server->id . ' to 1 MiB/s');
|
|
}
|
|
} catch (\Throwable $e) {
|
|
$this->error('Failed to throttle bandwidth for server ID ' . $server->id . ': ' . $e->getMessage());
|
|
}
|
|
}
|
|
}
|
|
|