refactor: improve Proxmox Firewall log parsing with enhanced handling for various log formats, detailed logging of parsing results, and better error management
This commit is contained in:
@@ -812,6 +812,9 @@ class Firewall extends Component
|
||||
$batchSize = 500;
|
||||
$maxLogs = $this->logLimit;
|
||||
$fetchedCount = 0;
|
||||
$totalBodyLength = 0;
|
||||
$lastContentType = '';
|
||||
$lastIsJson = false;
|
||||
|
||||
// Keep fetching until we have all logs or reach the limit
|
||||
do {
|
||||
@@ -846,36 +849,62 @@ class Firewall extends Component
|
||||
// Proxmox firewall log endpoint can return either text/plain or JSON format
|
||||
// Try to parse as JSON first, then fall back to text/plain
|
||||
$body = $response->body();
|
||||
$totalBodyLength += strlen($body);
|
||||
$batchRawLogs = [];
|
||||
|
||||
// Check if response is JSON
|
||||
$contentType = $response->header('Content-Type');
|
||||
$lastContentType = $contentType ?? '';
|
||||
$isJson = str_contains($contentType ?? '', 'json');
|
||||
$lastIsJson = $isJson;
|
||||
|
||||
if ($isJson) {
|
||||
try {
|
||||
$jsonData = $response->json();
|
||||
// Proxmox may return logs in different formats:
|
||||
// 1. { "data": [ {line}, {line}, ... ] }
|
||||
// 2. { "data": "line1\nline2\n..." }
|
||||
// 3. [ {line}, {line}, ... ]
|
||||
// 1. { "data": [ {"n": 1, "t": "log text"}, ... ] }
|
||||
// 2. { "data": [ "line1", "line2", ... ] }
|
||||
// 3. { "data": "line1\nline2\n..." }
|
||||
// 4. [ {"n": 1, "t": "log text"}, ... ]
|
||||
|
||||
if (isset($jsonData['data'])) {
|
||||
if (is_array($jsonData['data'])) {
|
||||
$batchRawLogs = $jsonData['data'];
|
||||
// Handle array of log objects or strings
|
||||
foreach ($jsonData['data'] as $item) {
|
||||
if (is_array($item) && isset($item['t'])) {
|
||||
// Proxmox format: {"n": 1, "t": "log text"}
|
||||
$batchRawLogs[] = $item['t'];
|
||||
} elseif (is_string($item)) {
|
||||
// Plain string format
|
||||
$batchRawLogs[] = $item;
|
||||
} elseif (is_array($item)) {
|
||||
// Other array format, try to get text from various fields
|
||||
$text = $item['text'] ?? $item['message'] ?? $item['line'] ?? json_encode($item);
|
||||
$batchRawLogs[] = $text;
|
||||
}
|
||||
}
|
||||
} elseif (is_string($jsonData['data'])) {
|
||||
// Multi-line string format
|
||||
$lines = preg_split('/\r\n|\r|\n/', $jsonData['data']);
|
||||
$batchRawLogs = array_filter($lines, static function ($line) {
|
||||
return ! empty(trim($line));
|
||||
});
|
||||
}
|
||||
} elseif (is_array($jsonData)) {
|
||||
$batchRawLogs = $jsonData;
|
||||
// Array at root level
|
||||
foreach ($jsonData as $item) {
|
||||
if (is_array($item) && isset($item['t'])) {
|
||||
$batchRawLogs[] = $item['t'];
|
||||
} elseif (is_string($item)) {
|
||||
$batchRawLogs[] = $item;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
// Fall through to text parsing
|
||||
\Log::warning('Failed to parse JSON response, falling back to text', [
|
||||
'error' => $e->getMessage()
|
||||
'error' => $e->getMessage(),
|
||||
'response_sample' => substr($body, 0, 500)
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -915,50 +944,78 @@ class Firewall extends Component
|
||||
|
||||
\Log::info('Firewall logs fetched from Proxmox API', [
|
||||
'raw_count' => count($rawLogs),
|
||||
'body_length' => strlen($body),
|
||||
'content_type' => $contentType,
|
||||
'is_json' => $isJson,
|
||||
'sample' => ! empty($rawLogs) ? substr(is_array($rawLogs[0]) ? json_encode($rawLogs[0]) : $rawLogs[0], 0, 200) : null,
|
||||
'first_5_lines' => array_slice($rawLogs, 0, 5),
|
||||
'total_body_length' => $totalBodyLength,
|
||||
'last_content_type' => $lastContentType,
|
||||
'last_is_json' => $lastIsJson,
|
||||
'batches_fetched' => ($start / $batchSize),
|
||||
'first_log_type' => ! empty($rawLogs) ? gettype($rawLogs[0]) : 'empty',
|
||||
'first_log_length' => ! empty($rawLogs) && is_string($rawLogs[0]) ? strlen($rawLogs[0]) : 'N/A',
|
||||
'sample' => ! empty($rawLogs) ? substr(is_array($rawLogs[0]) ? json_encode($rawLogs[0]) : $rawLogs[0], 0, 300) : null,
|
||||
'first_3_logs_full' => array_slice($rawLogs, 0, 3),
|
||||
]);
|
||||
|
||||
$this->firewallLogs = [];
|
||||
|
||||
foreach ($rawLogs as $logEntry) {
|
||||
$successfullyParsed = 0;
|
||||
$skippedEmpty = 0;
|
||||
$skippedInvalid = 0;
|
||||
|
||||
foreach ($rawLogs as $index => $logEntry) {
|
||||
$logText = '';
|
||||
|
||||
if (is_string($logEntry)) {
|
||||
$logText = trim($logEntry);
|
||||
} elseif (is_array($logEntry)) {
|
||||
// Handle various possible array formats from Proxmox API
|
||||
// Note: 't' is the text field, 'n' is the line number - prioritize 't'!
|
||||
$logText = trim(
|
||||
$logEntry['raw']
|
||||
?? $logEntry['n']
|
||||
?? $logEntry['t']
|
||||
?? $logEntry['text']
|
||||
?? $logEntry['log']
|
||||
?? $logEntry['message']
|
||||
?? $logEntry['line']
|
||||
?? json_encode($logEntry)
|
||||
$logEntry['t'] // Proxmox format: {"n": 1, "t": "log text"}
|
||||
?? $logEntry['text'] // Alternative text field
|
||||
?? $logEntry['raw'] // Raw log field
|
||||
?? $logEntry['message'] // Message field
|
||||
?? $logEntry['log'] // Log field
|
||||
?? $logEntry['line'] // Line field
|
||||
?? json_encode($logEntry) // Last resort: encode the whole thing
|
||||
);
|
||||
}
|
||||
|
||||
if ($logText === '' || strlen($logText) < 10) {
|
||||
$skippedEmpty++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
$parsed = $this->parseFirewallLog($logText);
|
||||
if ($parsed !== null) {
|
||||
// Check if the log was actually parsed (has at least action or source/destination)
|
||||
if ($parsed !== null && ($parsed['action'] !== null || $parsed['source'] !== null || $parsed['destination'] !== null)) {
|
||||
$this->firewallLogs[] = $parsed;
|
||||
$successfullyParsed++;
|
||||
} else {
|
||||
$skippedInvalid++;
|
||||
if ($index < 5) { // Log first 5 unparseable entries for debugging
|
||||
\Log::warning('Firewall log could not be parsed (missing key fields)', [
|
||||
'entry' => substr($logText, 0, 300),
|
||||
'parsed' => $parsed
|
||||
]);
|
||||
}
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
\Log::warning('Failed to parse firewall log entry', [
|
||||
'entry' => substr($logText, 0, 200),
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
$skippedInvalid++;
|
||||
if ($index < 5) { // Log first 5 errors for debugging
|
||||
\Log::warning('Failed to parse firewall log entry', [
|
||||
'entry' => substr($logText, 0, 300),
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
\Log::info('Firewall log parsing summary', [
|
||||
'total_raw_logs' => count($rawLogs),
|
||||
'successfully_parsed' => $successfullyParsed,
|
||||
'skipped_empty' => $skippedEmpty,
|
||||
'skipped_invalid' => $skippedInvalid
|
||||
]);
|
||||
|
||||
\Log::info('Firewall logs final count', [
|
||||
'total_logs' => count($this->firewallLogs),
|
||||
@@ -966,7 +1023,7 @@ class Firewall extends Component
|
||||
]);
|
||||
|
||||
if (empty($this->firewallLogs) && ! empty($rawLogs)) {
|
||||
$this->logError = 'Logs received but could not be processed.';
|
||||
$this->logError = 'Logs received (' . count($rawLogs) . ') but could not be processed. Successfully parsed: ' . $successfullyParsed . ', Skipped invalid: ' . $skippedInvalid . '. Check Laravel logs for details.';
|
||||
} elseif (empty($this->firewallLogs)) {
|
||||
$this->logError = 'No firewall logs available. Ensure firewall logging is enabled in Proxmox (Datacenter > Firewall > Options).';
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user