Skip to content

Commit

Permalink
Merge pull request #178 from Chronos-EaaS/job-log-overview-fix
Browse files Browse the repository at this point in the history 
Job log overview fix
  • Loading branch information
@PatrikBuetler
PatrikBuetler authored Aug 14, 2024
2 parents efef8af + bf17493 commit f73492d
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 64 deletions.
1 change: 0 additions & 1 deletion controllers/job.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,6 @@ public function detail() {
$logalyzer->examineEntireLog();
}


$this->view->assign('logWarningCount', Factory::getJobFactory()->getJobCountForLogLevel($job, 'warn', 'negative'));
$this->view->assign('logErrorCount', Factory::getJobFactory()->getJobCountForLogLevel($job, 'error', 'negative'));
if($job->getStatus()==Define::JOB_STATUS_FINISHED && !Factory::getJobFactory()->checkAllPositiveJobPatterns($job)) {
Expand Down
57 changes: 31 additions & 26 deletions libraries/dba/AbstractModelFactory.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -788,48 +788,53 @@ public function getDB($test = false) {
public function incrementJobCountAtomically($jobId, $resultCollection)
{
$dbh = self::getDB();
$dbh->beginTransaction();
try {
foreach ($resultCollection as $pattern => $amount) {
foreach ($resultCollection as $pattern) {
$stmt1 = $dbh->prepare("SELECT
JSON_UNQUOTE(
REPLACE(JSON_EXTRACT(
JSON_SEARCH(logalyzerResults, 'one', :pattern), '$[0]'), '].pattern', '].count'))
INTO @index
FROM Job
WHERE jobId = :jobId;");
$stmt1->bindParam(':pattern', $pattern, PDO::PARAM_STR);
$stmt1->bindParam(':pattern', $pattern['pattern'], PDO::PARAM_STR);
$stmt1->bindParam(':jobId', $jobId, PDO::PARAM_INT);
$stmt1->execute();

if (!$stmt1->execute()) {
file_put_contents(UPLOADED_DATA_PATH . 'log/' . $jobId . '.log', "\nError in execute() for Query 1\n", FILE_APPEND);
}
$helper = $dbh->query("SELECT @index");
$index = $helper->fetch(PDO::FETCH_ASSOC);

$incrementQuery = "UPDATE Job
SET logalyzerResults = JSON_SET(
logalyzerResults,
:index,
CAST(CAST(
JSON_UNQUOTE(
JSON_EXTRACT(logalyzerResults, :index)
) AS UNSIGNED) + :amount AS CHAR))
WHERE jobId = :jobId AND JSON_SEARCH(logalyzerResults, 'one', :pattern) is not null;";
$stmt2 = $dbh->prepare($incrementQuery);
if ($stmt2 === false) {
file_put_contents(UPLOADED_DATA_PATH . 'log/' . $jobId . '.log', "\nError in prepare()\n", FILE_APPEND);
}
$stmt2 = $dbh->prepare("UPDATE Job
SET logalyzerResults = JSON_SET(
logalyzerResults,
:index,
CAST(CAST(
JSON_UNQUOTE(
JSON_EXTRACT(logalyzerResults, :index)
) AS UNSIGNED) + :amount AS CHAR))
WHERE jobId = :jobId AND JSON_SEARCH(logalyzerResults, 'one', :pattern) is not null;");
$stmt2->bindParam(':index', $index['@index'], PDO::PARAM_STR);
$stmt2->bindParam(':pattern', $pattern, PDO::PARAM_STR);
$stmt2->bindParam(':amount', $amount, PDO::PARAM_INT);
$stmt2->bindParam(':pattern', $pattern['pattern'], PDO::PARAM_STR);
$stmt2->bindParam(':amount', $pattern['count'], PDO::PARAM_INT);
$stmt2->bindParam(':jobId', $jobId, PDO::PARAM_INT);
if (!$stmt2->execute()) {
file_put_contents(UPLOADED_DATA_PATH . 'log/' . $jobId . '.log', "\nError in execute()\n", FILE_APPEND);
file_put_contents(UPLOADED_DATA_PATH . 'log/' . $jobId . '.log', "\nError in execute()\n", FILE_APPEND);
}

# Used to query the database entry and append it to a log for debugging
#$checker2 = $dbh->prepare("SELECT * FROM Job WHERE jobId = :jobId AND JSON_SEARCH(logalyzerResults, 'one', :pattern) is not null;");
#$checker2->bindParam(':jobId', $jobId, PDO::PARAM_INT);
#$checker2->bindParam(':pattern', $pattern['pattern'], PDO::PARAM_STR);
#$checker2->execute();
#$fetch2 = $checker2->fetch(PDO::FETCH_ASSOC);
#file_put_contents(UPLOADED_DATA_PATH . 'log/' . $jobId . '.log', print_r($fetch2, true)."\n", FILE_APPEND);

}
$dbh->commit();
}
}
catch (PDOException $e) {
$dbh->rollback();
#$dbh->rollback();
file_put_contents(UPLOADED_DATA_PATH . 'log/' . $jobId . '.log', $e->getMessage(), FILE_APPEND);
}
}
Expand Down Expand Up @@ -888,11 +893,11 @@ public function logalyzerUpdateHash($jobId, $hash) {
public function getJobCountForLogLevel($job, $logLevel, $type) {
if($job->getLogalyzerResults() != null) {
$json = json_decode($job->getLogalyzerResults(), true);
$resultArray = $json['pattern'];
$resultArray = $json['result'];
$count = 0;
foreach ($resultArray as $element) {
if ($type === 'negative' && $element['type'] === 'negative' && $element['logLevel'] === $logLevel) {
$count += $element['count'];
if ($type === "negative" && $element["type"] === "negative" && $element["logLevel"] === $logLevel) {
$count += $element["count"];
}
}
return $count;
Expand Down
65 changes: 29 additions & 36 deletions libraries/logalyzer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,10 @@ public function examineEntireLog() {
$this->createEmptyJobLogalyzerResults();
$hash = $this->calculateHash();

foreach($this->data['pattern'] as $pattern) {
foreach($this->data['result'] as $pattern) {
$number = $this->countLogOccurances($pattern['pattern'], $this->log, $pattern['regex']);
$found = false;
foreach($this->results['pattern'] as $result) {
foreach($this->results['result'] as $result) {
//print_r($result);
if(isset($result['logLevel'],$result['pattern'],$result['regex'],$result['type']) && $pattern['logLevel'] === $result['logLevel'] && $pattern['pattern'] === $result['pattern'] && $pattern['regex'] === $result['regex'] && $pattern['type'] === $result['type']) {
$result['count'] += $number;
Expand All @@ -90,7 +90,7 @@ public function examineEntireLog() {
if(!$found) {
$pattern['count'] = $number;
//print_r($pattern);
$this->results['pattern'][] = $pattern;
$this->results['result'][] = $pattern;
}
}
$this->results['hash'] = $hash;
Expand All @@ -110,49 +110,42 @@ public function examineLogLine($logLine) {
if($json === null) {
return;
}
#$results = $this->job->getLogalyzerResults();
#if(isset($results)) {
# $this->results = json_decode($results, true);
#}
#else {
# $this->createEmptyJobLogalyzerResults();
#}
$this->createEmptyJobLogalyzerResults();
$json = $this->job->getLogalyzerResults();
if($json === null) {
$this->createEmptyJobLogalyzerResults();
}
else {
$this->results = json_decode($json, true);
}
$hash = $this->calculateHash();
$resultCollection = [];
$LOG_ERRORS_MAX = 10; // TODO change to constant from constants.php
foreach($this->data['pattern'] as $index => $pattern) {
$LOG_ERRORS_MAX = 50; // TODO change to constant from constants.php
foreach($this->data['result'] as $index => $pattern) {
$number = $this->countLogOccurances($pattern['pattern'], $logLine, $pattern['regex']);
$isInResultSet = false;
file_put_contents(UPLOADED_DATA_PATH . 'log/' . $this->job->getId() . '.log', print_r($this->results, true), FILE_APPEND);
foreach($this->results['pattern'] as $result) {
foreach($this->results['result'] as $result) {
// Check if the result has been previously set in the job's result
if (isset($result['logLevel'], $result['pattern'], $result['regex'], $result['type']) && $pattern['logLevel'] === $result['logLevel'] && $pattern['pattern'] === $result['pattern'] && $pattern['regex'] === $result['regex'] && $pattern['type'] === $result['type'] && $result['count'] < $LOG_ERRORS_MAX) {
$isInResultSet = true;
if ($number >= 1) {
$pattern['count'] = $number;
$resultCollection[] = $pattern;
file_put_contents(UPLOADED_DATA_PATH . 'log/' . $this->job->getId() . '.log', "\nadded to resultCollection\n", FILE_APPEND);

}
}
}
if(!$isInResultSet) {
file_put_contents(UPLOADED_DATA_PATH . 'log/' . $this->job->getId() . '.log', "\nnot in result set adding\n".$number, FILE_APPEND);
$pattern['count'] = $number;
$this->results['pattern'][] = $pattern;
$this->results['result'][] = $pattern;
if($this->results['hash'] === "" || $this->results['hash'] === null) {
$this->results['hash'] = $hash;
}
file_put_contents(UPLOADED_DATA_PATH . 'log/' . $this->job->getId() . '.log', print_r($this->results, true), FILE_APPEND);

$this->job->setLogalyzerResults(json_encode($this->results));
Factory::getJobFactory()->update($this->job);
}
}
if(!empty($resultCollection)) {
file_put_contents(UPLOADED_DATA_PATH . 'log/' . $this->job->getId() . '.log', "\ntrying to increment using query\n", FILE_APPEND);
Factory::getJobFactory()->incrementJobCountAtomically($this->job->getId(), $resultCollection);
Factory::getJobFactory()->update($this->job);
}
/*$end = microtime(true);
if($logLine == "SendMail\n") {
Expand Down Expand Up @@ -183,7 +176,7 @@ private function logTime($start, $end) {
*/
private function createBasicPatterns() {
$this->data['hash'] = "";
$this->data['pattern'] = array();
$this->data['result'] = array();
}

/**
Expand All @@ -198,10 +191,10 @@ public function getPatterns(string $logLevel, string $type) {
$this->createBasicPatterns();
}
if ($logLevel === 'all') {
return $this->data['pattern'];
return $this->data['result'];
} else {
$temp = [];
foreach ($this->data['pattern'] as $pattern) {
foreach ($this->data['result'] as $pattern) {
if ($pattern['logLevel'] == $logLevel && $pattern['type'] == $type) {
$temp[] = $pattern;
}
Expand Down Expand Up @@ -234,7 +227,7 @@ public function loadPatterns() {
* @return void
*/
private function savePatterns() {
$this->data['hash'] = hash('sha1', json_encode($this->data['pattern']));
$this->data['hash'] = hash('sha1', json_encode($this->data['result']));
$this->system->setLogalyzerPatterns(json_encode($this->data));
Factory::getSystemFactory()->update($this->system);
}
Expand All @@ -251,8 +244,8 @@ public function addKey(string $logLevel, string $pattern, string $regex, string
}
else {
$array = array('logLevel' => $logLevel, 'pattern' => $pattern, 'regex' => $regex, 'type' => $type);
if(!in_array($array, $this->data['pattern'])) {
$this->data['pattern'][] = $array;
if(!in_array($array, $this->data['result'])) {
$this->data['result'][] = $array;
$this->savePatterns();
}
}
Expand All @@ -270,16 +263,16 @@ public function removeKey(string $logLevel, string $pattern, string $type) {
}
else {
$array = array('logLevel' => $logLevel, 'pattern' => $pattern, 'regex' => 'string', 'type' => $type);
if(in_array($array, $this->data['pattern'])) {
$index = array_search($array, $this->data['pattern']);
unset($this->data['pattern'][$index]);
if(in_array($array, $this->data['result'])) {
$index = array_search($array, $this->data['result']);
unset($this->data['result'][$index]);
$this->savePatterns();
}
else {
$array['regex'] = 'regex';
if(in_array($array, $this->data['pattern'])) {
$index = array_search($array, $this->data['pattern']);
unset($this->data['pattern'][$index]);
if(in_array($array, $this->data['result'])) {
$index = array_search($array, $this->data['result']);
unset($this->data['result'][$index]);
$this->savePatterns();
}
}
Expand All @@ -292,13 +285,13 @@ public function removeKey(string $logLevel, string $pattern, string $type) {
*/
private function createEmptyJobLogalyzerResults() {
$this->results['hash'] = "";
$this->results['pattern'] = array();
$this->results['result'] = array();
}

/**
* @return string
*/
function calculateHash() {
return hash('sha1', json_encode($this->data['pattern']));
return hash('sha1', json_encode($this->data['result']));
}
}
2 changes: 1 addition & 1 deletion views/job/detail.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ function updateProgress() {
Errors: <?php if($data['logErrorCount'] > 10) { echo('> 10'); }
elseif($data['logErrorCount'] >= 0) { echo($data['logErrorCount']); } ?>
<br>
Warnings: <?php if($data['logErrorCount'] > 10) { echo('> 10'); }
Warnings: <?php if($data['logWarningCount'] > 10) { echo('> 10'); }
elseif($data['logWarningCount'] >= 0) { echo($data['logWarningCount']);} ?>
</div>
</div>
Expand Down

0 comments on commit f73492d

Please sign in to comment.