Skip to content

Commit 0173e0e

Browse files
nook24nook24
authored
Merge pull request #35 from statusengine/deadlock
Restart transaction on MySQL deadlocks
2 parents 0e388e3 + 58227fd commit 0173e0e

17 files changed

Lines changed: 67 additions & 47 deletions

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
3.7.4
1+
3.7.5

‎src/Backends/MySQL/MySQL.php‎

Lines changed: 43 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -280,31 +280,51 @@ public function dispatch() {
280280
* @param \PDOStatement $query
281281
* @return bool
282282
* @throws StorageBackendUnavailableExceptions
283-
*/
284-
public function executeQuery(\PDOStatement $query) {
285-
$result = false;
286-
try {
287-
$result = $query->execute();
288-
289-
} catch (\Exception $Exception) {
290-
$errorNo = $Exception->errorInfo[1];
291-
$errorString = $Exception->errorInfo[2];
292-
$this->Syslog->error(sprintf('[%s] %s', $errorNo, $errorString));
293-
$this->Syslog->error($query->queryString);
294-
$this->Syslog->error("Run the worker in foreground mode to see the full query: https://statusengine.org/worker/#debugging");
295-
296-
// This function has no return - so no log file -.-
297-
// If Statusengine is running via systemd systemd will write the messages to syslog
298-
if ($this->isDumpOfMysqlQueryParametersEnabled) {
299-
$query->debugDumpParams();
300-
}
301-
302-
if ($errorString == 'MySQL server has gone away') {
303-
$this->reconnect();
304-
throw new StorageBackendUnavailableExceptions($errorString);
283+
* @package string $caller
284+
*/
285+
public function executeQuery(\PDOStatement $query, $caller = 'Unknown') {
286+
// https://dev.mysql.com/doc/refman/8.0/en/innodb-deadlocks.html
287+
// https://dev.mysql.com/doc/refman/8.0/en/innodb-deadlocks-handling.html
288+
// The deadlock logic is ported from Statusengine 2
289+
// https://github.com/nook24/statusengine/blame/b5c86f0e02fd69a7045eb652f49dad20ee4d2b67/cakephp/app/src/BulkRepository.php#L185
290+
$retries = 10;
291+
for ($i = 1; $i < $retries; $i++) {
292+
try {
293+
return $query->execute();
294+
295+
} catch (\Exception $Exception) {
296+
$sqlstateErrorCode = $Exception->errorInfo[0]; // SQLSTATE error code (a five characters alphanumeric identifier defined in the ANSI SQL standard).
297+
$errorNo = $Exception->errorInfo[1]; // Driver-specific error code.
298+
$errorString = $Exception->errorInfo[2]; // Driver-specific error message.
299+
$this->Syslog->error(sprintf('[%s] %s', $errorNo, $errorString));
300+
301+
if ($i <= $retries && $sqlstateErrorCode == 40001 && $errorNo == 1213) {
302+
// This is a InnoDB deadlock - retry
303+
$sleep = 50000 + rand(0, 450000);
304+
$this->Syslog->info('Encountered MySQL Deadlock during transaction on ' . $caller . '. Retry transaction in ' . floor($sleep / 1000) . 'ms (try ' . ($i) . '/' . $retries . ')');
305+
usleep($sleep);
306+
} else if ($sqlstateErrorCode == 40001 && $errorNo == 1213) {
307+
// too many deadlocks
308+
$this->Syslog->info('Couldn\'t solve deadlock for ' . $caller . '. Ignore for now to prevent crash: Exception: ' . $Exception->getMessage());
309+
} else {
310+
// Any other error
311+
$this->Syslog->error($query->queryString);
312+
$this->Syslog->error("Run the worker in foreground mode to see the full query: https://statusengine.org/worker/#debugging");
313+
314+
// This function has no return - so no log file -.-
315+
// If Statusengine is running via systemd systemd will write the messages to syslog
316+
if ($this->isDumpOfMysqlQueryParametersEnabled) {
317+
$query->debugDumpParams();
318+
}
319+
320+
if ($errorString == 'MySQL server has gone away') {
321+
$this->reconnect();
322+
throw new StorageBackendUnavailableExceptions($errorString);
323+
}
324+
}
305325
}
306326
}
307-
return $result;
327+
return false;
308328
}
309329

310330
/**

‎src/Backends/MySQL/SqlObjects/MysqlHostAcknowledgement.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public function insert($isRecursion = false) {
7878
$query->bindValue($i++, (int)$this->Acknowledgement->isNotifyContacts());
7979

8080
try {
81-
return $this->MySQL->executeQuery($query);
81+
return $this->MySQL->executeQuery($query, 'MysqlHostAcknowledgement');
8282
} catch (StorageBackendUnavailableExceptions $Exceptions) {
8383
//Retry
8484
if ($isRecursion === false) {

‎src/Backends/MySQL/SqlObjects/MysqlHostDowntimehistory.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public function saveDowntime(Downtime $Downtime, $isRecursion = false) {
8383
}
8484

8585
try {
86-
return $this->MySQL->executeQuery($query);
86+
return $this->MySQL->executeQuery($query, 'MySQLHostDowntimehistory');
8787
} catch (StorageBackendUnavailableExceptions $Exceptions) {
8888
//Retry
8989
if ($isRecursion === false) {
@@ -108,7 +108,7 @@ public function deleteDowntime(Downtime $Downtime, $isRecursion = false) {
108108
$query->bindValue(4, $Downtime->getDowntimeId());
109109

110110
try {
111-
return $this->MySQL->executeQuery($query);
111+
return $this->MySQL->executeQuery($query, 'MySQLHostDowntimehistory');
112112
} catch (StorageBackendUnavailableExceptions $Exceptions) {
113113
//Retry
114114
if ($isRecursion === false) {

‎src/Backends/MySQL/SqlObjects/MysqlHostScheduleddowntime.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public function saveDowntime(Downtime $Downtime, $isRecursion = false) {
8888
}
8989

9090
try {
91-
return $this->MySQL->executeQuery($query);
91+
return $this->MySQL->executeQuery($query, 'MysqlHostScheduleddowntime');
9292
} catch (StorageBackendUnavailableExceptions $Exceptions) {
9393
//Retry
9494
if ($isRecursion === false) {
@@ -113,7 +113,7 @@ public function deleteDowntime(Downtime $Downtime, $isRecursion = false){
113113
$query->bindValue(4, $Downtime->getDowntimeId());
114114

115115
try {
116-
return $this->MySQL->executeQuery($query);
116+
return $this->MySQL->executeQuery($query, 'MysqlHostScheduleddowntime');
117117
} catch (StorageBackendUnavailableExceptions $Exceptions) {
118118
//Retry
119119
if ($isRecursion === false) {

‎src/Backends/MySQL/SqlObjects/MysqlHostcheck.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ public function insert($isRecursion = false){
9393
}
9494

9595
try {
96-
return $this->MySQL->executeQuery($query);
96+
return $this->MySQL->executeQuery($query, 'MysqlHostcheck');
9797
} catch (StorageBackendUnavailableExceptions $Exceptions) {
9898
//Retry
9999
if ($isRecursion === false) {

‎src/Backends/MySQL/SqlObjects/MysqlHoststatus.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ public function insert($isRecursion = false){
126126
}
127127

128128
try {
129-
return $this->MySQL->executeQuery($query);
129+
return $this->MySQL->executeQuery($query, 'MysqlHoststatus');
130130
} catch (StorageBackendUnavailableExceptions $Exceptions) {
131131
//Retry
132132
if ($isRecursion === false) {
@@ -143,7 +143,7 @@ public function truncate($isRecursion = false){
143143
$query = $this->MySQL->prepare('DELETE FROM statusengine_hoststatus WHERE node_name=?');
144144
$query->bindValue(1, $this->nodeName);
145145
try {
146-
return $this->MySQL->executeQuery($query);
146+
return $this->MySQL->executeQuery($query, 'MysqlHoststatus');
147147
} catch (StorageBackendUnavailableExceptions $Exceptions) {
148148
//Retry
149149
if ($isRecursion === false) {

‎src/Backends/MySQL/SqlObjects/MysqlLogentry.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public function insert($isRecursion = false){
8484
}
8585

8686
try {
87-
return $this->MySQL->executeQuery($query);
87+
return $this->MySQL->executeQuery($query, 'MysqlLogentry');
8888
} catch (StorageBackendUnavailableExceptions $Exceptions) {
8989
//Retry
9090
if ($isRecursion === false) {

‎src/Backends/MySQL/SqlObjects/MysqlNotification.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ public function getHostQuery($hostNotificationCache, $isRecursion = false) {
129129
}
130130

131131
try {
132-
return $this->MySQL->executeQuery($query);
132+
return $this->MySQL->executeQuery($query, 'MysqlNotification');
133133
} catch (StorageBackendUnavailableExceptions $Exceptions) {
134134
//Retry
135135
if ($isRecursion === false) {
@@ -169,7 +169,7 @@ public function getServiceQuery($serviceNotificationCache, $isRecursion = false)
169169
}
170170

171171
try {
172-
return $this->MySQL->executeQuery($query);
172+
return $this->MySQL->executeQuery($query, 'MysqlNotification');
173173
} catch (StorageBackendUnavailableExceptions $Exceptions) {
174174
//Retry
175175
if ($isRecursion === false) {

‎src/Backends/MySQL/SqlObjects/MysqlPerfdata.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public function insert($isRecursion = false) {
8282
}
8383

8484
try {
85-
return $this->MySQL->executeQuery($query);
85+
return $this->MySQL->executeQuery($query, 'MysqlPerfdata');
8686
} catch (StorageBackendUnavailableExceptions $Exceptions) {
8787
//Retry
8888
if ($isRecursion === false) {

0 commit comments

Comments
 (0)