You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
131 lines
4.8 KiB
131 lines
4.8 KiB
From d9833b68498e306d181be11adf9eee14b646a899 Mon Sep 17 00:00:00 2001 |
|
From: Damien Ciabrini <dciabrin@redhat.com> |
|
Date: Tue, 2 Feb 2016 14:34:36 +0100 |
|
Subject: [PATCH] galera: force crash recovery if needed during last commit |
|
detection |
|
|
|
--- |
|
heartbeat/galera | 90 +++++++++++++++++++++++++++++++++++++------------------- |
|
1 file changed, 60 insertions(+), 30 deletions(-) |
|
|
|
diff --git a/heartbeat/galera b/heartbeat/galera |
|
index 7be2b00..ca94c21 100755 |
|
--- a/heartbeat/galera |
|
+++ b/heartbeat/galera |
|
@@ -525,6 +525,58 @@ detect_first_master() |
|
set_bootstrap_node $best_node |
|
} |
|
|
|
+detect_last_commit() |
|
+{ |
|
+ local last_commit |
|
+ local recover_args="--defaults-file=$OCF_RESKEY_config \ |
|
+ --pid-file=$OCF_RESKEY_pid \ |
|
+ --socket=$OCF_RESKEY_socket \ |
|
+ --datadir=$OCF_RESKEY_datadir \ |
|
+ --user=$OCF_RESKEY_user" |
|
+ local recovered_position_regex='s/.*WSREP\:\s*[R|r]ecovered\s*position.*\:\(.*\)\s*$/\1/p' |
|
+ |
|
+ ocf_log info "attempting to detect last commit version by reading ${OCF_RESKEY_datadir}/grastate.dat" |
|
+ last_commit="$(cat ${OCF_RESKEY_datadir}/grastate.dat | sed -n 's/^seqno.\s*\(.*\)\s*$/\1/p')" |
|
+ if [ -z "$last_commit" ] || [ "$last_commit" = "-1" ]; then |
|
+ local tmp=$(mktemp) |
|
+ local tmperr=$(mktemp) |
|
+ |
|
+ ocf_log info "now attempting to detect last commit version using 'mysqld_safe --wsrep-recover'" |
|
+ |
|
+ ${OCF_RESKEY_binary} $recover_args --wsrep-recover > $tmp 2> $tmperr |
|
+ |
|
+ last_commit="$(cat $tmp | sed -n $recovered_position_regex)" |
|
+ if [ -z "$last_commit" ]; then |
|
+ # Galera uses InnoDB's 2pc transactions internally. If |
|
+ # server was stopped in the middle of a replication, the |
|
+ # recovery may find a "prepared" XA transaction in the |
|
+ # redo log, and mysql won't recover automatically |
|
+ |
|
+ cat $tmperr | grep -q -E '\[ERROR\]\s+Found\s+[0-9]+\s+prepared\s+transactions!' 2>/dev/null |
|
+ if [ $? -eq 0 ]; then |
|
+ # we can only rollback the transaction, but that's OK |
|
+ # since the DB will get resynchronized anyway |
|
+ ocf_log warn "local node <${NODENAME}> was not shutdown properly. Rollback stuck transaction with --tc-heuristic-recover" |
|
+ ${OCF_RESKEY_binary} $recover_args --wsrep-recover \ |
|
+ --tc-heuristic-recover=rollback > $tmp 2>/dev/null |
|
+ |
|
+ last_commit="$(cat $tmp | sed -n $recovered_position_regex)" |
|
+ fi |
|
+ fi |
|
+ rm -f $tmp $tmperr |
|
+ fi |
|
+ |
|
+ if [ ! -z "$last_commit" ]; then |
|
+ ocf_log info "Last commit version found: $last_commit" |
|
+ set_last_commit $last_commit |
|
+ return $OCF_SUCCESS |
|
+ else |
|
+ ocf_exit_reason "Unable to detect last known write sequence number" |
|
+ clear_last_commit |
|
+ return $OCF_ERR_GENERIC |
|
+ fi |
|
+} |
|
+ |
|
# For galera, promote is really start |
|
galera_promote() |
|
{ |
|
@@ -569,13 +620,15 @@ galera_demote() |
|
clear_bootstrap_node |
|
clear_last_commit |
|
|
|
- # record last commit by "starting" galera. start is just detection of the last sequence number |
|
- galera_start |
|
+ # record last commit for next promotion |
|
+ detect_last_commit |
|
+ rc=$? |
|
+ return $rc |
|
} |
|
|
|
galera_start() |
|
{ |
|
- local last_commit |
|
+ local rc |
|
|
|
echo $OCF_RESKEY_wsrep_cluster_address | grep -q $NODENAME |
|
if [ $? -ne 0 ]; then |
|
@@ -591,34 +644,11 @@ galera_start() |
|
|
|
mysql_common_prepare_dirs |
|
|
|
- ocf_log info "attempting to detect last commit version by reading ${OCF_RESKEY_datadir}/grastate.dat" |
|
- last_commit="$(cat ${OCF_RESKEY_datadir}/grastate.dat | sed -n 's/^seqno.\s*\(.*\)\s*$/\1/p')" |
|
- if [ -z "$last_commit" ] || [ "$last_commit" = "-1" ]; then |
|
- ocf_log info "now attempting to detect last commit version using 'mysqld_safe --wsrep-recover'" |
|
- local tmp=$(mktemp) |
|
- ${OCF_RESKEY_binary} --defaults-file=$OCF_RESKEY_config \ |
|
- --pid-file=$OCF_RESKEY_pid \ |
|
- --socket=$OCF_RESKEY_socket \ |
|
- --datadir=$OCF_RESKEY_datadir \ |
|
- --user=$OCF_RESKEY_user \ |
|
- --wsrep-recover > $tmp 2>&1 |
|
- |
|
- last_commit="$(cat $tmp | sed -n 's/.*WSREP\:\s*[R|r]ecovered\s*position.*\:\(.*\)\s*$/\1/p')" |
|
- rm -f $tmp |
|
- |
|
- if [ "$last_commit" = "-1" ]; then |
|
- last_commit="0" |
|
- fi |
|
- fi |
|
- |
|
- if [ -z "$last_commit" ]; then |
|
- ocf_exit_reason "Unable to detect last known write sequence number" |
|
- clear_last_commit |
|
- return $OCF_ERR_GENERIC |
|
+ detect_last_commit |
|
+ rc=$? |
|
+ if [ $rc -ne $OCF_SUCCESS ]; then |
|
+ return $rc |
|
fi |
|
- ocf_log info "Last commit version found: $last_commit" |
|
- |
|
- set_last_commit $last_commit |
|
|
|
master_exists |
|
if [ $? -eq 0 ]; then
|
|
|