From d9833b68498e306d181be11adf9eee14b646a899 Mon Sep 17 00:00:00 2001
From: Damien Ciabrini <dciabrin@redhat.com>
Date: Tue, 2 Feb 2016 14:34:36 +0100
Subject: [PATCH] galera: force crash recovery if needed during last commit
 detection

---
 heartbeat/galera | 90 +++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 60 insertions(+), 30 deletions(-)

diff --git a/heartbeat/galera b/heartbeat/galera
index 7be2b00..ca94c21 100755
--- a/heartbeat/galera
+++ b/heartbeat/galera
@@ -525,6 +525,58 @@ detect_first_master()
     set_bootstrap_node $best_node
 }
 
+detect_last_commit()
+{
+    local last_commit
+    local recover_args="--defaults-file=$OCF_RESKEY_config \
+                        --pid-file=$OCF_RESKEY_pid \
+                        --socket=$OCF_RESKEY_socket \
+                        --datadir=$OCF_RESKEY_datadir \
+                        --user=$OCF_RESKEY_user"
+    local recovered_position_regex='s/.*WSREP\:\s*[R|r]ecovered\s*position.*\:\(.*\)\s*$/\1/p'
+
+    ocf_log info "attempting to detect last commit version by reading ${OCF_RESKEY_datadir}/grastate.dat"
+    last_commit="$(cat ${OCF_RESKEY_datadir}/grastate.dat | sed -n 's/^seqno.\s*\(.*\)\s*$/\1/p')"
+    if [ -z "$last_commit" ] || [ "$last_commit" = "-1" ]; then
+        local tmp=$(mktemp)
+        local tmperr=$(mktemp)
+
+        ocf_log info "now attempting to detect last commit version using 'mysqld_safe --wsrep-recover'"
+
+        ${OCF_RESKEY_binary} $recover_args --wsrep-recover > $tmp 2> $tmperr
+
+        last_commit="$(cat $tmp | sed -n $recovered_position_regex)"
+        if [ -z "$last_commit" ]; then
+            # Galera uses InnoDB's 2pc transactions internally. If
+            # server was stopped in the middle of a replication, the
+            # recovery may find a "prepared" XA transaction in the
+            # redo log, and mysql won't recover automatically
+
+            cat $tmperr | grep -q -E '\[ERROR\]\s+Found\s+[0-9]+\s+prepared\s+transactions!' 2>/dev/null
+            if [ $? -eq 0 ]; then
+                # we can only rollback the transaction, but that's OK
+                # since the DB will get resynchronized anyway
+                ocf_log warn "local node <${NODENAME}> was not shutdown properly. Rollback stuck transaction with --tc-heuristic-recover"
+                ${OCF_RESKEY_binary} $recover_args --wsrep-recover \
+                                     --tc-heuristic-recover=rollback > $tmp 2>/dev/null
+
+                last_commit="$(cat $tmp | sed -n $recovered_position_regex)"
+            fi
+        fi
+        rm -f $tmp $tmperr
+    fi
+
+    if [ ! -z "$last_commit" ]; then
+        ocf_log info "Last commit version found:  $last_commit"
+        set_last_commit $last_commit
+        return $OCF_SUCCESS
+    else
+        ocf_exit_reason "Unable to detect last known write sequence number"
+        clear_last_commit
+        return $OCF_ERR_GENERIC
+    fi
+}
+
 # For galera, promote is really start
 galera_promote()
 {
@@ -569,13 +620,15 @@ galera_demote()
     clear_bootstrap_node
     clear_last_commit
 
-    # record last commit by "starting" galera. start is just detection of the last sequence number
-    galera_start
+    # record last commit for next promotion
+    detect_last_commit
+    rc=$?
+    return $rc
 }
 
 galera_start()
 {
-    local last_commit
+    local rc
 
     echo $OCF_RESKEY_wsrep_cluster_address | grep -q $NODENAME
     if [ $? -ne 0 ]; then
@@ -591,34 +644,11 @@ galera_start()
 
     mysql_common_prepare_dirs
 
-    ocf_log info "attempting to detect last commit version by reading ${OCF_RESKEY_datadir}/grastate.dat"
-    last_commit="$(cat ${OCF_RESKEY_datadir}/grastate.dat | sed -n 's/^seqno.\s*\(.*\)\s*$/\1/p')"
-    if [ -z "$last_commit" ] || [ "$last_commit" = "-1" ]; then
-        ocf_log info "now attempting to detect last commit version using 'mysqld_safe --wsrep-recover'"
-        local tmp=$(mktemp)
-        ${OCF_RESKEY_binary} --defaults-file=$OCF_RESKEY_config \
-            --pid-file=$OCF_RESKEY_pid \
-            --socket=$OCF_RESKEY_socket \
-            --datadir=$OCF_RESKEY_datadir \
-            --user=$OCF_RESKEY_user \
-            --wsrep-recover > $tmp 2>&1
-
-        last_commit="$(cat $tmp | sed -n 's/.*WSREP\:\s*[R|r]ecovered\s*position.*\:\(.*\)\s*$/\1/p')"
-        rm -f $tmp
-
-        if [ "$last_commit" = "-1" ]; then
-            last_commit="0"
-        fi
-    fi
-
-    if [ -z "$last_commit" ]; then
-        ocf_exit_reason "Unable to detect last known write sequence number"
-        clear_last_commit
-        return $OCF_ERR_GENERIC
+    detect_last_commit
+    rc=$?
+    if [ $rc -ne $OCF_SUCCESS ]; then
+        return $rc
     fi
-    ocf_log info "Last commit version found:  $last_commit"
-
-    set_last_commit $last_commit
 
     master_exists
     if [ $? -eq 0 ]; then