#!/bin/bash
#
# SAPHanaController
#
# Description:  Manages two SAP HANA Databases in System Replication
#
##############################################################################
#
# SAPHana
# Author:       Fabian Herschel, November 2013
# Support:      linux@sap.com
# License:      GNU General Public License (GPL)
# Copyright:    (c) 2013,2014 SUSE Linux Products GmbH
# Copyright:    (c) 2015-2016 SUSE Linux GmbH
# Copyright:    (c) 2017-2021 SUSE LLC
#
# An example usage:
#      See usage() function below for more details...
#
# OCF instance parameters:
#   OCF_RESKEY_SID
#   OCF_RESKEY_InstanceNumber
#   OCF_RESKEY_DIR_EXECUTABLE   (optional, well known directories will be searched by default)
#   OCF_RESKEY_DIR_PROFILE      (optional, well known directories will be searched by default)
#   OCF_RESKEY_INSTANCE_PROFILE (optional, well known directories will be searched by default)
#   OCF_RESKEY_PREFER_SITE_TAKEOVER (optional, default is no)
#   OCF_RESKEY_DUPLICATE_PRIMARY_TIMEOUT (optional, time difference needed between two last-primary-tiemstampe (lpt))
#
# HANA must support the following commands:
#     hdbnsutil -sr_stateConfiguration (unsure, if this means >= SPS110, SPS111 or SPS10x)
#     hdbnsutil -sr_takeover
#     hdbnsutil -sr_register
#     landscapeHostConfiguration.py
#     systemReplicationStatus.py (>= SPS090)
#
#######################################################################
SAPHanaControllerVersion="0.180.0.0628.1823"
# Resource Agent Generation
RAG="2.0"

#
# Initialization:
timeB=$(date '+%s')

: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs

#
#######################################################################
#
# KNOWN PROBLEMS TO BE FIXED:
# P001 - Setup with scale-out and PREFER_SITE_TAKEOVER=true, AUTOMATED_REGISTER=true. If you kill a primary instance it could happen that the primary sets itself to lpt=10 and the secondary will be set to SFAIL and lpt=10 this results in a WAITING4LPA situation. ==> A down/dying primary may never set SFAIL for a secondary!
# P002 - in the swarm non master-nameserver nodes may never set the lpt=date
# P003 - in the swarm non master nodes may NOT do a full master-walk
# P004 - Monitor on "dying" primary and failing systemReplicationStatus script set secondary to SFAIL, so local restart was processed instead of takeover
# DONE PRIO 1: AFTER(!) SAP HANA SPS12 is available we could use hdbnsutil --sr_stateConfiguration
log_attributes=false
if ocf_is_true "$log_attributes"; then
    log_attr_file="/var/log/fhATTRIBUTES"
else
    log_attr_file="/dev/null"
fi


HANA_STATE_PRIMARY=0
HANA_STATE_SECONDARY=1
HANA_STATE_STANDALONE=2
HANA_STATE_DEFECT=3
HANA_CALL_TIMEOUT=120
HANA_STD_ACTION_TIMEOUT=3600

debug_attributes=0

SH=/bin/sh

#
# function: super_ocf_log - wrapper function for ocf log in order catch usual logging into super log
# params:   LOG_MESSAGE
# globals:  SAPHanaFilter
function super_ocf_log() {
    local level="$1"
    local message="$2"
    local skip=1
    local mtype=""
    local search=0
    local shf="${SAPHanaFilter:-all}"
    # message levels: (dbg)|info|warn|err|error
    # message types:  (ACT|RA|FLOW|DBG|LPA|DEC|DBG2...
    case "$level" in
        debug | dbg | warn | err | error ) skip=0
        ;;
        info )
        case "$shf" in
            all) skip=0
            ;;
            none )
                skip=1
                ;;
            * ) mtype=${message%% *}
                mtype=${mtype%:}
                mtype=${mtype#fh}
                echo "$shf"|  grep -iq ${mtype}; search=$?
                if [ $search -eq 0 ]; then
                     skip=0
                else
                    skip=1
                fi
            ;;
        esac
        ;;
    esac
    if [ $skip -eq 0 ]; then
        ocf_log "$level" "$message"
    fi
}

#
# function: saphana_usage - short usage info
# params:   -
# globals:  $0(r)
#
function saphana_usage() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0
    methods=$(saphana_methods)
    methods=$(echo $methods | tr ' ' '|')
    cat <<-EOF
    usage: $0 ($methods)

    $0 manages two SAP HANA databases in system replication.

    The 'start'        operation starts the HANA instance or bring the "clone instance" to a WAITING status
    The 'stop'         operation stops the HANA instance
    The 'status'       operation reports whether the HANA instance is running
    The 'monitor'      operation reports whether the HANA instance seems to be working in multi-state it also needs to check the system replication status
    The 'promote'      operation either runs a takeover for a secondary or a just-nothing for a primary
    The 'demote'       operation nearly does nothing and just mark the instance as demoted
    The 'notify'       operation always returns SUCCESS
    The 'validate-all' operation reports whether the parameters are valid
    The 'methods'      operation reports on the methods $0 supports
    The 'reload'       operation allows to adapt resource parameters

EOF
    return $rc
}

function backup_global_and_nameserver() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0
    cp /hana/shared/${SID}/global/hdb/custom/config/global.ini /hana/shared/${SID}/global/hdb/custom/config/global.ini.$(date +"%s")
    cp /hana/shared/${SID}/global/hdb/custom/config/nameserver.ini /hana/shared/${SID}/global/hdb/custom/config/nameserver.ini.$(date +"%s")
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: saphana_meta_data - print resource agent meta-data for cluster
# params:   -
# globals:  -
#
function saphana_meta_data() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0
# TODO PRIO2: Description including parameters to be reviewed for scale-out
# TODO PRIO2: check whether default is PREFER_SITE_TAKEOVER=false, then correct this meta-data.
    cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="SAPHanaController">
<version>$SAPHanaControllerVersion</version>

<shortdesc lang="en">Manages two SAP HANA database systems in system replication (SR).</shortdesc>
<longdesc lang="en">
The SAPHanaSR resource agent manages two SAP HANA database systems which are configured
in system replication. SAPHanaController supports Scale-Out scenarios.

Managing the two SAP HANA database systems means that the resource agent controls the start/stop of the
instances. In addition the resource agent is able to monitor the SAP HANA databases to check their
availability on landscape host configuration level. For this monitoring the resource agent relies on interfaces
provided by SAP. A third task of the resource agent is to also check the synchronisation status
of the two SAP HANA databases. If the synchronisation is not "SOK", then the cluster avoids to
failover to the secondary side, if the primary fails. This is to improve the data consistency.

The resource agent uses the following four interfaces provided by SAP:

1. sapcontrol/sapstartsrv
   The interface sapcontrol/sapstartsrv is used to start/stop a HANA database instance/system

2. landscapeHostConfiguration
   The interface is used to monitor a HANA system. The python script is named landscapeHostConfiguration.py.
   landscapeHostConfiguration.py has some detailed output about HANA system status
   and node roles. For our monitor the overall status is relevant. This overall
   status is reported by the return code of the script:
   0: Internal Fatal, 1: ERROR, 2: WARNING, 3: INFO, 4: OK
   The SAPHanaController resource agent will interpret return codes 0 as FATAL, 1 as not-running or ERROR and and return codes 2+3+4 as RUNNING.

3. hdbnsutil
   The interface hdbnsutil is used to check the "topology" of the system replication as well as the current configuration
   (primary/secondary) of a SAP HANA database instance. A second task of the interface is the possibility to run a
   system replication takeover (sr_takeover) or to register a former primary to a newer one (sr_register).

4. hdbsql / systemReplicationStatus
   Interface is SQL query into HANA (system replication table).  The hdbsql query will be replaced by a python script
   "systemReplicationStatus.py" in SAP HANA SPS8 or 9.
   As long as we need to use hdbsql you need to set up secure store users for linux user root to be able to
   access the SAP HANA database. You need to configure a secure store user key "SAP HANA${SID}SR" which can connect the SAP
   HANA database:

5. saphostctrl
   The interface saphostctrl uses the function ListInstances to figure out the virtual host name of the
   SAP HANA instance. This is the hostname used during the HANA installation.

</longdesc>
<parameters>
    <parameter name="SID" unique="0" required="1">
        <longdesc lang="en">SAP System Identifier (SID) like "SLE" or "HAE"</longdesc>
        <shortdesc lang="en">SAP System Identifier (SID)</shortdesc>
        <content type="string" default="" />
    </parameter>
    <parameter name="InstanceNumber" unique="0" required="1">
        <longdesc lang="en">SAP instance number like "00" or "07"</longdesc>
        <shortdesc lang="en">SAP instance number</shortdesc>
        <content type="string" default="" />
    </parameter>
    <parameter name="PREFER_SITE_TAKEOVER" unique="0" required="0">
        <longdesc lang="en">Should cluster/RA prefer to switchover to secondary instance instead of restarting master locally? Default="yes"
        no: Do prefer restart locally
        yes: Do prefer takeover to remote site
        never: Do never run a sr_takeover (promote) at the secondary side. THIS VALUE IS CURRENTLY NOT SUPPORTED.
        </longdesc>
        <shortdesc lang="en">Local or site recover preferred?</shortdesc>
        <content type="string" default="false" />
    </parameter>
    <parameter name="AUTOMATED_REGISTER"  unique="0" required="0">
        <shortdesc lang="en">Define, if a former primary should automatically be registered.</shortdesc>
        <longdesc lang="en">The parameter AUTOMATED_REGISTER defines, whether a former primary instance should
             be registered automatically by the resource agent during cluster/resource start, if  the DUPLICATE_PRIMARY_TIMEOUT is expired... TDB
        </longdesc>
        <content type="boolean" default="false" />
    </parameter>
    <parameter name="DUPLICATE_PRIMARY_TIMEOUT" unique="0" required="0">
        <shortdesc lang="en">Time difference needed between to primary time stamps, if a dual-primary situation occurs</shortdesc>
        <longdesc lang="en">Time difference needed between to primary time stamps,
        if a dual-primary situation occurs. If the time difference is
        less than the time gap, then the cluster holds one or both instances in a "WAITING" status. This is to give an admin
        a chance to react on a failover. A failed former primary will be registered after the time difference is passed. After
        this registration to the new primary all data will be overwritten by the system replication.
        </longdesc>
        <content type="string" default="7200" />
    </parameter>
    <parameter name="HANA_CALL_TIMEOUT" unique="0" required="0">
        <shortdesc lang="en">Define timeout how long a call to HANA to receive information can take.</shortdesc>
        <longdesc lang="en">Define timeout how long a call to HANA to receive information can take. This could be eg landscapeHostConfiguration.py.
          There are some specific calls to HANA which have their own timeout values. For example the takeover command does not timeout (inf).
          If the timeout is reached, the return code will be 124. If you increase the timeouts for HANA calls you should also adjust the operation timeouts
          of your cluster resources.
        </longdesc>
        <content type="string" default="120" />
    </parameter>
    <parameter name="DIR_EXECUTABLE" unique="0" required="0">
        <longdesc lang="en">The full qualified path where to find sapstartsrv and sapcontrol. Specify this parameter, if you have changed the SAP kernel directory location after the default SAP installation.</longdesc>
        <shortdesc lang="en">Path of sapstartsrv and sapcontrol</shortdesc>
        <content type="string" default="" />
    </parameter>
    <parameter name="DIR_PROFILE" unique="0" required="0">
        <longdesc lang="en">The full qualified path where to find the SAP START profile. Specify this parameter, if you have changed the SAP profile directory location after the default SAP installation.</longdesc>
        <shortdesc lang="en">Path of start profile</shortdesc>
        <content type="string" default="" />
    </parameter>
    <parameter name="INSTANCE_PROFILE" unique="1" required="0">
        <longdesc lang="en">The name of the SAP HANA instance profile. Specify this parameter, if you have changed the name of the SAP HANA instance profile after the default SAP installation. Normally you do not need to set this parameter.</longdesc>
        <shortdesc lang="en">HANA instance profile name</shortdesc>
        <content type="string" default="" />
    </parameter>
</parameters>

<actions>
    <action name="start"   timeout="3600" />
    <action name="stop"    timeout="3600" />
    <action name="status"  timeout="60" />
    <action name="monitor" depth="0" timeout="700" interval="60" />
    <action name="monitor" depth="0" timeout="700" interval="61" role="Slave" />
    <action name="monitor" depth="0" timeout="700" interval="60" role="Master" />
    <action name="promote" timeout="3600" />
    <action name="demote"  timeout="320" />
    <action name="validate-all" timeout="5" />
    <action name="meta-data" timeout="5" />
    <action name="methods" timeout="5" />
    <action name="reload" timeout="5" />
</actions>
</resource-agent>
END
return $rc
}

#
# function: saphana_methods - report supported cluster methods
# params:   -
# globals:  -
# methods: What methods/operations do we support?
#
function saphana_methods() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0 m
    for m in start stop status monitor promote demote notify validate-all methods meta-data usage reload; do
        echo "$m"
    done
    return $rc
}

#
# function: dequote - filter: remove quotes (") from stdin
# params:   -
# globals:  -
function dequote()
{
    local rc=0; tr -d '"'; return $rc
}

# function: version: compare two HANA version strings
function ver_lt() {
    ocf_version_cmp $1 $2
    test $? -eq 0 && return 0 || return 1
}

function ver_le() {
    ocf_version_cmp $1 $2
    test $? -eq 0 -o $? -eq 1 && return 0 || return 1
}

function ver_gt() {
    ocf_version_cmp $1 $2
    test $? -eq 2 && return 0 || return 1
}

function ver_ge() {
    ocf_version_cmp $1 $2
    test $? -eq 2 -o $? -eq 1 && return 0 || return 1
}
#
# function: version: compare two HANA version strings
#
function version() {
    if [ $# -eq 3 ]; then
        case "$2" in
            LE | le | "<=" ) ver_le $1 $3;;
            LT | lt | "<" ) ver_lt $1 $3;;
            GE | ge | ">=" ) ver_ge $1 $3;;
            GT | gt | ">" ) ver_gt $1 $3;;
            * ) return 1;
        esac
    elif [ $# -ge 5 ]; then
        version $1 $2 $3 && shift 2 && version $*
    else
        return 1;
    fi
}

#
# function: get_saphana_waitfor_timeout
# output:  SAP HANA Waitfor... timeout
#
# timeOut = max(3600, 95%(actionTimeOut))
#
function get_saphana_waitfor_timeout() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    # $OCF_RESKEY_CRM_meta_timeout is the timeout of the current running action in ms
    local actionTimeOut="$OCF_RESKEY_CRM_meta_timeout"
    local stdTimeOut="$HANA_STD_ACTION_TIMEOUT"
    local actTimeOutPercent=95 # 95% to left 5% for the rest of the resource agent action
    local hanaAction="$1"
    if [ -z "$actionTimeOut" ]; then
        actionTimeOut="$stdTimeOut"
    else
        # actionTimeOut in seconds
        ((actionTimeOut = actionTimeOut/1000))
    fi

    # 95%(actionTimeOut)
    ((timeout = actionTimeOut * actTimeOutPercent/100))
    # max(3600, 95%(actionTimeOut))
    if [ -z "$timeout" ] || [ "$timeout" -lt "$stdTimeOut" ]; then
        timeout=$stdTimeOut
        super_ocf_log info "DEC: SAP HANA $hanaAction timeout is '$stdTimeOut'"
    else
        super_ocf_log info "DEC: SAP HANA $hanaAction timeout is '$timeout', was default '$stdTimeOut'"
    fi
    super_ocf_log info "FLOW $FUNCNAME"
    echo $timeout
}

#
# function: is_clone - report, if resource is configured as a clone (also multi-state)
# params:   -
# globals:  OCF_*(r)
# descript: is_clone : find out if we are configured to run in a multi-state configuration
# rc: 0: it is a clone, 1: it is not a clone
#
# DONE: PRIO2: For the first shipment (scale-out) we need to limit the clones to 2
#
function is_clone() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0
    #
    # is a clone config?
    #
    if [ -n "$OCF_RESKEY_CRM_meta_clone_max" ] \
       && [ "$OCF_RESKEY_CRM_meta_clone_max" -gt 0 ]; then
       #
       # yes it is a clone config - check, if its configured well
       #
        if [ "$OCF_RESKEY_CRM_meta_clone_node_max" -ne 1 ] || \
            [ "$OCF_RESKEY_CRM_meta_master_node_max" -ne 1 ] || \
            [ "$OCF_RESKEY_CRM_meta_master_max" -ne 1 ]; then
                super_ocf_log err "ACT: Clone options misconfigured. (expect: clone_node_max=1,master_node_max=1,master_max=1)"
                exit $OCF_NOT_RUNNING
        fi
        rc=0;
    else
        rc=1;
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: get_hana_attribute
# params:   NODE ATTR [STORE]
# globals:  -
# output:   attribute value
#
function get_hana_attribute()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0
    local attr_node=$1
    local attr_name=$2
    local attr_store=${3:-reboot} # DONE: PRIO5 get this (optional) from parameter
    local attr_default=${4:-}
    local dstr
    local attr_val=""
    dstr=$(date)
    case "$attr_store" in
        reboot | forever )
            if ocf_is_true "$log_attributes"; then
                echo "$dstr: SAPHanaController: crm_attribute -N ${attr_node} -G -n \"$attr_name\" -l $attr_store -q" >> $log_attr_file
            fi
            crm_attribute -N ${attr_node} -G -n "$attr_name" -l $attr_store -q -d "$attr_default" 2>>$log_attr_file; rc=$?
            ;;
        props )
            if ocf_is_true "$log_attributes"; then
                echo "$dstr: SAPHanaController: crm_attribute -G -n \"$attr_name\" -t crm_config -q" >> $log_attr_file
            fi
            crm_attribute -G -n "$attr_name" -t crm_config -q -d "$attr_default" 2>>$log_attr_file; rc=$?
            ;;
    esac
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: set_hana_attribute - set the multi-state status of a node
# params:   NODE VALUE ATTR [STORE]
# globals:  -
#
function set_hana_attribute()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local attr_node=$1
    local attr_value=$2
    local attr_name=$3
    local attr_store=${4:-reboot} # DONE: PRIO5 get this (optional) from parameter
    local attr_default=${5:-}
    local rc=1
    local attr_old=""
    attr_old=$(get_hana_attribute $attr_node $attr_name $attr_store $attr_default); get_rc=$?
    if [ "$attr_old" != "$attr_value" ]; then
        super_ocf_log debug "DBG: SET attribute $attr_name for node ${attr_node} to ${attr_value} former ($attr_old) get_rc=$get_rc "
        dstr=$(date)
        case "$attr_store" in
            reboot | forever )
                if ocf_is_true "$log_attributes"; then
                    echo "$dstr: SAPHanaController: crm_attribute -N $attr_node -v \"$attr_value\" -n \"$attr_name\" -l $attr_store" >> $log_attr_file
                fi
                crm_attribute -N $attr_node -v "$attr_value" -n "$attr_name" -l $attr_store 2>>$log_attr_file; rc=$?
                ;;
            props )
                if ocf_is_true "$log_attributes"; then
                    echo "$dstr: SAPHanaController: crm_attribute -v \"$attr_value\" -n \"$attr_name\" -t crm_config -s SAPHanaSR" >> $log_attr_file
                fi
                crm_attribute -v "$attr_value" -n "$attr_name" -t crm_config  -s SAPHanaSR 2>>$log_attr_file; rc=$?
                ;;
        esac
    else
        super_ocf_log debug "DBG: LET attribute $attr_name for node ${attr_node} still be ${attr_value}"
        rc=0
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: assert - quickly go out of here with minimal error/return code handling and log
# params:   MESSAGE
# globals:  OCF_*(r)
#
function assert() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local err_msg=$1
    local default_rc=$OCF_NOT_RUNNING
    # DONE: Check, if we need to distinguish between probe and others
    if ocf_is_probe; then
        default_exit=$OCF_NOT_RUNNING
    else
        default_exit=$OCF_NOT_RUNNING
    fi
    if [ "$ACTION" = "stop" ]; then
        cleanup_instance
        exit $OCF_SUCCESS
    fi
    super_ocf_log err "ACT: $err_msg"
    exit $OCF_NOT_RUNNING
}

#
# function: set_crm_master - set the crm master score of the local node
# params:   SCORE  [NODE]
# globals:  HA_SBIN_DIR(r), OCF_RESOURCE_INSTANCE(r)
#
function set_crm_master()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0
    local score=0
    local node=""
    if [ -n "$1" ]; then
        score=$1
    fi
    if [ -n "$2" ]; then
        node="-N $2"
    fi
    # DONE: PRIO2: Only adjust master if value is really different (try to check that)
    oldscore=$(${HA_SBIN_DIR}/crm_master $node -G -q -l reboot)
    if [ "$oldscore" != "$score" ]; then
       dstr=$(date)
       if ocf_is_true "$log_attributes"; then
           echo "$dstr: SAPHanaController: crm_master $node -v $score -l reboot " >> $log_attr_file
       fi
       super_ocf_log debug "DBG: SET crm master: $node $score (old: $oldscore)"
       ${HA_SBIN_DIR}/crm_master $node -v $score -l reboot; rc=$?
    else
       super_ocf_log debug "DBG: LET crm master: $score"
       rc=0
    fi
    return $rc
}

#
# function: scoring_crm_master - score instance due to role and sync match (table SCORING_TABLE)
# params:   NODE_ROLES NODE_SYNC_STATUS
# globals:  SCORING_TABLE[@],
#
scoring_crm_master()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local roles="$1"
    local sync="$2"
    local skip=0
    local myScore=""
    for scan in "${SCORING_TABLE[@]}"; do
        if [ $skip -eq 0 ]; then
            read rolePatt syncPatt score <<< $scan
            if grep "$rolePatt" <<< "$roles"; then
               if grep "$syncPatt" <<< "$sync"; then
                  super_ocf_log info "SCORE: scoring_crm_master: roles($roles) are matching pattern ($rolePatt)"
                  super_ocf_log info "SCORE: scoring_crm_master: sync($sync) is  matching syncPattern ($syncPatt)"
                  super_ocf_log info "SCORE: scoring_crm_master: set score $score"
                  skip=1
                  myScore=$score
               fi
            fi
        fi
    done
    super_ocf_log debug "DBG: scoring_crm_master adjust score $myScore"
    # DONE: PRIO1: DO Not Score, If we did not found our role/sync at this moment - bsc#919925
    if [ -n "$myScore" ]; then
        set_crm_master $myScore
    fi
}

#
# function: get_crm_master - get the crm master score of the local node
# params:   -
# globals:  HA_SBIN_DIR(r)
#
function get_crm_master()
{
  super_ocf_log info "FLOW $FUNCNAME ($*)"
  local rc=0
   ${HA_SBIN_DIR}/crm_master -G -q -l reboot; rc=$?
   return $rc
}

#
# function: HANA_CALL
# params:   timeout-in-seconds cmd-line
# globals:  sid(r), SID(r), InstanceName(r)
#
function HANA_CALL()
{
    #
    # TODO: PRIO 5: remove 'su - ${sidadm} later, when SAP HANA resoled issue with
    #       root-user-called hdbnsutil -sr_state (which creates root-owned shared memory file in /var/lib/hdb/SID/shmgrp)
    #       would hdbnsutil -sr_stateConfiguration also create such a shared memory file?
    # TODO: PRIO 5: Maybe make "su" optional by a parameter
    local timeOut=0
    local onTimeOut=""
    local rc=0
    local use_su=1 # Default to be changed later (see TODO above)
    local pre_cmd=""
    local cmd=""
    local pre_script=""
    local output=""
    while [ $# -gt 0 ]; do
        case "$1" in
            --timeout ) timeOut=$2; shift;;
            --use-su  ) use_su=1;;
            --on-timeout ) onTimeOut="$2"; shift;;
            --cmd ) shift; cmd="$*"; break;;
        esac
        shift
    done

    if [ $use_su -eq 1 ]; then
        pre_cmd="su - ${sid}adm -c"
        pre_script="true"
    else
        # as root user we need the library path to the SAP kernel to be able to call sapcontrol
        # check, if we already added DIR_EXECUTABLE at the beginning of LD_LIBRARY_PATH
        if [ "${LD_LIBRARY_PATH%%*:}" != "$DIR_EXECUTABLE" ]
        then
            MY_LD_LIBRARY_PATH=$DIR_EXECUTABLE${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH
        fi
        pre_cmd="bash -c"
        pre_script="LD_LIBRARY_PATH=$MY_LD_LIBRARY_PATH; export LD_LIBRARY_PATH"
    fi
    case $timeOut in
        0 | inf )
                  output=$($pre_cmd "$pre_script; /usr/sap/$SID/$InstanceName/HDBSettings.sh $cmd"); rc=$?
                  ;;
        *       )
                  output=$(timeout --foreground -s 9 $timeOut $pre_cmd "$pre_script; timeout -s 9 $timeOut /usr/sap/$SID/$InstanceName/HDBSettings.sh $cmd"); rc=$?
                  #
                  # on timeout ...
                  #
                  if [ "$rc" -eq 124 ]; then
                      super_ocf_log warn "RA: HANA_CALL TIMEOUT after $timeOut seconds running command '$cmd'"
                      if [ -n "$onTimeOut" ]; then
                          local second_output=""
                          second_output=$($pre_cmd "$pre_script; /usr/sap/$SID/$InstanceName/HDBSettings.sh $onTimeOut");
                      fi
                  fi
                 ;;
    esac
    super_ocf_log debug "DBG: HANA_CALL '$cmd' rc=$rc output=$output"
    echo "$output"
    return $rc;
}

# function get_SRHOOK
# fetches the global Hook SR status (for compatibility reason till RA update of all cluster nodes are done)
# or site specific Hook SR status or (as fallback) the polling-attribute
# params: site-name
# globals: ATTR_NAME_HANA_GLOB_SRHOOK[@](r), ATTR_NAME_HANA_SRHOOK[@](r), ATTR_NAME_HANA_SYNC_STATUS[@](r)
# locals:  ATTR[@], my_sync, old_sync, get_sr_name
#
function get_SRHOOK()
{
    local -a ATTR my_sync old_sync
    local get_sr_name="$1"
    # first check, if the new, site specific hook attribute is available
    # if yes, use this as 'my_sync', else check for old, global hook attribute
    # as the first fallback. If not available, our last resort is the
    # global sync state attribute
    ATTR=(${ATTR_NAME_HANA_SRHOOK[@]}); ATTR[0]="${ATTR[0]}_$get_sr_name"
    my_sync=$(get_hana_attribute "X" "${ATTR[@]}")
    super_ocf_log info "RA: SRHOOK0=$my_sync"
    if [ -z "$my_sync" ]; then
        # no new, site specific hook attribute found, try the old, global
        # hook attribute as fallback
        my_sync=$(get_hana_attribute "X" "${ATTR_NAME_HANA_GLOB_SRHOOK[@]}")
        super_ocf_log info "RA: SRHOOK1=$my_sync"
    fi
    # no new, site specific hook attribute found, but also no old, global
    # hook attribute found, so use global sync state attribute as fallback
    if [ -z "$my_sync" ] || [ "$my_sync" = "SWAIT" ]; then
        my_sync=$(get_hana_attribute "X" "${ATTR_NAME_HANA_SEC_SYNC_STATUS[@]}")
        super_ocf_log info "RA: SRHOOK2=$my_sync"
    fi
    # still unable to determine any SR attribute, so be pessimistic
    if [ -z "$my_sync" ]; then
        my_sync="SFAIL"
    fi
    super_ocf_log info "RA: SRHOOK3=$my_sync"
    echo "$my_sync"
    return 0
}

# function set_SRHOOK - sets the global or site specific Hook SR status
# params: site-name, value
# globals: ATTR_NAME_HANA_SRHOOK[@](r), ATTR_NAME_HANA_SYNC_STATUS[@](r)
# locals:  ATTR[@], my_sync, get_sr_name
#
function set_SRHOOK()
{
    local -a ATTR my_sync old_sync
    local get_sr_name="$1"
    local value="$2"
    # for compatibility reasons with 'old', non multi-target environments
    case "$value" in
    "SOK"|"SFAIL"|"SWAIT")
        set_global="ok" ;;
    *)
        set_global="nok" ;;
    esac
    mts=true
    srHook_gen=$(get_hana_attribute "${NODENAME}" "${ATTR_NAME_HANA_SRHOOK_GEN[@]}")
    if [ "$srHook_gen" == "" ] || [ "$srHook_gen" == "1.0" ]; then
        # no Hook is used or the old Hook from a previous package is used or
        # the new, but 'old-style' Hook is used
        # multi-target support not available, use global Hook attribute
        mts=false
    fi
    if ocf_is_true "$mts"; then
        super_ocf_log info "RA: set SRHOOK - multi-target support available, write site specific Hook attribute"
        old_sync=$(get_hana_attribute "X" "${ATTR_NAME_HANA_GLOB_SRHOOK[@]}")
        if [ -n "$old_sync" ] && [ "$set_global" == "ok" ]; then
            super_ocf_log info "RA: set SRHOOK - write still existing old, global Hook attribute too"
            # old, global attribute still exists, write both attributes
            set_hana_attribute "X" "$value" "${ATTR_NAME_HANA_GLOB_SRHOOK[@]}"
        fi
        ATTR=(${ATTR_NAME_HANA_SRHOOK[@]}); ATTR[0]="${ATTR[0]}_$get_sr_name"
        set_hana_attribute "X" "$value" "${ATTR[@]}"
    else
        if [ "$set_global" == "ok" ]; then
            super_ocf_log info "RA: set SRHOOK - no multi-target support, write old, global Hook attribute"
            set_hana_attribute "X" "$value" "${ATTR_NAME_HANA_GLOB_SRHOOK[@]}"
        fi
    fi
    return 0
}

#
# function: saphana_init - initialize variables for the resource agent
# params:   InstanceName
# globals:  OCF_*(r), SID(w), sid(rw), sidadm(w), InstanceName(w), InstanceNr(w), SAPVIRHOST(w), PreferSiteTakeover(w),
# globals:  sr_name(w), remoteHost(w), otherNodes(w), remSR_name(w)
# globals:  ATTR_NAME_HANA_SEC_SYNC_STATUS(w), ATTR_NAME_HANA_CLONE_STATE(w)
# globals:  DIR_EXECUTABLE(w), SAPSTARTSRV(w), SAPCONTROL(w), DIR_PROFILE(w), SAPSTARTPROFILE(w), LD_LIBRARY_PATH(w), PATH(w)
# globals:  LPA_DIRECTORY(w), SIDInstanceName(w), remoteNode(w), hdbSrQueryTimeout(w)
# globals:  NODENAME(w), vName(w), hdbver(w),
# saphana_init : Define global variables with default values, if optional parameters are not set
#
function saphana_init() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=$OCF_SUCCESS
    local clN
    # local site
    # two parameter models (for transition only)
    # OLD: InstanceName
    # NEW: SID InstanceNumber
    NODENAME=$(crm_node -n)
    SID=$OCF_RESKEY_SID
    InstanceNr=$OCF_RESKEY_InstanceNumber
    SIDInstanceName="${SID}_HDB${InstanceNr}"
    InstanceName="HDB${InstanceNr}"
    export SAPSYSTEMNAME=$SID
    HANA_CALL_TIMEOUT="${OCF_RESKEY_HANA_CALL_TIMEOUT:-120}"
    super_ocf_log debug "DBG: Used new method to get SID ($SID) and InstanceNr ($InstanceNr)"
    sid=$(echo "$SID" | tr [:upper:] [:lower:])
    sidadm="${sid}adm"
    # TODO PRIO3: Do we need a parameter for the RA to be able to adjust hdbSrQueryTimeout?
    hdbSrQueryTimeout=180
    # DONE: PRIO4: SAPVIRHOST might be different to NODENAME
    # DONE: PRIO1: ASK: Is the output format of ListInstances fix? Could we take that as an API? Answer: Yes
    # try to catch:  Inst Info : LNX - 42 - lv9041 - 740, patch 36, changelist 1444691
    # We rely on the following format: SID is word#4, SYSNR is word#6, vHost is word#8
    if [ -e /usr/sap/hostctrl/exe/saphostctrl ]; then
        vName=$(/usr/sap/hostctrl/exe/saphostctrl -function ListInstances \
            | awk '$4 == SID && $6 == SYSNR { print $8 }' SID=$SID SYSNR=$InstanceNr 2>/dev/null )
        super_ocf_log debug "DBG: ListInstances: $(/usr/sap/hostctrl/exe/saphostctrl -function ListInstances)"
    else
        super_ocf_log error "ERR: SAPHOSTAGENT is not installed at /usr/sap/hostctrl/exe (saphostctrl missing)"
    fi
    if [ -z "$vName" ]; then
       #
       # if saphostctrl does not know the answer, try to fallback to attribute provided by SAPHanaTopology
       #
       vName=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_VHOST[@]}" "$NODENAME");
    fi
    if [ -z "$vName" ]; then # last fallback if we are not able to figure out the virtual host name
       vName="$NODENAME"
    fi
    SAPVIRHOST=${vName}
    PreferSiteTakeover="$OCF_RESKEY_PREFER_SITE_TAKEOVER"
    AUTOMATED_REGISTER="${OCF_RESKEY_AUTOMATED_REGISTER:-false}"
    LPA_DIRECTORY=/var/lib/SAPHanaRA
    super_ocf_log debug "DBG: SID=$SID, sid=$sid, SIDInstanceName=$SIDInstanceName, InstanceName=$InstanceName, InstanceNr=$InstanceNr, SAPVIRHOST=$SAPVIRHOST"
    ocf_env=$(env | grep 'OCF_RESKEY_CRM')
    super_ocf_log debug "DBG: OCF: $ocf_env"
    #
    # attributes used to control the update from single-target replication
    # to mutli-target replication.
    # node specific, set, if the NewResourceAgent is running on the system
    # contains the resource agent generation ($RAG)
    ATTR_NAME_HANA_GRA=("hana_${sid}_gra" "forever")
    # attribute containing the current running srHook generation
    # node specific
    ATTR_NAME_HANA_SRHOOK_GEN=("hana_${sid}_gsh" "reboot")
    ATTR_NAME_HANA_CLONE_STATE=("hana_${sid}_clone_state" "reboot") # UNKNOWN?, DEMOTED, PROMOTED
    #ATTR_NAME_HANA_REMOTEHOST=("hana_${sid}_remoteHost" "forever")
    ATTR_NAME_HANA_SITE=("hana_${sid}_site" "forever")
    ATTR_NAME_HANA_ROLES=("hana_${sid}_roles" "reboot")
    ATTR_NAME_HANA_VHOST=("hana_${sid}_vhost" "forever")
    ATTR_NAME_HANA_STATUS=("hana_${sid}_status" "reboot")
    # TODO: PRIO0: Name of attribute
    ATTR_NAME_HANA_SRACTION=("hana_${sid}_sra" "reboot")   # Marks, if cluster in a phase requesting  a takeover (T?) or registration (R?)
    #
    # new "central" attributes
    #
    # new "central" attribute, containing the summarized status, if all
    # cluster nodes are already updated to the new RA or if there are still
    # nodes left running the old RA version
    # expected value "ok" - "nok", if there still old RAs running in the cluster
    ATTR_NAME_HANA_UPD_STATE=("hana_${sid}_glob_upd" "props")
    # attribut showing, if migration from global to site specific hook should
    # be done - customer choice
    ATTR_NAME_HANA_multiTargetSupport=("hana_${sid}_glob_mts" "props")
    #
    ATTR_NAME_HANA_PRIM=("hana_${sid}_glob_prim" "props")                        # SITE
    ATTR_NAME_HANA_SEC=("hana_${sid}_glob_sec" "props")                          # SITE
    ATTR_NAME_HANA_SEC_SYNC_STATUS=("hana_${sid}_glob_sync_state" "props")       # SOK, SFAIL
    ATTR_NAME_HANA_PRIMARY_AT=("hana_${sid}_glob_primary_at"   "props")          # Not used so far
    ATTR_NAME_HANA_SRMODE2=("hana_${sid}_glob_srmode" "props" "sync")            # sync syncmem async
    #
    # For MULTI-TARGET we need a site-specific attribute not a global one
    # but for compatibility we need to recognize the old global attribute too
    ATTR_NAME_HANA_GLOB_SRHOOK=("hana_${sid}_glob_srHook" "props")                    # external triggered SOK, SFAIL
    ATTR_NAME_HANA_FILTER=("hana_${sid}_glob_filter" "props" "ra-act-dec-lpa")
    ATTR_NAME_HANA_OPERATION_MODE=("hana_${sid}_glob_op_mode" "props" "logreplay")
    #
    # SITE based attributes
    #
    LPA_ATTR=("hana_${sid}_site_lpt" "props")
    #
    # for MULTI-TARGET the ra needs a per-site attribute - not only a global one
    #
    ATTR_NAME_HANA_SRHOOK=("hana_${sid}_site_srHook" "props")                    # external triggered SOK, SFAIL
    ATTR_NAME_HANA_LSS=("hana_${sid}_site_lss" "props")
    ATTR_NAME_HANA_SRR=("hana_${sid}_site_srr" "props")
    ATTR_NAME_HANA_MNS=("hana_${sid}_site_mns" "props")

    SAPHanaFilter=$(get_hana_attribute "X" "${ATTR_NAME_HANA_FILTER[@]}")
    #
    # TODO: PRIO4: Table for non-preferred-site-takeover
    #
    SCORING_TABLE_PREFERRED_SITE_TAKEOVER=(
       "[234]:P:master[123]:master     .*          150"
       "[234]:P:master[123]            .*          140"
       "[234]:P:master[123]:slave:.*:standby      .*          115"
       "[234]:P:master[123]:slave      .*          110"
       "[015]:P:master[123]:           .*           70"
       "[0-9]:P:master[123]:*:standby  .*           60"
       "[0-9]:P:slave:                 .*       -10000"
       "[234]:S:master[123]:master     SOK         100"
       "[234]:S:master[123]:slave      SOK          80"
       "[015]:S:master[123]:           SOK          70"
       "[0-9]:S:master[124]:*:standby  SFAIL    -22100"
       "[0-9]:S:slave:                 SOK      -12200"
       "[0-9]:S:slave:                 SFAIL    -22200"
       "[0-9]:S:                       .*       -32300"
       ".*                             .*       -33333"
    )
    SCORING_TABLE_PREFERRED_LOCAL_RESTART=(
       "[0-9]*:P:[^:]*:master          .*          150"
       "[0-9]*:P:[^:]*:slave           .*          140"
       "[0-9]*:P:[^:]*:\?              .*            0"
       "[0-9]*:P:[^:]*:-               .*            0"
       "[0-9]*:S:[^:]*:master          SOK         100"
       "[0-9]*:S:[^:]*:master          SFAIL -INFINITY"
       "[0-9]*:S:[^:]*:slave           SOK          10"
       "[0-9]*:S:[^:]*:slave           SFAIL -INFINITY"
       "[0-9]*:S:[^:]*:\?              .*            0"
       "[0-9]*:S:[^:]*:-               .*            0"
       ".*                             .*           -1"
    )
    SCORING_TABLE_PREFERRED_NEVER=(
       "[234]:P:master[123]:master     .*          150"
       "[234]:P:master[123]:slave      .*          110"
       "[015]:P:master[123]:           .*           70"
       "[0-9]:P:master[123]:*:standby  .*           60"
       "[0-9]:P:slave:                 .*       -10000"
       "[0-9]*:S:[^:]*:*:              SNA   -INFINITY"
       "[0-9]*:S:[^:]*:*:              .*    -INFINITY"
       ".*                             .*    -INFINITY"
    )
    SCORING_TABLE_PREFERRED_AGGRESSIVE=(
       "[34]:P:master[123]:master     .*          150"
       "[2]:P:master[123]:master     .*            50"
       "[34]:P:master[123]:slave      .*           30"
       "[2]:P:master[123]:slave      .*            20"
       "[015]:P:master[123]:           .*           20"
       "[0-9]:P:master[123]:*:standby  .*           20"
       "[0-9]:P:slave:                 .*       -10000"
       "[34]:S:master[123]:master     SOK         100"
       "[2]:S:master[123]:master      SOK          30"
       "[34]:S:master[123]:slave      SOK          80"
       "[2]:S:master[123]:slave       SOK          20"
       "[015]:S:master[123]:           SOK          20"
       "[0-9]:S:master[124]:*:standby  SFAIL    -22100"
       "[0-9]:S:slave:                 SOK      -12200"
       "[0-9]:S:slave:                 SFAIL    -22200"
       "[0-9]:S:                       .*       -32300"
       ".*                             .*       -33333"
    )
    if ocf_is_true $PreferSiteTakeover; then
        SCORING_TABLE=("${SCORING_TABLE_PREFERRED_SITE_TAKEOVER[@]}")
    else
        case "$PreferSiteTakeover" in
            never|NEVER|Never )
                SCORING_TABLE=("${SCORING_TABLE_PREFERRED_NEVER[@]}")
                ;;
            alex|ALEX|aggressive )
                SCORING_TABLE=("${SCORING_TABLE_PREFERRED_AGGRESSIVE[@]}")
                ;;
            * )
                SCORING_TABLE=("${SCORING_TABLE_PREFERRED_LOCAL_RESTART[@]}")
                ;;
        esac
    fi
    #
    DUPLICATE_PRIMARY_TIMEOUT="${OCF_RESKEY_DUPLICATE_PRIMARY_TIMEOUT:-7200}"
    super_ocf_log debug "DBG: DUPLICATE_PRIMARY_TIMEOUT=$DUPLICATE_PRIMARY_TIMEOUT"
    #
    # Determine list of other cluster nodes and store in otherNodes variable
    otherNodes=()
    case $(crm_attribute --type crm_config --name cluster-infrastructure -q) in
       *corosync* ) otherNodes=($(crm_node -l | awk '{ if ($2 != me) { print $2 }}' me=${NODENAME}));;
       *openais* ) otherNodes=($(crm_node -l | awk '$3 == "member" { if ($2 != me) { print $2 }}' me=${NODENAME}));;
       *cman*    ) otherNodes=($(crm_node -l | awk '{for (i=1; i<=NF; i++) { if ($i != me) { print $i }}}' me=${NODENAME}));;
    esac
    #
    #

    #  ATTR_NAME_HANA_SITE
    site=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_SITE[@]}");
    sr_name=${site%%:*}  # first field of site (separator :)
    sr_id=${site#*:}     # rest of site (separator :)
    sr_mode=$(get_hana_attribute "${NODENAME}" "${ATTR_NAME_HANA_SRMODE2[@]}")
    if [ -z "$sr_mode" ]; then
        sr_mode="sync"
    fi
    #
    # handle RA update state
    node_updated=$(get_hana_attribute "${NODENAME}" "${ATTR_NAME_HANA_GRA[@]}")
    onode_site=$(get_hana_attribute "${NODENAME}" "${ATTR_NAME_HANA_SITE[@]}")
    if [ -n "$onode_site" ] && [ "$node_updated" != "$RAG" ]; then
        # only set on HANA nodes, skip the decision maker / tiebreaker
        # attribute is empty or set by an older RA generation
        # update attribute to the new RA generation
        node_updated="$RAG"
        set_hana_attribute "${NODENAME}" "${node_updated}" "${ATTR_NAME_HANA_GRA[@]}"
    fi
    cluster_update_state=$(get_hana_attribute "X" "${ATTR_NAME_HANA_UPD_STATE[@]}")
    # evaluate and update/set the update state of the cluster only on a master
    # nameserver to prevent race conditions
    if is_master_nameserver; then
        nrOfNodes=1
        updNodes=1
        for onode in "${otherNodes[@]}"; do
            onode_site=$(get_hana_attribute "$onode" "${ATTR_NAME_HANA_SITE[@]}")
            if [ -z "$onode_site" ]; then
                # not a HANA node, may be the decision maker / tiebreaker
                continue
            fi
            (( nrOfNodes++ ))
            onode_updated=$(get_hana_attribute "$onode" "${ATTR_NAME_HANA_GRA[@]}")
            if [ -n "$onode_updated" ]; then
                if [ "$onode_updated" == "$RAG" ]; then
                    # only nodes with an equal RA generation as the local
                    # node are considered as 'updated nodes'
                    (( updNodes++ ))
                fi
            fi
        done
        if [ "$nrOfNodes" -eq "$updNodes" ]; then
            updState="ok"
        else
            oldNodes=$(("$nrOfNodes" - "$updNodes"))
            updState="nok"
        fi
        # if cluster_update_state is empty or the stored value does not match the
        # current running state 'updState' update the attribute and the variable
        if [ "$cluster_update_state" != "$updState" ]; then
            set_hana_attribute "X" "$updState" "${ATTR_NAME_HANA_UPD_STATE[@]}"
            cluster_update_state="$updState"
        fi
    fi
    super_ocf_log info "UPD: cluster update state is '$cluster_update_state'"
    # if NOT all cluster nodes are updated to the new RA, we still working
    # with the old, global Hook attribute
    if [ "$cluster_update_state" == "ok" ]; then
        # all cluster nodes are now updated to the new RA, now check, if
        # we can/need/should migrate the SRHook attribute
        # check which Hook generation is used on all cluster nodes
        srHook_gen=$(get_hana_attribute "${NODENAME}" "${ATTR_NAME_HANA_SRHOOK_GEN[@]}")
        if [ -z "$srHook_gen" ]; then
            # we are on the new RA package, but the new srHook code is currently
            # not active, no reload or restart of srHook on HANA side
            super_ocf_log info "HOOK: RA saphana_init - on the local cluster node '${NODENAME}' the srHook generation attribute is empty. May be the new srHook is currently not active, no reloaded or no restart of srHook on HANA side was done"
        fi
        for onode in "${otherNodes[@]}"; do
            onode_site=$(get_hana_attribute "$onode" "${ATTR_NAME_HANA_SITE[@]}")
            if [ -z "$onode_site" ]; then
                # not a HANA node, may be the decision maker / tiebreaker
                continue
            fi
            onode_srHook_gen=$(get_hana_attribute "$onode" "${ATTR_NAME_HANA_SRHOOK_GEN[@]}")
            if [ -z "$onode_srHook_gen" ]; then
                # we are on the new RA package, but the new srHook code is
                # currently not active, no reload or restart of srHook on HANA
                # side
                super_ocf_log info "HOOK: RA saphana_init - on cluster node '$onode' the srHook generation attribute is empty. May be the new srHook is currently not active, no reloaded or no restart of srHook on HANA side was done"
            fi
            if [ "$srHook_gen" != "$onode_srHook_gen" ]; then
                # the cluster nodes are running different Hook generations
                super_ocf_log info "HOOK: RA saphana_init - the cluster nodes '$NODENAME' and '$onode' are running different Hook generations ('$srHook_gen' - '$onode_srHook_gen')."
            fi
        done
        # be in mind: it may be that not all nodes running the same Hook generation
        multiTargetSupport=$(get_hana_attribute "X" "${ATTR_NAME_HANA_multiTargetSupport[@]}")
        if [ -z "$multiTargetSupport" ]; then
            # cluster attribute 'hana_${sid}_glob_mts' not set
            super_ocf_log info "RA: multiTargetSupport attribute not set. May be no Hook is configured or the old-style Hook is used."
        fi
        if ocf_is_true "$multiTargetSupport"; then
            super_ocf_log info "RA: multiTargetSupport attribute is set to 'true'"
            if [ "$srHook_gen" == "2.0" ]; then
                old_sync=$(get_hana_attribute "X" "${ATTR_NAME_HANA_GLOB_SRHOOK[@]}")
                if [ -n "$old_sync" ]; then
                    # old, global attribute still exists
                    super_ocf_log info "RA: The global Hook attribute is still available. Use cmd 'SAPHanaSR-manageAttr' to remove this attribute"
                fi
            fi
        fi
    fi
    #
    # get remote site name
    # PRIO1: WHAT TO DO, IF remSite could not be fetched?
    # TODO: PRIO1: Do not overwrite already found remSite, if next node does not provide this value
    # TODO: PRIO9: For later releases we might add the feature to differ between other remote Sites. Currently we only allow two SR sites of
    #              one SAP HANA Database in the same cluster. So only e.g. WALLDORF and ROT and not WALLDORF, ROT and NUSSLOCH.
    #              This limits us to only have sites A => B inside of the cluster and completely "ignore" site C
    #
    for clN in "${otherNodes[@]}"; do
	    nodeSite=$(get_hana_attribute $clN "${ATTR_NAME_HANA_SITE[@]}")
        super_ocf_log info "TOP: clN=$clN, nodeSite=$nodeSite"
        if [ -n "$nodeSite" -a "$nodeSite" != "$site" ]; then
            remSite=$nodeSite
            remSR_name=${remSite%%:*}
            remSR_id=${remSite#*:}
        fi
    done
    # It's always the master name server of the remote site which is really relevant, so we do not need any longer each nodes remoteHost
    ATTR=(${ATTR_NAME_HANA_MNS[@]}); ATTR[0]="${ATTR[0]}_$remSR_name"
    rem_mns=$(get_hana_attribute "X" "${ATTR[@]}")
    remoteHost=$rem_mns
    super_ocf_log debug "DBG: remoteHost is $remoteHost"
    remoteNode=$rem_mns
    super_ocf_log info "TOP: sr_name=$sr_name, remSR_name=$remSR_name"
    super_ocf_log debug "DBG: sr_name=$sr_name, remoteHost=$remoteHost, remoteNode=$remoteNode, sr_mode=$sr_mode"
    # optional OCF parameters, we try to guess which directories are correct
    if  [ -z "$OCF_RESKEY_DIR_EXECUTABLE" ]
    then
        if have_binary /usr/sap/$SID/$InstanceName/exe/sapstartsrv && have_binary /usr/sap/$SID/$InstanceName/exe/sapcontrol
        then
            DIR_EXECUTABLE="/usr/sap/$SID/$InstanceName/exe"
        fi
    else
        if have_binary "$OCF_RESKEY_DIR_EXECUTABLE/sapstartsrv" && have_binary "$OCF_RESKEY_DIR_EXECUTABLE/sapcontrol"
        then
            DIR_EXECUTABLE="$OCF_RESKEY_DIR_EXECUTABLE"
        fi
    fi
    SAPSTARTSRV="$DIR_EXECUTABLE/sapstartsrv"
    SAPCONTROL="$DIR_EXECUTABLE/sapcontrol"

    [ -z "$DIR_EXECUTABLE" ] && assert "Cannot find sapstartsrv and sapcontrol executable, please set DIR_EXECUTABLE parameter!"
    DIR_PROFILE="${OCF_RESKEY_DIR_PROFILE:-/usr/sap/$SID/SYS/profile}"
    # check, if the following fall-back is ok, or if there could be multiple profiles matching this pattern
    if [ -n "${SAPVIRHOST}" ]; then
        SAPSTARTPROFILE="$DIR_PROFILE/${OCF_RESKEY_INSTANCE_PROFILE:-${SID}_${InstanceName}_${SAPVIRHOST}}"
    else
        # check, if the following fall-back is ok, or if there could be multiple profiles matching this pattern
        # also take profile versions into account - they might break this fall-back
        # TODO: PRIO4: Check, if it makes sense to implement an additional last fall-back: get the SAPSTARTPROFILE from /usr/sap/sapservices
        #
        SAPSTARTPROFILE="$(ls -1 $DIR_PROFILE/${OCF_RESKEY_INSTANCE_PROFILE:-${SID}_${InstanceName}_*})"
    fi
    #PATH=${PATH}:${DIR_EXECUTABLE}; export PATH

    local standbyFilter=""
    if [ $ACTION = "stop" ]; then
       standbyFilter="--standbyFilter=off"
    fi
    if is_master_nameserver; then
       master_walk $standbyFilter
    elif is_active_nameserver_slave; then
       master_walk $standbyFilter
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$OCF_SUCCESS"
    #############################
    # DONE: PRIO9: To be able to call landscapeHostConfig.py without su (so as root)
    #############################
    return $OCF_SUCCESS
}


#
# function: check_sapstartsrv - check for sapstartsrv - optional start
# params:   -
# globals:  DIR_PROFILE(w), SAPSTARTPROFILE(r), SAPCONTROL(r), SID(r), InstanceName(r), InstanceNr(r), OCF_*(r)
# check_sapstartsrv : Before using sapcontrol we make sure that the sapstartsrv is running.
#
function check_sapstartsrv() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local restart=0
    local runninginst=""
    local rc=$OCF_SUCCESS
    local output=""
    if [ ! -S /tmp/.sapstream5${InstanceNr}13 ]; then
        super_ocf_log warn "ACT: sapstartsrv is not running for instance $SID-$InstanceName (no UDS), it will be started now"
        restart=1
    else
        output=$($SAPCONTROL -nr $InstanceNr -function ParameterValue INSTANCE_NAME -format script)
        if [ $? -eq 0 ]
        then
            runninginst=$(echo "$output" | grep '^0 : ' | cut -d' ' -f3)
            if [ "$runninginst" != "$InstanceName" ]
            then
                super_ocf_log warn "ACT: sapstartsrv is running for instance $runninginst, that service will be killed"
                restart=1
            else
                output=$($SAPCONTROL -nr $InstanceNr -function AccessCheck Start)
                if [ $? -ne 0 ]; then
                    super_ocf_log warn "ACT: FAILED - sapcontrol -nr $InstanceNr -function AccessCheck Start ($(ls -ld1 /tmp/.sapstream5${InstanceNr}13))"
                    super_ocf_log warn "ACT: sapstartsrv will be restarted to try to solve this situation, otherwise please check sapstsartsrv setup (SAP Note 927637)"
                    restart=1
                fi
            fi
        else
            super_ocf_log warn "ACT: sapstartsrv is not running for instance $SID-$InstanceName, it will be started now"
            restart=1
        fi
    fi
    if [ -z "$runninginst" ]; then runninginst=$InstanceName; fi
    if [ $restart -eq 1 ]
    then
        if [ -d /usr/sap/$SID/SYS/profile/ ]
        then
            DIR_PROFILE="/usr/sap/$SID/SYS/profile"
        else
            assert "Expected /usr/sap/$SID/SYS/profile/ to be a directory, please set DIR_PROFILE parameter!"
        fi
        [ ! -r $SAPSTARTPROFILE ] && assert "Expected $SAPSTARTPROFILE to be the instance START profile, please set INSTANCE_PROFILE parameter!"
        pkill -9 -f "sapstartsrv.*$runninginst"
        # removing the unix domain socket files as they might have wrong permissions
        # or ownership - they will be recreated by sapstartsrv during next start
	# TODO: PRIO2: Check, if we need to delete the socket files
        rm -f /tmp/.sapstream5${InstanceNr}13
        rm -f /tmp/.sapstream5${InstanceNr}14
        # DONE: PRI0: WE NEED LD_LIBRARY_PATH HERE!!
        (
          export PATH="$DIR_EXECUTABLE${PATH:+:}$PATH"
          export LD_LIBRARY_PATH="$DIR_EXECUTABLE${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH"
          $SAPSTARTSRV pf=$SAPSTARTPROFILE -D -u $sidadm
        )
        # now make sure the daemon has been started and is able to respond
        local srvrc=1
        while [ $srvrc -eq 1 -a $(pgrep -f "sapstartsrv.*$runninginst" | wc -l) -gt 0 ]
        do
            sleep 1
            $SAPCONTROL -nr $InstanceNr -function GetProcessList > /dev/null 2>&1
            srvrc=$?
        done
        if [ $srvrc -ne 1 ]
        then
            super_ocf_log info "ACT: sapstartsrv for instance $SID-$InstanceName was restarted!"
            rc=$OCF_SUCCESS
        else
            super_ocf_log error "ACT: sapstartsrv for instance $SID-$InstanceName could not be started!"
            rc=$OCF_ERR_GENERIC
            ocf_is_probe && rc=$OCF_NOT_RUNNING
        fi
    fi
    return $rc
}

#
# function: cleanup_instance - remove resources from a crashed instance
# params:   -
# globals:  -
#
function cleanup_instance() {
  super_ocf_log info "FLOW $FUNCNAME ($*)"
  local rc=0
  # TODO: PRIO5: Check, if we need HANA cleanup procedure (processes, ipc obj, pid files); Currently not needed
  super_ocf_log debug "DBG: cleanup_instance currently not implemented"
  rc=0
  super_ocf_log info "FLOW $FUNCNAME rc=$rc"
}

#
# function: check_for_primary - check if local SAP HANA is configured as primary
# params:   -
# globals:  HANA_STATE_PRIMARY(r), HANA_STATE_SECONDARY(r), HANA_STATE_DEFECT(r)
#
function check_for_primary() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=$HANA_STATE_DEFECT
    node_full_status=$(HANA_CALL --timeout $HANA_CALL_TIMEOUT --cmd "hdbnsutil -sr_stateConfiguration --sapcontrol=1" 2>/dev/null )
    node_status=$(echo "$node_full_status" | awk -F= '$1=="mode" {print $2}')
    super_ocf_log debug "DBG: check_for_primary: node_status=$node_status"
    # TODO: PRIO2: Maybe we need to use a fallback interface when hdbnsitil does not answer properly -> lookup in config files?
    #              This might also solve some problems when we could not figure-out the local or remote site name
    for i in  1 2 3; do
       case "$node_status" in
           primary )
               super_ocf_log info "FLOW: $FUNCNAME rc=HANA_STATE_PRIMARY"
               return $HANA_STATE_PRIMARY;;
           syncmem | sync | async )
               super_ocf_log info "FLOW: $FUNCNAME rc=HANA_STATE_SECONDARY"
               return $HANA_STATE_SECONDARY;;
           none ) # have seen that mode on second side BEFORE we registered it as replica
               super_ocf_log info "FLOW: $FUNCNAME rc=HANA_STATE_STANDALONE"
               return $HANA_STATE_STANDALONE;;
           * )
               super_ocf_log err "ACT: check_for_primary:  we didn't expect node_status to be: <$node_status>"
               dump=$( echo $node_status | hexdump -C );
               super_ocf_log err "ACT: check_for_primary:  we didn't expect node_status to be: DUMP <$dump>"
               node_full_status=$(HANA_CALL --timeout $HANA_CALL_TIMEOUT --cmd "hdbnsutil -sr_stateConfiguration --sapcontrol=1" 2>/dev/null )
               node_status=$(echo "$node_full_status" | awk -F= '$1=="mode" {print $2}')
               super_ocf_log debug "DEC: check_for_primary: loop=$i: node_status=$node_status"
               # TODO: PRIO1: Maybe we need to keep the old value for P/S/N, if hdbnsutil just crashes
       esac;
       sleep 20
    done
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

# function: analyze_hana_sync_statusSRS
# params:   -
# globals:  DIR_EXECUTABLE(r), FULL_SR_STATUS(w)
#
# systemReplicationStatus.py return-codes:
#    NoHSR        = 10
#    Error        = 11
#    Unkown       = 12
#    Initializing = 13
#    Syncing      = 14
#    Active       = 15
function analyze_hana_sync_statusSRS()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=-1 srRc=0 all_nodes_other_side="" local n="" siteParam=""
    if [ -n "$remSR_name" ]; then
       siteParam="--site=$remSR_name"
    fi
    FULL_SR_STATUS=$(HANA_CALL --timeout $HANA_CALL_TIMEOUT --cmd "systemReplicationStatus.py $siteParam" 2>/dev/null); srRc=$?
    super_ocf_log info "FLOW $FUNCNAME systemReplicationStatus.py (to site '$remSR_name')-> $srRc"
    #
    # TODO: PRIO2: Here we might also need to filter additional sites (if multi tier should be supported)
    #              And is the check for return code capable for chains?
    #
    if [ $srRc -eq 15 ]; then
       super_ocf_log info "FLOW $FUNCNAME SOK"
       set_hana_attribute "X" "SOK" "${ATTR_NAME_HANA_SEC_SYNC_STATUS[@]}"
       super_ocf_log debug "DBG: site=$sr_name, setting SOK for secondary (1)"
       lpa_set_lpt 30 "$remSR_name"
       rc=0;
    elif [ $srRc -le 11 ]; then # 11 and 10
       # if systemReplicationStatus is ERROR and landscapeHostConfiguration is down then do NOT set SFAIL
       get_hana_landscape_status; lss=$?
       if [ $lss -lt 2 ]; then
          # keep everything like it was
          rc=2
       else
          # ok we should be careful and set secondary to SFAIL
          super_ocf_log info "FLOW $FUNCNAME SFAIL"
          set_hana_attribute "X" "SFAIL" "${ATTR_NAME_HANA_SEC_SYNC_STATUS[@]}"
          super_ocf_log debug "DBG: site=$sr_name, setting SFAIL for secondary (5) - srRc=$srRc lss=$lss"
          # TODO: PRIO1 - P004: need to check LSS again to avoid dying primary to block (SFAIL) secondary
          super_ocf_log debug "DBG: 001 * lpa_set_lpt 10 $remSR_name"
          lpa_set_lpt 10 "$remSR_name"
          rc=1
       fi
    else
       super_ocf_log info "FLOW $FUNCNAME SFAIL"
       set_hana_attribute "X" "SFAIL" "${ATTR_NAME_HANA_SEC_SYNC_STATUS[@]}"
       super_ocf_log debug "DBG: site=$sr_name, setting SFAIL for secondary (2) - srRc=$srRc"
       # TODO: PRIO1 - P004: need to check LSS again to avoid dying primary to block (SFAIL) secondary
       super_ocf_log debug "DBG: 002 * lpa_set_lpt 10 $remSR_name"
       lpa_set_lpt 10 "$remSR_name"
       rc=1;
    fi
    super_ocf_log info "FLOW $FUNCNAME PRIM+LPA"
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}


#
# function: get_hana_landscape_status - figure out hana landscape status
# params:   -
# globals:  sidadm(r), DIR_EXECUTABLE(r), hana_LSS_Out(w)
#
function get_hana_landscape_status()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0
    hana_LSS_Out=$(HANA_CALL --timeout $HANA_CALL_TIMEOUT --cmd "landscapeHostConfiguration.py --sapcontrol=1" 2>/dev/null); rc=$?
    if [ $rc -eq 124 ]; then
        # landscape timeout
        super_ocf_log warn "RA: landscapeHostConfiguration.py TIMEOUT after $HANA_CALL_TIMEOUT seconds"
        sleep 20
        hana_LSS_Out=$(HANA_CALL --timeout $HANA_CALL_TIMEOUT --cmd "landscapeHostConfiguration.py --sapcontrol=1" 2>/dev/null); rc=$?
        if [ $rc -eq 124 ]; then
           super_ocf_log warn "RA: landscapeHostConfiguration.py second TIMEOUT after $HANA_CALL_TIMEOUT seconds"
           # TODO PRIO2: How to handle still hanging lss - current solution is to say "FATAL"
           rc=0
        fi
    fi
    return $rc;
}

#
# function: register_hana_secondary - register local hana as secondary to the other site
# params:   -
# globals:  sidadm(r), remoteHost(r), InstanceNr(r), sr_mode(r), sr_name(r)
# register_hana_secondary
#
function register_hana_secondary()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=2;
    local remoteInstance="";
    local remoteHanaHost="";
    if is_the_master_nameserver; then
        remoteInstance=$InstanceNr

        ges_ver=$(HANA_CALL --timeout 10 --cmd "HDB version" | tr -d " " | awk -F: '$1 == "version" {print $2}')
        hdbver=${ges_ver%.*.*}

        if version "$hdbver" ">=" "1.00.110"; then
            newParameter=1
        fi

        if ocf_is_true ${AUTOMATED_REGISTER}; then
           #
           # before we could register we stop HANA on all nodes of the swarm (maybe some still running)
           saphana_stop
           # TODO: PRIO1 change from remoteHost to remoteMSN (remote MasterNameServer)
           # TODO: PRIO2 How to get the remoteHost without cluster attribute remoteHost - could we use the master-list attribute?
           local hanaOM=""
           local hanaRM=""
           if [ $newParameter -eq 1 ]; then
               hanaOM=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_OPERATION_MODE[@]}")
               if [ -n "$hanaOM" ]; then
                   hanaOM="--operationMode=$hanaOM"
               fi
               hanaRM="--replicationMode=$sr_mode"
           else
               hanaOM=""
               hanaRM="--mode=$sr_mode"
           fi
           set_hana_attribute "$NODENAME" "$sr_name" "${ATTR_NAME_HANA_SEC[@]}"
           remoteHanaHost=$(get_hana_attribute "$remoteHost" "${ATTR_NAME_HANA_VHOST[@]}")
           if [ -z "$remoteHanaHost" ]; then
               remoteHanaHost=$remoteHost
           fi
           super_ocf_log info "ACT: SAP HANA REGISTER: hdbnsutil -sr_register --remoteHost=$remoteHanaHost --remoteInstance=$remoteInstance $hanaRM $hanaOM --name=$sr_name"
           set_hana_attribute "$NODENAME" "R" "${ATTR_NAME_HANA_SRACTION[@]}"
           HANA_CALL --timeout inf --use-su --cmd "hdbnsutil -sr_register --remoteHost=$remoteHanaHost --remoteInstance=$remoteInstance $hanaRM $hanaOM --name=$sr_name"; rc=$?
           set_hana_attribute "$NODENAME" "-" "${ATTR_NAME_HANA_SRACTION[@]}"
        else
           super_ocf_log info "ACT: SAP HANA: DROP REGISTER because AUTOMATED_REGISTER is set to FALSE"
           rc=1
        fi
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc;
}

#
# function: saphana_status - pure status check
# params:   -
# globals:  SIDInstanceName, OCF_*,
function saphana_status() {
    local binDeam="hdb.sap${SIDInstanceName}" rc=0
    binDeam=${binDeam:0:15}   # Process name is limited to the first 15 characters
    if pgrep $binDeam 1>/dev/null; then rc=$OCF_SUCCESS; else rc=$OCF_NOT_RUNNING; fi
    return $rc
}

#
# function: saphana_start - start a hana instance
# params:   -
# globals:  OCF_*, SAPCONTROL, InstanceNr, SID, InstanceName,
#
function saphana_start() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=$OCF_NOT_RUNNING
    local output=""
    local loopcount=0
    check_sapstartsrv
    rc=$?
    #
    # DONE: ASK: PRIO5: For SCALE-OUT - do we need to use another call like StartSystem? Or better to use the HDB command?
    #
    # set_SRHOOK() removed here. saphana_start() is used for primary and secondaries but would need different attribute values
    if is_the_master_nameserver; then
        if [ $rc -eq $OCF_SUCCESS ]; then
            output=$($SAPCONTROL -nr $InstanceNr -function StartSystem)
            rc=$?
            super_ocf_log info "ACT: SAP HANA START: Starting System $SID-$InstanceName: $output"
        fi
        if [ $rc -eq 0 ]
        then
            # DONE: PRIO9: something more dynamic than 3600 seconds in WaitforStarted
            HANA_ACTION_TIMEOUT=$(get_saphana_waitfor_timeout WaitforStarted)
            output=$($SAPCONTROL -nr $InstanceNr -function WaitforStarted $HANA_ACTION_TIMEOUT 1)
            if [ $? -eq 0 ]
            then
                for i in 1 2 3 4; do
                    super_ocf_log info "ACT: SAP HANA: System $SID-$InstanceName started: $output"
                    hanaANSWER=$(HANA_CALL --timeout $HANA_CALL_TIMEOUT --cmd "landscapeHostConfiguration.py --sapcontrol=1" 2>/dev/null); hanalrc="$?"
                    hanarole=$(echo "$hanaANSWER" | tr -d ' ' | \
                            awk -F= '
                            $1 == "host/"vName"/nameServerConfigRole"  {nsCR=$2}
                            $1 == "host/"vName"/nameServerActualRole"  {nsAR=$2}
                            $1 == "host/"vName"/indexServerConfigRole" {isCR=$2}
                            $1 == "host/"vName"/indexServerActualRole" {isAR=$2}
                            $1 == "nameServerConfigRole"  {nsCR=$2}
                            $1 == "nameServerActualRole"  {nsAR=$2}
                            $1 == "indexServerConfigRole" {isCR=$2}
                            $1 == "indexServerActualRole" {isAR=$2}
                            END { printf "%s:%s:%s:%s\n", nsCR, nsAR, isCR, isAR;  } ' vName=$vName )
                    hanaHostActive=$(echo "$hanaANSWER" | tr -d ' ' | \
                    awk -F= '
                                $1 ~ "host/[a-zA-Z].*/hostActive" {hostActive = hostActive ":" $2}
                                END { printf "%s\n", hostActive;  } ' vName=$vName )
                    set_hana_attribute ${NODENAME} "$hanarole" "${ATTR_NAME_HANA_ROLES[@]}"
                    ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
                    set_hana_attribute ${NODENAME} "$hanalrc" "${ATTR[@]}"
                    sleep 5
                done
                rc=$OCF_SUCCESS
            else
                super_ocf_log err  "ACT: SAP HANA: System $SID-$InstanceName start failed: $output"
                rc=$OCF_ERR_GENERIC
            fi
        else
            super_ocf_log err "ACT: SAP HANA: System $SID-$InstanceName start failed: $output"
            rc=$OCF_ERR_GENERIC
        fi
    else
        # TODO: PRIO1: Do we need to set a clone state here?
        rc=$OCF_SUCCESS
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

function saphana_stopSystem() {
    local rc=0 rcWfS=0
    check_sapstartsrv; rc="$?"
    if [ "$rc" -eq "$OCF_SUCCESS" ]; then
        ## TODO: PRIO 1: Only stop System, if I am the last master!
        ## TODO: PRIO 2: Do we need a "last-man-switch-off-the-light" detection?
        output=$($SAPCONTROL -nr $InstanceNr -function StopSystem)
        rc=$?
         super_ocf_log info "ACT: SAP HANA STOP: Stopping System $SID-$InstanceName: $output"
    fi
    if [ "$rc" -eq 0 ]; then
        HANA_ACTION_TIMEOUT=$(get_saphana_waitfor_timeout WaitforStopped)
        output=$($SAPCONTROL -nr $InstanceNr -function WaitforStopped $HANA_ACTION_TIMEOUT 1); rcWfS="$?"
        while true; do
            #
            # If the master-nameserver dies and does not have a failover candidate but there are still running worker nodes
            #    WaitforStopped terminates to early and does not wait till the worker nodes are down.
            #    So we need to wait till the last "partial" node is down.
            # TODO: PRIO1: differ return codes for WaitforStarted from the Number-Of-Partial-Nodes
            #
            rcWfS=0
            hanaANSWER=$(HANA_CALL --timeout $HANA_CALL_TIMEOUT --cmd "landscapeHostConfiguration.py --sapcontrol=1" 2>/dev/null);
            hAPart=$(echo "$hanaANSWER" | tr -d ' ' | \
            awk -F= '
               BEGIN { partials=0 }
                $1 ~ "host/[a-zA-Z].*/hostActive" {hostActive = hostActive "-" $2; if ( $2 == "partial" ) { partials=1 } }
                    END { printf "%s:%s\n", hostActive, partials;  } ' vName=$vName )
            hanaHostActive="${hAPart%:*}"
            partials="${hAPart#*:}"
            if [ "$partials" -eq 0 ]; then
                # terminate wait-for-all-nodes-down if no partial nodes are left any more
                break;
            else
                sleep 5
                timeNow=$(date '+%s')
                (( timeStop = timeNow - timeB ))
                (( timeRemain = HANA_ACTION_TIMEOUT - timeStop ))
                if [ "$timeRemain" -le 0 ]; then
                    # terminate wait-for-all-nodes-down if time-to-stop in sum expired HANA_ACTION_TIMEOUT
                    rcWfS=1
                    break;
                fi
            fi
        done
	    # DONE PRIO1: RC restore!!
        if [ "$rcWfS" -eq 0 ]; then
            super_ocf_log info "ACT: SAP HANA: System $SID-$InstanceName stopped: $output"
            rc="$OCF_SUCCESS"
        else
            super_ocf_log err "ACT: SAP HANA: System $SID-$InstanceName stop failed: $output, rc=$rc"
            rc="$OCF_ERR_GENERIC"
        fi
    else
        super_ocf_log err "ACT: SAP HANA: System $SID-$InstanceName stop failed: $output, rc=$rc"
        rc="$OCF_ERR_GENERIC"
    fi
    return "$rc"
}

#
# function: saphana_stop - stop a hana instance
# params:   -
# globals:  OCF_*(r), SAPCONTROL(r), SID(r), InstanceName(r)
# saphana_stop: Stop the SAP instance
#
function saphana_stop() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0
    local output=""
    if is_the_master_nameserver; then
        saphana_stopSystem; rc=$?
    elif is_active_nameserver_slave && [ -z "$the_master" ]; then
        saphana_stopSystem; rc=$?
    else
        # TODO: PRIO1: Do we need to set a clone state here?
        rc=$OCF_SUCCESS
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: saphana_validate - validation of (some) variables/parameters
# params:   -
# globals:  OCF_*(r), SID(r), InstanceName(r), InstanceNr(r), SAPVIRHOST(r)
# saphana_validate: Check the semantic of the input parameters
#
function saphana_validate() {
  super_ocf_log info "FLOW $FUNCNAME ($*)"
  local rc=$OCF_SUCCESS
  #
  # SID is Alpha-AlphaNumeric-Alphanumeric?
  #
  if [ $(echo "$SID" | grep -c '^[A-Z][A-Z0-9][A-Z0-9]$') -ne 1 ]
  then
    super_ocf_log err "ACT: Parsing instance profile name: '$SID' is not a valid SID!"
    rc=$OCF_ERR_ARGS
  fi
  #
  # InstanceNr is a two-Digit?
  #
  if [ $(echo "$InstanceNr" | grep -c '^[0-9][0-9]$') -ne 1 ]
  then
    super_ocf_log err "ACT: Parsing instance profile name: '$InstanceNr' is not a valid instance number!"
    rc=$OCF_ERR_ARGS
  fi
  super_ocf_log info "FLOW $FUNCNAME rc=$rc"
  return $rc
}

#
# function: saphana_start_primary - handle startup of PRIMARY in M/S
# params: current_landsscape_status
# globals: OCF_*(r), NODENAME, ATTR_NAME_*, HANA_STATE_*,
#
function saphana_start_primary()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local primary_status sync_attr score_master rc=$OCF_NOT_RUNNING
    local lss=$1
    local sqlrc=0;
    local rc=0
    local lpa_dec=4
    local lpa_advice=""
    #
    # we will be a PRIMARY so checking, if there is anOTHER master
    #
    super_ocf_log debug "DBG: saphana_primary - check_for_primary reports HANA_STATE_PRIMARY"
    #
    lpa_init_lpt $HANA_STATE_PRIMARY
    lpa_check_lpt_status; lpa_dec=$?
    # we take first lss from the parameter $1
    # get_hana_landscape_status; lss=$?
    # DONE: PRIO2: Do we need to differ 0 and 1 here? While 0 is a fatal SAP error, 1 for down/error
    if [ $lss -eq 0 ]; then
       super_ocf_log err "ACT: get_hana_landscape_status reports FATAL"
       # DONE: PRIO1: what to do for lss=0?
       # TODO: PRIO3: Check, if OCF_ERR_GENERIC is best reaction
       lpa_advice="skip"
       rc=$OCF_ERR_GENERIC
    fi
    ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
    my_lss=$(get_hana_attribute "X" "${ATTR[@]}")
    ATTR=(${ATTR_NAME_HANA_SRR[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
    my_srr=$(get_hana_attribute "X" "${ATTR[@]}")
    # saphana_start_primary - set SR attribute of primary to "PRIM"
    case "$lpa_dec" in
        0) lpa_advice="start" ;;
        1) lpa_advice="register" ;;
        2) lpa_advice="wait" ;;
        3) lpa_advice="wait" ;;
        4) lpa_advice="fail" ;;
        *) lpa_advice="fail" ;;
    esac

    case "$lpa_advice" in
        start )  # process a normal START
            set_SRHOOK "$sr_name" "PRIM"; my_sync="PRIM"
            case "$lss" in
                2 | 3 | 4 ) # as landscape says we are up - just set the scores and return code
                    super_ocf_log info "LPA: landscape: UP, LPA: start ==> keep running"
                    LPTloc=$(date '+%s')
                    lpa_set_lpt $LPTloc "$sr_name"
                    rc=$OCF_SUCCSESS
                    ;;
                1 ) # landscape says we are down, lets start and adjust scores and return code
                    super_ocf_log info "LPA: landscape: DOWN, LPA: start ==> start system"
                    saphana_start
                    rc=$?
                    LPTloc=$(date '+%s')
                    lpa_set_lpt $LPTloc "$sr_name"
                    ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
                    my_lss=$(get_hana_attribute "X" "${ATTR[@]}")
                    ;;
            esac
            my_role=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_ROLES[@]}")
            my_role="${my_lss}:${my_srr}:${my_role}"
            super_ocf_log info "SCORE: saphana_start_primary: scoring_crm_master($my_role,$my_sync)"
            scoring_crm_master "$my_role" "$my_sync"
            ;;
        register ) # process a REGISTER
            set_SRHOOK "$sr_name" "SREG"; my_sync="SREG"
            case "$lss" in
                2 | 3 | 4 ) # upps we are up - but shouldn't? - we could not register with started HDB
                    # DONE: PRIO3: check if this reaction is correct - tell cluster about failed start
                    super_ocf_log info "LPA: landscape: UP, LPA: register ==> take down"
                    set_crm_master -inf
                    rc=$OCF_NOT_RUNNING
                    ;;
                1 ) # lets try to register
                    # DONE: PRIO2: Like Action in start_secondary
                    super_ocf_log info "LPA: landscape: DOWN, LPA: register ==> try to register"
                    super_ocf_log info "DEC: ANOTHER HANA IS AVAILABLE ==> LETS REGISTER"
                    # DONE: PRIO0: For all master nodes of that site
                    set_crm_master  0
                    for m in "${masters[@]}"; do
                        set_crm_master  0 $m
                    done
                    if wait_for_primary_master 1; then
                        register_hana_secondary
                        check_for_primary; primary_status=$?
                        if [ $primary_status -eq $HANA_STATE_SECONDARY ]; then
                            super_ocf_log info "ACT: Register successful"
                            lpa_push_lpt 10
                            super_ocf_log debug "DBG: 003 * lpa_set_lpt 10 $sr_name"
                            lpa_set_lpt  10 "$sr_name"
                            set_crm_master  0
                            saphana_start_secondary $lss
                            rc=$?
                            super_ocf_log debug "DBG: 004 * lpa_set_lpt 10 $sr_name"
                            lpa_set_lpt  10 "$sr_name"
                        else
                            super_ocf_log err "ACT: Register failed"
                            rc=$OCF_NOT_RUNNING
                        fi
                    else
                        # lets check next monitor, if we can register
                        rc=$OCF_SUCCESS
                    fi
                    ;;
            esac
            ;;
        wait )  # process a WAIT
            set_SRHOOK "$sr_name" "PRIM"; my_sync="PRIM"
            case "$lss" in
                2 | 3 | 4 ) # TODO: PRIO3: as we ARE up we just keep it up
                    # TODO: PRIO3: I now change from "just keep it up to take that down"
                    # TODO: PRIO3: differ lpt_advice, if rc=3 will get implemted
                    # 2 => DOWN
                    # 3 => KEEP
                    # TODO: PRIO3: OCF_SUCCESS, OCF_NOT_RUNNING or OCF_ERR_xxxx ?
                    # lpa_dec
                    super_ocf_log err "LPA: Timestamp (lpt) attribute inconsistency detected"
                    set_crm_master -9000
                    #scoring_crm_master "$my_role" "$my_sync"
                    rc=$OCF_ERR_GENERIC
                    ;;
                1 ) # we are down, so we should wait --> followup in next monitor
                    super_ocf_log info "LPA: landscape: DOWN, LPA: wait ==> keep waiting"
                    # TODO: PRIO3: Check, if WAITING is correct here
                    if ocf_is_true "$AUTOMATED_REGISTER" ; then
                        super_ocf_log info "LPA: landscape: DOWN, LPA: wait ==> keep waiting"
                        set_hana_attribute ${NODENAME} "WAITING4LPA" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
                        set_crm_master -9000
                        rc=$OCF_SUCCESS
                    else
                        super_ocf_log warn "LPA: OLD primary needs manual registration (AUTOMATED_REGISTER='false')"
                        set_hana_attribute ${NODENAME} "WAITING4REG" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
                        set_crm_master -9000
                        rc=$OCF_NOT_RUNNING
                    fi
                    ;;
            esac
            ;;
        fail )   # process a lpa FAIL
            super_ocf_log info "LPA: LPA reports FAIL"
            set_crm_master -inf
            rc=$OCF_NOT_RUNNING
            ;;
    esac
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# check_for_primary_master
# params: -
# globals: ATTR_NAME_HANA_ROLES[@], NODENAME
#
check_for_primary_master()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=1
    local clN clN_role
    #
    # get actual list of cluster members
    #
    ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$remSR_name"
    rem_lss=$(get_hana_attribute "X" "${ATTR[@]}")
    ATTR=(${ATTR_NAME_HANA_SRR[@]}); ATTR[0]="${ATTR[0]}_$remSR_name"
    rem_srr=$(get_hana_attribute "X" "${ATTR[@]}")

    case "$rem_lss:$rem_srr" in
        3:P | 4:P ) rc=0;;
        * ) rc=1;;
    esac

    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# wait_for_primary_master: wait some time till a running primary master is shown in attributes
# params: optional: loop count - currently time in 10s waiting loop
# globals: -
#
wait_for_primary_master()
{
    local wait=1
    local rc=1
    local loops=${1:-0}
    local count=0
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    #
    # hana_ndb_roles=primary:master1:master:worker:master
    #
    while [ "$wait" -eq 1 ]; do
        if check_for_primary_master; then
           wait=0
           rc=0
        else
           if [ $loops -gt 0 ]; then
              (( count++ ))
              if [ $count -gt $loops ]; then
                 wait=0
                 rc=1
              fi
           fi
           sleep 10
        fi
    done
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: saphana_start_secondary - handle startup of PRIMARY in M/S
# params:
# globals: OCF_*(r), NODENAME, ATTR_NAME_*,
#
function saphana_start_secondary()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local primary_status sync_attr score_master rc=$OCF_NOT_RUNNING
    local sqlrc;
    case "$PreferSiteTakeover" in
        never | NEVER | Never )
            set_crm_master -INFINITY
            ;;
        * )
            set_crm_master 0
            ;;
    esac
    #
    ####### LPA - begin
    #
    lpa_push_lpt 10
    super_ocf_log debug "DBG: 005 * lpa_set_lpt 10 $sr_name"
    lpa_set_lpt  10 "$sr_name"
    #
    ####### LPA - end
    #
    #
    # we would be secondary
    # we first need to check, if there are Master Nodes, because the Secondary only starts
    # successfully, if the Primary is available. Therefore we mark the Secondary as "WAITING"
    # DONE: PRIO3: wait_for_primary_master 10 is just a test value: 10 loops x10 seconds, then go to WAITING
    # DONE: PRIO3: rename 'wait_for_primary_master' to match better the use case ("wait_some_time")
    #
    super_ocf_log debug "DBG: wait for promoted side"
    # TODO: PRIO3: Check if setting SFAIL during secondary start is ok
    set_hana_attribute "X" "SFAIL" "${ATTR_NAME_HANA_SEC_SYNC_STATUS[@]}"
    set_hana_attribute "${NODENAME}" "$sr_name" "${ATTR_NAME_HANA_SEC[@]}"
    if wait_for_primary_master 10; then
       saphana_start; rc=$?
       if [ $rc -ne $OCF_SUCCESS ]; then
           if ! wait_for_primary_master 1; then
               # It seems the stating secondary could not start because of stopping primary
               #    so this is a WAITING situation
               super_ocf_log info "ACT: PRIMARY seems to be down now ==> WAITING"
               set_hana_attribute ${NODENAME} "WAITING4PRIM" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
               set_crm_master -INFINITY
               rc=$OCF_SUCCESS
           fi
       else
           super_ocf_log debug "DBG: 006 * lpa_set_lpt 10 $sr_name"
           lpa_set_lpt  10 "$sr_name"
       fi
    else
       super_ocf_log info "ACT: wait_for_primary_master ==> WAITING"
       set_hana_attribute ${NODENAME} "WAITING4PRIM" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
       set_crm_master -INFINITY
       rc=$OCF_SUCCESS
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: saphana_check_local_instance
# params:
# output:
# rc:       rc=0 (UP) rc=1 (DOWN)
# globals:
#
function saphana_check_local_instance()
{
    local rc=1
    local count=0
    local SERVNO
    local output
    local MONITOR_SERVICES="hdbnameserver|hdbdaemon" # TODO: PRIO1: exact list of Services
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    check_sapstartsrv
    rc=$?
    if [ $rc -eq $OCF_SUCCESS ]
    then
      output=$($SAPCONTROL -nr $InstanceNr -function GetProcessList -format script)
      # we have to parse the output, because the return code doesn't tell anything about the instance status
      for SERVNO in `echo "$output" | grep '^[0-9] ' | cut -d' ' -f1 | sort -u`
      do
        local COLOR=`echo "$output" | grep "^$SERVNO dispstatus: " | cut -d' ' -f3`
        local SERVICE=`echo "$output" | grep "^$SERVNO name: " | cut -d' ' -f3`
        local STATE=0
        local SEARCH

        case $COLOR in
          GREEN|YELLOW)       STATE=$OCF_SUCCESS;;
          *)                  STATE=$OCF_NOT_RUNNING;;
        esac

        SEARCH=`echo "$MONITOR_SERVICES" | sed 's/\+/\\\+/g' | sed 's/\./\\\./g'`
        if [ `echo "$SERVICE" | egrep -c "$SEARCH"` -eq 1 ]
        then
          if [ $STATE -eq $OCF_NOT_RUNNING ]
          then
            [ "$MONLOG" != "NOLOG" ] && ocf_log err "SAP instance service $SERVICE status color is $COLOR !"
            rc=$STATE
          fi
          count=1
        fi
      done

      if [ $count -eq 0 -a $rc -eq $OCF_SUCCESS ]
      then
        if ocf_is_probe
        then
          rc=1
        else
          [ "$MONLOG" != "NOLOG" ] && ocf_log err "The SAP instance does not run any services which this RA could monitor!"
          rc=1
        fi
      fi
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: lpa_get_lpt - get lpt from cluster
# params:   SITE
# output:   LPT
# rc:       rc=0: OK, rc=1: InternalERROR, rc=2: ERROR
# globals:  LPA_ATTR_*,
#
function lpa_get_lpt() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=1
    local mysite=$1
    local lpt=""
    local -a ATTR
    ATTR=( "${LPA_ATTR[@]}" )
    ATTR[0]="${LPA_ATTR[0]}_${mysite}"
    super_ocf_log debug "DBG: $FUNCNAME get_hana_attribute X " "${ATTR[@]}"
    lpt=$(get_hana_attribute "X" "${ATTR[@]}")
    if [ -n "$lpt" ]; then
        rc=0
        echo $lpt
    else
        rc=2
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: lpa_set_lpt - set lpt in cluster
# params:   LPT [site]
# globals:  LPA_ATTR(r), NODENAME(r),
# rc:       rc=0: OK, rc=1: InternalERROR, rc=2: ERROR
#
function lpa_set_lpt() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=1
    local crm_rc=1
    local lpt=$1
    local clpt=-1
    local mysite=$2
    local -a ATTR
    ATTR=(${LPA_ATTR[@]})
    ATTR[0]="${LPA_ATTR[0]}_${mysite}"
    super_ocf_log debug "DBG: $FUNCNAME (set_hana_attribute X \"$lpt\"" "${ATTR[@]}" ")"
    if [ -n "$mysite" ]; then
        set_hana_attribute "X" "$lpt" "${ATTR[@]}"; crm_rc=$?
        clpt=$(lpa_get_lpt $mysite)
        if [ "$lpt" != "$clpt" ]; then
            super_ocf_log error "LPA: lpa_set_lpt failed for lpt=$lpt and mysite=$mysite"
            rc=2
        else
            rc=0
        fi
    else
       super_ocf_log info "DEC: lpa_set_lpt - ignore changing lpt due to lost site name"
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: lpa_pull_lpt - fetch lpt from file
# params:   -
# globals:  LPA_DIRECTORY(r), sid, NODENAME
# output:   LPT
# rc:       rc=0: OK, rc=1: InternalERROR, rc=2: ERROR
#
function lpa_pull_lpt() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=1
    local lpt=-1
    local lpa_file=$LPA_DIRECTORY/lpa_${sid}_${NODENAME}
    local -a lpt_file_words
    lpt_file_words=( $( head -1 $lpa_file ) )
    lpt=${lpt_file_words[0]}
    if [ -n "$lpt" ]; then
        rc=0
        echo $lpt
    else
        rc=2
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: lpa_push_lpt - put lpt to file
# params:   LPT
# globals:  LPA_DIRECTORY(r), sid, NODENAME
# output:   --
# rc:       rc=0: OK, rc=1: InternalERROR, rc=2: ERROR
#
function lpa_push_lpt() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local lpt=$1
    local clpt=-1
    local rc=1
    local lpa_file=$LPA_DIRECTORY/lpa_${sid}_${NODENAME}
    #
    mkdir -p $LPA_DIRECTORY
    echo "$lpt" > $lpa_file
    clpt=$(lpa_pull_lpt); lpt_rc=$?
    if [ "$clpt" != "$lpt" -o "$lpt_rc" -ne 0 ]; then
        rc=2
    else
        rc=0
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: lpa_init_lpt - initialize local lpt, if needed
# params:   HANA_STATE
# globals:  HANA_STATE_*(r), LPA_DIRECTORY(r), sid(r), NODENAME(r),
# lpa_init_lpt
#
# Returncodes:
#    rc=0: OK,  rc=1 InternalERROR,  rc=2: ERROR
#
# Initializing (if NO local LPT-file):
#    SECONDARY sets to 0
#    PRIMARY   sets to 1
#
function lpa_init_lpt() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=1
    local LPTloc=-1
    local LPTrem=-1
    local hana_state=$1
    local lpa_file=$LPA_DIRECTORY/lpa_${sid}_${NODENAME}
    mkdir -p $LPA_DIRECTORY
    LPTloc=$(lpa_get_lpt ${sr_name}) || LPTloc=$(lpa_pull_lpt) || \
        if   [ "$hana_state" -eq "$HANA_STATE_PRIMARY" ];  then    # Initialize for Primary
            # init primary
            LPTloc=20
            lpa_push_lpt "20"; rc=$?
        elif [ "$hana_state" -eq "$HANA_STATE_SECONDARY" ]; then   # Initialize for Secondary
            # init secondary
            LPTloc=10
            lpa_push_lpt "10"; rc=$?
        else
            rc=2
        fi
    lpa_set_lpt $LPTloc "$sr_name"
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: lpa_check_lpt_status - start a hana clone instance
# params:   -
# globals:  DUPLICATE_PRIMARY_TIMEOUT, NODENAME, remoteNode
# lpa_check_lpt_status
#
# Returncodes:
#    0: start
#    1: register    (then start)
#    2: wait4gab
#    3: wait4other
#
# Initializing (if NO local LPT-file):
#    SECONDARY sets to 10
#    PRIMARY   sets to 20
#
#    LPRlocal OR LPTremore ARE real lpt (>1000)
#        THEN:
#            Bigger LPR wins, if delta-gab is OK
#               LPTlocal >> LPTremore ===> rc=0 (start)
#               LPTRemote >> LPTlocal ===> rc=1 (register)
#            Stalemate in all other cases ==> STALEMATE-HANDLING ===> rc=2 (wait4gab)
#    LPRlocal AND LPTremore ARE NOT real lpt (<=1000)
#        THEN:
#            Bigger LPT wins
#               LPTlocal > LPTremore ===> rc=0 (start)
#               LPTRemote > LPTlocal ===> rc=1 (register)
#            Stalemate in all other cases ==> STALEMATE-HANDLING ===> rc=2 (wait4gab)
#    LPTRemote is not initialized (0)
#        THEN:
#            WAIT ==> LOST REMOTE HANDLING ===> rc=3 (wait4other)
#
function lpa_check_lpt_status() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0
    local LPTloc=-1
    local LPTrem=-1
    local LPTMark=1000
    local delta=0
    local remSn_name=""
    local remHost=""
    #
    # First GET LPT from ATTR-FILE-DEFAULT
    #
    LPTloc=$(lpa_get_lpt $sr_name); lparc=$?   # ATTR
    if [ "$lparc" -ne 0 ]; then
        # as a fallback try to fetch the value from external status file
        LPTloc=$(lpa_pull_lpt);                 # FILE
        lparc=$?
        if [ -z "$LPTloc" -o "$LPTloc" -eq -1 -o "$lparc" -ne 0 ]; then
            # last option - try to initialize as PRIMARY
            lpa_push_lpt 20
            lpa_set_lpt  20 "$sr_name"
            LPTloc=20                           # DEFAULT
        fi
    fi
    LPTrem=$(lpa_get_lpt $remSR_name); lparc=$?
    super_ocf_log info "LPA: LPTloc=$LPTloc LPTrem=$LPTrem"
    if [ $lparc -ne 0 ]; then
        # LPT of the other node could not be evaluated - LPA says WAIT
        super_ocf_log debug "DBG: LPA: LPTloc=$LPTloc, LPTrem undefined ==> WAIT"
        rc=2
    else
        super_ocf_log debug "DBG: LPA: LPTloc ($LPTloc) LPTrem ($LPTrem) delta ($delta)"
        if [ $LPTloc -lt $LPTMark -a $LPTrem -lt $LPTMark ]; then
           delta=0   # both lpts are not a real timestamp so just take the greater one
        else
           delta=$DUPLICATE_PRIMARY_TIMEOUT   # at least one of the lpts is a real timestamp so include delta-gap
        fi
        if (( delta < LPTloc - LPTrem )); then
            # We are the winner - LPA says STARTUP
            super_ocf_log debug "DBG: LPA: LPTloc wins $LPTloc > $LPTrem + $delta ==> START"
            if [ $rc -le 0 ]; then rc=0; fi
        elif (( delta < LPTrem - LPTloc )); then
            if ocf_is_true "$AUTOMATED_REGISTER" ; then
                # The other one has won - LPA says REGISTER
                super_ocf_log debug "DBG: LPA: LPTrem wins $LPTrem > $LPTloc + $delta ==> REGISTER"
                if [ $rc -le 1 ]; then rc=1; fi
                    else
                        super_ocf_log debug "DBG: LPA: LPTrem wins $LPTrem > $LPTloc + $delta BUT AUTOMATED_REGISTER='false' ==> WAIT"
                        rc=2
                    fi

                else
                    super_ocf_log debug "DBG: LPA: Difference between LPTloc and LPTrem is less than delta ($delta) ==> WAIT"
                    # TODO: PRIO3: ADD STALEMATE-HANDLING HERE; currently admin should set one of the lpa to 20
                    rc=2
                fi
            fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

# function: master_walk - find master node(s)
# params:   -
# globals:  srmode(r), the_master(w), masters[@](w), nr_site_node(w), nr_site_worker(w), nr_site_standby(w)
function master_walk() {
 #
 # TODO: PRIO1: EXCLUDE NON ONLINE CLUSTER NODES (offline/unclean/standby/maintenance)
 #
 # * crm node list se03 is online, se01 is standby+maintenance, se02 is offline
 # se01group1:
 #       standby: on
 #       maintenance: on
 # se02group1: (offline)
 # se03group1:
 # * crm_node -l
 # 1084752138 se01group1 member
 # 1084752148 se02group1 lost
 # 1084752154 se03group1 member
 # ====> othernodes contains only members (for openais)
 # * crm_attribute ...
 #
 # se01group1:~ # crm_attribute -N se01group1 -n standby -G -l forever -q
 # on
 # (rc=0)
 # se01group1:~ # crm_attribute -N se02group1 -n standby -G -l forever -q
 # Error performing operation: No such device or address
 # (rc=250)
 # se01group1:~ # crm_attribute -N se02group1 -n standby -G -l forever -q -d off
 # off
 #
 #
 # ==> First exclude non-members (maybe done already) by
 #       'list of nodes | grep member'
 # ==> then query the attributes 'standby' and 'maintenance
 # OR:
 # look at the scores:
 #
 # master_walk now also counts available nodes (same site) in the
    super_ocf_log info "FLOW $FUNCNAME ($*)"

    local active_master="" master1="" master2="" master3=""
    local nSite="" nRole=""
    local best_cold_master="" rest=""
    local standbyFilter=1
    nr_site_node=0     # is counting any node reporting a role like 'worker' or 'standby'
    nr_site_worker=0   # is counting 'configured' worker nodes
    nr_site_standby=0  # is counting 'configured' standby nodes
    while [ $# -gt 0 ]; do
        case "$1" in
            --standbyFilter=off ) standbyFilter=0;;
        esac
        shift
    done
    #
    # 2:S:master1:slave:worker:standby  MAINZ
    # 4:P:master1:master:worker:master  KOELN
    # 4:P:master2:slave:worker:slave    KOELN
    # 4:P:slave:slave:worker:slave      KOELN
    super_ocf_log debug "DBG: master walk"
    # walk over all known cluster nodes
    super_ocf_log debug "DBG: master walk for $srmode"
    for node in $NODENAME "${otherNodes[@]}"; do
        super_ocf_log debug "DBG: master walk node $node"
        nSite=$(get_hana_attribute ${node} "${ATTR_NAME_HANA_SITE[@]}")
        if [ "$site" = "$nSite" ]; then
            # node of same site found
            nStdBy=$(crm_attribute -N ${node} -n standby     -G -l forever -q -d off)
            nMaint=$(crm_attribute -N ${node} -n maintenance -G -l forever -q -d off)
            if [ "$nMaint" = "on" ]; then
                super_ocf_log debug "DBG: Filter node by cluster node maintenance mode"
            elif [ "$nStdBy" = "on" -a "$standbyFilter" = "1" ]; then
                super_ocf_log debug "DBG: Filter node by cluster node standby mode"
            else
                nRole=$(get_hana_attribute ${node} "${ATTR_NAME_HANA_ROLES[@]}")
                IFS=: read nNsConf nNsCurr nIsConf nIsCurr <<< "$nRole"
                super_ocf_log debug "DBG: site $site $nNsConf:$nNsCurr"
                case "$nNsConf:$nNsCurr" in
                    master1:master  ) master1=$node; active_master=$node
                        super_ocf_log debug "DBG: roles $nRole match master1:master"
                    ;;
                    master2:master  ) master2=$node; active_master=$node
                        super_ocf_log debug "DBG: roles $nRole match master2:master"
                    ;;
                    master3:master  ) master3=$node; active_master=$node
                        super_ocf_log debug "DBG: roles $nRole match master3:master"
                    ;;
                    master1:*       ) master1=$node;
                        super_ocf_log debug "DBG: roles $nRole match master1:STAR"
                    ;;
                    master2:*       ) master2=$node;
                        super_ocf_log debug "DBG: roles $nRole match master2:STAR"
                    ;;
                    master3:*       ) master3=$node;
                        super_ocf_log debug "DBG: roles $nRole match master3:STAR"
                    ;;
                    *:shtdown       )
                        super_ocf_log debug "DBG: roles $nRole ignore node (down)"
                    ;;
                    * )
                        super_ocf_log debug "DBG: roles $nRole NO match"
                    ;;
                esac
                case "$nIsConf" in
                    worker )  (( nr_site_worker++ ))
                              (( nr_site_node++ ))
                              ;;
                    standby ) (( nr_site_standby++ ))
                              (( nr_site_node++ ))
                              ;;
                esac
                super_ocf_log debug "DBG: $site: $nRole"
            fi
        fi
    done
    if [ -z "$active_master" ]; then
        masters=( $master1 $master2 $master3 )
        best_cold_master=${masters[0]}
        the_master=$best_cold_master
    else
        the_master=$active_master
    fi
    ATTR=(${ATTR_NAME_HANA_MNS[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
    set_hana_attribute "X" "$the_master" "${ATTR[@]}"

    super_ocf_log info "DEC: ===> master_walk: priorities for site $site master1=$master1 master2=$master2 master3=$master3 ==> active_master=$active_master best_cold_master=$best_cold_master"
    super_ocf_log info "DEC: ===> master_walk: the_master=$the_master;"
    super_ocf_log info "FLOW $FUNCNAME the_master=$the_master rc=0"
}

# function: is_active_nameserver_slave
# params:   -
# rc:       0: yes its an active nameserver slave (running)
#           1: else
# globals:
#
# true, if the node has an active (runnig) master nameserver slave role
#
is_active_nameserver_slave()
{
  super_ocf_log info "FLOW $FUNCNAME ($*)"
  local rc=1 nRole="" nLsc=""  nSrmode="" nNsConf="" nNsCurr="" nIsConf="" nIsCurr=""
  nRole=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_ROLES[@]}" | tr ':' ' ')
  read nNsConf nNsCurr nIsConf nIsCurr <<< $nRole
  case "$nNsConf:$nNsCurr" in
      slave:slave )
         rc=0
         ;;
      * )
         rc=1
         ;;
  esac
  super_ocf_log info "DEC: is_active_nameserver_slave rc=$rc"
  super_ocf_log info "FLOW $FUNCNAME rc=$rc"
  return $rc
}

# function: is_master_nameserver
# params:   -
# rc:       0: yes its a master nameserver
#           1: else
# globals:
#
# true, if the node has an active or configured master nameserver role
#
function is_master_nameserver()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=1 nRole="" nLsc=""  nSrmode="" nNsConf="" nNsCurr="" nIsConf="" nIsCurr=""
    nRole=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_ROLES[@]}" | tr ':' ' ')
    read nNsConf nNsCurr nIsConf nIsCurr <<< $nRole
    case "$nNsConf:$nNsCurr" in
        master[123]:master )
           rc=0
           ;;
        master[123]:* )
           rc=0
           ;;
        * )
           rc=1
           ;;
    esac
    super_ocf_log info "DEC: is_master_nameserver  rc=$rc"
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

# function: is_the_master_nameserver
# params:   -
# rc:       0: yes, local node is THE master nameserver
#           1: else
# globals:
function is_the_master_nameserver()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=1
    if [ "$the_master" = "$NODENAME" ]; then
        rc=0
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: saphana_start_clone - start a hana clone instance
# params:   -
# globals:  OCF_*, ATTR_NAME_*, HANA_STATE_*, NODENAME, hana_LSS_Out
# saphana_start_clone
#
function saphana_start_clone() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local primary_status sync_attr score_master rc=$OCF_NOT_RUNNING
    local sqlrc;
    local chkusr;
    if is_the_master_nameserver; then
        check_for_primary; primary_status=$?
        #
        # DONE: PRIO0: NOW! Insert get_hana_landscape_status HERE to check
        #       nr_site_nodes vs. landscape-workers?
        get_hana_landscape_status; lss=$?
        # DONE: PRIO0: NOW! Parse hana_LSS_Out
        #lss_worker=$(echo "$hana_LSS_Out" | tr -d ' ' | awk -F '|' '$12 == "worker" { w++ } END { print w }')
        lss_worker=$(echo "$hana_LSS_Out" | awk '/indexServerConfigRole=worker/ { w++; } END { print w; }')
        #
        # Only start HANA, if there are enough nodes in the cluster to fulfill the landscape:
        # $nr_site_node -ge $lss_worker means that we count all nodes (worker and standby) in the
        #   cluster attributes and this number must be greater than the expected worker nodes
        #   from the landscape.
        # NOTE: ALL NODES MUST BE IN THE SAME HANA FAILOVER GROUP (LIKE DEFAULT)
        #
        if [ $nr_site_node -ge $lss_worker ]; then
            if [ $primary_status -eq $HANA_STATE_PRIMARY ]; then
                set_hana_attribute ${NODENAME} "DEMOTED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
                saphana_start_primary $lss; rc=$?
            else
                set_hana_attribute ${NODENAME} "DEMOTED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
                super_ocf_log debug "DBG: 007 * lpa_set_lpt 10 $sr_name"
                lpa_set_lpt  10 "$sr_name"
                saphana_start_secondary $lss; rc=$?
            fi
        else
            super_ocf_log info "ACT: nr_site_worker < lss_worker- setting WAITING4NODES"
            set_hana_attribute ${NODENAME} "WAITING4NODES" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
            rc=$OCF_SUCCESS
        fi
    else
        set_hana_attribute ${NODENAME} "DEMOTED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
        local start=0
        local lss=0
        local my_role=0
        if saphana_check_local_instance; then
            super_ocf_log info "ACT: Local SAP HANA instance already up and running"
        else
            # IF LS>=2 AND ROLE STANDBY "RESTART" INSTANCE
            get_hana_landscape_status; lss=$?
            if [ $lss -gt 2 ]; then
                #
                # only restart standby instances
                #
                my_role=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_ROLES[@]}")
                echo "$my_role" | grep -q ":standby" && start=1
                output=$($SAPCONTROL -nr $InstanceNr -function RestartInstance)
            fi
            #
        fi
        set_crm_master -INFINITY
        rc=$OCF_SUCCESS
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: saphana_stop_clone - stop a hana clone instance
# params:   -
# globals:  NODENAME(r), HANA_STATE_*(r)
# saphana_stop_clone
#
function saphana_stop_clone() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=0
    local primary_status="x"
    set_hana_attribute ${NODENAME} "UNDEFINED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
    super_ocf_log debug "DBG: SET UNDEFINED"
    check_for_primary; primary_status=$?
    if [ $primary_status -eq $HANA_STATE_SECONDARY ]; then
        super_ocf_log debug "DBG: 008 * lpa_set_lpt 10 $sr_name"
        lpa_set_lpt  10 "$sr_name"
    fi
    saphana_stop; rc=$?
    return $rc
}

function recover_site_attributes_from_file()
{
    local srHookAttributeFile="$1"
    local site

    if [ -f "$srHookAttributeFile" ]; then
        #
        # need to read attribute from the file
        #
        # TODO: PRIO0: also allow a=b instead of a = b
        # TODO: PRIO5: also allow multiple attributes each in an own line - for now only one attribute is needed
        read srATTR EQS srVALUE <"$srHookAttributeFile"
        case "$srATTR" in
            hana_${sid}_site_srHook_* )
                #
                # get site name from site attribute-name
                #
                site=${srATTR#hana_${sid}_site_srHook_}
                super_ocf_log info "DEC: srHookAttribute file found - recover lost SAP HA/DR event ($srATTR=$srVALUE)"
                #
                # DONE: PRIO0 - add check for srATTR name here
                #
                case "$srVALUE" in
                SOK )
                      super_ocf_log info "DEC: recover attributes from file $srHookAttributeFile - calling set_SRHOOK $site SOK"
                      set_SRHOOK "$site" "SOK"
                      ;;
                SFAIL )
                      super_ocf_log info "DEC: recover attributes from file $srHookAttributeFile - calling set_SRHOOK $site SFAIL"
                      set_SRHOOK "$site" "SFAIL"
                      ;;
                * )
                      super_ocf_log info "DEC: failed recover attributes from file $srHookAttributeFile - Unknown value <<$srVALUE>>"
                      ;;
                esac
                ;;
        esac
        rm "$srHookAttributeFile"
    fi
}

#
# function: saphana_monitor_primary - monitor a hana clone instance
# params:   -
# globals:  HANA_STATE_*(r), remoteHost, NODENAME, ATTR_NAME_*, OCF_*, PreferSiteTakeover
#
function saphana_monitor_primary()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=$OCF_ERR_GENERIC
    local promoted=0
    local init_attribute=0
    local LPTloc=-1
    local lparc=4
    local lss
    local remoreSync=""
    local my_role=""
    local node=""
    local nodeSite=""
    #
    # OK, we are running/are configured as HANA PRIMARY
    #
    super_ocf_log debug "DBG: saphana_monitor_clone: HANA_STATE_PRIMARY"
    #
    ##### CHECK, IF WE ARE DEMOTED (CLUSTER NODE ATTRIBUTE)
    #
    promote_attr=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_CLONE_STATE[@]}")
    super_ocf_log debug "DBG: saphana_monitor_clone: $ATTR_NAME_HANA_CLONE_STATE=$promote_attr"
    if [ -z "$promote_attr" ]; then
        init_attribute=1
        promoted=0;
    else
        case "$promote_attr" in
            PROMOTED )
                promoted=1;
                ;;
            DEMOTED )
                promoted=0;
                ;;
            # TODO: PRIO5: Implement WAITING4LPA WAITING4REG WAITING4PRIM WAITING4NODES
            # WAITING4LPA WAITING4REG WAITING4PRIM WAITING4NODES
            WAITING* )
                # DONE: lpa_check_lpt_status to come out of here :)
                # DONE: PRIO2: CHECK IF THE FIX FOR COMING OUT OF WAITING IS CORRECT
                get_hana_landscape_status; lss=$?
                if [ $lss -ge 2 ]; then
                    # seems admin already decided that for us? -> we are running - set DEMOTED
                    promoted=0;
                    LPTloc=$(date '+%s')
                    lpa_set_lpt $LPTloc "$sr_name"
                fi
                lpa_check_lpt_status; lparc=$?
                # TODO: PRIO1: Need to differ lpa_check_lpt_status return codes
                if [ $lparc -lt 2 ]; then
                    # lpa - no need to wait any longer - lets try a new start
                    saphana_start_clone
                    rc=$?
                    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
                    return $rc
                else
                    lpa_init_lpt $HANA_STATE_PRIMARY
                    # still waiting for second site to report lpa-lpt
                    if ocf_is_true "$AUTOMATED_REGISTER" ; then
                       super_ocf_log info "LPA: Still waiting for remote site to report LPA status"
                    else
                       super_ocf_log info "LPA: Dual primary detected and AUTOMATED_REGISTER='false' ==> WAITING"
                    fi
                    return $OCF_SUCCESS
                fi
                promoted=0;
                ;;
            UNDEFINED )
                if ocf_is_probe; then
                   promoted=0;
                else
                   set_hana_attribute ${NODENAME} "DEMOTED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
                   promoted=0;
                fi
                ;;
            * )
                promoted=0;
                ;;
        esac
    fi
    get_hana_landscape_status; lss=$?
    super_ocf_log debug "DBG: saphana_monitor_clone: get_hana_landscape_status=$lss"
    case "$lss" in
        0 ) # FATAL or ERROR
            rc=$OCF_ERR_GENERIC
            ;;
        1 ) # DOWN or ERROR
            # DONE: PRIO2: Maybe we need to differ between 0 and 1. While 0 is a fatal sap error, 1 is down/error
            if ocf_is_probe; then
                #
                # leave master score untouched, only set return code
                #
                rc=$OCF_NOT_RUNNING
            else
                if [ "$promoted" -eq 1 ]; then
                    # INSTANCE IS FAILED PRIMARY IN PROMOTED STATE
                    # DONE: PRIO2: Adjust with set_crm_master?
                    #       For Migration it would be good to decrease master score
                    #       For Reload locally we should NOT adjust the master score
                    if ocf_is_true "${PreferSiteTakeover}" ; then
                        #
                        # DONE: PRIO1: first check, if remote site is already (and still) in sync
                        # TODO: PRIO4: Decide if penalty (-9000) or weak (5) is better here to cover situations where other clone is gone
                        #
                        # TODO PRIO1: REMOVE remoteNode dependency - get_sync_status
                        # saphana_monitor_primary - get_SRHOOK for "the" remote site (TODO: PRIO3: multi-tier may need multiple remote sites)
                        remoteSync=$(get_SRHOOK "$remSR_name")
                        super_ocf_log info "DEC: PRIMDEFECT: remoteNode=$remoteNode SyncStatus=$remoteSync"
                        case "$remoteSync" in
                            SOK | PRIM )
                                 super_ocf_log info "DEC: PreferSiteTakeover selected so decrease promotion score here (and reset lpa)"
                                 if is_the_master_nameserver; then
                                     super_ocf_log info "DEC: PreferSiteTakeover lpa_set_lpt 20 $sr_name"
                                     lpa_set_lpt 20 "$sr_name"
                                 fi
                                 #
                                 # DOWNSCORING CRM_MASTER TO DEMOTE THE PRIMARY MUST BE DONE AFTER THE LPT MARKING NOT TO RISK CANCELLATION OF THE MONITOR
                                 #
                                 set_crm_master 5
                                 ;;
                            SFAIL )
                                 super_ocf_log info "DEC: PreferSiteTakeover selected BUT remote site is not in sync (SFAIL) ==> local restart preferred"
                                 ;;
                            * )
                                 super_ocf_log info "DEC: PreferSiteTakeover selected BUT remote site is not in sync ($remoteSync) ==> local restart preferred"
                                 ;;
                        esac
                    else
                        # TODO:  PRIO5: SCALE-OUT ONLY? Implement for local restart
                        #        It maybe that for the local restart we only need to decrease the secondaries promotion score
                        #super_ocf_log info "DEC: PreferSiteTakeover selected so decrease promotion score here"
                        ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
                        my_lss=$(get_hana_attribute "X" "${ATTR[@]}")
                        ATTR=(${ATTR_NAME_HANA_SRR[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
                        my_srr=$(get_hana_attribute "X" "${ATTR[@]}")
                        my_role=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_ROLES[@]}")
                        # saphana_monitor_primary - get SR attribute of OWN site (TODO: PRIO3: allow multi targets in the future)
                        # TODO PRIO3: Check, if we need that query as for a real primary this should always be "PRIM"
                        my_sync=$(get_SRHOOK "$remSR_name")
                        my_role="${my_lss}:${my_srr}:${my_role}"
                        scoring_crm_master "$my_role" "$my_sync"
                        rc=$OCF_FAILED_MASTER
                    fi
                    rc=$OCF_FAILED_MASTER
                else
                    # INSTANCE IS FAILED PRIMARY IN DEMOTED STATE
                    # TODO: PRIO3: Adjust with set_crm_master?
                    #       Current decision: Do NOT adjust master score now as other
                    #       steps should already have done that
                    #
                    super_ocf_log info "DEC: PRIMDEFECT (in DEMOTED status)"
                    rc=$OCF_NOT_RUNNING
                fi
            fi
            ;;
        2 | 3 | 4 ) # WARN, INFO or OK
            ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
            my_lss=$(get_hana_attribute "X" "${ATTR[@]}")
            ATTR=(${ATTR_NAME_HANA_SRR[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
            my_srr=$(get_hana_attribute "X" "${ATTR[@]}")
            my_role=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_ROLES[@]}")
            set_SRHOOK "$sr_name" "PRIM"
            # saphana_monitor_primary - get SR attribute of "the" remote site (TODO: PRIO3: allow multi targets in the
            my_sync=$(get_SRHOOK "$remSR_name")
            my_hana_role="${my_lss}:${my_srr}:${my_role}"
            if ocf_is_probe; then
                rc=$OCF_SUCCESS
                super_ocf_log info "SCORE: saphana_monitor_primary: (1) scoring_crm_master($my_hana_role,$my_sync)"
                scoring_crm_master "$my_hana_role" "$my_sync"
            else
                LPTloc=$(date '+%s')
                lpa_set_lpt $LPTloc "$sr_name"
                lpa_push_lpt $LPTloc
                if [ "$promoted" -eq 1 ]; then
                    set_hana_attribute "X" "$sr_name" "${ATTR_NAME_HANA_PRIM[@]}"
                    rc=$OCF_RUNNING_MASTER
                else
                    if [ "$init_attribute" -eq 1 ]; then
                        set_hana_attribute ${NODENAME} "DEMOTED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
                        rc=$OCF_RUNNING_MASTER
                    else
                        rc=$OCF_SUCCESS
                    fi
                fi
                ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
                my_lss=$(get_hana_attribute "X" "${ATTR[@]}")
                ATTR=(${ATTR_NAME_HANA_SRR[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
                my_srr=$(get_hana_attribute "X" "${ATTR[@]}")
                my_role=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_ROLES[@]}")
                #
                # first check, if we have a pending HA/DR provider attribute in the interface files
                #
                recover_site_attributes_from_file "/hana/shared/$SID/$InstanceName/.crm_attribute.$remSR_name"
		#
		# second check, if we have a pending HA/DR provider attribute in the interface files of "other" (cluster external) sites
		#
		for rFile in $(ls "/hana/shared/$SID/$InstanceName/.crm_attribute."* 2>/dev/null); do
			recover_site_attributes_from_file "$rFile"
		done
                #
                # saphana_monitor_primary - get SR attribute of "the" remote site (TODO: PRIO3: allow multi targets in the
                #
                my_sync=$(get_SRHOOK "$remSR_name")
                my_role="${my_lss}:${my_srr}:${my_role}"
                case "$my_role" in
                    [12]:P:*:*:*  ) # primary is down or may not answer hdbsql query so drop analyze_hana_sync_status{SRS|SQL}
                        ;;
                    [34]:P:*:*:*  ) # primary is up and should now be able to answer hdbsql query
                            case "$PreferSiteTakeover" in
                                never | NEVER | Never )
                                    # TODO: PRIO 2: How to handle this attribute for srHOOK with NEVER?
                                    # Currently it is NOT allowed to implement the srHook together with the (unsupported) preferSitetakeover 'never'
                                    # DONE: PRIO0: Check, if we need also to set the HOOK value or not??
                                    set_hana_attribute "X" "SNA" "${ATTR_NAME_HANA_SEC_SYNC_STATUS[@]}"
                                    set_SRHOOK "$remSR_name" "SNA"
                                    ;;
                                * )
                                    # TODO: PRIO 5: We could limit the analyze_hana_sync_statusSRS later
                                    #       (only call analyze_hana_sync_statusSRS, if Hook is on SWAIT)
                                    #       currently we still poll the old method to track differences
                                    #       of results of the SAP interfaces
                                    analyze_hana_sync_statusSRS
                                    if [ "$my_sync" = "SWAIT" ]; then
                                        # DONE: PRIO0: Check, if we really should overwrite "SWAIT" or keep that value
                                        # if we still did not got a hook time since the cluster start-up maybe the
                                        # HANA was already in (stable) sync, so the HANA Hook would never been called till
                                        # a new event comes up. This is why we transfer the 'polled' attribute to the Hook attribute.
                                        my_sync=$(get_hana_attribute "X" "${ATTR_NAME_HANA_SEC_SYNC_STATUS[@]}")
                                        set_SRHOOK "$remSR_name" "$my_sync"
                                        super_ocf_log info "DEC: ${ATTR_NAME_HANA_SRHOOK[0]} was still SWAIT, so copy attribute ${ATTR_NAME_HANA_SEC_SYNC_STATUS[0]} ($my_sync)"
                                    fi
                                    ;;
                            esac
                        ;;
                esac
                ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$remSR_name"
                rem_lss=$(get_hana_attribute "X" "${ATTR[@]}")
                ATTR=(${ATTR_NAME_HANA_SRR[@]}); ATTR[0]="${ATTR[0]}_$remSR_name"
                rem_srr=$(get_hana_attribute "X" "${ATTR[@]}")
                # TODO PRIO1: change from remoteHost to remoteMNS
                rem_role=$(get_hana_attribute ${remoteHost} "${ATTR_NAME_HANA_ROLES[@]}")
# HERE
                rem_clone_status=$(get_hana_attribute ${remoteHost} "${ATTR_NAME_HANA_CLONE_STATE[@]}")
                rem_role="${rem_lss}:${rem_srr}:${rem_role}"
                if [ "$promote_attr" = "DEMOTED" -a "$rem_clone_status" = "PROMOTED" ]; then
                   case "$rem_role" in
                       [234]:P:* ) # dual primary, but other instance marked as PROMOTED by the cluster
                # TODO: PRIO1: Need to differ lpa_check_lpt_status return codes
                           lpa_check_lpt_status; again_lpa_rc=$?
                           if [ $again_lpa_rc -eq 2 ]; then
                              super_ocf_log info "DEC: Dual primary detected, other instance is PROMOTED and lpa stalemate ==> local restart"
                              super_ocf_log debug "DBG: 009 * lpa_set_lpt 10 $sr_name"
                              lpa_set_lpt 10 "$sr_name"
                              lpa_push_lpt 10
                              rc=$OCF_NOT_RUNNING
                           fi
                          ;;
                   esac
                fi
                super_ocf_log info "SCORE: saphana_monitor_primary: (2) scoring_crm_master($my_role,$my_sync)"
                scoring_crm_master "$my_role" "$my_sync"
            fi
            ;;
        * ) # UNDEFINED STATUS
            if ocf_is_probe; then
                rc=$OCF_NOT_RUNNING
            else
                if [ "$promoted" -eq 1 ]; then
                     rc=$OCF_FAILED_MASTER
                else
                     rc=$OCF_NOT_RUNNING
                fi
            fi
            ;;
    esac
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: saphana_monitor_secondary - monitor a hana clone instance
# params:   -
# globals:  OCF_*, ATTR_NAME_*, NODENAME
# saphana_monitor_secondary
#
function saphana_monitor_secondary()
{
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=$OCF_ERR_GENERIC
    local promoted=0
    local init_attribute=0
    local lss
    local my_role
    #
    # OK, we are running as HANA SECONDARY
    #
    if ! lpa_get_lpt ${sr_name}; then
        super_ocf_log debug "DBG: 010 * lpa_set_lpt 10 $sr_name"
        lpa_set_lpt  10 "$sr_name"
        lpa_push_lpt 10
    fi
    promote_attr=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_CLONE_STATE[@]}")
    super_ocf_log debug "DBG: saphana_monitor_secondary: $ATTR_NAME_HANA_CLONE_STATE=$promote_attr"
    if [ -z "$promote_attr" ]; then
        init_attribute=1
        #  DONE: PRIO3: do we need to initialise also the DEMOTED attribute value?
        set_hana_attribute ${NODENAME} "DEMOTED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
        promoted=0;
    else
        case "$promote_attr" in
            PROMOTED ) # However - PROMOTED should only happen when coming back from maintenance
                promoted=1;
                ;;
            DEMOTED )  # This is the status we expect
                promoted=0;
                ;;
            # DONE: PRIO0 - Need to differ logs for different WAITING4*
            # WAITING4LPA WAITING4REG WAITING4PRIM WAITING4NODES
            WAITING4PRIM ) # We are WAITING for PRIMARY so not testing the HANA engine now but check for a new start
                if check_for_primary_master; then
                    super_ocf_log info "ACT: SECONDARY still in status WAITING - Primary now available - try a new start"
                    saphana_start_clone
                    rc=$?
                else
                    super_ocf_log info "ACT: saphana_monitor_clone: SECONDARY still in status WAITING - Primary is still missing"
                    return $OCF_SUCCESS
                fi
                promoted=0;
                ;;
            WAITING4NODES ) # TODO: PRIO1: HOW TO HANDLE WAITING4NODES IN DETAIL
                super_ocf_log info "ACT: Site still in status WAITING4NODES."
                saphana_start_clone
                promoted=0;
                ;;
            WAITING* )
                # WAITING4REG (which should not happen for a secondary - answer by lss)
                # WAITING4LPA (which should not happen for a secondary - answer by lss)
                super_ocf_log info "ACT: SECONDARY still in status $promote_attr"
                ;;
            UNDEFINED | * )
                if ocf_is_probe; then
                   promoted=0;
                else
                   set_hana_attribute ${NODENAME} "DEMOTED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
                   promoted=0;
                fi
                ;;
        esac
    fi
    #
    super_ocf_log debug "DBG: saphana_monitor_clone: HANA_STATE_SECONDARY"
    #
    # old method was: saphana_monitor - new method is get_hana_landscape_status
    get_hana_landscape_status; lss=$?
    super_ocf_log debug "DBG: saphana_monitor_clone: get_hana_landscape_status=$lss"
    case "$lss" in
        0 ) # FATAL
            # DONE: PRIO1: Maybe we need to differ between 0 and 1. While 0 is a fatal sap error, 1 is down/error
            # TODO: PRIO3: is OCF_ERR_GENERIC best option?
            super_ocf_log debug "DBG: 011 * lpa_set_lpt 10 $sr_name"
            lpa_set_lpt  10 "$sr_name"
            rc=$OCF_ERR_GENERIC
            ;;
        1 ) # ERROR
            super_ocf_log debug "DBG: 012 * lpa_set_lpt 10 $sr_name"
            lpa_set_lpt  10 "$sr_name"
            rc=$OCF_NOT_RUNNING
            ;;
        2 | 3 | 4 ) # WARN INFO OK
            rc=$OCF_SUCCESS
            # saphana_monitor_secondary - get SR attribute of OWN site (TODO: PRIO3: do we need to allow multi targets)
            sync_attr=$(get_SRHOOK "$sr_name")
            local hanaOM=""
            local hanaOut1=""
            # TODO: PRIO 3: check, if using getParameter.py is the best option to analyze the set operationMode
            # TODO: DONE 3: Should we default to logreplay for SAP HANA >= SPS11 ?
            hanaOut1=$(HANA_CALL --timeout 10 --use-su --cmd "getParameter.py --key=global.ini/system_replication/operation_mode --sapcontrol=1")
            hanaFilter1=$(echo "$hanaOut1" | awk -F/ 'BEGIN {out=0} /^SAPCONTROL-OK: <begin>/ { out=1 } /^SAPCONTROL-OK: <end>/ { out=0 } /=/ {if (out==1) {print $3} }')
            hanaOM=$(echo "$hanaFilter1" | awk -F= '$1=="operation_mode" {print $2}')
            set_hana_attribute ${NODENAME} "$hanaOM" "${ATTR_NAME_HANA_OPERATION_MODE[@]}"
            super_ocf_log debug "DBG: sync_attr=$sync_attr"
            case "$sync_attr" in
                SOK ) # This is a possible node to promote, when primary is missing
                    lpa_set_lpt  30 "$sr_name"
                    super_ocf_log info "DEC: secondary with sync status SOK ==> possible takeover node"
                    ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
                    my_lss=$(get_hana_attribute "X" "${ATTR[@]}")
                    ATTR=(${ATTR_NAME_HANA_SRR[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
                    my_srr=$(get_hana_attribute "X" "${ATTR[@]}")
                    my_role=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_ROLES[@]}")
                    #
                    # for scoring using the scoring table we need a combined value of <landscape-status>:<sytem-replication-role>:<landscape-host-role>
                    #
                    my_hana_role="${my_lss}:${my_srr}:${my_role}"
                    super_ocf_log info "DEC: scoring $my_hana_role : $sync_attr"
                    super_ocf_log info "SCORE: saphana_monitor_secondary: scoring_crm_master($my_hana_role,$sync_attr)"
                    scoring_crm_master "$my_hana_role" "$sync_attr"
                    ;;
                SFAIL ) # This is currently NOT a possible node to promote
                    super_ocf_log debug "DBG: 013a * lpa_set_lpt 10 $sr_name"
                    lpa_set_lpt  10 "$sr_name"
                    super_ocf_log info "DEC: secondary with sync status FAILED ==> EXCLUDE as possible takeover node"
                    set_crm_master -INFINITY
                    ;;
                SNA ) # This is a never-takeover-node
                    super_ocf_log debug "DBG: 013b * lpa_set_lpt 10 $sr_name"
                    lpa_set_lpt  10 "$sr_name"
                    super_ocf_log info "DEC: secondary with sync status n/a ==> EXCLUDE as possible takeover node"
                    set_crm_master -INFINITY
                    ;;
                * ) # Unknown sync status
                    super_ocf_log debug "DBG: 014 * lpa_set_lpt 10 $sr_name"
                    lpa_set_lpt  10 "$sr_name"
                    super_ocf_log info "DEC: secondary with sync status UNKNOWN/UNDEFINED ==> EXCLUDE as possible takeover node"
                    set_crm_master -INFINITY
                    ;;
            esac
            ;;
        * ) # UNDEFINED STATUS
            rc=$OCF_NOT_RUNNING
            ;;
    esac
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: saphana_monitor_clone - monitor a hana clone instance
# params:   -
# globals:  OCF_*, ATTR_NAME_*, HOSTNAME, HANA_STATE_*
# saphana_monitor_clone
#
function saphana_monitor_clone() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    #
    # TODO: PRIO3: For the secondary, which is missing the primary (so in status WAITING) what is better:
    #       a) returning 7 here and force cluster a restart of the slave
    #       b) starting the instance here inside the monitor -> may result in longer runtime, timeouts
    #
    # first check with the status function (OS tools) if there could be something like a SAP instance running
    # as we do not know here, if we are in primary or secondary state we do not want to start our monitoring
    # agents (sapstartsrv) on the wrong host
    local rc="$OCF_ERR_GENERIC"
    local promoted=0
    local init_attribute=0
    local nState=""
    local primary_status="$HANA_STATE_DEFECT"

    ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
    my_lss=$(get_hana_attribute "X" "${ATTR[@]}")
    ATTR=(${ATTR_NAME_HANA_SRR[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
    my_srr=$(get_hana_attribute "X" "${ATTR[@]}")
    my_role=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_ROLES[@]}")

    my_role="${my_lss}:${my_srr}:${my_role}"

    # TODO: PRIO2: check where we need to differ probes and regularly monitor calls
    if ocf_is_probe; then
        super_ocf_log debug "DBG: PROBE ONLY"
        #
        # check during probe, if this instance is *NOT* running as master
        # setting clone_state to "DEMOTED" is needed to avoid misleading "PROMOTED"/"PROMOTED"
        #
        master_node_name=$(crm_resource -W -r "$OCF_RESOURCE_INSTANCE"  | awk '$7 == "Master" { print $6 }')
        if [ "$master_node_name" != "$NODENAME" ]; then
            if [ "$my_lss" -ge 2 ]; then
                # we are in a probe, hana is running and we need to reset the clone_state
                # TODO PRIO5: Only reset attribute, if it is currently set to "PROMOTED"?
                set_hana_attribute "${NODENAME}" "DEMOTED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
            fi
        fi
    else
        super_ocf_log debug "DBG: REGULAR MONITOR"
    fi
    if is_the_master_nameserver; then
        #
        # First check, if we are PRIMARY or SECONDARY
        #
        check_for_primary; primary_status=$?
        if [ $primary_status -eq $HANA_STATE_PRIMARY ]; then
            saphana_check_local_instance
            saphana_monitor_primary; rc=$?
        elif [ $primary_status -eq $HANA_STATE_SECONDARY  ]; then
            saphana_check_local_instance
            saphana_monitor_secondary; rc=$?
        else
            #
            # OK, we are neither HANA PRIMARY nor HANA SECONDARY
            #
            super_ocf_log warn "ACT: saphana_monitor_clone: HANA_STATE_DEFECT"
            # TODO: PRIO2: Or only set_crm_master -INFINITY ?
            # TODO: PRIO9: Currently pacemaker-remote nodes does not support 'reboot' attributes
            # DONE: PRIO1: OCF_ERR_GENERIC is quite fatal, if "only" hdbnsutil has a small problem
            nState=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_CLONE_STATE[@]}")
            if saphana_check_local_instance; then
                case "$nState" in
                        DEMOTED )   rc=$OCF_SUCCESS;;
                        PROMOTED )  rc=$OCF_RUNNING_MASTER;;
                        UNDEFINED ) rc=$OCF_NOT_RUNNING;;
                        * )         rc=$OCF_NOT_RUNNING;;
                    esac
            else
	            case "$nState" in
                        DEMOTED )   rc=$OCF_NOT_RUNNING;;
                        PROMOTED )  rc=$OCF_NOT_RUNNING;; ## TODO: PRIO 3: FAILED_MASTER?
                        UNDEFINED ) rc=$OCF_NOT_RUNNING;;
                        * )         rc=$OCF_NOT_RUNNING;;
                esac
            fi
        fi
    else
        #
        # instance is not THE master nameserver
        #
        nState=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_CLONE_STATE[@]}")
        # TODO: PRIO2: Check, if we need to (re) start a failed SAP HANA instance in a swarm
        if ! saphana_check_local_instance; then
            #
            # local instance is not THE master nameserver and is currently stopped
            #
            # IF LS>=2 AND ROLE STANDBY "RESTART" INSTANCE
            get_hana_landscape_status; lss=$?
            if [ $lss -gt 2 ]; then
                super_ocf_log info "DEC: local instance is down landscape is up (lss=$lss)"
                rc=$OCF_NOT_RUNNING
            else
                #
                # local instance AND landscape are down
                #
                super_ocf_log info "DEC: local instance AND landscape are down (lss=$lss)"
                case "$nState" in
                    DEMOTED )
                                super_ocf_log info "DEC: DEMOTED => OCF_SUCCESS"
                                rc=$OCF_SUCCESS;;          # landscape is down and local instance is down -> report SUCCESS
                    PROMOTED )
                                super_ocf_log info "DEC: PROMOTED => OCF_FAILED_MASTER"
                                rc=$OCF_FAILED_MASTER;;   # same, but for MASTER -> report FAILED_MASTER
                    UNDEFINED )
                                super_ocf_log info "DEC: UNDEFINED => OCF_NOT_RUNNING"
                                rc=$OCF_NOT_RUNNING;;    # same is DEMOTED, but target should be down -> report NOT_RUNNING
                    * )
                                super_ocf_log info "DEC: <<UNKNOWN>> => OCF_NOT_RUNNING"
                                rc=$OCF_NOT_RUNNING;;
                esac
            fi
            #
        else
            #
            # local instance not THE master nameserver, but up-and-running
            #
            super_ocf_log info "DEC: the_master=<<$the_master>>"
            if [ -z "$the_master"  ]; then
                #
                # code for missing ALL master nameserver candidates
                #
                if is_active_nameserver_slave; then
                    #
                    # missing ALL master nameserver candidates, but local instance still running -> we need to trigger the cluster to take us down
                    #
                    super_ocf_log info "DEC: left-over instance ???"
                    get_hana_landscape_status; lss=$?
                    if [ $lss -le 1 ]; then
                        #
                        # landscape already reports down / error but local instance is up and running
                        # we are a left-over active instance without master nameserver nodes - report an error to the cluster to get this instance stopped
                        #
                        super_ocf_log info "DEC: left-over instance and lss=$lss - report generic error for lost instance"
                        rc=$OCF_ERR_GENERIC
                    else
                        #
                        # landscape still reports a running swarm AND local instance is up and running
                        # we are a left-over active instance without master nameserver nodes - wait till landscape status falls down to error
                        #
                        super_ocf_log info "DEC: left-over instance and lss=$lss - wait till lss falls down to error and SAP HANA did a final decision"
                        rc=$OCF_SUCCESS
                    fi
                else
                    #
                    # missing ALL master nameserver candidates AND local instance is also stopped -> just inform the cluster we are down
                    #
                    super_ocf_log info "DEC: left-over instance but already down"
                    rc=$OCF_NOT_RUNNING
                fi
            else
              #
              # code, if we still have ANY master nameserver candidates
              #
              # for self-scoring always use the local sites SR-attribute
              #
              ATTR=(${ATTR_NAME_HANA_LSS[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
              my_lss=$(get_hana_attribute "X" "${ATTR[@]}")
              ATTR=(${ATTR_NAME_HANA_SRR[@]}); ATTR[0]="${ATTR[0]}_$sr_name"
              my_srr=$(get_hana_attribute "X" "${ATTR[@]}")
              my_role=$(get_hana_attribute ${NODENAME} "${ATTR_NAME_HANA_ROLES[@]}")
              my_sync=$(get_SRHOOK "$sr_name")
              #
              # for scoring using the scoring table we need a combined value of <landscape-status>:<sytem-replication-role>:<landscape-host-role>
              #
              my_hana_role="${my_lss}:${my_srr}:${my_role}"
              super_ocf_log info "DEC: scoring $my_hana_role : $my_sync"
              scoring_crm_master "$my_hana_role" "$my_sync"
              # TODO: PRIO2: So far we do NOT monitor the swarm, if we are NOT the master nameserver
              #              This reflects the situation that swarm members get be started (as resource)
              #              before the master nameserver runs the StartSystem command
              #           => However we need to check, if it could happen that NO started (warm) master nameserver
              #              is available and we miss to monitor the degraded HANA on another node
              case "$nState" in
                  DEMOTED ) rc=$OCF_SUCCESS;;              # local instance is up -> report SUCCESS
                  PROMOTED ) rc=$OCF_RUNNING_MASTER;;      # same but state is MASTER -> report RUNNING_MASTER
                  UNDEFINED ) rc=$OCF_NOT_RUNNING;;        # same but state is DOWN -> report NOT_RUNNING
                  * ) rc=$OCF_NOT_RUNNING;;
              esac
            fi
        fi
    fi
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: saphana_promote_clone - promote a hana clone
# params:   -
# globals:  OCF_*(r), NODENAME(r), HANA_STATE_*, SID(r), InstanceName(r),
# saphana_promote_clone:
#    In a multi-state configuration get Master being the primary OR by running hana takeover
#
function saphana_promote_clone() {
  super_ocf_log info "FLOW $FUNCNAME ($*)"
  local rc=$OCF_ERR_GENERIC;
  local hana_sync;
  local primary_status=$HANA_STATE_DEFECT;
  #
  # first check, if we WILL be PRIMARY (checking HANA status)
  #
  set_hana_attribute ${NODENAME} "PROMOTED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
  check_for_primary; primary_status=$?
  #
  if [ $primary_status -eq $HANA_STATE_PRIMARY ]; then
     #
     # as we are already planned to be PRIMARY we only mark the node as PROMOTED
     #
     super_ocf_log info "ACT: Promoted $SID-$InstanceName as master (no hdbnsutil action needed)."
     rc=$OCF_SUCCESS;
  else
     if [ $primary_status -eq $HANA_STATE_SECONDARY ]; then
        #
        # we are SECONDARY and need to takeover ...  promote on the replica (secondary) side...
        #
        case "$PreferSiteTakeover" in
            never | Never | NEVER )
                super_ocf_log err "ACT: PreferSiteTakeover is set to never, so refusing promote on secondary side"
                rc=$OCF_ERR_GENERIC
                ;;
            * )
                hana_sync=$(get_SRHOOK "$sr_name")
                case "$hana_sync" in
                     SOK )
                        if is_the_master_nameserver; then
                            super_ocf_log info "ACT: SAP HANA TAKEOVER: - Promote REPLICA $SID-$InstanceName to be primary."
                            LPTloc=$(date '+%s')
                            # lpa_set_lpt 20 $remoteNode
                            lpa_set_lpt $LPTloc "$sr_name"
                            lpa_push_lpt $LPTloc
                            backup_global_and_nameserver
                            set_hana_attribute "$NODENAME" "$sr_name" "${ATTR_NAME_HANA_PRIM[@]}"
                            # TODO PRIO5: 145 should not be static but come from the scoring table
                            set_crm_master 145
                            set_hana_attribute "$NODENAME" "-" "${ATTR_NAME_HANA_SEC[@]}"
                            set_SRHOOK "$sr_name" "PRIM"
                            super_ocf_log info "ACT: SAP HANA TAKEOVER: SET MARK"
                            set_hana_attribute "$NODENAME" "T" "${ATTR_NAME_HANA_SRACTION[@]}"
                            su - $sidadm -c "hdbnsutil -sr_takeover"; srt_rc=$?
                            super_ocf_log info "ACT: SAP HANA TAKEOVER: srt_rc=$srt_rc"
                            set_hana_attribute "$NODENAME" "-" "${ATTR_NAME_HANA_SRACTION[@]}"
                            super_ocf_log info "ACT: SAP HANA TAKEOVER: REMOVE MARK"
                            backup_global_and_nameserver
                            #
                            # now gain check, if we are primary NOW
                            #
                            # TODO: PRIO3: check, if we need to distinguish between HANA_STATE_PRIMARY, HANA_STATE_SECONDARY, HANA_STATE_DEFECT
                            #
                        fi
                        if check_for_primary; then
                            rc=$OCF_SUCCESS;
                            # DONE: PRIO0: !!!
                            super_ocf_log info "ACT: SAP HANA TAKEOVER: MARK FORMER PRIMARY AS SWAIT"
                            set_SRHOOK "$remSR_name" "SWAIT"
                        else
                            super_ocf_log info "ACT: SAP HANA TAKEOVER: TAKEOVER FAILED? STILL NOT PRIMARY"
                            rc=$OCF_FAILED_MASTER
                        fi
                        ;;
                     * )
                        super_ocf_log err "ACT: HANA SYNC STATUS IS NOT 'SOK' SO THIS HANA SITE COULD NOT BE PROMOTED"
                        rc=$OCF_ERR_GENERIC
                        ;;
                esac
                ;;
        esac
     else
        #
        # neither PRIMARY nor SECONDARY - This clone instance seems to be broken!!
        #
        rc=$OCF_ERR_GENERIC
     fi
  fi
  rc=$?
  super_ocf_log info "FLOW $FUNCNAME rc=$rc"
  return $rc
}

#
# function: saphana_demote_clone - demote a hana clone instance
# params:   -
# globals:  OCF_*(r), NODENAME(r),
# saphana_demote_clone
#   the HANA System Replication (SR) runs in a multi-state configuration
#   While we could not change a HANA instance to be really demoted, we only mark the status for
#   correct monitor return codes
#
function saphana_demote_clone() {
    super_ocf_log info "FLOW $FUNCNAME ($*)"
    local rc=$OCF_ERR_GENERIC;
    set_hana_attribute ${NODENAME} "DEMOTED" "${ATTR_NAME_HANA_CLONE_STATE[@]}"
    rc=$OCF_SUCCESS;
    super_ocf_log info "ACT: Demoted $SID-$InstanceName."
    super_ocf_log info "FLOW $FUNCNAME rc=$rc"
    return $rc
}

#
# function: main - main function to operate
# params:   ACTION
# globals:  OCF_*(r), SID(w), sidadm(w), InstanceName(w), SAPVIRHOST(w), DIR_EXECUTABLE(w),
# globals:  SAPSTARTSRV(w), SAPCONTROL(w), DIR_PROFILE(w), SAPSTARTPROFILE(w), ACTION(w), CLACT(w), ra_rc(rw), $0(r), %ENV(r)
#

## GLOBALS
SID=""
sidadm=""
InstanceName=""
InstanceNr=""
SAPVIRHOST=""
DIR_EXECUTABLE=""
SAPSTARTSRV=""
SAPCONTROL=""
DIR_PROFILE=""
SAPSTARTPROFILE=""
SAPHanaFilter="ra-act-dec-lpa"



if [ $# -ne 1 ]
then
  saphana_usage
  exit $OCF_ERR_ARGS
fi

ACTION=$1
if [ "$ACTION" = "status" ]; then
    ACTION=monitor
fi

# These operations don't require OCF parameters to be set
# TODO: PRIO5: check, if notify is still not needing OCF parameters
case "$ACTION" in
    usage|methods)  saphana_$ACTION
                    exit $OCF_SUCCESS;;
    meta-data)      saphana_meta_data
                    exit $OCF_SUCCESS;;
    notify)         #saphana_notify
                    exit $OCF_SUCCESS;;
    *);;
esac
saphana_init

if ! ocf_is_root
then
    super_ocf_log err "ACT: $0 must be run as root"
    exit $OCF_ERR_PERM
fi

# parameter check
if  [ -z "$OCF_RESKEY_SID" ]
then
    super_ocf_log err "ACT: Please set parameter SID!"
    exit $OCF_ERR_ARGS
fi

if  [ -z "$OCF_RESKEY_InstanceNumber" ]
then
    super_ocf_log err "ACT: Please set parameter InstanceNumber!"
    exit $OCF_ERR_ARGS
fi

if is_clone
then
    CLACT=_clone
else
    if [ "$ACTION" = "promote" -o "$ACTION" = "demote" ]
    then
        super_ocf_log err "ACT: $ACTION called in a non multi-state environment"
        exit $OCF_ERR_ARGS
    fi
fi

# What kind of method was invoked?
super_ocf_log info "RA ==== begin action $ACTION$CLACT ($SAPHanaControllerVersion) ===="
ra_rc=$OCF_ERR_UNIMPLEMENTED
case "$ACTION" in
    start|stop|monitor|promote|demote) # Standard controlling actions
        saphana_$ACTION$CLACT
        ra_rc=$?
        ;;
    validate-all)
        saphana_validate
        ra_rc=$?
        ;;
    lpa_check)
        lpa_check_lpt_status
        ra_rc=$?
        ;;
    reload)
        ra_rc=$OCF_SUCCESS
        ;;
    *)  # seems to be an unknown request
        saphana_methods
        ra_rc=$OCF_ERR_UNIMPLEMENTED
        ;;
esac
timeE=$(date '+%s')
(( timeR = timeE - timeB ))
super_ocf_log debug "DBG: ==== SAPHanaFilter=$SAPHanaFilter"
super_ocf_log info "RA ==== end action $ACTION$CLACT with rc=${ra_rc} ($SAPHanaControllerVersion) (${timeR}s)===="
exit ${ra_rc}
# set ts=4 sw=4 sts=4 et
