Sunday, January 9, 2011

Cross Server Multipath

I was asked to come up with a method to compare multipath devices across an Oracle RAC cluster to ensure they are consistent. Beyond this there was one catch; it had to be run as a regular user. I would have normally run a multipath command but unfortunately that isn't available without elevated privileges. So my solution looks at /etc/multipath.conf, which is readable by a user, compare the entries across all nodes, and output a table at the end. You can see an example of the output in the initial comments.

#!/bin/bash                                                                                                                                                                         
# Script Information
# Version: 1.0
# Created By: Michael England
                                                                                    
# Parses /etc/multipath.conf on any number of nodes (with ssh) and produces a report comparing all of the entries
# e.g.
# WWID                                    node1                         node2                         node3                         Status
# 360060e8005be08000000be08000011ab       apps_001_40g                  --                            --                            INVALID
# 360060e8005be08000000be08000011ac       apps_002_40g                  --                            --                            INVALID
# 360060e8005be08000000be0800004000       ora_shared_data_001_407g      ora_shared_data_001_407g      ora_shared_data_001_407g      OK
# 360060e8005be08000000be0800004001       ora_shared_data_002_407g      ora_shared_data_002_407g      ora_shared_data_002_407g      OK

######
### function read_multipath
### reads a file looking for wwid and alias pairs
### e.g.
### multipath {
###     wwid    36006016015a019004e9820d8b56cde11
###     alias   vote_disk1
### }
### becomes
###     wwid 36006016015a019004e9820d8b56cde11 alias vote_disk1
### if it is missing a wwid or alias that part will be blank
### e.g.
###     wwid 36006016015a019004e9820d8b56cde11
### The first entry will be blank as we increment COUNT on the first multipath
######
function parse_multipath {
        COUNT=0
        unset MPATH_ENTRIES
        oldIFS=$IFS
        IFS=$'\n'
        for line in echo $NODE_MPATH_RESULT
        do
                # skip anything beginning with a comment
                # after bash 3.2 we can't use "" anymore on right hand side
                if [[ $line == \#* ]]
                then
                        continue
                fi
                # look for multipath keyword and increment array counter
                # \ .* excludes multipaths
                if [[ $line =~ multipath\ .* ]]
                then

                        (( COUNT++ ))
                # anything that starts with wwid or alias and add it to the current array element
                elif [[ $line =~ wwid* ]] || [[ $line =~ alias* ]]
                then
                        MPATH_ENTRIES[$COUNT]=`echo "${MPATH_ENTRIES[$COUNT]} $line"`
                fi
        done
        IFS=$oldIFS
}

######
### function search_wwid
### Args: 
###     $1 search key
### Searches the WWID_MAP array for a given key and returns the array position of that element
######
function search_wwid {
        key="$1"
        for index in ${!WWID_MAP[@]}
        do
                if [[ ${WWID_MAP[$index]} =~ $key ]]
                then
                        echo $index
                        # exit the function
                        exit
                fi
        done
        echo -1
}

######
### function search_node
### Args:
###     $1 search key
### Searches the NODE_ARRAY array for a given key and returns the array position of that element
######
function search_node {
        key="\<$1\>"
        for index in ${!NODE_ARRAY[@]}
        do
                if [[ ${NODE_ARRAY[index]} =~ $key ]]
                then
                        echo $index
                        exit
                fi
        done
        echo -1
}

######
### function assign_wwid
### populates WWID_MAP array with a common view to all nodes multipath entries
### reads from the current MPATH_ENTRIES which should be the output from parse_multipath for one node only
### array will look like the following
### WWID        ALIAS_node1     ALIAS_node2     ALIAS_node3     etc
### it does not include any status information
######
function assign_wwid {
        for index in ${!MPATH_ENTRIES[@]}
        do
                # count the number of items starting from 1 (wc -w is word)
                ITEM_COUNT=`echo ${MPATH_ENTRIES[index]} | wc -w`
                # make sure our entry is 4 words long (wwid  alias ), this is really an error condition as every wwid should have an alias
                # if it doesn't pad it with "--"
                for (( i=$ITEM_COUNT; $i <= 4; i++ ))
                do
                        MPATH_ENTRIES[$index]="${MPATH_ENTRIES[index]} --"
                done

                unset FILLER
                # set the key to the WWID and search the WWID_MAP to see if one already exists
                key=`echo ${MPATH_ENTRIES[$index]} | awk '{print $2}'`
                RETVAL=`search_wwid $key`
                if [[ $RETVAL -lt 0 ]]
                then
                        # this is a new WWID
                        # check to see what node ordinal this is (first, second, third, etc...)
                        # if this isn't the first node we will have to fill in -- for the ones before to indicate this WWID doesn't exist on previous nodes
                        NODE_POSITION=`search_node $1`
                        for (( i=0; $i < $NODE_POSITION; i++ ))
                        do
                                FILLER="$FILLER --"
                        done
                        # tack on a new element with the wwid, and filler required, and then the node alias
                        WWID_MAP[${#WWID_MAP[@]}]=`echo ${MPATH_ENTRIES[index]} | awk '{ print $2, filler, $4 }' filler="$FILLER"`
                else
                        # a WWID already exists, just add on
                        (( NODE_POSITION=`search_node $1` + 1 ))
                        MAP_LENGTH=`echo ${WWID_MAP[RETVAL]} | wc -w`
                        # the node position will be greater than the length if we have a hole... plug it with filler
                        for (( i=$MAP_LENGTH; $i < $NODE_POSITION; i++ ))
                        do
                                FILLER="$FILLER --"
                        done
                        # append any required filler and then the alias for this node
                        WWID_MAP[$RETVAL]=${WWID_MAP[$RETVAL]}\ $FILLER\ `echo ${MPATH_ENTRIES[$index]} | awk '{print $4}'`
                fi
        done
}

######
### function check_status
### -ensures each WWID_MAP is the correct length by first finding the longest entry
###  and then padding all others to that length with --
### -creates a WWID_STATUS array, each element aligns with a WWID_MAP element 
###  it checks if all aliases in WWID_MAP are the same, if so marks a green OK, if not a red INVALID
### WWID_STATUS array will have one entry for each WWID_MAP
###   OK
###   OK
###   INVALID
###   etc
######
function check_status {
        # find the longest element in WWID_MAP
        LONGEST=0
        for index in ${!WWID_MAP[@]}
        do
                LENGTH=`echo ${WWID_MAP[index]} | wc -w`
                if [[ $LENGTH > $LONGEST ]]
                then
                        LONGEST=$LENGTH
                fi
        done
        for index in ${!WWID_MAP[@]}
        do
                # count the number of items in this element
                COUNT=`echo ${WWID_MAP[index]} | wc -w`
                # another way to do this is convert to an array and then count the array elements
                # ARRAY=( $(echo ${WWID_MAP[index]}) )
                # for i in `seq ${#ARRAY[@]} $(( $LONGEST - 1 ))` or for i in `seq 2 $(( ${#ARRAY[@]} - 1))` when math rquired
                # if the array is shorter, pad it
                # longest is reduced by 1 because arrays are 0 based
                for i in `seq $COUNT $(( $LONGEST - 1 ))`
                do
                        WWID_MAP[$index]="${WWID_MAP[$index]} --"

                done

                # recount the element as its size may have just changed
                COUNT=`echo ${WWID_MAP[index]} | wc -w`
                if [ $COUNT -eq 2 ]
                then
                        WWID_STATUS[$index]="\e[0;32mOK\e[0;30m"
                fi
                # for all items starting at 2 (the third item) to the end - 1 (zero based)
                # compare with the item previous (i.e. 3:2, 4:3, 5:4, etc)
                for i in `seq 2 $(( $COUNT - 1))`
                do
                        ARRAY=( $(echo ${WWID_MAP[index]}) )
                        # if they don't match, or this item is "--" mark it as invalid
                        if [[ "${ARRAY[i]}" != "${ARRAY[i-1]}" ]] || [[ "${ARRAY[i]}" = "--" ]]
                        then
                                WWID_STATUS[$index]="\e[0;31mINVALID\e[0;30m"
                                break
                        else
                                WWID_STATUS[$index]="\e[0;32mOK\e[0;30m"
                        fi
                done
        done
}

######
### function exchange
### helper for bubble sort to swap both WWID_MAP and WWID_STATUS entries
######
function exchange {
        local temp=${WWID_MAP[$1]}
        local status_temp=${WWID_STATUS[$1]}
        WWID_MAP[$1]=${WWID_MAP[$2]}
        WWID_MAP[$2]=$temp

        WWID_STATUS[$1]=${WWID_STATUS[$2]}
        WWID_STATUS[$2]=$status_temp
}

######
### function sort_results
### a bubble sort of both WWID_MAP and WWID_STATUS based on a user specified sort field (SORT_FIELD)
### if SORT_FIELD = a digit, uses awk to compare values for that column
### if SORT_FIELD = valid | invalid simply compares the WWID_STATUS array elements to be either > or <
######
function sort_results {
        number_of_elements=${#WWID_MAP[@]}
        (( comparisons = $number_of_elements - 1 ))
        count=1
        while [ "$comparisons" -gt 0 ]
        do
                index=0
                while [ "$index" -lt "$comparisons" ]
                do
                        if [[ $SORT_FIELD = [[:digit:]]* ]]
                        then
                                if [[ `echo ${WWID_MAP[index]} | awk '{print $i}' i=$SORT_FIELD` > `echo ${WWID_MAP[ (( $index + 1 ))]} | awk '{print $i}' i=$SORT_FIELD` ]]
                                then
                                        exchange $index `expr $index + 1`
                                fi
                        elif [[ $SORT_FIELD = "invalid" ]]
                        then
                                if [[ ${WWID_STATUS[index]} > ${WWID_STATUS[`expr $index + 1`]} ]]
                                then
                                        exchange $index `expr $index + 1`
                                fi
                        elif [[ $SORT_FIELD = "valid" ]]
                        then
                                if [[ ${WWID_STATUS[index]} < ${WWID_STATUS[`expr $index + 1`]} ]]
                                then
                                        exchange $index `expr $index + 1`
                                fi
                        fi
                        (( index += 1 ))
                done
                (( comparisons -= 1 ))
                (( count += 1 ))
        done
}

######
### function echo_results
### outputs the collected results in WWID_MAP and WWID_STATUS to the screen
######
function echo_results {
        # put together a header row
        # WWID      Status
        RESULT_SET="WWID"
        for node in $NODE_LIST
        do
                RESULT_SET="$RESULT_SET $node"
        done
        RESULT_SET="$RESULT_SET Status"
        # print the header row, wwid (1st column) is 40 characters wide, everything else is 30
        echo -e $RESULT_SET | awk '{ printf "%-40s", $1 }'
        echo -e $RESULT_SET | awk '{ for (i=2; i<=NF; i++) printf "%-30s", $i }'
        awk 'BEGIN {printf "\n"}'
        echo "---------------------"
        # for each element print the first at 40 characters, all others (colume 2 - NF) at 30, then the status (using echo so the colours work)
        # NF is Number of Fields
        for index in ${!WWID_MAP[@]}
        do
                echo ${WWID_MAP[index]} | awk '{ printf  "%-40s", $1 }'
                echo ${WWID_MAP[index]} | awk '{ for (i=2; i<=NF; i++) printf "%-30s", $i }'
                echo -e "${WWID_STATUS[index]}"
        done
}

######
### function usage
### outputs help message
######
function usage {
        echo -e "Usage: `basename $0` -n {node1,node2,node3,etc} [-sort {field} | -sort_invalid | -sort_valid] [-user {username}]"
        echo -e "\t-n is a node list separated by commas, the script will ssh and read /etc/multipath.conf on each"
        echo -e "\t-sort {field} sorts the output. 0 sorts the WWID number, 1..x sorts for a specific node"
        echo -e "\t-sort_invalid places invalid alias entries at the top"
        echo -e "\t-sort_valid places valid alias entries at the top"
        echo -e "\t-user {username} allows a different username to be used than the one currently logged in"
        exit
}

######
### parse the command line for options
######
if [ -z "$1" ]
then
        usage
fi
until [ -z "$1" ]
do
        case "$1" in
        -n)
                shift
                NODE_LIST=`echo $1 | tr "," " "`
                ;;
        -sort)
                shift
                SORT_FIELD=`expr $1 + 1`
                ;;
        -sort_invalid)
                SORT_FIELD="invalid"
                ;;
        -sort_valid)
                SORT_FIELD="valid"
                ;;
        -user)
                shift
                USERNAME="-l $1"
                ;;
        *)
                usage
                ;;
        esac
        shift
done

# convert NODE_LIST to an array so we can search the position
NODE_ARRAY=( $(echo "$NODE_LIST") )

# for each node in the list
#  - grab its multipath.conf (ssh)
#  - parse out wwid and alias pairs (parse_multipath)
#  - add it to the master WWID list (assign_wwid)
# then fill in any blank holes and the overall status (check_status)
# sort the results as requested (sort_results)
# then output to display (echo_results)
for node in $NODE_LIST
do
        NODE_MPATH_RESULT=`ssh $node $USERNAME -C "cat /etc/multipath.conf" 2> /dev/null`
        RETVAL=$?
        if [ $RETVAL != 0 ]
        then
                echo "--- Error retrieving /etc/multipath.conf from node $node ---"
        fi
        parse_multipath $node
        assign_wwid $node
done
check_status
sort_results
echo_results

No comments:

Post a Comment