#!/bin/bash
# Script Information
# Version: 1.0
# Created By: Michael England
# Parses /etc/multipath.conf on any number of nodes (with ssh) and produces a report comparing all of the entries
# e.g.
# WWID node1 node2 node3 Status
# 360060e8005be08000000be08000011ab apps_001_40g -- -- INVALID
# 360060e8005be08000000be08000011ac apps_002_40g -- -- INVALID
# 360060e8005be08000000be0800004000 ora_shared_data_001_407g ora_shared_data_001_407g ora_shared_data_001_407g OK
# 360060e8005be08000000be0800004001 ora_shared_data_002_407g ora_shared_data_002_407g ora_shared_data_002_407g OK
######
### function read_multipath
### reads a file looking for wwid and alias pairs
### e.g.
### multipath {
### wwid 36006016015a019004e9820d8b56cde11
### alias vote_disk1
### }
### becomes
### wwid 36006016015a019004e9820d8b56cde11 alias vote_disk1
### if it is missing a wwid or alias that part will be blank
### e.g.
### wwid 36006016015a019004e9820d8b56cde11
### The first entry will be blank as we increment COUNT on the first multipath
######
function parse_multipath {
COUNT=0
unset MPATH_ENTRIES
oldIFS=$IFS
IFS=$'\n'
for line in echo $NODE_MPATH_RESULT
do
# skip anything beginning with a comment
# after bash 3.2 we can't use "" anymore on right hand side
if [[ $line == \#* ]]
then
continue
fi
# look for multipath keyword and increment array counter
# \ .* excludes multipaths
if [[ $line =~ multipath\ .* ]]
then
(( COUNT++ ))
# anything that starts with wwid or alias and add it to the current array element
elif [[ $line =~ wwid* ]] || [[ $line =~ alias* ]]
then
MPATH_ENTRIES[$COUNT]=`echo "${MPATH_ENTRIES[$COUNT]} $line"`
fi
done
IFS=$oldIFS
}
######
### function search_wwid
### Args:
### $1 search key
### Searches the WWID_MAP array for a given key and returns the array position of that element
######
function search_wwid {
key="$1"
for index in ${!WWID_MAP[@]}
do
if [[ ${WWID_MAP[$index]} =~ $key ]]
then
echo $index
# exit the function
exit
fi
done
echo -1
}
######
### function search_node
### Args:
### $1 search key
### Searches the NODE_ARRAY array for a given key and returns the array position of that element
######
function search_node {
key="\<$1\>"
for index in ${!NODE_ARRAY[@]}
do
if [[ ${NODE_ARRAY[index]} =~ $key ]]
then
echo $index
exit
fi
done
echo -1
}
######
### function assign_wwid
### populates WWID_MAP array with a common view to all nodes multipath entries
### reads from the current MPATH_ENTRIES which should be the output from parse_multipath for one node only
### array will look like the following
### WWID ALIAS_node1 ALIAS_node2 ALIAS_node3 etc
### it does not include any status information
######
function assign_wwid {
for index in ${!MPATH_ENTRIES[@]}
do
# count the number of items starting from 1 (wc -w is word)
ITEM_COUNT=`echo ${MPATH_ENTRIES[index]} | wc -w`
# make sure our entry is 4 words long (wwid alias ), this is really an error condition as every wwid should have an alias
# if it doesn't pad it with "--"
for (( i=$ITEM_COUNT; $i <= 4; i++ ))
do
MPATH_ENTRIES[$index]="${MPATH_ENTRIES[index]} --"
done
unset FILLER
# set the key to the WWID and search the WWID_MAP to see if one already exists
key=`echo ${MPATH_ENTRIES[$index]} | awk '{print $2}'`
RETVAL=`search_wwid $key`
if [[ $RETVAL -lt 0 ]]
then
# this is a new WWID
# check to see what node ordinal this is (first, second, third, etc...)
# if this isn't the first node we will have to fill in -- for the ones before to indicate this WWID doesn't exist on previous nodes
NODE_POSITION=`search_node $1`
for (( i=0; $i < $NODE_POSITION; i++ ))
do
FILLER="$FILLER --"
done
# tack on a new element with the wwid, and filler required, and then the node alias
WWID_MAP[${#WWID_MAP[@]}]=`echo ${MPATH_ENTRIES[index]} | awk '{ print $2, filler, $4 }' filler="$FILLER"`
else
# a WWID already exists, just add on
(( NODE_POSITION=`search_node $1` + 1 ))
MAP_LENGTH=`echo ${WWID_MAP[RETVAL]} | wc -w`
# the node position will be greater than the length if we have a hole... plug it with filler
for (( i=$MAP_LENGTH; $i < $NODE_POSITION; i++ ))
do
FILLER="$FILLER --"
done
# append any required filler and then the alias for this node
WWID_MAP[$RETVAL]=${WWID_MAP[$RETVAL]}\ $FILLER\ `echo ${MPATH_ENTRIES[$index]} | awk '{print $4}'`
fi
done
}
######
### function check_status
### -ensures each WWID_MAP is the correct length by first finding the longest entry
### and then padding all others to that length with --
### -creates a WWID_STATUS array, each element aligns with a WWID_MAP element
### it checks if all aliases in WWID_MAP are the same, if so marks a green OK, if not a red INVALID
### WWID_STATUS array will have one entry for each WWID_MAP
### OK
### OK
### INVALID
### etc
######
function check_status {
# find the longest element in WWID_MAP
LONGEST=0
for index in ${!WWID_MAP[@]}
do
LENGTH=`echo ${WWID_MAP[index]} | wc -w`
if [[ $LENGTH > $LONGEST ]]
then
LONGEST=$LENGTH
fi
done
for index in ${!WWID_MAP[@]}
do
# count the number of items in this element
COUNT=`echo ${WWID_MAP[index]} | wc -w`
# another way to do this is convert to an array and then count the array elements
# ARRAY=( $(echo ${WWID_MAP[index]}) )
# for i in `seq ${#ARRAY[@]} $(( $LONGEST - 1 ))` or for i in `seq 2 $(( ${#ARRAY[@]} - 1))` when math rquired
# if the array is shorter, pad it
# longest is reduced by 1 because arrays are 0 based
for i in `seq $COUNT $(( $LONGEST - 1 ))`
do
WWID_MAP[$index]="${WWID_MAP[$index]} --"
done
# recount the element as its size may have just changed
COUNT=`echo ${WWID_MAP[index]} | wc -w`
if [ $COUNT -eq 2 ]
then
WWID_STATUS[$index]="\e[0;32mOK\e[0;30m"
fi
# for all items starting at 2 (the third item) to the end - 1 (zero based)
# compare with the item previous (i.e. 3:2, 4:3, 5:4, etc)
for i in `seq 2 $(( $COUNT - 1))`
do
ARRAY=( $(echo ${WWID_MAP[index]}) )
# if they don't match, or this item is "--" mark it as invalid
if [[ "${ARRAY[i]}" != "${ARRAY[i-1]}" ]] || [[ "${ARRAY[i]}" = "--" ]]
then
WWID_STATUS[$index]="\e[0;31mINVALID\e[0;30m"
break
else
WWID_STATUS[$index]="\e[0;32mOK\e[0;30m"
fi
done
done
}
######
### function exchange
### helper for bubble sort to swap both WWID_MAP and WWID_STATUS entries
######
function exchange {
local temp=${WWID_MAP[$1]}
local status_temp=${WWID_STATUS[$1]}
WWID_MAP[$1]=${WWID_MAP[$2]}
WWID_MAP[$2]=$temp
WWID_STATUS[$1]=${WWID_STATUS[$2]}
WWID_STATUS[$2]=$status_temp
}
######
### function sort_results
### a bubble sort of both WWID_MAP and WWID_STATUS based on a user specified sort field (SORT_FIELD)
### if SORT_FIELD = a digit, uses awk to compare values for that column
### if SORT_FIELD = valid | invalid simply compares the WWID_STATUS array elements to be either > or <
######
function sort_results {
number_of_elements=${#WWID_MAP[@]}
(( comparisons = $number_of_elements - 1 ))
count=1
while [ "$comparisons" -gt 0 ]
do
index=0
while [ "$index" -lt "$comparisons" ]
do
if [[ $SORT_FIELD = [[:digit:]]* ]]
then
if [[ `echo ${WWID_MAP[index]} | awk '{print $i}' i=$SORT_FIELD` > `echo ${WWID_MAP[ (( $index + 1 ))]} | awk '{print $i}' i=$SORT_FIELD` ]]
then
exchange $index `expr $index + 1`
fi
elif [[ $SORT_FIELD = "invalid" ]]
then
if [[ ${WWID_STATUS[index]} > ${WWID_STATUS[`expr $index + 1`]} ]]
then
exchange $index `expr $index + 1`
fi
elif [[ $SORT_FIELD = "valid" ]]
then
if [[ ${WWID_STATUS[index]} < ${WWID_STATUS[`expr $index + 1`]} ]]
then
exchange $index `expr $index + 1`
fi
fi
(( index += 1 ))
done
(( comparisons -= 1 ))
(( count += 1 ))
done
}
######
### function echo_results
### outputs the collected results in WWID_MAP and WWID_STATUS to the screen
######
function echo_results {
# put together a header row
# WWID Status
RESULT_SET="WWID"
for node in $NODE_LIST
do
RESULT_SET="$RESULT_SET $node"
done
RESULT_SET="$RESULT_SET Status"
# print the header row, wwid (1st column) is 40 characters wide, everything else is 30
echo -e $RESULT_SET | awk '{ printf "%-40s", $1 }'
echo -e $RESULT_SET | awk '{ for (i=2; i<=NF; i++) printf "%-30s", $i }'
awk 'BEGIN {printf "\n"}'
echo "---------------------"
# for each element print the first at 40 characters, all others (colume 2 - NF) at 30, then the status (using echo so the colours work)
# NF is Number of Fields
for index in ${!WWID_MAP[@]}
do
echo ${WWID_MAP[index]} | awk '{ printf "%-40s", $1 }'
echo ${WWID_MAP[index]} | awk '{ for (i=2; i<=NF; i++) printf "%-30s", $i }'
echo -e "${WWID_STATUS[index]}"
done
}
######
### function usage
### outputs help message
######
function usage {
echo -e "Usage: `basename $0` -n {node1,node2,node3,etc} [-sort {field} | -sort_invalid | -sort_valid] [-user {username}]"
echo -e "\t-n is a node list separated by commas, the script will ssh and read /etc/multipath.conf on each"
echo -e "\t-sort {field} sorts the output. 0 sorts the WWID number, 1..x sorts for a specific node"
echo -e "\t-sort_invalid places invalid alias entries at the top"
echo -e "\t-sort_valid places valid alias entries at the top"
echo -e "\t-user {username} allows a different username to be used than the one currently logged in"
exit
}
######
### parse the command line for options
######
if [ -z "$1" ]
then
usage
fi
until [ -z "$1" ]
do
case "$1" in
-n)
shift
NODE_LIST=`echo $1 | tr "," " "`
;;
-sort)
shift
SORT_FIELD=`expr $1 + 1`
;;
-sort_invalid)
SORT_FIELD="invalid"
;;
-sort_valid)
SORT_FIELD="valid"
;;
-user)
shift
USERNAME="-l $1"
;;
*)
usage
;;
esac
shift
done
# convert NODE_LIST to an array so we can search the position
NODE_ARRAY=( $(echo "$NODE_LIST") )
# for each node in the list
# - grab its multipath.conf (ssh)
# - parse out wwid and alias pairs (parse_multipath)
# - add it to the master WWID list (assign_wwid)
# then fill in any blank holes and the overall status (check_status)
# sort the results as requested (sort_results)
# then output to display (echo_results)
for node in $NODE_LIST
do
NODE_MPATH_RESULT=`ssh $node $USERNAME -C "cat /etc/multipath.conf" 2> /dev/null`
RETVAL=$?
if [ $RETVAL != 0 ]
then
echo "--- Error retrieving /etc/multipath.conf from node $node ---"
fi
parse_multipath $node
assign_wwid $node
done
check_status
sort_results
echo_results
Sunday, January 9, 2011
Cross Server Multipath
I was asked to come up with a method to compare multipath devices across an Oracle RAC cluster to ensure they are consistent. Beyond this there was one catch; it had to be run as a regular user. I would have normally run a multipath command but unfortunately that isn't available without elevated privileges. So my solution looks at /etc/multipath.conf, which is readable by a user, compare the entries across all nodes, and output a table at the end. You can see an example of the output in the initial comments.
Subscribe to:
Posts (Atom)