#!/bin/bash
# Script Information
# Version: 1.0
# Created By: Michael England
# Parses /etc/multipath.conf on any number of nodes (with ssh) and produces a report comparing all of the entries
# e.g.
# WWID node1 node2 node3 Status
# 360060e8005be08000000be08000011ab apps_001_40g -- -- INVALID
# 360060e8005be08000000be08000011ac apps_002_40g -- -- INVALID
# 360060e8005be08000000be0800004000 ora_shared_data_001_407g ora_shared_data_001_407g ora_shared_data_001_407g OK
# 360060e8005be08000000be0800004001 ora_shared_data_002_407g ora_shared_data_002_407g ora_shared_data_002_407g OK
######
### function read_multipath
### reads a file looking for wwid and alias pairs
### e.g.
### multipath {
### wwid 36006016015a019004e9820d8b56cde11
### alias vote_disk1
### }
### becomes
### wwid 36006016015a019004e9820d8b56cde11 alias vote_disk1
### if it is missing a wwid or alias that part will be blank
### e.g.
### wwid 36006016015a019004e9820d8b56cde11
### The first entry will be blank as we increment COUNT on the first multipath
######
function parse_multipath {
COUNT=0
unset MPATH_ENTRIES
oldIFS=$IFS
IFS=$'\n'
for line in echo $NODE_MPATH_RESULT
do
# skip anything beginning with a comment
# after bash 3.2 we can't use "" anymore on right hand side
if [[ $line == \#* ]]
then
continue
fi
# look for multipath keyword and increment array counter
# \ .* excludes multipaths
if [[ $line =~ multipath\ .* ]]
then
(( COUNT++ ))
# anything that starts with wwid or alias and add it to the current array element
elif [[ $line =~ wwid* ]] || [[ $line =~ alias* ]]
then
MPATH_ENTRIES[$COUNT]=`echo "${MPATH_ENTRIES[$COUNT]} $line"`
fi
done
IFS=$oldIFS
}
######
### function search_wwid
### Args:
### $1 search key
### Searches the WWID_MAP array for a given key and returns the array position of that element
######
function search_wwid {
key="$1"
for index in ${!WWID_MAP[@]}
do
if [[ ${WWID_MAP[$index]} =~ $key ]]
then
echo $index
# exit the function
exit
fi
done
echo -1
}
######
### function search_node
### Args:
### $1 search key
### Searches the NODE_ARRAY array for a given key and returns the array position of that element
######
function search_node {
key="\<$1\>"
for index in ${!NODE_ARRAY[@]}
do
if [[ ${NODE_ARRAY[index]} =~ $key ]]
then
echo $index
exit
fi
done
echo -1
}
######
### function assign_wwid
### populates WWID_MAP array with a common view to all nodes multipath entries
### reads from the current MPATH_ENTRIES which should be the output from parse_multipath for one node only
### array will look like the following
### WWID ALIAS_node1 ALIAS_node2 ALIAS_node3 etc
### it does not include any status information
######
function assign_wwid {
for index in ${!MPATH_ENTRIES[@]}
do
# count the number of items starting from 1 (wc -w is word)
ITEM_COUNT=`echo ${MPATH_ENTRIES[index]} | wc -w`
# make sure our entry is 4 words long (wwid alias ), this is really an error condition as every wwid should have an alias
# if it doesn't pad it with "--"
for (( i=$ITEM_COUNT; $i <= 4; i++ ))
do
MPATH_ENTRIES[$index]="${MPATH_ENTRIES[index]} --"
done
unset FILLER
# set the key to the WWID and search the WWID_MAP to see if one already exists
key=`echo ${MPATH_ENTRIES[$index]} | awk '{print $2}'`
RETVAL=`search_wwid $key`
if [[ $RETVAL -lt 0 ]]
then
# this is a new WWID
# check to see what node ordinal this is (first, second, third, etc...)
# if this isn't the first node we will have to fill in -- for the ones before to indicate this WWID doesn't exist on previous nodes
NODE_POSITION=`search_node $1`
for (( i=0; $i < $NODE_POSITION; i++ ))
do
FILLER="$FILLER --"
done
# tack on a new element with the wwid, and filler required, and then the node alias
WWID_MAP[${#WWID_MAP[@]}]=`echo ${MPATH_ENTRIES[index]} | awk '{ print $2, filler, $4 }' filler="$FILLER"`
else
# a WWID already exists, just add on
(( NODE_POSITION=`search_node $1` + 1 ))
MAP_LENGTH=`echo ${WWID_MAP[RETVAL]} | wc -w`
# the node position will be greater than the length if we have a hole... plug it with filler
for (( i=$MAP_LENGTH; $i < $NODE_POSITION; i++ ))
do
FILLER="$FILLER --"
done
# append any required filler and then the alias for this node
WWID_MAP[$RETVAL]=${WWID_MAP[$RETVAL]}\ $FILLER\ `echo ${MPATH_ENTRIES[$index]} | awk '{print $4}'`
fi
done
}
######
### function check_status
### -ensures each WWID_MAP is the correct length by first finding the longest entry
### and then padding all others to that length with --
### -creates a WWID_STATUS array, each element aligns with a WWID_MAP element
### it checks if all aliases in WWID_MAP are the same, if so marks a green OK, if not a red INVALID
### WWID_STATUS array will have one entry for each WWID_MAP
### OK
### OK
### INVALID
### etc
######
function check_status {
# find the longest element in WWID_MAP
LONGEST=0
for index in ${!WWID_MAP[@]}
do
LENGTH=`echo ${WWID_MAP[index]} | wc -w`
if [[ $LENGTH > $LONGEST ]]
then
LONGEST=$LENGTH
fi
done
for index in ${!WWID_MAP[@]}
do
# count the number of items in this element
COUNT=`echo ${WWID_MAP[index]} | wc -w`
# another way to do this is convert to an array and then count the array elements
# ARRAY=( $(echo ${WWID_MAP[index]}) )
# for i in `seq ${#ARRAY[@]} $(( $LONGEST - 1 ))` or for i in `seq 2 $(( ${#ARRAY[@]} - 1))` when math rquired
# if the array is shorter, pad it
# longest is reduced by 1 because arrays are 0 based
for i in `seq $COUNT $(( $LONGEST - 1 ))`
do
WWID_MAP[$index]="${WWID_MAP[$index]} --"
done
# recount the element as its size may have just changed
COUNT=`echo ${WWID_MAP[index]} | wc -w`
if [ $COUNT -eq 2 ]
then
WWID_STATUS[$index]="\e[0;32mOK\e[0;30m"
fi
# for all items starting at 2 (the third item) to the end - 1 (zero based)
# compare with the item previous (i.e. 3:2, 4:3, 5:4, etc)
for i in `seq 2 $(( $COUNT - 1))`
do
ARRAY=( $(echo ${WWID_MAP[index]}) )
# if they don't match, or this item is "--" mark it as invalid
if [[ "${ARRAY[i]}" != "${ARRAY[i-1]}" ]] || [[ "${ARRAY[i]}" = "--" ]]
then
WWID_STATUS[$index]="\e[0;31mINVALID\e[0;30m"
break
else
WWID_STATUS[$index]="\e[0;32mOK\e[0;30m"
fi
done
done
}
######
### function exchange
### helper for bubble sort to swap both WWID_MAP and WWID_STATUS entries
######
function exchange {
local temp=${WWID_MAP[$1]}
local status_temp=${WWID_STATUS[$1]}
WWID_MAP[$1]=${WWID_MAP[$2]}
WWID_MAP[$2]=$temp
WWID_STATUS[$1]=${WWID_STATUS[$2]}
WWID_STATUS[$2]=$status_temp
}
######
### function sort_results
### a bubble sort of both WWID_MAP and WWID_STATUS based on a user specified sort field (SORT_FIELD)
### if SORT_FIELD = a digit, uses awk to compare values for that column
### if SORT_FIELD = valid | invalid simply compares the WWID_STATUS array elements to be either > or <
######
function sort_results {
number_of_elements=${#WWID_MAP[@]}
(( comparisons = $number_of_elements - 1 ))
count=1
while [ "$comparisons" -gt 0 ]
do
index=0
while [ "$index" -lt "$comparisons" ]
do
if [[ $SORT_FIELD = [[:digit:]]* ]]
then
if [[ `echo ${WWID_MAP[index]} | awk '{print $i}' i=$SORT_FIELD` > `echo ${WWID_MAP[ (( $index + 1 ))]} | awk '{print $i}' i=$SORT_FIELD` ]]
then
exchange $index `expr $index + 1`
fi
elif [[ $SORT_FIELD = "invalid" ]]
then
if [[ ${WWID_STATUS[index]} > ${WWID_STATUS[`expr $index + 1`]} ]]
then
exchange $index `expr $index + 1`
fi
elif [[ $SORT_FIELD = "valid" ]]
then
if [[ ${WWID_STATUS[index]} < ${WWID_STATUS[`expr $index + 1`]} ]]
then
exchange $index `expr $index + 1`
fi
fi
(( index += 1 ))
done
(( comparisons -= 1 ))
(( count += 1 ))
done
}
######
### function echo_results
### outputs the collected results in WWID_MAP and WWID_STATUS to the screen
######
function echo_results {
# put together a header row
# WWID Status
RESULT_SET="WWID"
for node in $NODE_LIST
do
RESULT_SET="$RESULT_SET $node"
done
RESULT_SET="$RESULT_SET Status"
# print the header row, wwid (1st column) is 40 characters wide, everything else is 30
echo -e $RESULT_SET | awk '{ printf "%-40s", $1 }'
echo -e $RESULT_SET | awk '{ for (i=2; i<=NF; i++) printf "%-30s", $i }'
awk 'BEGIN {printf "\n"}'
echo "---------------------"
# for each element print the first at 40 characters, all others (colume 2 - NF) at 30, then the status (using echo so the colours work)
# NF is Number of Fields
for index in ${!WWID_MAP[@]}
do
echo ${WWID_MAP[index]} | awk '{ printf "%-40s", $1 }'
echo ${WWID_MAP[index]} | awk '{ for (i=2; i<=NF; i++) printf "%-30s", $i }'
echo -e "${WWID_STATUS[index]}"
done
}
######
### function usage
### outputs help message
######
function usage {
echo -e "Usage: `basename $0` -n {node1,node2,node3,etc} [-sort {field} | -sort_invalid | -sort_valid] [-user {username}]"
echo -e "\t-n is a node list separated by commas, the script will ssh and read /etc/multipath.conf on each"
echo -e "\t-sort {field} sorts the output. 0 sorts the WWID number, 1..x sorts for a specific node"
echo -e "\t-sort_invalid places invalid alias entries at the top"
echo -e "\t-sort_valid places valid alias entries at the top"
echo -e "\t-user {username} allows a different username to be used than the one currently logged in"
exit
}
######
### parse the command line for options
######
if [ -z "$1" ]
then
usage
fi
until [ -z "$1" ]
do
case "$1" in
-n)
shift
NODE_LIST=`echo $1 | tr "," " "`
;;
-sort)
shift
SORT_FIELD=`expr $1 + 1`
;;
-sort_invalid)
SORT_FIELD="invalid"
;;
-sort_valid)
SORT_FIELD="valid"
;;
-user)
shift
USERNAME="-l $1"
;;
*)
usage
;;
esac
shift
done
# convert NODE_LIST to an array so we can search the position
NODE_ARRAY=( $(echo "$NODE_LIST") )
# for each node in the list
# - grab its multipath.conf (ssh)
# - parse out wwid and alias pairs (parse_multipath)
# - add it to the master WWID list (assign_wwid)
# then fill in any blank holes and the overall status (check_status)
# sort the results as requested (sort_results)
# then output to display (echo_results)
for node in $NODE_LIST
do
NODE_MPATH_RESULT=`ssh $node $USERNAME -C "cat /etc/multipath.conf" 2> /dev/null`
RETVAL=$?
if [ $RETVAL != 0 ]
then
echo "--- Error retrieving /etc/multipath.conf from node $node ---"
fi
parse_multipath $node
assign_wwid $node
done
check_status
sort_results
echo_results
Showing posts with label multipath. Show all posts
Showing posts with label multipath. Show all posts
Sunday, January 9, 2011
Cross Server Multipath
I was asked to come up with a method to compare multipath devices across an Oracle RAC cluster to ensure they are consistent. Beyond this there was one catch; it had to be run as a regular user. I would have normally run a multipath command but unfortunately that isn't available without elevated privileges. So my solution looks at /etc/multipath.conf, which is readable by a user, compare the entries across all nodes, and output a table at the end. You can see an example of the output in the initial comments.
Tuesday, May 18, 2010
Dynamic Linux Disk
To start with, I am going to assume udev and multipath are setup as per my last post. Udev isn't required for scanning or device naming but it is responsible for permissions and device location (directory). Device naming is actually controlled by the multipath driver, which in a modern Linux distribution is conveniently included in the kernel.
The second assumption is that multipath has the basic setup for your particular storage frame. Now, lets ensure multipath is running and set to start on every boot:
As in the last post, I am dealing with RedHat 5.4 and an EMC CLARiiON array. Without any LUNs allocated multipath -ll should look something like this:
These entries are the four paths available to CX controllers.
Adding Devices
No Existing Devices
I have so far been unable to use the simple scan method on an HBA without any devices at all, so this process will unload and reload the adapter driver. It's a disruptive process on the fibre channel bus but there aren't any devices anyway, so it shouldn't matter.
First find the driver you are using, it is likely either an Emulex (lpfc) or Qlogic (qla). In this example I am using an Emulex card.
Instead of modprobe, you can also use insmod. The difference being insmod will only load the specified driver and modprobe will load the driver and any dependent drivers.
This will allow the device(s) to show under /dev/ora_rdsk but won't create any multipath entries. To do that we simply run multipath.
Existing Devices
If you have at least one Fibre device existing, you can simply rescan the bus. This will not take down the existing devices and is able to operate one path at a time ensuring I/O can continue to flow. You will need to know which host devices are your fibre HBAs. To find that, we can list the know fibre adapters as follows and then issue a scan for each.
Renaming Devices
The newly scanned device has a WWID name which isn't terribly useful for something like Oracle as we want udev to apply appropriate permissions. To do this, cut and paste the ID into /etc/multipath.conf so it looks something like this:
And then remove the old device name and re-import it into multipath
If you get an error "must provide a map name to remove" when running multipath -f, make sure you don't have a shell inside /dev/ora_rdsk directory. Also, be careful not to use multipath -F as that will remove all devices, probably not what you want.
Removing Devices
Before doing the actual removal you will need to note several pieces of information; the multipath device name and all block devices assigned to it. All of which can be obtained from multipath -ll.
The second assumption is that multipath has the basic setup for your particular storage frame. Now, lets ensure multipath is running and set to start on every boot:
# service multipathd status
multipathd is stopped
# chkconfig --list multipathd
multipathd 0:off 1:off 2:off 3:off 4:off 5:off 6:off
# chkconfig multipathd on
# chkconfig --list multipathd
multipathd 0:off 1:off 2:off 3:on 4:off 5:on 6:off
# service multipathd start
As in the last post, I am dealing with RedHat 5.4 and an EMC CLARiiON array. Without any LUNs allocated multipath -ll should look something like this:
# multipath -ll
sdb: checker msg is "emc_clariion_checker: Logical Unit is umbound or LUNZ"
sdc: checker msg is "emc_clariion_checker: Logical Unit is umbound or LUNZ"
sdd: checker msg is "emc_clariion_checker: Logical Unit is umbound or LUNZ"
sde: checker msg is "emc_clariion_checker: Logical Unit is umbound or LUNZ"
These entries are the four paths available to CX controllers.
Adding Devices
No Existing Devices
I have so far been unable to use the simple scan method on an HBA without any devices at all, so this process will unload and reload the adapter driver. It's a disruptive process on the fibre channel bus but there aren't any devices anyway, so it shouldn't matter.
First find the driver you are using, it is likely either an Emulex (lpfc) or Qlogic (qla). In this example I am using an Emulex card.
# lsmod | grep lpfc
lpfc 352909 0
scsi_transport_fc 73801 1 lpfc
scsi_mod 196569 10 scsi_dh,sr_mod,sg,usb_storage,lpfc,scsi_transport_fc,mptsas,mptscsih,scsi_transport_sas,sd_mod
remove the module
# rmmod lpfc
insert the module
# modprobe lpfc
Instead of modprobe, you can also use insmod. The difference being insmod will only load the specified driver and modprobe will load the driver and any dependent drivers.
This will allow the device(s) to show under /dev/ora_rdsk but won't create any multipath entries. To do that we simply run multipath.
# multipath
reload: 36006016015a01900796464949a36df11 DGC,RAID 5
[size=50G][features=1 queue_if_no_path|features=1
queue_if_no_path][hwhandler=1 emc][n/a]
\_ round-robin 0 [prio=2][undef]
\_ 4:0:1:0 sdc 8:32 [active][ready]
\_ 5:0:1:0 sde 8:64 [undef][ready]
\_ round-robin 0 [prio=0][undef]
\_ 4:0:0:0 sdb 8:16 [undef][ready]
\_ 5:0:0:0 sdd 8:48 [undef][ready]
Existing Devices
If you have at least one Fibre device existing, you can simply rescan the bus. This will not take down the existing devices and is able to operate one path at a time ensuring I/O can continue to flow. You will need to know which host devices are your fibre HBAs. To find that, we can list the know fibre adapters as follows and then issue a scan for each.
# ls -l /sys/class/fc_host
drwxr-xr-x 3 root root 0 Apr 17 08:50 host4
drwxr-xr-x 3 root root 0 Apr 17 08:50 host5
# echo "- - -" > /sys/class/scsi_host/host4/scan
# multipath -ll
36006016015a01900e2acd0d4a549df11 dm-3 DGC,RAID 5
[size=8.0G][features=1 queue_if_no_path|features=1
queue_if_no_path][hwhandler=1 emc][rw]
\_ round-robin 0 [prio=1][active]
\_ 1:0:0:1 sdh 8:112 [active][ready]
\_ round-robin 0 [prio=0][enabled]
\_ 1:0:1:1 sdi 8:128 [active][ready]
# echo "- - -" > /sys/class/scsi_host/host5/scan
# multipath -ll
36006016015a01900e2acd0d4a549df11 dm-3 DGC,RAID 5
[size=8.0G][features=1 queue_if_no_path|features=1
queue_if_no_path][hwhandler=1 emc][rw]
\_ round-robin 0 [prio=2][enabled]
\_ 1:0:0:1 sdh 8:112 [active][ready]
\_ 2:0:0:1 sdj 8:144 [active][ready]
\_ round-robin 0 [prio=0][enabled]
\_ 1:0:1:1 sdi 8:128 [active][ready]
\_ 2:0:1:1 sdk 8:160 [active][ready]
# ls -lL /dev/ora_rdsk
brw-rw---- 1 root root 253, 3 Apr 17 15:57 36006016015a01900e2acd0d4a549df11
Renaming Devices
The newly scanned device has a WWID name which isn't terribly useful for something like Oracle as we want udev to apply appropriate permissions. To do this, cut and paste the ID into /etc/multipath.conf so it looks something like this:
multipath {
wwid 36006016015a01900796464949a36df11
alias ora_test
}
And then remove the old device name and re-import it into multipath
# multipath -f 36006016015a01900796464949a36df11
# multipath
create: ora_test (36006016015a01900796464949a36df11) DGC,RAID 5
[size=50G][features=1 queue_if_no_path|features=1
queue_if_no_path][hwhandler=1 emc][n/a]
\_ round-robin 0 [prio=2][undef]
\_ 1:0:1:0 sdc 8:32 [undef][ready]
\_ 2:0:1:0 sde 8:64 [undef][ready]
\_ round-robin 0 [prio=0][undef]
\_ 1:0:0:0 sdb 8:16 [undef][ready]
\_ 2:0:0:0 sdd 8:48 [undef][ready]
# ls \-lL /dev/ora_rdsk
brw-rw---- 1 oracle dba 253, 2 Apr 17 09:44 ora_test
If you get an error "must provide a map name to remove" when running multipath -f, make sure you don't have a shell inside /dev/ora_rdsk directory. Also, be careful not to use multipath -F as that will remove all devices, probably not what you want.
Removing Devices
Before doing the actual removal you will need to note several pieces of information; the multipath device name and all block devices assigned to it. All of which can be obtained from multipath -ll.
# multipath -ll
*ora_test2* (36006016015a01900e2acd0d4a549df11) dm-3 DGC,RAID 5
[size=8.0G][features=1 queue_if_no_path|features=1
queue_if_no_path][hwhandler=1 emc][rw]
\_ round-robin 0 [prio=2][active]
\_ 1:0:0:1 *sdh* 8:112 [active][ready]
\_ 2:0:0:1 *sdj* 8:144 [active][ready]
\_ round-robin 0 [prio=0][enabled]
\_ 1:0:1:1 *sdi* 8:128 [active][ready]
\_ 2:0:1:1 *sdk* 8:160 [active][ready]
To remove the multipath device
# multipath -f ora_test2
Then remove the appropriate block devices from the system with 'echo 1 >
/sys/block/*dev*/device/delete'
# echo 1 > /sys/block/sdh/device/delete
# echo 1 > /sys/block/sdj/device/delete
# echo 1 > /sys/block/sdi/device/delete
# echo 1 > /sys/block/sdk/device/delete
Sunday, May 9, 2010
Linux udev and multipath
While technically /dev is always under the control of udev, as system administrators we rarely need to think about it let alone control it. However, the best use case I have come up with is when configuring storage for Oracle RAC. Oracle requires specific permissions on the devices, you can either use ASMlib (which I think a lot of companies do) or you can use udev. Obviously I like the udev approach as it eliminates yet another vendor supplied software package that needs to be maintained. Being built into Linux means compatibility and updates are handled just like any other OS patch.
A similar argument can be made with multipath (device mapper) although use cases for this spread far beyond just Oracle. Many vendors tout the use of their own multipath software, and while they can have some extra features, I am of the opinion that 99% of the what is required today is available in this free and integrated solution.
Multipath Setup
First off, make sure your storage array is supported by multipath. From my knowledge, all of the major vendors are, but you may have one that isn't on the list. My setup is RedHat 5.4 with an EMC CX array which is convenient as multipath has integrated support for the CLARiiON line. The key section to note from your vendor is any special device entry under /etc/multipath.conf they may need.
Next is to ensure multipath is installed. For RedHat this is in the package device-mapper-multipath, for SuSE it is multipath-tools. Do a package search for multipath and you should find it.
Configuration File
device for you. I hate these. They are shorter than a WWID but they aren't guaranteed to be consistent across reboots and certainly not across multiple servers. So by specifying “no” you will end up with a device named after the World Wide Identifier (WWID) of the SAN target.
The multipath entries take this WWID and turns it into any name specified by the alias entry. Because we want to change the device permissions, a specific name or prefix is used. In this case I have chosen “ora_” but you could use whatever you like, or multiple prefixes for different functions. For example you could have a vote_ or an ocr_ prefix using different permissions although those disks aren't really required anymore with 11gR2.
You can get WWIDs from the storage array or simply let Linux tell you when it scans them, we'll cover that in a later post.
Optional Configuration
Now if you have a different storage array you may also have to add the vendor specified devices entry, perhaps something like this:
This is only an example, check with your vendor. If they support multipath, they have documentation with the correct entry for the current features and models.
You can also include a blacklist entry for your local devices but I haven't found it to be required. However, if you like an entry such as this would do:
Multipath Daemon
The last step is to get the daemon running. Its job is to reconfigure paths when something breaks or when a link is put back in service.
Under RedHat / SuSE this is relatively simple to do:
You can validate this with:
And finally startup the daemon immediately with:
Udev Configuration
Unfortunately distributions can vary greatly in how their udev rules are laid out, so here is the configuration for RedHat, your mileage may vary. I am going to store my new configuration as part of the multipath rules under /etc/udev/rules.d/40-multipath.rules. The changes from the default are highlighted:
The ora_rdsk entry is a way of keeping the oracle disk (multipath disk) in a unique location. The default is mpath, its up to you. This means that all of my multipath devices will show up under /dev/ora_rdsk.
GOTO=”end_mpath” is an entry which acts as a catch for devices that aren't specifically sent to update_oracle_devs which of course follows. This is where the magic happens, it basically does a match based on a certain prefix assigns an owner, a group, and permissions. You can get quite creative here if you like with different regular expressions or device entries for different functions.
Strictly speaking you don't require partitions to use ASM under Oracle RAC, however, if you would like to, they will show up as your designated alias plus a p. For example ora_data1p1, ora_data1p2, etc. I generally don't use them but if you want to change this it is controlled earlier in the file when kpartx is run.
Putting it All Together
Just allocate your disk, put the entries into /dev/multipath.conf and you are off to the races. Your devices should all magically show up in /dev/ora_rdsk/. Next time I'll cover off how to add and remove devices dynamically so you don't have to know your WWIDs up front or do any nasty rebooting.
A similar argument can be made with multipath (device mapper) although use cases for this spread far beyond just Oracle. Many vendors tout the use of their own multipath software, and while they can have some extra features, I am of the opinion that 99% of the what is required today is available in this free and integrated solution.
Multipath Setup
First off, make sure your storage array is supported by multipath. From my knowledge, all of the major vendors are, but you may have one that isn't on the list. My setup is RedHat 5.4 with an EMC CX array which is convenient as multipath has integrated support for the CLARiiON line. The key section to note from your vendor is any special device entry under /etc/multipath.conf they may need.
Next is to ensure multipath is installed. For RedHat this is in the package device-mapper-multipath, for SuSE it is multipath-tools. Do a package search for multipath and you should find it.
Configuration File
defaults {
user_friendly_names no
}
multipaths {
multipath {
wwid wwid (e.g. 36006016015a0190038585d690b53df11)
alias ora_data1
}
multipath {
wwid wwid (e.g. 36006016015a0190038585d690b53df12)
alias ora_data2
}
}
The multipath entries take this WWID and turns it into any name specified by the alias entry. Because we want to change the device permissions, a specific name or prefix is used. In this case I have chosen “ora_” but you could use whatever you like, or multiple prefixes for different functions. For example you could have a vote_ or an ocr_ prefix using different permissions although those disks aren't really required anymore with 11gR2.
You can get WWIDs from the storage array or simply let Linux tell you when it scans them, we'll cover that in a later post.
Optional Configuration
Now if you have a different storage array you may also have to add the vendor specified devices entry, perhaps something like this:
devices {
device {
vendor “NETAPP”
product “LUN”
path_group_policy multibus
getuid_callout “/sbin/scsi_id -g -u -s /block/%n”
prio_callout “/sbin/mpath_prio_ontap /dev/%n”
features "1 queue_if_no_path"
path_checker directio
failback immediate
flush_on_last_del yes
}
}
This is only an example, check with your vendor. If they support multipath, they have documentation with the correct entry for the current features and models.
You can also include a blacklist entry for your local devices but I haven't found it to be required. However, if you like an entry such as this would do:
blacklist
{
devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
devnode "^hd[a-z]"
devnode "^cciss!c[0-9]d[0-9]*[p[0-9]*]"
}
Multipath Daemon
The last step is to get the daemon running. Its job is to reconfigure paths when something breaks or when a link is put back in service.
Under RedHat / SuSE this is relatively simple to do:
chkconfig multipathd on
You can validate this with:
chkconfig --list multipathd
And finally startup the daemon immediately with:
service multipathd start
- or -
/etc/init.d/multipathd start
Udev Configuration
Unfortunately distributions can vary greatly in how their udev rules are laid out, so here is the configuration for RedHat, your mileage may vary. I am going to store my new configuration as part of the multipath rules under /etc/udev/rules.d/40-multipath.rules. The changes from the default are highlighted:
# multipath wants the devmaps presented as meaninglful device names
# so name them after their devmap name
SUBSYSTEM!="block", GOTO="end_mpath"
KERNEL!="dm-[0-9]*", ACTION=="add", PROGRAM=="/bin/bash -c '/sbin/lsmod | /bin/grep ^dm_multipath'", RUN+="/sbin/multipath -v0 %M:%m"
KERNEL!="dm-[0-9]*", GOTO="end_mpath"
PROGRAM!="/sbin/mpath_wait %M %m", GOTO="end_mpath"
ACTION=="add", RUN+="/sbin/dmsetup ls --target multipath --exec '/sbin/kpartx -a -p p' -j %M -m %m"
PROGRAM=="/sbin/dmsetup ls --target multipath --exec /bin/basename -j %M -m %m", RESULT=="?*", NAME="%k", SYMLINK="ora_rdsk/%c", GOTO="update_oracle_devs"
PROGRAM!="/bin/bash -c '/sbin/dmsetup info -c --noheadings -j %M -m %m | /bin/grep -q .*:.*:.*:.*:.*:.*:.*:part[0-9]*-mpath-'", GOTO="end_mpath"
PROGRAM=="/sbin/dmsetup ls --target linear --exec /bin/basename -j %M -m %m", NAME="%k", RESULT=="?*", SYMLINK="ora_rdsk/%c", GOTO="update_oracle_devs"
GOTO="end_mpath"
LABEL="update_oracle_devs"
RESULT=="vote*",OWNER="oracle",GROUP="oinstall",MODE="644"
RESULT=="ocr*",GROUP="oinstall",MODE="640"
RESULT=="ora*",OWNER="oracle",GROUP="dba",MODE="660"
OPTIONS="last_rule"
LABEL="end_mpath"
The ora_rdsk entry is a way of keeping the oracle disk (multipath disk) in a unique location. The default is mpath, its up to you. This means that all of my multipath devices will show up under /dev/ora_rdsk.
GOTO=”end_mpath” is an entry which acts as a catch for devices that aren't specifically sent to update_oracle_devs which of course follows. This is where the magic happens, it basically does a match based on a certain prefix assigns an owner, a group, and permissions. You can get quite creative here if you like with different regular expressions or device entries for different functions.
Strictly speaking you don't require partitions to use ASM under Oracle RAC, however, if you would like to, they will show up as your designated alias plus a p
Putting it All Together
Just allocate your disk, put the entries into /dev/multipath.conf and you are off to the races. Your devices should all magically show up in /dev/ora_rdsk/. Next time I'll cover off how to add and remove devices dynamically so you don't have to know your WWIDs up front or do any nasty rebooting.
Subscribe to:
Posts (Atom)