Viewing: lustre_start.in
#!/usr/bin/bash
# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
#
# lustre_start - start or stop multiple Lustre servers from a CSV file
#
# This script is used to parse each line of a CSV (Comma-Separated Value) file
# and execute remote command to start/stop the service on every Lustre server
# target that will be part of the Lustre cluster.
#
################################################################################
# Usage
usage() {
cat <<EOF
Usage: $(basename $0) [options] <-a|-w|-x> <CSV file>
This script is used to start or stop multiple Lustre servers from a
CSV file.
Options:
-a select all the nodes from the CSV file to operate on
-w hostname,hostname,...
select the specified list of nodes (separated by commas) to
operate on rather than all the nodes in the CSV file
-x hostname,hostname,...
exclude the specified list of nodes (separated by commas)
-n no net - don't verify network connectivity and hostnames
in the cluster
-m pass "mount options" item in the CSV file to mount command line
-k stop the services on Lustre server targets
-v verbose mode
-h help
CSV file a comma-separated value file that contains configuration
parameters for each target in a Lustre cluster
Please refer to "lustre_config -h" for the description of CSV file formats.
EOF
}
# Get the library of functions
. @scriptlibdir@/lc_common
SPECIFY_MNTOPTS=false
STOP_SERVICE=false
# Get and check the positional parameters
while getopts "aw:x:nmkhv" OPTION; do
case $OPTION in
a)
[ -z "$SPECIFIED_NODELIST" ] && [ -z "$EXCLUDED_NODELIST" ] \
&& USE_ALLNODES=true
NODELIST_OPT="$NODELIST_OPT -a"
;;
w)
USE_ALLNODES=false
SPECIFIED_NODELIST=$OPTARG
NODELIST_OPT="$NODELIST_OPT -w $SPECIFIED_NODELIST"
;;
x)
USE_ALLNODES=false
EXCLUDED_NODELIST=$OPTARG
NODELIST_OPT="$NODELIST_OPT -x $EXCLUDED_NODELIST"
;;
n)
VERIFY_CONNECT=false
;;
m)
SPECIFY_MNTOPTS=true
;;
k)
STOP_SERVICE=true
;;
h)
usage
exit 0
;;
v)
VERBOSE_OPT="-v"
VERBOSE_OUTPUT=true
;;
?)
usage 1>&2
exit 1
;;
esac
done
# Toss out the parameters we've already processed
shift $((OPTIND - 1))
# Here we expect the CSV file
if [ $# -eq 0 ]; then
error_output "Missing CSV file!"
usage 1>&2
exit 1
fi
CSV_FILE=$1
# get_fstab_mntopts host_name device_name mount_point
# Get the mount options from the /etc/fstab file
get_fstab_mntopts() {
local host_name=$1
local dev_name=$2
local mnt_pnt=$3
local mnt_opts=""
if [ -z "$host_name" -o -z "$dev_name" -o -z "$mnt_pnt" ]; then
echo "get_fstab_mntopts(): Missing argument!"
return 1
fi
# Execute remote command to get the mount options from /etc/fstab file
mnt_opts=$($REMOTE $host_name "grep -w ^$dev_name /etc/fstab | \
grep -w $mnt_pnt | awk '{print \$4}'" 2>/dev/null)
mnt_opts=${mnt_opts//$host_name: /}
echo $mnt_opts
return 0
}
# Start the service on one Lustre server target
start_service() {
declare -i i=$1
shift
local extra_mntopts="$*"
local mntopts=""
# Get mount options
if $SPECIFY_MNTOPTS; then
# Use the "mount options" item from the CSV file
[ -n "${MOUNT_OPTIONS[i]}" ] && mntopts=${MOUNT_OPTIONS[i]}
else
# Do not use the "mount options" item from the CSV file
mntopts=$(get_fstab_mntopts ${HOST_NAME[i]} ${DEVICE_NAME[i]} \
${MOUNT_POINT[i]})
[ ${PIPESTATUS[0]} -ne 0 ] && error_output "$mntopts" && return 1
fi
[ -n "$mntopts" ] && mntopts="-o $mntopts"
[ -n "$extra_mntopts" ] && mntopts="$mntopts $extra_mntopts"
# Strip of any leading space
mntopts=${mntopts# }
# Execute remote command to start the service
verbose_output "Mounting Lustre ${DEVICE_TYPE[i]} target"\
"${DEVICE_NAME[i]} (opts: $mntopts) on ${HOST_NAME[i]}:${MOUNT_POINT[i]}..."
$REMOTE ${HOST_NAME[i]} "PATH=\$PATH:/sbin:/usr/sbin
error() { set +x; echo \"ERROR: \$2: \$1\"; echo \"XXRETCODE:\$1\"; exit \$1; }
mkdir -p ${MOUNT_POINT[i]} || \\
error \${PIPESTATUS[0]} \"failed to mkdir ${MOUNT_POINT[i]}\"
mount -t $FS_TYPE $mntopts ${DEVICE_NAME[i]} ${MOUNT_POINT[i]} || \\
error \${PIPESTATUS[0]} \\
\"failed to mount ${DEVICE_NAME[i]} on host ${HOST_NAME[i]}\""
return ${PIPESTATUS[0]}
}
# Stop the service on one Lustre server target
stop_service() {
declare -i i=$1
# Execute remote command to stop the service
verbose_output "Unmounting Lustre ${DEVICE_TYPE[i]} target"\
"${DEVICE_NAME[i]} on ${HOST_NAME[i]}:${MOUNT_POINT[i]}..."
$REMOTE ${HOST_NAME[i]} "PATH=\$PATH:/sbin:/usr/sbin
error() { set +x; echo \"ERROR: \$2: \$1\"; echo \"XXRETCODE:\$1\"; exit \$1; }
if grep -q \" ${MOUNT_POINT[i]} \" /proc/mounts; then
umount -d -f ${MOUNT_POINT[i]} || \\
error \${PIPESTATUS[0]} \\
\"failed to unmount ${DEVICE_NAME[i]} on host ${HOST_NAME[i]}\"
else
echo \"${DEVICE_NAME[i]} was not mounted on\"\\
\"${HOST_NAME[i]}:${MOUNT_POINT[i]}\"
fi"
return ${PIPESTATUS[0]}
}
# mass_op op_type target_type
# Start/stop the services on Lustre server targets in parallel
mass_op() {
local op_type=$1
local target_type=$2
local op_func
declare -i i
declare -i pid_num=0
declare -a REMOTE_PID
local RC=0
if [ -z "$op_type" -o -z "$target_type" ]; then
error_output "mass_op(): Missing argument!"
return 1
fi
case "$op_type" in
"start") op_func=start_service;;
"stop") op_func=stop_service;;
*) error_output "mass_op(): Invalid op type \"$op_type\"!" && return 1;;
esac
for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
if [ "${DEVICE_TYPE[i]}" = "$target_type" ] \
&& [[ "${FORMAT_OPTIONS[i]}" != *noformat* ]]; then
eval "$op_func $i &"
REMOTE_PID[$pid_num]=$!
let pid_num=$pid_num+1
fi
done
for ((pid_num = 0; pid_num < ${#REMOTE_PID[@]}; pid_num++)); do
wait ${REMOTE_PID[${pid_num}]}
local RC1=${PIPESTATUS[0]}
[ $RC1 -ne 0 ] && RC=$RC1
done
[ $RC -ne 0 ] && return $RC
return 0
}
# Unload the modules to make cleanup
unload_modules() {
local command
local host
local host_list
host_list=$(comma_list "${HOST_NAME[@]}")
[ -z "$host_list" ] && return 0
command="PATH=\$PATH:/sbin:/usr/sbin
if grep -q libcfs /proc/modules; then
lctl net down 1>/dev/null 2>&1
lustre_rmmod
fi"
if is_pdsh; then
$REMOTE $host_list "$command"
else
for host in ${host_list//,/ }; do
$REMOTE $host "$command"
done
fi
}
# Start the services on Lustre server targets
mass_start() {
declare -i i
local combo_mgs_mdt=false
if [ ${#HOST_NAME[@]} -eq 0 ]; then
verbose_output "There are no Lustre targets specified."
return 0
fi
# Start lnet network on the MGS node
start_mgs_lnet || return ${PIPESTATUS[0]}
local checked_hosts=""
for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
host_in_hostlist ${HOST_NAME[i]} $checked_hosts && continue
if ! is_mgs_node ${HOST_NAME[i]}; then
# Add module options to the module configuration file
add_module_options $i ${HOST_NAME[i]} || return ${PIPESTATUS[0]}
# Check lnet networks
check_lnet $i || return ${PIPESTATUS[0]}
checked_hosts="$checked_hosts,${HOST_NAME[i]}"
fi
done
# Start MGS or the MGS service on combo MGS/MDT (with "-o nosvc -n" options)
if [ -n "${MGS_NODENAME[0]}" ]; then
local idx=${MGS_IDX[0]}
if [ "${DEVICE_TYPE[idx]#*mdt*}" != "${DEVICE_TYPE[idx]}" ]; then
# Combo MGS/MDT
combo_mgs_mdt=true
start_service ${MGS_IDX[0]} "-o nosvc -n" || return ${PIPESTATUS[0]}
else
start_service ${MGS_IDX[0]} || return ${PIPESTATUS[0]}
fi
fi
# Start OST(s)
mass_op "start" "ost" || return ${PIPESTATUS[0]}
# Start the MDT service on combo MGS/MDT (with "-o nomgs" option)
if $combo_mgs_mdt; then
start_service ${MGS_IDX[0]} "-o nomgs" || return ${PIPESTATUS[0]}
fi
# Start MDT(s)
mass_op "start" "mdt" || return ${PIPESTATUS[0]}
verbose_output "Success on all Lustre targets!"
return 0
}
# Stop the services on Lustre server targets
mass_stop() {
declare -i i
if [ ${#HOST_NAME[@]} -eq 0 ]; then
verbose_output "There are no Lustre targets specified."
return 0
fi
# Stop MDT(s)
mass_op "stop" "mdt" || return ${PIPESTATUS[0]}
# Stop the MDT service on combo MGS/MDT
if [ -n "${MGS_NODENAME[0]}" ]; then
local idx=${MGS_IDX[0]}
if [ "${DEVICE_TYPE[idx]#*mdt*}" != "${DEVICE_TYPE[idx]}" ]; then
# Combo MGS/MDT
stop_service ${MGS_IDX[0]} || return ${PIPESTATUS[0]}
fi
fi
# Stop OST(s)
mass_op "stop" "ost" || return ${PIPESTATUS[0]}
# Stop MGS or the MGS service on combo MGS/MDT
if [ -n "${MGS_NODENAME[0]}" ]; then
stop_service ${MGS_IDX[0]} || return ${PIPESTATUS[0]}
fi
unload_modules
return 0
}
#********************************* Main Flow **********************************#
# Check the CSV file
check_file $CSV_FILE || exit ${PIPESTATUS[0]}
# Get the list of nodes to be operated on
NODES_TO_USE=$(get_nodelist) || error_exit ${PIPESTATUS[0]} "$NODES_TO_USE"
# Check the node list
check_nodelist $NODES_TO_USE || exit ${PIPESTATUS[0]}
# Check the network connectivity and hostnames
if $VERIFY_CONNECT; then
verbose_output "Checking the cluster network connectivity and hostnames..."
$VERIFY_CLUSTER_NET $NODELIST_OPT $VERBOSE_OPT $CSV_FILE || \
exit ${PIPESTATUS[0]}
verbose_output "Check the cluster network connectivity and hostnames OK!"
fi
# Configure the Lustre cluster
echo "$(basename $0): ******** Lustre cluster configuration BEGIN ********"
get_lustre_items $CSV_FILE || exit ${PIPESTATUS[0]}
check_mgs || exit ${PIPESTATUS[0]}
if ! $STOP_SERVICE; then
mass_start || exit ${PIPESTATUS[0]}
else
mass_stop || exit ${PIPESTATUS[0]}
fi
echo "$(basename $0): ******** Lustre cluster configuration END **********"
exit 0
