Viewing: lfsck-performance.sh
#!/usr/bin/bash
set -e
ONLY=${ONLY:-"$*"}
LUSTRE=${LUSTRE:-$(dirname $0)/..}
. $LUSTRE/tests/test-framework.sh
init_test_env "$@"
init_logging
ALWAYS_EXCEPT="$LFSCK_PERFORMANCE_EXCEPT"
build_test_filter
require_dsh_mds || exit 0
require_dsh_ost || exit 0
[ "$SLOW" = "no" ] &&
skip "skip lfsck performance test under non-SLOW mode"
NTHREADS=${NTHREADS:-0}
UNIT=${UNIT:-8192}
MINCOUNT=${MINCOUNT:-4096}
MAXCOUNT=${MAXCOUNT:-8192}
MINCOUNT_REPAIR=${MINCOUNT_REPAIR:-4096}
MAXCOUNT_REPAIR=${MAXCOUNT_REPAIR:-8192}
BASE_COUNT=${BASE_COUNT:-8192}
FACTOR=${FACTOR:-2}
INCFACTOR=${INCFACTOR:-25} #percent
MINSUBDIR=${MINSUBDIR:-1}
MAXSUBDIR=${MAXSUBDIR:-2}
TOTSUBDIR=${TOTSUBDIR:-2}
WTIME=${WTIME:-86400}
RCMD="do_facet ${SINGLEMDS}"
RLCTL="${RCMD} ${LCTL}"
MDT_DEV="${FSNAME}-MDT0000"
MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
START_NAMESPACE="${RLCTL} lfsck_start -M ${MDT_DEV} -t namespace"
STOP_LFSCK="${RLCTL} lfsck_stop -M ${MDT_DEV} -A"
SHOW_NAMESPACE="${RLCTL} get_param -n mdd.${MDT_DEV}.lfsck_namespace"
MNTOPTS_NOSCRUB="-o user_xattr,noscrub"
remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"
if [ ${NTHREADS} -eq 0 ]; then
CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
NTHREADS=$((CPUCORE * 2))
fi
lfsck_attach() {
${RCMD} "modprobe obdecho"
${ECHOCMD} "${LCTL} <<-EOF
attach echo_client lfsck-MDT0000 lfsck-MDT0000_UUID
setup ${MDT_DEV} mdd
EOF"
}
lfsck_detach() {
${ECHOCMD} "${LCTL} <<-EOF
--device lfsck-MDT0000 cleanup
--device lfsck-MDT0000 detach
EOF"
}
lfsck_create() {
local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
local j
${ECHOCMD} "${LCTL} --device ${echodev} test_mkdir ${tdir}"
for ((j = 1; j < ${threads}; j++)); do
${ECHOCMD} "${LCTL} --device ${echodev} test_mkdir ${tdir}${j}"
done
${ECHOCMD} "${LCTL} --device ${echodev} \
--threads ${threads} 0 ${echodev} \
test_create -d${tdir} -D${threads} -b${lbase} -c0 -n${usize}"
}
lfsck_cleanup() {
do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
REFORMAT="yes" cleanup_and_setup_lustre
}
lfsck_create_nfiles() {
local total=$1
local lbase=$2
local threads=$3
local linkea=$4
local ldir="/test-${lbase}"
local cycle=0
local count=${UNIT}
while true; do
[ ${count} -eq 0 -o ${count} -gt ${total} ] && count=${total}
local usize=$((count / NTHREADS))
[ ${usize} -eq 0 ] && break
local tdir=${ldir}-${cycle}-
echo "[cycle: ${cycle}] [threads: ${threads}]"\
"[files: ${count}] [basedir: ${tdir}]"
start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB ||
error "Fail to start MDS!"
#define OBD_FAIL_FID_IGIF 0x1504
[ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x1504
lfsck_attach
lfsck_create
lfsck_detach
[ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x0
stop ${SINGLEMDS} || error "Fail to stop MDS!"
total=$((total - usize * NTHREADS))
[ ${total} -eq 0 ] && break
lbase=$((lbase + usize))
cycle=$((cycle + 1))
done
}
format_start_mgs () {
do_rpc_nodes $(facet_active_host mgs) load_modules_local
format_mgs
start mgs $(mgsdevname) $MGS_MOUNT_OPTS
}
test_0() {
local BCOUNT=0
local i
stopall
combined_mgs_mds || format_start_mgs
do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
format_mdt $(facet_number $SINGLEMDS)
for ((i = $MINCOUNT; i <= $MAXCOUNT; i = $((i * FACTOR)))); do
local nfiles=$((i - BCOUNT))
echo "+++ start to create for ${i} files set at: $(date) +++"
lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
error "Fail to create files!"
echo "+++ end to create for ${i} files set at: $(date) +++"
BCOUNT=${i}
start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
error "Fail to start MDS!"
echo "start lfsck_namespace for ${i} files set at: $(date)"
$START_NAMESPACE || error "Fail to start lfsck_namespace!"
while true; do
local STATUS=$($SHOW_NAMESPACE |
awk '/^status/ { print $2 }')
[ "$STATUS" == "completed" ] && break
sleep 3 # check status every 3 seconds
done
echo "end lfsck_namespace for ${i} files set at: $(date)"
SPEED=$($SHOW_NAMESPACE |
awk '/^average_speed_phase1/ { print $2 }')
echo "lfsck_namespace speed is ${SPEED}/sec"
stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
done
combined_mgs_mds || stop mgs
}
run_test 0 "lfsck namespace performance (routine case) without load"
test_1() {
[ "$mds1_FSTYPE" != ldiskfs ] &&
skip_env "not implemented for ZFS"
local BCOUNT=0
local i
stopall
combined_mgs_mds || format_start_mgs
do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
format_mdt $(facet_number $SINGLEMDS)
for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
i = $((i * FACTOR)))); do
local nfiles=$((i - BCOUNT))
echo "+++ start to create for ${i} files set at: $(date) +++"
lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
error "Fail to create files!"
echo "+++ end to create for ${i} files set at: $(date) +++"
BCOUNT=${i}
local stime=$(date +%s)
echo "backup/restore ${i} files start at: $(date)"
mds_backup_restore $SINGLEMDS || error "Fail to backup/restore!"
echo "backup/restore ${i} files end at: $(date)"
local etime=$(date +%s)
local delta=$((etime - stime))
[ $delta -gt 0 ] || delta=1
echo "backup/restore ${i} files used ${delta} seconds"
echo "backup/restore speed is $((i / delta))/sec"
start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
error "Fail to start MDS!"
echo "start lfsck_namespace for ${i} files set at: $(date)"
$START_NAMESPACE || error "Fail to start lfsck_namespace!"
while true; do
local STATUS=$($SHOW_NAMESPACE |
awk '/^status/ { print $2 }')
[ "$STATUS" == "completed" ] && break
sleep 3 # check status every 3 seconds
done
echo "end lfsck_namespace for ${i} files set at: $(date)"
local SPEED=$($SHOW_NAMESPACE |
awk '/^average_speed_phase1/ { print $2 }')
echo "lfsck_namespace speed is ${SPEED}/sec"
stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
done
combined_mgs_mds || stop mgs
}
run_test 1 "lfsck namespace performance (backup/restore) without load"
test_2() {
local i
for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
i = $((i * FACTOR)))); do
stopall
combined_mgs_mds || format_start_mgs
do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
format_mdt $(facet_number $SINGLEMDS)
echo "+++ start to create for ${i} files set at: $(date) +++"
lfsck_create_nfiles ${i} 0 ${NTHREADS} 1 ||
error "Fail to create files!"
echo "+++ end to create for ${i} files set at: $(date) +++"
start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
error "Fail to start MDS!"
echo "start lfsck_namespace for ${i} files set at: $(date)"
$START_NAMESPACE || error "Fail to start lfsck_namespace!"
while true; do
local STATUS=$($SHOW_NAMESPACE |
awk '/^status/ { print $2 }')
[ "$STATUS" == "completed" ] && break
sleep 3 # check status every 3 seconds
done
echo "end lfsck_namespace for ${i} files set at: $(date)"
local SPEED=$($SHOW_NAMESPACE |
awk '/^average_speed_phase1/ { print $2 }')
echo "lfsck_namespace speed is ${SPEED}/sec"
stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
done
combined_mgs_mds || stop mgs
}
run_test 2 "lfsck namespace performance (upgrade from 1.8) without load"
test_3() {
[ $INCFACTOR -gt 25 ] && INCFACTOR=25
local inc_count=$((BASE_COUNT * INCFACTOR / 100))
local BCOUNT=0
local i
stopall
combined_mgs_mds || format_start_mgs
do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
format_mdt $(facet_number $SINGLEMDS)
for ((i = $inc_count; i <= $BASE_COUNT; i = $((i + inc_count)))); do
local nfiles=$((i - BCOUNT))
echo "+++ start to create for ${i} files set at: $(date) +++"
lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
error "Fail to create files!"
echo "+++ end to create for ${i} files set at: $(date) +++"
BCOUNT=${i}
done
start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
error "Fail to start MDS!"
echo "start lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
$START_NAMESPACE || error "Fail to start lfsck_namespace!"
while true; do
local STATUS=$($SHOW_NAMESPACE |
awk '/^status/ { print $2 }')
[ "$STATUS" == "completed" ] && break
sleep 3 # check status every 3 seconds
done
echo "end lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
local FULL_SPEED=$($SHOW_NAMESPACE |
awk '/^average_speed_phase1/ { print $2 }')
echo "lfsck_namespace full_speed is ${FULL_SPEED}/sec"
stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
local inc_speed=$((FULL_SPEED * INCFACTOR / 100))
local j
for ((j = $inc_speed; j < $FULL_SPEED; j = $((j + inc_speed)))); do
start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
error "Fail to start MDS!"
$STOP_LFSCK > /dev/null 2>&1
echo "start lfsck_namespace with speed ${j} at: $(date)"
$START_NAMESPACE --reset -s ${j} ||
error "Fail to start lfsck_namespace with speed ${j}!"
# lfsck_namespace will be paused when MDS stop,
# and will be restarted automatically when mount up again.
stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
local nfiles=$(((i - BCOUNT) / 2))
echo "+++ start to create for ${i} files set at: $(date) +++"
lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
error "Fail to create files!"
echo "+++ end to create for ${i} files set at: $(date) +++"
BCOUNT=${i}
i=$((i + inc_count))
done
start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
error "Fail to start MDS!"
$STOP_LFSCK /dev/null 2>&1
echo "start lfsck_namespace with full speed at: $(date)"
$START_NAMESPACE --reset -s 0 ||
error "Fail to start lfsck_namespace with full speed!"
stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
local nfiles=$(((i - BCOUNT) / 2))
echo "+++ start to create for ${i} files set at: $(date) +++"
lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
error "Fail to create files!"
echo "+++ end to create for ${i} files set at: $(date) +++"
combined_mgs_mds || stop mgs
}
run_test 3 "lfsck namespace impact on create performance"
show_layout() {
local idx=$1
do_facet mds${idx} \
"$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_layout"
}
layout_test_one() {
echo "***** Start layout LFSCK on all devices at: $(date) *****"
$RLCTL lfsck_start -M ${MDT_DEV} -t layout -A -r || return 21
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_layout |
awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
show_layout 1
return 22
}
echo "***** End layout LFSCK on all devices at: $(date) *****"
for n in $(seq $MDSCOUNT); do
show_layout ${n}
local SPEED=$(show_layout ${n} |
awk '/^average_speed_phase1/ { print $2 }')
echo
echo "lfsck_layout speed on MDS_${n} is $SPEED objs/sec"
echo
done
}
layout_gen_one() {
local idx1=$1
local idx2=$2
local mntpt="/mnt/lustre_lfsck_${idx1}_${idx2}"
local basedir="$mntpt/$tdir/$idx1/$idx2"
mkdir -p $mntpt || {
error_noexit "(11) Fail to mkdir $mntpt"
return 11
}
mount_client $mntpt || {
error_noexit "(12) Fail to mount $mntpt"
return 12
}
mkdir $basedir || {
umount_client $mntpt
error_noexit "(13) Fail to mkdir $basedir"
return 13
}
echo "&&&&& Start create $UNIT files under $basedir at: $(date) &&&&&"
createmany -o ${basedir}/f $UNIT || {
umount_client $mntpt
error_noexit "(14) Fail to gen $UNIT files under $basedir"
return 14
}
echo "&&&&& End create $UNIT files under $basedir at: $(date) &&&&&"
umount_client $mntpt
}
layout_gen_set() {
local cnt=$1
echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
for ((k = 0; k < $MDSCOUNT; k++)); do
$LFS mkdir -i ${k} $LFSCKDIR/${k} || return 10
for ((l = 1; l <= $cnt; l++)); do
layout_gen_one ${k} ${l} &
done
done
wait
echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
}
t4_test() {
local saved_mdscount=$MDSCOUNT
local saved_ostcount=$OSTCOUNT
echo "stopall"
stopall > /dev/null || error "(1) Fail to stopall"
LFSCKDIR="$DIR/$tdir"
MDSCOUNT=1
for ((i = 1; i <= $saved_ostcount; i = $((i * 2)))); do
OSTCOUNT=${i}
echo "+++++ Start cycle ostcount=$OSTCOUNT at: $(date) +++++"
echo
for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
echo "formatall"
formatall > /dev/null ||
error "(2) Fail to formatall, subdirs=${j}"
echo "setupall"
setupall > /dev/null ||
error "(3) Fail to setupall, subdirs=${j}"
mkdir $LFSCKDIR ||
error "(4) mkdir $LFSCKDIR, subdirs=${j}"
$LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
local RC=0
layout_gen_set ${j} || RC=$?
[ $RC -eq 0 ] ||
error "(6) generate set $RC, subdirs=${j}"
RC=0
layout_test_one || RC=$?
[ $RC -eq 0 ] ||
error "(7) LFSCK failed with $RC, subdirs=${j}"
done
echo "stopall"
stopall > /dev/null || error "(8) Fail to stopall, subdirs=${j}"
echo
echo "----- Stop cycle ostcount=$OSTCOUNT at: $(date) -----"
done
MDSCOUNT=$saved_mdscount
OSTCOUNT=$saved_ostcount
}
test_4a() {
t4_test
}
run_test 4a "Single MDS lfsck layout performance (routine case) without load"
test_4b() {
echo "Inject failure stub to simulate dangling reference"
#define OBD_FAIL_LFSCK_DANGLING 0x1610
do_nodes $(osts_nodes) "$LCTL set_param fail_loc=0x1610"
t4_test
}
run_test 4b "Single MDS lfsck layout performance (repairing case) without load"
t5_test() {
local saved_mdscount=$MDSCOUNT
echo "stopall"
stopall > /dev/null || error "(1) Fail to stopall"
LFSCKDIR="$DIR/$tdir"
for ((i = 1; i <= $saved_mdscount; i++)); do
MDSCOUNT=${i}
echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
echo
for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
echo "formatall"
formatall > /dev/null ||
error "(2) Fail to formatall, subdirs=${j}"
echo "setupall"
setupall > /dev/null ||
error "(3) Fail to setupall, subdirs=${j}"
# minimal number of free inodes among all OSTs
local avail=$($LFS df -i | grep "\[OST:.*\]" | sort -nk 4 |
head -1 | awk '{print $4}')
local need=$((j * UNIT * i))
# add 10% to take unevenness of precreates among OSPs
((need += need / 10))
if ((avail < need))
then
echo "$avail, need $need at least"
i=$saved_mdscount
break;
fi
mkdir $LFSCKDIR ||
error "(4) mkdir $LFSCKDIR, subdirs=${j}"
$LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
error "(5) setstripe on $LFSCKDIR, subdirs=${j}"
local RC=0
layout_gen_set ${j} || RC=$?
[ $RC -eq 0 ] ||
error "(6) generate set $RC, subdirs=${j}"
RC=0
layout_test_one || RC=$?
[ $RC -eq 0 ] ||
error "(7) LFSCK failed with $RC, subdirs=${j}"
done
echo "stopall"
stopall > /dev/null || error "(8) Fail to stopall"
echo
echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
done
MDSCOUNT=$saved_mdscount
}
test_5a() {
t5_test
}
run_test 5a "lfsck layout performance (routine case) without load for DNE"
test_5b() {
echo "Inject failure stub to simulate dangling reference"
#define OBD_FAIL_LFSCK_DANGLING 0x1610
do_nodes $(osts_nodes) "$LCTL set_param fail_loc=0x1610"
t5_test
}
run_test 5b "lfsck layout performance (repairing case) without load for DNE"
lfsck_fast_create() {
local total=$1
local lbase=$2
local threads=$3
local ldir="/test-${lbase}"
local cycle=0
local count=$UNIT
while true; do
[ $count -eq 0 -o $count -gt ${total} ] && count=$total
local usize=$((count / NTHREADS))
[ ${usize} -eq 0 ] && break
local tdir=${ldir}-${cycle}-
echo "[cycle: $cycle] [threads: $threads]"\
"[files: $count] [basedir: $tdir]"
lfsck_create
total=$((total - usize * NTHREADS))
[ $total -eq 0 ] && break
lbase=$((lbase + usize))
cycle=$((cycle + 1))
done
}
lfsck_detach_error() {
lfsck_detach
error "$@"
}
test_6() {
[ $INCFACTOR -gt 25 ] && INCFACTOR=25
echo "stopall"
stopall > /dev/null || error "(1) Fail to stopall"
local saved_mdscount=$MDSCOUNT
LFSCKDIR="$DIR/$tdir"
MDSCOUNT=1
echo "formatall"
formatall > /dev/null || error "(2) Fail to formatall"
echo "setupall"
setupall > /dev/null || error "(3) Fail to setupall"
mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
$LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
error "(5) Fail to setstripe on $LFSCKDIR"
local RC=0
layout_gen_set $TOTSUBDIR || RC=$?
[ $RC -eq 0 ] ||
error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
echo
echo "***** Start layout LFSCK on single MDS at: $(date) *****"
$RLCTL lfsck_start -M ${MDT_DEV} -t layout -r ||
error "(7) Fail to start layout LFSCK"
wait_update_facet $SINGLEMDS "$LCTL get_param -n \
mdd.${MDT_DEV}.lfsck_layout |
awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
show_layout 1
error "(8) layout LFSCK cannot finished in time"
}
echo "***** End layout LFSCK on single MDS at: $(date) *****"
local SPEED=$(show_layout 1 |
awk '/^average_speed_phase1/ { print $2 }')
echo "lfsck_layout full_speed is $SPEED objs/sec"
local inc_count=$((BASE_COUNT * INCFACTOR / 100))
local nfiles=$((inc_count / 2))
lfsck_attach
for ((m = 0, n = $INCFACTOR; n < 100;
m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
local sl=$((SPEED * n / 100))
$STOP_LFSCK > /dev/null 2>&1
echo
echo "start lfsck_layout with speed ${sl} at: $(date)"
$RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s ${sl} ||
lfsck_detach_error \
"(9) Fail to start lfsck_layout with speed ${sl}"
echo "&&&&& Start create files set from ${m} at: $(date) &&&&&"
lfsck_fast_create $nfiles ${m} $NTHREADS ||
lfsck_detach_error "(10) Fail to create files"
echo "&&&&& End create files set from ${m} at: $(date) &&&&&"
done
$STOP_LFSCK > /dev/null 2>&1
echo
echo "start lfsck_layout with full speed at: $(date)"
$RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s 0 ||
lfsck_detach_error \
"(11) Fail to start lfsck_layout with full speed"
echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
lfsck_fast_create $nfiles ${m} $NTHREADS ||
lfsck_detach_error "(12) Fail to create files"
echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
m=$((m + inc_count))
$STOP_LFSCK > /dev/null 2>&1
echo
echo "create without lfsck_layout run back-ground"
echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
lfsck_fast_create $nfiles ${m} $NTHREADS ||
lfsck_detach_error "(13) Fail to create files"
echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"
lfsck_detach
echo
echo "stopall"
stopall > /dev/null || error "(14) Fail to stopall"
MDSCOUNT=$saved_mdscount
}
run_test 6 "lfsck layout impact on create performance"
show_namespace() {
local idx=$1
do_facet mds${idx} \
"$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_namespace"
}
namespace_test_one() {
echo "***** Start namespace LFSCK on all devices at: $(date) *****"
$RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r || return 21
for n in $(seq $MDSCOUNT); do
wait_update_facet mds${n} "$LCTL get_param -n \
mdd.$(facet_svc mds${n}).lfsck_namespace |
awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
show_namespace ${n}
return 22
}
done
echo "***** End namespace LFSCK on all devices at: $(date) *****"
for n in $(seq $MDSCOUNT); do
show_namespace ${n}
local SPEED=$(show_namespace ${n} |
awk '/^average_speed_total/ { print $2 }')
echo
echo "lfsck_namespace speed on MDS_${n} is $SPEED objs/sec"
echo
done
}
namespace_gen_one() {
local idx1=$1
local idx2=$2
local idx3=$(((idx1 + 1) % MDSCOUNT))
local base_mntpt="/mnt/lustre_lfsck_${idx1}"
local show_dir="$LFSCKDIR/${idx1}/${idx2}"
local work_dir="${base_mntpt}_0/$tdir/${idx1}/${idx2}"
mkdir $show_dir || return 20
local count=$((UNIT * 78 / 100)) # 78% regular files
local sub_count=$((count / NTHREADS))
echo "Creating $count regular files under $show_dir at: $(date)"
for ((m = 0; m < $NTHREADS; m++)); do
local sub_dir="${base_mntpt}_${m}/$tdir/${idx1}/${idx2}"
createmany -o ${sub_dir}/f_${m}_ $sub_count > /dev/null &
done
wait || {
error_noexit "(21) Fail to gen regular files under $show_dir"
return 21
}
count=$((UNIT * 3 / 100)) # 3% local sub-dirs
echo "Creating $count local sub-dirs under $show_dir at: $(date)"
createmany -d $work_dir/d_l_ $count > /dev/null || {
error_noexit "(22) Fail to gen local sub-dir under $show_dir"
return 22
}
# 3% * 5 = 15% regular files under local sub-dirs
echo "Creating 5 regular files under each local sub-dir at: $(date)"
for ((m = 0; m < $count; m++)); do
createmany -o $work_dir/d_l_${m}/f_l_ 5 > /dev/null || {
error_noexit \
"(23) Fail to gen regular under $work_dir/d_l_${m}"
return 23
}
done
count=$((UNIT * 4 / 1000)) # 0.4% multiple hard-links
echo "Creating $count multiple hard-links under $show_dir at: $(date)"
for ((m = 0; m < $count; m++)); do
ln $work_dir/f_0_${m} $work_dir/f_m_${m} || {
error_noexit \
"(24) Fail to hardlink to $work_dir/f_0_${m}"
return 24
}
done
count=$((UNIT * 3 / 1000)) # 0.3% remote sub-dirs
echo "Creating $count remote sub-dirs under $show_dir, and 4 regular" \
"files under each remote sub-dir at: $(date)"
for ((m = 0; m < $count; m++)); do
$LFS mkdir -i ${idx3} $work_dir/d_r_${m} || {
error_noexit \
"(25) Fail to remote mkdir $work_dir/d_r_${m}"
return 25
}
# 0.3% * 4 = 1.2% regular files under remote sub-dirs
createmany -o $work_dir/d_r_${m}/f_r_ 4 > /dev/null || {
error_noexit \
"(26) Fail to gen regular under $work_dir/d_r_${m}"
return 26
}
done
# 0.3% 2-striped sub-dirs + 0.6% shards of the 2-striped sub-dirs
count=$((UNIT * 3 / 1000))
echo "Creating $count 2-striped sub-dirs under $show_dir," \
"and 4 regular files under each striped sub-dir at: $(date)"
for ((m = 0; m < $count; m++)); do
$LFS setdirstripe -i ${idx1} -c 2 -H all_char \
$work_dir/d_s_${m} || {
error_noexit \
"(27) Fail to make striped-dir $work_dir/d_s_${m}"
return 27
}
# 0.3% * 4 = 1.2% regular files under striped sub-dirs
createmany -o $work_dir/d_s_${m}/f_s_ 4 > /dev/null || {
error_noexit \
"(28) Fail to gen regular under $work_dir/d_s_${m}"
return 28
}
done
}
namespace_gen_mdt() {
local mdt_idx=$1
local dir_cnt=$2
local base_mntpt="/mnt/lustre_lfsck_${mdt_idx}"
$LFS mkdir -i ${mdt_idx} $LFSCKDIR/${mdt_idx} || return 10
for ((m = 0; m < $NTHREADS; m++)); do
local mntpt="${base_mntpt}_${m}"
mkdir -p $mntpt || {
umount ${base_mntpt}_*
error_noexit "(11) Fail to mkdir $mntpt"
return 11
}
mount_client $mntpt || {
umount ${base_mntpt}_*
error_noexit "(12) Fail to mount $mntpt"
return 12
}
done
for ((l = 0; l < $dir_cnt; l++)); do
namespace_gen_one ${mdt_idx} ${l}
done
umount ${base_mntpt}_*
}
namespace_gen_set() {
local cnt=$1
echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
for ((k = 0; k < $MDSCOUNT; k++)); do
namespace_gen_mdt ${k} ${cnt} &
done
wait
echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
}
t7_test() {
local local_loc=$1
local saved_mdscount=$MDSCOUNT
local mdts=$(mdts_nodes)
[ $MDSCOUNT -le 8 ] ||
error "Too much MDT, test data set on each MDT may be unbalance"
echo "stopall"
stopall > /dev/null || error "(1) Fail to stopall"
LFSCKDIR="$DIR/$tdir"
for ((i = 2; i <= $saved_mdscount; i = $((i + 2)))); do
MDSCOUNT=${i}
echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
echo
for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j += MINSUBDIR)); do
echo "formatall"
formatall > /dev/null ||
error "(2) Fail to formatall, subdirs=${j}"
echo "setupall"
setupall > /dev/null ||
error "(3) Fail to setupall, subdirs=${j}"
mkdir $LFSCKDIR ||
error "(4) mkdir $LFSCKDIR, subdirs=${j}"
$LFS setstripe -c 1 -i -1 $LFSCKDIR ||
error "(5) Fail to setstripe on $LFSCKDIR"
do_nodes $mdts $LCTL set_param fail_loc=$local_loc
local RC=0
namespace_gen_set ${j} || RC=$?
[ $RC -eq 0 ] ||
error "(6) generate set $RC, subdirs=${j}"
RC=0
namespace_test_one || RC=$?
[ $RC -eq 0 ] ||
error "(7) LFSCK failed with $RC, subdirs=${j}"
do_nodes $mdts $LCTL set_param fail_loc=0
done
echo "stopall"
stopall > /dev/null || error "(8) Fail to stopall"
echo
echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
done
MDSCOUNT=$saved_mdscount
}
test_7a() {
t7_test 0
}
run_test 7a "namespace LFSCK performance (routine check) without load for DNE"
test_7b() {
echo "Inject failure stub to simulate the case of lost linkEA"
#define OBD_FAIL_LFSCK_NO_LINKEA 0x161d
t7_test 0x161d
}
run_test 7b "namespace LFSCK performance (repairing lost linkEA) for DNE"
test_7c() {
echo "Inject failure stub to simulate the case of bad FID-in-dirent"
#define OBD_FAIL_FID_INDIR 0x1501
t7_test 0x1501
}
run_test 7c "namespace LFSCK performance (repairing bad FID-in-dirent) for DNE"
test_8() {
[ $MDSCOUNT -lt 2 ] && skip_env "needs >= 2 MDTs"
[ $INCFACTOR -gt 25 ] && INCFACTOR=25
echo "stopall"
stopall > /dev/null || error "(1) Fail to stopall"
local saved_mdscount=$MDSCOUNT
LFSCKDIR="$DIR/$tdir"
MDSCOUNT=2
echo "formatall"
formatall > /dev/null || error "(2) Fail to formatall"
echo "setupall"
setupall > /dev/null || error "(3) Fail to setupall"
mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"
$LFS setstripe -c 1 -i 0 $LFSCKDIR ||
error "(5) Fail to setstripe on $LFSCKDIR"
local RC=0
namespace_gen_set $TOTSUBDIR || RC=$?
[ $RC -eq 0 ] ||
error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"
echo
echo "***** Start namespace LFSCK at: $(date) *****"
$RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r ||
error "(7) Fail to start namespace LFSCK"
for n in $(seq $MDSCOUNT); do
wait_update_facet mds${n} "$LCTL get_param -n \
mdd.$(facet_svc mds${n}).lfsck_namespace |
awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
show_namespace ${n}
error "(8) namespace LFSCK cannot finished in time"
}
done
echo "***** End namespace LFSCK at: $(date) *****"
local SPEED=$(show_namespace 1 |
awk '/^average_speed_phase1/ { print $2 }')
echo "lfsck_namespace full_speed is $SPEED objs/sec"
echo
local inc_count=$((BASE_COUNT * INCFACTOR / 100))
local nfiles=$((inc_count / 2))
local m=0
lfsck_attach
local stime=$(date +%s)
lfsck_fast_create $nfiles ${m} $NTHREADS ||
lfsck_detach_error "(9) Fail to create files"
local etime=$(date +%s)
echo "created $nfiles without lfsck_namespace run back-ground used" \
"$((etime - stime)) seconds"
echo
for ((m = nfiles, n = $INCFACTOR; n < 100;
m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
local sl=$((SPEED * n / 100))
$STOP_LFSCK > /dev/null 2>&1
echo "start lfsck_namespace with speed ${sl} at: $(date)"
$RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s ${sl} ||
lfsck_detach_error \
"(10) Fail to start lfsck_namespace with speed ${sl}"
stime=$(date +%s)
lfsck_fast_create $nfiles ${m} $NTHREADS ||
lfsck_detach_error "(11) Fail to create files"
etime=$(date +%s)
echo "created $nfiles with namespace LFSCK run with the" \
"speed limit of ${n}% of full speed used" \
"$((etime - stime)) seconds"
echo
done
$STOP_LFSCK > /dev/null 2>&1
echo "start lfsck_namespace with full speed at: $(date)"
$RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s 0 ||
lfsck_detach_error \
"(12) Fail to start lfsck_namespace with full speed"
stime=$(date +%s)
lfsck_fast_create $nfiles ${m} $NTHREADS ||
lfsck_detach_error "(13) Fail to create files"
etime=$(date +%s)
echo "created $nfiles with namespace LFSCK run with full speed used" \
"$((etime - stime)) seconds"
echo
$STOP_LFSCK > /dev/null 2>&1
lfsck_detach
echo "stopall"
stopall > /dev/null || error "(14) Fail to stopall"
MDSCOUNT=$saved_mdscount
}
run_test 8 "lfsck namespace impact on create performance"
# cleanup the system at last
lfsck_cleanup
complete_test $SECONDS
check_and_cleanup_lustre
exit_status
