mirror of
https://github.com/markusressel/zfs-inplace-rebalancing
synced 2026-02-05 05:24:07 +00:00
Merge branch 'master' into HEAD
This commit is contained in:
Colin Hebert
commit
adf4cf60d0
33
.github/workflows/test.yml
vendored
33
.github/workflows/test.yml
vendored
@ -9,11 +9,40 @@ on:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
shellcheck:
|
||||
name: Test
|
||||
linuxTest:
|
||||
name: Test on Linux
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Run testing script
|
||||
run: ./testing.sh
|
||||
|
||||
macOsTest:
|
||||
name: Test on macOS
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install coreutils
|
||||
run: brew install coreutils
|
||||
|
||||
- name: Run testing script on macOS
|
||||
run: ./testing.sh
|
||||
|
||||
FreeBSDTest:
|
||||
name: Test on FreeBSD
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Test in FreeBSD
|
||||
id: test
|
||||
uses: vmactions/freebsd-vm@v1
|
||||
with:
|
||||
usesh: true
|
||||
prepare: |
|
||||
pkg install -y bash
|
||||
run: |
|
||||
./testing.sh
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@ -1,4 +1,5 @@
|
||||
test.log
|
||||
error.log
|
||||
rebalance_db.txt
|
||||
testing_data
|
||||
testing_data
|
||||
.vscode
|
||||
117
testing.sh
117
testing.sh
@ -9,6 +9,27 @@ log_std_file=./test.log
|
||||
log_error_file=./error.log
|
||||
test_data_src=./test/pool
|
||||
test_pool_data_path=./testing_data
|
||||
test_pool_data_size_path=$test_pool_data_path/size
|
||||
|
||||
## Color Constants
|
||||
|
||||
# Reset
|
||||
Color_Off='\033[0m' # Text Reset
|
||||
|
||||
# Regular Colors
|
||||
Red='\033[0;31m' # Red
|
||||
Green='\033[0;32m' # Green
|
||||
Yellow='\033[0;33m' # Yellow
|
||||
Cyan='\033[0;36m' # Cyan
|
||||
|
||||
## Functions
|
||||
|
||||
# print a given text entirely in a given color
|
||||
function color_echo () {
|
||||
color=$1
|
||||
text=$2
|
||||
echo -e "${color}${text}${Color_Off}"
|
||||
}
|
||||
|
||||
function prepare() {
|
||||
# cleanup
|
||||
@ -21,10 +42,25 @@ function prepare() {
|
||||
cp -rf $test_data_src $test_pool_data_path
|
||||
}
|
||||
|
||||
# return time to the milisecond
|
||||
function get_time() {
|
||||
|
||||
case "$OSTYPE" in
|
||||
darwin*)
|
||||
date=$(gdate +%s%N)
|
||||
;;
|
||||
*)
|
||||
date=$(date +%s%N)
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "$date"
|
||||
}
|
||||
|
||||
function assertions() {
|
||||
# check error log is empty
|
||||
if grep -q '[^[:space:]]' $log_error_file; then
|
||||
echo "error log is not empty!"
|
||||
color_echo "$Red" "error log is not empty!"
|
||||
cat $log_error_file
|
||||
exit 1
|
||||
fi
|
||||
@ -44,21 +80,38 @@ function assert_matching_file_not_copied() {
|
||||
fi
|
||||
}
|
||||
|
||||
function print_time_taken(){
|
||||
time_taken=$1
|
||||
minute=$((time_taken / 60000))
|
||||
seconde=$((time_taken % 60000 / 1000))
|
||||
miliseconde=$((time_taken % 1000))
|
||||
color_echo "$Yellow" "Time taken: ${minute}m ${seconde}s ${miliseconde}ms"
|
||||
}
|
||||
|
||||
color_echo "$Cyan" "Running tests..."
|
||||
|
||||
color_echo "$Cyan" "Running tests with default options..."
|
||||
prepare
|
||||
./zfs-inplace-rebalancing.sh $test_pool_data_path >> $log_std_file 2>> $log_error_file
|
||||
cat $log_std_file
|
||||
assertions
|
||||
color_echo "$Green" "Tests passed!"
|
||||
|
||||
color_echo "$Cyan" "Running tests with checksum true and 1 pass..."
|
||||
prepare
|
||||
./zfs-inplace-rebalancing.sh --checksum true --passes 1 $test_pool_data_path >> $log_std_file 2>> $log_error_file
|
||||
cat $log_std_file
|
||||
assertions
|
||||
color_echo "$Green" "Tests passed!"
|
||||
|
||||
color_echo "$Cyan" "Running tests with checksum false..."
|
||||
prepare
|
||||
./zfs-inplace-rebalancing.sh --checksum false $test_pool_data_path >> $log_std_file 2>> $log_error_file
|
||||
cat $log_std_file
|
||||
assertions
|
||||
color_echo "$Green" "Tests passed!"
|
||||
|
||||
color_echo "$Cyan" "Running tests with skip-hardlinks false..."
|
||||
prepare
|
||||
ln "$test_pool_data_path/projects/[2020] some project/mp4.txt" "$test_pool_data_path/projects/[2020] some project/mp4.txt.link"
|
||||
./zfs-inplace-rebalancing.sh --skip-hardlinks false $test_pool_data_path >> $log_std_file 2>> $log_error_file
|
||||
@ -67,7 +120,9 @@ cat $log_std_file
|
||||
assert_matching_file_copied "mp4.txt"
|
||||
assert_matching_file_copied "mp4.txt.link"
|
||||
assertions
|
||||
color_echo "$Green" "Tests passed!"
|
||||
|
||||
color_echo "$Cyan" "Running tests with skip-hardlinks true..."
|
||||
prepare
|
||||
ln "$test_pool_data_path/projects/[2020] some project/mp4.txt" "$test_pool_data_path/projects/[2020] some project/mp4.txt.link"
|
||||
./zfs-inplace-rebalancing.sh --skip-hardlinks true $test_pool_data_path >> $log_std_file 2>> $log_error_file
|
||||
@ -76,3 +131,63 @@ cat $log_std_file
|
||||
assert_matching_file_not_copied "mp4.txt.link"
|
||||
assert_matching_file_not_copied "mp4.txt"
|
||||
assertions
|
||||
color_echo "$Green" "Tests passed!"
|
||||
|
||||
color_echo "$Cyan" "Running tests with different file count and size..."
|
||||
prepare
|
||||
|
||||
mkdir -p $test_pool_data_size_path
|
||||
|
||||
color_echo "$Cyan" "Creating 1000 files of 1KB each..."
|
||||
mkdir -p $test_pool_data_size_path/small
|
||||
for i in {1..1000}; do
|
||||
dd if=/dev/urandom of=$test_pool_data_size_path/small/file_"$i".txt bs=1024 count=1 >> /dev/null 2>&1
|
||||
done
|
||||
|
||||
color_echo "$Cyan" "Creating 5 file of 1GB each..."
|
||||
mkdir -p $test_pool_data_size_path/big
|
||||
for i in {1..5}; do
|
||||
dd if=/dev/urandom of=$test_pool_data_size_path/big/file_"$i".txt bs=1024 count=1048576 >> /dev/null 2>&1
|
||||
done
|
||||
|
||||
color_echo "$Green" "Files created!"
|
||||
|
||||
echo "Running rebalancing on small files..."
|
||||
# measure time taken
|
||||
start_time=$(get_time)
|
||||
./zfs-inplace-rebalancing.sh $test_pool_data_size_path/small >> $log_std_file 2>> $log_error_file
|
||||
end_time=$(get_time)
|
||||
time_taken=$(( (end_time - start_time) / 1000000 ))
|
||||
print_time_taken $time_taken
|
||||
assertions
|
||||
color_echo "$Green" "Tests passed!"
|
||||
|
||||
echo "Running rebalancing on big files..."
|
||||
rm -f rebalance_db.txt
|
||||
# measure time taken
|
||||
start_time=$(get_time)
|
||||
./zfs-inplace-rebalancing.sh $test_pool_data_size_path/big >> $log_std_file 2>> $log_error_file
|
||||
end_time=$(get_time)
|
||||
time_taken=$(( (end_time - start_time) / 1000000 ))
|
||||
print_time_taken $time_taken
|
||||
assertions
|
||||
color_echo "$Green" "Tests passed!"
|
||||
|
||||
echo "Running rebalancing on all files..."
|
||||
rm -f rebalance_db.txt
|
||||
# measure time taken
|
||||
start_time=$(get_time)
|
||||
./zfs-inplace-rebalancing.sh $test_pool_data_size_path >> $log_std_file 2>> $log_error_file
|
||||
end_time=$(get_time)
|
||||
time_taken=$(( (end_time - start_time) / 1000000 ))
|
||||
print_time_taken $time_taken
|
||||
assertions
|
||||
color_echo "$Green" "Tests passed!"
|
||||
|
||||
color_echo "$Green" "All tests passed!"
|
||||
color_echo "$Cyan" "Cleaning"
|
||||
rm -f $log_std_file
|
||||
rm -f $log_error_file
|
||||
rm -f rebalance_db.txt
|
||||
rm -rf $test_pool_data_path
|
||||
|
||||
|
||||
@ -14,29 +14,29 @@ current_index=0
|
||||
## Color Constants
|
||||
|
||||
# Reset
|
||||
Color_Off='\033[0m' # Text Reset
|
||||
Color_Off='\033[0m' # Text Reset
|
||||
|
||||
# Regular Colors
|
||||
Red='\033[0;31m' # Red
|
||||
Green='\033[0;32m' # Green
|
||||
Yellow='\033[0;33m' # Yellow
|
||||
Cyan='\033[0;36m' # Cyan
|
||||
Red='\033[0;31m' # Red
|
||||
Green='\033[0;32m' # Green
|
||||
Yellow='\033[0;33m' # Yellow
|
||||
Cyan='\033[0;36m' # Cyan
|
||||
|
||||
## Functions
|
||||
|
||||
# Print a help message
|
||||
function print_usage() {
|
||||
echo "Usage: zfs-inplace-rebalancing.sh --checksum true --passes 1 --debug false /my/pool"
|
||||
echo "Usage: zfs-inplace-rebalancing.sh --checksum true --passes 1 --debug false /my/pool"
|
||||
}
|
||||
|
||||
# Print a given text entirely in a given color
|
||||
function color_echo () {
|
||||
function color_echo() {
|
||||
color=$1
|
||||
text=$2
|
||||
echo -e "${color}${text}${Color_Off}"
|
||||
}
|
||||
|
||||
function get_rebalance_count () {
|
||||
function get_rebalance_count() {
|
||||
file_path="$1"
|
||||
|
||||
line_nr=$(grep -xF -n -e "${file_path}" "./${rebalance_db_file_name}" | head -n 1 | cut -d: -f1)
|
||||
@ -103,14 +103,14 @@ function process_inode_group() {
|
||||
if [ "$debug_flag" = true ]; then
|
||||
echo "Executing copy command:"
|
||||
fi
|
||||
if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then
|
||||
if [[ "${OSName}" == "linux-gnu"* ]]; then
|
||||
# Linux
|
||||
cmd=(cp --reflink=never -ax "${main_file}" "${tmp_file_path}")
|
||||
if [ "$debug_flag" = true ]; then
|
||||
echo "${cmd[@]}"
|
||||
fi
|
||||
"${cmd[@]}"
|
||||
elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then
|
||||
elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then
|
||||
# Mac OS and FreeBSD
|
||||
cmd=(cp -ax "${main_file}" "${tmp_file_path}")
|
||||
if [ "$debug_flag" = true ]; then
|
||||
@ -123,30 +123,57 @@ function process_inode_group() {
|
||||
fi
|
||||
|
||||
# Compare copy against original to make sure nothing went wrong
|
||||
if [[ "${checksum_flag,,}" == "true"* ]]; then
|
||||
if [[ "${checksum_flag}" == "true"* ]]; then
|
||||
echo "Comparing copy against original..."
|
||||
if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then
|
||||
if [[ "${OSName}" == "linux-gnu"* ]]; then
|
||||
# Linux
|
||||
original_md5=$(md5sum -b "${main_file}" | awk '{print $1}')
|
||||
copy_md5=$(md5sum -b "${tmp_file_path}" | awk '{print $1}')
|
||||
elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then
|
||||
# Mac OS and FreeBSD
|
||||
original_md5=$(md5 -q "${main_file}")
|
||||
copy_md5=$(md5 -q "${tmp_file_path}")
|
||||
|
||||
# file attributes
|
||||
original_perms=$(lsattr "${main_file}")
|
||||
# remove anything after the last space
|
||||
original_perms=${original_perms% *}
|
||||
# file permissions, owner, group, size, modification time
|
||||
original_perms="${original_perms} $(stat -c "%A %U %G %s %Y" "${main_file}")"
|
||||
|
||||
|
||||
# file attributes
|
||||
copy_perms=$(lsattr "${tmp_file_path}")
|
||||
# remove anything after the last space
|
||||
copy_perms=${copy_perms% *}
|
||||
# file permissions, owner, group, size, modification time
|
||||
copy_perms="${copy_perms} $(stat -c "%A %U %G %s %Y" "${tmp_file_path}")"
|
||||
elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then
|
||||
# Mac OS
|
||||
# FreeBSD
|
||||
|
||||
# note: no lsattr on Mac OS or FreeBSD
|
||||
|
||||
# file permissions, owner, group size, modification time
|
||||
original_perms="$(stat -f "%Sp %Su %Sg %z %m" "${main_file}")"
|
||||
|
||||
# file permissions, owner, group size, modification time
|
||||
copy_perms="$(stat -f "%Sp %Su %Sg %z %m" "${tmp_file_path}")"
|
||||
else
|
||||
echo "Unsupported OS type: $OSTYPE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$debug_flag" = true ]; then
|
||||
echo "Original MD5: $original_md5"
|
||||
echo "Copy MD5: $copy_md5"
|
||||
echo "Original perms: $original_perms"
|
||||
echo "Copy perms: $copy_perms"
|
||||
fi
|
||||
|
||||
if [[ "${original_md5}" == "${copy_md5}" ]]; then
|
||||
color_echo "${Green}" "MD5 OK"
|
||||
if [[ "${original_perms}" == "${copy_perms}"* ]]; then
|
||||
color_echo "${Green}" "Attribute and permission check OK"
|
||||
else
|
||||
color_echo "${Red}" "MD5 FAILED: ${original_md5} != ${copy_md5}"
|
||||
color_echo "${Red}" "Attribute and permission check FAILED: ${original_perms} != ${copy_perms}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if cmp -s "${main_file}" "${tmp_file_path}"; then
|
||||
color_echo "${Green}" "File content check OK"
|
||||
else
|
||||
color_echo "${Red}" "File content check FAILED"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
@ -206,40 +233,42 @@ if [[ "$#" -eq 0 ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while true ; do
|
||||
while true; do
|
||||
case "$1" in
|
||||
-h | --help )
|
||||
print_usage
|
||||
exit 0
|
||||
-h | --help)
|
||||
print_usage
|
||||
exit 0
|
||||
;;
|
||||
-c | --checksum )
|
||||
if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then
|
||||
checksum_flag="true"
|
||||
else
|
||||
checksum_flag="false"
|
||||
fi
|
||||
shift 2
|
||||
-c | --checksum)
|
||||
if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then
|
||||
checksum_flag="true"
|
||||
else
|
||||
checksum_flag="false"
|
||||
fi
|
||||
shift 2
|
||||
;;
|
||||
-p | --passes )
|
||||
passes_flag=$2
|
||||
shift 2
|
||||
-p | --passes)
|
||||
passes_flag=$2
|
||||
shift 2
|
||||
;;
|
||||
--debug )
|
||||
if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then
|
||||
debug_flag="true"
|
||||
else
|
||||
debug_flag="false"
|
||||
fi
|
||||
shift 2
|
||||
--debug)
|
||||
if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then
|
||||
debug_flag="true"
|
||||
else
|
||||
debug_flag="false"
|
||||
fi
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
break
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done;
|
||||
esac
|
||||
done
|
||||
|
||||
root_path=$1
|
||||
|
||||
OSName=$(echo "$OSTYPE" | tr '[:upper:]' '[:lower:]')
|
||||
|
||||
color_echo "$Cyan" "Start rebalancing $(date):"
|
||||
color_echo "$Cyan" " Path: ${root_path}"
|
||||
color_echo "$Cyan" " Rebalancing Passes: ${passes_flag}"
|
||||
@ -247,10 +276,10 @@ color_echo "$Cyan" " Use Checksum: ${checksum_flag}"
|
||||
color_echo "$Cyan" " Debug Mode: ${debug_flag}"
|
||||
|
||||
# Generate files_list.txt with device and inode numbers using stat, separated by a pipe '|'
|
||||
if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then
|
||||
if [[ "${OSName}" == "linux-gnu"* ]]; then
|
||||
# Linux
|
||||
find "$root_path" -type f -not -path '*/.zfs/*' -exec stat --printf '%d:%i|%n\n' {} \; > files_list.txt
|
||||
elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then
|
||||
elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then
|
||||
# Mac OS and FreeBSD
|
||||
find "$root_path" -type f -not -path '*/.zfs/*' -exec sh -c 'stat -f "%d:%i|%N" "$0"' {} \; {} \; > files_list.txt
|
||||
else
|
||||
|
||||
Loading…
Reference in New Issue
Block a user