diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4239539..34d1b13 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,11 +9,40 @@ on: branches: [ master ] jobs: - shellcheck: - name: Test + linuxTest: + name: Test on Linux runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Run testing script run: ./testing.sh + + macOsTest: + name: Test on macOS + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - name: Install coreutils + run: brew install coreutils + + - name: Run testing script on macOS + run: ./testing.sh + + FreeBSDTest: + name: Test on FreeBSD + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Test in FreeBSD + id: test + uses: vmactions/freebsd-vm@v1 + with: + usesh: true + prepare: | + pkg install -y bash + run: | + ./testing.sh \ No newline at end of file diff --git a/.gitignore b/.gitignore index a00bc94..629c291 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ test.log error.log rebalance_db.txt -testing_data \ No newline at end of file +testing_data +.vscode \ No newline at end of file diff --git a/testing.sh b/testing.sh index b46f74f..f531e1e 100755 --- a/testing.sh +++ b/testing.sh @@ -9,6 +9,27 @@ log_std_file=./test.log log_error_file=./error.log test_data_src=./test/pool test_pool_data_path=./testing_data +test_pool_data_size_path=$test_pool_data_path/size + +## Color Constants + +# Reset +Color_Off='\033[0m' # Text Reset + +# Regular Colors +Red='\033[0;31m' # Red +Green='\033[0;32m' # Green +Yellow='\033[0;33m' # Yellow +Cyan='\033[0;36m' # Cyan + +## Functions + +# print a given text entirely in a given color +function color_echo () { + color=$1 + text=$2 + echo -e "${color}${text}${Color_Off}" +} function prepare() { # cleanup @@ -21,10 +42,25 @@ function prepare() { cp -rf $test_data_src $test_pool_data_path } +# return time to the milisecond +function get_time() { + + case "$OSTYPE" in + darwin*) + date=$(gdate +%s%N) + ;; + *) + date=$(date +%s%N) + ;; + esac + + echo "$date" +} + function assertions() { # check error log is empty if grep -q '[^[:space:]]' $log_error_file; then - echo "error log is not empty!" + color_echo "$Red" "error log is not empty!" cat $log_error_file exit 1 fi @@ -44,21 +80,38 @@ function assert_matching_file_not_copied() { fi } +function print_time_taken(){ + time_taken=$1 + minute=$((time_taken / 60000)) + seconde=$((time_taken % 60000 / 1000)) + miliseconde=$((time_taken % 1000)) + color_echo "$Yellow" "Time taken: ${minute}m ${seconde}s ${miliseconde}ms" +} + +color_echo "$Cyan" "Running tests..." + +color_echo "$Cyan" "Running tests with default options..." prepare ./zfs-inplace-rebalancing.sh $test_pool_data_path >> $log_std_file 2>> $log_error_file cat $log_std_file assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with checksum true and 1 pass..." prepare ./zfs-inplace-rebalancing.sh --checksum true --passes 1 $test_pool_data_path >> $log_std_file 2>> $log_error_file cat $log_std_file assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with checksum false..." prepare ./zfs-inplace-rebalancing.sh --checksum false $test_pool_data_path >> $log_std_file 2>> $log_error_file cat $log_std_file assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with skip-hardlinks false..." prepare ln "$test_pool_data_path/projects/[2020] some project/mp4.txt" "$test_pool_data_path/projects/[2020] some project/mp4.txt.link" ./zfs-inplace-rebalancing.sh --skip-hardlinks false $test_pool_data_path >> $log_std_file 2>> $log_error_file @@ -67,7 +120,9 @@ cat $log_std_file assert_matching_file_copied "mp4.txt" assert_matching_file_copied "mp4.txt.link" assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with skip-hardlinks true..." prepare ln "$test_pool_data_path/projects/[2020] some project/mp4.txt" "$test_pool_data_path/projects/[2020] some project/mp4.txt.link" ./zfs-inplace-rebalancing.sh --skip-hardlinks true $test_pool_data_path >> $log_std_file 2>> $log_error_file @@ -76,3 +131,63 @@ cat $log_std_file assert_matching_file_not_copied "mp4.txt.link" assert_matching_file_not_copied "mp4.txt" assertions +color_echo "$Green" "Tests passed!" + +color_echo "$Cyan" "Running tests with different file count and size..." +prepare + +mkdir -p $test_pool_data_size_path + +color_echo "$Cyan" "Creating 1000 files of 1KB each..." +mkdir -p $test_pool_data_size_path/small +for i in {1..1000}; do + dd if=/dev/urandom of=$test_pool_data_size_path/small/file_"$i".txt bs=1024 count=1 >> /dev/null 2>&1 +done + +color_echo "$Cyan" "Creating 5 file of 1GB each..." +mkdir -p $test_pool_data_size_path/big +for i in {1..5}; do + dd if=/dev/urandom of=$test_pool_data_size_path/big/file_"$i".txt bs=1024 count=1048576 >> /dev/null 2>&1 +done + +color_echo "$Green" "Files created!" + +echo "Running rebalancing on small files..." +# measure time taken +start_time=$(get_time) +./zfs-inplace-rebalancing.sh $test_pool_data_size_path/small >> $log_std_file 2>> $log_error_file +end_time=$(get_time) +time_taken=$(( (end_time - start_time) / 1000000 )) +print_time_taken $time_taken +assertions +color_echo "$Green" "Tests passed!" + +echo "Running rebalancing on big files..." +rm -f rebalance_db.txt +# measure time taken +start_time=$(get_time) +./zfs-inplace-rebalancing.sh $test_pool_data_size_path/big >> $log_std_file 2>> $log_error_file +end_time=$(get_time) +time_taken=$(( (end_time - start_time) / 1000000 )) +print_time_taken $time_taken +assertions +color_echo "$Green" "Tests passed!" + +echo "Running rebalancing on all files..." +rm -f rebalance_db.txt +# measure time taken +start_time=$(get_time) +./zfs-inplace-rebalancing.sh $test_pool_data_size_path >> $log_std_file 2>> $log_error_file +end_time=$(get_time) +time_taken=$(( (end_time - start_time) / 1000000 )) +print_time_taken $time_taken +assertions +color_echo "$Green" "Tests passed!" + +color_echo "$Green" "All tests passed!" +color_echo "$Cyan" "Cleaning" +rm -f $log_std_file +rm -f $log_error_file +rm -f rebalance_db.txt +rm -rf $test_pool_data_path + diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index 2346aad..a8cf824 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -14,29 +14,29 @@ current_index=0 ## Color Constants # Reset -Color_Off='\033[0m' # Text Reset +Color_Off='\033[0m' # Text Reset # Regular Colors -Red='\033[0;31m' # Red -Green='\033[0;32m' # Green -Yellow='\033[0;33m' # Yellow -Cyan='\033[0;36m' # Cyan +Red='\033[0;31m' # Red +Green='\033[0;32m' # Green +Yellow='\033[0;33m' # Yellow +Cyan='\033[0;36m' # Cyan ## Functions # Print a help message function print_usage() { - echo "Usage: zfs-inplace-rebalancing.sh --checksum true --passes 1 --debug false /my/pool" + echo "Usage: zfs-inplace-rebalancing.sh --checksum true --passes 1 --debug false /my/pool" } # Print a given text entirely in a given color -function color_echo () { +function color_echo() { color=$1 text=$2 echo -e "${color}${text}${Color_Off}" } -function get_rebalance_count () { +function get_rebalance_count() { file_path="$1" line_nr=$(grep -xF -n -e "${file_path}" "./${rebalance_db_file_name}" | head -n 1 | cut -d: -f1) @@ -103,14 +103,14 @@ function process_inode_group() { if [ "$debug_flag" = true ]; then echo "Executing copy command:" fi - if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then + if [[ "${OSName}" == "linux-gnu"* ]]; then # Linux cmd=(cp --reflink=never -ax "${main_file}" "${tmp_file_path}") if [ "$debug_flag" = true ]; then echo "${cmd[@]}" fi "${cmd[@]}" - elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then + elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then # Mac OS and FreeBSD cmd=(cp -ax "${main_file}" "${tmp_file_path}") if [ "$debug_flag" = true ]; then @@ -123,30 +123,57 @@ function process_inode_group() { fi # Compare copy against original to make sure nothing went wrong - if [[ "${checksum_flag,,}" == "true"* ]]; then + if [[ "${checksum_flag}" == "true"* ]]; then echo "Comparing copy against original..." - if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then + if [[ "${OSName}" == "linux-gnu"* ]]; then # Linux - original_md5=$(md5sum -b "${main_file}" | awk '{print $1}') - copy_md5=$(md5sum -b "${tmp_file_path}" | awk '{print $1}') - elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then - # Mac OS and FreeBSD - original_md5=$(md5 -q "${main_file}") - copy_md5=$(md5 -q "${tmp_file_path}") + + # file attributes + original_perms=$(lsattr "${main_file}") + # remove anything after the last space + original_perms=${original_perms% *} + # file permissions, owner, group, size, modification time + original_perms="${original_perms} $(stat -c "%A %U %G %s %Y" "${main_file}")" + + + # file attributes + copy_perms=$(lsattr "${tmp_file_path}") + # remove anything after the last space + copy_perms=${copy_perms% *} + # file permissions, owner, group, size, modification time + copy_perms="${copy_perms} $(stat -c "%A %U %G %s %Y" "${tmp_file_path}")" + elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then + # Mac OS + # FreeBSD + + # note: no lsattr on Mac OS or FreeBSD + + # file permissions, owner, group size, modification time + original_perms="$(stat -f "%Sp %Su %Sg %z %m" "${main_file}")" + + # file permissions, owner, group size, modification time + copy_perms="$(stat -f "%Sp %Su %Sg %z %m" "${tmp_file_path}")" else echo "Unsupported OS type: $OSTYPE" exit 1 fi if [ "$debug_flag" = true ]; then - echo "Original MD5: $original_md5" - echo "Copy MD5: $copy_md5" + echo "Original perms: $original_perms" + echo "Copy perms: $copy_perms" fi - if [[ "${original_md5}" == "${copy_md5}" ]]; then - color_echo "${Green}" "MD5 OK" + if [[ "${original_perms}" == "${copy_perms}"* ]]; then + color_echo "${Green}" "Attribute and permission check OK" else - color_echo "${Red}" "MD5 FAILED: ${original_md5} != ${copy_md5}" + color_echo "${Red}" "Attribute and permission check FAILED: ${original_perms} != ${copy_perms}" + exit 1 + fi + + if cmp -s "${main_file}" "${tmp_file_path}"; then + color_echo "${Green}" "File content check OK" + else + color_echo "${Red}" "File content check FAILED" exit 1 fi fi @@ -206,40 +233,42 @@ if [[ "$#" -eq 0 ]]; then exit 0 fi -while true ; do +while true; do case "$1" in - -h | --help ) - print_usage - exit 0 + -h | --help) + print_usage + exit 0 ;; - -c | --checksum ) - if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then - checksum_flag="true" - else - checksum_flag="false" - fi - shift 2 + -c | --checksum) + if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then + checksum_flag="true" + else + checksum_flag="false" + fi + shift 2 ;; - -p | --passes ) - passes_flag=$2 - shift 2 + -p | --passes) + passes_flag=$2 + shift 2 ;; - --debug ) - if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then - debug_flag="true" - else - debug_flag="false" - fi - shift 2 + --debug) + if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then + debug_flag="true" + else + debug_flag="false" + fi + shift 2 ;; - *) - break + *) + break ;; - esac -done; + esac +done root_path=$1 +OSName=$(echo "$OSTYPE" | tr '[:upper:]' '[:lower:]') + color_echo "$Cyan" "Start rebalancing $(date):" color_echo "$Cyan" " Path: ${root_path}" color_echo "$Cyan" " Rebalancing Passes: ${passes_flag}" @@ -247,10 +276,10 @@ color_echo "$Cyan" " Use Checksum: ${checksum_flag}" color_echo "$Cyan" " Debug Mode: ${debug_flag}" # Generate files_list.txt with device and inode numbers using stat, separated by a pipe '|' -if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then +if [[ "${OSName}" == "linux-gnu"* ]]; then # Linux find "$root_path" -type f -not -path '*/.zfs/*' -exec stat --printf '%d:%i|%n\n' {} \; > files_list.txt -elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then +elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then # Mac OS and FreeBSD find "$root_path" -type f -not -path '*/.zfs/*' -exec sh -c 'stat -f "%d:%i|%N" "$0"' {} \; {} \; > files_list.txt else