From 922531829ef5fc56ef3b8220e5a2cb1289947b5a Mon Sep 17 00:00:00 2001 From: meo <83820503+meo-pill@users.noreply.github.com> Date: Sat, 14 Dec 2024 02:13:11 +0100 Subject: [PATCH 1/8] refactor: add regularity to spaces (#52) 1) switch all the miscellaneous indentation to 4 spaces 2) remove the space before the parenthesis of function 3) add indentation inside of if 4) remove miscellaneous spaces at the end of lines --- zfs-inplace-rebalancing.sh | 139 ++++++++++++++++++------------------- 1 file changed, 69 insertions(+), 70 deletions(-) diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index b9fe6ce..07fb3ac 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -14,30 +14,29 @@ current_index=0 ## Color Constants # Reset -Color_Off='\033[0m' # Text Reset +Color_Off='\033[0m' # Text Reset # Regular Colors -Red='\033[0;31m' # Red -Green='\033[0;32m' # Green -Yellow='\033[0;33m' # Yellow -Cyan='\033[0;36m' # Cyan +Red='\033[0;31m' # Red +Green='\033[0;32m' # Green +Yellow='\033[0;33m' # Yellow +Cyan='\033[0;36m' # Cyan ## Functions # print a help message function print_usage() { - echo "Usage: zfs-inplace-rebalancing --checksum true --skip-hardlinks false --passes 1 /my/pool" + echo "Usage: zfs-inplace-rebalancing --checksum true --skip-hardlinks false --passes 1 /my/pool" } # print a given text entirely in a given color -function color_echo () { +function color_echo() { color=$1 text=$2 echo -e "${color}${text}${Color_Off}" } - -function get_rebalance_count () { +function get_rebalance_count() { file_path=$1 line_nr=$(grep -xF -n "${file_path}" "./${rebalance_db_file_name}" | head -n 1 | cut -d: -f1) @@ -53,58 +52,58 @@ function get_rebalance_count () { } # rebalance a specific file -function rebalance () { +function rebalance() { file_path=$1 # check if file has >=2 links in the case of --skip-hardlinks # this shouldn't be needed in the typical case of `find` only finding files with links == 1 # but this can run for a long time, so it's good to double check if something changed if [[ "${skip_hardlinks_flag,,}" == "true"* ]]; then - if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then - # Linux - # - # -c --format=FORMAT - # use the specified FORMAT instead of the default; output a - # newline after each use of FORMAT - # %h number of hard links + if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then + # Linux + # + # -c --format=FORMAT + # use the specified FORMAT instead of the default; output a + # newline after each use of FORMAT + # %h number of hard links - hardlink_count=$(stat -c "%h" "${file_path}") - elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then - # Mac OS - # FreeBSD - # -f format - # Display information using the specified format - # l Number of hard links to file (st_nlink) + hardlink_count=$(stat -c "%h" "${file_path}") + elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then + # Mac OS + # FreeBSD + # -f format + # Display information using the specified format + # l Number of hard links to file (st_nlink) - hardlink_count=$(stat -f %l "${file_path}") - else - echo "Unsupported OS type: $OSTYPE" - exit 1 - fi + hardlink_count=$(stat -f %l "${file_path}") + else + echo "Unsupported OS type: $OSTYPE" + exit 1 + fi - if [ "${hardlink_count}" -ge 2 ]; then + if [ "${hardlink_count}" -ge 2 ]; then echo "Skipping hard-linked file: ${file_path}" return fi fi current_index="$((current_index + 1))" - progress_percent=$(printf '%0.2f' "$((current_index*10000/file_count))e-2") - color_echo "${Cyan}" "Progress -- Files: ${current_index}/${file_count} (${progress_percent}%)" + progress_percent=$(printf '%0.2f' "$((current_index * 10000 / file_count))e-2") + color_echo "${Cyan}" "Progress -- Files: ${current_index}/${file_count} (${progress_percent}%)" if [[ ! -f "${file_path}" ]]; then - color_echo "${Yellow}" "File is missing, skipping: ${file_path}" + color_echo "${Yellow}" "File is missing, skipping: ${file_path}" fi if [ "${passes_flag}" -ge 1 ]; then # check if target rebalance count is reached rebalance_count=$(get_rebalance_count "${file_path}") if [ "${rebalance_count}" -ge "${passes_flag}" ]; then - color_echo "${Yellow}" "Rebalance count (${passes_flag}) reached, skipping: ${file_path}" - return + color_echo "${Yellow}" "Rebalance count (${passes_flag}) reached, skipping: ${file_path}" + return fi fi - + tmp_extension=".balance" tmp_file_path="${file_path}${tmp_extension}" @@ -113,7 +112,7 @@ function rebalance () { # Linux # --reflink=never -- force standard copy (see ZFS Block Cloning) - # -a -- keep attributes, includes -d -- keep symlinks (dont copy target) and + # -a -- keep attributes, includes -d -- keep symlinks (dont copy target) and # -p -- preserve ACLs to # -x -- stay on one system cp --reflink=never -ax "${file_path}" "${tmp_file_path}" @@ -121,8 +120,8 @@ function rebalance () { # Mac OS # FreeBSD - # -a -- Archive mode. Same as -RpP. Includes preservation of modification - # time, access time, file flags, file mode, ACL, user ID, and group + # -a -- Archive mode. Same as -RpP. Includes preservation of modification + # time, access time, file flags, file mode, ACL, user ID, and group # ID, as allowed by permissions. # -x -- File system mount points are not traversed. cp -ax "${file_path}" "${tmp_file_path}" @@ -194,13 +193,13 @@ function rebalance () { # update rebalance "database" line_nr=$(grep -xF -n "${file_path}" "./${rebalance_db_file_name}" | head -n 1 | cut -d: -f1) if [ -z "${line_nr}" ]; then - rebalance_count=1 - echo "${file_path}" >> "./${rebalance_db_file_name}" - echo "${rebalance_count}" >> "./${rebalance_db_file_name}" + rebalance_count=1 + echo "${file_path}" >>"./${rebalance_db_file_name}" + echo "${rebalance_count}" >>"./${rebalance_db_file_name}" else - rebalance_count_line_nr="$((line_nr + 1))" - rebalance_count="$((rebalance_count + 1))" - sed -i '' "${rebalance_count_line_nr}s/.*/${rebalance_count}/" "./${rebalance_db_file_name}" + rebalance_count_line_nr="$((line_nr + 1))" + rebalance_count="$((rebalance_count + 1))" + sed -i '' "${rebalance_count_line_nr}s/.*/${rebalance_count}/" "./${rebalance_db_file_name}" fi fi } @@ -214,37 +213,37 @@ if [[ "$#" -eq 0 ]]; then exit 0 fi -while true ; do +while true; do case "$1" in - -h | --help ) - print_usage - exit 0 + -h | --help) + print_usage + exit 0 ;; - -c | --checksum ) - if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then - checksum_flag="true" - else - checksum_flag="false" - fi - shift 2 + -c | --checksum) + if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then + checksum_flag="true" + else + checksum_flag="false" + fi + shift 2 ;; - --skip-hardlinks ) - if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then - skip_hardlinks_flag="true" - else - skip_hardlinks_flag="false" - fi - shift 2 + --skip-hardlinks) + if [[ "$2" == 1 || "$2" =~ (on|true|yes) ]]; then + skip_hardlinks_flag="true" + else + skip_hardlinks_flag="false" + fi + shift 2 ;; - -p | --passes ) - passes_flag=$2 - shift 2 + -p | --passes) + passes_flag=$2 + shift 2 ;; - *) - break + *) + break ;; - esac -done; + esac +done root_path=$1 From bd8d447150419337bc49f0872038cb0da59c4203 Mon Sep 17 00:00:00 2001 From: meo <83820503+meo-pill@users.noreply.github.com> Date: Tue, 17 Dec 2024 00:33:35 +0100 Subject: [PATCH 2/8] Test mac os (#53) * add: test on MacOS Add test for MacOS in Github action Maybe see this to add test for FreeBSD : https://github.com/marketplace/actions/freebsd-vm * add: log to test adding log to the test for better understanding * fix: shellcheck remove the yellow color that is unused in the test * fix: fail on lower-casting the `"${variable,,}"` seam to not work on macOS so y remove it for the `skip_hardlink_flag` because it's always lowercase * fix: macOS remove all the lower-case casting * fix(perf): going to `stat` remove `lsattr` macOS The command `lsattr` is not supported on macOS so I have remove it switch to `stat` instead of `ls` for better performance and no subshell use of parameter expansion with `%` in place of `awk` for performance and no subshell * fix: forgot to remove variable --- .github/workflows/test.yml | 13 ++++++-- testing.sh | 35 ++++++++++++++++++++- zfs-inplace-rebalancing.sh | 63 ++++++++++++++++++++------------------ 3 files changed, 78 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4239539..8bc609f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,11 +9,20 @@ on: branches: [ master ] jobs: - shellcheck: - name: Test + linuxTest: + name: Test on Linux runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Run testing script run: ./testing.sh + + macOsTest: + name: Test on macOS + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - name: Run testing script on macOS + run: ./testing.sh \ No newline at end of file diff --git a/testing.sh b/testing.sh index b46f74f..4fe7333 100755 --- a/testing.sh +++ b/testing.sh @@ -10,6 +10,25 @@ log_error_file=./error.log test_data_src=./test/pool test_pool_data_path=./testing_data +## Color Constants + +# Reset +Color_Off='\033[0m' # Text Reset + +# Regular Colors +Red='\033[0;31m' # Red +Green='\033[0;32m' # Green +Cyan='\033[0;36m' # Cyan + +## Functions + +# print a given text entirely in a given color +function color_echo () { + color=$1 + text=$2 + echo -e "${color}${text}${Color_Off}" +} + function prepare() { # cleanup rm -f $log_std_file @@ -24,7 +43,7 @@ function prepare() { function assertions() { # check error log is empty if grep -q '[^[:space:]]' $log_error_file; then - echo "error log is not empty!" + color_echo "$Red" "error log is not empty!" cat $log_error_file exit 1 fi @@ -44,21 +63,30 @@ function assert_matching_file_not_copied() { fi } +color_echo "$Cyan" "Running tests..." + +color_echo "$Cyan" "Running tests with default options..." prepare ./zfs-inplace-rebalancing.sh $test_pool_data_path >> $log_std_file 2>> $log_error_file cat $log_std_file assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with checksum true and 1 pass..." prepare ./zfs-inplace-rebalancing.sh --checksum true --passes 1 $test_pool_data_path >> $log_std_file 2>> $log_error_file cat $log_std_file assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with checksum false..." prepare ./zfs-inplace-rebalancing.sh --checksum false $test_pool_data_path >> $log_std_file 2>> $log_error_file cat $log_std_file assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with skip-hardlinks false..." prepare ln "$test_pool_data_path/projects/[2020] some project/mp4.txt" "$test_pool_data_path/projects/[2020] some project/mp4.txt.link" ./zfs-inplace-rebalancing.sh --skip-hardlinks false $test_pool_data_path >> $log_std_file 2>> $log_error_file @@ -67,7 +95,9 @@ cat $log_std_file assert_matching_file_copied "mp4.txt" assert_matching_file_copied "mp4.txt.link" assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with skip-hardlinks true..." prepare ln "$test_pool_data_path/projects/[2020] some project/mp4.txt" "$test_pool_data_path/projects/[2020] some project/mp4.txt.link" ./zfs-inplace-rebalancing.sh --skip-hardlinks true $test_pool_data_path >> $log_std_file 2>> $log_error_file @@ -76,3 +106,6 @@ cat $log_std_file assert_matching_file_not_copied "mp4.txt.link" assert_matching_file_not_copied "mp4.txt" assertions +color_echo "$Green" "Tests passed!" + +color_echo "$Green" "All tests passed!" \ No newline at end of file diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index 07fb3ac..262d0f6 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -58,8 +58,8 @@ function rebalance() { # check if file has >=2 links in the case of --skip-hardlinks # this shouldn't be needed in the typical case of `find` only finding files with links == 1 # but this can run for a long time, so it's good to double check if something changed - if [[ "${skip_hardlinks_flag,,}" == "true"* ]]; then - if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then + if [[ "${skip_hardlinks_flag}" == "true"* ]]; then + if [[ "${OSTYPE}" == "linux-gnu"* ]]; then # Linux # # -c --format=FORMAT @@ -68,7 +68,7 @@ function rebalance() { # %h number of hard links hardlink_count=$(stat -c "%h" "${file_path}") - elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then + elif [[ "${OSTYPE}" == "darwin"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then # Mac OS # FreeBSD # -f format @@ -108,7 +108,7 @@ function rebalance() { tmp_file_path="${file_path}${tmp_extension}" echo "Copying '${file_path}' to '${tmp_file_path}'..." - if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then + if [[ "${OSTYPE}" == "linux-gnu"* ]]; then # Linux # --reflink=never -- force standard copy (see ZFS Block Cloning) @@ -116,7 +116,7 @@ function rebalance() { # -p -- preserve ACLs to # -x -- stay on one system cp --reflink=never -ax "${file_path}" "${tmp_file_path}" - elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then + elif [[ "${OSTYPE}" == "darwin"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then # Mac OS # FreeBSD @@ -131,45 +131,48 @@ function rebalance() { fi # compare copy against original to make sure nothing went wrong - if [[ "${checksum_flag,,}" == "true"* ]]; then + if [[ "${checksum_flag}" == "true"* ]]; then echo "Comparing copy against original..." - if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then + if [[ "${OSTYPE}" == "linux-gnu"* ]]; then # Linux # file attributes - original_md5=$(lsattr "${file_path}" | awk '{print $1}') - # file permissions, owner, group - # shellcheck disable=SC2012 - original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')" + original_md5=$(lsattr "${file_path}") + # remove anything after the last space + original_md5=${original_md5% *} + # file permissions, owner, group, size, modification time + original_md5="${original_md5} $(stat -c "%A %U %G %s %Y" "${file_path}")" # file content - original_md5="${original_md5} $(md5sum -b "${file_path}" | awk '{print $1}')" + original_md5="${original_md5} $(md5sum -b "${file_path}")" + # file attributes - copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}') - # file permissions, owner, group - # shellcheck disable=SC2012 - copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')" + copy_md5=$(lsattr "${tmp_file_path}") + # remove anything after the last space + copy_md5=${copy_md5% *} + # file permissions, owner, group, size, modification time + copy_md5="${copy_md5} $(stat -c "%A %U %G %s %Y" "${tmp_file_path}")" # file content - copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}" | awk '{print $1}')" - elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then + copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}")" + # remove the temporary extension + copy_md5=${copy_md5%"${tmp_extension}"} + elif [[ "${OSTYPE}" == "darwin"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then # Mac OS # FreeBSD - # file attributes - original_md5=$(lsattr "${file_path}" | awk '{print $1}') - # file permissions, owner, group - # shellcheck disable=SC2012 - original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')" + # note: no lsattr on Mac OS or FreeBSD + + # file permissions, owner, group size, modification time + original_md5="$(stat -f "%Sp %Su %Sg %z %m" "${file_path}")" # file content original_md5="${original_md5} $(md5 -q "${file_path}")" - # file attributes - copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}') - # file permissions, owner, group - # shellcheck disable=SC2012 - copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')" + # file permissions, owner, group size, modification time + copy_md5="$(stat -f "%Sp %Su %Sg %z %m" "${tmp_file_path}")" # file content copy_md5="${copy_md5} $(md5 -q "${tmp_file_path}")" + # remove the temporary extension + copy_md5=${copy_md5%"${tmp_extension}"} else echo "Unsupported OS type: $OSTYPE" exit 1 @@ -254,7 +257,7 @@ color_echo "$Cyan" " Use Checksum: ${checksum_flag}" color_echo "$Cyan" " Skip Hardlinks: ${skip_hardlinks_flag}" # count files -if [[ "${skip_hardlinks_flag,,}" == "true"* ]]; then +if [[ "${skip_hardlinks_flag}" == "true"* ]]; then file_count=$(find "${root_path}" -type f -links 1 | wc -l) else file_count=$(find "${root_path}" -type f | wc -l) @@ -269,7 +272,7 @@ fi # recursively scan through files and execute "rebalance" procedure # in the case of --skip-hardlinks, only find files with links == 1 -if [[ "${skip_hardlinks_flag,,}" == "true"* ]]; then +if [[ "${skip_hardlinks_flag}" == "true"* ]]; then find "$root_path" -type f -links 1 -print0 | while IFS= read -r -d '' file; do rebalance "$file"; done else find "$root_path" -type f -print0 | while IFS= read -r -d '' file; do rebalance "$file"; done From 393cbe2a12c8d429e4205e280b0fdf35b5a437b8 Mon Sep 17 00:00:00 2001 From: meo <83820503+meo-pill@users.noreply.github.com> Date: Mon, 6 Jan 2025 00:43:43 +0100 Subject: [PATCH 3/8] FreeBSD fix and test (#55) * add: test on freebsd * test fix vm * fix: transform to lowercase transform the `OSTYPE` to a lowercase and store it inside `OSName` * add: caching of the vm --- .github/workflows/test.yml | 27 ++++++++++++++++++++++++++- zfs-inplace-rebalancing.sh | 14 ++++++++------ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8bc609f..02e3b01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,4 +25,29 @@ jobs: - uses: actions/checkout@v4 - name: Run testing script on macOS - run: ./testing.sh \ No newline at end of file + run: ./testing.sh + + FreeBSDTest: + name: Test on FreeBSD + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: ~/.cache + key: ${{ runner.os }}-cache-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-cache- + + - name: Test in FreeBSD + id: test + uses: vmactions/freebsd-vm@v1 + with: + usesh: true + prepare: | + pkg install -y bash + run: | + ./testing.sh \ No newline at end of file diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index 262d0f6..9497103 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -59,7 +59,7 @@ function rebalance() { # this shouldn't be needed in the typical case of `find` only finding files with links == 1 # but this can run for a long time, so it's good to double check if something changed if [[ "${skip_hardlinks_flag}" == "true"* ]]; then - if [[ "${OSTYPE}" == "linux-gnu"* ]]; then + if [[ "${OSName}" == "linux-gnu"* ]]; then # Linux # # -c --format=FORMAT @@ -68,7 +68,7 @@ function rebalance() { # %h number of hard links hardlink_count=$(stat -c "%h" "${file_path}") - elif [[ "${OSTYPE}" == "darwin"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then + elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then # Mac OS # FreeBSD # -f format @@ -108,7 +108,7 @@ function rebalance() { tmp_file_path="${file_path}${tmp_extension}" echo "Copying '${file_path}' to '${tmp_file_path}'..." - if [[ "${OSTYPE}" == "linux-gnu"* ]]; then + if [[ "${OSName}" == "linux-gnu"* ]]; then # Linux # --reflink=never -- force standard copy (see ZFS Block Cloning) @@ -116,7 +116,7 @@ function rebalance() { # -p -- preserve ACLs to # -x -- stay on one system cp --reflink=never -ax "${file_path}" "${tmp_file_path}" - elif [[ "${OSTYPE}" == "darwin"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then + elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then # Mac OS # FreeBSD @@ -133,7 +133,7 @@ function rebalance() { # compare copy against original to make sure nothing went wrong if [[ "${checksum_flag}" == "true"* ]]; then echo "Comparing copy against original..." - if [[ "${OSTYPE}" == "linux-gnu"* ]]; then + if [[ "${OSName}" == "linux-gnu"* ]]; then # Linux # file attributes @@ -156,7 +156,7 @@ function rebalance() { copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}")" # remove the temporary extension copy_md5=${copy_md5%"${tmp_extension}"} - elif [[ "${OSTYPE}" == "darwin"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then + elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then # Mac OS # FreeBSD @@ -250,6 +250,8 @@ done root_path=$1 +OSName=$(echo "$OSTYPE" | tr '[:upper:]' '[:lower:]') + color_echo "$Cyan" "Start rebalancing $(date):" color_echo "$Cyan" " Path: ${root_path}" color_echo "$Cyan" " Rebalancing Passes: ${passes_flag}" From 21d718259e355ccd37f0e0e3ed88f006be49330c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:39:07 +0100 Subject: [PATCH 4/8] Bump actions/cache from 3 to 4 (#58) Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 02e3b01..455b0f3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v4 - name: Cache dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache key: ${{ runner.os }}-cache-${{ hashFiles('**/package-lock.json') }} From 9e5aadff04af3825b9c1b23913c07a6ab2e7e649 Mon Sep 17 00:00:00 2001 From: meo <83820503+meo-pill@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:58:26 +0100 Subject: [PATCH 5/8] fix: remove the caching (#60) the caching for the vm do not work so i remove it --- .github/workflows/test.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 455b0f3..58970d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,14 +33,6 @@ jobs: steps: - uses: actions/checkout@v4 - - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: ~/.cache - key: ${{ runner.os }}-cache-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-cache- - name: Test in FreeBSD id: test From 9b233fdeaff7b92a281989494c7ebf352210d033 Mon Sep 17 00:00:00 2001 From: Adi Date: Thu, 9 Jan 2025 16:00:01 +0000 Subject: [PATCH 6/8] Replace MD5 with CRC32 for 2x checksum speed (#57) * Replace MD5 with CRC32 for 2x checksum speed * Use % builtin instead of awk for better performance --- zfs-inplace-rebalancing.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index 9497103..9c280df 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -137,25 +137,25 @@ function rebalance() { # Linux # file attributes - original_md5=$(lsattr "${file_path}") + original_checksum=$(lsattr "${file_path}") # remove anything after the last space - original_md5=${original_md5% *} + original_checksum=${original_checksum% *} # file permissions, owner, group, size, modification time - original_md5="${original_md5} $(stat -c "%A %U %G %s %Y" "${file_path}")" + original_checksum="${original_checksum} $(stat -c "%A %U %G %s %Y" "${file_path}")" # file content - original_md5="${original_md5} $(md5sum -b "${file_path}")" + original_checksum="${original_checksum} $(cksum "${file_path}")" # file attributes - copy_md5=$(lsattr "${tmp_file_path}") + copy_checksum=$(lsattr "${tmp_file_path}") # remove anything after the last space - copy_md5=${copy_md5% *} + copy_checksum=${copy_checksum% *} # file permissions, owner, group, size, modification time - copy_md5="${copy_md5} $(stat -c "%A %U %G %s %Y" "${tmp_file_path}")" + copy_checksum="${copy_checksum} $(stat -c "%A %U %G %s %Y" "${tmp_file_path}")" # file content - copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}")" + copy_checksum="${copy_checksum} $(cksum "${file_path}")" # remove the temporary extension - copy_md5=${copy_md5%"${tmp_extension}"} + copy_checksum=${copy_checksum%"${tmp_extension}"} elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then # Mac OS # FreeBSD @@ -163,25 +163,25 @@ function rebalance() { # note: no lsattr on Mac OS or FreeBSD # file permissions, owner, group size, modification time - original_md5="$(stat -f "%Sp %Su %Sg %z %m" "${file_path}")" + original_checksum="$(stat -f "%Sp %Su %Sg %z %m" "${file_path}")" # file content - original_md5="${original_md5} $(md5 -q "${file_path}")" + original_checksum="${original_checksum} $(cksum "${file_path}")" # file permissions, owner, group size, modification time - copy_md5="$(stat -f "%Sp %Su %Sg %z %m" "${tmp_file_path}")" + copy_checksum="$(stat -f "%Sp %Su %Sg %z %m" "${tmp_file_path}")" # file content - copy_md5="${copy_md5} $(md5 -q "${tmp_file_path}")" + copy_checksum="${copy_checksum} $(cksum "${file_path}")" # remove the temporary extension - copy_md5=${copy_md5%"${tmp_extension}"} + copy_checksum=${copy_checksum%"${tmp_extension}"} else echo "Unsupported OS type: $OSTYPE" exit 1 fi - if [[ "${original_md5}" == "${copy_md5}"* ]]; then - color_echo "${Green}" "MD5 OK" + if [[ "${original_checksum}" == "${copy_checksum}"* ]]; then + color_echo "${Green}" "Checksum OK" else - color_echo "${Red}" "MD5 FAILED: ${original_md5} != ${copy_md5}" + color_echo "${Red}" "Checksum FAILED: ${original_checksum} != ${copy_checksum}" exit 1 fi fi From d190faf49ce25096ab03ea7d583e4773a9907944 Mon Sep 17 00:00:00 2001 From: meo <83820503+meo-pill@users.noreply.github.com> Date: Fri, 17 Jan 2025 22:50:33 +0100 Subject: [PATCH 7/8] Expand test (#59) * add(ignore): ignore of .vscode file add the vscode config file to gitignore * add(test): add set of test - test 1000 1Ko file - test 5 1 Go file - test whit both * fix: spellcheck prevent wordspliting * fix: remove of db remove the db before testing for more accurate time * fix: time in macOS and freeBSD - use of gdate (coreutils) for macOS - switch to `ns` in place of `ms` for freeBSD --- .github/workflows/test.yml | 3 ++ .gitignore | 3 +- testing.sh | 84 +++++++++++++++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 58970d8..34d1b13 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install coreutils + run: brew install coreutils + - name: Run testing script on macOS run: ./testing.sh diff --git a/.gitignore b/.gitignore index a00bc94..629c291 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ test.log error.log rebalance_db.txt -testing_data \ No newline at end of file +testing_data +.vscode \ No newline at end of file diff --git a/testing.sh b/testing.sh index 4fe7333..f531e1e 100755 --- a/testing.sh +++ b/testing.sh @@ -9,6 +9,7 @@ log_std_file=./test.log log_error_file=./error.log test_data_src=./test/pool test_pool_data_path=./testing_data +test_pool_data_size_path=$test_pool_data_path/size ## Color Constants @@ -18,6 +19,7 @@ Color_Off='\033[0m' # Text Reset # Regular Colors Red='\033[0;31m' # Red Green='\033[0;32m' # Green +Yellow='\033[0;33m' # Yellow Cyan='\033[0;36m' # Cyan ## Functions @@ -40,6 +42,21 @@ function prepare() { cp -rf $test_data_src $test_pool_data_path } +# return time to the milisecond +function get_time() { + + case "$OSTYPE" in + darwin*) + date=$(gdate +%s%N) + ;; + *) + date=$(date +%s%N) + ;; + esac + + echo "$date" +} + function assertions() { # check error log is empty if grep -q '[^[:space:]]' $log_error_file; then @@ -63,6 +80,14 @@ function assert_matching_file_not_copied() { fi } +function print_time_taken(){ + time_taken=$1 + minute=$((time_taken / 60000)) + seconde=$((time_taken % 60000 / 1000)) + miliseconde=$((time_taken % 1000)) + color_echo "$Yellow" "Time taken: ${minute}m ${seconde}s ${miliseconde}ms" +} + color_echo "$Cyan" "Running tests..." color_echo "$Cyan" "Running tests with default options..." @@ -108,4 +133,61 @@ assert_matching_file_not_copied "mp4.txt" assertions color_echo "$Green" "Tests passed!" -color_echo "$Green" "All tests passed!" \ No newline at end of file +color_echo "$Cyan" "Running tests with different file count and size..." +prepare + +mkdir -p $test_pool_data_size_path + +color_echo "$Cyan" "Creating 1000 files of 1KB each..." +mkdir -p $test_pool_data_size_path/small +for i in {1..1000}; do + dd if=/dev/urandom of=$test_pool_data_size_path/small/file_"$i".txt bs=1024 count=1 >> /dev/null 2>&1 +done + +color_echo "$Cyan" "Creating 5 file of 1GB each..." +mkdir -p $test_pool_data_size_path/big +for i in {1..5}; do + dd if=/dev/urandom of=$test_pool_data_size_path/big/file_"$i".txt bs=1024 count=1048576 >> /dev/null 2>&1 +done + +color_echo "$Green" "Files created!" + +echo "Running rebalancing on small files..." +# measure time taken +start_time=$(get_time) +./zfs-inplace-rebalancing.sh $test_pool_data_size_path/small >> $log_std_file 2>> $log_error_file +end_time=$(get_time) +time_taken=$(( (end_time - start_time) / 1000000 )) +print_time_taken $time_taken +assertions +color_echo "$Green" "Tests passed!" + +echo "Running rebalancing on big files..." +rm -f rebalance_db.txt +# measure time taken +start_time=$(get_time) +./zfs-inplace-rebalancing.sh $test_pool_data_size_path/big >> $log_std_file 2>> $log_error_file +end_time=$(get_time) +time_taken=$(( (end_time - start_time) / 1000000 )) +print_time_taken $time_taken +assertions +color_echo "$Green" "Tests passed!" + +echo "Running rebalancing on all files..." +rm -f rebalance_db.txt +# measure time taken +start_time=$(get_time) +./zfs-inplace-rebalancing.sh $test_pool_data_size_path >> $log_std_file 2>> $log_error_file +end_time=$(get_time) +time_taken=$(( (end_time - start_time) / 1000000 )) +print_time_taken $time_taken +assertions +color_echo "$Green" "Tests passed!" + +color_echo "$Green" "All tests passed!" +color_echo "$Cyan" "Cleaning" +rm -f $log_std_file +rm -f $log_error_file +rm -f rebalance_db.txt +rm -rf $test_pool_data_path + From 97d11fdb0b3f1e6bad22f84094e926390fb32db0 Mon Sep 17 00:00:00 2001 From: Adi Date: Sat, 18 Jan 2025 12:48:12 +0200 Subject: [PATCH 8/8] 1.5x speed increase by using cmp instead of crc32 (#61) * 2x speed increase by using cmp instead of crc32 * Remove redundant cmp check --- zfs-inplace-rebalancing.sh | 41 +++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index 9c280df..43ae85d 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -137,25 +137,19 @@ function rebalance() { # Linux # file attributes - original_checksum=$(lsattr "${file_path}") + original_perms=$(lsattr "${file_path}") # remove anything after the last space - original_checksum=${original_checksum% *} + original_perms=${original_perms% *} # file permissions, owner, group, size, modification time - original_checksum="${original_checksum} $(stat -c "%A %U %G %s %Y" "${file_path}")" - # file content - original_checksum="${original_checksum} $(cksum "${file_path}")" + original_perms="${original_perms} $(stat -c "%A %U %G %s %Y" "${file_path}")" # file attributes - copy_checksum=$(lsattr "${tmp_file_path}") + copy_perms=$(lsattr "${tmp_file_path}") # remove anything after the last space - copy_checksum=${copy_checksum% *} + copy_perms=${copy_perms% *} # file permissions, owner, group, size, modification time - copy_checksum="${copy_checksum} $(stat -c "%A %U %G %s %Y" "${tmp_file_path}")" - # file content - copy_checksum="${copy_checksum} $(cksum "${file_path}")" - # remove the temporary extension - copy_checksum=${copy_checksum%"${tmp_extension}"} + copy_perms="${copy_perms} $(stat -c "%A %U %G %s %Y" "${tmp_file_path}")" elif [[ "${OSName}" == "darwin"* ]] || [[ "${OSName}" == "freebsd"* ]]; then # Mac OS # FreeBSD @@ -163,25 +157,26 @@ function rebalance() { # note: no lsattr on Mac OS or FreeBSD # file permissions, owner, group size, modification time - original_checksum="$(stat -f "%Sp %Su %Sg %z %m" "${file_path}")" - # file content - original_checksum="${original_checksum} $(cksum "${file_path}")" + original_perms="$(stat -f "%Sp %Su %Sg %z %m" "${file_path}")" # file permissions, owner, group size, modification time - copy_checksum="$(stat -f "%Sp %Su %Sg %z %m" "${tmp_file_path}")" - # file content - copy_checksum="${copy_checksum} $(cksum "${file_path}")" - # remove the temporary extension - copy_checksum=${copy_checksum%"${tmp_extension}"} + copy_perms="$(stat -f "%Sp %Su %Sg %z %m" "${tmp_file_path}")" else echo "Unsupported OS type: $OSTYPE" exit 1 fi - if [[ "${original_checksum}" == "${copy_checksum}"* ]]; then - color_echo "${Green}" "Checksum OK" + if [[ "${original_perms}" == "${copy_perms}"* ]]; then + color_echo "${Green}" "Attribute and permission check OK" else - color_echo "${Red}" "Checksum FAILED: ${original_checksum} != ${copy_checksum}" + color_echo "${Red}" "Attribute and permission check FAILED: ${original_perms} != ${copy_perms}" + exit 1 + fi + + if cmp -s "${file_path}" "${tmp_file_path}"; then + color_echo "${Green}" "File content check OK" + else + color_echo "${Red}" "File content check FAILED" exit 1 fi fi