From 3d02d6c2746ae1f99160bbb696533fb601c00cc7 Mon Sep 17 00:00:00 2001 From: Markus Ressel Date: Thu, 30 Jul 2020 17:38:02 +0200 Subject: [PATCH] added -checksum and -passes parameters, added logic to skip checksum, added logic to limit max passes, print configuration on startup, added help message --- README.md | 21 ++++- zfs-inplace-rebalancing.sh | 155 ++++++++++++++++++++++++++----------- 2 files changed, 128 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 3c1fd6e..3cd671f 100644 --- a/README.md +++ b/README.md @@ -46,15 +46,34 @@ Due to the working principle of this script, it is crucial that you **only run i **ALWAYS HAVE A BACKUP OF YOUR DATA!** +You can print a help message by running the script without any parameters: + ``` chmod +x ./zfs-inplace-rebalancing.sh -./zfs-inplace-rebalancing.sh /pool/path/to/rebalance +./zfs-inplace-rebalancing.sh ``` +### Parameters + +| Name | Description | Default | +|-----------|-------------|---------| +| -checksum | Whether to compare the copy using an **MD5** checksum | `true` | +| -passes | The maximum number of rebalance passes per file | `1` | + +### Example + +``` +./zfs-inplace-rebalancing.sh -checksum true -passes 1 /pool/path/to/rebalance +``` + +### Things to consider + Although this script **does** have a progress output (files as well as percentage) it might be a good idea to try a small subfolder first, or process your pool folder layout in manually selected badges. This can also limit the damage done, if anything bad happens. When aborting the script midway through, be sure to check the last lines of its output. When cancelling before or during the renaming process a ".rebalance" file might be left and you have to rename it manually. +Although the `-passes` paramter can be used to limit the maximum amount of rebalance passes per file, it is only meant to speedup aborted runs. Individual files will **not be process multiple times automatically**. To reach multiple passes you have to run the script on the same target directory multiple times. + ## Attributions This script was inspired by [zfs-balancer](https://github.com/programster/zfs-balancer). diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index d2cba07..7570319 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -17,7 +17,6 @@ current_index=0 Color_Off='\033[0m' # Text Reset # Regular Colors -Black='\033[0;30m' # Black Red='\033[0;31m' # Red Green='\033[0;32m' # Green Yellow='\033[0;33m' # Yellow @@ -25,6 +24,11 @@ Cyan='\033[0;36m' # Cyan ## Functions +# print a help message +function print_usage() { + echo "Usage: zfs-inplace-rebalancing -checksum true -passes 1 /my/pool" +} + # print a given text entirely in a given color function color_echo () { color=$1 @@ -32,13 +36,36 @@ function color_echo () { echo -e "${color}${text}${Color_Off}" } + +function get_rebalance_count () { + file_path=$1 + + line_nr=$(grep -n "${file_path}" "./${rebalance_db_file_name}" | head -n 1 | cut -d: -f1) + if [ -z "${line_nr}" ]; then + echo "0" + return + else + rebalance_count_line_nr="$((line_nr + 1))" + rebalance_count=$(awk "NR == ${rebalance_count_line_nr}" "./${rebalance_db_file_name}") + echo "${rebalance_count}" + return + fi +} + # rebalance a specific file function rebalance () { file_path=$1 current_index="$((current_index + 1))" progress_percent=$(echo "scale=2; ${current_index}*100/${file_count}" | bc) - color_echo "$Cyan" "Progress -- Files: ${current_index}/${file_count} (${progress_percent}%)" + color_echo "${Cyan}" "Progress -- Files: ${current_index}/${file_count} (${progress_percent}%)" + + # check if target rebalance count is reached + rebalance_count=$(get_rebalance_count "${file_path}") + if [ "${rebalance_count}" -ge "${passes_flag}" ]; then + color_echo "${Yellow}" "Rebalance count (${passes_flag}) reached, skipping: ${file_path}" + return + fi tmp_extension=".balance" tmp_file_path="${file_path}${tmp_extension}" @@ -68,52 +95,52 @@ function rebalance () { fi # compare copy against original to make sure nothing went wrong - echo "Comparing copy against original..." - if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then - # Linux + if [[ "${checksum_flag,,}" == "true"* ]]; then + echo "Comparing copy against original..." + if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then + # Linux - # file attributes - original_md5=$(lsattr "${file_path}" | awk '{print $1}') - # file permissions, owner, group - original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')" - # file content - original_md5="${original_md5} $(md5sum -b "${file_path}" | awk '{print $1}')" + # file attributes + original_md5=$(lsattr "${file_path}" | awk '{print $1}') + # file permissions, owner, group + original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')" + # file content + original_md5="${original_md5} $(md5sum -b "${file_path}" | awk '{print $1}')" - # file attributes - copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}') - # file permissions, owner, group - copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')" - # file content - copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}" | awk '{print $1}')" + # file attributes + copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}') + # file permissions, owner, group + copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')" + # file content + copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}" | awk '{print $1}')" + elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then + # Mac OS + # FreeBSD - elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then - # Mac OS - # FreeBSD + # file attributes + original_md5=$(lsattr "${file_path}" | awk '{print $1}') + # file permissions, owner, group + original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')" + # file content + original_md5="${original_md5} $(md5 -q "${file_path}")" - # file attributes - original_md5=$(lsattr "${file_path}" | awk '{print $1}') - # file permissions, owner, group - original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')" - # file content - original_md5="${original_md5} $(md5 -q "${file_path}")" + # file attributes + copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}') + # file permissions, owner, group + copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')" + # file content + copy_md5="${copy_md5} $(md5 -q "${tmp_file_path}")" + else + echo "Unsupported OS type: $OSTYPE" + exit 1 + fi - # file attributes - copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}') - # file permissions, owner, group - copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')" - # file content - copy_md5="${copy_md5} $(md5 -q "${tmp_file_path}")" - - else - echo "Unsupported OS type: $OSTYPE" - exit 1 - fi - - if [[ "${original_md5}" == "${copy_md5}"* ]]; then - color_echo "${Green}" "MD5 OK" - else - color_echo "${Red}" "MD5 FAILED: ${original_md5} != ${copy_md5}" - exit 1 + if [[ "${original_md5}" == "${copy_md5}"* ]]; then + color_echo "${Green}" "MD5 OK" + else + color_echo "${Red}" "MD5 FAILED: ${original_md5} != ${copy_md5}" + exit 1 + fi fi echo "Removing original '${file_path}'..." @@ -123,7 +150,6 @@ function rebalance () { mv "${tmp_file_path}" "${file_path}" # update rebalance "database" - touch "./${rebalance_db_file_name}" line_nr=$(grep -n "${file_path}" "./${rebalance_db_file_name}" | head -n 1 | cut -d: -f1) if [ -z "${line_nr}" ]; then rebalance_count=1 @@ -131,17 +157,52 @@ function rebalance () { echo "${rebalance_count}" >> "./${rebalance_db_file_name}" else rebalance_count_line_nr="$((line_nr + 1))" - rebalance_count=$(awk "NR == ${rebalance_count_line_nr}" "./${rebalance_db_file_name}") rebalance_count="$((rebalance_count + 1))" sed -i "${rebalance_count_line_nr}s/.*/${rebalance_count}/" "./${rebalance_db_file_name}" fi } +checksum_flag='true' +passes_flag='1' + +if [ "$#" -eq 0 ]; then + print_usage + exit 0 +fi + +while true ; do + case "$1" in + -checksum ) + if [ "$2" -eq 1 ] || [[ "$2" =~ (on|true|yes) ]]; then + checksum_flag="true" + else + checksum_flag="false" + fi + shift 2 + ;; + -count ) + passes_flag=$2 + shift 2 + ;; + *) + break + ;; + esac +done; + root_path=$1 +color_echo "$Cyan" "Start rebalancing:" +color_echo "$Cyan" " Path: ${root_path}" +color_echo "$Cyan" " Rebalancing Passes: ${passes_flag}" +color_echo "$Cyan" " Use Checksum: ${checksum_flag}" + # count files -file_count=$(find "$root_path" -type f | wc -l) -echo "Files to rebalance: $file_count" +file_count=$(find "${root_path}" -type f | wc -l) +color_echo "$Cyan" " File count: ${file_count}" + +# create db file +touch "./${rebalance_db_file_name}" # recursively scan through files and execute "rebalance" procedure find "$root_path" -type f -print0 | while IFS= read -r -d '' file; do rebalance "$file"; done