mirror of
https://github.com/markusressel/zfs-inplace-rebalancing
synced 2026-02-05 05:24:07 +00:00
added -checksum and -passes parameters, added logic to skip checksum, added logic to limit max passes, print configuration on startup, added help message
This commit is contained in:
Markus Ressel
parent
67a75a52b3
commit
3d02d6c274
21
README.md
21
README.md
@ -46,15 +46,34 @@ Due to the working principle of this script, it is crucial that you **only run i
|
||||
|
||||
**ALWAYS HAVE A BACKUP OF YOUR DATA!**
|
||||
|
||||
You can print a help message by running the script without any parameters:
|
||||
|
||||
```
|
||||
chmod +x ./zfs-inplace-rebalancing.sh
|
||||
./zfs-inplace-rebalancing.sh /pool/path/to/rebalance
|
||||
./zfs-inplace-rebalancing.sh
|
||||
```
|
||||
|
||||
### Parameters
|
||||
|
||||
| Name | Description | Default |
|
||||
|-----------|-------------|---------|
|
||||
| -checksum | Whether to compare the copy using an **MD5** checksum | `true` |
|
||||
| -passes | The maximum number of rebalance passes per file | `1` |
|
||||
|
||||
### Example
|
||||
|
||||
```
|
||||
./zfs-inplace-rebalancing.sh -checksum true -passes 1 /pool/path/to/rebalance
|
||||
```
|
||||
|
||||
### Things to consider
|
||||
|
||||
Although this script **does** have a progress output (files as well as percentage) it might be a good idea to try a small subfolder first, or process your pool folder layout in manually selected badges. This can also limit the damage done, if anything bad happens.
|
||||
|
||||
When aborting the script midway through, be sure to check the last lines of its output. When cancelling before or during the renaming process a ".rebalance" file might be left and you have to rename it manually.
|
||||
|
||||
Although the `-passes` paramter can be used to limit the maximum amount of rebalance passes per file, it is only meant to speedup aborted runs. Individual files will **not be process multiple times automatically**. To reach multiple passes you have to run the script on the same target directory multiple times.
|
||||
|
||||
## Attributions
|
||||
|
||||
This script was inspired by [zfs-balancer](https://github.com/programster/zfs-balancer).
|
||||
|
||||
@ -17,7 +17,6 @@ current_index=0
|
||||
Color_Off='\033[0m' # Text Reset
|
||||
|
||||
# Regular Colors
|
||||
Black='\033[0;30m' # Black
|
||||
Red='\033[0;31m' # Red
|
||||
Green='\033[0;32m' # Green
|
||||
Yellow='\033[0;33m' # Yellow
|
||||
@ -25,6 +24,11 @@ Cyan='\033[0;36m' # Cyan
|
||||
|
||||
## Functions
|
||||
|
||||
# print a help message
|
||||
function print_usage() {
|
||||
echo "Usage: zfs-inplace-rebalancing -checksum true -passes 1 /my/pool"
|
||||
}
|
||||
|
||||
# print a given text entirely in a given color
|
||||
function color_echo () {
|
||||
color=$1
|
||||
@ -32,13 +36,36 @@ function color_echo () {
|
||||
echo -e "${color}${text}${Color_Off}"
|
||||
}
|
||||
|
||||
|
||||
function get_rebalance_count () {
|
||||
file_path=$1
|
||||
|
||||
line_nr=$(grep -n "${file_path}" "./${rebalance_db_file_name}" | head -n 1 | cut -d: -f1)
|
||||
if [ -z "${line_nr}" ]; then
|
||||
echo "0"
|
||||
return
|
||||
else
|
||||
rebalance_count_line_nr="$((line_nr + 1))"
|
||||
rebalance_count=$(awk "NR == ${rebalance_count_line_nr}" "./${rebalance_db_file_name}")
|
||||
echo "${rebalance_count}"
|
||||
return
|
||||
fi
|
||||
}
|
||||
|
||||
# rebalance a specific file
|
||||
function rebalance () {
|
||||
file_path=$1
|
||||
|
||||
current_index="$((current_index + 1))"
|
||||
progress_percent=$(echo "scale=2; ${current_index}*100/${file_count}" | bc)
|
||||
color_echo "$Cyan" "Progress -- Files: ${current_index}/${file_count} (${progress_percent}%)"
|
||||
color_echo "${Cyan}" "Progress -- Files: ${current_index}/${file_count} (${progress_percent}%)"
|
||||
|
||||
# check if target rebalance count is reached
|
||||
rebalance_count=$(get_rebalance_count "${file_path}")
|
||||
if [ "${rebalance_count}" -ge "${passes_flag}" ]; then
|
||||
color_echo "${Yellow}" "Rebalance count (${passes_flag}) reached, skipping: ${file_path}"
|
||||
return
|
||||
fi
|
||||
|
||||
tmp_extension=".balance"
|
||||
tmp_file_path="${file_path}${tmp_extension}"
|
||||
@ -68,52 +95,52 @@ function rebalance () {
|
||||
fi
|
||||
|
||||
# compare copy against original to make sure nothing went wrong
|
||||
echo "Comparing copy against original..."
|
||||
if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then
|
||||
# Linux
|
||||
if [[ "${checksum_flag,,}" == "true"* ]]; then
|
||||
echo "Comparing copy against original..."
|
||||
if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then
|
||||
# Linux
|
||||
|
||||
# file attributes
|
||||
original_md5=$(lsattr "${file_path}" | awk '{print $1}')
|
||||
# file permissions, owner, group
|
||||
original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')"
|
||||
# file content
|
||||
original_md5="${original_md5} $(md5sum -b "${file_path}" | awk '{print $1}')"
|
||||
# file attributes
|
||||
original_md5=$(lsattr "${file_path}" | awk '{print $1}')
|
||||
# file permissions, owner, group
|
||||
original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')"
|
||||
# file content
|
||||
original_md5="${original_md5} $(md5sum -b "${file_path}" | awk '{print $1}')"
|
||||
|
||||
# file attributes
|
||||
copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}')
|
||||
# file permissions, owner, group
|
||||
copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')"
|
||||
# file content
|
||||
copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}" | awk '{print $1}')"
|
||||
# file attributes
|
||||
copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}')
|
||||
# file permissions, owner, group
|
||||
copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')"
|
||||
# file content
|
||||
copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}" | awk '{print $1}')"
|
||||
elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then
|
||||
# Mac OS
|
||||
# FreeBSD
|
||||
|
||||
elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then
|
||||
# Mac OS
|
||||
# FreeBSD
|
||||
# file attributes
|
||||
original_md5=$(lsattr "${file_path}" | awk '{print $1}')
|
||||
# file permissions, owner, group
|
||||
original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')"
|
||||
# file content
|
||||
original_md5="${original_md5} $(md5 -q "${file_path}")"
|
||||
|
||||
# file attributes
|
||||
original_md5=$(lsattr "${file_path}" | awk '{print $1}')
|
||||
# file permissions, owner, group
|
||||
original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')"
|
||||
# file content
|
||||
original_md5="${original_md5} $(md5 -q "${file_path}")"
|
||||
# file attributes
|
||||
copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}')
|
||||
# file permissions, owner, group
|
||||
copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')"
|
||||
# file content
|
||||
copy_md5="${copy_md5} $(md5 -q "${tmp_file_path}")"
|
||||
else
|
||||
echo "Unsupported OS type: $OSTYPE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# file attributes
|
||||
copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}')
|
||||
# file permissions, owner, group
|
||||
copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')"
|
||||
# file content
|
||||
copy_md5="${copy_md5} $(md5 -q "${tmp_file_path}")"
|
||||
|
||||
else
|
||||
echo "Unsupported OS type: $OSTYPE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${original_md5}" == "${copy_md5}"* ]]; then
|
||||
color_echo "${Green}" "MD5 OK"
|
||||
else
|
||||
color_echo "${Red}" "MD5 FAILED: ${original_md5} != ${copy_md5}"
|
||||
exit 1
|
||||
if [[ "${original_md5}" == "${copy_md5}"* ]]; then
|
||||
color_echo "${Green}" "MD5 OK"
|
||||
else
|
||||
color_echo "${Red}" "MD5 FAILED: ${original_md5} != ${copy_md5}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Removing original '${file_path}'..."
|
||||
@ -123,7 +150,6 @@ function rebalance () {
|
||||
mv "${tmp_file_path}" "${file_path}"
|
||||
|
||||
# update rebalance "database"
|
||||
touch "./${rebalance_db_file_name}"
|
||||
line_nr=$(grep -n "${file_path}" "./${rebalance_db_file_name}" | head -n 1 | cut -d: -f1)
|
||||
if [ -z "${line_nr}" ]; then
|
||||
rebalance_count=1
|
||||
@ -131,17 +157,52 @@ function rebalance () {
|
||||
echo "${rebalance_count}" >> "./${rebalance_db_file_name}"
|
||||
else
|
||||
rebalance_count_line_nr="$((line_nr + 1))"
|
||||
rebalance_count=$(awk "NR == ${rebalance_count_line_nr}" "./${rebalance_db_file_name}")
|
||||
rebalance_count="$((rebalance_count + 1))"
|
||||
sed -i "${rebalance_count_line_nr}s/.*/${rebalance_count}/" "./${rebalance_db_file_name}"
|
||||
fi
|
||||
}
|
||||
|
||||
checksum_flag='true'
|
||||
passes_flag='1'
|
||||
|
||||
if [ "$#" -eq 0 ]; then
|
||||
print_usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while true ; do
|
||||
case "$1" in
|
||||
-checksum )
|
||||
if [ "$2" -eq 1 ] || [[ "$2" =~ (on|true|yes) ]]; then
|
||||
checksum_flag="true"
|
||||
else
|
||||
checksum_flag="false"
|
||||
fi
|
||||
shift 2
|
||||
;;
|
||||
-count )
|
||||
passes_flag=$2
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done;
|
||||
|
||||
root_path=$1
|
||||
|
||||
color_echo "$Cyan" "Start rebalancing:"
|
||||
color_echo "$Cyan" " Path: ${root_path}"
|
||||
color_echo "$Cyan" " Rebalancing Passes: ${passes_flag}"
|
||||
color_echo "$Cyan" " Use Checksum: ${checksum_flag}"
|
||||
|
||||
# count files
|
||||
file_count=$(find "$root_path" -type f | wc -l)
|
||||
echo "Files to rebalance: $file_count"
|
||||
file_count=$(find "${root_path}" -type f | wc -l)
|
||||
color_echo "$Cyan" " File count: ${file_count}"
|
||||
|
||||
# create db file
|
||||
touch "./${rebalance_db_file_name}"
|
||||
|
||||
# recursively scan through files and execute "rebalance" procedure
|
||||
find "$root_path" -type f -print0 | while IFS= read -r -d '' file; do rebalance "$file"; done
|
||||
|
||||
Loading…
Reference in New Issue
Block a user