hoostine/zfs-inplace-rebalancing-sqlite
1
0
Fork 0
forked from mirror/zfs-inplace-rebalancing
Code Pull Requests Activity

added -checksum and -passes parameters, added logic to skip checksum, added logic to limit max passes, print configuration on startup, added help message

Browse Source
This commit is contained in:
Markus Ressel 2020-07-30 17:38:02 +02:00
parent 67a75a52b3
commit 3d02d6c274
2 changed files with 128 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
README.md
View File

@ -46,15 +46,34 @@ Due to the working principle of this script, it is crucial that you **only run i
**ALWAYS HAVE A BACKUP OF YOUR DATA!**
You can print a help message by running the script without any parameters:
```
chmod +x ./zfs-inplace-rebalancing.sh
./zfs-inplace-rebalancing.sh /pool/path/to/rebalance
./zfs-inplace-rebalancing.sh
```
### Parameters
| Name | Description | Default |
|-----------|-------------|---------|
| -checksum | Whether to compare the copy using an **MD5** checksum | `true` |
| -passes | The maximum number of rebalance passes per file | `1` |
### Example
```
./zfs-inplace-rebalancing.sh -checksum true -passes 1 /pool/path/to/rebalance
```
### Things to consider
Although this script **does** have a progress output (files as well as percentage) it might be a good idea to try a small subfolder first, or process your pool folder layout in manually selected badges. This can also limit the damage done, if anything bad happens.
When aborting the script midway through, be sure to check the last lines of its output. When cancelling before or during the renaming process a ".rebalance" file might be left and you have to rename it manually.
Although the `-passes` paramter can be used to limit the maximum amount of rebalance passes per file, it is only meant to speedup aborted runs. Individual files will **not be process multiple times automatically**. To reach multiple passes you have to run the script on the same target directory multiple times.
## Attributions
This script was inspired by [zfs-balancer](https://github.com/programster/zfs-balancer).

155
zfs-inplace-rebalancing.sh
View File

@ -17,7 +17,6 @@ current_index=0
Color_Off='\033[0m' # Text Reset
# Regular Colors
Black='\033[0;30m' # Black
Red='\033[0;31m' # Red
Green='\033[0;32m' # Green
Yellow='\033[0;33m' # Yellow
@ -25,6 +24,11 @@ Cyan='\033[0;36m' # Cyan
## Functions
# print a help message
function print_usage() {
echo "Usage: zfs-inplace-rebalancing -checksum true -passes 1 /my/pool"
}
# print a given text entirely in a given color
function color_echo () {
color=$1
@ -32,13 +36,36 @@ function color_echo () {
echo -e "${color}${text}${Color_Off}"
}
function get_rebalance_count () {
file_path=$1
line_nr=$(grep -n "${file_path}" "./${rebalance_db_file_name}" | head -n 1 | cut -d: -f1)
if [ -z "${line_nr}" ]; then
echo "0"
return
else
rebalance_count_line_nr="$((line_nr + 1))"
rebalance_count=$(awk "NR == ${rebalance_count_line_nr}" "./${rebalance_db_file_name}")
echo "${rebalance_count}"
return
fi
}
# rebalance a specific file
function rebalance () {
file_path=$1
current_index="$((current_index + 1))"
progress_percent=$(echo "scale=2; ${current_index}*100/${file_count}" | bc)
color_echo "$Cyan" "Progress -- Files: ${current_index}/${file_count} (${progress_percent}%)"
color_echo "${Cyan}" "Progress -- Files: ${current_index}/${file_count} (${progress_percent}%)"
# check if target rebalance count is reached
rebalance_count=$(get_rebalance_count "${file_path}")
if [ "${rebalance_count}" -ge "${passes_flag}" ]; then
color_echo "${Yellow}" "Rebalance count (${passes_flag}) reached, skipping: ${file_path}"
return
fi
tmp_extension=".balance"
tmp_file_path="${file_path}${tmp_extension}"
@ -68,52 +95,52 @@ function rebalance () {
fi
# compare copy against original to make sure nothing went wrong
echo "Comparing copy against original..."
if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then
# Linux
if [[ "${checksum_flag,,}" == "true"* ]]; then
echo "Comparing copy against original..."
if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then
# Linux
# file attributes
original_md5=$(lsattr "${file_path}" | awk '{print $1}')
# file permissions, owner, group
original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')"
# file content
original_md5="${original_md5} $(md5sum -b "${file_path}" | awk '{print $1}')"
# file attributes
original_md5=$(lsattr "${file_path}" | awk '{print $1}')
# file permissions, owner, group
original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')"
# file content
original_md5="${original_md5} $(md5sum -b "${file_path}" | awk '{print $1}')"
# file attributes
copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}')
# file permissions, owner, group
copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')"
# file content
copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}" | awk '{print $1}')"
# file attributes
copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}')
# file permissions, owner, group
copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')"
# file content
copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}" | awk '{print $1}')"
elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then
# Mac OS
# FreeBSD
elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then
# Mac OS
# FreeBSD
# file attributes
original_md5=$(lsattr "${file_path}" | awk '{print $1}')
# file permissions, owner, group
original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')"
# file content
original_md5="${original_md5} $(md5 -q "${file_path}")"
# file attributes
original_md5=$(lsattr "${file_path}" | awk '{print $1}')
# file permissions, owner, group
original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')"
# file content
original_md5="${original_md5} $(md5 -q "${file_path}")"
# file attributes
copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}')
# file permissions, owner, group
copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')"
# file content
copy_md5="${copy_md5} $(md5 -q "${tmp_file_path}")"
else
echo "Unsupported OS type: $OSTYPE"
exit 1
fi
# file attributes
copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}')
# file permissions, owner, group
copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')"
# file content
copy_md5="${copy_md5} $(md5 -q "${tmp_file_path}")"
else
echo "Unsupported OS type: $OSTYPE"
exit 1
fi
if [[ "${original_md5}" == "${copy_md5}"* ]]; then
color_echo "${Green}" "MD5 OK"
else
color_echo "${Red}" "MD5 FAILED: ${original_md5} != ${copy_md5}"
exit 1
if [[ "${original_md5}" == "${copy_md5}"* ]]; then
color_echo "${Green}" "MD5 OK"
else
color_echo "${Red}" "MD5 FAILED: ${original_md5} != ${copy_md5}"
exit 1
fi
fi
echo "Removing original '${file_path}'..."
@ -123,7 +150,6 @@ function rebalance () {
mv "${tmp_file_path}" "${file_path}"
# update rebalance "database"
touch "./${rebalance_db_file_name}"
line_nr=$(grep -n "${file_path}" "./${rebalance_db_file_name}" | head -n 1 | cut -d: -f1)
if [ -z "${line_nr}" ]; then
rebalance_count=1
@ -131,17 +157,52 @@ function rebalance () {
echo "${rebalance_count}" >> "./${rebalance_db_file_name}"
else
rebalance_count_line_nr="$((line_nr + 1))"
rebalance_count=$(awk "NR == ${rebalance_count_line_nr}" "./${rebalance_db_file_name}")
rebalance_count="$((rebalance_count + 1))"
sed -i "${rebalance_count_line_nr}s/.*/${rebalance_count}/" "./${rebalance_db_file_name}"
fi
}
checksum_flag='true'
passes_flag='1'
if [ "$#" -eq 0 ]; then
print_usage
exit 0
fi
while true ; do
case "$1" in
-checksum )
if [ "$2" -eq 1 ] || [[ "$2" =~ (on|true|yes) ]]; then
checksum_flag="true"
else
checksum_flag="false"
fi
shift 2
;;
-count )
passes_flag=$2
shift 2
;;
*)
break
;;
esac
done;
root_path=$1
color_echo "$Cyan" "Start rebalancing:"
color_echo "$Cyan" " Path: ${root_path}"
color_echo "$Cyan" " Rebalancing Passes: ${passes_flag}"
color_echo "$Cyan" " Use Checksum: ${checksum_flag}"
# count files
file_count=$(find "$root_path" -type f | wc -l)
echo "Files to rebalance: $file_count"
file_count=$(find "${root_path}" -type f | wc -l)
color_echo "$Cyan" " File count: ${file_count}"
# create db file
touch "./${rebalance_db_file_name}"
# recursively scan through files and execute "rebalance" procedure
find "$root_path" -type f -print0 | while IFS= read -r -d '' file; do rebalance "$file"; done