From 2f98c317457fc905e9e4bb5da130c5d06bd1255f Mon Sep 17 00:00:00 2001 From: Frede H Date: Wed, 11 Sep 2024 16:14:01 +0200 Subject: [PATCH] update rules - use auto calc for buffers based on hardware and speed --- 99-usb-sync.rules | 19 ++++------ LICENSE | 2 +- PKGBUILD | 16 ++++---- README.md | 94 ++++++++++++++++------------------------------ udev-usb-sync | 81 ++++++++++++++++++++------------------- udev-usb-sync.conf | 30 ++++++--------- 6 files changed, 104 insertions(+), 138 deletions(-) diff --git a/99-usb-sync.rules b/99-usb-sync.rules index 8b97fd0..292bdd6 100644 --- a/99-usb-sync.rules +++ b/99-usb-sync.rules @@ -31,16 +31,13 @@ # : @linux-aarhus (Manjaro Forum) # inspired by : @kwg (EndeavourOS Forum) -# rule to disable write cache for usb storage -# requires hdparm to be installed -ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", RUN+="/usr/bin/hdparm -W 0 /dev/%k" +KERNEL!="sd[a-z]", GOTO="usb_limit_write_cache_end" +ENV{ID_USB_TYPE}!="disk", GOTO="usb_limit_write_cache_end" +ACTION!="add|change", GOTO="usb_limit_write_cache_end" -# rules to impose buffer limits on USB devices -# implemented using: /usr/bin/udev-usb-sync -# see configuration: /etc/udev-usb-sync/udev-usb-sync.conf -# Works since Linux 6.2 -ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", ATTRS{speed}=="12", RUN+="/usr/bin/udev-usb-sync %k 12" -ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", ATTRS{speed}=="480", RUN+="/usr/bin/udev-usb-sync %k 480" -ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", ATTRS{speed}=="5000", RUN+="/usr/bin/udev-usb-sync %k 5000" -ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", ATTRS{speed}=="10000", RUN+="/usr/bin/udev-usb-sync %k 10000" +ATTRS{bDeviceClass}!="09", ATTRS{speed}=="10000", RUN+="/usr/bin/udev-usb-sync %k %s{speed}", GOTO="usb_limit_write_cache_end" +ATTRS{bDeviceClass}!="09", ATTRS{speed}=="5000", RUN+="/usr/bin/udev-usb-sync %k %s{speed}", GOTO="usb_limit_write_cache_end" +ATTRS{bDeviceClass}!="09", ATTRS{speed}=="480", RUN+="/usr/bin/udev-usb-sync %k %s{speed}", GOTO="usb_limit_write_cache_end" +ATTRS{bDeviceClass}!="09", ATTRS{speed}=="12", RUN+="/usr/bin/udev-usb-sync %k %s{speed}", GOTO="usb_limit_write_cache_end" +LABEL="usb_limit_write_cache_end" diff --git a/LICENSE b/LICENSE index 9112a7a..2071b23 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2023 Frede Hundewadt (@linux-aarhus) +Copyright (c) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/PKGBUILD b/PKGBUILD index a5c659d..edb93ce 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -1,18 +1,18 @@ # Maintainer: root.nix.dk pkgname='udev-usb-sync' -pkgver=0.8 +pkgver=0.9 pkgrel=1 -pkgdesc='Fine tune write cache when USB storage device is plugged' +pkgdesc='Fine tune write cache and impose buffer limites when USB storage device is plugged' arch=('any') -url='https://codeberg.org/wonky/udev-usb-sync' +url='https://gitlab.manjaro.org/fhdk/udev-usb-sync' license=('MIT') -depends=('hdparm') +depends=('hdparm' 'bc') backup=("etc/${pkgname}/${pkgname}.conf") install="${pkgname}.install" source=('99-usb-sync.rules' 'udev-usb-sync' 'udev-usb-sync.conf') -sha256sums=('4f5888647d9be47a8992a7f5ea52eadd5baac0295a39751baa496815bddb065f' - 'b514e3bf1ea55f5e1dc4f5af46da3b5f9f2409da1efe7b36a11647704faf0a8f' - '7cf194b2e3767f8ce4dcbf98b665e46d67624034d5ec0f4cb89f359b677c3687') +sha256sums=('14ff9fa9783f72cb321792ea8a44051d0eb0aaf244edec773d7fa16fdf8dc023' + 'd2987ed0e88d027024ea6c060e42e5557505216d97029930620bd60fccc22b8b' + '4666e62ef38fe4c9c982411f92a99e8e8e5cf73eb08e83252afaf5ec0b527acc') package() { install -d -m755 "$pkgdir/etc/udev/rules.d" @@ -21,4 +21,4 @@ package() { cp "${srcdir}/99-usb-sync.rules" "${pkgdir}/etc/udev/rules.d" cp "${srcdir}/${pkgname}" "${pkgdir}/usr/bin/${pkgname}" cp "${srcdir}/${pkgname}.conf" "${pkgdir}/etc/${pkgname}" -} \ No newline at end of file +} diff --git a/README.md b/README.md index 5ba5302..dca3cb9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,30 @@ -# Disable write cache for usb storage devices - -[Manjaro Forum topic][0] - ## Linux filesystem cache Linux agressively caches files in order to improve overall performance. +* https://linuxatemyram.com + When copying large amount of files to an USB storage this often results in some wait time until the device can be safely removed. How long you have to wait depends on your system and the quality of the USB storage device. Numerous issues regarding this has resulted in various suggestions involving sysctl tweaks and trim. -udev rule to disable write-cache + +Examples: + +* https://forum.manjaro.org/t/unbelievably-slow-file-transfer-speeds/123222 +* https://forum.manjaro.org/t/decrease-dirty-bytes-for-more-reliable-usb-transfer/120798 +* https://forum.manjaro.org/t/slow-usb-transfers/146863 +* https://forum.manjaro.org/t/usb-transfer-speed-5mb-sec-dropping/146777 +* https://forum.manjaro.org/t/extremely-slow-file-transfer-speeds-from-usb-3-0-to-external-hdd/39339 +* https://forum.manjaro.org/t/transfer-speed-slows-down-to-a-crawl-after-10gb-transfered/43453 +* https://forum.manjaro.org/t/usb-transfer-data-drops-drastically-and-maximum-number-of-clients-are-reached/93278 +* https://forum.manjaro.org/t/file-copy-on-a-usb-2-slows-down-to-a-halt/36316 +* https://forum.manjaro.org/t/dolphin-large-file-transfer-to-usb-stick-indicates-completed-after-caching-is-done-transfer-still-ongoing/132634 ## udev rule to disable write-cache -In another place one user threw in an udev rule which would disable write-cache for devices when they were added and while it was just an idea - it triggered my curiosity. +In [another place][2] one user threw in an udev rule which would disable write-cache for devices when they were added and while it was just an idea - it triggered my curiosity. I dug into the intricacies of udev and found a method to only target USB storage devices. @@ -23,57 +32,13 @@ I dug into the intricacies of udev and found a method to only target USB storage The rule has gotten a major overhaul and now consist of the rule, a config file and a script -### The rule **/etc/udev/rules.d/99-usb-sync.rules** -``` -# rule to disable write cache for usb storage -# requires hdparm to be installed -ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", RUN+="/usr/bin/hdparm -W 0 /dev/%K" +https://gitlab.manjaro.org/fhdk/udev-usb-sync/-/blob/master/99-usb-sync.rules -# -# the following rules is introduced with kernel 6.2 -# https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-strict-limit -# https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-max-ratio -# https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-max-bytes -ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", RUN+="/usr/bin/udev-usb-sync %k" -``` -### The config **/etc/usb-dev-sync/udev-usb-sync.conf** -``` -# default values -#use_tweaks=1 -#max_bytes=16777216 -#max_ratio=50 -#strict_limit=1 -``` -### The script **/usr/bin/udev-usb-sync** -``` -#!/usr/bin/bash -# -# script to tweak USB storage device filesystem sync -# -# sources /etc/usb-dev-sync/usb-dev-sync.conf -# +https://gitlab.manjaro.org/fhdk/udev-usb-sync/-/blob/master/udev-usb-sync.conf -use_tweaks=1 -max_bytes=16777216 -max_ratio=50 -strict_limit=1 - -# read user config -source /etc/udev-usb-sync/udev-usb-sync.conf - -if [[ "$use_tweaks" = 0 ]]; then - exit 0 -fi - -if [[ -z "$1" ]]; then - exit 1 -fi - -echo "$max_bytes" > "/sys/block/$1/bdi/max_bytes" -echo "$max_ratio" > "/sys/block/$1/bdi/max_ratio" -echo "$strict_limit" > "/sys/block/$1/bdi/strict_limit" -``` +https://gitlab.manjaro.org/fhdk/udev-usb-sync/-/blob/master/udev-usb-sync +## How it works The rule activates when udev detects * add or change @@ -86,7 +51,9 @@ The rule activates when udev detects - read config and apply user values - if use_tweaks=0 the script exits - if use_tweaks=1 the applies the values (default or config) + - set a max_bytes value based on device speed +## How to setup manually Create a file in **/etc/udev/rules.d/99-usb-sync.rules** and paste the rule into it. Create a file in **/etc/udev-usb-sync/udev-usb-sync.conf** and paste the default values. Create a file in **/usr/bin/udev-usb-sync** and paste the script content. @@ -99,15 +66,20 @@ Reload udev sudo udevadm control --reload -Then plug an usb device - open in your file manager - copy a huge amout of files to the device - when the copy is done - click eject in the file manager - note how quick the device is ejected. +Then plug an usb device - open in your file manager - copy a huge amount of files to the device - when the copy is done - click eject in the file manager - note how quick the device is ejected. -For those preferring the package manager, I have created a [PKGBUILD][4] which will pull the **hdparm** dependency upon installation. +For those preferring the package manager, I have created a [PKGBUILD][1] which will pull the **hdparm** dependency upon installation. pamac build udev-usb-sync -Another idea by [@megavolt][1] at [Manjaro Forum][2] which does not require hdparm. +Another fine utility script provided by @cscs fine-tunes a number of system parameters with the option to input your own values when the script is run -[0]: https://forum.manjaro.org/t/root-tip-how-to-bypass-write-cache-for-usb-storage-devices/135566 -[1]: https://forum.manjaro.org/u/megavolt -[2]: https://forum.manjaro.org/t/decrease-dirty-bytes-for-more-reliable-usb-transfer/120798/4 -[4]: https://aur.archlinux.org/packages/udev-usb-sync \ No newline at end of file +https://gitlab.com/cscs/maxperfwiz + +https://gitlab.manjaro.org/fhdk/udev-usb-sync + +* Cross posted - https://root.nix.dk/en/system-configuration/disable-write-cache-for-usb-devices +* Related topic with tests - [Slow USB transfers - #12 by linux-aarhus](https://forum.manjaro.org/t/slow-usb-transfers/146863/12) + +[1]: https://aur.archlinux.org/packages/udev-usb-sync +[2]: https://forum.endeavouros.com/t/how-come-everything-is-written-when-i-decide-to-unmount-a-usbdrive/37178/10 \ No newline at end of file diff --git a/udev-usb-sync b/udev-usb-sync index b14ce65..46a1df2 100755 --- a/udev-usb-sync +++ b/udev-usb-sync @@ -18,7 +18,7 @@ # @linux-aarhus - root.nix.dk # # configuration : /etc/usb-dev-sync/usb-dev-sync.conf -# triggered by : /usr/lib/udev/rules.d/99-usb-sync.rules +# triggered by : /etc/udev/rules.d/99-usb-sync.rules # # contributors: @megavolt (Manjaro Forum) # : @linux-aarhus (Manjaro Forum) @@ -28,46 +28,51 @@ # $1: usb block device # $2: usb bandwidth reported by device # -# default values (override in configuration file) -# -# block device is passed in $1 -# speed is passed as $2 +set -euo pipefail -# defaults -use_tweaks=1 -use_bandwith=1 -max_ratio=50 -strict_limit=1 -max_bytes=16777216 -# speed defined values -max_bytes_12=10485 -max_bytes_480=629145 -max_bytes_5000=655369 -max_bytes_10000=13107200 +LANG=C +LC_NUMERIC=C -# source configuratoin to override default values -source /etc/udev-usb-sync/udev-usb-sync.conf -if [[ "$use_tweaks" = 0 ]]; then - exit 0 -fi +AUTOCALC=${AUTOCALC:-1} +CONFIG='/etc/udev-usb-sync/udev-usb-sync.conf' -if [[ -z "$1" ]]; then - exit 1 -fi +[[ -f $CONFIG ]] && source $CONFIG -# if speed value is not present use default max_bytes -[[ -z $2 ]] && use_bandwith=0 +BLOCKDEVICE="$1" +SPEED="$2" +KERNEL_MAJOR_VERSION=$(uname -r | awk -F'.' '{print $1}') -# apply max_ratio -echo "$max_ratio" > "/sys/block/$1/bdi/max_ratio" -# apply strict limit -echo "$strict_limit" > "/sys/block/$1/bdi/strict_limit" -# apply max_bytes depending on speed value -if [[ use_bandwith == 0 ]]; then - # apply default value - echo "$max_bytes" > "/sys/block/$1/bdi/max_bytes" -else - # apply bandwidth defined value - echo "$max_bytes_$2" > "/sys/block/%1/bdi/max_bytes" -fi +# disable write cache for device if possible +[[ -n $(which hdparm) ]] && $(which hdparm) -W 0 /dev/$BLOCKDEVICE + +if [[ $KERNEL_MAJOR_VERSION -le 5 ]]; then + # the following rules is introduced with kernel 2.6 + # https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-max-ratio + + # 1% of available RAM -> 8046522kB -> 80.465kB -> 80MB + echo 1 > /sys/block/$BLOCKDEVICE/bdi/max_ratio +elif [[ $KERNEL_MAJOR_VERSION -ge 6 ]]; then + # the following rules is introduced with kernel 6.1 + # https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-strict-limit + # https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-max-bytes + + # apply strict limit + echo 1 > /sys/block/$BLOCKDEVICE/bdi/strict_limit + + if [[ ${AUTOCALC} == 0 ]]; then + # apply 16M as max_bytes + echo 16777216 > /sys/block/$BLOCKDEVICE/bdi/max_bytes + elif [[ ${AUTOCALC} == 1 ]]; then + BUFFER_TIME=${BUFFER_TIME:-"0.05"} + SAFETY_FACTOR=${SAFETY_FACTOR:-"1.3"} + BUFFER_SIZE=$(printf '%.0f' `echo "( ($SPEED / 8) * $BUFFER_TIME * $SAFETY_FACTOR) * 1024 * 1024" | bc`) + # for x in 12 480 5000 10000; do echo -n "$x -> " ;printf "%.0f\n" ` echo "(($x / 8) * 0.05 * 1.3) * 1024 * 1024" | bc`; done + # 62915 + # 4089446 + # 42593157 + # 85196800 + # apply calculated buffer size + echo "$BUFFER_SIZE" > /sys/block/$BLOCKDEVICE/bdi/max_bytes + fi +fi \ No newline at end of file diff --git a/udev-usb-sync.conf b/udev-usb-sync.conf index 42d8b2f..ee3beab 100644 --- a/udev-usb-sync.conf +++ b/udev-usb-sync.conf @@ -1,4 +1,4 @@ -# # The configuration file is part of udev-usb-sync package +# The configuration file is part of udev-usb-sync package # # MIT License # @@ -24,8 +24,8 @@ # # @linux-aarhus - root.nix.dk # -# configuration : /etc/usb-dev-sync/usb-dev-sync.conf -# triggered by : /usr/lib/udev/rules.d/99-usb-sync.rules +# trigger script: /usr/bin/usb-dev-sync +# triggered by : /etc/udev/rules.d/99-usb-sync.rules # # contributors: @megavolt (Manjaro Forum) # : @linux-aarhus (Manjaro Forum) @@ -33,21 +33,13 @@ # # The values used can be modified if you need to do so # The commented default usually works well -#use_tweaks=1 -#use_bandwith=1 +# +# +## Let it calculate based on given bandwidth +#AUTOCALC=1 -#max_ratio=50 -# strict limit requires kernel >= 6.2 -#strict_limit=1 +## The time in seconds to hold data in RAM +#BUFFER_TIME='0.05' -# a sane default bandwidth value -#max_bytes=16777216 - -# bandwith based limitation requires >= 6.1 -# values as reported by the device ATTRS{speed} -# These values seem to be good and stable. -# It buffers 0,01s of the current bandwidth. -#max_bytes_12=10485 -#max_bytes_480=629145 -#max_bytes_5000=655369 -#max_bytes_10000=13107200 +## multiplicator for safety reasons. +#SAFETY_FACTOR='1.3'