root/udev-usb-sync
1
0
Fork 0
mirror of https://codeberg.org/wonky/udev-usb-sync.git synced 2024-09-19 23:30:37 +02:00
Code Issues Projects Releases Packages Wiki Activity

update rules - use auto calc for buffers based on hardware and speed

Browse source
This commit is contained in:
Frede H 2024-09-11 16:14:01 +02:00
parent dac3ec2e23
commit 2f98c31745
No known key found for this signature in database
GPG key ID: 3629B5D280E47F0A
6 changed files with 104 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
99-usb-sync.rules
View file

@ -31,16 +31,13 @@
# : @linux-aarhus (Manjaro Forum) # : @linux-aarhus (Manjaro Forum)
# inspired by : @kwg (EndeavourOS Forum) # inspired by : @kwg (EndeavourOS Forum)
# rule to disable write cache for usb storage KERNEL!="sd[a-z]", GOTO="usb_limit_write_cache_end"
# requires hdparm to be installed ENV{ID_USB_TYPE}!="disk", GOTO="usb_limit_write_cache_end"
ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", RUN+="/usr/bin/hdparm -W 0 /dev/%k" ACTION!="add|change", GOTO="usb_limit_write_cache_end"
# rules to impose buffer limits on USB devices ATTRS{bDeviceClass}!="09", ATTRS{speed}=="10000", RUN+="/usr/bin/udev-usb-sync %k %s{speed}", GOTO="usb_limit_write_cache_end"
# implemented using: /usr/bin/udev-usb-sync ATTRS{bDeviceClass}!="09", ATTRS{speed}=="5000", RUN+="/usr/bin/udev-usb-sync %k %s{speed}", GOTO="usb_limit_write_cache_end"
# see configuration: /etc/udev-usb-sync/udev-usb-sync.conf ATTRS{bDeviceClass}!="09", ATTRS{speed}=="480", RUN+="/usr/bin/udev-usb-sync %k %s{speed}", GOTO="usb_limit_write_cache_end"
# Works since Linux 6.2 ATTRS{bDeviceClass}!="09", ATTRS{speed}=="12", RUN+="/usr/bin/udev-usb-sync %k %s{speed}", GOTO="usb_limit_write_cache_end"
ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", ATTRS{speed}=="12", RUN+="/usr/bin/udev-usb-sync %k 12"
ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", ATTRS{speed}=="480", RUN+="/usr/bin/udev-usb-sync %k 480"
ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", ATTRS{speed}=="5000", RUN+="/usr/bin/udev-usb-sync %k 5000"
ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", ATTRS{speed}=="10000", RUN+="/usr/bin/udev-usb-sync %k 10000"
LABEL="usb_limit_write_cache_end"

2
LICENSE
View file

@ -1,6 +1,6 @@
MIT License MIT License
Copyright (c) 2023 Frede Hundewadt (@linux-aarhus) Copyright (c) <year> <copyright holders>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

14
PKGBUILD
View file

@ -1,18 +1,18 @@
# Maintainer: root.nix.dk # Maintainer: root.nix.dk
pkgname='udev-usb-sync' pkgname='udev-usb-sync'
pkgver=0.8 pkgver=0.9
pkgrel=1 pkgrel=1
pkgdesc='Fine tune write cache when USB storage device is plugged' pkgdesc='Fine tune write cache and impose buffer limites when USB storage device is plugged'
arch=('any') arch=('any')
url='https://codeberg.org/wonky/udev-usb-sync' url='https://gitlab.manjaro.org/fhdk/udev-usb-sync'
license=('MIT') license=('MIT')
depends=('hdparm') depends=('hdparm' 'bc')
backup=("etc/${pkgname}/${pkgname}.conf") backup=("etc/${pkgname}/${pkgname}.conf")
install="${pkgname}.install" install="${pkgname}.install"
source=('99-usb-sync.rules' 'udev-usb-sync' 'udev-usb-sync.conf') source=('99-usb-sync.rules' 'udev-usb-sync' 'udev-usb-sync.conf')
sha256sums=('4f5888647d9be47a8992a7f5ea52eadd5baac0295a39751baa496815bddb065f' sha256sums=('14ff9fa9783f72cb321792ea8a44051d0eb0aaf244edec773d7fa16fdf8dc023'
'b514e3bf1ea55f5e1dc4f5af46da3b5f9f2409da1efe7b36a11647704faf0a8f' 'd2987ed0e88d027024ea6c060e42e5557505216d97029930620bd60fccc22b8b'
'7cf194b2e3767f8ce4dcbf98b665e46d67624034d5ec0f4cb89f359b677c3687') '4666e62ef38fe4c9c982411f92a99e8e8e5cf73eb08e83252afaf5ec0b527acc')
package() { package() {
install -d -m755 "$pkgdir/etc/udev/rules.d" install -d -m755 "$pkgdir/etc/udev/rules.d"

94
README.md
View file

@ -1,21 +1,30 @@
# Disable write cache for usb storage devices
[Manjaro Forum topic][0]
## Linux filesystem cache ## Linux filesystem cache
Linux agressively caches files in order to improve overall performance. Linux agressively caches files in order to improve overall performance.
* https://linuxatemyram.com
When copying large amount of files to an USB storage this often results in some wait time until the device can be safely removed. When copying large amount of files to an USB storage this often results in some wait time until the device can be safely removed.
How long you have to wait depends on your system and the quality of the USB storage device. How long you have to wait depends on your system and the quality of the USB storage device.
Numerous issues regarding this has resulted in various suggestions involving sysctl tweaks and trim. Numerous issues regarding this has resulted in various suggestions involving sysctl tweaks and trim.
udev rule to disable write-cache
Examples:
* https://forum.manjaro.org/t/unbelievably-slow-file-transfer-speeds/123222
* https://forum.manjaro.org/t/decrease-dirty-bytes-for-more-reliable-usb-transfer/120798
* https://forum.manjaro.org/t/slow-usb-transfers/146863
* https://forum.manjaro.org/t/usb-transfer-speed-5mb-sec-dropping/146777
* https://forum.manjaro.org/t/extremely-slow-file-transfer-speeds-from-usb-3-0-to-external-hdd/39339
* https://forum.manjaro.org/t/transfer-speed-slows-down-to-a-crawl-after-10gb-transfered/43453
* https://forum.manjaro.org/t/usb-transfer-data-drops-drastically-and-maximum-number-of-clients-are-reached/93278
* https://forum.manjaro.org/t/file-copy-on-a-usb-2-slows-down-to-a-halt/36316
* https://forum.manjaro.org/t/dolphin-large-file-transfer-to-usb-stick-indicates-completed-after-caching-is-done-transfer-still-ongoing/132634
## udev rule to disable write-cache ## udev rule to disable write-cache
In another place one user threw in an udev rule which would disable write-cache for devices when they were added and while it was just an idea - it triggered my curiosity. In [another place][2] one user threw in an udev rule which would disable write-cache for devices when they were added and while it was just an idea - it triggered my curiosity.
I dug into the intricacies of udev and found a method to only target USB storage devices. I dug into the intricacies of udev and found a method to only target USB storage devices.
@ -23,57 +32,13 @@ I dug into the intricacies of udev and found a method to only target USB storage
The rule has gotten a major overhaul and now consist of the rule, a config file and a script The rule has gotten a major overhaul and now consist of the rule, a config file and a script
### The rule **/etc/udev/rules.d/99-usb-sync.rules** https://gitlab.manjaro.org/fhdk/udev-usb-sync/-/blob/master/99-usb-sync.rules
```
# rule to disable write cache for usb storage
# requires hdparm to be installed
ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", RUN+="/usr/bin/hdparm -W 0 /dev/%K"
# https://gitlab.manjaro.org/fhdk/udev-usb-sync/-/blob/master/udev-usb-sync.conf
# the following rules is introduced with kernel 6.2
# https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-strict-limit
# https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-max-ratio
# https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-max-bytes
ACTION=="add|change", KERNEL=="sd[a-z]", ENV{ID_USB_TYPE}=="disk", RUN+="/usr/bin/udev-usb-sync %k"
```
### The config **/etc/usb-dev-sync/udev-usb-sync.conf**
```
# default values
#use_tweaks=1
#max_bytes=16777216
#max_ratio=50
#strict_limit=1
```
### The script **/usr/bin/udev-usb-sync**
```
#!/usr/bin/bash
#
# script to tweak USB storage device filesystem sync
#
# sources /etc/usb-dev-sync/usb-dev-sync.conf
#
use_tweaks=1 https://gitlab.manjaro.org/fhdk/udev-usb-sync/-/blob/master/udev-usb-sync
max_bytes=16777216
max_ratio=50
strict_limit=1
# read user config
source /etc/udev-usb-sync/udev-usb-sync.conf
if [[ "$use_tweaks" = 0 ]]; then
exit 0
fi
if [[ -z "$1" ]]; then
exit 1
fi
echo "$max_bytes" > "/sys/block/$1/bdi/max_bytes"
echo "$max_ratio" > "/sys/block/$1/bdi/max_ratio"
echo "$strict_limit" > "/sys/block/$1/bdi/strict_limit"
```
## How it works
The rule activates when udev detects The rule activates when udev detects
* add or change * add or change
@ -86,7 +51,9 @@ The rule activates when udev detects
- read config and apply user values - read config and apply user values
- if use_tweaks=0 the script exits - if use_tweaks=0 the script exits
- if use_tweaks=1 the applies the values (default or config) - if use_tweaks=1 the applies the values (default or config)
- set a max_bytes value based on device speed
## How to setup manually
Create a file in **/etc/udev/rules.d/99-usb-sync.rules** and paste the rule into it. Create a file in **/etc/udev/rules.d/99-usb-sync.rules** and paste the rule into it.
Create a file in **/etc/udev-usb-sync/udev-usb-sync.conf** and paste the default values. Create a file in **/etc/udev-usb-sync/udev-usb-sync.conf** and paste the default values.
Create a file in **/usr/bin/udev-usb-sync** and paste the script content. Create a file in **/usr/bin/udev-usb-sync** and paste the script content.
@ -99,15 +66,20 @@ Reload udev
sudo udevadm control --reload sudo udevadm control --reload
Then plug an usb device - open in your file manager - copy a huge amout of files to the device - when the copy is done - click eject in the file manager - note how quick the device is ejected. Then plug an usb device - open in your file manager - copy a huge amount of files to the device - when the copy is done - click eject in the file manager - note how quick the device is ejected.
For those preferring the package manager, I have created a [PKGBUILD][4] which will pull the **hdparm** dependency upon installation. For those preferring the package manager, I have created a [PKGBUILD][1] which will pull the **hdparm** dependency upon installation.
pamac build udev-usb-sync pamac build udev-usb-sync
Another idea by [@megavolt][1] at [Manjaro Forum][2] which does not require hdparm. Another fine utility script provided by @cscs fine-tunes a number of system parameters with the option to input your own values when the script is run
[0]: https://forum.manjaro.org/t/root-tip-how-to-bypass-write-cache-for-usb-storage-devices/135566 https://gitlab.com/cscs/maxperfwiz
[1]: https://forum.manjaro.org/u/megavolt
[2]: https://forum.manjaro.org/t/decrease-dirty-bytes-for-more-reliable-usb-transfer/120798/4 https://gitlab.manjaro.org/fhdk/udev-usb-sync
[4]: https://aur.archlinux.org/packages/udev-usb-sync
* Cross posted - https://root.nix.dk/en/system-configuration/disable-write-cache-for-usb-devices
* Related topic with tests - [Slow USB transfers - #12 by linux-aarhus](https://forum.manjaro.org/t/slow-usb-transfers/146863/12)
[1]: https://aur.archlinux.org/packages/udev-usb-sync
[2]: https://forum.endeavouros.com/t/how-come-everything-is-written-when-i-decide-to-unmount-a-usbdrive/37178/10

85
udev-usb-sync
View file

@ -18,7 +18,7 @@
# @linux-aarhus - root.nix.dk # @linux-aarhus - root.nix.dk
# #
# configuration : /etc/usb-dev-sync/usb-dev-sync.conf # configuration : /etc/usb-dev-sync/usb-dev-sync.conf
# triggered by : /usr/lib/udev/rules.d/99-usb-sync.rules # triggered by : /etc/udev/rules.d/99-usb-sync.rules
# #
# contributors: @megavolt (Manjaro Forum) # contributors: @megavolt (Manjaro Forum)
# : @linux-aarhus (Manjaro Forum) # : @linux-aarhus (Manjaro Forum)
@ -28,46 +28,51 @@
# $1: usb block device # $1: usb block device
# $2: usb bandwidth reported by device # $2: usb bandwidth reported by device
# #
# default values (override in configuration file)
#
# block device is passed in $1 set -euo pipefail
# speed is passed as $2
# defaults LANG=C
use_tweaks=1 LC_NUMERIC=C
use_bandwith=1
max_ratio=50
strict_limit=1
max_bytes=16777216
# speed defined values
max_bytes_12=10485
max_bytes_480=629145
max_bytes_5000=655369
max_bytes_10000=13107200
# source configuratoin to override default values AUTOCALC=${AUTOCALC:-1}
source /etc/udev-usb-sync/udev-usb-sync.conf CONFIG='/etc/udev-usb-sync/udev-usb-sync.conf'
if [[ "$use_tweaks" = 0 ]]; then
exit 0 [[ -f $CONFIG ]] && source $CONFIG
fi
BLOCKDEVICE="$1"
if [[ -z "$1" ]]; then SPEED="$2"
exit 1 KERNEL_MAJOR_VERSION=$(uname -r | awk -F'.' '{print $1}')
fi
# disable write cache for device if possible
# if speed value is not present use default max_bytes [[ -n $(which hdparm) ]] && $(which hdparm) -W 0 /dev/$BLOCKDEVICE
[[ -z $2 ]] && use_bandwith=0
if [[ $KERNEL_MAJOR_VERSION -le 5 ]]; then
# apply max_ratio # the following rules is introduced with kernel 2.6
echo "$max_ratio" > "/sys/block/$1/bdi/max_ratio" # https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-max-ratio
# apply strict limit
echo "$strict_limit" > "/sys/block/$1/bdi/strict_limit" # 1% of available RAM -> 8046522kB -> 80.465kB -> 80MB
# apply max_bytes depending on speed value echo 1 > /sys/block/$BLOCKDEVICE/bdi/max_ratio
if [[ use_bandwith == 0 ]]; then elif [[ $KERNEL_MAJOR_VERSION -ge 6 ]]; then
# apply default value # the following rules is introduced with kernel 6.1
echo "$max_bytes" > "/sys/block/$1/bdi/max_bytes" # https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-strict-limit
else # https://docs.kernel.org/admin-guide/abi-testing.html#abi-sys-class-bdi-bdi-max-bytes
# apply bandwidth defined value
echo "$max_bytes_$2" > "/sys/block/%1/bdi/max_bytes" # apply strict limit
echo 1 > /sys/block/$BLOCKDEVICE/bdi/strict_limit
if [[ ${AUTOCALC} == 0 ]]; then
# apply 16M as max_bytes
echo 16777216 > /sys/block/$BLOCKDEVICE/bdi/max_bytes
elif [[ ${AUTOCALC} == 1 ]]; then
BUFFER_TIME=${BUFFER_TIME:-"0.05"}
SAFETY_FACTOR=${SAFETY_FACTOR:-"1.3"}
BUFFER_SIZE=$(printf '%.0f' `echo "( ($SPEED / 8) * $BUFFER_TIME * $SAFETY_FACTOR) * 1024 * 1024" | bc`)
# for x in 12 480 5000 10000; do echo -n "$x -> " ;printf "%.0f\n" ` echo "(($x / 8) * 0.05 * 1.3) * 1024 * 1024" | bc`; done
# 62915
# 4089446
# 42593157
# 85196800
# apply calculated buffer size
echo "$BUFFER_SIZE" > /sys/block/$BLOCKDEVICE/bdi/max_bytes
fi
fi fi

30
udev-usb-sync.conf
View file

@ -1,4 +1,4 @@
# # The configuration file is part of udev-usb-sync package # The configuration file is part of udev-usb-sync package
# #
# MIT License # MIT License
# #
@ -24,8 +24,8 @@
# #
# @linux-aarhus - root.nix.dk # @linux-aarhus - root.nix.dk
# #
# configuration : /etc/usb-dev-sync/usb-dev-sync.conf # trigger script: /usr/bin/usb-dev-sync
# triggered by : /usr/lib/udev/rules.d/99-usb-sync.rules # triggered by : /etc/udev/rules.d/99-usb-sync.rules
# #
# contributors: @megavolt (Manjaro Forum) # contributors: @megavolt (Manjaro Forum)
# : @linux-aarhus (Manjaro Forum) # : @linux-aarhus (Manjaro Forum)
@ -33,21 +33,13 @@
# #
# The values used can be modified if you need to do so # The values used can be modified if you need to do so
# The commented default usually works well # The commented default usually works well
#use_tweaks=1 #
#use_bandwith=1 #
## Let it calculate based on given bandwidth
#AUTOCALC=1
#max_ratio=50 ## The time in seconds to hold data in RAM
# strict limit requires kernel >= 6.2 #BUFFER_TIME='0.05'
#strict_limit=1
# a sane default bandwidth value ## multiplicator for safety reasons.
#max_bytes=16777216 #SAFETY_FACTOR='1.3'
# bandwith based limitation requires >= 6.1
# values as reported by the device ATTRS{speed}
# These values seem to be good and stable.
# It buffers 0,01s of the current bandwidth.
#max_bytes_12=10485
#max_bytes_480=629145
#max_bytes_5000=655369
#max_bytes_10000=13107200