From 4180b449cb82f0c19a5e24d2ed69f5dfdbde2158 Mon Sep 17 00:00:00 2001 From: Frede Hundewadt Date: Sat, 29 Feb 2020 10:59:50 +0100 Subject: [PATCH 1/2] removed limitation on which latest to build buildiso -l - create permalink for iso deployiso -z - create and upload permalinks --- README.md | 10 +++++--- bin/buildiso.in | 34 ++++++++++++++----------- bin/deployiso.in | 6 ++++- lib/util-iso.sh | 37 ++++++++++++++++----------- lib/util-publish.sh | 61 +++++++++++++++++++++++++++++++++++---------- 5 files changed, 101 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 2c72514..6ef8a61 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ overriding # log_dir='/var/log/manjaro-tools' # custom build mirror server -# build_mirror=https://mirror.netzspielplatz.de/manjaro/packages +# build_mirror=https://manjaro.moson.eu ################ buildtree ############### @@ -95,13 +95,13 @@ overriding # build_list_iso=default # the dist release; default: auto -# dist_release=17.1 +# dist_release=auto # the branding; default: auto -# dist_branding="MJRO" +# dist_branding="MANJARO" # unset defaults to given value -# kernel="linux414" +# kernel="linux54" # gpg key; leave empty or commented to skip sfs signing # gpgkey="" @@ -240,6 +240,7 @@ Usage: buildiso [options] -h This help -k Kernel to use [default: linux49] + -l Create permalink -m Set SquashFS image mode to persistence -p Buildset or profile [default: default] -q Query settings and pretend build @@ -405,6 +406,7 @@ Usage: deployiso [options] -t Create ISO torrent -u Update remote directory -v Verbose output + -z Upload permalinks (shell.osdn.net) ~~~ ###### * upload official build list, ie all built iso defined in a build list diff --git a/bin/buildiso.in b/bin/buildiso.in index aedeb66..471e16a 100755 --- a/bin/buildiso.in +++ b/bin/buildiso.in @@ -29,6 +29,7 @@ show_profile(){ [[ ${target_arch} == 'x86_64' ]] && msg2 "multilib: %s" "${multilib}" msg2 "extra: %s" "${extra}" + msg2 "permalink: %s" "${permalink}" msg2 "netinstall: %s" "${netinstall}" msg2 "chrootcfg: %s" "${chrootcfg}" @@ -78,6 +79,8 @@ display_settings(){ msg2 "images_only: %s" "${images_only}" msg2 "iso_only: %s" "${iso_only}" msg2 "persist: %s" "${persist}" + msg2 "extra: %s" "${full_iso}" + msg2 "permalink: %s" "${permalink}" msg "DIST SETTINGS:" msg2 "dist_name: %s" "${dist_name}" @@ -107,31 +110,33 @@ iso_only=false verbose=false persist=false initialize=false +permalink=false usage() { echo "Usage: ${0##*/} [options]" - echo " -i Initialize iso-profiles repo [default: ${branch}]" - echo " -p Buildset or profile [default: ${build_list_iso}]" echo " -a Arch [default: ${target_arch}]" echo " -b Branch [default: ${target_branch}]" + echo ' -c Disable clean work dir' + echo ' -d Compression used for build ISO: gzip, lzma, lz4, lzo, xz, zstd' + echo " [default: ${iso_compression}]" + echo ' -f Build full ISO (extra=true)' + echo ' -g The gpg key for sfs signing' + echo " [default: ${gpgkey}]" + echo " -i Initialize iso-profiles repo [default: ${branch}]" + echo ' -k Kernel to use' + echo " [default: ${kernel}]" + echo ' -l Create permalink' + echo ' -m Set SquashFS image mode to persistence' + echo " -p Buildset or profile [default: ${build_list_iso}]" + echo ' -q Query settings and pretend build' echo ' -r Chroots directory' echo " [default: ${chroots_iso}]" echo ' -t Target directory' echo " [default: ${cache_dir_iso}]" - echo ' -k Kernel to use' - echo " [default: ${kernel}]" - echo ' -g The gpg key for sfs signing' - echo " [default: ${gpgkey}]" - echo ' -m Set SquashFS image mode to persistence' - echo ' -c Disable clean work dir' - echo ' -f Build full ISO (extra=true)' - echo ' -d Compression used for build ISO: gzip, lzma, lz4, lzo, xz, zstd' - echo " [default: ${iso_compression}]" + echo ' -v Verbose output to log file, show profile detail (-q)' echo ' -x Build images only' echo ' -z Generate iso only' echo ' Requires pre built images (-x)' - echo ' -v Verbose output to log file, show profile detail (-q)' - echo ' -q Query settings and pretend build' echo ' -h This help' echo '' echo '' @@ -140,7 +145,7 @@ usage() { orig_argv=("$@") -opts='p:a:b:r:t:k:g:d:cfzxmvqhi' +opts='p:a:b:r:t:k:g:d:cfzxmvqhil' while getopts "${opts}" arg; do case "${arg}" in @@ -160,6 +165,7 @@ while getopts "${opts}" arg; do v) verbose=true ;; x) images_only=true ;; z) iso_only=true ;; + l) permalink=true ;; h|?) usage 0 ;; *) echo "invalid argument '${arg}'"; usage 1 ;; esac diff --git a/bin/deployiso.in b/bin/deployiso.in index f4c584f..673d702 100644 --- a/bin/deployiso.in +++ b/bin/deployiso.in @@ -45,6 +45,7 @@ display_settings(){ msg2 "verbose: %s" "${verbose}" msg2 "signiso: %s" "${sign}" msg2 "torrent: %s" "${torrent}" + msg2 "shell_upload: %s" "${shell_upload}" if ${torrent}; then msg2 "tracker_url: %s" "${tracker_url}" @@ -75,6 +76,7 @@ update=false verbose=false sign=false alt_storage=false +shell_upload=false rsync_args=(-aP -e ssh) @@ -90,12 +92,13 @@ usage() { echo ' -t Create ISO torrent' echo ' -u Update remote directory' echo ' -v Verbose output' + echo ' -z Upload permalinks (shell.osdn.net)' echo '' echo '' exit $1 } -opts='adhl:p:qstuv' +opts='adhl:p:qstuvz' while getopts "${opts}" arg; do case "${arg}" in @@ -108,6 +111,7 @@ while getopts "${opts}" arg; do t) torrent=true ;; u) update=true; rsync_args+=(-u) ;; v) verbose=true; rsync_args+=(-v --stats) ;; + z) shell_upload=true ;; h|?) usage 0 ;; *) echo "invalid argument '${arg}'"; usage 1 ;; esac diff --git a/lib/util-iso.sh b/lib/util-iso.sh index b29c05c..212944e 100644 --- a/lib/util-iso.sh +++ b/lib/util-iso.sh @@ -215,31 +215,38 @@ make_iso() { fi assemble_iso - [[ ${target_branch} == "stable" ]] && [[ ${extra} == "true" ]] && gen_latest_html + ${permalink} && gen_permalink msg "Done [Build ISO]" } -gen_latest_html(){ +gen_permalink(){ if [[ ${edition} == "community" ]] || [[ ${edition} == "manjaro" ]]; then - if [[ -f "${iso_dir}/${iso_file}" ]]; then - msg2 "Creating download link ..." + if [[ -f "${iso_dir}/${iso_file}" ]]; then + msg2 "Creating download link ..." direct_url="https://osdn.net/dl/${edition}/${iso_file}" - [[ ${edition} == "community" ]] && direct_url="https://osdn.net/dl/manjaro-${edition}/${iso_file}" - html_doc="" - html_doc+="" - html_doc+="" - html_doc+="" - html_doc+="Download Redirection" - html_doc+="If you are not redirected automatically, follow the link to latest iso" - echo ${html_doc} > "${iso_dir}/.latest" - + [[ ${edition} == "community" ]] && direct_url="https://osdn.net/dl/manjaro-${edition}/${iso_file}" + ## html permalink + html_doc="" + html_doc+="" + html_doc+="" + html_doc+="" + html_doc+="Download Redirection" + html_doc+="If you are not redirected automatically, follow the link to latest iso" + ## php permalink php_doc="" - echo ${php_doc} > "${iso_dir}/.latest.php" - fi + ## write files + if [[ ${extra} == "true" ]]; then + echo ${html_doc} > "${iso_dir}/.latest" + echo ${php_doc} > "${iso_dir}/.latest.php" + else + echo ${html_doc} > "${iso_dir}/.latest-minimal" + echo ${php_doc} > "${iso_dir}/.latest-minimal.php" + fi + fi fi } diff --git a/lib/util-publish.sh b/lib/util-publish.sh index f9f6bac..2e0fe45 100644 --- a/lib/util-publish.sh +++ b/lib/util-publish.sh @@ -9,6 +9,8 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +local MINIMAL + connect(){ ${alt_storage} && server="storage-in" || server="storage" local storage="@${server}.osdn.net:/storage/groups/m/ma/" @@ -92,10 +94,7 @@ sync_dir(){ else count=$(($max_count + 1)) - # sync latest files - # manjaro.osdn.io - # manjaro-community.osdn.io - sync_web_shell + ${shell_upload} && upload_permalinks msg "Done upload [%s]" "$1" show_elapsed_time "${FUNCNAME}" "${timer_start}" @@ -104,35 +103,65 @@ sync_dir(){ } -sync_web_shell(){ +upload_permalinks(){ + ## permalinks for full ISO if [[ -f "${src_dir}/.latest" ]]; then - LINKS="links.txt" + msg "Uploading permalinks ..." LATEST_ISO=$(sed -e 's/\"/\n/g' < "${src_dir}/.latest" | grep -Eo 'http.*iso$' -m1 | awk '{split($0,x,"/"); print x[6]}') PKGLIST="${LATEST_ISO/.iso/-pkgs.txt}" + ## upload redirector files [[ -f "${src_dir}/.latest" ]] && sync_latest_html [[ -f "${src_dir}/.latest.php" ]] && sync_latest_php + ## upload verification files, torrent and package list [[ -f "${src_dir}/${LATEST_ISO}.torrent" ]] && sync_latest_torrent [[ -f "${src_dir}/${LATEST_ISO}.sig" ]] && sync_latest_signature [[ -f "${src_dir}/${LATEST_ISO}.sha1" ]] && sync_latest_checksum_sha1 [[ -f "${src_dir}/${LATEST_ISO}.sha256" ]] && sync_latest_checksum_sha256 [[ -f "${src_dir}/${PKGLIST}" ]] && sync_latest_pkg_list - #sync_latest_index - + ## remove uploaded files rm -f "${src_dir}/latest" rm -f "${src_dir}/latest.php" - rm -f "${src_dir}/*.iso.torrent" - rm -f "${src_dir}/*.iso.sig" - rm -f "${src_dir}/*.iso.sha1" - rm -f "${src_dir}/*.iso.sha256" + rm -f "${src_dir}/${LATEST_ISO}.torrent" + rm -f "${src_dir}/${LATEST_ISO}.sig" + rm -f "${src_dir}/${LATEST_ISO}.sha1" + rm -f "${src_dir}/${LATEST_ISO}.sha256" + fi + + ## permalinks for minimal ISO + if [[ -f "${src_dir}/.latest-minimal" ]]; then + msg "Uploading permalinks (minimal) ..." + MINIMAL="yes" + LATEST_ISO=$(sed -e 's/\"/\n/g' < "${src_dir}/.latest-minimal" | grep -Eo 'http.*iso$' -m1 | awk '{split($0,x,"/"); print x[6]}') + PKGLIST="${LATEST_ISO/.iso/-pkgs.txt}" + + ## upload redirector files + [[ -f "${src_dir}/.latest-minimal" ]] && sync_latest_html + [[ -f "${src_dir}/.latest-minimal.php" ]] && sync_latest_php + + ## upload verification files, torrent and package list + [[ -f "${src_dir}/${LATEST_ISO}.torrent" ]] && sync_latest_torrent + [[ -f "${src_dir}/${LATEST_ISO}.sig" ]] && sync_latest_signature + [[ -f "${src_dir}/${LATEST_ISO}.sha1" ]] && sync_latest_checksum_sha1 + [[ -f "${src_dir}/${LATEST_ISO}.sha256" ]] && sync_latest_checksum_sha256 + [[ -f "${src_dir}/${PKGLIST}" ]] && sync_latest_pkg_list + + ## remove uploaded files + rm -f "${src_dir}/latest-minimal" + rm -f "${src_dir}/latest-minimal.php" + rm -f "${src_dir}/${LATEST_ISO}.torrent" + rm -f "${src_dir}/${LATEST_ISO}.sig" + rm -f "${src_dir}/${LATEST_ISO}.sha1" + rm -f "${src_dir}/${LATEST_ISO}.sha256" fi } sync_latest_pkg_list(){ msg2 "Uploading package list ..." local pkglist="latest-pkgs.txt" + [[ ${MINIMAL} == "yes" ]] && pkglist="latest-minimal-pkgs.txt" scp "${src_dir}/${PKGLIST}" "${webshell}/${htdocs}/${pkglist}" } @@ -140,6 +169,7 @@ sync_latest_checksum_sha256(){ msg2 "Uploading sha256 checksum file ..." local filename="${LATEST_ISO}.sha256" local checksum_file="latest.sha256" + [[ ${MINIMAL} == "yes" ]] && checksum_file="latest-minimal.sha256" scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${checksum_file}" } @@ -147,6 +177,7 @@ sync_latest_checksum_sha1(){ msg2 "Uploading sha1 checksum file ..." local filename="${LATEST_ISO}.sha1" local checksum_file="latest.sha1" + [[ ${MINIMAL} == "yes" ]] && checksum_file="latest-minimal.sha1" scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${checksum_file}" } @@ -154,6 +185,7 @@ sync_latest_signature(){ msg2 "Uploading signature file ..." local filename="${LATEST_ISO}.sig" local signature="latest.sig" + [[ ${MINIMAL} == "yes" ]] && signature="latest-minimal.sig" scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${signature}" } @@ -161,17 +193,20 @@ sync_latest_torrent(){ msg2 "Uploading torrent file ..." local filename="${LATEST_ISO}.torrent" local torrent="latest.torrent" + [[ ${MINIMAL} == "yes" ]] && torrent="latest-minimal.torrent" scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${torrent}" } sync_latest_php(){ msg2 "Uploading php redirector ..." local php="latest.php" + [[ ${MINIMAL} == "yes" ]] && php="latest-minimal.php" scp "${src_dir}/.${php}" "${webshell}/${htdocs}/${php}" } sync_latest_html(){ msg2 "Uploading url redirector ..." local html="latest" - scp "${src_dir}/.latest" "${webshell}/${htdocs}/${html}" + [[ ${MINIMAL} == "yes" ]] && html="latest-minimal" + scp "${src_dir}/.${html}" "${webshell}/${htdocs}/${html}" } From 5a889a805899298759d2032c984ec0d544c7bd36 Mon Sep 17 00:00:00 2001 From: Frede Hundewadt Date: Sat, 29 Feb 2020 17:42:20 +0100 Subject: [PATCH 2/2] set permissions on uploaded file use sftp -p to transfer the file --- lib/util-publish.sh | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/lib/util-publish.sh b/lib/util-publish.sh index 2e0fe45..a611a3c 100644 --- a/lib/util-publish.sh +++ b/lib/util-publish.sh @@ -162,7 +162,8 @@ sync_latest_pkg_list(){ msg2 "Uploading package list ..." local pkglist="latest-pkgs.txt" [[ ${MINIMAL} == "yes" ]] && pkglist="latest-minimal-pkgs.txt" - scp "${src_dir}/${PKGLIST}" "${webshell}/${htdocs}/${pkglist}" + chmod g+w "${src_dir}/${PKGLIST}" + sftp -p "${webshell}/${htdocs}/${pkglist} <<<'put ${src_dir}/${PKGLIST}'" } sync_latest_checksum_sha256(){ @@ -170,7 +171,9 @@ sync_latest_checksum_sha256(){ local filename="${LATEST_ISO}.sha256" local checksum_file="latest.sha256" [[ ${MINIMAL} == "yes" ]] && checksum_file="latest-minimal.sha256" - scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${checksum_file}" + #scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${checksum_file}" + chmod g+w "${src_dir}/${filename}" + sftp -p "${webshell}/${htdocs}/${checksum_file} <<<'put ${src_dir}/${filename}'" } sync_latest_checksum_sha1(){ @@ -178,7 +181,9 @@ sync_latest_checksum_sha1(){ local filename="${LATEST_ISO}.sha1" local checksum_file="latest.sha1" [[ ${MINIMAL} == "yes" ]] && checksum_file="latest-minimal.sha1" - scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${checksum_file}" + #scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${checksum_file}" + chmod g+w "${src_dir}/${filename}" + sftp -p "${webshell}/${htdocs}/${checksum_file} <<<'put ${src_dir}/${filename}'" } sync_latest_signature(){ @@ -186,7 +191,9 @@ sync_latest_signature(){ local filename="${LATEST_ISO}.sig" local signature="latest.sig" [[ ${MINIMAL} == "yes" ]] && signature="latest-minimal.sig" - scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${signature}" + #scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${signature}" + chmod g+w "${src_dir}/${filename}" + sftp -p "${webshell}/${htdocs}/${signature} <<<'put ${src_dir}/${filename}'" } sync_latest_torrent(){ @@ -194,19 +201,27 @@ sync_latest_torrent(){ local filename="${LATEST_ISO}.torrent" local torrent="latest.torrent" [[ ${MINIMAL} == "yes" ]] && torrent="latest-minimal.torrent" - scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${torrent}" + #scp "${src_dir}/${filename}" "${webshell}/${htdocs}/${torrent}" + chmod g+w "${src_dir}/${filename}" + sftp -p "${webshell}/${htdocs}/${torrent} <<<'put ${src_dir}/${filename}'" } sync_latest_php(){ msg2 "Uploading php redirector ..." + local filename=".latest.php" local php="latest.php" [[ ${MINIMAL} == "yes" ]] && php="latest-minimal.php" - scp "${src_dir}/.${php}" "${webshell}/${htdocs}/${php}" + #scp "${src_dir}/.${php}" "${webshell}/${htdocs}/${php}" + chmod g+w "${src_dir}/${filename}" + sftp -p "${webshell}/${htdocs}/${php} <<<'put ${src_dir}/${filename}'" } sync_latest_html(){ msg2 "Uploading url redirector ..." + local filename=".latest" local html="latest" [[ ${MINIMAL} == "yes" ]] && html="latest-minimal" - scp "${src_dir}/.${html}" "${webshell}/${htdocs}/${html}" + #scp "${src_dir}/.${html}" "${webshell}/${htdocs}/${html}" + chmod g+w "${src_dir}/${filename}" + sftp -p "${webshell}/${htdocs}/${html} <<<'put ${src_dir}/${filename}'" }