Context Navigation

Back to Ticket #68

Ticket #68: getdelta-paludis.2.sh

File getdelta-paludis.2.sh, 29.7 KB (added by eternaleye@…, 3 years ago)
A getdelta.sh that respects (as best I could) safe-resume

Line
1	#!/bin/sh
2	# getdelta.sh
3	# A download-wrapper script for gentoo that tries to get dtu files
4	# created by deltup instead of downloading complete source-files
5	# to save bandwidth.
6	#
7	# (C) 2004-2006 Nicolai Lissner <nlissne@linux01.gwdg.de>
8	# This script is free software; you can redistribute it and/or modify
9	# it under the terms of the GNU General Public License as published by
10	# the Free Software Foundation; either version 2 of the License , or
11	# (at your option) any later version.
12	#
13	# This program is distributed in the hope that it will be useful,
14	# but WITHOUT ANY WARRANTY; without even the implied warranty of
15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16	# GNU General Public License for more details.
17	#
18	# You should have received a copy of the GNU General Public License
19	# along with this program. If not, write to
20	#
21	# The Free Software Foundation, Inc.
22	# 59 Temple Place, Suite 330
23	# Boston, MA 02111, USA.
24
25	VERSION="0.7"
26
27	# Changelog
28	# version 0.7.7 2007/01/30
29	# - added support for more than one local mirror in getdelta.rc
30	# thanks to Alexander Polozov, who sent me the patch for this
31	# version 0.7.6 2006/10/08
32	# - fixed a typo - thanks to Andrey, who reported this problem in gentoo bug #150426
33	# version 0.7.5 2006/10/03
34	# - added support for changing timeout based on expected filesize
35	# if configured it will reduce the waiting timeout to the expected download-time (based on size of old version)
36	# version 0.7.4 2006/09/06
37	# - do not remove log file but reset only to make this work with FEATURE userfetch
38	# version 0.7.3
39	# 2006/09/01
40	# - fixed a bug in detection of original url (sometimes no url was found)
41	# version 0.7.2
42	# 2006/08/18
43	# - improved method of chosing the right candidate
44	# version 0.7.1
45	# 2006/08/08
46	# - fixed a problem in counting differences in filenames (aka bug #105011)
47	# -
48	# version 0.7
49	# 2005/05/09
50	# - servers create dtu files based on bdelta instead of xdelta
51	# this happens for two reasons: smaller dtu-files and amd64-compatibilty
52	# (yes, it's true - welcome to all new amd64 users of the servers)
53	# this is the reason for major update - and update forcing - you really
54	# NEED bdelta now to use the servers, while you do not need xdelta anymore :)
55	# - integrity change of old candidate is optional now and disabled by default
56	# if you want this time consuming "safe way" re-enable it via the config-file
57	# - added some files to DO_NOT_REMOVE file
58	# - added "&time=<timestamp>" to prevent ANY proxy from returning results from cache
59	# instead of asking the server - the server does not use this parameter - it just
60	# exists to create unique request-URLS (as proposed by bodo "bothie" thiesen)
61	# version 0.6.9 2005/03/11
62	# � exit with exitcode of wget to signal to portage if
63	# download was successful
64	# - handle metadata.dtd as exception (repoman uses FETCHCOMMAND)
65	# - you can disable fetching of dtu-files now by setting
66	# the environment variable GETDELTA=0
67	# version 0.6.8 2005/01/09
68	# - init frontmatch and backmatch with 0 (thanks, Torsten Veller)
69	# I wonder when it disappeared from the script
70	# version 0.6.7 2004/12/22
71	# - corrected the formula for the saved size in percent
72	# as reported by Torsten Veller
73	# version 0.6.6 2004/12/21
74	# - ignore "try" in MPlayer filename
75	# version 0.6.5 2004/11/15
76	# - added information about the saved traffic
77	# - fix: use NORMAL color after first waiting for retry
78	# and QUEUETIMEOUT works now (thanks to Bodo Thiesen for the patch)
79	# - ignore "PR" in filename if the filename starts with firefox
80	# version 0.6.4 2004/11/08
81	# - inserted "break" to the TSTAMP>=QTMOUNT condition, too
82	# (reported by Torsten Veller)
83	# version 0.6.3 2004/11/08
84	# - added "^bash" and "^gtk-engines" to the default
85	# do_not_remove file
86	# - inserted a "break" to prevent infinite looping
87	# when the server returns a queueposition higher than
88	# the allowed number (as reported by James Rowe and others)
89	# version 0.6.2 2004/10/22
90	# - changed QUERY_URL to get the last URL instead the first
91	# since that's the original server (not a mirror)
92	# version 0.6.1 2004/10/18
93	# - give better original URL to the server
94	# - enhanced detection of former version (thanks to Jimmy Wennlund)
95	# - the COLOR variable didn't work since 0.5.3, because
96	# the config file was not read before evaluating the variable
97	# - check, if the user have set RESUMECOMMAND to getdelta.sh
98	# and if so complain about this
99	#
100	# version 0.6 2004/10/12
101	# - dropped the client-side mirror-detection
102	# - Jimmy Wennlund <jimmy@jw.dyndns.org> sent me patch to
103	# make getdelta.sh work in an own tempdir and to
104	# remove any temporary files even when user pressed
105	# ctrl-c -- I really like that. Thanks, Jimmy.
106	# version 0.5.4 2004/10/11
107	# - fixed a security leak (possible symlink attack)
108	# thanks to Raimund Specht <raimund@spemaus.de> for
109	# reporting the problem and sending some possible solutions.
110	# version 0.5.3.5 2004/10/02
111	# - the DO_NOT_REMOVE-file was overwritten with defaults
112	# fixed.
113	# version 0.5.3.4 2004/09/20
114	# - fixed wrong path-detection with thirdpartymirrors
115	# version 0.5.3.3 2004/09/12
116	# - changed the way the script finds the mirror-group to use
117	# - setting GENTOO_MIRRORS="" is not necessary anymore
118	# removed check and warning about that
119	# version 0.5.3.2 2004/09/12
120	# - fixed a bug in the ouput of remove() (thx to wiebel)
121	# version 0.5.3.1 2004/09/12
122	# - fixed a typo (FILESDIR)
123	# - fixed a missing "]"
124	# version 0.5.3 2004/09/12
125	# - some code cleanups
126	# - use a separated config file now
127	# - old file in DISTDIR is tested on corruption before trying to download a dtu
128	# (thanks to pkunk)
129	# - check for GENTOO_MIRRORS=""
130	# - new LOCAL_MIRROR to check before requesting a dtu
131	# - non existing DO_NOT_REMOVE-file is created with some defaults
132	# - found a severe bug in finding candidates when updating files beginning with "lib"
133	# - added MAXIMUM_ACCEPTABLE_QUEUEPOS
134	# version 0.5.2.3 2004/09/06
135	# - new variable QUERY_RETRY
136	# - dont remove file added
137	# version 0.5.2.2 2004/08/30
138	# - fixed a typo
139	# version 0.5.2.1 2004/08/29
140	# - fixed "too many arguments" as suggested by NoUseForAName
141	# in posting http://forums.gentoo.org/viewtopic.php?p=1480776#1480776
142	# version 0.5.2 2003/08/27
143	# - server sends a queued-message including queue-position now
144	# show this.
145	# version 0.5.1 2003/08/24
146	# - for some reason a "broken pipe" message appears when
147	# this script is called by portage/python, caused by
148	# "ls -c\|head -n1" - Ok, that IS a broken pipe, "head"
149	# would not read anything more than 1 line, but I do not
150	# really understand, why it does not happen when the script
151	# is called manually -- ANY use of "head" in a pipe-construction
152	# would result in a "broken-pipe", but bash itself never
153	# complains about that. a cosmetic change to make the
154	# output clean and the script-code ugly :-/
155	# version 0.5.0 2003/08/21
156	# - the exception handling for kde changed to the server
157	# - this script now checks if it got a dtu or xdelta
158	# - added timeout (to prevent endless loops in case of problems)
159	#
160	# version 0.4.0 2003/07/06
161	# the deltup-server queues requests now
162	# and sends back a document "deltup-queued"
163	# the client then waits 10 seconds and tries
164	# again until it either gets the dtu or a file
165	# named *.failed
166	# version 0.3.3 2003/05/06
167	# transmit version to server
168	# receive important messages from server
169	# version 0.3.2 2003/05/05
170	# correct handle of src-archives of X11-org
171	# version 0.3.1 2003/04/26
172	# fixed path to kde-sources on kde-mirror
173	# version 0.3.0 2004/04/20
174	# exception: get kde-version as xdelta-files from kde-mirror
175	#
176	# version 0.2.4 2004/04/15
177	# colors are now optional
178	# candidates named lib* are found faster now
179	# little enhancements on verbosity
180	# version 0.2.3 2004/04/14
181	# colorized verbosity
182	# fixed a bug that leaded to wrong candidates and error-outputs
183	# version 0.2.2 2004/04/14
184	# verbosity added by wiebel
185	# initialize frontmatch / backmatch with 0
186	# version 0.2.1 2004/04/14
187	# ignore spaces and "+" in filename-mask, too
188	# option REMOVE_OLD added
189	# version 0.2 2004/04/13
190	# old files can differ by one char from the wanted file
191	# to catch versions with letters in it
192	#
193	# version 0.1.1 2004/04/08
194	# changed method to determine which mirror to use
195	#
196	# version 0.1
197	# initial version 2004/04/06
198	#
199	#
200	####################################################
201	# NO variables to set here in the script anymore #
202	# we use a config-file instead which is created #
203	# and filled with some default values on first run #
204	# This file: #
205	####################################################
206
207	GETDELTA_CONFIGFILE=/etc/deltup-paludis/getdelta.rc
208	#
209
210	splitversion(){
211	# $1: the version string
212	# output: the splitted version (1.2.3 -> 1 2 3, 10.11.12b -> 10 11 12 b)
213	local vstr=$1
214	shopt -s extglob
215	while [ -n "$vstr" ]
216	do
217	case ${vstr:0:1} in
218	[[:digit:]])
219	echo "${vstr%%[^[:digit:]]*}"
220	vstr="${vstr##+([[:digit:]])}"
221	;;
222	[[:alpha:]])
223	nomatch="${vstr##+([[:alpha:]])*([[:digit:]])}"
224	echo "${vstr:0:$((${#vstr} - ${#nomatch}))}"
225	vstr="${nomatch}"
226	;;
227	*) vstr="${vstr:1}"
228	;;
229	esac
230	done
231	shopt -u extglob
232	}
233
234	ver2ser(){
235	local x=$(splitversion $1)
236	x=($x)
237	vser=""
238	shopt -s extglob
239	for ((i=0;i<${#x[@]};i++))
240	do
241	case ${x[$i]:0:1} in
242	[[:digit:]])
243	# let ad=${x[$i]##+(0)}+1
244	[ "${x[$i]##+(0)}" ] && let ad=${x[$i]##+(0)} \|\| ad=0
245	vs=$(printf "%02x" ${ad})
246	vser="${vser}${vs}"
247	;;
248	[[:alpha:]])
249	calced=0
250	if [ "${x[$i]:0:3}" = "pre" ]
251	then
252	vs=${x[$i]:3}
253	let vser=0x${vser}00-40+${vs:-0}
254	vser=$(printf "%02x" $vser)
255	calced=1
256	fi
257	if [ "${x[$i]:0:2}" = "rc" ]
258	then
259	vs=${x[$i]:2}
260	let vser=0x${vser}00-40+${vs:-0}
261	vser=$(printf "%02x" $vser)
262	calced=1
263	fi
264	if [ "${x[$i]:0:5}" = "alpha" ]
265	then
266	vs=${x[$i]:5}
267	let vser=0x${vser}00-80+${vs:-0}
268	vser=$(printf "%02x" $vser)
269	calced=1
270	fi
271	if [ "${x[$i]:0:4}" = "beta" ]
272	then
273	vs=${x[$i]:4}
274	let vser=0x${vser}00-60+${vs:-0}
275	vser=$(printf "%02x" $vser)
276	calced=1
277	fi
278	if [ "$calced" = "0" ]
279	then
280	vs=$(echo -n ${x[$i]} \| od -t x1 \| head -n1\| cut -d" " -f2-\| tr -d " ")
281	vser="${vser}${vs}"
282	fi
283	esac
284	done
285	shopt -u extglob
286	let m=${#vser}%2
287	[ "$m" = "1" ] && vser="0${vser}"
288	echo $vser
289	}
290
291
292	# some colors for colored output
293	output() {
294	${VERBOSITY} && echo -e "$1${NORMAL}" \| tee -a $LOGFILE
295	}
296
297	# this checks for a variable in our config-file and adds it if does not exist
298	# $1 is the name of the variable, $2 the default content of the variable
299	# $3 a description line for the variable
300	add_to_configfile() {
301	GETDELTA_CONFIGDIR=$(dirname $GETDELTA_CONFIGFILE )
302	[ -e $GETDELTA_CONFIGFILE ] \|\| ( mkdir -p $GETDELTA_CONFIGDIR && touch $GETDELTA_CONFIGFILE )
303	if ! grep -q "$1" $GETDELTA_CONFIGFILE
304	then
305	echo -e "\n# ${3}\n${1}=\"${2}\"" >>$GETDELTA_CONFIGFILE
306	output "${CYAN}Added new variable ${YELLOW}$1${CYAN} to config file ${GETDELTA_CONFIGFILE}\n"
307	output "please check if it fits your needs\n"
308	fi
309	}
310
311	# this checks for an entry in our do_not_remove-file and adds it if does not exist
312	# $1 is the name (as grep regexp) of the file not to be removed
313	add_to_donotremove() {
314
315	if ! grep -q "^${1}" $DO_NOT_REMOVE
316	then
317	echo "${1}" >>$DO_NOT_REMOVE
318	output "${CYAN}Added new grep-regex \"${1}\" to config file ${DO_NOT_REMOVE}\n"
319	fi
320	}
321
322
323	remove() {
324	output "${GREEN}You have chosen to remove ${CYAN}$1\n"
325	pushd ${DISTDIR} >/dev/null 2>&1
326	removeme=true
327	for n in $(grep -v "^#" ${DO_NOT_REMOVE})
328	do
329	grep -q $n <<< "$1" && removeme=false && output "${CYAN}${1}${RED} is not deleted, since it matches ${n} in ${DO_NOT_REMOVE}"
330	done
331	$removeme && rm -f $1
332	popd >/dev/null 2>&1
333	}
334
335
336	mask_name() {
337	MASK_FILENAME=$1
338	# do some "blackmagic" with the src-files of xorg
339
340	if [ $(cut -c 1-6 <<< $MASK_FILENAME) = "X11R6." ]
341	then
342	MASK_FILENAME=$(sed -e "s/src1/srcAAA/g" \
343	-e "s/src2/srcBBB/g" \
344	-e "s/src3/srcCCC/g" \
345	-e "s/src4/srcDDD/g" \
346	-e "s/src5/srcEEE/g" \
347	-e "s/src6/srcFFF/g" \
348	-e "s/src7/srcGGG/g" <<< $MASK_FILENAME)
349	fi
350
351	# ignore PR for src-files of firefox
352	if [ $(cut -c 1-7 <<< $MASK_FILENAME) = "firefox" ]
353	then
354	MASK_FILENAME=$(sed -e "s/PR//g" <<< $MASK_FILENAME)
355	fi
356
357	# ignore "try" with new mplayer
358	if [ $(cut -c 1-7 <<< $MASK_FILENAME) = "MPlayer" ]
359	then
360	MASK_FILENAME=$(sed -e "s/try//g" <<< $MASK_FILENAME)
361	fi
362
363
364	# ignore some strings in any filename
365	echo $(sed -e "s/\.bz2$//g" \
366	-e "s/\.gz$//g" \
367	-e "s/[0-9]//g" \
368	-e "s/pre//g" \
369	-e "s/preview//g" \
370	-e "s/beta//g" \
371	-e "s/rc//g" \
372	-e "s/[\._-]//g" \
373	-e "s/\+//g" \
374	-e "s/ //g" <<< $MASK_FILENAME)
375	}
376
377	# create or update a config-file
378
379	add_to_configfile KDE_MIRROR "ftp://ftp.kde.org/pub/kde/stable" "we de not get kde-deltas from a delta-up-server, since kde provides own xdelta-files"
380	add_to_configfile LOCAL_MIRROR "" "set this to one or more (space separated) URI ending with '/' if you want to check one or more local mirror(s) first\n# most people just leave it empty."
381	add_to_configfile DELTUP_SERVER "http://linux01.gwdg.de/~nlissne/deltup.php" "deltup-server to use"
382	add_to_configfile FETCH "/usr/bin/wget -t 1 --passive-ftp" "command to use for downloading"
383	add_to_configfile QUEUERETRY 15 "number of seconds to wait before a queued request is retried"
384	add_to_configfile MAXIMUM_ACCEPTABLE_QUEUEPOS "15" "the maximum queuepos you would accept (if higher download full archive instead)"
385	add_to_configfile QUEUETIMEOUT 900 "when a dtu-request is queued - how long should we wait max. before downloading the original archive instead (in seconds)"
386	add_to_configfile CHECK_OLD_FILE "false" "set to \"true\", if you want getdelta.sh to use Pkunk's integrity check for the old file before downloading dtu-files"
387	add_to_configfile REMOVE_OLD "false" "set to \"true\", if you want getdelta.sh to delete the old file, if patch was succesful"
388	add_to_configfile DO_NOT_REMOVE "/etc/deltup/do_not_remove" "a list of files not to be removed by REMOVE_OLD feature"
389	add_to_configfile REMOVE_INCOMPLETE_OLD_FILES "false" "set this to \"true\" if you want getdelta.sh to delete old versions that seems to be corrupt,\n# or to \"false\" if you want to delete them manually\n# note: getdelta.sh will not use these files anyway"
390	add_to_configfile VERBOSITY true "set to \"true\", if you want verbose outputs (later to be set to a level [0-3])"
391	add_to_configfile COLOR true "set to \"true\", if you want colorful messages, \"false\" if not."
392	add_to_configfile LOGFILE "/var/log/getdelta.log" "set to a writable file (or to \"/dev/null\" if you do not want this) this is not used, if VERBOSITY is false"
393	add_to_configfile DELETE_LOG true "set to \"true\" if you want a temporarily log only (deleted when getdelta is finished)"
394	add_to_configfile SEPARATED_WINDOW "false" "set to \"true\", if you want messages from this script in a separate window\n# set to \"false\", if you do not start getdelta.sh from an Xsession or if you \n# do not have permissions to open terminals on the Xserver"
395	add_to_configfile TERM_APP "aterm -tr -trsb -fg white -bg black -sh 70 -e tail -f ${LOGFILE}" "the terminal application to use for the separated window"
396	add_to_configfile BANDWIDTH 1 "the bandwidth in bytes per second. configure this if you want to reduce timeouts on small files"
397
398	source $GETDELTA_CONFIGFILE
399
400	# create or update DO_NOT_REMOVE file
401	# these files have "old" versions that are needed to build the new versions
402	# so they should never removed by the REMOVE_OLD feature
403	DO_NOT_REMOVE_DIR=$(dirname $DO_NOT_REMOVE)
404	if [ ! -e $DO_NOT_REMOVE ]
405	then
406	mkdir -p $DO_NOT_REMOVE_DIR
407	echo "# This file contains regexp in 'grep-style' for files that should not be removed" >$DO_NOT_REMOVE
408	echo "# if REMOVE_OLD is set to 'true'" >>$DO_NOT_REMOVE
409	echo "# Some examples (actually these files are known to result" >>$DO_NOT_REMOVE
410	echo "# in problems if getdelta.sh is used with REMOVE_OLD=true" >>$DO_NOT_REMOVE
411	fi
412	add_to_donotremove "^font-arial-iso-8859"
413	add_to_donotremove "^libtool"
414	add_to_donotremove "^readline"
415	add_to_donotremove "^gtk-engines"
416	add_to_donotremove "^bash"
417	add_to_donotremove "^openssl"
418	add_to_donotremove "^curl"
419	add_to_donotremove "^festvox"
420	add_to_donotremove "^rp-pppoe"
421
422
423	if [ -z $1 ]
424	then
425	COLOR=true
426	echo -e "${YELLOW}getdelta.sh version ${VERSION}"
427	echo "This script has to be called like this:"
428	echo -e "${CYAN}$0 <URI>"
429	echo -e "\n${YELLOW}To use it, you should just put the following line into your /etc/make.conf"
430	echo -e "${GREEN}FETCHCOMMAND=\"$0 \\\${URI}\""
431	echo -e "\n${YELLOW}There is a config-file ${CYAN}${GETDELTA_CONFIGFILE}${YELLOW} with some variables to control the behaviour of this script."
432	echo -e "Edit it to your needs.${NORMAL}"
433	exit 1
434	fi
435	# include variables from gentoo make.globals and make.conf
436	source /etc/make.globals
437	source /etc/make.conf
438
439
440	if ${COLOR}
441	then
442	RED="\033[01;31m"
443	GREEN="\033[01;32m"
444	YELLOW="\033[01;33m"
445	BLUE="\033[01;34m"
446	MAGENTA="\033[01;35m"
447	CYAN="\033[01;36m"
448	NORMAL="\033[00m"
449	else
450	RED=""
451	GREEN=""
452	YELLOW=""
453	BLUE=""
454	MAGENTA=""
455	CYAN=""
456	NORMAL=""
457	fi
458	grep -q "getdelta.sh" <<< "${RESUMECOMMAND}" &&
459	output "${RED}do NOT set RESUMECOMMAND to use getdelta.sh" &&
460	output "use getdelta.sh for your FETCHCOMMAND, only." &&
461	sleep 5 && exit 1
462
463	pushd $DISTDIR >/dev/null 2>/dev/null
464	ORIG_URI=$1
465	NEW_FILE=$(basename $ORIG_URI)
466
467	# repoman downloads metadata.dtd with FETCHCOMMAND
468	# this should not be done with getdelta - so just fetch the file and exit
469
470	# Check if env.variable GETDELTA is set to 0 to disable fetching of
471	# dtu files.
472	if [ "${NEW_FILE}" = "metadata.dtd" ] \|\| [ "$GETDELTA" = "0" ]
473	then
474	$FETCH -O ${NEW_FILE}.-PARTIAL- $ORIG_URI
475	exit $?
476	fi
477
478
479	[ -e deltup-server.msg ] && rm -f deltup-server.msg
480
481	# if output should go to an additional window start it
482	if $SEPARATED_WINDOW
483	then
484	touch $LOGFILE
485	$TERM_APP &
486	termpid=$!
487	echo -e "\x1b]1;\x07\x1b]2;getdelta.sh trying to get dtu for ${NEW_FILE}\x07"
488	fi
489
490	# First of all: check if LOCAL_MIRROR is set and provides the file in question already
491	for localn in $LOCAL_MIRROR
492	do
493	output "${YELLOW}Trying to get ${CYAN}${NEW_FILE}${YELLOW} from local mirror ${CYAN}${localn}\n"
494	if $FETCH -O ${NEW_FILE}.-PARTIAL- "${localn}${NEW_FILE}"
495	then
496	output "${GREEN}success.\n"
497	exit 0
498	else
499	output "${RED}failed${YELLOW}\n"
500	fi
501	done
502
503	#
504	# find an old file in $DISTDIR that matches the new one. This is tricky,
505	# and probably it will fail sometimes.
506	#
507	# we just ignore any occurence of
508	# "pre","rc","[0-9]","_","-","." in the filenames and test
509	# if they are the same (or VERY similar (differ only in 1 char)).
510	# to reduce the files to check, we only check files
511	# with the same beginning
512	#
513	output "${GREEN}Searching for a previously downloaded file in ${YELLOW}${DISTDIR}\n"
514
515	first_chars=$(sed 's/[[:digit:]][[:print:]]*$//' <<< $NEW_FILE)
516	length_first_chars=$(wc -c <<< $first_chars)
517	[ $length_first_chars -lt 3 ] && first_chars=$(cut -c 1-2 <<< $NEW_FILE)
518
519	# if filename is lib* use first 4 letters to increase performance
520	[ "$( cut -c 1-3 <<< $NEW_FILE )" = "lib" ] &&
521	[ $length_first_chars -lt 5 ] && first_chars=$(cut -c 1-4 <<< $NEW_FILE)
522
523	mask=$(mask_name "${NEW_FILE}")
524	let len1=$(wc -c <<< $mask)-1
525	filelist=""
526
527	for name in $( ls ${first_chars}* 2>/dev/null )
528	do
529	mask2=$(mask_name "${name}")
530	# add any file, that results in the same mask or differ not more than two letters
531	let len2=$(wc -c <<< $mask2)-1
532	if [ $len1 -gt $len2 ]
533	then
534	max=${len1}
535	let min=${len2}
536	else
537	let min=${len1}
538	max=${len2}
539	fi
540	let df=${max}-${min}
541
542	# if masks differ in length more than 1 they cannot match
543	if [ $df -le 1 ]
544	then
545	let frontmatch=0
546	let backmatch=0
547	for ((ch=1;ch<=min;ch++))
548	do
549	if [ $(cut -c ${ch} <<< ${mask}) = $(cut -c ${ch} <<< ${mask2}) ]
550	then frontmatch=${ch}
551	else break
552	fi
553	done
554
555	# now backwards
556	mask=$(rev <<< ${mask})
557	mask2=$(rev <<< ${mask2})
558	for ((ch=1;ch<=min;ch++))
559	do
560	if [ $(cut -c ${ch} <<< ${mask} ) = $(cut -c ${ch} <<< ${mask2}) ]
561	then backmatch=${ch}
562	else break
563	fi
564	done
565
566	# forwards for mask again (need this for the next run of the loop)
567	mask=$(rev <<< ${mask})
568
569	let matchall=${frontmatch}+${backmatch}
570	let minmatch=${min}-1
571	[ ${matchall} -ge ${minmatch} ] && filelist="${filelist} $name"
572	fi
573	done
574
575	if ! [ -z "$filelist" ]
576	then
577	# we have got a list of candidates in $filelist now. find the best match .
578	output "${GREEN}We have the following candidates to choose from \n${YELLOW}`sed -e \"s/\ /\\n/g\" <<< $filelist` \n"
579
580	# find matching part of filename - first: frontmatch
581	x=0;
582	a=($NEW_FILE $filelist)
583	match=""
584	while [ -z "$match" ]
585	do
586	for ((i=0;i<${#a[@]};i++))
587	do
588	[ ${a[0]:${x}:1} != ${a[$i]:${x}:1} ] && match=$x
589	done
590	((x++))
591	done
592	frontmatch=${a[0]:0:${match}}
593
594	# find matching part of filename - second: backmatch
595	x=1;
596	match=""
597	while [ -z "$match" ]
598	do
599	for ((i=0;i<${#a[@]};i++))
600	do
601	[ ${a[0]:${#a[0]}-${x}:1} != ${a[$i]:${#a[$i]}-${x}:1} ] && match=$x
602	done
603	((x++))
604	done
605	((match--))
606	backmatch=${a[0]:${#a[0]}-${match}}
607
608	# isolate version from filename (foobar-1.2.3.tar.gz -> 1.2.3)
609	new_version=${NEW_FILE#${frontmatch}}
610	new_version=${new_version%${backmatch}}
611	new_serial=$(ver2ser $new_version)
612	# find length for comparison
613	maxlength=0
614	for name in $filelist
615	do
616	old_version=${name#${frontmatch}}
617	old_version=${old_version%${backmatch}}
618	old_serial=$(ver2ser $old_version)
619	cm1=$new_serial
620	cm2=$old_serial
621	while [ ${#cm1} -gt ${#cm2} ] ; do cm2="${cm2}00" ; done
622	while [ ${#cm2} -gt ${#cm1} ] ; do cm1="${cm1}00" ; done
623	[ ${#cm1} -gt ${maxlength} ] && maxlength=${#cm1}
624	done
625	# add 00 until length of serial matches maxlength
626	while [ ${#new_serial} -lt ${maxlength} ] ; do new_serial="${new_serial}00"; done
627	# now find the candidate with the lowest difference to new_serial
628	for name in $filelist
629	do
630	old_version=${name#${frontmatch}}
631	old_version=${old_version%${backmatch}}
632	old_serial=$(ver2ser $old_version)
633	while [ ${#old_serial} -lt ${maxlength} ] ; do old_serial="${old_serial}00"; done
634	let new_s=0x${new_serial}
635	let old_s=0x${old_serial}
636	if [ $new_s -gt $old_s ]
637	then
638	let serial_diff=0x${new_serial}-0x${old_serial}
639	else
640	let serial_diff=0x${old_serial}-0x${new_serial}
641	fi
642	if [ $serial_diff -le ${minimal_diff:-${serial_diff}} ]
643	then
644	best_candidate="$name"
645	minimal_diff=${serial_diff}
646	fi
647	done
648
649	output "${GREEN}The best of all is ... ${CYAN}${best_candidate}\n"
650	output "${YELLOW}Checking if this file is OK.\n"
651
652	# this part is based on Pkunk's code posted on http://bugs.gentoo.org/show_bug.cgi?id=63525
653	# but with some changes
654	FILE_IS_CORRUPT=false
655	if $CHECK_OLD_FILE
656	then
657	file_digest=$(grep -h ${best_candidate} ${FILESDIR}/digest-* \| sed -n 1p)
658	if [ ! -z "$file_digest" ]
659	then
660	file_md5=$(cut -d ' ' -f2 <<< $file_digest)
661	file_origsize=$(cut -d ' ' -f4 <<< $file_digest)
662	file_currentsize=$(stat -c %s ${best_candidate})
663	if [ $file_currentsize -ne $file_origsize ]
664	then
665	output "${RED}Found ${best_candidate}, but filesize ${CYAN}${file_currentsize} ${RED} does not match ${CYAN}${file_origsize} (found in digest-file)\n"
666	FILE_IS_CORRUPT=true
667	fi
668	else
669	if [ $(rev <<< ${best_candidate} \| cut -d. -f2 \| rev) = "tar" ]
670	then
671	output "${YELLOW}Could not find a digest-file for ${CYAN}${best_candidate}${YELLOW}. Testing file integrity with tar.\n"
672	case $(rev <<< ${best_candidate} \| cut -d. -f1 \| rev) in
673	gz) tarparm=z
674	;;
675	bz2) tarparm=j
676	;;
677	esac
678
679	if ! tar -${tarparm}tf ${best_candidate} >/dev/null
680	then
681	output "${RED}reported an error while testing ${CYAN}${best_candidate}${RED} - so this file is unusable.\n"
682	FILE_IS_CORRUPT=true
683	fi
684
685	if $FILE_IS_CORRUPT && $REMOVE_INCOMPLETE_OLD_FILES
686	then
687	output "${YELLOW}You have chosen to automatically delete such broken files from your distfiles-directory, so here we go...\n"
688	remove ${best_candidate}
689	fi
690	fi
691	fi
692	fi
693	# end of file-corruption check for $best_candidate found in distfiles
694	if ! $FILE_IS_CORRUPT
695	then
696
697	QUERY_URL=$(GENTOO_MIRRORS="" emerge -fp =${CATEGORY}/${PF} 2>&1 \|
698	sed -e "s/ /\\n/g" \| egrep "(http\|ftp)://" \|
699	grep "${NEW_FILE}" \| tail -n 1)
700	query="?have=${best_candidate}&want=${NEW_FILE}&url=${QUERY_URL}&version=${VERSION}&time=$(date +%s)"
701	output "${GREEN}Trying to download ${YELLOW}${best_candidate}-${NEW_FILE}.dtu\n"
702
703	# Remember where we are, and go to a new dir there we can work
704	tmp_dwn_dest="${DISTDIR}/.getdelta-`date +%N`-tmp"
705	mkdir ${tmp_dwn_dest}
706	# If user abort Ctrl+C (signal 2), remove tmp-dir; enabable trap again and send it again to stop wget
707	trap "rm -r ${tmp_dwn_dest}; trap 2; kill -2 $$" 2
708	pushd ${tmp_dwn_dest} >/dev/null 2>&1
709
710	# thanks to MATSUI Fe2+ Tetsushi for idea and patch
711	FILESIZE=$(stat -c %s "${DISTDIR}/${best_candidate}")
712	let TIMELIMIT=${FILESIZE}/${BANDWIDTH}
713	[[ $TIMELIMIT -lt $QUEUETIMEOUT ]] && QUEUETIMEOUT=$TIMELIMIT
714
715	if $FETCH "${DELTUP_SERVER}${query}"
716	then
717	# thanks to deelkar for this much more elegant solution to the "broken pipe" problem with "head -n1"
718	GOTFILE=$(ls -c \| sed -n 1p)
719	output "${YELLOW}GOT ${CYAN}${GOTFILE}\n"
720
721	# There are some possibilities what the deltup-server
722	# may have sento to us.
723
724	# first: the request have been queued
725	if [ "${GOTFILE}" = "deltup-queued" ]
726	then
727	let QTMOUT=$(date +%s)+QUEUETIMEOUT
728	while [ -f deltup-queued ]
729	do
730	output "${GREEN}destination file: ${CYAN}${NEW_FILE}\n"
731	output "${YELLOW}$(cat deltup-queued)"
732	QUEUEPOS=$(grep "has been queued" deltup-queued \| cut -d. -f2 \| cut -d")" -f1)
733	rm -f deltup-queued
734	TSTAMP=$(date +%s)
735	if ((TSTAMP<QTMOUT)) && ((QUEUEPOS<=MAXIMUM_ACCEPTABLE_QUEUEPOS))
736	then
737	for ((sec=QUEUERETRY;sec>0;sec--))
738	do
739	if ((sec>1))
740	then
741	${VERBOSITY} && echo -n -e "${YELLOW} I will try again in ${sec} seconds. \r"
742	else
743	${VERBOSITY} && echo -n -e "${YELLOW} I will try again in ${sec} second. \r"
744	fi
745	sleep 1
746	done
747	echo -n -e "${NORMAL}"
748	$FETCH "${DELTUP_SERVER}${query}"
749	GOTFILE=$(ls -c \| sed -n 1p)
750	else
751	if ((TSTAMP>=QTMOUT))
752	then
753	GOTFILE="timeout"
754	output "\n${RED}TIMEOUT exceeded.\n"
755	break
756	fi
757	if ((QUEUEPOS>MAXIMUM_ACCEPTABLE_QUEUEPOS))
758	then
759	GOTFILE="unacceptable"
760	output "\n${RED}You have configured getdelta.sh not to accept this queue-position.\n"
761	output "${YELLOW}We are going to download the ${RED}full archive${YELLOW} instead.\n"
762	break
763	fi
764	fi
765	done
766	fi
767
768	if [ -f ${best_candidate}-${NEW_FILE}.failed ]
769	then
770	output "\n${RED}The server could not build the dtu-file for ${NEW_FILE}\n"
771	output "${YELLOW}reason:\n${RED}$(cat ${best_candidate}-${NEW_FILE}.failed)\n"
772	rm -rf ${best_candidate}-${NEW_FILE}.failed
773	fi
774
775	if [ -f ${best_candidate}-${NEW_FILE}.dtu ]
776	then
777	output "${GREEN}Successfully fetched the dtu-file - let's build ${NEW_FILE}...\n"
778	downloadsize=$(stat -c %s ${best_candidate}-${NEW_FILE}.dtu)
779	if deltup -p -v -D ${DISTDIR} ${best_candidate}-${NEW_FILE}.dtu
780	then
781	newsize=$(stat -c %s ${NEW_FILE})
782	let savedsize=${newsize}-${downloadsize}
783	let percent=${savedsize}*100/${newsize}
784	unit="bytes"
785	[ $savedsize -gt 1024 ] && let savedsize=$savedsize/1024 && unit="kB"
786	[ $savedsize -gt 1024 ] && let savedsize=$savedsize/1024 && unit="MB"
787
788	case $unit in
789	bytes) UCOLOR=${RED}
790	;;
791	kB) UCOLOR=${YELLOW}
792	;;
793	MB) UCOLOR=${GREEN}
794	;;
795	esac
796	output "${YELLOW}This dtu-file saved ${UCOLOR}${savedsize} ${unit} (${percent}%)${YELLOW} download size.\n"
797	fi
798	mv -f ${NEW_FILE} ${DISTDIR} &&
799	${REMOVE_OLD} && remove "${best_candidate}"
800	fi
801
802	FILEEXT=$(rev <<< $GOTFILE \| cut -c 1-11 \| rev)
803	if [ $FILEEXT = ".tar.xdelta" ]
804	then
805	# we haven't received a dtu-file, but an xdelta instead
806	# this means the deltup-server redirected us to ftp.kde.org
807	# to get the official delta-file from there
808	output "${GREEN}This is an xdelta from ftp.kde.org...\n"
809	output "${GREEN}Applying...\n"
810
811	bunzip2 ${DISTDIR}/${best_candidate}
812	xdelta patch $GOTFILE
813	if ${REMOVE_OLD}
814	then
815	remove "$(rev <<< ${best_candidate} \| cut -c 5- \| rev)"
816	else
817	bzip2 $(rev <<< ${best_candidate} \| cut -c 5- \| rev)
818	fi
819	bzip2 $(rev <<< $NEW_FILE \| cut -c 5- \| rev)
820	rm -f $GOTFILE
821	mv -f ${NEW_FILE} ${DISTDIR}
822	output "${GREEN}Succesfully done\n"
823	fi
824	fi # if $FETCH "${DELTUP_SERVER}${query}"
825
826	# Clean up.
827	# We might got an important message
828	if [ -f ${tmp_dwn_dest}/deltup-server.msg ]
829	then
830	echo -e "${RED}IMPORTANT MESSAGE FROM DELTUP-SERVER${YELLOW}$(cat ${tmp_dwn_dest}/deltup-server.msg)\n"
831	for ((i=1;i<=5;i++))
832	do
833	echo -n -e "\a"
834	sleep 1
835	done
836	echo -e "${YELLOW}PRESS ENTER TO DOWNLOAD FROM ORIGINAL URL"
837	echo -e "${GREEN}or CTRL-C to cancel${NORMAL}"
838	read
839	fi
840	popd >/dev/null 2>&1
841	rm -rf ${tmp_dwn_dest}
842	#stop respond to trap2
843	trap 2
844	fi # if ! FILE_IS_CORRUPT
845	else # if ! [ -z "$filelist" ]
846	# No filelist - probably we do not have an old version of the file
847	output "${RED}No old version of the requested file found.\n"
848	fi
849
850
851	# Ok, once we are here, we should have got the delta (and used it)
852	# or we still have to download the full file
853	if ! [ -f ${DISTDIR}/${NEW_FILE} ]
854	then
855	output "${RED}The dtu could not be fetched,${YELLOW} downloading full file from original URL\n"
856	$FETCH -O ${NEW_FILE}.-PARTIAL- $ORIG_URI
857	# remember we had a fallback to use correct exitcode for portage
858	FALLBACK=$?
859	fi
860
861
862	if $SEPARATED_WINDOW
863	then
864	sleep 3
865	kill $termpid
866	fi
867
868	$DELETE_LOG && : >$LOGFILE
869
870	popd >/dev/null 2>&1
871
872
873	if ! [ -z $FALLBACK ]
874	then
875	exit $FALLBACK
876	fi

Download in other formats:

Original Format