Context Navigation

Back to Ticket #68

Ticket #68: getdelta-paludis.sh

File getdelta-paludis.sh, 29.5 KB (added by eternaleye@…, 3 years ago)
A modified getdelta.sh script that plays nice with paludis and --safe-resume

Line
1	#!/bin/sh
2	# getdelta.sh
3	# A download-wrapper script for gentoo that tries to get dtu files
4	# created by deltup instead of downloading complete source-files
5	# to save bandwidth.
6	#
7	# (C) 2004-2006 Nicolai Lissner <nlissne@linux01.gwdg.de>
8	# This script is free software; you can redistribute it and/or modify
9	# it under the terms of the GNU General Public License as published by
10	# the Free Software Foundation; either version 2 of the License , or
11	# (at your option) any later version.
12	#
13	# This program is distributed in the hope that it will be useful,
14	# but WITHOUT ANY WARRANTY; without even the implied warranty of
15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16	# GNU General Public License for more details.
17	#
18	# You should have received a copy of the GNU General Public License
19	# along with this program. If not, write to
20	#
21	# The Free Software Foundation, Inc.
22	# 59 Temple Place, Suite 330
23	# Boston, MA 02111, USA.
24
25	VERSION="0.7"
26
27	# Changelog
28	# version 0.7.6 2006/10/08
29	# - fixed a typo - thanks to Andrey, who reported this problem in gentoo bug #150426
30	# version 0.7.5 2006/10/03
31	# - added support for changing timeout based on expected filesize
32	# if configured it will reduce the waiting timeout to the expected download-time (based on size of old version)
33	# version 0.7.4 2006/09/06
34	# - do not remove log file but reset only to make this work with FEATURE userfetch
35	# version 0.7.3
36	# 2006/09/01
37	# - fixed a bug in detection of original url (sometimes no url was found)
38	# version 0.7.2
39	# 2006/08/18
40	# - improved method of chosing the right candidate
41	# version 0.7.1
42	# 2006/08/08
43	# - fixed a problem in counting differences in filenames (aka bug #105011)
44	# -
45	# version 0.7
46	# 2005/05/09
47	# - servers create dtu files based on bdelta instead of xdelta
48	# this happens for two reasons: smaller dtu-files and amd64-compatibilty
49	# (yes, it's true - welcome to all new amd64 users of the servers)
50	# this is the reason for major update - and update forcing - you really
51	# NEED bdelta now to use the servers, while you do not need xdelta anymore :)
52	# - integrity change of old candidate is optional now and disabled by default
53	# if you want this time consuming "safe way" re-enable it via the config-file
54	# - added some files to DO_NOT_REMOVE file
55	# - added "&time=<timestamp>" to prevent ANY proxy from returning results from cache
56	# instead of asking the server - the server does not use this parameter - it just
57	# exists to create unique request-URLS (as proposed by bodo "bothie" thiesen)
58	# version 0.6.9 2005/03/11
59	# � exit with exitcode of wget to signal to portage if
60	# download was successful
61	# - handle metadata.dtd as exception (repoman uses FETCHCOMMAND)
62	# - you can disable fetching of dtu-files now by setting
63	# the environment variable GETDELTA=0
64	# version 0.6.8 2005/01/09
65	# - init frontmatch and backmatch with 0 (thanks, Torsten Veller)
66	# I wonder when it disappeared from the script
67	# version 0.6.7 2004/12/22
68	# - corrected the formula for the saved size in percent
69	# as reported by Torsten Veller
70	# version 0.6.6 2004/12/21
71	# - ignore "try" in MPlayer filename
72	# version 0.6.5 2004/11/15
73	# - added information about the saved traffic
74	# - fix: use NORMAL color after first waiting for retry
75	# and QUEUETIMEOUT works now (thanks to Bodo Thiesen for the patch)
76	# - ignore "PR" in filename if the filename starts with firefox
77	# version 0.6.4 2004/11/08
78	# - inserted "break" to the TSTAMP>=QTMOUNT condition, too
79	# (reported by Torsten Veller)
80	# version 0.6.3 2004/11/08
81	# - added "^bash" and "^gtk-engines" to the default
82	# do_not_remove file
83	# - inserted a "break" to prevent infinite looping
84	# when the server returns a queueposition higher than
85	# the allowed number (as reported by James Rowe and others)
86	# version 0.6.2 2004/10/22
87	# - changed QUERY_URL to get the last URL instead the first
88	# since that's the original server (not a mirror)
89	# version 0.6.1 2004/10/18
90	# - give better original URL to the server
91	# - enhanced detection of former version (thanks to Jimmy Wennlund)
92	# - the COLOR variable didn't work since 0.5.3, because
93	# the config file was not read before evaluating the variable
94	# - check, if the user have set RESUMECOMMAND to getdelta.sh
95	# and if so complain about this
96	#
97	# version 0.6 2004/10/12
98	# - dropped the client-side mirror-detection
99	# - Jimmy Wennlund <jimmy@jw.dyndns.org> sent me patch to
100	# make getdelta.sh work in an own tempdir and to
101	# remove any temporary files even when user pressed
102	# ctrl-c -- I really like that. Thanks, Jimmy.
103	# version 0.5.4 2004/10/11
104	# - fixed a security leak (possible symlink attack)
105	# thanks to Raimund Specht <raimund@spemaus.de> for
106	# reporting the problem and sending some possible solutions.
107	# version 0.5.3.5 2004/10/02
108	# - the DO_NOT_REMOVE-file was overwritten with defaults
109	# fixed.
110	# version 0.5.3.4 2004/09/20
111	# - fixed wrong path-detection with thirdpartymirrors
112	# version 0.5.3.3 2004/09/12
113	# - changed the way the script finds the mirror-group to use
114	# - setting GENTOO_MIRRORS="" is not necessary anymore
115	# removed check and warning about that
116	# version 0.5.3.2 2004/09/12
117	# - fixed a bug in the ouput of remove() (thx to wiebel)
118	# version 0.5.3.1 2004/09/12
119	# - fixed a typo (FILESDIR)
120	# - fixed a missing "]"
121	# version 0.5.3 2004/09/12
122	# - some code cleanups
123	# - use a separated config file now
124	# - old file in DISTDIR is tested on corruption before trying to download a dtu
125	# (thanks to pkunk)
126	# - check for GENTOO_MIRRORS=""
127	# - new LOCAL_MIRROR to check before requesting a dtu
128	# - non existing DO_NOT_REMOVE-file is created with some defaults
129	# - found a severe bug in finding candidates when updating files beginning with "lib"
130	# - added MAXIMUM_ACCEPTABLE_QUEUEPOS
131	# version 0.5.2.3 2004/09/06
132	# - new variable QUERY_RETRY
133	# - dont remove file added
134	# version 0.5.2.2 2004/08/30
135	# - fixed a typo
136	# version 0.5.2.1 2004/08/29
137	# - fixed "too many arguments" as suggested by NoUseForAName
138	# in posting http://forums.gentoo.org/viewtopic.php?p=1480776#1480776
139	# version 0.5.2 2003/08/27
140	# - server sends a queued-message including queue-position now
141	# show this.
142	# version 0.5.1 2003/08/24
143	# - for some reason a "broken pipe" message appears when
144	# this script is called by portage/python, caused by
145	# "ls -c\|head -n1" - Ok, that IS a broken pipe, "head"
146	# would not read anything more than 1 line, but I do not
147	# really understand, why it does not happen when the script
148	# is called manually -- ANY use of "head" in a pipe-construction
149	# would result in a "broken-pipe", but bash itself never
150	# complains about that. a cosmetic change to make the
151	# output clean and the script-code ugly :-/
152	# version 0.5.0 2003/08/21
153	# - the exception handling for kde changed to the server
154	# - this script now checks if it got a dtu or xdelta
155	# - added timeout (to prevent endless loops in case of problems)
156	#
157	# version 0.4.0 2003/07/06
158	# the deltup-server queues requests now
159	# and sends back a document "deltup-queued"
160	# the client then waits 10 seconds and tries
161	# again until it either gets the dtu or a file
162	# named *.failed
163	# version 0.3.3 2003/05/06
164	# transmit version to server
165	# receive important messages from server
166	# version 0.3.2 2003/05/05
167	# correct handle of src-archives of X11-org
168	# version 0.3.1 2003/04/26
169	# fixed path to kde-sources on kde-mirror
170	# version 0.3.0 2004/04/20
171	# exception: get kde-version as xdelta-files from kde-mirror
172	#
173	# version 0.2.4 2004/04/15
174	# colors are now optional
175	# candidates named lib* are found faster now
176	# little enhancements on verbosity
177	# version 0.2.3 2004/04/14
178	# colorized verbosity
179	# fixed a bug that leaded to wrong candidates and error-outputs
180	# version 0.2.2 2004/04/14
181	# verbosity added by wiebel
182	# initialize frontmatch / backmatch with 0
183	# version 0.2.1 2004/04/14
184	# ignore spaces and "+" in filename-mask, too
185	# option REMOVE_OLD added
186	# version 0.2 2004/04/13
187	# old files can differ by one char from the wanted file
188	# to catch versions with letters in it
189	#
190	# version 0.1.1 2004/04/08
191	# changed method to determine which mirror to use
192	#
193	# version 0.1
194	# initial version 2004/04/06
195	#
196	#
197	####################################################
198	# NO variables to set here in the script anymore #
199	# we use a config-file instead which is created #
200	# and filled with some default values on first run #
201	# This file: #
202	####################################################
203
204	GETDELTA_CONFIGFILE=/etc/deltup/getdelta.rc
205	#
206
207	splitversion(){
208	# $1: the version string
209	# output: the splitted version (1.2.3 -> 1 2 3, 10.11.12b -> 10 11 12 b)
210	local vstr=$1
211	shopt -s extglob
212	while [ -n "$vstr" ]
213	do
214	case ${vstr:0:1} in
215	[[:digit:]])
216	echo "${vstr%%[^[:digit:]]*}"
217	vstr="${vstr##+([[:digit:]])}"
218	;;
219	[[:alpha:]])
220	nomatch="${vstr##+([[:alpha:]])*([[:digit:]])}"
221	echo "${vstr:0:$((${#vstr} - ${#nomatch}))}"
222	vstr="${nomatch}"
223	;;
224	*) vstr="${vstr:1}"
225	;;
226	esac
227	done
228	shopt -u extglob
229	}
230
231	ver2ser(){
232	local x=$(splitversion $1)
233	x=($x)
234	vser=""
235	shopt -s extglob
236	for ((i=0;i<${#x[@]};i++))
237	do
238	case ${x[$i]:0:1} in
239	[[:digit:]])
240	# let ad=${x[$i]##+(0)}+1
241	[ "${x[$i]##+(0)}" ] && let ad=${x[$i]##+(0)} \|\| ad=0
242	vs=$(printf "%02x" ${ad})
243	vser="${vser}${vs}"
244	;;
245	[[:alpha:]])
246	calced=0
247	if [ "${x[$i]:0:3}" = "pre" ]
248	then
249	vs=${x[$i]:3}
250	let vser=0x${vser}00-40+${vs:-0}
251	vser=$(printf "%02x" $vser)
252	calced=1
253	fi
254	if [ "${x[$i]:0:2}" = "rc" ]
255	then
256	vs=${x[$i]:2}
257	let vser=0x${vser}00-40+${vs:-0}
258	vser=$(printf "%02x" $vser)
259	calced=1
260	fi
261	if [ "${x[$i]:0:5}" = "alpha" ]
262	then
263	vs=${x[$i]:5}
264	let vser=0x${vser}00-80+${vs:-0}
265	vser=$(printf "%02x" $vser)
266	calced=1
267	fi
268	if [ "${x[$i]:0:4}" = "beta" ]
269	then
270	vs=${x[$i]:4}
271	let vser=0x${vser}00-60+${vs:-0}
272	vser=$(printf "%02x" $vser)
273	calced=1
274	fi
275	if [ "$calced" = "0" ]
276	then
277	vs=$(echo -n ${x[$i]} \| od -t x1 \| head -n1\| cut -d" " -f2-\| tr -d " ")
278	vser="${vser}${vs}"
279	fi
280	esac
281	done
282	shopt -u extglob
283	let m=${#vser}%2
284	[ "$m" = "1" ] && vser="0${vser}"
285	echo $vser
286	}
287
288
289	# some colors for colored output
290	output() {
291	${VERBOSITY} && echo -e "$1${NORMAL}" \| tee -a $LOGFILE
292	}
293
294	# this checks for a variable in our config-file and adds it if does not exist
295	# $1 is the name of the variable, $2 the default content of the variable
296	# $3 a description line for the variable
297	add_to_configfile() {
298	GETDELTA_CONFIGDIR=$(dirname $GETDELTA_CONFIGFILE )
299	[ -e $GETDELTA_CONFIGFILE ] \|\| ( mkdir -p $GETDELTA_CONFIGDIR && touch $GETDELTA_CONFIGFILE )
300	if ! grep -q "$1" $GETDELTA_CONFIGFILE
301	then
302	echo -e "\n# ${3}\n${1}=\"${2}\"" >>$GETDELTA_CONFIGFILE
303	output "${CYAN}Added new variable ${YELLOW}$1${CYAN} to config file ${GETDELTA_CONFIGFILE}\n"
304	output "please check if it fits your needs\n"
305	fi
306	}
307
308	# this checks for an entry in our do_not_remove-file and adds it if does not exist
309	# $1 is the name (as grep regexp) of the file not to be removed
310	add_to_donotremove() {
311
312	if ! grep -q "^${1}" $DO_NOT_REMOVE
313	then
314	echo "${1}" >>$DO_NOT_REMOVE
315	output "${CYAN}Added new grep-regex \"${1}\" to config file ${DO_NOT_REMOVE}\n"
316	fi
317	}
318
319
320	remove() {
321	output "${GREEN}You have chosen to remove ${CYAN}$1\n"
322	pushd ${DISTDIR} >/dev/null 2>&1
323	removeme=true
324	for n in $(grep -v "^#" ${DO_NOT_REMOVE})
325	do
326	grep -q $n <<< "$1" && removeme=false && output "${CYAN}${1}${RED} is not deleted, since it matches ${n} in ${DO_NOT_REMOVE}"
327	done
328	$removeme && rm -f $1
329	popd >/dev/null 2>&1
330	}
331
332
333	mask_name() {
334	MASK_FILENAME=$1
335	# do some "blackmagic" with the src-files of xorg
336
337	if [ $(cut -c 1-6 <<< $MASK_FILENAME) = "X11R6." ]
338	then
339	MASK_FILENAME=$(sed -e "s/src1/srcAAA/g" \
340	-e "s/src2/srcBBB/g" \
341	-e "s/src3/srcCCC/g" \
342	-e "s/src4/srcDDD/g" \
343	-e "s/src5/srcEEE/g" \
344	-e "s/src6/srcFFF/g" \
345	-e "s/src7/srcGGG/g" <<< $MASK_FILENAME)
346	fi
347
348	# ignore PR for src-files of firefox
349	if [ $(cut -c 1-7 <<< $MASK_FILENAME) = "firefox" ]
350	then
351	MASK_FILENAME=$(sed -e "s/PR//g" <<< $MASK_FILENAME)
352	fi
353
354	# ignore "try" with new mplayer
355	if [ $(cut -c 1-7 <<< $MASK_FILENAME) = "MPlayer" ]
356	then
357	MASK_FILENAME=$(sed -e "s/try//g" <<< $MASK_FILENAME)
358	fi
359
360
361	# ignore some strings in any filename
362	echo $(sed -e "s/\.bz2$//g" \
363	-e "s/\.gz$//g" \
364	-e "s/[0-9]//g" \
365	-e "s/pre//g" \
366	-e "s/preview//g" \
367	-e "s/beta//g" \
368	-e "s/rc//g" \
369	-e "s/[\._-]//g" \
370	-e "s/\+//g" \
371	-e "s/ //g" <<< $MASK_FILENAME)
372	}
373
374	# create or update a config-file
375
376	add_to_configfile KDE_MIRROR "ftp://ftp.kde.org/pub/kde/stable" "we de not get kde-deltas from a delta-up-server, since kde provides own xdelta-files"
377	add_to_configfile LOCAL_MIRROR "" "set this to an URI ending with '/' if you want to check a local mirror first\n# most people just leave it empty."
378	add_to_configfile DELTUP_SERVER "http://linux01.gwdg.de/~nlissne/deltup.php" "deltup-server to use"
379	add_to_configfile FETCH "/usr/bin/wget -t 1 --passive-ftp" "command to use for downloading"
380	add_to_configfile QUEUERETRY 15 "number of seconds to wait before a queued request is retried"
381	add_to_configfile MAXIMUM_ACCEPTABLE_QUEUEPOS "15" "the maximum queuepos you would accept (if higher download full archive instead)"
382	add_to_configfile QUEUETIMEOUT 900 "when a dtu-request is queued - how long should we wait max. before downloading the original archive instead (in seconds)"
383	add_to_configfile CHECK_OLD_FILE "false" "set to \"true\", if you want getdelta.sh to use Pkunk's integrity check for the old file before downloading dtu-files"
384	add_to_configfile REMOVE_OLD "false" "set to \"true\", if you want getdelta.sh to delete the old file, if patch was succesful"
385	add_to_configfile DO_NOT_REMOVE "/etc/deltup/do_not_remove" "a list of files not to be removed by REMOVE_OLD feature"
386	add_to_configfile REMOVE_INCOMPLETE_OLD_FILES "false" "set this to \"true\" if you want getdelta.sh to delete old versions that seems to be corrupt,\n# or to \"false\" if you want to delete them manually\n# note: getdelta.sh will not use these files anyway"
387	add_to_configfile VERBOSITY true "set to \"true\", if you want verbose outputs (later to be set to a level [0-3])"
388	add_to_configfile COLOR true "set to \"true\", if you want colorful messages, \"false\" if not."
389	add_to_configfile LOGFILE "/var/log/getdelta.log" "set to a writable file (or to \"/dev/null\" if you do not want this) this is not used, if VERBOSITY is false"
390	add_to_configfile DELETE_LOG true "set to \"true\" if you want a temporarily log only (deleted when getdelta is finished)"
391	add_to_configfile SEPARATED_WINDOW "false" "set to \"true\", if you want messages from this script in a separate window\n# set to \"false\", if you do not start getdelta.sh from an Xsession or if you \n# do not have permissions to open terminals on the Xserver"
392	add_to_configfile TERM_APP "aterm -tr -trsb -fg white -bg black -sh 70 -e tail -f ${LOGFILE}" "the terminal application to use for the separated window"
393	add_to_configfile BANDWIDTH 1 "the bandwidth in bytes per second. configure this if you want to reduce timeouts on small files"
394
395	source $GETDELTA_CONFIGFILE
396
397	# create or update DO_NOT_REMOVE file
398	# these files have "old" versions that are needed to build the new versions
399	# so they should never removed by the REMOVE_OLD feature
400	DO_NOT_REMOVE_DIR=$(dirname $DO_NOT_REMOVE)
401	if [ ! -e $DO_NOT_REMOVE ]
402	then
403	mkdir -p $DO_NOT_REMOVE_DIR
404	echo "# This file contains regexp in 'grep-style' for files that should not be removed" >$DO_NOT_REMOVE
405	echo "# if REMOVE_OLD is set to 'true'" >>$DO_NOT_REMOVE
406	echo "# Some examples (actually these files are known to result" >>$DO_NOT_REMOVE
407	echo "# in problems if getdelta.sh is used with REMOVE_OLD=true" >>$DO_NOT_REMOVE
408	fi
409	add_to_donotremove "^font-arial-iso-8859"
410	add_to_donotremove "^libtool"
411	add_to_donotremove "^readline"
412	add_to_donotremove "^gtk-engines"
413	add_to_donotremove "^bash"
414	add_to_donotremove "^openssl"
415	add_to_donotremove "^curl"
416	add_to_donotremove "^festvox"
417	add_to_donotremove "^rp-pppoe"
418
419
420	if [ -z $1 ]
421	then
422	COLOR=true
423	echo -e "${YELLOW}getdelta.sh version ${VERSION}"
424	echo "This script has to be called like this:"
425	echo -e "${CYAN}$0 <URI>"
426	echo -e "\n${YELLOW}To use it, you should just put the following line into your /etc/make.conf"
427	echo -e "${GREEN}FETCHCOMMAND=\"$0 \\\${URI}\""
428	echo -e "\n${YELLOW}There is a config-file ${CYAN}${GETDELTA_CONFIGFILE}${YELLOW} with some variables to control the behaviour of this script."
429	echo -e "Edit it to your needs.${NORMAL}"
430	exit 1
431	fi
432	# include variables from gentoo make.globals and make.conf
433	source /etc/make.globals
434	source /etc/make.conf
435
436
437	if ${COLOR}
438	then
439	RED="\033[01;31m"
440	GREEN="\033[01;32m"
441	YELLOW="\033[01;33m"
442	BLUE="\033[01;34m"
443	MAGENTA="\033[01;35m"
444	CYAN="\033[01;36m"
445	NORMAL="\033[00m"
446	else
447	RED=""
448	GREEN=""
449	YELLOW=""
450	BLUE=""
451	MAGENTA=""
452	CYAN=""
453	NORMAL=""
454	fi
455	grep -q "getdelta.sh" <<< "${RESUMECOMMAND}" &&
456	output "${RED}do NOT set RESUMECOMMAND to use getdelta.sh" &&
457	output "use getdelta.sh for your FETCHCOMMAND, only." &&
458	sleep 5 && exit 1
459
460	pushd $DISTDIR >/dev/null 2>/dev/null
461	ORIG_URI=$1
462	DEST_FILE=$2
463	NEW_FILE=$(basename $ORIG_URI)
464
465	# repoman downloads metadata.dtd with FETCHCOMMAND
466	# this should not be done with getdelta - so just fetch the file and exit
467
468	# Check if env.variable GETDELTA is set to 0 to disable fetching of
469	# dtu files.
470	if [ "${NEW_FILE}" = "metadata.dtd" ] \|\| [ "$GETDELTA" = "0" ]
471	then
472	$FETCH $@
473	exit $?
474	fi
475
476
477	[ -e deltup-server.msg ] && rm -f deltup-server.msg
478
479	# if output should go to an additional window start it
480	if $SEPARATED_WINDOW
481	then
482	touch $LOGFILE
483	$TERM_APP &
484	termpid=$!
485	echo -e "\x1b]1;\x07\x1b]2;getdelta.sh trying to get dtu for ${NEW_FILE}\x07"
486	fi
487
488	# First of all: check if LOCAL_MIRROR is set and provides the file in question already
489
490	if [ ! -z $LOCAL_MIRROR ]
491	then
492	output "${YELLOW}Trying to get ${CYAN}${NEW_FILE}${YELLOW} from local mirror ${CYAN}${LOCAL_MIRROR}\n"
493	if $FETCH "${LOCAL_MIRROR}${NEW_FILE}"
494	then
495	output "${GREEN}success.\n"
496	exit 0
497	else
498	output "${RED}failed${YELLOW} - need to get a delta file\n"
499	fi
500	fi
501
502
503	#
504	# find an old file in $DISTDIR that matches the new one. This is tricky,
505	# and probably it will fail sometimes.
506	#
507	# we just ignore any occurence of
508	# "pre","rc","[0-9]","_","-","." in the filenames and test
509	# if they are the same (or VERY similar (differ only in 1 char)).
510	# to reduce the files to check, we only check files
511	# with the same beginning
512	#
513	output "${GREEN}Searching for a previously downloaded file in ${YELLOW}${DISTDIR}\n"
514
515	first_chars=$(sed 's/[[:digit:]][[:print:]]*$//' <<< $NEW_FILE)
516	length_first_chars=$(wc -c <<< $first_chars)
517	[ $length_first_chars -lt 3 ] && first_chars=$(cut -c 1-2 <<< $NEW_FILE)
518
519	# if filename is lib* use first 4 letters to increase performance
520	[ "$( cut -c 1-3 <<< $NEW_FILE )" = "lib" ] &&
521	[ $length_first_chars -lt 5 ] && first_chars=$(cut -c 1-4 <<< $NEW_FILE)
522
523	mask=$(mask_name "${NEW_FILE}")
524	let len1=$(wc -c <<< $mask)-1
525	filelist=""
526
527	for name in $( ls ${first_chars}* 2>/dev/null )
528	do
529	mask2=$(mask_name "${name}")
530	# add any file, that results in the same mask or differ not more than two letters
531	let len2=$(wc -c <<< $mask2)-1
532	if [ $len1 -gt $len2 ]
533	then
534	max=${len1}
535	let min=${len2}
536	else
537	let min=${len1}
538	max=${len2}
539	fi
540	let df=${max}-${min}
541
542	# if masks differ in length more than 1 they cannot match
543	if [ $df -le 1 ]
544	then
545	let frontmatch=0
546	let backmatch=0
547	for ((ch=1;ch<=min;ch++))
548	do
549	if [ $(cut -c ${ch} <<< ${mask}) = $(cut -c ${ch} <<< ${mask2}) ]
550	then frontmatch=${ch}
551	else break
552	fi
553	done
554
555	# now backwards
556	mask=$(rev <<< ${mask})
557	mask2=$(rev <<< ${mask2})
558	for ((ch=1;ch<=min;ch++))
559	do
560	if [ $(cut -c ${ch} <<< ${mask} ) = $(cut -c ${ch} <<< ${mask2}) ]
561	then backmatch=${ch}
562	else break
563	fi
564	done
565
566	# forwards for mask again (need this for the next run of the loop)
567	mask=$(rev <<< ${mask})
568
569	let matchall=${frontmatch}+${backmatch}
570	let minmatch=${min}-1
571	[ ${matchall} -ge ${minmatch} ] && filelist="${filelist} $name"
572	fi
573	done
574
575	if ! [ -z "$filelist" ]
576	then
577	# we have got a list of candidates in $filelist now. find the best match .
578	output "${GREEN}We have the following candidates to choose from \n${YELLOW}`sed -e \"s/\ /\\n/g\" <<< $filelist` \n"
579
580	# find matching part of filename - first: frontmatch
581	x=0;
582	a=($NEW_FILE $filelist)
583	match=""
584	while [ -z "$match" ]
585	do
586	for ((i=0;i<${#a[@]};i++))
587	do
588	[ ${a[0]:${x}:1} != ${a[$i]:${x}:1} ] && match=$x
589	done
590	((x++))
591	done
592	frontmatch=${a[0]:0:${match}}
593
594	# find matching part of filename - second: backmatch
595	x=1;
596	match=""
597	while [ -z "$match" ]
598	do
599	for ((i=0;i<${#a[@]};i++))
600	do
601	[ ${a[0]:${#a[0]}-${x}:1} != ${a[$i]:${#a[$i]}-${x}:1} ] && match=$x
602	done
603	((x++))
604	done
605	((match--))
606	backmatch=${a[0]:${#a[0]}-${match}}
607
608	# isolate version from filename (foobar-1.2.3.tar.gz -> 1.2.3)
609	new_version=${NEW_FILE#${frontmatch}}
610	new_version=${new_version%${backmatch}}
611	new_serial=$(ver2ser $new_version)
612	# find length for comparison
613	maxlength=0
614	for name in $filelist
615	do
616	old_version=${name#${frontmatch}}
617	old_version=${old_version%${backmatch}}
618	old_serial=$(ver2ser $old_version)
619	cm1=$new_serial
620	cm2=$old_serial
621	while [ ${#cm1} -gt ${#cm2} ] ; do cm2="${cm2}00" ; done
622	while [ ${#cm2} -gt ${#cm1} ] ; do cm1="${cm1}00" ; done
623	[ ${#cm1} -gt ${maxlength} ] && maxlength=${#cm1}
624	done
625	# add 00 until length of serial matches maxlength
626	while [ ${#new_serial} -lt ${maxlength} ] ; do new_serial="${new_serial}00"; done
627	# now find the candidate with the lowest difference to new_serial
628	for name in $filelist
629	do
630	old_version=${name#${frontmatch}}
631	old_version=${old_version%${backmatch}}
632	old_serial=$(ver2ser $old_version)
633	while [ ${#old_serial} -lt ${maxlength} ] ; do old_serial="${old_serial}00"; done
634	let new_s=0x${new_serial}
635	let old_s=0x${old_serial}
636	if [ $new_s -gt $old_s ]
637	then
638	let serial_diff=0x${new_serial}-0x${old_serial}
639	else
640	let serial_diff=0x${old_serial}-0x${new_serial}
641	fi
642	if [ $serial_diff -le ${minimal_diff:-${serial_diff}} ]
643	then
644	best_candidate="$name"
645	minimal_diff=${serial_diff}
646	fi
647	done
648
649	output "${GREEN}The best of all is ... ${CYAN}${best_candidate}\n"
650	output "${YELLOW}Checking if this file is OK.\n"
651
652	# this part is based on Pkunk's code posted on http://bugs.gentoo.org/show_bug.cgi?id=63525
653	# but with some changes
654	FILE_IS_CORRUPT=false
655	if $CHECK_OLD_FILE
656	then
657	file_digest=$(grep -h ${best_candidate} ${FILESDIR}/digest-* \| sed -n 1p)
658	if [ ! -z "$file_digest" ]
659	then
660	file_md5=$(cut -d ' ' -f2 <<< $file_digest)
661	file_origsize=$(cut -d ' ' -f4 <<< $file_digest)
662	file_currentsize=$(stat -c %s ${best_candidate})
663	if [ $file_currentsize -ne $file_origsize ]
664	then
665	output "${RED}Found ${best_candidate}, but filesize ${CYAN}${file_currentsize} ${RED} does not match ${CYAN}${file_origsize} (found in digest-file)\n"
666	FILE_IS_CORRUPT=true
667	fi
668	else
669	if [ $(rev <<< ${best_candidate} \| cut -d. -f2 \| rev) = "tar" ]
670	then
671	output "${YELLOW}Could not find a digest-file for ${CYAN}${best_candidate}${YELLOW}. Testing file integrity with tar.\n"
672	case $(rev <<< ${best_candidate} \| cut -d. -f1 \| rev) in
673	gz) tarparm=z
674	;;
675	bz2) tarparm=j
676	;;
677	esac
678
679	if ! tar -${tarparm}tf ${best_candidate} >/dev/null
680	then
681	output "${RED}reported an error while testing ${CYAN}${best_candidate}${RED} - so this file is unusable.\n"
682	FILE_IS_CORRUPT=true
683	fi
684
685	if $FILE_IS_CORRUPT && $REMOVE_INCOMPLETE_OLD_FILES
686	then
687	output "${YELLOW}You have chosen to automatically delete such broken files from your distfiles-directory, so here we go...\n"
688	remove ${best_candidate}
689	fi
690	fi
691	fi
692	fi
693	# end of file-corruption check for $best_candidate found in distfiles
694	if ! $FILE_IS_CORRUPT
695	then
696
697	QUERY_URL=$(GENTOO_MIRRORS="" emerge -fp =${CATEGORY}/${PF} 2>&1 \|
698	sed -e "s/ /\\n/g" \| egrep "(http\|ftp)://" \|
699	grep "${NEW_FILE}" \| tail -n 1)
700	query="?have=${best_candidate}&want=${NEW_FILE}&url=${QUERY_URL}&version=${VERSION}&time=$(date +%s)"
701	output "${GREEN}Trying to download ${YELLOW}${best_candidate}-${NEW_FILE}.dtu\n"
702
703	# Remember where we are, and go to a new dir there we can work
704	tmp_dwn_dest="${DISTDIR}/.getdelta-`date +%N`-tmp"
705	mkdir ${tmp_dwn_dest}
706	# If user abort Ctrl+C (signal 2), remove tmp-dir; enabable trap again and send it again to stop wget
707	trap "rm -r ${tmp_dwn_dest}; trap 2; kill -2 $$" 2
708	pushd ${tmp_dwn_dest} >/dev/null 2>&1
709
710	# thanks to MATSUI Fe2+ Tetsushi for idea and patch
711	FILESIZE=$(stat -c %s "${DISTDIR}/${best_candidate}")
712	let TIMELIMIT=${FILESIZE}/${BANDWIDTH}
713	[[ $TIMELIMIT -lt $QUEUETIMEOUT ]] && QUEUETIMEOUT=$TIMELIMIT
714
715	if $FETCH "${DELTUP_SERVER}${query}"
716	then
717	# thanks to deelkar for this much more elegant solution to the "broken pipe" problem with "head -n1"
718	GOTFILE=$(ls -c \| sed -n 1p)
719	output "${YELLOW}GOT ${CYAN}${GOTFILE}\n"
720
721	# There are some possibilities what the deltup-server
722	# may have sento to us.
723
724	# first: the request have been queued
725	if [ "${GOTFILE}" = "deltup-queued" ]
726	then
727	let QTMOUT=$(date +%s)+QUEUETIMEOUT
728	while [ -f deltup-queued ]
729	do
730	output "${GREEN}destination file: ${CYAN}${NEW_FILE}\n"
731	output "${YELLOW}$(cat deltup-queued)"
732	QUEUEPOS=$(grep "has been queued" deltup-queued \| cut -d. -f2 \| cut -d")" -f1)
733	rm -f deltup-queued
734	TSTAMP=$(date +%s)
735	if ((TSTAMP<QTMOUT)) && ((QUEUEPOS<=MAXIMUM_ACCEPTABLE_QUEUEPOS))
736	then
737	for ((sec=QUEUERETRY;sec>0;sec--))
738	do
739	if ((sec>1))
740	then
741	${VERBOSITY} && echo -n -e "${YELLOW} I will try again in ${sec} seconds. \r"
742	else
743	${VERBOSITY} && echo -n -e "${YELLOW} I will try again in ${sec} second. \r"
744	fi
745	sleep 1
746	done
747	echo -n -e "${NORMAL}"
748	$FETCH "${DELTUP_SERVER}${query}"
749	GOTFILE=$(ls -c \| sed -n 1p)
750	else
751	if ((TSTAMP>=QTMOUT))
752	then
753	GOTFILE="timeout"
754	output "\n${RED}TIMEOUT exceeded.\n"
755	break
756	fi
757	if ((QUEUEPOS>MAXIMUM_ACCEPTABLE_QUEUEPOS))
758	then
759	GOTFILE="unacceptable"
760	output "\n${RED}You have configured getdelta.sh not to accept this queue-position.\n"
761	output "${YELLOW}We are going to download the ${RED}full archive${YELLOW} instead.\n"
762	break
763	fi
764	fi
765	done
766	fi
767
768	if [ -f ${best_candidate}-${NEW_FILE}.failed ]
769	then
770	output "\n${RED}The server could not build the dtu-file for ${NEW_FILE}\n"
771	output "${YELLOW}reason:\n${RED}$(cat ${best_candidate}-${NEW_FILE}.failed)\n"
772	rm -rf ${best_candidate}-${NEW_FILE}.failed
773	fi
774
775	if [ -f ${best_candidate}-${NEW_FILE}.dtu ]
776	then
777	output "${GREEN}Successfully fetched the dtu-file - let's build ${NEW_FILE}...\n"
778	downloadsize=$(stat -c %s ${best_candidate}-${NEW_FILE}.dtu)
779	if deltup -p -v -D ${DISTDIR} ${best_candidate}-${NEW_FILE}.dtu
780	then
781	newsize=$(stat -c %s ${NEW_FILE})
782	let savedsize=${newsize}-${downloadsize}
783	let percent=${savedsize}*100/${newsize}
784	unit="bytes"
785	[ $savedsize -gt 1024 ] && let savedsize=$savedsize/1024 && unit="kB"
786	[ $savedsize -gt 1024 ] && let savedsize=$savedsize/1024 && unit="MB"
787
788	case $unit in
789	bytes) UCOLOR=${RED}
790	;;
791	kB) UCOLOR=${YELLOW}
792	;;
793	MB) UCOLOR=${GREEN}
794	;;
795	esac
796	output "${YELLOW}This dtu-file saved ${UCOLOR}${savedsize} ${unit} (${percent}%)${YELLOW} download size.\n"
797	fi
798	mv -f ${NEW_FILE} ${DISTDIR} &&
799	${REMOVE_OLD} && remove "${best_candidate}"
800	fi
801
802	FILEEXT=$(rev <<< $GOTFILE \| cut -c 1-11 \| rev)
803	if [ $FILEEXT = ".tar.xdelta" ]
804	then
805	# we haven't received a dtu-file, but an xdelta instead
806	# this means the deltup-server redirected us to ftp.kde.org
807	# to get the official delta-file from there
808	output "${GREEN}This is an xdelta from ftp.kde.org...\n"
809	output "${GREEN}Applying...\n"
810
811	bunzip2 ${DISTDIR}/${best_candidate}
812	xdelta patch $GOTFILE
813	if ${REMOVE_OLD}
814	then
815	remove "$(rev <<< ${best_candidate} \| cut -c 5- \| rev)"
816	else
817	bzip2 $(rev <<< ${best_candidate} \| cut -c 5- \| rev)
818	fi
819	bzip2 $(rev <<< $NEW_FILE \| cut -c 5- \| rev)
820	rm -f $GOTFILE
821	mv -f ${NEW_FILE} ${DISTDIR}
822	output "${GREEN}Succesfully done\n"
823	fi
824	fi # if $FETCH "${DELTUP_SERVER}${query}"
825
826	# Clean up.
827	# We might got an important message
828	if [ -f ${tmp_dwn_dest}/deltup-server.msg ]
829	then
830	echo -e "${RED}IMPORTANT MESSAGE FROM DELTUP-SERVER${YELLOW}$(cat ${tmp_dwn_dest}/deltup-server.msg)\n"
831	for ((i=1;i<=5;i++))
832	do
833	echo -n -e "\a"
834	sleep 1
835	done
836	echo -e "${YELLOW}PRESS ENTER TO DOWNLOAD FROM ORIGINAL URL"
837	echo -e "${GREEN}or CTRL-C to cancel${NORMAL}"
838	read
839	fi
840	popd >/dev/null 2>&1
841	rm -rf ${tmp_dwn_dest}
842	#stop respond to trap2
843	trap 2
844	fi # if ! FILE_IS_CORRUPT
845	else # if ! [ -z "$filelist" ]
846	# No filelist - probably we do not have an old version of the file
847	output "${RED}No old version of the requested file found.\n"
848	fi
849
850
851	# Ok, once we are here, we should have got the delta (and used it)
852	# or we still have to download the full file
853	if ! [ -f ${DISTDIR}/${NEW_FILE} ]
854	then
855	output "${RED}The dtu could not be fetched,${YELLOW} downloading full file from original URL\n"
856	$FETCH $DEST_FILE $ORIG_URI
857	# remember we had a fallback to use correct exitcode for portage
858	FALLBACK=$?
859	fi
860
861
862	if $SEPARATED_WINDOW
863	then
864	sleep 3
865	kill $termpid
866	fi
867
868	$DELETE_LOG && : >$LOGFILE
869
870	popd >/dev/null 2>&1
871
872
873	if ! [ -z $FALLBACK ]
874	then
875	exit $FALLBACK
876	fi

Download in other formats:

Original Format