Date: Sat, 21 Jan 2006 22:00:43 -0600 From: "Kelly D. Grills" <kdgrills@the-grills.com> To: freebsd-questions@freebsd.org Subject: cvsup6.us.freebsd.org / /sysutils/fastest_cvsup Message-ID: <20060122040002.GA52928@the-grills.com>
next in thread | raw e-mail | index | archive | help
--HWvPVVuAAfuRc6SZ
Content-Type: multipart/mixed; boundary="Izn7cH1Com+I3R9J"
Content-Disposition: inline
--Izn7cH1Com+I3R9J
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
Greetings,
Recently there was a thread regarding cvsup6.us.freebsd.org
accepting connections, and offering no content, which breaks
fastest_cvsup.
I've done a little hacking on fastest_cvsup version 0.2.9
to enable verification of the desired collections existence.
Please see attached patch
--
Kelly D. Grills
kdgrills@the-grills.com
--Izn7cH1Com+I3R9J
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="fastest_cvsup.patch"
Content-Transfer-Encoding: quoted-printable
*** old/fastest_cvsup Sat Jan 21 21:12:05 2006
--- new/fastest_cvsup Sat Jan 21 21:52:31 2006
***************
*** 8,13 ****
--- 8,34 ----
#------------------------------------------------------------------------=
-----#
#
# Changes:
+ # 0.2.9.1 - kdgrills@the-grills.com, Jan, 2006:
+ # added a -v command line argument to enable verification of the
+ # collections existence on the server. We create a dummy cvsup
+ # configuration with a refuse file which refuses everything,
+ # in order to check the server. If the server is accepting
+ # connections but doesn't carry our collection we'll get an err=
or,
+ # and can discard the server. This has been the case with
+ # cvsup6.us.freebsd.org for a while now:
+ #
+ # Connecting to cvsup6.us.freebsd.org
+ # Connected to cvsup6.us.freebsd.org
+ # Server software version: SNAP_16_1h
+ # Negotiating file attribute support
+ # Exchanging collection information
+ # Server message: Collection "ports-all" release "cvs" is not a=
vailable here
+ # Establishing multiplexed-mode data connection
+ # Running
+ # Skipping collection ports-all/cvs
+ # Shutting down connection to server
+ # Finished successfully
+ #
# 0.2.9 - added 'us' country code for USA hosts and 'tld' for top-level
# domain machines (can be in any country)
# 0.2.8 - fixed the progress meter, timestamped %FREEBSD_SERVERS
***************
*** 71,76 ****
--- 92,119 ----
# in the online Handbook? (only root can do this)
=20
#------------------------------------------------------------------------=
-----#
+ # kdgrills@the-grills.com, Variables for verification
+ #------------------------------------------------------------------------=
-----#
+=20
+ # User Configurable Variables (you can change these):
+ # Set either ports or src, not both
+=20
+ my $CVSUP_CMD =3D "/usr/local/bin/cvsup -g -L 2"; # cvsup command line
+ my $CVSUP_BASE =3D "/tmp/fastest_cvsup"; # directory for our du=
mmy cvsup configuration
+=20
+ my $COLLECTION =3D "ports-all"; # check for the ports =
collection
+ my $PREFIX =3D "/ports"; # ports would go here,=
cvsup won't run without it
+ my $TAG =3D "."; # tag for ports-* coll=
ection
+=20
+ #my $COLLECTION =3D "src-all"; # check for the sourc=
e collection
+ #my $PREFIX =3D "/src"; # source would go her=
e, cvsup won't run without it
+ #my $TAG =3D "RELENG_6_0"; # tag for release bra=
nch
+=20
+ #------------------------------------------------------------------------=
-----#
+ # End kdgrills@the-grills.com modifications
+ #------------------------------------------------------------------------=
-----#
+=20
+ #------------------------------------------------------------------------=
-----#
# Other variables
=20
# How many FreeBSD CVSup servers has each country got? (well, not strictly
***************
*** 179,184 ****
--- 222,228 ----
=20
#------------------------------------------------------------------------=
-----#
# Usage() sub
+ # kdgrills@the-grills.com, added -v option
=20
sub Usage {
my $error =3D shift;
***************
*** 189,194 ****
--- 233,239 ----
" -q quiet mode, only outputs fastest server\n",
" -Q very quiet mode, no progress meter\n",
" -r uses remote server list from FreeBSD Handb=
ook\n",
+ " -v verifies existence of the collection on se=
rver\n",
" -c aa,bb,cc queries servers in countries aa,bb,cc\n",
" tld queries servers in the top level domain\n",
" local queries servers set as local in the script=
\n",
***************
*** 201,209 ****
# get user supplied options
=20
# we need the ':' to store the actual values of the -c switch
=20
my %opt =3D ();
! getopts("hrqQc:", \%opt);
=20
# if no args given, or help page requested
=20
--- 246,255 ----
# get user supplied options
=20
# we need the ':' to store the actual values of the -c switch
+ # kdgrills@the-grills.com, added -v option
=20
my %opt =3D ();
! getopts("hrqQvc:", \%opt);
=20
# if no args given, or help page requested
=20
***************
*** 222,227 ****
--- 268,284 ----
my $REMOTE_SERVERLIST =3D 0; # default is no, use internal list
if ( $opt{'r'} ) { $REMOTE_SERVERLIST =3D 1; }
=20
+ #------------------------------------------------------------------------=
-----#
+ # kdgrills@the-grills.com, check for verification argument, default to no
+ #------------------------------------------------------------------------=
-----#
+=20
+ my $VERIFY =3D 0; # default to no
+ if ( $opt{'v'} ) { $VERIFY =3D 1 } # Enable verify, if "v" argument pass=
ed
+=20
+ #------------------------------------------------------------------------=
-----#
+ # End kdgrills@the-grills.com modifications
+ #------------------------------------------------------------------------=
-----#
+=20
# which servers shall we query?
=20
my @countries =3D ();
***************
*** 295,301 ****
=20
# extract the country name and the rest (what a mess!)
=20
- s/^<a[^>]+><\/a>//;
if ( m/^(\w+)(\s{1}\w+)?\s*(.*)/s ) {
my $country =3D $1.$2; # for two word names
my $mess =3D $3; # the rest
--- 352,357 ----
***************
*** 420,425 ****
--- 476,519 ----
print "Failed! (Using internal list)\n" if $VERBOSE;
}
}
+ #------------------------------------------------------------------------=
-------------#
+ # kdgrills@the-grills.com, create directory / file structure necessary fo=
r verification
+ #------------------------------------------------------------------------=
-------------#
+=20
+ my $SUP_FILE =3D "$CVSUP_BASE/fastest_cvsup_supfile"; # path to our su=
p file
+ my $REFUSE_FILE =3D "$CVSUP_BASE/sup/refuse"; # path to our re=
fuse file
+=20
+ # if verification is specified...
+=20
+ if ( $VERIFY ) {=20
+=20
+ # Create the directory structure, if it doesn't exist
+=20
+ if ( ! -d "$CVSUP_BASE" ) { # base dir
+ mkdir "$CVSUP_BASE" , 0755 or die "can't mkdir $CVSUP_BASE: $!";
+ }
+ if ( ! -d "$CVSUP_BASE/sup" ) { # sup dir
+ mkdir "$CVSUP_BASE/sup" , 0755 or die "can't mkdir $CVSUP_BASE/sup: $=
!";
+ }
+ if ( ! -d "$CVSUP_BASE$PREFIX" ) { # prefix dir
+ mkdir "$CVSUP_BASE$PREFIX" , 0755 or die "can't mkdir $CVSUP_BASE$PRE=
FIX: $!";
+ }
+=20
+ # Create the sup and refuse files
+=20
+ open (F, "> $SUP_FILE") or die "Couldn't open $SUP_FILE for writing: $!=
";
+ print F "*default base=3D$CVSUP_BASE \n *default prefix=3D$CVSUP_BASE \=
n *default release=3Dcvs tag=3D$TAG \n *default delete use-rel-suffix \n $C=
OLLECTION " or die "Couldn't write to $SUP_FILE : $!";
+=20
+ close (F) or die "Couldn't close $SUP_FILE : $!";
+=20
+ open (F, "> $REFUSE_FILE") or die "Couldn't open $REFUSE_FILE for writi=
ng: $!";
+ print F "*" or die "Couldn't write to $REFUSE_FILE : $!";
+ close (F) or die "Couldn't close $REFUSE_FILE: $!";
+=20
+ }
+ #------------------------------------------------------------------------=
-----#
+ # End kdgrills@the-grills.com modifications
+ #------------------------------------------------------------------------=
-----#
=20
#------------------------------------------------------------------------=
-----#
# now we build an array of servers
***************
*** 536,551 ****
close $remote; # close connection
my $time_taken =3D $time_after - $time_before; # calculate time
=20
! # if the server replied 'OK', i.e. not overloaded, save the timing
! $time{$server} =3D $time_taken if ( $reply =3D~ m/^OK/ );
=20
! if ( $VERBOSE ) { # print pretty display
=20
! my $ms =3D sprintf("%.2f", 1000 * $time_taken); # time in ms
! print $SERVER_RESULT, "server replied: $reply\n";
! print $SERVER_RESULT, "time taken: $ms ms\n";
=20
}
}
}
=20
--- 630,686 ----
close $remote; # close connection
my $time_taken =3D $time_after - $time_before; # calculate time
=20
! #--------------------------------------------------------------------=
---------#
! # kdgrills@the-grills.com, verify the existence of our collection
! #--------------------------------------------------------------------=
---------#
! =20
! # if server said "OK", and verify is specified...
!=20
! if ( $reply =3D~ m/^OK/ && $VERIFY) {
!=20
! # try to sup our collection
!=20
! my @full_cvsup_response =3D `$CVSUP_CMD -h $server $SUP_FILE` or di=
e "$CVSUP_CMD failed: $!";
=20
! my @cvsup_response =3D grep /not available/, @full_cvsup_response;
=20
! # if we didn't get a "not available" message, add the server to the=
timing
=20
+ if ( @cvsup_response !~ /not available/ ) {
+ $time{$server} =3D $time_taken;
+ }
+=20
+ # if we're being noisy, print the results
+ =20
+ if ( $VERBOSE ) { # print pretty display
+ my $ms =3D sprintf("%.2f", 1000 * $time_taken); # time in ms
+ print $SERVER_RESULT, "server replied: $reply\n";
+ if ( @cvsup_response ) {
+ print $SERVER_ERROR, @cvsup_response;
+ }
+ print $SERVER_RESULT, "time taken: $ms ms\n";
+ }
+ }
+=20
+ # if server said "OK", and we're not doing verification...
+=20
+ else{
+=20
+ # add the server to the timing
+=20
+ $time{$server} =3D $time_taken if ( $reply =3D~ m/^OK/ );
+=20
+ # if we're being noisy, print the results
+=20
+ if ( $VERBOSE ) { # print pretty display
+ my $ms =3D sprintf("%.2f", 1000 * $time_taken); # time in ms
+ print $SERVER_RESULT, "server replied: $reply\n";
+ print $SERVER_RESULT, "time taken: $ms ms\n";
+ }
}
+ #--------------------------------------------------------------------=
---------#
+ # End kdgrills@the-grills.com modifications
+ #--------------------------------------------------------------------=
---------#
}
}
=20
--Izn7cH1Com+I3R9J--
--HWvPVVuAAfuRc6SZ
Content-Type: application/pgp-signature
Content-Disposition: inline
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.2 (FreeBSD)
Comment: PGP key: mailto:kdgrills-pgpkey@the-grills.com
iD8DBQFD0wNf7inS5LzF7HMRAuyMAJ4vt2oSoQdVvbeXpUVwWhC4DP/ZVgCeKpMP
/UDA2au5W4hKrNysxYTmaRQ=
=K32c
-----END PGP SIGNATURE-----
--HWvPVVuAAfuRc6SZ--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20060122040002.GA52928>