From owner-svn-src-stable-12@freebsd.org Thu Dec 6 10:48:48 2018 Return-Path: Delivered-To: svn-src-stable-12@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id EE3361327294; Thu, 6 Dec 2018 10:48:47 +0000 (UTC) (envelope-from yuripv@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 9E6958B2E1; Thu, 6 Dec 2018 10:48:47 +0000 (UTC) (envelope-from yuripv@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 7F5EB226CC; Thu, 6 Dec 2018 10:48:47 +0000 (UTC) (envelope-from yuripv@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id wB6Aml5T040412; Thu, 6 Dec 2018 10:48:47 GMT (envelope-from yuripv@FreeBSD.org) Received: (from yuripv@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id wB6Amkj8040387; Thu, 6 Dec 2018 10:48:46 GMT (envelope-from yuripv@FreeBSD.org) Message-Id: <201812061048.wB6Amkj8040387@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: yuripv set sender to yuripv@FreeBSD.org using -f From: Yuri Pankov Date: Thu, 6 Dec 2018 10:48:46 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r341628 - in stable/12/tools/tools/locale: . etc tools X-SVN-Group: stable-12 X-SVN-Commit-Author: yuripv X-SVN-Commit-Paths: in stable/12/tools/tools/locale: . etc tools X-SVN-Commit-Revision: 341628 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Rspamd-Queue-Id: 9E6958B2E1 X-Spamd-Result: default: False [-1.02 / 15.00]; local_wl_from(0.00)[FreeBSD.org]; NEURAL_HAM_MEDIUM(-0.43)[-0.433,0]; NEURAL_HAM_SHORT(-0.54)[-0.542,0]; NEURAL_HAM_LONG(-0.04)[-0.040,0]; ASN(0.00)[asn:11403, ipnet:2610:1c1:1::/48, country:US] X-Rspamd-Server: mx1.freebsd.org X-BeenThere: svn-src-stable-12@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: SVN commit messages for only the 12-stable src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 06 Dec 2018 10:48:48 -0000 Author: yuripv Date: Thu Dec 6 10:48:46 2018 New Revision: 341628 URL: https://svnweb.freebsd.org/changeset/base/341628 Log: MFC r340204: Cleanup locale tools: - Simplify the source dir specification, and update README appropriately - Drop the LC (doonly) processing, it's broken, and even if fixed, not really useful - Don't remove the target directories while installing new data as it removes Makefile.depend which we don't manage; only rm the files we are going to add/replace/delete instead - Restrict adding bsd.endian.mk to colldef and ctypedef Makefiles, it's not needed in other (text-only) categories - GC unused scripts; they don't seem to be particularly helpful standalone as well Reviewed by: bapt Differential Revision: https://reviews.freebsd.org/D17858 Deleted: stable/12/tools/tools/locale/etc/unicode.conf stable/12/tools/tools/locale/tools/UTF82encoding.pl stable/12/tools/tools/locale/tools/changeoffset.pl stable/12/tools/tools/locale/tools/unicode2src.pl stable/12/tools/tools/locale/tools/whatis.pl Modified: stable/12/tools/tools/locale/Makefile stable/12/tools/tools/locale/README stable/12/tools/tools/locale/tools/cldr2def.pl stable/12/tools/tools/locale/tools/finalize stable/12/tools/tools/locale/tools/utf8-rollup.pl Directory Properties: stable/12/ (props changed) Modified: stable/12/tools/tools/locale/Makefile ============================================================================== --- stable/12/tools/tools/locale/Makefile Thu Dec 6 10:41:22 2018 (r341627) +++ stable/12/tools/tools/locale/Makefile Thu Dec 6 10:48:46 2018 (r341628) @@ -10,13 +10,10 @@ .OBJDIR: . -.if !defined(CLDRDIR) -CLDRDIR!= grep ^cldr etc/unicode.conf | cut -f 2 -d " " +.if !defined(UNIDIR) +.error UNIDIR is not set .endif -.if !defined(UNIDATADIR) -UNIDATADIR!= grep ^unidata etc/unicode.conf | cut -f 2 -d " " -.endif -PASSON= CLDRDIR="${CLDRDIR}" UNIDATADIR="${UNIDATADIR}" +PASSON= UNIDIR="${UNIDIR}" ETCDIR= ${.CURDIR}/etc @@ -49,12 +46,8 @@ COLLATIONS_SPECIAL_ENV+= ${area}.${enc} .endfor PASSON+= COLLATIONS_SPECIAL="${COLLATIONS_SPECIAL_ENV}" -.if defined(LC) -LC:= --lc=${LC} -.endif - all: - cp ${ETCDIR}/common.UTF-8.src ${CLDRDIR}/posix/xx_Comm_C.UTF-8.src + cp ${ETCDIR}/common.UTF-8.src ${UNIDIR}/posix/xx_Comm_C.UTF-8.src .for t in ${TYPES} . if ${KNOWN:M${t}} test -d ${t} || mkdir ${t} @@ -69,8 +62,9 @@ install: install-${t} install-${t}: . if ${KNOWN:M${t}} rm -rf ${.CURDIR}/${t}.draft - rm -rf ${.CURDIR}/../../../share/${t} - mv ${.CURDIR}/${t} ${.CURDIR}/../../../share/${t} + rm -f ${.CURDIR}/../../../share/${t}/Makefile + rm -f ${.CURDIR}/../../../share/${t}/*.src + mv ${.CURDIR}/${t}/* ${.CURDIR}/../../../share/${t}/ . endif .endfor @@ -86,10 +80,9 @@ post-install: gen-${t}: mkdir -p ${t} ${t}.draft perl -I tools tools/cldr2def.pl \ - --cldr=$$(realpath ${CLDRDIR}) \ - --unidata=$$(realpath ${UNIDATADIR}) \ + --unidir=$$(realpath ${UNIDIR}) \ --etc=$$(realpath ${ETCDIR}) \ - --type=${t} ${LC} + --type=${t} build-${t}: gen-${t} env ${PASSON} tools/finalize ${t} @@ -101,15 +94,16 @@ build-colldef: static-colldef static-colldef: .for area enc in ${COLLATION_SPECIAL} - awk -f tools/extract-colldef.awk ${CLDRDIR}/posix/${area}.${enc}.src > colldef.draft/${area}.${enc}.src + awk -f tools/extract-colldef.awk ${UNIDIR}/posix/${area}.${enc}.src > \ + colldef.draft/${area}.${enc}.src .endfor transfer-rollup: - cp ${ETCDIR}/common.UTF-8.src ${CLDRDIR}/posix/xx_Comm_C.UTF-8.src + cp ${ETCDIR}/common.UTF-8.src ${UNIDIR}/posix/xx_Comm_C.UTF-8.src rollup: perl -I tools tools/utf8-rollup.pl \ - --cldr=$$(realpath ${CLDRDIR}) \ + --unidir=$$(realpath ${UNIDIR}) \ --etc=$$(realpath ${ETCDIR}) clean: @@ -159,33 +153,33 @@ ENCODINGS= Big5 \ POSIX: -.if exists (${CLDRDIR}/tools/java/cldr.jar) - mkdir -p ${CLDRDIR}/posix +.if exists (${UNIDIR}/tools/java/cldr.jar) + mkdir -p ${UNIDIR}/posix . for area in ${BASE_LOCALES_OF_INTEREST} -. if !exists(${CLDRDIR}/posix/${area}.UTF-8.src) - java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \ +. if !exists(${UNIDIR}/posix/${area}.UTF-8.src) + java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \ org.unicode.cldr.posix.GeneratePOSIX \ - -d ${CLDRDIR}/posix -m ${area} -c UTF-8 + -d ${UNIDIR}/posix -m ${area} -c UTF-8 . endif . endfor . for area encoding in ${COLLATION_SPECIAL} -. if !exists(${CLDRDIR}/posix/${area}.${encoding}.src) - java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \ +. if !exists(${UNIDIR}/posix/${area}.${encoding}.src) + java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \ org.unicode.cldr.posix.GeneratePOSIX \ - -d ${CLDRDIR}/posix -m ${area} -c ${encoding} + -d ${UNIDIR}/posix -m ${area} -c ${encoding} . endif . endfor . for enc in ${ENCODINGS} -. if !exists(${CLDRDIR}/posix/${enc}.cm) - java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \ +. if !exists(${UNIDIR}/posix/${enc}.cm) + java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \ org.unicode.cldr.posix.GenerateCharmap \ - -d ${CLDRDIR}/posix -c ${enc} + -d ${UNIDIR}/posix -c ${enc} . endif . endfor .else @echo "Please install CLDR toolset for the desired release" - @echo "It should go at ${CLDRDIR}/tools" + @echo "It should go at ${UNIDIR}/tools" .endif clean-POSIX: - rm -f ${CLDRDIR}/posix/* + rm -f ${UNIDIR}/posix/* Modified: stable/12/tools/tools/locale/README ============================================================================== --- stable/12/tools/tools/locale/README Thu Dec 6 10:41:22 2018 (r341627) +++ stable/12/tools/tools/locale/README Thu Dec 6 10:48:46 2018 (r341628) @@ -9,30 +9,23 @@ Tools needed: devel/p5-Tie-IxHash textproc/p5-XML-Parser -Fetch CLDR data from: http://unicode.org/Public/cldr/. You need all of the +1. Fetch CLDR data from: http://unicode.org/Public/cldr/. You need all of the core.zip, keyboards.zip, and tools.zip. - -Extract: - mkdir -p ~/unicode/cldr/v33.0 - cd ~/unicode/cldr/v33.0 - unzip ~/core.zip ~/keyboards.zip ~/tools.zip - -Fetch unidata (UCD.zip) from http://www.unicode.org/Public/zipped/latest. - -Extract: - mkdir -p ~/unicode/UNIDATA/11.0.0 - cd ~/unicode/UNIDATA/11.0.0 +2. Fetch unidata (UCD.zip) from http://www.unicode.org/Public/zipped/latest. +3. Extract: + mkdir -p ~/unicode + cd ~/unicode + unzip ~/core.zip + unzip ~/keyboards.zip + unzip ~/tools.zip unzip ~/UCD.zip - -Either modify tools/tools/locales/etc/unicode.conf or export variables: - CLDRDIR=~/unicode/cldr/v33.0; export CLDRDIR - UNIDATADIR=~/unicode/UNIDATA/9.0.0; export UNIDATADIR - -Build the CLDR tools: - cd $CLDRDIR/tools/java +4. Export variable: + UNIDIR=~/unicode; export UNIDIR +5. Build the CLDR tools: + cd $UNIDIR/tools/java ant jar - -Run: +6. Build POSIX data files from CLDR data: make POSIX +7. Build and install new locale data: make make install Modified: stable/12/tools/tools/locale/tools/cldr2def.pl ============================================================================== --- stable/12/tools/tools/locale/tools/cldr2def.pl Thu Dec 6 10:41:22 2018 (r341627) +++ stable/12/tools/tools/locale/tools/cldr2def.pl Thu Dec 6 10:48:46 2018 (r341628) @@ -6,32 +6,27 @@ use File::Copy; use XML::Parser; use Tie::IxHash; use Text::Iconv; -use Data::Dumper; +#use Data::Dumper; use Getopt::Long; use Digest::SHA qw(sha1_hex); require "charmaps.pm"; if ($#ARGV < 2) { - print "Usage: $0 --cldr= --unidata= --etc= --type= [--lc=]\n"; + print "Usage: $0 --unidir= --etc= --type=\n"; exit(1); } my $DEFENCODING = "UTF-8"; -my @filter = (); -my $CLDRDIR = undef; -my $UNIDATADIR = undef; +my $UNIDIR = undef; my $ETCDIR = undef; my $TYPE = undef; -my $doonly = undef; my $result = GetOptions ( - "cldr=s" => \$CLDRDIR, - "unidata=s" => \$UNIDATADIR, + "unidir=s" => \$UNIDIR, "etc=s" => \$ETCDIR, "type=s" => \$TYPE, - "lc=s" => \$doonly ); my %convertors = (); @@ -47,8 +42,8 @@ get_languages(); my %utf8map = (); my %utf8aliases = (); -get_unidata($UNIDATADIR); -get_utf8map("$CLDRDIR/posix/$DEFENCODING.cm"); +get_unidata($UNIDIR); +get_utf8map("$UNIDIR/posix/$DEFENCODING.cm"); get_encodings("$ETCDIR/charmaps"); my %keys = (); @@ -397,22 +392,6 @@ sub get_languages { %translations = %{$data{T}}; %alternativemonths = %{$data{AM}}; %encodings = %{$data{E}}; - - return if (!defined $doonly); - - my @a = split(/_/, $doonly); - if ($#a == 1) { - $filter[0] = $a[0]; - $filter[1] = "x"; - $filter[2] = $a[1]; - } elsif ($#a == 2) { - $filter[0] = $a[0]; - $filter[1] = $a[1]; - $filter[2] = $a[2]; - } - - print Dumper(@filter); - return; } sub transform_ctypes { @@ -422,8 +401,6 @@ sub transform_ctypes { foreach my $l (sort keys(%languages)) { foreach my $f (sort keys(%{$languages{$l}})) { foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { - next if ($#filter == 2 && ($filter[0] ne $l - || $filter[1] ne $f || $filter[2] ne $c)); next if (defined $languages{$l}{$f}{definitions} && $languages{$l}{$f}{definitions} !~ /$TYPE/); $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread @@ -432,7 +409,7 @@ sub transform_ctypes { $file .= "_" . $c if ($c ne "x"); my $actfile = $file; - my $filename = "$CLDRDIR/posix/xx_Comm_C.UTF-8.src"; + my $filename = "$UNIDIR/posix/xx_Comm_C.UTF-8.src"; if (! -f $filename) { print STDERR "Cannot open $filename\n"; next; @@ -455,7 +432,7 @@ sub transform_ctypes { close(FOUT); foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) { next if ($enc eq $DEFENCODING); - $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; + $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"; if (! -f $filename) { print STDERR "Cannot open $filename\n"; next; @@ -494,8 +471,6 @@ sub transform_collation { foreach my $l (sort keys(%languages)) { foreach my $f (sort keys(%{$languages{$l}})) { foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { - next if ($#filter == 2 && ($filter[0] ne $l - || $filter[1] ne $f || $filter[2] ne $c)); next if (defined $languages{$l}{$f}{definitions} && $languages{$l}{$f}{definitions} !~ /$TYPE/); $languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0; # unread @@ -505,15 +480,15 @@ sub transform_collation { $file .= $c; my $actfile = $file; - my $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; + my $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"; $filename = "$ETCDIR/$file.$DEFENCODING.src" if (! -f $filename); if (! -f $filename && defined $languages{$l}{$f}{fallback}) { $file = $languages{$l}{$f}{fallback}; - $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; + $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"; } - $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src" + $filename = "$UNIDIR/posix/$file.$DEFENCODING.src" if (! -f $filename); if (! -f $filename) { print STDERR @@ -564,8 +539,6 @@ sub get_fields { foreach my $l (sort keys(%languages)) { foreach my $f (sort keys(%{$languages{$l}})) { foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { - next if ($#filter == 2 && ($filter[0] ne $l - || $filter[1] ne $f || $filter[2] ne $c)); next if (defined $languages{$l}{$f}{definitions} && $languages{$l}{$f}{definitions} !~ /$TYPE/); @@ -575,15 +548,15 @@ sub get_fields { $file .= $f . "_" if ($f ne "x"); $file .= $c; - my $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; + my $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"; $filename = "$ETCDIR/$file.$DEFENCODING.src" if (! -f $filename); if (! -f $filename && defined $languages{$l}{$f}{fallback}) { $file = $languages{$l}{$f}{fallback}; - $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"; + $filename = "$UNIDIR/posix/$file.$DEFENCODING.src"; } - $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src" + $filename = "$UNIDIR/posix/$file.$DEFENCODING.src" if (! -f $filename); if (! -f $filename) { print STDERR @@ -703,8 +676,6 @@ sub print_fields { foreach my $l (sort keys(%languages)) { foreach my $f (sort keys(%{$languages{$l}})) { foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) { - next if ($#filter == 2 && ($filter[0] ne $l - || $filter[1] ne $f || $filter[2] ne $c)); next if (defined $languages{$l}{$f}{definitions} && $languages{$l}{$f}{definitions} !~ /$TYPE/); foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) { @@ -851,7 +822,6 @@ EOF } sub make_makefile { - return if ($#filter > -1); print "Creating Makefile for $TYPE\n"; my $SRCOUT; my $SRCOUT2; @@ -913,8 +883,16 @@ LOCALEDIR= \${SHAREDIR}/locale FILESNAME= $FILESNAMES{$TYPE} .SUFFIXES: .src .${SRCOUT2} ${MAPLOC} +EOF + + if ($TYPE eq "colldef" || $TYPE eq "ctypedef") { + print FOUT < +EOF + } + + print FOUT < ${TEMP3} rm -f ${TEMP2} /usr/bin/sed -E -e 's/[ ]+/ /g' \ - ${CLDRDIR}/posix/UTF-8.cm \ + ${UNIDIR}/posix/UTF-8.cm \ > ${base}/../etc/final-maps/map.UTF-8 /usr/bin/sed -E -e 's/[ ]+/ /g' \ - ${CLDRDIR}/posix/eucCN.cm \ + ${UNIDIR}/posix/eucCN.cm \ > ${base}/../etc/final-maps/map.eucCN /usr/bin/sed -E -e 's/[ ]+/ /g' \ - ${CLDRDIR}/posix/eucCN.cm \ + ${UNIDIR}/posix/eucCN.cm \ > ${base}/../etc/final-maps/map.GB2312 CHARMAPS="ARMSCII-8 Big5 CP1131 CP1251 \ CP866 GBK ISCII-DEV ISO8859-1 \ Modified: stable/12/tools/tools/locale/tools/utf8-rollup.pl ============================================================================== --- stable/12/tools/tools/locale/tools/utf8-rollup.pl Thu Dec 6 10:41:22 2018 (r341627) +++ stable/12/tools/tools/locale/tools/utf8-rollup.pl Thu Dec 6 10:48:46 2018 (r341628) @@ -1,4 +1,5 @@ #!/usr/local/bin/perl -wC +# $FreeBSD$ use strict; #use File::Copy; @@ -11,15 +12,15 @@ use Getopt::Long; if ($#ARGV != 1) { - print "Usage: $0 --cldr= --etc=\n"; + print "Usage: $0 --unidir= --etc=\n"; exit(1); } -my $CLDRDIR = undef; +my $UNIDIR = undef; my $ETCDIR = undef; my $result = GetOptions ( - "cldr=s" => \$CLDRDIR, + "unidir=s" => \$UNIDIR, "etc=s" => \$ETCDIR, ); @@ -118,7 +119,7 @@ my $outfilename = "$ETCDIR/common.UTF-8.src"; my $manual_file = "$ETCDIR/manual-input.UTF-8"; my $stars = "**********************************************************************\n"; -get_utf8map("$CLDRDIR/posix/UTF-8.cm"); +get_utf8map("$UNIDIR/posix/UTF-8.cm"); generate_header (); generate_sections (); generate_footer (); @@ -252,7 +253,7 @@ sub compress_ctype { my @lines = initialize_lines ($territory); - my $filename = "$CLDRDIR/posix/$territory.UTF-8.src"; + my $filename = "$UNIDIR/posix/$territory.UTF-8.src"; if (! -f $filename) { print STDERR "Cannot open $filename\n"; return;