Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 6 Dec 2018 10:48:46 +0000 (UTC)
From:      Yuri Pankov <yuripv@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org
Subject:   svn commit: r341628 - in stable/12/tools/tools/locale: . etc tools
Message-ID:  <201812061048.wB6Amkj8040387@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: yuripv
Date: Thu Dec  6 10:48:46 2018
New Revision: 341628
URL: https://svnweb.freebsd.org/changeset/base/341628

Log:
  MFC r340204:
  Cleanup locale tools:
  
  - Simplify the source dir specification, and update README
    appropriately
  - Drop the LC (doonly) processing, it's broken, and even if fixed, not
    really useful
  - Don't remove the target directories while installing new data as it
    removes Makefile.depend which we don't manage; only rm the files we
    are going to add/replace/delete instead
  - Restrict adding bsd.endian.mk to colldef and ctypedef Makefiles, it's
    not needed in other (text-only) categories
  - GC unused scripts; they don't seem to be particularly helpful standalone
    as well
  
  Reviewed by:	bapt
  Differential Revision:	https://reviews.freebsd.org/D17858

Deleted:
  stable/12/tools/tools/locale/etc/unicode.conf
  stable/12/tools/tools/locale/tools/UTF82encoding.pl
  stable/12/tools/tools/locale/tools/changeoffset.pl
  stable/12/tools/tools/locale/tools/unicode2src.pl
  stable/12/tools/tools/locale/tools/whatis.pl
Modified:
  stable/12/tools/tools/locale/Makefile
  stable/12/tools/tools/locale/README
  stable/12/tools/tools/locale/tools/cldr2def.pl
  stable/12/tools/tools/locale/tools/finalize
  stable/12/tools/tools/locale/tools/utf8-rollup.pl
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/tools/tools/locale/Makefile
==============================================================================
--- stable/12/tools/tools/locale/Makefile	Thu Dec  6 10:41:22 2018	(r341627)
+++ stable/12/tools/tools/locale/Makefile	Thu Dec  6 10:48:46 2018	(r341628)
@@ -10,13 +10,10 @@
 
 .OBJDIR:	.
 
-.if !defined(CLDRDIR)
-CLDRDIR!=	grep ^cldr etc/unicode.conf | cut -f 2 -d " "
+.if !defined(UNIDIR)
+.error UNIDIR is not set
 .endif
-.if !defined(UNIDATADIR)
-UNIDATADIR!=	grep ^unidata etc/unicode.conf | cut -f 2 -d " "
-.endif
-PASSON=		CLDRDIR="${CLDRDIR}" UNIDATADIR="${UNIDATADIR}"
+PASSON=		UNIDIR="${UNIDIR}"
 
 ETCDIR=		${.CURDIR}/etc
 
@@ -49,12 +46,8 @@ COLLATIONS_SPECIAL_ENV+=	${area}.${enc}
 .endfor
 PASSON+=	COLLATIONS_SPECIAL="${COLLATIONS_SPECIAL_ENV}"
 
-.if defined(LC)
-LC:=	--lc=${LC}
-.endif
-
 all:
-	cp ${ETCDIR}/common.UTF-8.src ${CLDRDIR}/posix/xx_Comm_C.UTF-8.src
+	cp ${ETCDIR}/common.UTF-8.src ${UNIDIR}/posix/xx_Comm_C.UTF-8.src
 .for t in ${TYPES}
 .  if ${KNOWN:M${t}}
 	test -d ${t} || mkdir ${t}
@@ -69,8 +62,9 @@ install: install-${t}
 install-${t}:
 .  if ${KNOWN:M${t}}
 	rm -rf ${.CURDIR}/${t}.draft
-	rm -rf ${.CURDIR}/../../../share/${t}
-	mv ${.CURDIR}/${t} ${.CURDIR}/../../../share/${t}
+	rm -f ${.CURDIR}/../../../share/${t}/Makefile
+	rm -f ${.CURDIR}/../../../share/${t}/*.src
+	mv ${.CURDIR}/${t}/* ${.CURDIR}/../../../share/${t}/
 .  endif
 .endfor
 
@@ -86,10 +80,9 @@ post-install:
 gen-${t}:
 	mkdir -p ${t} ${t}.draft
 	perl -I tools tools/cldr2def.pl \
-		--cldr=$$(realpath ${CLDRDIR}) \
-		--unidata=$$(realpath ${UNIDATADIR}) \
+		--unidir=$$(realpath ${UNIDIR}) \
 		--etc=$$(realpath ${ETCDIR}) \
-		--type=${t} ${LC}
+		--type=${t}
 
 build-${t}: gen-${t}
 	env ${PASSON} tools/finalize ${t}
@@ -101,15 +94,16 @@ build-colldef:	static-colldef
 
 static-colldef:
 .for area enc in ${COLLATION_SPECIAL}
-	awk -f tools/extract-colldef.awk ${CLDRDIR}/posix/${area}.${enc}.src > colldef.draft/${area}.${enc}.src
+	awk -f tools/extract-colldef.awk ${UNIDIR}/posix/${area}.${enc}.src > \
+	    colldef.draft/${area}.${enc}.src
 .endfor
 
 transfer-rollup:
-	cp ${ETCDIR}/common.UTF-8.src ${CLDRDIR}/posix/xx_Comm_C.UTF-8.src
+	cp ${ETCDIR}/common.UTF-8.src ${UNIDIR}/posix/xx_Comm_C.UTF-8.src
 
 rollup:
 	perl -I tools tools/utf8-rollup.pl \
-		--cldr=$$(realpath ${CLDRDIR}) \
+		--unidir=$$(realpath ${UNIDIR}) \
 		--etc=$$(realpath ${ETCDIR})
 
 clean:
@@ -159,33 +153,33 @@ ENCODINGS=	Big5 \
 
 
 POSIX:
-.if exists (${CLDRDIR}/tools/java/cldr.jar)
-	mkdir -p ${CLDRDIR}/posix
+.if exists (${UNIDIR}/tools/java/cldr.jar)
+	mkdir -p ${UNIDIR}/posix
 .  for area in ${BASE_LOCALES_OF_INTEREST}
-.    if !exists(${CLDRDIR}/posix/${area}.UTF-8.src)
-	java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \
+.    if !exists(${UNIDIR}/posix/${area}.UTF-8.src)
+	java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \
 		org.unicode.cldr.posix.GeneratePOSIX \
-		-d ${CLDRDIR}/posix -m ${area} -c UTF-8
+		-d ${UNIDIR}/posix -m ${area} -c UTF-8
 .    endif
 .  endfor
 .  for area encoding in ${COLLATION_SPECIAL}
-.    if !exists(${CLDRDIR}/posix/${area}.${encoding}.src)
-	java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \
+.    if !exists(${UNIDIR}/posix/${area}.${encoding}.src)
+	java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \
 		org.unicode.cldr.posix.GeneratePOSIX \
-		-d ${CLDRDIR}/posix -m ${area} -c ${encoding}
+		-d ${UNIDIR}/posix -m ${area} -c ${encoding}
 .    endif
 .  endfor
 .  for enc in ${ENCODINGS}
-.  if !exists(${CLDRDIR}/posix/${enc}.cm)
-	java -DCLDR_DIR=${CLDRDIR:Q} -jar ${CLDRDIR}/tools/java/cldr.jar \
+.  if !exists(${UNIDIR}/posix/${enc}.cm)
+	java -DCLDR_DIR=${UNIDIR:Q} -jar ${UNIDIR}/tools/java/cldr.jar \
 		org.unicode.cldr.posix.GenerateCharmap \
-		-d ${CLDRDIR}/posix -c ${enc}
+		-d ${UNIDIR}/posix -c ${enc}
 .  endif
 .  endfor
 .else
 	@echo "Please install CLDR toolset for the desired release"
-	@echo "It should go at ${CLDRDIR}/tools"
+	@echo "It should go at ${UNIDIR}/tools"
 .endif
 
 clean-POSIX:
-	rm -f ${CLDRDIR}/posix/*
+	rm -f ${UNIDIR}/posix/*

Modified: stable/12/tools/tools/locale/README
==============================================================================
--- stable/12/tools/tools/locale/README	Thu Dec  6 10:41:22 2018	(r341627)
+++ stable/12/tools/tools/locale/README	Thu Dec  6 10:48:46 2018	(r341628)
@@ -9,30 +9,23 @@ Tools needed:
 	devel/p5-Tie-IxHash
 	textproc/p5-XML-Parser
 
-Fetch CLDR data from: http://unicode.org/Public/cldr/.  You need all of the
+1. Fetch CLDR data from: http://unicode.org/Public/cldr/.  You need all of the
 core.zip, keyboards.zip, and tools.zip.
-
-Extract:
-	mkdir -p ~/unicode/cldr/v33.0
-	cd ~/unicode/cldr/v33.0
-	unzip ~/core.zip ~/keyboards.zip ~/tools.zip
-
-Fetch unidata (UCD.zip) from http://www.unicode.org/Public/zipped/latest.
-
-Extract:
-	mkdir -p ~/unicode/UNIDATA/11.0.0
-	cd ~/unicode/UNIDATA/11.0.0
+2. Fetch unidata (UCD.zip) from http://www.unicode.org/Public/zipped/latest.
+3. Extract:
+	mkdir -p ~/unicode
+	cd ~/unicode
+	unzip ~/core.zip
+	unzip ~/keyboards.zip
+	unzip ~/tools.zip
 	unzip ~/UCD.zip
-
-Either modify tools/tools/locales/etc/unicode.conf or export variables:
-	CLDRDIR=~/unicode/cldr/v33.0; export CLDRDIR
-	UNIDATADIR=~/unicode/UNIDATA/9.0.0; export UNIDATADIR
-
-Build the CLDR tools:
-	cd $CLDRDIR/tools/java
+4. Export variable:
+	UNIDIR=~/unicode; export UNIDIR
+5. Build the CLDR tools:
+	cd $UNIDIR/tools/java
 	ant jar
-
-Run:
+6. Build POSIX data files from CLDR data:
 	make POSIX
+7. Build and install new locale data:
 	make
 	make install

Modified: stable/12/tools/tools/locale/tools/cldr2def.pl
==============================================================================
--- stable/12/tools/tools/locale/tools/cldr2def.pl	Thu Dec  6 10:41:22 2018	(r341627)
+++ stable/12/tools/tools/locale/tools/cldr2def.pl	Thu Dec  6 10:48:46 2018	(r341628)
@@ -6,32 +6,27 @@ use File::Copy;
 use XML::Parser;
 use Tie::IxHash;
 use Text::Iconv;
-use Data::Dumper;
+#use Data::Dumper;
 use Getopt::Long;
 use Digest::SHA qw(sha1_hex);
 require "charmaps.pm";
 
 
 if ($#ARGV < 2) {
-	print "Usage: $0 --cldr=<cldrdir> --unidata=<unidatadir> --etc=<etcdir> --type=<type> [--lc=<la_CC>]\n";
+	print "Usage: $0 --unidir=<unidir> --etc=<etcdir> --type=<type>\n";
 	exit(1);
 }
 
 my $DEFENCODING = "UTF-8";
-my @filter = ();
 
-my $CLDRDIR = undef;
-my $UNIDATADIR = undef;
+my $UNIDIR = undef;
 my $ETCDIR = undef;
 my $TYPE = undef;
-my $doonly = undef;
 
 my $result = GetOptions (
-		"cldr=s"	=> \$CLDRDIR,
-		"unidata=s"	=> \$UNIDATADIR,
+		"unidir=s"	=> \$UNIDIR,
 		"etc=s"		=> \$ETCDIR,
 		"type=s"	=> \$TYPE,
-		"lc=s"		=> \$doonly
 	    );
 
 my %convertors = ();
@@ -47,8 +42,8 @@ get_languages();
 
 my %utf8map = ();
 my %utf8aliases = ();
-get_unidata($UNIDATADIR);
-get_utf8map("$CLDRDIR/posix/$DEFENCODING.cm");
+get_unidata($UNIDIR);
+get_utf8map("$UNIDIR/posix/$DEFENCODING.cm");
 get_encodings("$ETCDIR/charmaps");
 
 my %keys = ();
@@ -397,22 +392,6 @@ sub get_languages {
 	%translations = %{$data{T}}; 
 	%alternativemonths = %{$data{AM}}; 
 	%encodings = %{$data{E}}; 
-
-	return if (!defined $doonly);
-
-	my @a = split(/_/, $doonly);
-	if ($#a == 1) {
-		$filter[0] = $a[0];
-		$filter[1] = "x";
-		$filter[2] = $a[1];
-	} elsif ($#a == 2) {
-		$filter[0] = $a[0];
-		$filter[1] = $a[1];
-		$filter[2] = $a[2];
-	}
-
-	print Dumper(@filter);
-	return;
 }
 
 sub transform_ctypes {
@@ -422,8 +401,6 @@ sub transform_ctypes {
 	foreach my $l (sort keys(%languages)) {
 	foreach my $f (sort keys(%{$languages{$l}})) {
 	foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) {
-		next if ($#filter == 2 && ($filter[0] ne $l
-		    || $filter[1] ne $f || $filter[2] ne $c));
 		next if (defined $languages{$l}{$f}{definitions}
 		    && $languages{$l}{$f}{definitions} !~ /$TYPE/);
 		$languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0;	# unread
@@ -432,7 +409,7 @@ sub transform_ctypes {
 		$file .= "_" . $c if ($c ne "x");
 		my $actfile = $file;
 
-		my $filename = "$CLDRDIR/posix/xx_Comm_C.UTF-8.src";
+		my $filename = "$UNIDIR/posix/xx_Comm_C.UTF-8.src";
 		if (! -f $filename) {
 			print STDERR "Cannot open $filename\n";
 			next;
@@ -455,7 +432,7 @@ sub transform_ctypes {
 		close(FOUT);
 		foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) {
 			next if ($enc eq $DEFENCODING);
-			$filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+			$filename = "$UNIDIR/posix/$file.$DEFENCODING.src";
 			if (! -f $filename) {
 				print STDERR "Cannot open $filename\n";
 				next;
@@ -494,8 +471,6 @@ sub transform_collation {
 	foreach my $l (sort keys(%languages)) {
 	foreach my $f (sort keys(%{$languages{$l}})) {
 	foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) {
-		next if ($#filter == 2 && ($filter[0] ne $l
-		    || $filter[1] ne $f || $filter[2] ne $c));
 		next if (defined $languages{$l}{$f}{definitions}
 		    && $languages{$l}{$f}{definitions} !~ /$TYPE/);
 		$languages{$l}{$f}{data}{$c}{$DEFENCODING} = 0;	# unread
@@ -505,15 +480,15 @@ sub transform_collation {
 		$file .= $c;
 		my $actfile = $file;
 
-		my $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+		my $filename = "$UNIDIR/posix/$file.$DEFENCODING.src";
 		$filename = "$ETCDIR/$file.$DEFENCODING.src"
 		    if (! -f $filename);
 		if (! -f $filename
 		 && defined $languages{$l}{$f}{fallback}) {
 			$file = $languages{$l}{$f}{fallback};
-			$filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+			$filename = "$UNIDIR/posix/$file.$DEFENCODING.src";
 		}
-		$filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"
+		$filename = "$UNIDIR/posix/$file.$DEFENCODING.src"
 		    if (! -f $filename);
 		if (! -f $filename) {
 			print STDERR
@@ -564,8 +539,6 @@ sub get_fields {
 	foreach my $l (sort keys(%languages)) {
 	foreach my $f (sort keys(%{$languages{$l}})) {
 	foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) {
-		next if ($#filter == 2 && ($filter[0] ne $l
-		    || $filter[1] ne $f || $filter[2] ne $c));
 		next if (defined $languages{$l}{$f}{definitions}
 		    && $languages{$l}{$f}{definitions} !~ /$TYPE/);
 
@@ -575,15 +548,15 @@ sub get_fields {
 		$file .= $f . "_" if ($f ne "x");
 		$file .= $c;
 
-		my $filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+		my $filename = "$UNIDIR/posix/$file.$DEFENCODING.src";
 		$filename = "$ETCDIR/$file.$DEFENCODING.src"
 		    if (! -f $filename);
 		if (! -f $filename
 		 && defined $languages{$l}{$f}{fallback}) {
 			$file = $languages{$l}{$f}{fallback};
-			$filename = "$CLDRDIR/posix/$file.$DEFENCODING.src";
+			$filename = "$UNIDIR/posix/$file.$DEFENCODING.src";
 		}
-		$filename = "$CLDRDIR/posix/$file.$DEFENCODING.src"
+		$filename = "$UNIDIR/posix/$file.$DEFENCODING.src"
 		    if (! -f $filename);
 		if (! -f $filename) {
 			print STDERR
@@ -703,8 +676,6 @@ sub print_fields {
 	foreach my $l (sort keys(%languages)) {
 	foreach my $f (sort keys(%{$languages{$l}})) {
 	foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) {
-		next if ($#filter == 2 && ($filter[0] ne $l
-		    || $filter[1] ne $f || $filter[2] ne $c));
 		next if (defined $languages{$l}{$f}{definitions}
 		    && $languages{$l}{$f}{definitions} !~ /$TYPE/);
 		foreach my $enc (sort keys(%{$languages{$l}{$f}{data}{$c}})) {
@@ -851,7 +822,6 @@ EOF
 }
 
 sub make_makefile {
-	return if ($#filter > -1);
 	print "Creating Makefile for $TYPE\n";
 	my $SRCOUT;
 	my $SRCOUT2;
@@ -913,8 +883,16 @@ LOCALEDIR=	\${SHAREDIR}/locale
 FILESNAME=	$FILESNAMES{$TYPE}
 .SUFFIXES:	.src .${SRCOUT2}
 ${MAPLOC}
+EOF
+
+	if ($TYPE eq "colldef" || $TYPE eq "ctypedef") {
+		print FOUT <<EOF;
 .include <bsd.endian.mk>
 
+EOF
+	}
+
+	print FOUT <<EOF;
 .src.${SRCOUT2}:
 	$SRCOUT
 
@@ -979,8 +957,6 @@ EOF
 	foreach my $l (sort keys(%languages)) {
 	foreach my $f (sort keys(%{$languages{$l}})) {
 	foreach my $c (sort keys(%{$languages{$l}{$f}{data}})) {
-		next if ($#filter == 2 && ($filter[0] ne $l
-		    || $filter[1] ne $f || $filter[2] ne $c));
 		next if (defined $languages{$l}{$f}{definitions}
 		    && $languages{$l}{$f}{definitions} !~ /$TYPE/);
 		if (defined $languages{$l}{$f}{data}{$c}{$DEFENCODING}

Modified: stable/12/tools/tools/locale/tools/finalize
==============================================================================
--- stable/12/tools/tools/locale/tools/finalize	Thu Dec  6 10:41:22 2018	(r341627)
+++ stable/12/tools/tools/locale/tools/finalize	Thu Dec  6 10:48:46 2018	(r341628)
@@ -116,13 +116,13 @@ then
 	else {last1 = $1; last2 = $2}}' ${TEMP2} > ${TEMP3}
 	rm -f ${TEMP2}
 	/usr/bin/sed -E -e 's/[ ]+/ /g' \
-		${CLDRDIR}/posix/UTF-8.cm \
+		${UNIDIR}/posix/UTF-8.cm \
 		> ${base}/../etc/final-maps/map.UTF-8
 	/usr/bin/sed -E -e 's/[ ]+/ /g' \
-		${CLDRDIR}/posix/eucCN.cm \
+		${UNIDIR}/posix/eucCN.cm \
 		> ${base}/../etc/final-maps/map.eucCN
 	/usr/bin/sed -E -e 's/[ ]+/ /g' \
-		${CLDRDIR}/posix/eucCN.cm \
+		${UNIDIR}/posix/eucCN.cm \
 		> ${base}/../etc/final-maps/map.GB2312
 	CHARMAPS="ARMSCII-8 Big5 CP1131 CP1251 \
 		CP866 GBK ISCII-DEV ISO8859-1 \

Modified: stable/12/tools/tools/locale/tools/utf8-rollup.pl
==============================================================================
--- stable/12/tools/tools/locale/tools/utf8-rollup.pl	Thu Dec  6 10:41:22 2018	(r341627)
+++ stable/12/tools/tools/locale/tools/utf8-rollup.pl	Thu Dec  6 10:48:46 2018	(r341628)
@@ -1,4 +1,5 @@
 #!/usr/local/bin/perl -wC
+# $FreeBSD$
 
 use strict;
 #use File::Copy;
@@ -11,15 +12,15 @@ use Getopt::Long;
 
 
 if ($#ARGV != 1) {
-	print "Usage: $0 --cldr=<cldrdir> --etc=<etcdir>\n";
+	print "Usage: $0 --unidir=<unidir> --etc=<etcdir>\n";
 	exit(1);
 }
 
-my $CLDRDIR = undef;
+my $UNIDIR = undef;
 my $ETCDIR = undef;
 
 my $result = GetOptions (
-		"cldr=s"	=> \$CLDRDIR,
+		"unidir=s"	=> \$UNIDIR,
 		"etc=s"		=> \$ETCDIR,
 	    );
 
@@ -118,7 +119,7 @@ my $outfilename = "$ETCDIR/common.UTF-8.src";
 my $manual_file = "$ETCDIR/manual-input.UTF-8";
 my $stars = "**********************************************************************\n";
 
-get_utf8map("$CLDRDIR/posix/UTF-8.cm");
+get_utf8map("$UNIDIR/posix/UTF-8.cm");
 generate_header ();
 generate_sections ();
 generate_footer ();
@@ -252,7 +253,7 @@ sub compress_ctype {
 
 	my @lines = initialize_lines ($territory);
 
-	my $filename = "$CLDRDIR/posix/$territory.UTF-8.src";
+	my $filename = "$UNIDIR/posix/$territory.UTF-8.src";
 	if (! -f $filename) {
 		print STDERR "Cannot open $filename\n";
 		return;



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201812061048.wB6Amkj8040387>