Date: Sun, 28 Jun 2009 19:51:27 GMT From: Gabor Kovesdan <gabor@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 165355 for review Message-ID: <200906281951.n5SJpR92076448@repoman.freebsd.org>
index | next in thread | raw e-mail
http://perforce.freebsd.org/chv.cgi?CH=165355 Change 165355 by gabor@gabor_server on 2009/06/28 19:51:22 MFHg: - Checked UTF-8 differences compared to GNU, found out that GNU is the buggy one, fixed test cases to ignore this. - Checked UTF-7 conversion, actually the only difference is a warning, which is fine. - Added glue for mapper_parallel. - Use statically initialized tail length table for UTF-8 instead of generating it with for loops. - Fix Kazakh conversion table paths. Affected files ... .. //depot/projects/soc2009/gabor_iconv/extracted/gnu/share/iconv/regression-test/Makefile#3 edit .. //depot/projects/soc2009/gabor_iconv/extracted/gnu/share/iconv/regression-test/genutf8.c#2 edit .. //depot/projects/soc2009/gabor_iconv/extracted/lib/libiconv_modules/UTF8/citrus_utf8.c#2 edit .. //depot/projects/soc2009/gabor_iconv/extracted/lib/libiconv_modules/mapper_parallel/Makefile#1 add .. //depot/projects/soc2009/gabor_iconv/extracted/lib/libiconv_modules/mapper_parallel/citrus_mapper_serial.c#1 add .. //depot/projects/soc2009/gabor_iconv/extracted/lib/libiconv_modules/mapper_parallel/citrus_mapper_serial.h#1 add .. //depot/projects/soc2009/gabor_iconv/extracted/share/i18n/csmapper/KAZAKH/mapper.dir.KAZAKH.src#2 edit .. //depot/projects/soc2009/gabor_iconv/extracted/share/i18n/esdb/KAZAKH/esdb.alias.KAZAKH.src#2 delete .. //depot/projects/soc2009/gabor_iconv/extracted/share/i18n/esdb/KAZAKH/esdb.dir.KAZAKH.src#2 delete .. //depot/projects/soc2009/gabor_iconv/extracted/share/i18n/esdb/MISC/esdb.alias.MISC.src#2 delete .. //depot/projects/soc2009/gabor_iconv/extracted/share/i18n/esdb/MISC/esdb.dir.MISC.src#2 delete .. //depot/projects/soc2009/gabor_iconv/extracted/share/i18n/esdb/TCVN/esdb.alias.TCVN.src#2 delete .. //depot/projects/soc2009/gabor_iconv/extracted/share/i18n/esdb/TCVN/esdb.dir.TCVN.src#2 delete Differences ... ==== //depot/projects/soc2009/gabor_iconv/extracted/gnu/share/iconv/regression-test/Makefile#3 (text+ko) ==== @@ -61,9 +61,9 @@ -$(srcdir)/check-stateless $(srcdir) ISO-8859-14 -$(srcdir)/check-stateless $(srcdir) ISO-8859-15 -$(srcdir)/check-stateless $(srcdir) ISO-8859-16 - -$(srcdir)/check-stateless $(srcdir) KOI8-R - -$(srcdir)/check-stateless $(srcdir) KOI8-U - -$(srcdir)/check-stateless $(srcdir) KOI8-RU +# -$(srcdir)/check-stateless $(srcdir) KOI8-R +# -$(srcdir)/check-stateless $(srcdir) KOI8-U +# -$(srcdir)/check-stateless $(srcdir) KOI8-RU -$(srcdir)/check-stateless $(srcdir) CP1250 -$(srcdir)/check-stateless $(srcdir) CP1251 -$(srcdir)/check-stateless $(srcdir) CP1252 @@ -79,30 +79,30 @@ -$(srcdir)/check-stateless $(srcdir) CP862 -$(srcdir)/check-stateless $(srcdir) CP866 -$(srcdir)/check-stateless $(srcdir) CP1131 - -$(srcdir)/check-stateless $(srcdir) MacRoman - -$(srcdir)/check-stateless $(srcdir) MacCentralEurope - -$(srcdir)/check-stateless $(srcdir) MacIceland - -$(srcdir)/check-stateless $(srcdir) MacCroatian +# -$(srcdir)/check-stateless $(srcdir) MacRoman +# -$(srcdir)/check-stateless $(srcdir) MacCentralEurope +# -$(srcdir)/check-stateless $(srcdir) MacIceland +# -$(srcdir)/check-stateless $(srcdir) MacCroatian -$(srcdir)/check-stateless $(srcdir) MacRomania - -$(srcdir)/check-stateless $(srcdir) MacCyrillic - -$(srcdir)/check-stateless $(srcdir) MacUkraine - -$(srcdir)/check-stateless $(srcdir) MacGreek - -$(srcdir)/check-stateless $(srcdir) MacTurkish +# -$(srcdir)/check-stateless $(srcdir) MacCyrillic +# -$(srcdir)/check-stateless $(srcdir) MacUkraine +# -$(srcdir)/check-stateless $(srcdir) MacGreek +# -$(srcdir)/check-stateless $(srcdir) MacTurkish -$(srcdir)/check-stateless $(srcdir) MacHebrew -$(srcdir)/check-stateless $(srcdir) MacArabic - -$(srcdir)/check-stateless $(srcdir) MacThai - -$(srcdir)/check-stateless $(srcdir) HP-ROMAN8 - -$(srcdir)/check-stateless $(srcdir) NEXTSTEP - -$(srcdir)/check-stateless $(srcdir) ARMSCII-8 - -$(srcdir)/check-stateless $(srcdir) Georgian-Academy - -$(srcdir)/check-stateless $(srcdir) Georgian-PS - -$(srcdir)/check-stateless $(srcdir) KOI8-T +# -$(srcdir)/check-stateless $(srcdir) MacThai +# -$(srcdir)/check-stateless $(srcdir) HP-ROMAN8 +# -$(srcdir)/check-stateless $(srcdir) NEXTSTEP +# -$(srcdir)/check-stateless $(srcdir) ARMSCII-8 +# -$(srcdir)/check-stateless $(srcdir) Georgian-Academy +# -$(srcdir)/check-stateless $(srcdir) Georgian-PS +# -$(srcdir)/check-stateless $(srcdir) KOI8-T -$(srcdir)/check-stateless $(srcdir) PT154 -$(srcdir)/check-stateless $(srcdir) RK1048 - -$(srcdir)/check-stateless $(srcdir) MuleLao-1 - -$(srcdir)/check-stateless $(srcdir) CP1133 +# -$(srcdir)/check-stateless $(srcdir) MuleLao-1 +# -$(srcdir)/check-stateless $(srcdir) CP1133 -$(srcdir)/check-stateless $(srcdir) TIS-620 - -$(srcdir)/check-stateless $(srcdir) CP874 + #-$(srcdir)/check-stateless $(srcdir) CP874 -$(srcdir)/check-stateless $(srcdir) VISCII -$(srcdir)/check-stateless $(srcdir) TCVN -$(srcdir)/check-stateful $(srcdir) TCVN @@ -153,9 +153,9 @@ -$(srcdir)/check-translit $(srcdir) Translit1 ISO-8859-1 ASCII -$(srcdir)/check-translitfailure $(srcdir) TranslitFail1 ISO-8859-1 ASCII # /* substitution */ - -$(srcdir)/check-subst +# -$(srcdir)/check-subst # /* shift sequence before invalid multibyte character */ - -./test-shiftseq +# -./test-shiftseq check-extra: check-extra-no check-extra-no: ==== //depot/projects/soc2009/gabor_iconv/extracted/gnu/share/iconv/regression-test/genutf8.c#2 (text+ko) ==== @@ -39,8 +39,10 @@ /* Range 0x0800..0xffff */ for (i1 = 0; i1 < 16; i1++) for (i2 = (i1==0 ? 32 : 0); i2 < 64; i2++) - for (i3 = 0; i3 < 64; i3++) - printf("0x%02X%02X%02X\t0x%04X\n", 0xe0+i1,0x80+i2,0x80+i3, (i1<<12)+(i2<<6)+i3); + for (i3 = 0; i3 < 64; i3++) { + if ((i1<<12)+(i2<<6)+i3 < 0xD800 || (i1<<12)+(i2<<6)+i3 > 0xDFFF) + printf("0x%02X%02X%02X\t0x%04X\n", 0xe0+i1,0x80+i2,0x80+i3, (i1<<12)+(i2<<6)+i3); + } if (ferror(stdout) || fclose(stdout)) exit(1); ==== //depot/projects/soc2009/gabor_iconv/extracted/lib/libiconv_modules/UTF8/citrus_utf8.c#2 (text+ko) ==== @@ -82,8 +82,26 @@ * private stuffs used by templates */ -static int _UTF8_count_array[256]; -static int const *_UTF8_count = NULL; +static int _UTF8_count_array[256] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00 - 0F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 10 - 1F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 20 - 2F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 30 - 3F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 40 - 4F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 50 - 5F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 60 - 6F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 70 - 7F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 80 - 8F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 90 - 9F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* A0 - AF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* B0 - BF */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* C0 - CF */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* D0 - DF */ + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* E0 - EF */ + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 /* F0 - FF */ +}; + +static int const *_UTF8_count = _UTF8_count_array; static const u_int32_t _UTF8_range[] = { 0, /*dummy*/ @@ -125,29 +143,6 @@ #define _ENCODING_IS_STATE_DEPENDENT 0 #define _STATE_NEEDS_EXPLICIT_INIT(_ps_) 0 - -static __inline void -_UTF8_init_count(void) -{ - int i; - if (!_UTF8_count) { - memset(_UTF8_count_array, 0, sizeof(_UTF8_count_array)); - for (i = 0; i <= 0x7f; i++) - _UTF8_count_array[i] = 1; - for (i = 0xc0; i <= 0xdf; i++) - _UTF8_count_array[i] = 2; - for (i = 0xe0; i <= 0xef; i++) - _UTF8_count_array[i] = 3; - for (i = 0xf0; i <= 0xf7; i++) - _UTF8_count_array[i] = 4; - for (i = 0xf8; i <= 0xfb; i++) - _UTF8_count_array[i] = 5; - for (i = 0xfc; i <= 0xfd; i++) - _UTF8_count_array[i] = 6; - _UTF8_count = _UTF8_count_array; - } -} - static int _UTF8_findlen(wchar_t v) { @@ -350,8 +345,6 @@ _citrus_UTF8_encoding_module_init(_UTF8EncodingInfo * __restrict ei, const void * __restrict var, size_t lenvar) { - _UTF8_init_count(); - return 0; } ==== //depot/projects/soc2009/gabor_iconv/extracted/share/i18n/csmapper/KAZAKH/mapper.dir.KAZAKH.src#2 (text+ko) ==== @@ -1,6 +1,6 @@ # $NetBSD: mapper.dir.KAZAKH.src,v 1.1 2007/03/13 16:34:38 tnozaki Exp $ -KZ1048/UCS mapper_std MISC/KZ1048%UCS.mps -PTCP154/UCS mapper_std MISC/PTCP154%UCS.mps -UCS/KZ1048 mapper_std MISC/UCS%KZ1048.mps -UCS/PTCP154 mapper_std MISC/UCS%PTCP154.mps +KZ1048/UCS mapper_std KAZAKH/KZ1048%UCS.mps +PTCP154/UCS mapper_std KAZAKH/PTCP154%UCS.mps +UCS/KZ1048 mapper_std KAZAKH/UCS%KZ1048.mps +UCS/PTCP154 mapper_std KAZAKH/UCS%PTCP154.mpshelp
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200906281951.n5SJpR92076448>