Date: Tue, 9 May 2017 16:51:07 +0000 (UTC) From: Jung-uk Kim <jkim@FreeBSD.org> To: ports-committers@freebsd.org, svn-ports-all@freebsd.org, svn-ports-branches@freebsd.org Subject: svn commit: r440506 - in branches/2017Q2/devel/icu: . files Message-ID: <201705091651.v49Gp7bZ041027@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jkim Date: Tue May 9 16:51:06 2017 New Revision: 440506 URL: https://svnweb.freebsd.org/changeset/ports/440506 Log: MFH: r437961, r440117 - Behave same on BSDs as on Darwin in that UTF-8 shall be used instead of ASCII outside the POSIX 'C' locale and UTF-8 is deafult in case anything should call ucnv_getDefaultName() prior to calling setlocale(). This change fixes problems that occur in multiple Qt5 applications when handling files with names containing non-ASCII characters. - Apply upstream patches to fix CVE-2017-7867 and CVE-2017-7868. Approved by: ports-secteam (miwi) Added: branches/2017Q2/devel/icu/files/patch-common_putil.cpp - copied unchanged from r437961, head/devel/icu/files/patch-common_putil.cpp branches/2017Q2/devel/icu/files/patch-r39671 - copied unchanged from r440117, head/devel/icu/files/patch-r39671 Modified: branches/2017Q2/devel/icu/Makefile Directory Properties: branches/2017Q2/ (props changed) Modified: branches/2017Q2/devel/icu/Makefile ============================================================================== --- branches/2017Q2/devel/icu/Makefile Tue May 9 16:48:19 2017 (r440505) +++ branches/2017Q2/devel/icu/Makefile Tue May 9 16:51:06 2017 (r440506) @@ -3,7 +3,7 @@ PORTNAME= icu DISTVERSION= 58_2 -PORTREVISION?= 0 # keep for icu-lx +PORTREVISION?= 2 # keep for icu-lx PORTEPOCH?= 1 CATEGORIES?= devel MASTER_SITES= http://download.icu-project.org/files/icu4c/${PORTVERSION}/ Copied: branches/2017Q2/devel/icu/files/patch-common_putil.cpp (from r437961, head/devel/icu/files/patch-common_putil.cpp) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ branches/2017Q2/devel/icu/files/patch-common_putil.cpp Tue May 9 16:51:06 2017 (r440506, copy of r437961, head/devel/icu/files/patch-common_putil.cpp) @@ -0,0 +1,32 @@ +--- common/putil.cpp.orig 2016-10-19 17:20:56 UTC ++++ common/putil.cpp +@@ -1789,7 +1789,7 @@ remapPlatformDependentCodepage(const cha + */ + name = "ISO-8859-1"; + } +-#elif U_PLATFORM_IS_DARWIN_BASED ++#elif U_PLATFORM_IS_DARWIN_BASED || U_PLATFORM == U_PF_BSD + if (locale == NULL && *name == 0) { + /* + No locale was specified, and an empty name was passed in. +@@ -1808,11 +1808,6 @@ remapPlatformDependentCodepage(const cha + */ + name = "UTF-8"; + } +-#elif U_PLATFORM == U_PF_BSD +- if (uprv_strcmp(name, "CP949") == 0) { +- /* Remap CP949 to a similar codepage to avoid issues with backslash and won symbol. */ +- name = "EUC-KR"; +- } + #elif U_PLATFORM == U_PF_HPUX + if (locale != NULL && uprv_strcmp(locale, "zh_HK") == 0 && uprv_strcmp(name, "big5") == 0) { + /* HP decided to extend big5 as hkbig5 even though it's not compatible :-( */ +@@ -1942,7 +1937,7 @@ int_getDefaultCodepage() + nl_langinfo may use the same buffer as setlocale. */ + { + const char *codeset = nl_langinfo(U_NL_LANGINFO_CODESET); +-#if U_PLATFORM_IS_DARWIN_BASED || U_PLATFORM_IS_LINUX_BASED ++#if U_PLATFORM_IS_DARWIN_BASED || U_PLATFORM_IS_LINUX_BASED || U_PLATFORM == U_PF_BSD + /* + * On Linux and MacOSX, ensure that default codepage for non C/POSIX locale is UTF-8 + * instead of ASCII. Copied: branches/2017Q2/devel/icu/files/patch-r39671 (from r440117, head/devel/icu/files/patch-r39671) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ branches/2017Q2/devel/icu/files/patch-r39671 Tue May 9 16:51:06 2017 (r440506, copy of r440117, head/devel/icu/files/patch-r39671) @@ -0,0 +1,176 @@ +Index: test/intltest/utxttest.h +=================================================================== +--- test/intltest/utxttest.h (revision 39670) ++++ test/intltest/utxttest.h (revision 39671) +@@ -38,6 +38,7 @@ class UTextTest : public IntlTest { + void Ticket10562(); + void Ticket10983(); + void Ticket12130(); ++ void Ticket12888(); + + private: + struct m { // Map between native indices & code points. +Index: test/intltest/utxttest.cpp +=================================================================== +--- test/intltest/utxttest.cpp (revision 39670) ++++ test/intltest/utxttest.cpp (revision 39671) +@@ -67,6 +67,8 @@ UTextTest::runIndexedTest(int32_t index, UBool exe + if (exec) Ticket10983(); break; + case 7: name = "Ticket12130"; + if (exec) Ticket12130(); break; ++ case 8: name = "Ticket12888"; ++ if (exec) Ticket12888(); break; + default: name = ""; break; + } + } +@@ -1583,3 +1585,63 @@ void UTextTest::Ticket12130() { + } + utext_close(&ut); + } ++ ++// Ticket 12888: bad handling of illegal utf-8 containing many instances of the archaic, now illegal, ++// six byte utf-8 forms. Original implementation had an assumption that ++// there would be at most three utf-8 bytes per UTF-16 code unit. ++// The five and six byte sequences map to a single replacement character. ++ ++void UTextTest::Ticket12888() { ++ const char *badString = ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" ++ "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"; ++ ++ UErrorCode status = U_ZERO_ERROR; ++ LocalUTextPointer ut(utext_openUTF8(NULL, badString, -1, &status)); ++ TEST_SUCCESS(status); ++ for (;;) { ++ UChar32 c = utext_next32(ut.getAlias()); ++ if (c == U_SENTINEL) { ++ break; ++ } ++ } ++ int32_t endIdx = utext_getNativeIndex(ut.getAlias()); ++ if (endIdx != (int32_t)strlen(badString)) { ++ errln("%s:%d expected=%d, actual=%d", __FILE__, __LINE__, strlen(badString), endIdx); ++ return; ++ } ++ ++ for (int32_t prevIndex = endIdx; prevIndex>0;) { ++ UChar32 c = utext_previous32(ut.getAlias()); ++ int32_t currentIndex = utext_getNativeIndex(ut.getAlias()); ++ if (c != 0xfffd) { ++ errln("%s:%d (expected, actual, index) = (%d, %d, %d)\n", ++ __FILE__, __LINE__, 0xfffd, c, currentIndex); ++ break; ++ } ++ if (currentIndex != prevIndex - 6) { ++ errln("%s:%d: wrong index. Expected, actual = %d, %d", ++ __FILE__, __LINE__, prevIndex - 6, currentIndex); ++ break; ++ } ++ prevIndex = currentIndex; ++ } ++} +Index: common/utext.cpp +=================================================================== +--- common/utext.cpp (revision 39670) ++++ common/utext.cpp (revision 39671) +@@ -847,9 +847,15 @@ U_CDECL_END + //------------------------------------------------------------------------------ + + // Chunk size. +-// Must be less than 85, because of byte mapping from UChar indexes to native indexes. +-// Worst case is three native bytes to one UChar. (Supplemenaries are 4 native bytes +-// to two UChars.) ++// Must be less than 42 (256/6), because of byte mapping from UChar indexes to native indexes. ++// Worst case there are six UTF-8 bytes per UChar. ++// obsolete 6 byte form fd + 5 trails maps to fffd ++// obsolete 5 byte form fc + 4 trails maps to fffd ++// non-shortest 4 byte forms maps to fffd ++// normal supplementaries map to a pair of utf-16, two utf8 bytes per utf-16 unit ++// mapToUChars array size must allow for the worst case, 6. ++// This could be brought down to 4, by treating fd and fc as pure illegal, ++// rather than obsolete lead bytes. But that is not compatible with the utf-8 access macros. + // + enum { UTF8_TEXT_CHUNK_SIZE=32 }; + +@@ -889,7 +895,7 @@ struct UTF8Buf { + // Requires two extra slots, + // one for a supplementary starting in the last normal position, + // and one for an entry for the buffer limit position. +- uint8_t mapToUChars[UTF8_TEXT_CHUNK_SIZE*3+6]; // Map native offset from bufNativeStart to ++ uint8_t mapToUChars[UTF8_TEXT_CHUNK_SIZE*6+6]; // Map native offset from bufNativeStart to + // correspoding offset in filled part of buf. + int32_t align; + }; +@@ -1032,6 +1038,7 @@ utf8TextAccess(UText *ut, int64_t index, UBool for + // Requested index is in this buffer. + u8b = (UTF8Buf *)ut->p; // the current buffer + mapIndex = ix - u8b->toUCharsMapStart; ++ U_ASSERT(mapIndex < (int32_t)sizeof(UTF8Buf::mapToUChars)); + ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx; + return TRUE; + +@@ -1298,6 +1305,10 @@ fillReverse: + // Can only do this if the incoming index is somewhere in the interior of the string. + // If index is at the end, there is no character there to look at. + if (ix != ut->b) { ++ // Note: this function will only move the index back if it is on a trail byte ++ // and there is a preceding lead byte and the sequence from the lead ++ // through this trail could be part of a valid UTF-8 sequence ++ // Otherwise the index remains unchanged. + U8_SET_CP_START(s8, 0, ix); + } + +@@ -1311,7 +1322,10 @@ fillReverse: + UChar *buf = u8b->buf; + uint8_t *mapToNative = u8b->mapToNative; + uint8_t *mapToUChars = u8b->mapToUChars; +- int32_t toUCharsMapStart = ix - (UTF8_TEXT_CHUNK_SIZE*3 + 1); ++ int32_t toUCharsMapStart = ix - sizeof(UTF8Buf::mapToUChars) + 1; ++ // Note that toUCharsMapStart can be negative. Happens when the remaining ++ // text from current position to the beginning is less than the buffer size. ++ // + 1 because mapToUChars must have a slot at the end for the bufNativeLimit entry. + int32_t destIx = UTF8_TEXT_CHUNK_SIZE+2; // Start in the overflow region + // at end of buffer to leave room + // for a surrogate pair at the +@@ -1338,6 +1352,7 @@ fillReverse: + if (c<0x80) { + // Special case ASCII range for speed. + buf[destIx] = (UChar)c; ++ U_ASSERT(toUCharsMapStart <= srcIx); + mapToUChars[srcIx - toUCharsMapStart] = (uint8_t)destIx; + mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart); + } else { +@@ -1367,6 +1382,7 @@ fillReverse: + do { + mapToUChars[sIx-- - toUCharsMapStart] = (uint8_t)destIx; + } while (sIx >= srcIx); ++ U_ASSERT(toUCharsMapStart <= (srcIx+1)); + + // Set native indexing limit to be the current position. + // We are processing a non-ascii, non-native-indexing char now; +@@ -1541,6 +1557,7 @@ utf8TextMapIndexToUTF16(const UText *ut, int64_t i + U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit); + U_ASSERT(index<=ut->chunkNativeLimit); + int32_t mapIndex = index - u8b->toUCharsMapStart; ++ U_ASSERT(mapIndex < (int32_t)sizeof(UTF8Buf::mapToUChars)); + int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx; + U_ASSERT(offset>=0 && offset<=ut->chunkLength); + return offset;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201705091651.v49Gp7bZ041027>