Date: Fri, 9 Nov 2018 03:32:54 +0000 (UTC) From: Yuri Pankov <yuripv@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r340276 - in head: contrib/netbsd-tests/lib/libc/locale lib/libc/locale Message-ID: <201811090332.wA93WsYM068423@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: yuripv Date: Fri Nov 9 03:32:53 2018 New Revision: 340276 URL: https://svnweb.freebsd.org/changeset/base/340276 Log: Reset persistent mbstates when rune locale encoding changes. This was shown to be a problem by side effect of now-enabled test case, which was going through C, en_US.UTF-8, ja_JP.SJIS, and ja_JP.eucJP, and failing eventually as data in mbrtowc's mbstate, that was perfectly correct for en_US.UTF-8 was treated as incorrect for ja_JP.SJIS, failing the entire test case. This makes the persistent mbstates to be per ctype-component, and not per-locale so we could easily reset the mbstates when only LC_CTYPE is changed. Reviewed by: bapt, pfg Approved by: kib (mentor, implicit) Differential Revision: https://reviews.freebsd.org/D17796 Modified: head/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c head/lib/libc/locale/cXXrtomb_iconv.h head/lib/libc/locale/mblen.c head/lib/libc/locale/mblocal.h head/lib/libc/locale/mbrlen.c head/lib/libc/locale/mbrtocXX_iconv.h head/lib/libc/locale/mbrtowc.c head/lib/libc/locale/mbsnrtowcs.c head/lib/libc/locale/mbsrtowcs.c head/lib/libc/locale/mbtowc.c head/lib/libc/locale/setrunelocale.c head/lib/libc/locale/wcrtomb.c head/lib/libc/locale/wcsnrtombs.c head/lib/libc/locale/wcsrtombs.c head/lib/libc/locale/wctomb.c head/lib/libc/locale/xlocale_private.h Modified: head/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c ============================================================================== --- head/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c Fri Nov 9 03:32:53 2018 (r340276) @@ -243,9 +243,6 @@ ATF_TC_BODY(mbrtowc_internal, tc) { struct test *t; -#ifdef __FreeBSD__ - atf_tc_expect_fail("ja_* locale fails"); -#endif for (t = &tests[0]; t->data != NULL; ++t) h_ctype2(t, false); } Modified: head/lib/libc/locale/cXXrtomb_iconv.h ============================================================================== --- head/lib/libc/locale/cXXrtomb_iconv.h Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/cXXrtomb_iconv.h Fri Nov 9 03:32:53 2018 (r340276) @@ -39,7 +39,7 @@ __FBSDID("$FreeBSD$"); #include "../iconv/citrus_hash.h" #include "../iconv/citrus_module.h" #include "../iconv/citrus_iconv.h" -#include "xlocale_private.h" +#include "mblocal.h" typedef struct { bool initialized; @@ -65,7 +65,7 @@ cXXrtomb_l(char * __restrict s, charXX_t c, mbstate_t FIX_LOCALE(locale); if (ps == NULL) - ps = &locale->cXXrtomb; + ps = &(XLOCALE_CTYPE(locale)->cXXrtomb); cs = (_ConversionState *)ps; handle = &cs->iconv; Modified: head/lib/libc/locale/mblen.c ============================================================================== --- head/lib/libc/locale/mblen.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/mblen.c Fri Nov 9 03:32:53 2018 (r340276) @@ -47,10 +47,11 @@ mblen_l(const char *s, size_t n, locale_t locale) if (s == NULL) { /* No support for state dependent encodings. */ - locale->mblen = initial; + XLOCALE_CTYPE(locale)->mblen = initial; return (0); } - rval = XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, &locale->mblen); + rval = XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, + &(XLOCALE_CTYPE(locale)->mblen)); if (rval == (size_t)-1 || rval == (size_t)-2) return (-1); return ((int)rval); Modified: head/lib/libc/locale/mblocal.h ============================================================================== --- head/lib/libc/locale/mblocal.h Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/mblocal.h Fri Nov 9 03:32:53 2018 (r340276) @@ -60,6 +60,34 @@ struct xlocale_ctype { size_t, size_t, mbstate_t * __restrict); int __mb_cur_max; int __mb_sb_limit; + /** Persistent state used by mblen() calls. */ + __mbstate_t mblen; + /** Persistent state used by mbrlen() calls. */ + __mbstate_t mbrlen; + /** Persistent state used by mbrtoc16() calls. */ + __mbstate_t mbrtoc16; + /** Persistent state used by mbrtoc32() calls. */ + __mbstate_t mbrtoc32; + /** Persistent state used by mbrtowc() calls. */ + __mbstate_t mbrtowc; + /** Persistent state used by mbsnrtowcs() calls. */ + __mbstate_t mbsnrtowcs; + /** Persistent state used by mbsrtowcs() calls. */ + __mbstate_t mbsrtowcs; + /** Persistent state used by mbtowc() calls. */ + __mbstate_t mbtowc; + /** Persistent state used by c16rtomb() calls. */ + __mbstate_t c16rtomb; + /** Persistent state used by c32rtomb() calls. */ + __mbstate_t c32rtomb; + /** Persistent state used by wcrtomb() calls. */ + __mbstate_t wcrtomb; + /** Persistent state used by wcsnrtombs() calls. */ + __mbstate_t wcsnrtombs; + /** Persistent state used by wcsrtombs() calls. */ + __mbstate_t wcsrtombs; + /** Persistent state used by wctomb() calls. */ + __mbstate_t wctomb; }; #define XLOCALE_CTYPE(x) ((struct xlocale_ctype*)(x)->components[XLC_CTYPE]) extern struct xlocale_ctype __xlocale_global_ctype; Modified: head/lib/libc/locale/mbrlen.c ============================================================================== --- head/lib/libc/locale/mbrlen.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/mbrlen.c Fri Nov 9 03:32:53 2018 (r340276) @@ -42,7 +42,7 @@ mbrlen_l(const char * __restrict s, size_t n, mbstate_ { FIX_LOCALE(locale); if (ps == NULL) - ps = &locale->mbrlen; + ps = &(XLOCALE_CTYPE(locale)->mbrlen); return (XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, ps)); } Modified: head/lib/libc/locale/mbrtocXX_iconv.h ============================================================================== --- head/lib/libc/locale/mbrtocXX_iconv.h Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/mbrtocXX_iconv.h Fri Nov 9 03:32:53 2018 (r340276) @@ -41,7 +41,7 @@ __FBSDID("$FreeBSD$"); #include "../iconv/citrus_hash.h" #include "../iconv/citrus_module.h" #include "../iconv/citrus_iconv.h" -#include "xlocale_private.h" +#include "mblocal.h" typedef struct { bool initialized; @@ -68,7 +68,7 @@ mbrtocXX_l(charXX_t * __restrict pc, const char * __re FIX_LOCALE(locale); if (ps == NULL) - ps = &locale->mbrtocXX; + ps = &(XLOCALE_CTYPE(locale)->mbrtocXX); cs = (_ConversionState *)ps; handle = &cs->iconv; Modified: head/lib/libc/locale/mbrtowc.c ============================================================================== --- head/lib/libc/locale/mbrtowc.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/mbrtowc.c Fri Nov 9 03:32:53 2018 (r340276) @@ -43,7 +43,7 @@ mbrtowc_l(wchar_t * __restrict pwc, const char * __res { FIX_LOCALE(locale); if (ps == NULL) - ps = &locale->mbrtowc; + ps = &(XLOCALE_CTYPE(locale)->mbrtowc); return (XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, ps)); } Modified: head/lib/libc/locale/mbsnrtowcs.c ============================================================================== --- head/lib/libc/locale/mbsnrtowcs.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/mbsnrtowcs.c Fri Nov 9 03:32:53 2018 (r340276) @@ -48,7 +48,7 @@ mbsnrtowcs_l(wchar_t * __restrict dst, const char ** _ { FIX_LOCALE(locale); if (ps == NULL) - ps = &locale->mbsnrtowcs; + ps = &(XLOCALE_CTYPE(locale)->mbsnrtowcs); return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(dst, src, nms, len, ps)); } size_t Modified: head/lib/libc/locale/mbsrtowcs.c ============================================================================== --- head/lib/libc/locale/mbsrtowcs.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/mbsrtowcs.c Fri Nov 9 03:32:53 2018 (r340276) @@ -46,7 +46,7 @@ mbsrtowcs_l(wchar_t * __restrict dst, const char ** __ { FIX_LOCALE(locale); if (ps == NULL) - ps = &locale->mbsrtowcs; + ps = &(XLOCALE_CTYPE(locale)->mbsrtowcs); return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(dst, src, SIZE_T_MAX, len, ps)); } size_t Modified: head/lib/libc/locale/mbtowc.c ============================================================================== --- head/lib/libc/locale/mbtowc.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/mbtowc.c Fri Nov 9 03:32:53 2018 (r340276) @@ -48,10 +48,11 @@ mbtowc_l(wchar_t * __restrict pwc, const char * __rest if (s == NULL) { /* No support for state dependent encodings. */ - locale->mbtowc = initial; + XLOCALE_CTYPE(locale)->mbtowc = initial; return (0); } - rval = XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, &locale->mbtowc); + rval = XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, + &(XLOCALE_CTYPE(locale)->mbtowc)); switch (rval) { case (size_t)-2: errno = EILSEQ; Modified: head/lib/libc/locale/setrunelocale.c ============================================================================== --- head/lib/libc/locale/setrunelocale.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/setrunelocale.c Fri Nov 9 03:32:53 2018 (r340276) @@ -160,6 +160,21 @@ __setrunelocale(struct xlocale_ctype *l, const char *e if (ret == 0) { /* Free the old runes if it exists. */ free_runes(saved.runes); + /* Reset the mbstates */ + memset(&l->c16rtomb, 0, sizeof(l->c16rtomb)); + memset(&l->c32rtomb, 0, sizeof(l->c32rtomb)); + memset(&l->mblen, 0, sizeof(l->mblen)); + memset(&l->mbrlen, 0, sizeof(l->mbrlen)); + memset(&l->mbrtoc16, 0, sizeof(l->mbrtoc16)); + memset(&l->mbrtoc32, 0, sizeof(l->mbrtoc32)); + memset(&l->mbrtowc, 0, sizeof(l->mbrtowc)); + memset(&l->mbsnrtowcs, 0, sizeof(l->mbsnrtowcs)); + memset(&l->mbsrtowcs, 0, sizeof(l->mbsrtowcs)); + memset(&l->mbtowc, 0, sizeof(l->mbtowc)); + memset(&l->wcrtomb, 0, sizeof(l->wcrtomb)); + memset(&l->wcsnrtombs, 0, sizeof(l->wcsnrtombs)); + memset(&l->wcsrtombs, 0, sizeof(l->wcsrtombs)); + memset(&l->wctomb, 0, sizeof(l->wctomb)); } else { /* Restore the saved version if this failed. */ memcpy(l, &saved, sizeof(struct xlocale_ctype)); Modified: head/lib/libc/locale/wcrtomb.c ============================================================================== --- head/lib/libc/locale/wcrtomb.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/wcrtomb.c Fri Nov 9 03:32:53 2018 (r340276) @@ -43,7 +43,7 @@ wcrtomb_l(char * __restrict s, wchar_t wc, mbstate_t * { FIX_LOCALE(locale); if (ps == NULL) - ps = &locale->wcrtomb; + ps = &(XLOCALE_CTYPE(locale)->wcrtomb); return (XLOCALE_CTYPE(locale)->__wcrtomb(s, wc, ps)); } Modified: head/lib/libc/locale/wcsnrtombs.c ============================================================================== --- head/lib/libc/locale/wcsnrtombs.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/wcsnrtombs.c Fri Nov 9 03:32:53 2018 (r340276) @@ -48,7 +48,7 @@ wcsnrtombs_l(char * __restrict dst, const wchar_t ** _ { FIX_LOCALE(locale); if (ps == NULL) - ps = &locale->wcsnrtombs; + ps = &(XLOCALE_CTYPE(locale)->wcsnrtombs); return (XLOCALE_CTYPE(locale)->__wcsnrtombs(dst, src, nwc, len, ps)); } size_t Modified: head/lib/libc/locale/wcsrtombs.c ============================================================================== --- head/lib/libc/locale/wcsrtombs.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/wcsrtombs.c Fri Nov 9 03:32:53 2018 (r340276) @@ -46,7 +46,7 @@ wcsrtombs_l(char * __restrict dst, const wchar_t ** __ { FIX_LOCALE(locale); if (ps == NULL) - ps = &locale->wcsrtombs; + ps = &(XLOCALE_CTYPE(locale)->wcsrtombs); return (XLOCALE_CTYPE(locale)->__wcsnrtombs(dst, src, SIZE_T_MAX, len, ps)); } Modified: head/lib/libc/locale/wctomb.c ============================================================================== --- head/lib/libc/locale/wctomb.c Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/wctomb.c Fri Nov 9 03:32:53 2018 (r340276) @@ -47,10 +47,11 @@ wctomb_l(char *s, wchar_t wchar, locale_t locale) if (s == NULL) { /* No support for state dependent encodings. */ - locale->wctomb = initial; + XLOCALE_CTYPE(locale)->wctomb = initial; return (0); } - if ((rval = XLOCALE_CTYPE(locale)->__wcrtomb(s, wchar, &locale->wctomb)) == (size_t)-1) + if ((rval = XLOCALE_CTYPE(locale)->__wcrtomb(s, wchar, + &(XLOCALE_CTYPE(locale)->wctomb))) == (size_t)-1) return (-1); return ((int)rval); } Modified: head/lib/libc/locale/xlocale_private.h ============================================================================== --- head/lib/libc/locale/xlocale_private.h Fri Nov 9 00:46:50 2018 (r340275) +++ head/lib/libc/locale/xlocale_private.h Fri Nov 9 03:32:53 2018 (r340276) @@ -128,34 +128,6 @@ struct _xlocale { int using_messages_locale; /** The structure to be returned from localeconv_l() for this locale. */ struct lconv lconv; - /** Persistent state used by mblen() calls. */ - __mbstate_t mblen; - /** Persistent state used by mbrlen() calls. */ - __mbstate_t mbrlen; - /** Persistent state used by mbrtoc16() calls. */ - __mbstate_t mbrtoc16; - /** Persistent state used by mbrtoc32() calls. */ - __mbstate_t mbrtoc32; - /** Persistent state used by mbrtowc() calls. */ - __mbstate_t mbrtowc; - /** Persistent state used by mbsnrtowcs() calls. */ - __mbstate_t mbsnrtowcs; - /** Persistent state used by mbsrtowcs() calls. */ - __mbstate_t mbsrtowcs; - /** Persistent state used by mbtowc() calls. */ - __mbstate_t mbtowc; - /** Persistent state used by c16rtomb() calls. */ - __mbstate_t c16rtomb; - /** Persistent state used by c32rtomb() calls. */ - __mbstate_t c32rtomb; - /** Persistent state used by wcrtomb() calls. */ - __mbstate_t wcrtomb; - /** Persistent state used by wcsnrtombs() calls. */ - __mbstate_t wcsnrtombs; - /** Persistent state used by wcsrtombs() calls. */ - __mbstate_t wcsrtombs; - /** Persistent state used by wctomb() calls. */ - __mbstate_t wctomb; /** Buffer used by nl_langinfo_l() */ char *csym; };
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201811090332.wA93WsYM068423>