Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 9 Nov 2018 03:32:54 +0000 (UTC)
From:      Yuri Pankov <yuripv@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r340276 - in head: contrib/netbsd-tests/lib/libc/locale lib/libc/locale
Message-ID:  <201811090332.wA93WsYM068423@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: yuripv
Date: Fri Nov  9 03:32:53 2018
New Revision: 340276
URL: https://svnweb.freebsd.org/changeset/base/340276

Log:
  Reset persistent mbstates when rune locale encoding changes.
  
  This was shown to be a problem by side effect of now-enabled test case,
  which was going through C, en_US.UTF-8, ja_JP.SJIS, and ja_JP.eucJP,
  and failing eventually as data in mbrtowc's mbstate, that was
  perfectly correct for en_US.UTF-8 was treated as incorrect for
  ja_JP.SJIS, failing the entire test case.
  
  This makes the persistent mbstates to be per ctype-component,
  and not per-locale so we could easily reset the mbstates when
  only LC_CTYPE is changed.
  
  Reviewed by:	bapt, pfg
  Approved by:	kib (mentor, implicit)
  Differential Revision:	https://reviews.freebsd.org/D17796

Modified:
  head/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c
  head/lib/libc/locale/cXXrtomb_iconv.h
  head/lib/libc/locale/mblen.c
  head/lib/libc/locale/mblocal.h
  head/lib/libc/locale/mbrlen.c
  head/lib/libc/locale/mbrtocXX_iconv.h
  head/lib/libc/locale/mbrtowc.c
  head/lib/libc/locale/mbsnrtowcs.c
  head/lib/libc/locale/mbsrtowcs.c
  head/lib/libc/locale/mbtowc.c
  head/lib/libc/locale/setrunelocale.c
  head/lib/libc/locale/wcrtomb.c
  head/lib/libc/locale/wcsnrtombs.c
  head/lib/libc/locale/wcsrtombs.c
  head/lib/libc/locale/wctomb.c
  head/lib/libc/locale/xlocale_private.h

Modified: head/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c
==============================================================================
--- head/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -243,9 +243,6 @@ ATF_TC_BODY(mbrtowc_internal, tc)
 {
 	struct test *t;
 
-#ifdef __FreeBSD__
-	atf_tc_expect_fail("ja_* locale fails");
-#endif
 	for (t = &tests[0]; t->data != NULL; ++t)
 		h_ctype2(t, false);
 }

Modified: head/lib/libc/locale/cXXrtomb_iconv.h
==============================================================================
--- head/lib/libc/locale/cXXrtomb_iconv.h	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/cXXrtomb_iconv.h	Fri Nov  9 03:32:53 2018	(r340276)
@@ -39,7 +39,7 @@ __FBSDID("$FreeBSD$");
 #include "../iconv/citrus_hash.h"
 #include "../iconv/citrus_module.h"
 #include "../iconv/citrus_iconv.h"
-#include "xlocale_private.h"
+#include "mblocal.h"
 
 typedef struct {
 	bool			initialized;
@@ -65,7 +65,7 @@ cXXrtomb_l(char * __restrict s, charXX_t c, mbstate_t 
 
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->cXXrtomb;
+		ps = &(XLOCALE_CTYPE(locale)->cXXrtomb);
 	cs = (_ConversionState *)ps;
 	handle = &cs->iconv;
 

Modified: head/lib/libc/locale/mblen.c
==============================================================================
--- head/lib/libc/locale/mblen.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/mblen.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -47,10 +47,11 @@ mblen_l(const char *s, size_t n, locale_t locale)
 
 	if (s == NULL) {
 		/* No support for state dependent encodings. */
-		locale->mblen = initial;
+		XLOCALE_CTYPE(locale)->mblen = initial;
 		return (0);
 	}
-	rval = XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, &locale->mblen);
+	rval = XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n,
+	    &(XLOCALE_CTYPE(locale)->mblen));
 	if (rval == (size_t)-1 || rval == (size_t)-2)
 		return (-1);
 	return ((int)rval);

Modified: head/lib/libc/locale/mblocal.h
==============================================================================
--- head/lib/libc/locale/mblocal.h	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/mblocal.h	Fri Nov  9 03:32:53 2018	(r340276)
@@ -60,6 +60,34 @@ struct xlocale_ctype {
 		size_t, size_t, mbstate_t * __restrict);
 	int __mb_cur_max;
 	int __mb_sb_limit;
+	/** Persistent state used by mblen() calls. */
+	__mbstate_t mblen;
+	/** Persistent state used by mbrlen() calls. */
+	__mbstate_t mbrlen;
+	/** Persistent state used by mbrtoc16() calls. */
+	__mbstate_t mbrtoc16;
+	/** Persistent state used by mbrtoc32() calls. */
+	__mbstate_t mbrtoc32;
+	/** Persistent state used by mbrtowc() calls. */
+	__mbstate_t mbrtowc;
+	/** Persistent state used by mbsnrtowcs() calls. */
+	__mbstate_t mbsnrtowcs;
+	/** Persistent state used by mbsrtowcs() calls. */
+	__mbstate_t mbsrtowcs;
+	/** Persistent state used by mbtowc() calls. */
+	__mbstate_t mbtowc;
+	/** Persistent state used by c16rtomb() calls. */
+	__mbstate_t c16rtomb;
+	/** Persistent state used by c32rtomb() calls. */
+	__mbstate_t c32rtomb;
+	/** Persistent state used by wcrtomb() calls. */
+	__mbstate_t wcrtomb;
+	/** Persistent state used by wcsnrtombs() calls. */
+	__mbstate_t wcsnrtombs;
+	/** Persistent state used by wcsrtombs() calls. */
+	__mbstate_t wcsrtombs;
+	/** Persistent state used by wctomb() calls. */
+	__mbstate_t wctomb;
 };
 #define XLOCALE_CTYPE(x) ((struct xlocale_ctype*)(x)->components[XLC_CTYPE])
 extern struct xlocale_ctype __xlocale_global_ctype;

Modified: head/lib/libc/locale/mbrlen.c
==============================================================================
--- head/lib/libc/locale/mbrlen.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/mbrlen.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -42,7 +42,7 @@ mbrlen_l(const char * __restrict s, size_t n, mbstate_
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->mbrlen;
+		ps = &(XLOCALE_CTYPE(locale)->mbrlen);
 	return (XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, ps));
 }
 

Modified: head/lib/libc/locale/mbrtocXX_iconv.h
==============================================================================
--- head/lib/libc/locale/mbrtocXX_iconv.h	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/mbrtocXX_iconv.h	Fri Nov  9 03:32:53 2018	(r340276)
@@ -41,7 +41,7 @@ __FBSDID("$FreeBSD$");
 #include "../iconv/citrus_hash.h"
 #include "../iconv/citrus_module.h"
 #include "../iconv/citrus_iconv.h"
-#include "xlocale_private.h"
+#include "mblocal.h"
 
 typedef struct {
 	bool			initialized;
@@ -68,7 +68,7 @@ mbrtocXX_l(charXX_t * __restrict pc, const char * __re
 
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->mbrtocXX;
+		ps = &(XLOCALE_CTYPE(locale)->mbrtocXX);
 	cs = (_ConversionState *)ps;
 	handle = &cs->iconv;
 

Modified: head/lib/libc/locale/mbrtowc.c
==============================================================================
--- head/lib/libc/locale/mbrtowc.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/mbrtowc.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -43,7 +43,7 @@ mbrtowc_l(wchar_t * __restrict pwc, const char * __res
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->mbrtowc;
+		ps = &(XLOCALE_CTYPE(locale)->mbrtowc);
 	return (XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, ps));
 }
 

Modified: head/lib/libc/locale/mbsnrtowcs.c
==============================================================================
--- head/lib/libc/locale/mbsnrtowcs.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/mbsnrtowcs.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -48,7 +48,7 @@ mbsnrtowcs_l(wchar_t * __restrict dst, const char ** _
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->mbsnrtowcs;
+		ps = &(XLOCALE_CTYPE(locale)->mbsnrtowcs);
 	return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(dst, src, nms, len, ps));
 }
 size_t

Modified: head/lib/libc/locale/mbsrtowcs.c
==============================================================================
--- head/lib/libc/locale/mbsrtowcs.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/mbsrtowcs.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -46,7 +46,7 @@ mbsrtowcs_l(wchar_t * __restrict dst, const char ** __
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->mbsrtowcs;
+		ps = &(XLOCALE_CTYPE(locale)->mbsrtowcs);
 	return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(dst, src, SIZE_T_MAX, len, ps));
 }
 size_t

Modified: head/lib/libc/locale/mbtowc.c
==============================================================================
--- head/lib/libc/locale/mbtowc.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/mbtowc.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -48,10 +48,11 @@ mbtowc_l(wchar_t * __restrict pwc, const char * __rest
 
 	if (s == NULL) {
 		/* No support for state dependent encodings. */
-		locale->mbtowc = initial;
+		XLOCALE_CTYPE(locale)->mbtowc = initial;
 		return (0);
 	}
-	rval = XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, &locale->mbtowc);
+	rval = XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n,
+	    &(XLOCALE_CTYPE(locale)->mbtowc));
 	switch (rval) {
 	case (size_t)-2:
 		errno = EILSEQ;

Modified: head/lib/libc/locale/setrunelocale.c
==============================================================================
--- head/lib/libc/locale/setrunelocale.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/setrunelocale.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -160,6 +160,21 @@ __setrunelocale(struct xlocale_ctype *l, const char *e
 	if (ret == 0) {
 		/* Free the old runes if it exists. */
 		free_runes(saved.runes);
+		/* Reset the mbstates */
+		memset(&l->c16rtomb, 0, sizeof(l->c16rtomb));
+		memset(&l->c32rtomb, 0, sizeof(l->c32rtomb));
+		memset(&l->mblen, 0, sizeof(l->mblen));
+		memset(&l->mbrlen, 0, sizeof(l->mbrlen));
+		memset(&l->mbrtoc16, 0, sizeof(l->mbrtoc16));
+		memset(&l->mbrtoc32, 0, sizeof(l->mbrtoc32));
+		memset(&l->mbrtowc, 0, sizeof(l->mbrtowc));
+		memset(&l->mbsnrtowcs, 0, sizeof(l->mbsnrtowcs));
+		memset(&l->mbsrtowcs, 0, sizeof(l->mbsrtowcs));
+		memset(&l->mbtowc, 0, sizeof(l->mbtowc));
+		memset(&l->wcrtomb, 0, sizeof(l->wcrtomb));
+		memset(&l->wcsnrtombs, 0, sizeof(l->wcsnrtombs));
+		memset(&l->wcsrtombs, 0, sizeof(l->wcsrtombs));
+		memset(&l->wctomb, 0, sizeof(l->wctomb));
 	} else {
 		/* Restore the saved version if this failed. */
 		memcpy(l, &saved, sizeof(struct xlocale_ctype));

Modified: head/lib/libc/locale/wcrtomb.c
==============================================================================
--- head/lib/libc/locale/wcrtomb.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/wcrtomb.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -43,7 +43,7 @@ wcrtomb_l(char * __restrict s, wchar_t wc, mbstate_t *
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->wcrtomb;
+		ps = &(XLOCALE_CTYPE(locale)->wcrtomb);
 	return (XLOCALE_CTYPE(locale)->__wcrtomb(s, wc, ps));
 }
 

Modified: head/lib/libc/locale/wcsnrtombs.c
==============================================================================
--- head/lib/libc/locale/wcsnrtombs.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/wcsnrtombs.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -48,7 +48,7 @@ wcsnrtombs_l(char * __restrict dst, const wchar_t ** _
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->wcsnrtombs;
+		ps = &(XLOCALE_CTYPE(locale)->wcsnrtombs);
 	return (XLOCALE_CTYPE(locale)->__wcsnrtombs(dst, src, nwc, len, ps));
 }
 size_t

Modified: head/lib/libc/locale/wcsrtombs.c
==============================================================================
--- head/lib/libc/locale/wcsrtombs.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/wcsrtombs.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -46,7 +46,7 @@ wcsrtombs_l(char * __restrict dst, const wchar_t ** __
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->wcsrtombs;
+		ps = &(XLOCALE_CTYPE(locale)->wcsrtombs);
 	return (XLOCALE_CTYPE(locale)->__wcsnrtombs(dst, src, SIZE_T_MAX, len, ps));
 }
 

Modified: head/lib/libc/locale/wctomb.c
==============================================================================
--- head/lib/libc/locale/wctomb.c	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/wctomb.c	Fri Nov  9 03:32:53 2018	(r340276)
@@ -47,10 +47,11 @@ wctomb_l(char *s, wchar_t wchar, locale_t locale)
 
 	if (s == NULL) {
 		/* No support for state dependent encodings. */
-		locale->wctomb = initial;
+		XLOCALE_CTYPE(locale)->wctomb = initial;
 		return (0);
 	}
-	if ((rval = XLOCALE_CTYPE(locale)->__wcrtomb(s, wchar, &locale->wctomb)) == (size_t)-1)
+	if ((rval = XLOCALE_CTYPE(locale)->__wcrtomb(s, wchar,
+	    &(XLOCALE_CTYPE(locale)->wctomb))) == (size_t)-1)
 		return (-1);
 	return ((int)rval);
 }

Modified: head/lib/libc/locale/xlocale_private.h
==============================================================================
--- head/lib/libc/locale/xlocale_private.h	Fri Nov  9 00:46:50 2018	(r340275)
+++ head/lib/libc/locale/xlocale_private.h	Fri Nov  9 03:32:53 2018	(r340276)
@@ -128,34 +128,6 @@ struct _xlocale {
 	int using_messages_locale;
 	/** The structure to be returned from localeconv_l() for this locale. */
 	struct lconv lconv;
-	/** Persistent state used by mblen() calls. */
-	__mbstate_t mblen;
-	/** Persistent state used by mbrlen() calls. */
-	__mbstate_t mbrlen;
-	/** Persistent state used by mbrtoc16() calls. */
-	__mbstate_t mbrtoc16;
-	/** Persistent state used by mbrtoc32() calls. */
-	__mbstate_t mbrtoc32;
-	/** Persistent state used by mbrtowc() calls. */
-	__mbstate_t mbrtowc;
-	/** Persistent state used by mbsnrtowcs() calls. */
-	__mbstate_t mbsnrtowcs;
-	/** Persistent state used by mbsrtowcs() calls. */
-	__mbstate_t mbsrtowcs;
-	/** Persistent state used by mbtowc() calls. */
-	__mbstate_t mbtowc;
-	/** Persistent state used by c16rtomb() calls. */
-	__mbstate_t c16rtomb;
-	/** Persistent state used by c32rtomb() calls. */
-	__mbstate_t c32rtomb;
-	/** Persistent state used by wcrtomb() calls. */
-	__mbstate_t wcrtomb;
-	/** Persistent state used by wcsnrtombs() calls. */
-	__mbstate_t wcsnrtombs;
-	/** Persistent state used by wcsrtombs() calls. */
-	__mbstate_t wcsrtombs;
-	/** Persistent state used by wctomb() calls. */
-	__mbstate_t wctomb;
 	/** Buffer used by nl_langinfo_l() */
 	char *csym;
 };



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201811090332.wA93WsYM068423>