From owner-svn-soc-all@FreeBSD.ORG Tue Aug 12 18:05:26 2014 Return-Path: Delivered-To: svn-soc-all@FreeBSD.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 8D1C7831 for ; Tue, 12 Aug 2014 18:05:26 +0000 (UTC) Received: from socsvn.freebsd.org (socsvn.freebsd.org [IPv6:2001:1900:2254:206a::50:2]) (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 5C63A308F for ; Tue, 12 Aug 2014 17:29:24 +0000 (UTC) Received: from socsvn.freebsd.org ([127.0.1.124]) by socsvn.freebsd.org (8.14.9/8.14.9) with ESMTP id s7CHTOAf040160 for ; Tue, 12 Aug 2014 17:29:24 GMT (envelope-from ghostmansd@FreeBSD.org) Received: (from www@localhost) by socsvn.freebsd.org (8.14.9/8.14.9/Submit) id s7CHTK23040135 for svn-soc-all@FreeBSD.org; Tue, 12 Aug 2014 17:29:20 GMT (envelope-from ghostmansd@FreeBSD.org) Date: Tue, 12 Aug 2014 17:29:20 GMT Message-Id: <201408121729.s7CHTK23040135@socsvn.freebsd.org> X-Authentication-Warning: socsvn.freebsd.org: www set sender to ghostmansd@FreeBSD.org using -f From: ghostmansd@FreeBSD.org To: svn-soc-all@FreeBSD.org Subject: socsvn commit: r272296 - in soc2014/ghostmansd/head: contrib/colldb lib lib/libc lib/libc/locale lib/libc/unicode lib/libcolldb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-soc-all@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: SVN commit messages for the entire Summer of Code repository List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 12 Aug 2014 18:05:26 -0000 Author: ghostmansd Date: Tue Aug 12 17:29:20 2014 New Revision: 272296 URL: http://svnweb.FreeBSD.org/socsvn/?view=rev&rev=272296 Log: putting everything together libc now supports UNICODE flag. If it is defined, then the entire libc is compiled with -D_UNICODE_SOURCE, thus supporting Unicode Collation Algorithm as well as Unicode normalization and canonicalization using hidden __ucsnorm() and __ucscanon() functions. Collation Database Library (libcolldb) moved into contrib/, though it has its own Makefile inside lib/ directory. Collation Database Library provides colldb script, which is used to transform Unicode collation files into Collation Database format. Added: soc2014/ghostmansd/head/contrib/colldb/ - copied from r272251, soc2014/ghostmansd/head/lib/libcolldb/ soc2014/ghostmansd/head/contrib/colldb/colldb - copied, changed from r272251, soc2014/ghostmansd/head/lib/libcolldb/colldb.py soc2014/ghostmansd/head/lib/libc/unicode/Makefile.inc soc2014/ghostmansd/head/lib/libcolldb/Makefile Replaced: soc2014/ghostmansd/head/lib/libcolldb/ Deleted: soc2014/ghostmansd/head/contrib/colldb/colldb.py soc2014/ghostmansd/head/lib/libc/unicode/uchash.c Modified: soc2014/ghostmansd/head/contrib/colldb/colldb.c soc2014/ghostmansd/head/contrib/colldb/colldb.h soc2014/ghostmansd/head/lib/Makefile soc2014/ghostmansd/head/lib/libc/Makefile soc2014/ghostmansd/head/lib/libc/locale/collate.c soc2014/ghostmansd/head/lib/libc/locale/xlocale.c soc2014/ghostmansd/head/lib/libc/locale/xlocale_private.h soc2014/ghostmansd/head/lib/libc/unicode/coll.h soc2014/ghostmansd/head/lib/libc/unicode/norm.h soc2014/ghostmansd/head/lib/libc/unicode/ucscoll.c soc2014/ghostmansd/head/lib/libc/unicode/ucsnorm.c soc2014/ghostmansd/head/lib/libc/unicode/ucsxfrm.c Copied and modified: soc2014/ghostmansd/head/contrib/colldb/colldb (from r272251, soc2014/ghostmansd/head/lib/libcolldb/colldb.py) ============================================================================== --- soc2014/ghostmansd/head/lib/libcolldb/colldb.py Tue Aug 12 01:40:11 2014 (r272251, copy source) +++ soc2014/ghostmansd/head/contrib/colldb/colldb Tue Aug 12 17:29:20 2014 (r272296) @@ -1,11 +1,12 @@ #!/usr/bin/env python # *-* coding: utf-8 *-* -"""Python bindings for FreeBSD libcoll2db""" +"""Python bindings for libcolldb""" import codecs import ctypes import os import re +import sys __author__ = "Dmitry Selyutin" @@ -14,43 +15,12 @@ __maintainer__ = "Dmitry Selyutin" -_dll_ = ctypes.CDLL("libcoll2db.so") +#_dll_ = ctypes.CDLL("libcolldb.so") +_dll_ = ctypes.CDLL("./colldb.so") -_create_ = _dll_.__collation_create -_create_.argtypes = [ctypes.POINTER(ctypes.c_char), ctypes.c_int] -_create_.restype = ctypes.POINTER(CollationDB) - -_open_ = _dll_.__collation_open -_open_.argtypes = [ctypes.POINTER(ctypes.c_char)] -_open_.restype = ctypes.POINTER(CollationDB) -_sync_ = _dll_.__collation_close -_sync_.argtypes = [ctypes.POINTER(CollationDB)] -_sync_.restype = ctypes.c_int - -_close_ = _dll_.__collation_close -_close_.argtypes = [ctypes.POINTER(CollationDB)] -_close_.restype = ctypes.c_int - -_get_ = _dll_.__collation_get -_get_.argtypes = [ - ctypes.POINTER(CollationDB), - ctypes.POINTER(_CollatonKey_), - ctypes.POINTER(_CollationValue_), -] -_get_.restype = ctypes.c_int - -_put_ = _dll_.__collation_put -_put_.argtypes = [ - ctypes.POINTER(CollationDB), - ctypes.POINTER(_CollatonKey_), - ctypes.POINTER(_CollationValue_), -] -_put_.restype = ctypes.c_int - - -class _CollationWeight_(ctypes.Structure): - """struct __collation_weight { +class _CollDB_Weight_(ctypes.Structure): + """struct colldb_weight { uint8_t alternate; uint32_t level1; uint32_t level2; @@ -66,8 +36,8 @@ ] -class _CollatonKey_(ctypes.Structure): - """struct __collation_key { +class _CollDB_Key_(ctypes.Structure): + """struct colldb_key { uint8_t count; uint32_t *chars; };""" @@ -77,33 +47,55 @@ ] -class _CollationValue_(ctypes.Structure): - """struct __collation_value { +class _CollDB_Value_(ctypes.Structure): + """struct colldb_value { uint8_t count; - struct __collation_weight *weights; + struct colldb_weight *weights; };""" _fields_ = [ ("count", ctypes.c_uint8), - ("weights", ctypes.POINTER(_CollationWeight_)), + ("weights", ctypes.POINTER(_CollDB_Weight_)), ] -class CollationDB(ctypes.Structure): - """struct __collation { - uint32_t version; - void *handle; - };""" - _fields_ = [ - ("version", ctypes.c_uint32), - ("handle", ctypes.c_void_p), - ] +_create_ = _dll_.colldb_create +_create_.argtypes = [ctypes.c_char_p, ctypes.c_int] +_create_.restype = ctypes.c_void_p + +_open_ = _dll_.colldb_open +_open_.argtypes = [ctypes.c_char_p] +_open_.restype = ctypes.c_void_p +_sync_ = _dll_.colldb_sync +_sync_.argtypes = [ctypes.c_void_p] +_sync_.restype = ctypes.c_int + +_close_ = _dll_.colldb_close +_close_.argtypes = [ctypes.c_void_p] +_close_.restype = ctypes.c_int -class Collation(object): - """Collation object manages collation databases using C API.""" +_get_ = _dll_.colldb_get +_get_.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(_CollDB_Key_), + ctypes.POINTER(_CollDB_Value_), +] +_get_.restype = ctypes.c_int + +_put_ = _dll_.colldb_put +_put_.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(_CollDB_Key_), + ctypes.POINTER(_CollDB_Value_), +] +_put_.restype = ctypes.c_int + + +class CollDB(object): + """CollDB object manages collation databases using C API.""" def __init__(self, path, mode): - """Collation.__init__(path, mode) -> Collation""" + """CollDB.__init__(path, mode) -> Collation""" path = path.encode("UTF-8") if mode == "w": self._db_ = _create_(path, 511) @@ -111,6 +103,8 @@ self._db_ = _open_(path) else: raise ValueError("mode must be either 'r' or 'w'") + if not self._db_: + raise OSError(os.strerror(ctypes.get_errno())) def __enter__(self): return self @@ -125,37 +119,41 @@ def get(self, key): """Get the value which corresponds to the given key.""" - ckey = _CollatonKey_() - cvalue = _CollationValue_() - weights = ([_CollationWeight_()] * 16) + ckey = _CollDB_Key_() + cvalue = _CollDB_Value_() + weights = ([_CollDB_Weight_()] * 16) ckey.count = len(key) cvalue.count = len(weights) ckey.chars = (ctypes.c_uint32 * len(key))(*key) - cvalue.weights = (_CollationWeight_ * len(weights))(*weights) - state = _get_(self._db_, ctypes.byref(ckey), ctypes.byref(cvalue)) + cvalue.weights = (_CollDB_Weight_ * len(weights))(*weights) + state = _get_( + self._db_, + ctypes.byref(ckey), + ctypes.byref(cvalue) + ) if state == -1: raise OSError(os.strerror(ctypes.get_errno())) elif state == +1: return KeyError("".join([chr(i) for i in key])) value = [] for i in range(cvalue.count): - alternate = cvalue.weights[i][0] + alt = cvalue.weights[i][0] level1 = cvalue.weights[i][1] level2 = cvalue.weights[i][2] level3 = cvalue.weights[i][3] level4 = cvalue.weights[i][4] - value.append([alternate, level1, level2, level3, level4]) + value.append([alt, level1, level2, level3, level4]) return value def put(self, key, value): """Associate the given key with the given value.""" weights = [] - ckey = _CollatonKey_() - cvalue = _CollationValue_() + ckey = _CollDB_Key_() + cvalue = _CollDB_Value_() ckey.count = len(key) cvalue.count = len(value) for i in range(len(value)): - weight = _CollationWeight_() + weight = _CollDB_Weight_() weight.alternate = value[i][0] weight.level1 = value[i][1] weight.level2 = value[i][2] @@ -163,8 +161,12 @@ weight.level4 = value[i][4] weights.append(weight) ckey.chars = (ctypes.c_uint32 * len(key))(*key) - cvalue.weights = (_CollationWeight_ * len(value))(*weights) - state = _put_(self._db_, ctypes.byref(ckey), ctypes.byref(cvalue)) + cvalue.weights = (_CollDB_Weight_ * len(value))(*weights) + state = _put_( + self._db_, + ctypes.byref(ckey), + ctypes.byref(cvalue) + ) if state != 0: raise OSError(os.strerror(ctypes.get_errno())) @@ -182,8 +184,8 @@ """, re.X) -def coll2db(src, dst): - collation = Collation(dst, "w") +def colldb(src, dst): + collation = CollDB(dst, "w") stream = codecs.open(src, "rb", "UTF-8") for line in stream: line = line.strip() @@ -209,3 +211,11 @@ collation.put(sequence, elements) collation.close() stream.close() + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.stderr.write("usage: colldb \n") + exit(-1) + colldb(sys.argv[1], sys.argv[2]) + exit(0) Modified: soc2014/ghostmansd/head/contrib/colldb/colldb.c ============================================================================== --- soc2014/ghostmansd/head/lib/libcolldb/colldb.c Tue Aug 12 01:40:11 2014 (r272251) +++ soc2014/ghostmansd/head/contrib/colldb/colldb.c Tue Aug 12 17:29:20 2014 (r272296) @@ -25,23 +25,30 @@ */ #include +#include #include #include #include "colldb.h" -colldb_t * +struct colldb { + uint32_t version; + void *handle; +}; + + +void * colldb_create(const char *path, int mode) { DBT key; DBT value; DB *db = NULL; int error = 0; + void *colldb = NULL; uint32_t version = 0; int flags = (O_RDWR | O_CREAT | O_TRUNC); - colldb_t *colldb = NULL; - colldb = calloc(1, sizeof(*colldb)); + colldb = calloc(1, sizeof(struct colldb)); if (colldb == NULL) { errno = ENOMEM; @@ -55,7 +62,7 @@ errno = error; return (NULL); } - colldb->version = __COLLATION_VERSION; + ((struct colldb*)colldb)->version = COLLDB_VERSION; key.data = "TYPE"; value.data = "COLLATION"; @@ -68,17 +75,17 @@ } key.data = "VERSION"; - version = htonl(colldb->version); + version = htonl(((struct colldb*)colldb)->version); value.data = &version; key.size = (strlen("VERSION") + 1); - value.size = sizeof(colldb->version); + value.size = sizeof(((struct colldb*)colldb)->version); if (db->put(db, &key, &value, 0) == -1) { error = errno; goto failure; } - colldb->handle = db; + ((struct colldb*)colldb)->handle = db; return (colldb); failure: @@ -89,7 +96,7 @@ } -colldb_t * +void * colldb_open(const char *path) { DBT key; @@ -97,10 +104,10 @@ DB *db = NULL; int error = 0; int state = 0; + void *colldb = NULL; int flags = O_RDONLY; - colldb_t *colldb = NULL; - colldb = calloc(1, sizeof(*colldb)); + colldb = calloc(1, sizeof(struct colldb)); if (colldb == NULL) { errno = ENOMEM; @@ -143,9 +150,10 @@ error = EFTYPE; goto failure; } - colldb->version = ntohl(*(const uint32_t*)value.data); + ((struct colldb*)colldb)->version = + ntohl(*(const uint32_t*)value.data); - colldb->handle = db; + ((struct colldb*)colldb)->handle = db; return (colldb); failure: @@ -157,7 +165,7 @@ int -colldb_close(colldb_t *colldb) +colldb_close(void *colldb) { DB *db = NULL; int error = 0; @@ -167,7 +175,7 @@ errno = EINVAL; return (-1); } - db = colldb->handle; + db = ((struct colldb*)colldb)->handle; if (db == NULL) { errno = EINVAL; @@ -186,7 +194,7 @@ int -colldb_sync(colldb_t *colldb) +colldb_sync(void *colldb) { DB *db = NULL; @@ -195,7 +203,7 @@ errno = EINVAL; return (-1); } - db = colldb->handle; + db = ((struct colldb*)colldb)->handle; if (db == NULL) { errno = EINVAL; @@ -206,9 +214,9 @@ int -colldb_get(colldb_t *colldb, - struct colldb_key *key, - struct colldb_value *value) +colldb_get(void *colldb, + struct colldb_key *key, + struct colldb_value *value) { DBT dbkey; DBT dbvalue; @@ -223,7 +231,7 @@ errno = EINVAL; return (-1); } - db = colldb->handle; + db = ((struct colldb*)colldb)->handle; if ((db == NULL) || (key->chars == NULL) || (key->count == 0)) { errno = EINVAL; @@ -273,9 +281,9 @@ int -colldb_put(colldb_t *colldb, - struct colldb_key *key, - struct colldb_value *value) +colldb_put(void *colldb, + struct colldb_key *key, + struct colldb_value *value) { DBT dbkey; DBT dbvalue; @@ -291,7 +299,7 @@ errno = EINVAL; return (-1); } - db = colldb->handle; + db = ((struct colldb*)colldb)->handle; if ((db == NULL) || (key->chars == NULL) || (key->count == 0)) { errno = EINVAL; Modified: soc2014/ghostmansd/head/contrib/colldb/colldb.h ============================================================================== --- soc2014/ghostmansd/head/lib/libcolldb/colldb.h Tue Aug 12 01:40:11 2014 (r272251) +++ soc2014/ghostmansd/head/contrib/colldb/colldb.h Tue Aug 12 17:29:20 2014 (r272296) @@ -49,6 +49,14 @@ * errno to ERANGE to indicate that struct colldb_value must allocate * a larger weights buffer. */ + +#include +#include +#include +#include +#include + + #define COLLDB_VERSION 0x00000001 #define COLLDB_WEIGHTS_MAX 10 @@ -70,17 +78,15 @@ struct colldb_weight *weights; }; -typedef struct colldb colldb_t; - -colldb_t* colldb_create(const char*, int mode); -colldb_t* colldb_open(const char*); -int colldb_close(colldb_t*); -int colldb_sync(colldb_t*); -int colldb_get(colldb_t*, +void* colldb_create(const char*, int mode); +void* colldb_open(const char*); +int colldb_close(void*); +int colldb_sync(void*); +int colldb_get(void*, struct colldb_key*, struct colldb_value*); -int colldb_put(colldb_t*, +int colldb_put(void*, struct colldb_key*, struct colldb_value*); Modified: soc2014/ghostmansd/head/lib/Makefile ============================================================================== --- soc2014/ghostmansd/head/lib/Makefile Tue Aug 12 16:51:37 2014 (r272295) +++ soc2014/ghostmansd/head/lib/Makefile Tue Aug 12 17:29:20 2014 (r272296) @@ -114,6 +114,10 @@ ${_cuse} \ ${_tests} +.ifdef UNICODE +SUBDIR+= libcolldb +.endif + # Inter-library dependencies. When the makefile for a library contains LDADD # libraries, those libraries should be listed as build order dependencies here. Modified: soc2014/ghostmansd/head/lib/libc/Makefile ============================================================================== --- soc2014/ghostmansd/head/lib/libc/Makefile Tue Aug 12 16:51:37 2014 (r272295) +++ soc2014/ghostmansd/head/lib/libc/Makefile Tue Aug 12 17:29:20 2014 (r272296) @@ -94,6 +94,10 @@ .include "${LIBC_SRCTOP}/stdlib/jemalloc/Makefile.inc" .include "${LIBC_SRCTOP}/stdtime/Makefile.inc" .include "${LIBC_SRCTOP}/string/Makefile.inc" +.ifdef UNICODE +CFLAGS+= -D_UNICODE_SOURCE +.include "${LIBC_SRCTOP}/unicode/Makefile.inc" +.endif .include "${LIBC_SRCTOP}/sys/Makefile.inc" .include "${LIBC_SRCTOP}/rpc/Makefile.inc" .include "${LIBC_SRCTOP}/uuid/Makefile.inc" Modified: soc2014/ghostmansd/head/lib/libc/locale/collate.c ============================================================================== --- soc2014/ghostmansd/head/lib/libc/locale/collate.c Tue Aug 12 16:51:37 2014 (r272295) +++ soc2014/ghostmansd/head/lib/libc/locale/collate.c Tue Aug 12 17:29:20 2014 (r272296) @@ -51,18 +51,50 @@ #ifdef _UNICODE_SOURCE -__colldb_t * +#include +#include +#include +#include +#include + + +struct __colldb { + uint32_t version; + void *handle; +}; + + +static void *handle = NULL; +static pthread_once_t once = PTHREAD_ONCE_INIT; + + +static void +__colldb_root_init(void) +{ + handle = __colldb_open("/usr/share/colldb/root.db"); +} + + +void * +__colldb_root_handle(void) +{ + pthread_once(&once, &__colldb_root_init); + return (handle); +} + + +void * __colldb_create(const char *path, int mode) { DBT key; DBT value; DB *db = NULL; int error = 0; + void *colldb = NULL; uint32_t version = 0; int flags = (O_RDWR | O_CREAT | O_TRUNC); - __colldb_t *colldb = NULL; - colldb = calloc(1, sizeof(*colldb)); + colldb = calloc(1, sizeof(struct __colldb)); if (colldb == NULL) { errno = ENOMEM; @@ -76,7 +108,7 @@ errno = error; return (NULL); } - colldb->version = __COLLATION_VERSION; + ((struct __colldb*)colldb)->version = __COLLDB_VERSION; key.data = "TYPE"; value.data = "COLLATION"; @@ -89,17 +121,17 @@ } key.data = "VERSION"; - version = htonl(colldb->version); + version = htonl(((struct __colldb*)colldb)->version); value.data = &version; key.size = (strlen("VERSION") + 1); - value.size = sizeof(colldb->version); + value.size = sizeof(((struct __colldb*)colldb)->version); if (db->put(db, &key, &value, 0) == -1) { error = errno; goto failure; } - colldb->handle = db; + ((struct __colldb*)colldb)->handle = db; return (colldb); failure: @@ -110,7 +142,7 @@ } -__colldb_t * +void * __colldb_open(const char *path) { DBT key; @@ -118,10 +150,10 @@ DB *db = NULL; int error = 0; int state = 0; + void *colldb = NULL; int flags = O_RDONLY; - __colldb_t *colldb = NULL; - colldb = calloc(1, sizeof(*colldb)); + colldb = calloc(1, sizeof(struct __colldb)); if (colldb == NULL) { errno = ENOMEM; @@ -164,9 +196,10 @@ error = EFTYPE; goto failure; } - colldb->version = ntohl(*(const uint32_t*)value.data); + ((struct __colldb*)colldb)->version = + ntohl(*(const uint32_t*)value.data); - colldb->handle = db; + ((struct __colldb*)colldb)->handle = db; return (colldb); failure: @@ -178,7 +211,7 @@ int -__colldb_close(__colldb_t *colldb) +__colldb_close(void *colldb) { DB *db = NULL; int error = 0; @@ -188,7 +221,7 @@ errno = EINVAL; return (-1); } - db = colldb->handle; + db = ((struct __colldb*)colldb)->handle; if (db == NULL) { errno = EINVAL; @@ -207,7 +240,7 @@ int -__colldb_sync(__colldb_t *colldb) +__colldb_sync(void *colldb) { DB *db = NULL; @@ -216,7 +249,7 @@ errno = EINVAL; return (-1); } - db = colldb->handle; + db = ((struct __colldb*)colldb)->handle; if (db == NULL) { errno = EINVAL; @@ -227,14 +260,15 @@ int -__colldb_get(__colldb_t *colldb, - struct __colldb_key *key, - struct __colldb_value *value) +__colldb_get(void *colldb, + struct __colldb_key *key, + struct __colldb_value *value) { DBT dbkey; DBT dbvalue; DB *db = NULL; size_t i = 0; + int error = 0; int state = 0; uint32_t *keybuf = NULL; struct __colldb_weight *weights = NULL; @@ -244,37 +278,37 @@ errno = EINVAL; return (-1); } - db = colldb->handle; + db = ((struct __colldb*)colldb)->handle; if ((db == NULL) || (key->chars == NULL) || (key->count == 0)) { errno = EINVAL; return (-1); } - keybuf = key->chars; - if (__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__) + keybuf = malloc(key->count * sizeof(*key->chars)); + if (keybuf == NULL) { - keybuf = malloc(key->count * sizeof(*key->chars)); - if (keybuf == NULL) - { - errno = ENOMEM; - return (-1); - } - for (i = 0; i < key->count; ++i) - keybuf[i] = htonl(key->chars[i]); + errno = ENOMEM; + return (-1); } + for (i = 0; i < key->count; ++i) + keybuf[i] = htonl(key->chars[i]); dbkey.data = keybuf; dbkey.size = (key->count * sizeof(*key->chars)); state = db->get(db, &dbkey, &dbvalue, 0); if (state != 0) + { + error = errno; + free(keybuf); + errno = error; return (state); + } weights = dbvalue.data; if ((dbvalue.size / sizeof(*weights)) > value->count) { - if (__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__) - free(keybuf); + free(keybuf); errno = ERANGE; return (-1); } @@ -287,16 +321,15 @@ value->weights[i].level3 = ntohl(weights[i].level3); value->weights[i].level4 = ntohl(weights[i].level4); } - if (__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__) - free(keybuf); + free(keybuf); return (0); } int -__colldb_put(__colldb_t *colldb, - struct __colldb_key *key, - struct __colldb_value *value) +__colldb_put(void *colldb, + struct __colldb_key *key, + struct __colldb_value *value) { DBT dbkey; DBT dbvalue; @@ -312,34 +345,30 @@ errno = EINVAL; return (-1); } - db = colldb->handle; + db = ((struct __colldb*)colldb)->handle; if ((db == NULL) || (key->chars == NULL) || (key->count == 0)) { errno = EINVAL; return (-1); } - keybuf = key->chars; valuebuf = value->weights; - if (__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__) + keybuf = malloc(key->count * sizeof(*key->chars)); + valuebuf = malloc(value->count * sizeof(*value->weights)); + if ((keybuf == NULL) || (valuebuf == NULL)) { - keybuf = malloc(key->count * sizeof(*key->chars)); - valuebuf = malloc(value->count * sizeof(*value->weights)); - if ((keybuf == NULL) || (valuebuf == NULL)) - { - errno = ENOMEM; - return (-1); - } - for (i = 0; i < key->count; ++i) - keybuf[i] = htonl(key->chars[i]); - for (i = 0; i < value->count; ++i) - { - valuebuf[i].alternate = value->weights[i].alternate; - valuebuf[i].level1 = htonl(value->weights[i].level1); - valuebuf[i].level2 = htonl(value->weights[i].level2); - valuebuf[i].level3 = htonl(value->weights[i].level3); - valuebuf[i].level4 = htonl(value->weights[i].level4); - } + errno = ENOMEM; + return (-1); + } + for (i = 0; i < key->count; ++i) + keybuf[i] = htonl(key->chars[i]); + for (i = 0; i < value->count; ++i) + { + valuebuf[i].alternate = value->weights[i].alternate; + valuebuf[i].level1 = htonl(value->weights[i].level1); + valuebuf[i].level2 = htonl(value->weights[i].level2); + valuebuf[i].level3 = htonl(value->weights[i].level3); + valuebuf[i].level4 = htonl(value->weights[i].level4); } dbkey.data = keybuf; @@ -347,7 +376,7 @@ dbkey.size = (key->count * sizeof(*key->chars)); dbvalue.size = (value->count * sizeof(*value->weights)); state = db->put(db, &dbkey, &dbvalue, 0); - if (__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__) + if (state != 0) { error = errno; free(keybuf); @@ -504,7 +533,7 @@ return (_LDP_ERROR); } -#define FREAD(a, b, c, d) \ +#define COLL_FREAD(a, b, c, d) \ { \ if (fread(a, b, c, d) != c) { \ saverr = errno; \ @@ -517,9 +546,9 @@ } \ } - FREAD(TMP_substitute_table, sizeof(__collate_substitute_table), 1, fp); - FREAD(TMP_char_pri_table, sizeof(__collate_char_pri_table), 1, fp); - FREAD(TMP_chain_pri_table, + COLL_FREAD(TMP_substitute_table, sizeof(__collate_substitute_table), 1, fp); + COLL_FREAD(TMP_char_pri_table, sizeof(__collate_char_pri_table), 1, fp); + COLL_FREAD(TMP_chain_pri_table, sizeof(*__collate_chain_pri_table), chains, fp); (void)fclose(fp); Modified: soc2014/ghostmansd/head/lib/libc/locale/xlocale.c ============================================================================== --- soc2014/ghostmansd/head/lib/libc/locale/xlocale.c Tue Aug 12 16:51:37 2014 (r272295) +++ soc2014/ghostmansd/head/lib/libc/locale/xlocale.c Tue Aug 12 17:29:20 2014 (r272296) @@ -81,6 +81,9 @@ &__xlocale_global_time, &__xlocale_global_messages }, +#ifdef _UNICODE_SOURCE + NULL, +#endif 1, 0, 1, @@ -94,6 +97,9 @@ &__xlocale_C_ctype, 0, 0, 0, 0 }, +#ifdef _UNICODE_SOURCE + NULL, +#endif 1, 0, 1, Modified: soc2014/ghostmansd/head/lib/libc/locale/xlocale_private.h ============================================================================== --- soc2014/ghostmansd/head/lib/libc/locale/xlocale_private.h Tue Aug 12 16:51:37 2014 (r272295) +++ soc2014/ghostmansd/head/lib/libc/locale/xlocale_private.h Tue Aug 12 17:29:20 2014 (r272296) @@ -56,8 +56,8 @@ #define __UC_NFKC 4 size_t __ucsnorm(uint32_t*, const uint32_t*, size_t, int); -#define COLLDB_VERSION 0x00000001 -#define COLLDB_WEIGHTS_MAX 10 +#define __COLLDB_VERSION 0x00000001 +#define __COLLDB_WEIGHTS_MAX 10 struct __colldb_weight { uint8_t alternate; uint32_t level1; @@ -67,28 +67,28 @@ }; struct __colldb_key { size_t count; - uint32_t *chars; + const uint32_t *chars; }; struct __colldb_value { size_t count; struct __colldb_weight *weights; }; -typedef struct __colldb __colldb_t; -#define __colldb_root (__colldb_root_handle()) +void *__colldb_root_handle(void); +#define __colldb_root (__colldb_root_handle()) -__colldb_t* __colldb_create(const char*, int mode); -__colldb_t* __colldb_open(const char*); -int __colldb_close(__colldb_t*); -int __colldb_sync(__colldb_t*); -int __colldb_get(__colldb_t*, +void* __colldb_create(const char*, int mode); +void* __colldb_open(const char*); +int __colldb_close(void*); +int __colldb_sync(void*); +int __colldb_get(void*, struct __colldb_key*, struct __colldb_value*); -int __colldb_put(__colldb_t*, +int __colldb_put(void*, struct __colldb_key*, struct __colldb_value*); -size_t __ucsxfrm(uint32_t*, const uint32_t*, size_t, const __colldb_t*); -int __ucscoll(const uint32_t*, const uint32_t*, const __colldb_t*); +size_t __ucsxfrm(uint32_t*, const uint32_t*, size_t, void*); +int __ucscoll(const uint32_t*, const uint32_t*, void*); #endif @@ -138,9 +138,8 @@ /** Components for the locale. */ struct xlocale_component *components[XLC_LAST]; #ifdef _UNICODE_SOURCE - /** Collation table (NULL means DUCET). */ - size_t collsize; - struct __collation *colltable; + /** Collation Database handle. */ + void *colldb; #endif /** Flag indicating if components[XLC_MONETARY] has changed since the * last call to localeconv_l() with this locale. */ Added: soc2014/ghostmansd/head/lib/libc/unicode/Makefile.inc ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ soc2014/ghostmansd/head/lib/libc/unicode/Makefile.inc Tue Aug 12 17:29:20 2014 (r272296) @@ -0,0 +1,6 @@ +# from @(#)Makefile.inc 8.1 (Berkeley) 6/4/93 + +# unicode sources +.PATH: ${LIBC_SRCTOP}/${LIBC_ARCH}/unicode ${LIBC_SRCTOP}/unicode + +SRCS+= uccclass.c ucscanon.c ucscoll.c ucsnorm.c ucsxfrm.c Modified: soc2014/ghostmansd/head/lib/libc/unicode/coll.h ============================================================================== --- soc2014/ghostmansd/head/lib/libc/unicode/coll.h Tue Aug 12 16:51:37 2014 (r272295) +++ soc2014/ghostmansd/head/lib/libc/unicode/coll.h Tue Aug 12 17:29:20 2014 (r272296) @@ -26,28 +26,11 @@ #include #include "xlocale_private.h" -static __colldb_t *handle = NULL; -static pthread_once_t once = PTHREAD_ONCE_INIT; - - -static __colldb_t -__colldb_root_init(void) -{ - handle = __colldb_open("/usr/share/colldb/root.db"); -} - - -__colldb_t * -__colldb_root_handle(void) -{ - pthread_once(&once, &__colldb_root_init); - return handle; -} static size_t -__coll_iter(const uint32_t *iter, __colldb_t *colldb, struct __colldb_value *val, - struct __colldb_weight *default_weights[2]) +__coll_iter(const uint32_t *iter, void *colldb, struct __colldb_value *val, + struct __colldb_weight default_weights[2]) { int state = 0; size_t shift = 0; @@ -58,7 +41,7 @@ if (*(iter + shift - 1) == 0) break; key.count = shift; - key.chars = norm; + key.chars = iter; state = __colldb_get(colldb, &key, val); if (state == -1) return (0); @@ -73,7 +56,7 @@ if (*(iter + shift - 1) == 0) break; key.count = shift; - key.chars = norm; + key.chars = iter; state = __colldb_get(__colldb_root, &key, val); if (state == -1) return (0); @@ -85,7 +68,7 @@ if (state != 0) { shift = 1; - (*default_weights)[0].level1 = 0xFBC0; + default_weights[0].level1 = 0xFBC0; if (((0x4E00 <= *iter) && (*iter <= 0x9FCC)) || (*iter == 0xFA0E) || (*iter == 0xFA0F) || (*iter == 0xFA11) || (*iter == 0xFA13) || @@ -93,15 +76,15 @@ (*iter == 0xFA21) || (*iter == 0xFA23) || (*iter == 0xFA24) || (*iter == 0xFA27) || (*iter == 0xFA28) || (*iter == 0xFA29)) - (*default_weights)[0].level1 = 0xFB40; + default_weights[0].level1 = 0xFB40; else if (((0x3400 <= *iter) && (*iter <= 0x4DB5)) || ((0x20000 <= *iter) && (*iter <= 0x2A6D6)) || ((0x2A700 <= *iter) && (*iter <= 0x2B734)) || ((0x2B740 <= *iter) && (*iter <= 0x2B81D))) - (*default_weights)[0].level1 = 0xFB80; - (*default_weights)[0].level1 = ((*default_weights)[0].level1 + (*iter >> 15)); - (*default_weights)[1].level1 = ((*iter & 0x7FFF) | 0x8000); - val->weights = *default_weights; + default_weights[0].level1 = 0xFB80; + default_weights[0].level1 = (default_weights[0].level1 + (*iter >> 15)); + default_weights[1].level1 = ((*iter & 0x7FFF) | 0x8000); + val->weights = default_weights; val->count = 2; } *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***