Date: Wed, 5 May 2021 07:24:11 GMT From: Baptiste Daroussin <bapt@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org Subject: git: c57a440af2da - stable/12 - libucl: vendor import snapshort 20210314 Message-ID: <202105050724.1457OB7C048539@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch stable/12 has been updated by bapt: URL: https://cgit.FreeBSD.org/src/commit/?id=c57a440af2daf752a99d4e3d7791226734090f06 commit c57a440af2daf752a99d4e3d7791226734090f06 Author: Baptiste Daroussin <bapt@FreeBSD.org> AuthorDate: 2021-03-22 14:13:02 +0000 Commit: Baptiste Daroussin <bapt@FreeBSD.org> CommitDate: 2021-05-05 07:18:53 +0000 libucl: vendor import snapshort 20210314 (cherry picked from commit a0409676120c1e558d0ade943019934e0f15118d) --- contrib/libucl/CMakeLists.txt | 105 ++- contrib/libucl/ChangeLog.md | 38 +- contrib/libucl/README.md | 64 +- contrib/libucl/configure.ac | 8 +- contrib/libucl/doc/api.md | 17 +- contrib/libucl/doc/libucl.3 | 26 +- contrib/libucl/doc/lua_api.md | 4 +- contrib/libucl/include/lua_ucl.h | 20 +- contrib/libucl/include/ucl++.h | 191 ++++- contrib/libucl/include/ucl.h | 191 ++++- contrib/libucl/klib/kvec.h | 78 ++- contrib/libucl/lua/lua_ucl.c | 450 ++++++++++-- contrib/libucl/python/MANIFEST.in | 5 + contrib/libucl/python/setup.py | 42 +- contrib/libucl/python/src/uclmodule.c | 3 +- contrib/libucl/python/tests/test_example.py | 59 ++ contrib/libucl/python/tests/test_load.py | 17 +- contrib/libucl/src/mum.h | 8 +- contrib/libucl/src/ucl_chartable.h | 4 +- contrib/libucl/src/ucl_emitter.c | 12 +- contrib/libucl/src/ucl_emitter_utils.c | 57 +- contrib/libucl/src/ucl_hash.c | 218 +++++- contrib/libucl/src/ucl_hash.h | 20 +- contrib/libucl/src/ucl_internal.h | 105 ++- contrib/libucl/src/ucl_msgpack.c | 82 ++- contrib/libucl/src/ucl_parser.c | 552 ++++++++++++--- contrib/libucl/src/ucl_schema.c | 29 +- contrib/libucl/src/ucl_util.c | 780 ++++++++++++++++----- contrib/libucl/tests/basic.test | 2 +- contrib/libucl/tests/basic/13.in | 2 +- contrib/libucl/tests/basic/20.in | 2 - contrib/libucl/tests/basic/20.res | 5 - contrib/libucl/tests/basic/21.in | 2 - contrib/libucl/tests/basic/21.res | 10 - contrib/libucl/tests/basic/9.in | 2 +- contrib/libucl/tests/basic/9.res | 8 +- contrib/libucl/tests/basic/squote.in | 8 + contrib/libucl/tests/basic/squote.res | 7 + .../libucl/tests/fuzzers/ucl_add_string_fuzzer.c | 25 + contrib/libucl/tests/fuzzers/ucl_msgpack_fuzzer.c | 29 + contrib/libucl/tests/generate.test | 2 +- contrib/libucl/tests/run_tests.sh | 4 +- contrib/libucl/tests/streamline.test | 2 +- contrib/libucl/tests/test_basic.c | 11 +- contrib/libucl/tests/test_generate.c | 15 +- contrib/libucl/tests/test_msgpack.c | 1 + contrib/libucl/utils/CMakeLists.txt | 12 + contrib/libucl/utils/objdump.c | 17 +- contrib/libucl/utils/ucl-tool.c | 100 +-- 49 files changed, 2820 insertions(+), 631 deletions(-) diff --git a/contrib/libucl/CMakeLists.txt b/contrib/libucl/CMakeLists.txt index 7b55faf8243f..5fe772a30f88 100644 --- a/contrib/libucl/CMakeLists.txt +++ b/contrib/libucl/CMakeLists.txt @@ -9,13 +9,16 @@ SET(LIBUCL_VERSION "${LIBUCL_VERSION_MAJOR}.${LIBUCL_VERSION_MINOR}.${LIBUCL_VERSION_PATCH}") INCLUDE(CheckCCompilerFlag) +INCLUDE(CheckCSourceCompiles) INCLUDE(FindOpenSSL) +INCLUDE(GNUInstallDirs) OPTION(ENABLE_URL_INCLUDE "Enable urls in ucl includes (requires libcurl or libfetch) [default: OFF]" OFF) OPTION(ENABLE_URL_SIGN "Enable signatures check in ucl includes (requires openssl) [default: OFF]" OFF) OPTION(BUILD_SHARED_LIBS "Build Shared Libraries [default: OFF]" OFF) OPTION(ENABLE_LUA "Enable lua support [default: OFF]" OFF) OPTION(ENABLE_LUAJIT "Enable luajit support [default: OFF]" OFF) +OPTION(ENABLE_UTILS "Enable building utility binaries [default: OFF]" OFF) # Find lua installation MACRO(FindLua) @@ -150,40 +153,47 @@ IF(ENABLE_URL_INCLUDE MATCHES "ON") DOC "Path to libfetch header") ELSE(LIBFETCH_LIBRARY) # Try to find libcurl - ProcessPackage(CURL libcurl) + FIND_PACKAGE(CURL) IF(NOT CURL_FOUND) MESSAGE(WARNING "Neither libcurl nor libfetch were found, no support of URL includes in configuration") ENDIF(NOT CURL_FOUND) ENDIF(LIBFETCH_LIBRARY) ENDIF(ENABLE_URL_INCLUDE MATCHES "ON") +set(SYNC_BUILTINS_TEST_SOURCE [====[ +int main() +{ + unsigned long val; + + __sync_bool_compare_and_swap(&val, 0, 1); + __sync_add_and_fetch(&val, 1); + __sync_fetch_and_add(&val, 0); + __sync_sub_and_fetch(&val, 1); + + return 0; +} +]====]) + +CHECK_C_SOURCE_COMPILES("${SYNC_BUILTINS_TEST_SOURCE}" HAVE_ATOMIC_BUILTINS) +IF(NOT HAVE_ATOMIC_BUILTINS) + MESSAGE(WARNING "Libucl references could be thread-unsafe because atomic builtins are missing") +ENDIF(NOT HAVE_ATOMIC_BUILTINS) + SET(CMAKE_C_WARN_FLAGS "") -CHECK_C_COMPILER_FLAG(-Wall SUPPORT_WALL) CHECK_C_COMPILER_FLAG(-W SUPPORT_W) -CHECK_C_COMPILER_FLAG(-Wno-unused-parameter SUPPORT_WPARAM) CHECK_C_COMPILER_FLAG(-Wno-pointer-sign SUPPORT_WPOINTER_SIGN) -CHECK_C_COMPILER_FLAG(-Wstrict-prototypes SUPPORT_WSTRICT_PROTOTYPES) -IF(NOT "${CMAKE_C_COMPILER_ID}" MATCHES SunPro) - CHECK_C_COMPILER_FLAG("-std=c99" SUPPORT_STD_FLAG) -ENDIF(NOT "${CMAKE_C_COMPILER_ID}" MATCHES SunPro) +CHECK_C_COMPILER_FLAG(-Wno-unused-parameter SUPPORT_WUNUSED_PARAMETER) IF(SUPPORT_W) - SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -W") + SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -W") ENDIF(SUPPORT_W) -IF(SUPPORT_WALL) - SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wall") -ENDIF(SUPPORT_WALL) -IF(SUPPORT_WPARAM) - SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-unused-parameter") -ENDIF(SUPPORT_WPARAM) IF(SUPPORT_WPOINTER_SIGN) SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-pointer-sign") ENDIF(SUPPORT_WPOINTER_SIGN) -IF(SUPPORT_WSTRICT_PROTOTYPES) - SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wstrict-prototypes") -ENDIF(SUPPORT_WSTRICT_PROTOTYPES) -IF(SUPPORT_STD_FLAG) - SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -std=c99") -ENDIF(SUPPORT_STD_FLAG) +IF(SUPPORT_WUNUSED_PARAMETER) + SET(CMAKE_C_WARN_FLAGS "${CMAKE_C_WARN_FLAGS} -Wno-unused-parameter") +ENDIF(SUPPORT_WUNUSED_PARAMETER) + +SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_WARN_FLAGS}" ) IF(ENABLE_URL_SIGN MATCHES "ON") IF(OPENSSL_FOUND) @@ -192,10 +202,19 @@ IF(ENABLE_URL_SIGN MATCHES "ON") ENDIF(OPENSSL_FOUND) ENDIF(ENABLE_URL_SIGN MATCHES "ON") -INCLUDE_DIRECTORIES("src") -INCLUDE_DIRECTORIES("include") -INCLUDE_DIRECTORIES("uthash") -INCLUDE_DIRECTORIES("klib") +SET(UCL_COMPILE_DEFS) +IF(HAVE_FETCH_H) + LIST(APPEND UCL_COMPILE_DEFS -DHAVE_FETCH_H=1) +ENDIF(HAVE_FETCH_H) +IF(CURL_FOUND) + LIST(APPEND UCL_COMPILE_DEFS -DCURL_FOUND=1) +ENDIF(CURL_FOUND) +IF(HAVE_OPENSSL) + LIST(APPEND UCL_COMPILE_DEFS -DHAVE_OPENSSL=1) +ENDIF(HAVE_OPENSSL) +IF(HAVE_ATOMIC_BUILTINS) + LIST(APPEND UCL_COMPILE_DEFS -DHAVE_ATOMIC_BUILTINS=1) +ENDIF(HAVE_ATOMIC_BUILTINS) SET(UCLSRC src/ucl_util.c src/ucl_parser.c @@ -207,13 +226,27 @@ SET(UCLSRC src/ucl_util.c src/ucl_msgpack.c src/ucl_sexp.c) +SET(UCLHDR include/ucl.h + include/ucl++.h) SET (LIB_TYPE STATIC) IF (BUILD_SHARED_LIBS) SET (LIB_TYPE SHARED) ENDIF (BUILD_SHARED_LIBS) ADD_LIBRARY(ucl ${LIB_TYPE} ${UCLSRC}) +ADD_LIBRARY(ucl::ucl ALIAS ucl) SET_TARGET_PROPERTIES(ucl PROPERTIES VERSION ${LIBUCL_VERSION} SOVERSION ${LIBUCL_VERSION_MAJOR}) +TARGET_INCLUDE_DIRECTORIES(ucl + PUBLIC + include + PRIVATE + src + uthash + klib) +TARGET_COMPILE_DEFINITIONS(ucl + PRIVATE + ${UCL_COMPILE_DEFS} +) IF(ENABLE_LUA MATCHES "ON") IF(ENABLE_LUAJIT MATCHES "ON") @@ -236,13 +269,20 @@ IF(ENABLE_LUA MATCHES "ON") ENDIF(ENABLE_LUAJIT MATCHES "ON") SET(UCL_LUA_SRC lua/lua_ucl.c) ADD_LIBRARY(lua-ucl ${LIB_TYPE} ${UCL_LUA_SRC}) + ADD_LIBRARY(ucl::lua ALIAS lua-ucl) IF(ENABLE_LUAJIT MATCHES "ON") TARGET_LINK_LIBRARIES(lua-ucl "${LUAJIT_LIBRARY}") ELSE(ENABLE_LUAJIT MATCHES "ON") TARGET_LINK_LIBRARIES(lua-ucl "${LUA_LIBRARY}") ENDIF(ENABLE_LUAJIT MATCHES "ON") TARGET_LINK_LIBRARIES(lua-ucl ucl) - SET_TARGET_PROPERTIES(lua-ucl PROPERTIES VERSION ${LIBUCL_VERSION} SOVERSION ${LIBUCL_VERSION_MAJOR}) + TARGET_INCLUDE_DIRECTORIES(lua-ucl PUBLIC include PRIVATE src uthash) + SET_TARGET_PROPERTIES(lua-ucl PROPERTIES + VERSION ${LIBUCL_VERSION} + SOVERSION ${LIBUCL_VERSION_MAJOR} + PUBLIC_HEADER include/lua_ucl.h) + INSTALL(TARGETS lua-ucl DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) ENDIF() IF(HAVE_FETCH_H) @@ -257,3 +297,18 @@ IF(ENABLE_URL_SIGN MATCHES "ON") TARGET_LINK_LIBRARIES(ucl ${OPENSSL_LIBRARIES}) ENDIF(OPENSSL_FOUND) ENDIF(ENABLE_URL_SIGN MATCHES "ON") + +IF(UNIX) + TARGET_LINK_LIBRARIES(ucl -lm) +ENDIF(UNIX) + +SET_TARGET_PROPERTIES(ucl PROPERTIES + PUBLIC_HEADER "${UCLHDR}") + +INSTALL(TARGETS ucl DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +IF(ENABLE_UTILS MATCHES "ON") + ADD_SUBDIRECTORY(utils) +ENDIF() + diff --git a/contrib/libucl/ChangeLog.md b/contrib/libucl/ChangeLog.md index e4c1263bccb7..cba29aa9a7b5 100644 --- a/contrib/libucl/ChangeLog.md +++ b/contrib/libucl/ChangeLog.md @@ -64,4 +64,40 @@ **Incompatible changes**: -- `ucl_object_emit_full` now accepts additional argument `comments` that could be used to emit comments with UCL output \ No newline at end of file +- `ucl_object_emit_full` now accepts additional argument `comments` that could be used to emit comments with UCL output + +### Libucl 0.8.1 + +- Create ucl_parser_add_file_full() to be able to specify merge mode and parser type (by Allan Jude) +- C++ wrapper improvements (by @ftilde) +- C++ wrapper: add convenience method at() and lookup() (by Yonghee Kim) +- C++ wrapper: add assignment operator to Ucl class (by Yonghee Kim) +- C++ wrapper: support variables in parser (by Yonghee Kim) +- C++ wrapper: refactoring C++ interface (by Yonghee Kim): + - use auto variables (if possible) + - remove dangling expressions + - use std::set::emplace instead of std::set::insert + - not use std::move in return statement; considering copy elision +- C++ wrapper: fix compilation error and warnings (by Zhe Wang) +- C++ wrapper: fix iteration over objects in which the first value is `false` (by Zhe Wang) +- C++ wrapper: Macro helper functions (by Chris Meacham) +- C++ wrapper: Changing the duplicate strategy in the C++ API (by Chris Meacham) +- C++ wrapper: Added access functions for the size of a UCL_ARRAY (by Chris Meacham) +- Fix caseless comparison +- Fix include when EPERM is issued +- Fix Windows build +- Allow to reserve space in arrays and hashes +- Fix bug with including of empty files +- Move to mum_hash from xxhash +- Fix msgpack on non-x86 +- python: Add support to Python 3 (by Denis Volpato Martins) +- python: Add support for Python 2.6 tests (by Denis Volpato Martins) +- python: Implement validation function and tests (by Denis Volpato Martins) +- python: Added UCL_NULL handling and tests (by Denis Volpato Martins) +- Fix schema validation for patternProperties with object data (by Denis Volpato Martins) +- Remove the dependency on NBBY, add missing <strings.h> include (by Ed Schouten) +- Allow to emit msgpack from Lua +- Performance improvements in Lua API +- Allow to pass opaque objects in Lua API for transparent C passthrough +- Various bugs fixed +- Couple of memory leaks plugged \ No newline at end of file diff --git a/contrib/libucl/README.md b/contrib/libucl/README.md index 44983c57d643..53d8a651d73b 100644 --- a/contrib/libucl/README.md +++ b/contrib/libucl/README.md @@ -1,6 +1,6 @@ # LIBUCL -[](https://travis-ci.org/vstakhov/libucl) +[](https://circleci.com/gh/vstakhov/libucl) [](https://scan.coverity.com/projects/4138) [](https://coveralls.io/github/vstakhov/libucl?branch=master) @@ -18,6 +18,7 @@ - [Macros support](#macros-support) - [Variables support](#variables-support) - [Multiline strings](#multiline-strings) + - [Single quoted strings](#single-quoted-strings) - [Emitter](#emitter) - [Validation](#validation) - [Performance](#performance) @@ -65,19 +66,25 @@ section { ```json { "param": "value", - "param1": "value1", - "flag": true, - "subsection": { - "host": [ - { - "host": "hostname", - "port": 900 - }, - { - "host": "hostname", - "port": 901 + "section": { + "param": "value", + "param1": "value1", + "flag": true, + "number": 10000, + "time": "0.2s", + "string": "something", + "subsection": { + "host": [ + { + "host": "hostname", + "port": 900 + }, + { + "host": "hostname", + "port": 901 + } + ] } - ] } } ``` @@ -288,7 +295,22 @@ as following: By default, the priority of top-level object is set to zero (lowest priority). Currently, you can define up to 16 priorities (from 0 to 15). Includes with bigger priorities will -rewrite keys from the objects with lower priorities as specified by the policy. +rewrite keys from the objects with lower priorities as specified by the policy. The priority +of the top-level or any other object can be changed with the `.priority` macro, which has no +options and takes the new priority: + +``` +# Default priority: 0. +foo = 6 +.priority 5 +# The following will have priority 5. +bar = 6 +baz = 7 +# The following will be included with a priority of 3, 5, and 6 respectively. +.include(priority=3) "path.conf" +.include(priority=5) "equivalent-path.conf" +.include(priority=6) "highpriority-path.conf" +``` ### Variables support @@ -333,9 +355,21 @@ text EOD ``` +### Single quoted strings + +It is possible to use single quoted strings to simplify escaping rules. All values passed in single quoted strings are *NOT* escaped, with two exceptions: a single `'` character just before `\` character, and a newline character just after `\` character that is ignored. + +``` +key = 'value'; # Read as value +key = 'value\n\'; # Read as value\n\ +key = 'value\''; # Read as value' +key = 'value\ +bla'; # Read as valuebla +``` + ## Emitter -Each UCL object can be serialized to one of the three supported formats: +Each UCL object can be serialized to one of the four supported formats: * `JSON` - canonic json notation (with spaces indented structure); * `Compacted JSON` - compact json notation (without spaces or newlines); diff --git a/contrib/libucl/configure.ac b/contrib/libucl/configure.ac index 6457268854ac..731b7113e689 100644 --- a/contrib/libucl/configure.ac +++ b/contrib/libucl/configure.ac @@ -1,7 +1,7 @@ m4_define([maj_ver], [0]) m4_define([med_ver], [8]) -m4_define([min_ver], [0]) -m4_define([so_version], [6:0:0]) +m4_define([min_ver], [1]) +m4_define([so_version], [6:0:1]) m4_define([ucl_version], [maj_ver.med_ver.min_ver]) AC_INIT([libucl],[ucl_version],[https://github.com/vstakhov/libucl],[libucl]) @@ -73,11 +73,11 @@ AC_ARG_ENABLE([utils], AM_CONDITIONAL([UTILS], [test x$utils = xtrue]) AS_IF([test "x$enable_signatures" = "xyes"], [ - AC_SEARCH_LIBS([EVP_MD_CTX_create], [crypto], [ + AC_SEARCH_LIBS([CRYPTO_new_ex_data], [crypto], [ AC_DEFINE(HAVE_OPENSSL, 1, [Define to 1 if you have the 'crypto' library (-lcrypto).]) LIBCRYPTO_LIB="-lcrypto" LIBS_EXTRA="${LIBS_EXTRA} -lcrypto" - ], [AC_MSG_ERROR([unable to find the EVP_MD_CTX_create() function])]) + ], [AC_MSG_ERROR([unable to find the CRYPTO_new_ex_data() function])]) ]) AC_SUBST(LIBCRYPTO_LIB) AC_PATH_PROG(PANDOC, pandoc, [/non/existent]) diff --git a/contrib/libucl/doc/api.md b/contrib/libucl/doc/api.md index 75b954bb302c..a0d33c0e68a9 100644 --- a/contrib/libucl/doc/api.md +++ b/contrib/libucl/doc/api.md @@ -243,7 +243,7 @@ return ret; # Emitting functions -Libucl can transform UCL objects to a number of tectual formats: +Libucl can transform UCL objects to a number of textual formats: - configuration (`UCL_EMIT_CONFIG`) - nginx like human readable configuration file where implicit arrays are transformed to the duplicate keys - compact json: `UCL_EMIT_JSON_COMPACT` - single line valid json without spaces @@ -349,7 +349,7 @@ This object should be released by caller. Libucl provides the functions similar to inverse conversion functions called with the specific C type: - `ucl_object_fromint` - converts `int64_t` to UCL object - `ucl_object_fromdouble` - converts `double` to UCL object -- `ucl_object_fromboolean` - converts `bool` to UCL object +- `ucl_object_frombool` - converts `bool` to UCL object - `ucl_object_fromstring` - converts `const char *` to UCL object (this string should be NULL terminated) - `ucl_object_fromlstring` - converts `const char *` and `size_t` len to UCL object (string does not need to be NULL terminated) @@ -432,7 +432,8 @@ UCL defines the following functions to manage safe iterators: - `ucl_object_iterate_new` - creates new safe iterator - `ucl_object_iterate_reset` - resets iterator to a new object -- `ucl_object_iterate_safe` - safely iterate the object inside iterator +- `ucl_object_iterate_safe` - safely iterate the object inside iterator. Note: function may allocate and free memory during its operation. Therefore it returns `NULL` either while trying to access item after the last one or when exception (such as memory allocation failure) happens. +- `ucl_object_iter_chk_excpn` - check if the last call to `ucl_object_iterate_safe` ended up in unrecoverable exception (e.g. `ENOMEM`). - `ucl_object_iterate_free` - free memory associated with the safe iterator Please note that unlike unsafe iterators, safe iterators *must* be explicitly initialized and freed. @@ -447,6 +448,11 @@ it = ucl_object_iterate_new (obj); while ((cur = ucl_object_iterate_safe (it, true)) != NULL) { /* Do something */ } +/* Check error condition */ +if (ucl_object_iter_chk_excpn (it)) { + ucl_object_iterate_free (it); + exit (1); +} /* Switch to another object */ it = ucl_object_iterate_reset (it, another_obj); @@ -454,6 +460,11 @@ it = ucl_object_iterate_reset (it, another_obj); while ((cur = ucl_object_iterate_safe (it, true)) != NULL) { /* Do something else */ } +/* Check error condition */ +if (ucl_object_iter_chk_excpn (it)) { + ucl_object_iterate_free (it); + exit (1); +} ucl_object_iterate_free (it); ~~~ diff --git a/contrib/libucl/doc/libucl.3 b/contrib/libucl/doc/libucl.3 index ec5046325700..b5fef09f7691 100644 --- a/contrib/libucl/doc/libucl.3 +++ b/contrib/libucl/doc/libucl.3 @@ -612,15 +612,23 @@ Iteration\ without\ expansion: .PP UCL defines the following functions to manage safe iterators: .IP \[bu] 2 -\f[C]ucl_object_iterate_new\f[] \- creates new safe iterator +\f[C]ucl_object_iterate_new\f[] \- creates new safe iterator. .IP \[bu] 2 -\f[C]ucl_object_iterate_reset\f[] \- resets iterator to a new object +\f[C]ucl_object_iterate_reset\f[] \- resets iterator to a new object. .IP \[bu] 2 \f[C]ucl_object_iterate_safe\f[] \- safely iterate the object inside -iterator +iterator. +Note: function may allocate and free memory during its operation. +Therefore it returns \f[C]NULL\f[] either while trying to access item +after the last one or when exception (such as memory allocation +failure) happens. +.IP \[bu] 2 +\f[C]ucl_object_iter_chk_excpn\f[] \- check if the last call to +\f[C]ucl_object_iterate_safe\f[] ended up in unrecoverable exception +(e.g. \f[C]ENOMEM\f[]). .IP \[bu] 2 \f[C]ucl_object_iterate_free\f[] \- free memory associated with the safe -iterator +iterator. .PP Please note that unlike unsafe iterators, safe iterators \f[I]must\f[] be explicitly initialized and freed. @@ -637,6 +645,11 @@ it\ =\ ucl_object_iterate_new\ (obj); while\ ((cur\ =\ ucl_object_iterate_safe\ (it,\ true))\ !=\ NULL)\ { \ \ \ \ /*\ Do\ something\ */ } +/*\ Check\ error\ condition\ */ +if\ (ucl_object_iter_chk_excpn\ (it))\ { +\ \ \ \ ucl_object_iterate_free\ (it); +\ \ \ \ exit\ (1); +} /*\ Switch\ to\ another\ object\ */ it\ =\ ucl_object_iterate_reset\ (it,\ another_obj); @@ -644,6 +657,11 @@ it\ =\ ucl_object_iterate_reset\ (it,\ another_obj); while\ ((cur\ =\ ucl_object_iterate_safe\ (it,\ true))\ !=\ NULL)\ { \ \ \ \ /*\ Do\ something\ else\ */ } +/*\ Check\ error\ condition\ */ +if\ (ucl_object_iter_chk_excpn\ (it))\ { +\ \ \ \ ucl_object_iterate_free\ (it); +\ \ \ \ exit\ (1); +} ucl_object_iterate_free\ (it); \f[] diff --git a/contrib/libucl/doc/lua_api.md b/contrib/libucl/doc/lua_api.md index f7af3caffff4..7da414903b01 100644 --- a/contrib/libucl/doc/lua_api.md +++ b/contrib/libucl/doc/lua_api.md @@ -69,8 +69,8 @@ converts `obj` to lua representation using the following conversions: - *scalar* values are directly presented by lua objects - *userdata* values are converted to lua function objects using `LUA_REGISTRYINDEX`, this can be used to pass functions from lua to c and vice-versa -- *arrays* are converted to lua tables with numeric indicies suitable for `ipairs` iterations -- *objects* are converted to lua tables with string indicies +- *arrays* are converted to lua tables with numeric indices suitable for `ipairs` iterations +- *objects* are converted to lua tables with string indices **Parameters:** diff --git a/contrib/libucl/include/lua_ucl.h b/contrib/libucl/include/lua_ucl.h index 38e74d3f619d..5b7f88e031e1 100644 --- a/contrib/libucl/include/lua_ucl.h +++ b/contrib/libucl/include/lua_ucl.h @@ -54,6 +54,14 @@ UCL_EXTERN int luaopen_ucl (lua_State *L); */ UCL_EXTERN ucl_object_t* ucl_object_lua_import (lua_State *L, int idx); +/** + * Import UCL object from lua state, escaping JSON strings + * @param L lua state + * @param idx index of object at the lua stack to convert to UCL + * @return new UCL object or NULL, the caller should unref object after using + */ +UCL_EXTERN ucl_object_t* ucl_object_lua_import_escape (lua_State *L, int idx); + /** * Push an object to lua * @param L lua state @@ -62,8 +70,16 @@ UCL_EXTERN ucl_object_t* ucl_object_lua_import (lua_State *L, int idx); */ UCL_EXTERN int ucl_object_push_lua (lua_State *L, const ucl_object_t *obj, bool allow_array); +/** + * Push an object to lua replacing all ucl.null with `false` + * @param L lua state + * @param obj object to push + * @param allow_array traverse over implicit arrays + */ +UCL_EXTERN int ucl_object_push_lua_filter_nil (lua_State *L, + const ucl_object_t *obj, + bool allow_array); -UCL_EXTERN struct ucl_lua_funcdata* ucl_object_toclosure ( - const ucl_object_t *obj); +UCL_EXTERN struct ucl_lua_funcdata* ucl_object_toclosure (const ucl_object_t *obj); #endif /* LUA_UCL_H_ */ diff --git a/contrib/libucl/include/ucl++.h b/contrib/libucl/include/ucl++.h index 2c2bdde51559..fb63430d400d 100644 --- a/contrib/libucl/include/ucl++.h +++ b/contrib/libucl/include/ucl++.h @@ -29,6 +29,7 @@ #include <set> #include <memory> #include <iostream> +#include <tuple> #include "ucl.h" @@ -106,7 +107,7 @@ private: static bool ucl_variable_getter(const unsigned char *data, size_t len, unsigned char ** /*replace*/, size_t * /*replace_len*/, bool *need_free, void* ud) { - *need_free = false; + *need_free = false; auto vars = reinterpret_cast<std::set<std::string> *>(ud); if (vars && data && len != 0) { @@ -123,17 +124,17 @@ private: auto replacer = reinterpret_cast<variable_replacer *>(ud); if (!replacer) { return false; - } + } std::string var_name (data, data + len); if (!replacer->is_variable (var_name)) { return false; - } + } std::string var_value = replacer->replace (var_name); if (var_value.empty ()) { return false; - } + } *replace = (unsigned char *)UCL_ALLOC (var_value.size ()); memcpy (*replace, var_value.data (), var_value.size ()); @@ -152,7 +153,8 @@ private: config_func (parser); if (!parse_func (parser)) { - err.assign (ucl_parser_get_error (parser)); + const char *error = ucl_parser_get_error (parser); //Assigning here without checking result first causes a + if( error != NULL ) err.assign(error); // crash if ucl_parser_get_error returns NULL ucl_parser_free (parser); return nullptr; @@ -168,6 +170,16 @@ private: std::unique_ptr<ucl_object_t, ucl_deleter> obj; public: + struct macro_handler_s { + ucl_macro_handler handler; + ucl_context_macro_handler ctx_handler; + }; + + struct macro_userdata_s { + ucl_parser *parser; + void *userdata; + }; + class const_iterator { private: struct ucl_iter_deleter { @@ -184,7 +196,7 @@ public: it = std::shared_ptr<void>(ucl_object_iterate_new (obj.obj.get()), ucl_iter_deleter()); cur.reset (new Ucl(ucl_object_iterate_safe (it.get(), true))); - if (cur->type() == UCL_NULL) { + if (!cur->obj) { it.reset (); cur.reset (); } @@ -218,7 +230,7 @@ public: cur.reset (new Ucl(ucl_object_iterate_safe (it.get(), true))); } - if (cur && cur->type() == UCL_NULL) { + if (cur && !cur->obj) { it.reset (); cur.reset (); } @@ -330,7 +342,7 @@ public: return UCL_NULL; } - const std::string key () const { + std::string key () const { std::string res; if (obj->key) { @@ -373,7 +385,7 @@ public: return default_val; } - const std::string string_value (const std::string& default_val = "") const + std::string string_value (const std::string& default_val = "") const { const char* res = nullptr; @@ -384,7 +396,16 @@ public: return default_val; } - const Ucl at (size_t i) const + size_t size () const + { + if (type () == UCL_ARRAY) { + return ucl_array_size (obj.get()); + } + + return 0; + } + + Ucl at (size_t i) const { if (type () == UCL_ARRAY) { return Ucl (ucl_array_find_index (obj.get(), i)); @@ -393,7 +414,7 @@ public: return Ucl (nullptr); } - const Ucl lookup (const std::string &key) const + Ucl lookup (const std::string &key) const { if (type () == UCL_OBJECT) { return Ucl (ucl_object_lookup_len (obj.get(), @@ -403,12 +424,12 @@ public: return Ucl (nullptr); } - inline const Ucl operator[] (size_t i) const + inline Ucl operator[] (size_t i) const { return at(i); } - inline const Ucl operator[](const std::string &key) const + inline Ucl operator[](const std::string &key) const { return lookup(key); } @@ -432,43 +453,116 @@ public: return out; } - static Ucl parse (const std::string &in, std::string &err) + static Ucl parse (const std::string &in, std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) { - return parse (in, std::map<std::string, std::string>(), err); + return parse (in, std::map<std::string, std::string>(), err, duplicate_strategy); } - static Ucl parse (const std::string &in, const std::map<std::string, std::string> &vars, std::string &err) + static Ucl parse (const std::string &in, const std::map<std::string, std::string> &vars, + std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) { - auto config_func = [&vars] (ucl_parser *parser) { + std::vector< std::tuple< std::string, macro_handler_s, void * > > emptyVector; + return parse ( in, vars, emptyVector, err, duplicate_strategy ); + } + + //Macro handler will receive a macro_userdata_s as void *ud + static Ucl parse (const std::string &in, + std::vector< std::tuple< std::string /*name*/, macro_handler_s, void * /*userdata*/ > > ¯os, + std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) + { + return parse (in, std::map<std::string, std::string>(), macros, err, duplicate_strategy); + } + + //Macro handler will receive a macro_userdata_s as void *ud + static Ucl parse (const std::string &in, const std::map<std::string, std::string> &vars, + std::vector< std::tuple< std::string /*name*/, macro_handler_s, void * /*userdata*/ > > ¯os, + std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) + { + //Preserve macro_userdata_s memory for later use in parse_with_strategy_function() + std::vector<macro_userdata_s> userdata_list; + userdata_list.reserve (macros.size()); + auto config_func = [&userdata_list, &vars, ¯os] (ucl_parser *parser) { for (const auto & item : vars) { ucl_parser_register_variable (parser, item.first.c_str (), item.second.c_str ()); - } + } + for (auto & macro : macros) { + userdata_list.push_back ({parser, std::get<2>(macro)}); + if (std::get<1>(macro).handler != NULL) { + ucl_parser_register_macro (parser, + std::get<0>(macro).c_str(), + std::get<1>(macro).handler, + reinterpret_cast<void*>(&userdata_list.back())); + } + else if (std::get<1>(macro).ctx_handler != NULL) { + ucl_parser_register_context_macro (parser, + std::get<0>(macro).c_str(), + std::get<1>(macro).ctx_handler, + reinterpret_cast<void*>(&userdata_list.back())); + } + } }; - auto parse_func = [&in] (ucl_parser *parser) { - return ucl_parser_add_chunk (parser, (unsigned char *)in.data (), in.size ()); + auto parse_func = [&in, &duplicate_strategy] (struct ucl_parser *parser) -> bool { + return ucl_parser_add_chunk_full (parser, + (unsigned char *) in.data (), + in.size (), + (unsigned int)ucl_parser_get_default_priority (parser), + duplicate_strategy, + UCL_PARSE_UCL); }; return parse_with_strategy_function (config_func, parse_func, err); } - static Ucl parse (const std::string &in, const variable_replacer &replacer, std::string &err) - { - auto config_func = [&replacer] (ucl_parser *parser) { - ucl_parser_set_variables_handler (parser, ucl_variable_replacer, - &const_cast<variable_replacer &>(replacer)); + static Ucl parse (const std::string &in, const variable_replacer &replacer, + std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) + { + std::vector< std::tuple< std::string, macro_handler_s, void * > > emptyVector; + return parse ( in, replacer, emptyVector, err, duplicate_strategy ); + } + + //Macro handler will receive a macro_userdata_s as void *ud + static Ucl parse (const std::string &in, const variable_replacer &replacer, + std::vector< std::tuple< std::string /*name*/, macro_handler_s, void * /*userdata*/ > > ¯os, + std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) + { + //Preserve macro_userdata_s memory for later use in parse_with_strategy_function() + std::vector<macro_userdata_s> userdata_list; + userdata_list.reserve (macros.size()); + auto config_func = [&userdata_list, &replacer, ¯os] (ucl_parser *parser) { + ucl_parser_set_variables_handler (parser, ucl_variable_replacer, &const_cast<variable_replacer &>(replacer)); + for (auto & macro : macros) { + userdata_list.push_back ({parser, std::get<2>(macro)}); + if (std::get<1>(macro).handler != NULL) { + ucl_parser_register_macro (parser, + std::get<0>(macro).c_str(), + std::get<1>(macro).handler, + reinterpret_cast<void*>(&userdata_list.back())); + } + else if (std::get<1>(macro).ctx_handler != NULL) { + ucl_parser_register_context_macro (parser, + std::get<0>(macro).c_str(), + std::get<1>(macro).ctx_handler, + reinterpret_cast<void*>(&userdata_list.back())); + } + } }; - auto parse_func = [&in] (ucl_parser *parser) { - return ucl_parser_add_chunk (parser, (unsigned char *) in.data (), in.size ()); + auto parse_func = [&in, &duplicate_strategy] (struct ucl_parser *parser) -> bool { + return ucl_parser_add_chunk_full (parser, + (unsigned char *) in.data (), + in.size (), + (unsigned int)ucl_parser_get_default_priority (parser), + duplicate_strategy, + UCL_PARSE_UCL); }; return parse_with_strategy_function (config_func, parse_func, err); } - static Ucl parse (const char *in, std::string &err) + static Ucl parse (const char *in, std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) { - return parse (in, std::map<std::string, std::string>(), err); + return parse (in, std::map<std::string, std::string>(), err, duplicate_strategy); } static Ucl parse (const char *in, const std::map<std::string, std::string> &vars, std::string &err) @@ -480,13 +574,46 @@ public: return parse (std::string (in), vars, err); } - static Ucl parse (const char *in, const variable_replacer &replacer, std::string &err) + //Macro handler will receive a macro_userdata_s as void *ud + static Ucl parse (const char *in, + std::vector< std::tuple< std::string /*name*/, macro_handler_s, void * /*userdata*/ > > ¯os, + std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) + { + return parse (in, std::map<std::string, std::string>(), macros, err, duplicate_strategy); + } + + //Macro handler will receive a macro_userdata_s as void *ud + static Ucl parse (const char *in, const std::map<std::string, std::string> &vars, + std::vector< std::tuple< std::string /*name*/, macro_handler_s, void * /*userdata*/ > > ¯os, + std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) { if (!in) { err = "null input"; return nullptr; } - return parse (std::string(in), replacer, err); + return parse (std::string (in), vars, macros, err, duplicate_strategy); + } + + static Ucl parse (const char *in, const variable_replacer &replacer, + std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) + { + if (!in) { + err = "null input"; + return nullptr; + } + return parse (std::string(in), replacer, err, duplicate_strategy); + } + + //Macro handler will receive a macro_userdata_s as void *ud + static Ucl parse (const char *in, const variable_replacer &replacer, + std::vector< std::tuple< std::string /*name*/, macro_handler_s, void * /*userdata*/ > > ¯os, + std::string &err, enum ucl_duplicate_strategy duplicate_strategy = UCL_DUPLICATE_APPEND) + { + if (!in) { + err = "null input"; + return nullptr; + } + return parse (std::string (in), replacer, macros, err, duplicate_strategy); } static Ucl parse_from_file (const std::string &filename, std::string &err) @@ -556,7 +683,7 @@ public: std::vector<std::string> result; std::move (vars.begin (), vars.end (), std::back_inserter (result)); - return std::move (result); + return result; } Ucl& operator= (Ucl rhs) diff --git a/contrib/libucl/include/ucl.h b/contrib/libucl/include/ucl.h index fccf6fcb2237..39da2593648d 100644 --- a/contrib/libucl/include/ucl.h +++ b/contrib/libucl/include/ucl.h @@ -105,10 +105,11 @@ typedef enum ucl_error { UCL_EIO, /**< IO error occurred during parsing */ UCL_ESTATE, /**< Invalid state machine state */ UCL_ENESTED, /**< Input has too many recursion levels */ + UCL_EUNPAIRED, /**< Input has too many recursion levels */ UCL_EMACRO, /**< Error processing a macro */ UCL_EINTERNAL, /**< Internal unclassified error */ UCL_ESSL, /**< SSL error */ - UCL_EMERGE /**< A merge error occured */ + UCL_EMERGE /**< A merge error occurred */ } ucl_error_t; /** @@ -177,7 +178,8 @@ typedef enum ucl_string_flags { } ucl_string_flags_t; /** - * Basic flags for an object + * Basic flags for an object (can use up to 12 bits as higher 4 bits are used + * for priorities) */ typedef enum ucl_object_flags { UCL_OBJECT_ALLOCATED_KEY = (1 << 0), /**< An object has key allocated internally */ @@ -187,7 +189,8 @@ typedef enum ucl_object_flags { UCL_OBJECT_MULTILINE = (1 << 4), /**< String should be displayed as multiline string */ UCL_OBJECT_MULTIVALUE = (1 << 5), /**< Object is a key with multiple values */ UCL_OBJECT_INHERITED = (1 << 6), /**< Object has been inherited from another */ - UCL_OBJECT_BINARY = (1 << 7) /**< Object contains raw binary data */ + UCL_OBJECT_BINARY = (1 << 7), /**< Object contains raw binary data */ + UCL_OBJECT_SQUOTED = (1 << 8) /**< Object has been enclosed in single quotes */ } ucl_object_flags_t; /** @@ -462,6 +465,14 @@ UCL_EXTERN ucl_object_t* ucl_object_pop_key (ucl_object_t *top, const char *key) UCL_EXTERN bool ucl_object_insert_key_merged (ucl_object_t *top, ucl_object_t *elt, const char *key, size_t keylen, bool copy_key); +/** + * Reserve space in ucl array or object for `elt` elements + * @param obj object to reserve + * @param reserved size to reserve in an object + * @return 0 on success, -1 on failure (i.e. ENOMEM) + */ +UCL_EXTERN bool ucl_object_reserve (ucl_object_t *obj, size_t reserved); + /** * Append an element to the end of array object * @param top destination object (must NOT be NULL) @@ -533,6 +544,13 @@ UCL_EXTERN ucl_object_t* ucl_array_pop_last (ucl_object_t *top); */ UCL_EXTERN ucl_object_t* ucl_array_pop_first (ucl_object_t *top); *** 5210 LINES SKIPPED ***
](https://travis-ci.org/vstakhov/libucl){kind=link}
](https://circleci.com/gh/vstakhov/libucl){kind=link}
](https://scan.coverity.com/projects/4138){kind=link}
](https://coveralls.io/github/vstakhov/libucl?branch=master){kind=link}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202105050724.1457OB7C048539>