Date: Thu, 10 Sep 2020 15:21:27 +0000 (UTC) From: Ruslan Makhmatkhanov <rm@FreeBSD.org> To: ports-committers@freebsd.org, svn-ports-all@freebsd.org, svn-ports-head@freebsd.org Subject: svn commit: r548207 - in head/databases/clickhouse: . files Message-ID: <202009101521.08AFLRki047588@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rm Date: Thu Sep 10 15:21:26 2020 New Revision: 548207 URL: https://svnweb.freebsd.org/changeset/ports/548207 Log: databases/clickhouse: apply patch from #13859: MemoryTracking support Clickhouse for FreeBSD updating no memory usage stats. In the case build without NDEBUG, allocator use 4KB chunk, those dont account when freeing and MemoryTracking metric grow up to ~6TB, after this clickhouse stop query processing with error "Memory limit (total) exceeded: would use 6.24 TiB" due skipping calc allocation < 4MB Patch still not upstreamed, obtained from: https://github.com/ClickHouse/ClickHouse/issues/13859 + https://github.com/ClickHouse/ClickHouse/pull/13869 reported and tested by: Vyacheslav (via Clickhouse telegram) PR: 249205 Submitted by: olevole@olevole.ru (maintainer) Obtained from: clickhouse repository Added: head/databases/clickhouse/files/patch-cmake_freebsd_default__libs.cmake (contents, props changed) head/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.cpp (contents, props changed) head/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.h (contents, props changed) head/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.cpp (contents, props changed) head/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.h (contents, props changed) Modified: head/databases/clickhouse/Makefile Modified: head/databases/clickhouse/Makefile ============================================================================== --- head/databases/clickhouse/Makefile Thu Sep 10 14:07:49 2020 (r548206) +++ head/databases/clickhouse/Makefile Thu Sep 10 15:21:26 2020 (r548207) @@ -5,6 +5,7 @@ PORTNAME= clickhouse PORTVERSION= 20.7.2.30 DISTVERSIONPREFIX= v DISTVERSIONSUFFIX= -stable +PORTREVISION= 1 CATEGORIES= databases MAINTAINER= olevole@olevole.ru Added: head/databases/clickhouse/files/patch-cmake_freebsd_default__libs.cmake ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/databases/clickhouse/files/patch-cmake_freebsd_default__libs.cmake Thu Sep 10 15:21:26 2020 (r548207) @@ -0,0 +1,22 @@ +--- cmake/freebsd/default_libs.cmake.orig 2020-08-31 16:22:57 UTC ++++ cmake/freebsd/default_libs.cmake +@@ -4,13 +4,13 @@ if (NOT COMPILER_CLANG) + message (FATAL_ERROR "FreeBSD build is supported only for Clang") + endif () + +-if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "amd64") +- execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-x86_64.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) +-else () +- execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) +-endif () ++#if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "amd64") ++# execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-x86_64.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) ++#else () ++# execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) ++#endif () + +-set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread") ++set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -lcompiler_rt -lprocstat") + + message(STATUS "Default libraries: ${DEFAULT_LIBS}") + Added: head/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.cpp ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.cpp Thu Sep 10 15:21:26 2020 (r548207) @@ -0,0 +1,127 @@ +--- src/Common/MemoryStatisticsOS.cpp.orig 2020-08-31 16:22:57 UTC ++++ src/Common/MemoryStatisticsOS.cpp +@@ -1,7 +1,13 @@ +-#if defined(OS_LINUX) +- + #include <sys/types.h> + #include <sys/stat.h> ++ ++#ifdef OS_FREEBSD ++#include <sys/param.h> ++#include <sys/sysctl.h> ++#include <sys/user.h> ++#include <libprocstat.h> ++#endif ++ + #include <fcntl.h> + #include <unistd.h> + #include <cassert> +@@ -11,9 +17,10 @@ + #include <Common/Exception.h> + #include <IO/ReadBufferFromMemory.h> + #include <IO/ReadHelpers.h> +-#include <common/logger_useful.h> + + ++ ++ + namespace DB + { + +@@ -23,21 +30,40 @@ namespace ErrorCodes + extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR; + extern const int CANNOT_CLOSE_FILE; ++#ifdef OS_FREEBSD ++ extern const int CANNOT_ALLOCATE_MEMORY; ++#endif + } + ++#ifndef OS_FREEBSD + static constexpr auto filename = "/proc/self/statm"; + static constexpr size_t PAGE_SIZE = 4096; ++#endif + + MemoryStatisticsOS::MemoryStatisticsOS() + { ++#ifdef OS_FREEBSD ++ pstat = ::procstat_open_sysctl(); ++ if (NULL == pstat) ++ { ++ throwFromErrno("Cannot open sysctl", ErrorCodes::CANNOT_ALLOCATE_MEMORY); ++ } ++#else + fd = ::open(filename, O_RDONLY | O_CLOEXEC); + + if (-1 == fd) + throwFromErrno("Cannot open file " + std::string(filename), errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE); ++#endif + } + + MemoryStatisticsOS::~MemoryStatisticsOS() + { ++#ifdef OS_FREEBSD ++ if (NULL != pstat) ++ { ++ ::procstat_close(pstat); ++ } ++#else + if (0 != ::close(fd)) + { + try +@@ -51,12 +77,43 @@ MemoryStatisticsOS::~MemoryStatisticsOS() + DB::tryLogCurrentException(__PRETTY_FUNCTION__); + } + } ++#endif + } + + MemoryStatisticsOS::Data MemoryStatisticsOS::get() const + { + Data data; + ++#ifdef OS_FREEBSD ++ size_t pagesize = ::getpagesize(); ++ unsigned int count = 0; ++ ++ struct kinfo_proc *kp; ++ struct kinfo_vmentry *kve; ++ ++ kp = ::procstat_getprocs(pstat, KERN_PROC_PID, ::getpid(), &count); ++ if (NULL == kp) ++ { ++ throwFromErrno("Cannot get proc info", ErrorCodes::CANNOT_ALLOCATE_MEMORY); ++ } ++ ++ kve = ::procstat_getvmmap(pstat, kp, &count); ++ if (NULL == kve) ++ { ++ ::procstat_freeprocs(pstat, kp); ++ throwFromErrno("Cannot get vmmap info", ErrorCodes::CANNOT_ALLOCATE_MEMORY); ++ } ++ ++ data.virt = kp->ki_size; ++ data.resident = kp->ki_rssize * pagesize; ++ data.shared = (kp->ki_rssize - kve->kve_private_resident) * pagesize; ++ data.code = kp->ki_tsize * pagesize; ++ data.data_and_stack = (kp->ki_dsize + kp->ki_ssize) * pagesize; ++ ++ ::procstat_freevmmap(pstat, kve); ++ ::procstat_freeprocs(pstat, kp); ++#else ++ + constexpr size_t buf_size = 1024; + char buf[buf_size]; + +@@ -98,10 +155,8 @@ MemoryStatisticsOS::Data MemoryStatisticsOS::get() con + data.shared *= PAGE_SIZE; + data.code *= PAGE_SIZE; + data.data_and_stack *= PAGE_SIZE; +- ++#endif + return data; + } + + } +- +-#endif Added: head/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/databases/clickhouse/files/patch-src_Common_MemoryStatisticsOS.h Thu Sep 10 15:21:26 2020 (r548207) @@ -0,0 +1,22 @@ +--- src/Common/MemoryStatisticsOS.h.orig 2020-08-31 16:22:57 UTC ++++ src/Common/MemoryStatisticsOS.h +@@ -1,5 +1,4 @@ + #pragma once +-#if defined(OS_LINUX) + #include <cstdint> + + +@@ -35,9 +34,12 @@ class MemoryStatisticsOS (public) + Data get() const; + + private: ++#ifdef OS_FREEBSD ++ struct procstat * pstat; ++#else + int fd; ++#endif + }; + + } + +-#endif Added: head/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.cpp ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.cpp Thu Sep 10 15:21:26 2020 (r548207) @@ -0,0 +1,11 @@ +--- src/Interpreters/AsynchronousMetrics.cpp.orig 2020-08-31 16:22:57 UTC ++++ src/Interpreters/AsynchronousMetrics.cpp +@@ -194,7 +194,7 @@ void AsynchronousMetrics::update() + new_values["Uptime"] = context.getUptimeSeconds(); + + /// Process memory usage according to OS +-#if defined(OS_LINUX) ++#if defined(OS_LINUX) || defined(OS_FREEBSD) + { + MemoryStatisticsOS::Data data = memory_stat.get(); + Added: head/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/databases/clickhouse/files/patch-src_Interpreters_AsynchronousMetrics.h Thu Sep 10 15:21:26 2020 (r548207) @@ -0,0 +1,11 @@ +--- src/Interpreters/AsynchronousMetrics.h.orig 2020-08-31 16:22:57 UTC ++++ src/Interpreters/AsynchronousMetrics.h +@@ -50,7 +50,7 @@ class AsynchronousMetrics (private) + bool quit {false}; + AsynchronousMetricValues values; + +-#if defined(OS_LINUX) ++#if defined(OS_LINUX) || defined(OS_FREEBSD) + MemoryStatisticsOS memory_stat; + #endif +
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202009101521.08AFLRki047588>