Date: Wed, 14 Oct 2020 18:09:42 +0000 (UTC) From: Kyle Evans <kevans@FreeBSD.org> To: ports-committers@freebsd.org, svn-ports-all@freebsd.org, svn-ports-head@freebsd.org Subject: svn commit: r552351 - in head/lang: python36 python36/files python37 python37/files python38 python38/files python39 python39/files Message-ID: <202010141809.09EI9ghd033448@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kevans Date: Wed Oct 14 18:09:41 2020 New Revision: 552351 URL: https://svnweb.freebsd.org/changeset/ports/552351 Log: lang/python3{6,7,8,9}: Backport close_range patches Worked out over BPO-40422 and BPO-40423, this is the culmination of months of work to coordinate with Linux and get close_range(2) added to FreeBSD, then the usage accepted into CPython. It has landed for Python 3.10 and here I've backported it locally to all the supported Python 3 versions we have. Note that this does include and supercede our previous closefrom(2) patches. There was a lot of intersection between the work done, so this patch against the ports tree does remove those patches from each of the ports in favor of this patch. All the patches involved have been accepted and merged upstream. This patch will bring a performance boost in some more situations on 12.2 and 13.0, as close_range exists there. There is one additional patch sitting in an upstream PR that shuffles the _Py_closerange implementation into a different file -- this is not important for the backport, and the absence of that patch here will not realistically cause any issues. PR: 250322 Approved by: lwhsu (python) Added: head/lang/python36/files/patch-issue40422_issue40423 (contents, props changed) head/lang/python37/files/patch-issue40422_issue40423 (contents, props changed) head/lang/python38/files/patch-issue40422_issue40423 (contents, props changed) head/lang/python39/files/patch-issue40422_issue40423 (contents, props changed) Deleted: head/lang/python36/files/patch-Modules___posixsubprocess.c head/lang/python36/files/patch-Modules_posixmodule.c head/lang/python37/files/patch-Modules___posixsubprocess.c head/lang/python37/files/patch-Modules_posixmodule.c head/lang/python38/files/patch-Modules___posixsubprocess.c head/lang/python38/files/patch-Modules_posixmodule.c Modified: head/lang/python36/Makefile head/lang/python37/Makefile head/lang/python38/Makefile head/lang/python39/Makefile Modified: head/lang/python36/Makefile ============================================================================== --- head/lang/python36/Makefile Wed Oct 14 17:54:50 2020 (r552350) +++ head/lang/python36/Makefile Wed Oct 14 18:09:41 2020 (r552351) @@ -3,7 +3,7 @@ PORTNAME= python PORTVERSION= ${PYTHON_PORTVERSION} -PORTREVISION= 1 +PORTREVISION= 2 CATEGORIES= lang python MASTER_SITES= PYTHON/ftp/python/${PORTVERSION} PKGNAMESUFFIX= ${PYTHON_SUFFIX} Added: head/lang/python36/files/patch-issue40422_issue40423 ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/lang/python36/files/patch-issue40422_issue40423 Wed Oct 14 18:09:41 2020 (r552351) @@ -0,0 +1,207 @@ +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index 07dd54dba9..ef1b7a8ad5 100644 +--- Modules/_posixsubprocess.c ++++ Modules/_posixsubprocess.c +@@ -21,6 +21,8 @@ + #include <dirent.h> + #endif + ++#include "posixmodule.h" ++ + #ifdef _Py_MEMORY_SANITIZER + # include <sanitizer/msan_interface.h> + #endif +@@ -222,7 +222,6 @@ _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) + long end_fd = safe_get_max_fd(); + Py_ssize_t num_fds_to_keep = PyTuple_GET_SIZE(py_fds_to_keep); + Py_ssize_t keep_seq_idx; +- int fd_num; + /* As py_fds_to_keep is sorted we can loop through the list closing + * fds inbetween any in the keep list falling within our range. */ + for (keep_seq_idx = 0; keep_seq_idx < num_fds_to_keep; ++keep_seq_idx) { +@@ -230,15 +229,11 @@ _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) + int keep_fd = PyLong_AsLong(py_keep_fd); + if (keep_fd < start_fd) + continue; +- for (fd_num = start_fd; fd_num < keep_fd; ++fd_num) { +- close(fd_num); +- } ++ _Py_closerange(start_fd, keep_fd - 1); + start_fd = keep_fd + 1; + } + if (start_fd <= end_fd) { +- for (fd_num = start_fd; fd_num < end_fd; ++fd_num) { +- close(fd_num); +- } ++ _Py_closerange(start_fd, end_fd); + } + } + +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index 776a3d249a..f46df9ddf7 100644 +--- Modules/posixmodule.c ++++ Modules/posixmodule.c +@@ -7667,6 +7667,78 @@ os_close_impl(PyObject *module, int fd) + Py_RETURN_NONE; + } + ++/* Our selection logic for which function to use is as follows: ++ * 1. If close_range(2) is available, always prefer that; it's better for ++ * contiguous ranges like this than fdwalk(3) which entails iterating over ++ * the entire fd space and simply doing nothing for those outside the range. ++ * 2. If closefrom(2) is available, we'll attempt to use that next if we're ++ * closing up to sysconf(_SC_OPEN_MAX). ++ * 2a. Fallback to fdwalk(3) if we're not closing up to sysconf(_SC_OPEN_MAX), ++ * as that will be more performant if the range happens to have any chunk of ++ * non-opened fd in the middle. ++ * 2b. If fdwalk(3) isn't available, just do a plain close(2) loop. ++ */ ++#ifdef __FreeBSD__ ++#define USE_CLOSEFROM ++#endif /* __FreeBSD__ */ ++ ++#ifdef HAVE_FDWALK ++#define USE_FDWALK ++#endif /* HAVE_FDWALK */ ++ ++#ifdef USE_FDWALK ++static int ++_fdwalk_close_func(void *lohi, int fd) ++{ ++ int lo = ((int *)lohi)[0]; ++ int hi = ((int *)lohi)[1]; ++ ++ if (fd >= hi) ++ return 1; ++ else if (fd >= lo) ++ close(fd); ++ return 0; ++} ++#endif /* USE_FDWALK */ ++ ++/* Closes all file descriptors in [first, last], ignoring errors. */ ++void ++_Py_closerange(int first, int last) ++{ ++ first = Py_MAX(first, 0); ++ _Py_BEGIN_SUPPRESS_IPH ++#ifdef HAVE_CLOSE_RANGE ++ if (close_range(first, last, 0) == 0 || errno != ENOSYS) { ++ /* Any errors encountered while closing file descriptors are ignored; ++ * ENOSYS means no kernel support, though, ++ * so we'll fallback to the other methods. */ ++ } ++ else ++#endif /* HAVE_CLOSE_RANGE */ ++#ifdef USE_CLOSEFROM ++ if (last >= sysconf(_SC_OPEN_MAX)) { ++ /* Any errors encountered while closing file descriptors are ignored */ ++ closefrom(first); ++ } ++ else ++#endif /* USE_CLOSEFROM */ ++#ifdef USE_FDWALK ++ { ++ int lohi[2]; ++ lohi[0] = first; ++ lohi[1] = last + 1; ++ fdwalk(_fdwalk_close_func, lohi); ++ } ++#else ++ { ++ for (int i = first; i <= last; i++) { ++ /* Ignore errors */ ++ (void)close(i); ++ } ++ } ++#endif /* USE_FDWALK */ ++ _Py_END_SUPPRESS_IPH ++} + + /*[clinic input] + os.closerange +@@ -7682,12 +7754,8 @@ static PyObject * + os_closerange_impl(PyObject *module, int fd_low, int fd_high) + /*[clinic end generated code: output=0ce5c20fcda681c2 input=5855a3d053ebd4ec]*/ + { +- int i; + Py_BEGIN_ALLOW_THREADS +- _Py_BEGIN_SUPPRESS_IPH +- for (i = Py_MAX(fd_low, 0); i < fd_high; i++) +- close(i); +- _Py_END_SUPPRESS_IPH ++ _Py_closerange(fd_low, fd_high - 1); + Py_END_ALLOW_THREADS + Py_RETURN_NONE; + } +diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h +index 1ec1833825..be2308ea86 100644 +--- Modules/posixmodule.h ++++ Modules/posixmodule.h +@@ -19,6 +19,8 @@ PyAPI_FUNC(int) _Py_Gid_Converter(PyObject *, void *); + #endif /* MS_WINDOWS */ + #endif + ++PyAPI_FUNC(void) _Py_closerange(int first, int last); ++ + #ifdef __cplusplus + } + #endif +diff --git a/configure b/configure +index e39c16eee2..1f34b4de74 100755 +--- configure ++++ configure +@@ -11420,9 +11420,9 @@ fi + + # checks for library functions + for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ +- clock confstr ctermid dup3 execv faccessat fchmod fchmodat fchown fchownat \ +- fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \ +- futimens futimes gai_strerror getentropy \ ++ clock close_range confstr ctermid dup3 execv faccessat fchmod fchmodat fchown \ ++ fchownat fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate \ ++ futimesat futimens futimes gai_strerror getentropy \ + getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ + getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ + initgroups kill killpg lchown linkat lstat lutimes mmap \ +diff --git a/configure.ac b/configure.ac +index cf280506bd..09886eaeef 100644 +--- configure.ac ++++ configure.ac +@@ -3531,9 +3531,9 @@ fi + + # checks for library functions + AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ +- clock confstr ctermid dup3 execv faccessat fchmod fchmodat fchown fchownat \ +- fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \ +- futimens futimes gai_strerror getentropy \ ++ clock close_range confstr ctermid dup3 execv faccessat fchmod fchmodat fchown \ ++ fchownat fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate \ ++ futimesat futimens futimes gai_strerror getentropy \ + getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ + getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ + initgroups kill killpg lchown linkat lstat lutimes mmap \ +diff --git a/pyconfig.h.in b/pyconfig.h.in +index 2fbbb6d20d..a4488cbe32 100644 +--- pyconfig.h.in ++++ pyconfig.h.in +@@ -128,6 +128,9 @@ + /* Define to 1 if you have the `clock_settime' function. */ + #undef HAVE_CLOCK_SETTIME + ++/* Define to 1 if you have the `close_range' function. */ ++#undef HAVE_CLOSE_RANGE ++ + /* Define if the C compiler supports computed gotos. */ + #undef HAVE_COMPUTED_GOTOS + +@@ -317,6 +320,9 @@ + /* Define to 1 if you have the `fdopendir' function. */ + #undef HAVE_FDOPENDIR + ++/* Define to 1 if you have the `fdwalk' function. */ ++#undef HAVE_FDWALK ++ + /* Define to 1 if you have the `fexecve' function. */ + #undef HAVE_FEXECVE + Modified: head/lang/python37/Makefile ============================================================================== --- head/lang/python37/Makefile Wed Oct 14 17:54:50 2020 (r552350) +++ head/lang/python37/Makefile Wed Oct 14 18:09:41 2020 (r552351) @@ -3,6 +3,7 @@ PORTNAME= python PORTVERSION= ${PYTHON_PORTVERSION} +PORTREVISION= 1 CATEGORIES= lang python MASTER_SITES= PYTHON/ftp/python/${PORTVERSION} PKGNAMESUFFIX= ${PYTHON_SUFFIX} Added: head/lang/python37/files/patch-issue40422_issue40423 ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/lang/python37/files/patch-issue40422_issue40423 Wed Oct 14 18:09:41 2020 (r552351) @@ -0,0 +1,207 @@ +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index 3cf0683ad9..fb070e4cc4 100644 +--- Modules/_posixsubprocess.c ++++ Modules/_posixsubprocess.c +@@ -21,6 +21,8 @@ + #include <dirent.h> + #endif + ++#include "posixmodule.h" ++ + #ifdef _Py_MEMORY_SANITIZER + # include <sanitizer/msan_interface.h> + #endif +@@ -222,7 +222,6 @@ _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) + long end_fd = safe_get_max_fd(); + Py_ssize_t num_fds_to_keep = PyTuple_GET_SIZE(py_fds_to_keep); + Py_ssize_t keep_seq_idx; +- int fd_num; + /* As py_fds_to_keep is sorted we can loop through the list closing + * fds in between any in the keep list falling within our range. */ + for (keep_seq_idx = 0; keep_seq_idx < num_fds_to_keep; ++keep_seq_idx) { +@@ -230,15 +229,11 @@ _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) + int keep_fd = PyLong_AsLong(py_keep_fd); + if (keep_fd < start_fd) + continue; +- for (fd_num = start_fd; fd_num < keep_fd; ++fd_num) { +- close(fd_num); +- } ++ _Py_closerange(start_fd, keep_fd - 1); + start_fd = keep_fd + 1; + } + if (start_fd <= end_fd) { +- for (fd_num = start_fd; fd_num < end_fd; ++fd_num) { +- close(fd_num); +- } ++ _Py_closerange(start_fd, end_fd); + } + } + +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index 43d4302b92..7878517353 100644 +--- Modules/posixmodule.c ++++ Modules/posixmodule.c +@@ -7820,6 +7820,78 @@ os_close_impl(PyObject *module, int fd) + Py_RETURN_NONE; + } + ++/* Our selection logic for which function to use is as follows: ++ * 1. If close_range(2) is available, always prefer that; it's better for ++ * contiguous ranges like this than fdwalk(3) which entails iterating over ++ * the entire fd space and simply doing nothing for those outside the range. ++ * 2. If closefrom(2) is available, we'll attempt to use that next if we're ++ * closing up to sysconf(_SC_OPEN_MAX). ++ * 2a. Fallback to fdwalk(3) if we're not closing up to sysconf(_SC_OPEN_MAX), ++ * as that will be more performant if the range happens to have any chunk of ++ * non-opened fd in the middle. ++ * 2b. If fdwalk(3) isn't available, just do a plain close(2) loop. ++ */ ++#ifdef __FreeBSD__ ++#define USE_CLOSEFROM ++#endif /* __FreeBSD__ */ ++ ++#ifdef HAVE_FDWALK ++#define USE_FDWALK ++#endif /* HAVE_FDWALK */ ++ ++#ifdef USE_FDWALK ++static int ++_fdwalk_close_func(void *lohi, int fd) ++{ ++ int lo = ((int *)lohi)[0]; ++ int hi = ((int *)lohi)[1]; ++ ++ if (fd >= hi) ++ return 1; ++ else if (fd >= lo) ++ close(fd); ++ return 0; ++} ++#endif /* USE_FDWALK */ ++ ++/* Closes all file descriptors in [first, last], ignoring errors. */ ++void ++_Py_closerange(int first, int last) ++{ ++ first = Py_MAX(first, 0); ++ _Py_BEGIN_SUPPRESS_IPH ++#ifdef HAVE_CLOSE_RANGE ++ if (close_range(first, last, 0) == 0 || errno != ENOSYS) { ++ /* Any errors encountered while closing file descriptors are ignored; ++ * ENOSYS means no kernel support, though, ++ * so we'll fallback to the other methods. */ ++ } ++ else ++#endif /* HAVE_CLOSE_RANGE */ ++#ifdef USE_CLOSEFROM ++ if (last >= sysconf(_SC_OPEN_MAX)) { ++ /* Any errors encountered while closing file descriptors are ignored */ ++ closefrom(first); ++ } ++ else ++#endif /* USE_CLOSEFROM */ ++#ifdef USE_FDWALK ++ { ++ int lohi[2]; ++ lohi[0] = first; ++ lohi[1] = last + 1; ++ fdwalk(_fdwalk_close_func, lohi); ++ } ++#else ++ { ++ for (int i = first; i <= last; i++) { ++ /* Ignore errors */ ++ (void)close(i); ++ } ++ } ++#endif /* USE_FDWALK */ ++ _Py_END_SUPPRESS_IPH ++} + + /*[clinic input] + os.closerange +@@ -7835,12 +7907,8 @@ static PyObject * + os_closerange_impl(PyObject *module, int fd_low, int fd_high) + /*[clinic end generated code: output=0ce5c20fcda681c2 input=5855a3d053ebd4ec]*/ + { +- int i; + Py_BEGIN_ALLOW_THREADS +- _Py_BEGIN_SUPPRESS_IPH +- for (i = Py_MAX(fd_low, 0); i < fd_high; i++) +- close(i); +- _Py_END_SUPPRESS_IPH ++ _Py_closerange(fd_low, fd_high - 1); + Py_END_ALLOW_THREADS + Py_RETURN_NONE; + } +diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h +index 1ec1833825..be2308ea86 100644 +--- Modules/posixmodule.h ++++ Modules/posixmodule.h +@@ -19,6 +19,8 @@ PyAPI_FUNC(int) _Py_Gid_Converter(PyObject *, void *); + #endif /* MS_WINDOWS */ + #endif + ++PyAPI_FUNC(void) _Py_closerange(int first, int last); ++ + #ifdef __cplusplus + } + #endif +diff --git a/configure b/configure +index 829dd69bb8..2eeadec5f0 100755 +--- configure ++++ configure +@@ -11490,9 +11490,9 @@ fi + + # checks for library functions + for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ +- clock confstr ctermid dup3 execv faccessat fchmod fchmodat fchown fchownat \ +- fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \ +- futimens futimes gai_strerror getentropy \ ++ clock close_range confstr ctermid dup3 execv faccessat fchmod fchmodat fchown \ ++ fchownat fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate \ ++ futimesat futimens futimes gai_strerror getentropy \ + getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ + getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ + if_nameindex \ +diff --git a/configure.ac b/configure.ac +index f1cc8e9bcb..80952290b7 100644 +--- configure.ac ++++ configure.ac +@@ -3574,9 +3574,9 @@ fi + + # checks for library functions + AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ +- clock confstr ctermid dup3 execv faccessat fchmod fchmodat fchown fchownat \ +- fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \ +- futimens futimes gai_strerror getentropy \ ++ clock close_range confstr ctermid dup3 execv faccessat fchmod fchmodat fchown \ ++ fchownat fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate \ ++ futimesat futimens futimes gai_strerror getentropy \ + getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ + getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ + if_nameindex \ +diff --git a/pyconfig.h.in b/pyconfig.h.in +index ebab5ff518..e1d659059c 100644 +--- pyconfig.h.in ++++ pyconfig.h.in +@@ -128,6 +128,9 @@ + /* Define to 1 if you have the `clock_settime' function. */ + #undef HAVE_CLOCK_SETTIME + ++/* Define to 1 if you have the `close_range' function. */ ++#undef HAVE_CLOSE_RANGE ++ + /* Define if the C compiler supports computed gotos. */ + #undef HAVE_COMPUTED_GOTOS + +@@ -324,6 +327,9 @@ + /* Define to 1 if you have the `fdopendir' function. */ + #undef HAVE_FDOPENDIR + ++/* Define to 1 if you have the `fdwalk' function. */ ++#undef HAVE_FDWALK ++ + /* Define to 1 if you have the `fexecve' function. */ + #undef HAVE_FEXECVE + Modified: head/lang/python38/Makefile ============================================================================== --- head/lang/python38/Makefile Wed Oct 14 17:54:50 2020 (r552350) +++ head/lang/python38/Makefile Wed Oct 14 18:09:41 2020 (r552351) @@ -3,6 +3,7 @@ PORTNAME= python PORTVERSION= ${PYTHON_PORTVERSION} +PORTREVISION= 1 CATEGORIES= lang python MASTER_SITES= PYTHON/ftp/python/${PORTVERSION} PKGNAMESUFFIX= ${PYTHON_SUFFIX} Added: head/lang/python38/files/patch-issue40422_issue40423 ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/lang/python38/files/patch-issue40422_issue40423 Wed Oct 14 18:09:41 2020 (r552351) @@ -0,0 +1,199 @@ +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index e693e53206..a26ef96e91 100644 +--- Modules/_posixsubprocess.c ++++ Modules/_posixsubprocess.c +@@ -21,6 +21,8 @@ + #include <dirent.h> + #endif + ++#include "posixmodule.h" ++ + #ifdef _Py_MEMORY_SANITIZER + # include <sanitizer/msan_interface.h> + #endif +@@ -222,7 +222,6 @@ _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) + long end_fd = safe_get_max_fd(); + Py_ssize_t num_fds_to_keep = PyTuple_GET_SIZE(py_fds_to_keep); + Py_ssize_t keep_seq_idx; +- int fd_num; + /* As py_fds_to_keep is sorted we can loop through the list closing + * fds in between any in the keep list falling within our range. */ + for (keep_seq_idx = 0; keep_seq_idx < num_fds_to_keep; ++keep_seq_idx) { +@@ -230,15 +229,11 @@ _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) + int keep_fd = PyLong_AsLong(py_keep_fd); + if (keep_fd < start_fd) + continue; +- for (fd_num = start_fd; fd_num < keep_fd; ++fd_num) { +- close(fd_num); +- } ++ _Py_closerange(start_fd, keep_fd - 1); + start_fd = keep_fd + 1; + } + if (start_fd <= end_fd) { +- for (fd_num = start_fd; fd_num < end_fd; ++fd_num) { +- close(fd_num); +- } ++ _Py_closerange(start_fd, end_fd); + } + } + +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index 726e3723f9..7ef2dbc854 100644 +--- Modules/posixmodule.c ++++ Modules/posixmodule.c +@@ -8555,8 +8555,26 @@ os_close_impl(PyObject *module, int fd) + Py_RETURN_NONE; + } + ++/* Our selection logic for which function to use is as follows: ++ * 1. If close_range(2) is available, always prefer that; it's better for ++ * contiguous ranges like this than fdwalk(3) which entails iterating over ++ * the entire fd space and simply doing nothing for those outside the range. ++ * 2. If closefrom(2) is available, we'll attempt to use that next if we're ++ * closing up to sysconf(_SC_OPEN_MAX). ++ * 2a. Fallback to fdwalk(3) if we're not closing up to sysconf(_SC_OPEN_MAX), ++ * as that will be more performant if the range happens to have any chunk of ++ * non-opened fd in the middle. ++ * 2b. If fdwalk(3) isn't available, just do a plain close(2) loop. ++ */ ++#ifdef __FreeBSD__ ++#define USE_CLOSEFROM ++#endif /* __FreeBSD__ */ + + #ifdef HAVE_FDWALK ++#define USE_FDWALK ++#endif /* HAVE_FDWALK */ ++ ++#ifdef USE_FDWALK + static int + _fdwalk_close_func(void *lohi, int fd) + { +@@ -8569,7 +8587,46 @@ _fdwalk_close_func(void *lohi, int fd) + close(fd); + return 0; + } +-#endif /* HAVE_FDWALK */ ++#endif /* USE_FDWALK */ ++ ++/* Closes all file descriptors in [first, last], ignoring errors. */ ++void ++_Py_closerange(int first, int last) ++{ ++ first = Py_MAX(first, 0); ++ _Py_BEGIN_SUPPRESS_IPH ++#ifdef HAVE_CLOSE_RANGE ++ if (close_range(first, last, 0) == 0 || errno != ENOSYS) { ++ /* Any errors encountered while closing file descriptors are ignored; ++ * ENOSYS means no kernel support, though, ++ * so we'll fallback to the other methods. */ ++ } ++ else ++#endif /* HAVE_CLOSE_RANGE */ ++#ifdef USE_CLOSEFROM ++ if (last >= sysconf(_SC_OPEN_MAX)) { ++ /* Any errors encountered while closing file descriptors are ignored */ ++ closefrom(first); ++ } ++ else ++#endif /* USE_CLOSEFROM */ ++#ifdef USE_FDWALK ++ { ++ int lohi[2]; ++ lohi[0] = first; ++ lohi[1] = last + 1; ++ fdwalk(_fdwalk_close_func, lohi); ++ } ++#else ++ { ++ for (int i = first; i <= last; i++) { ++ /* Ignore errors */ ++ (void)close(i); ++ } ++ } ++#endif /* USE_FDWALK */ ++ _Py_END_SUPPRESS_IPH ++} + + /*[clinic input] + os.closerange +@@ -8585,22 +8642,8 @@ static PyObject * + os_closerange_impl(PyObject *module, int fd_low, int fd_high) + /*[clinic end generated code: output=0ce5c20fcda681c2 input=5855a3d053ebd4ec]*/ + { +-#ifdef HAVE_FDWALK +- int lohi[2]; +-#else +- int i; +-#endif + Py_BEGIN_ALLOW_THREADS +- _Py_BEGIN_SUPPRESS_IPH +-#ifdef HAVE_FDWALK +- lohi[0] = Py_MAX(fd_low, 0); +- lohi[1] = fd_high; +- fdwalk(_fdwalk_close_func, lohi); +-#else +- for (i = Py_MAX(fd_low, 0); i < fd_high; i++) +- close(i); +-#endif +- _Py_END_SUPPRESS_IPH ++ _Py_closerange(fd_low, fd_high - 1); + Py_END_ALLOW_THREADS + Py_RETURN_NONE; + } +diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h +index 1e00562abc..749833f71c 100644 +--- Modules/posixmodule.h ++++ Modules/posixmodule.h +@@ -28,6 +28,8 @@ PyAPI_FUNC(int) _Py_Sigset_Converter(PyObject *, void *); + #endif /* HAVE_SIGSET_T */ + #endif /* Py_LIMITED_API */ + ++PyAPI_FUNC(void) _Py_closerange(int first, int last); ++ + #ifdef __cplusplus + } + #endif +diff --git a/configure b/configure +index 96dcd0dcd5..90b61f9f8f 100755 +--- configure ++++ configure +@@ -11503,8 +11503,8 @@ fi + + # checks for library functions + for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ +- clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \ +- faccessat fchmod fchmodat fchown fchownat \ ++ clock confstr close_range copy_file_range ctermid dup3 execv explicit_bzero \ ++ explicit_memset faccessat fchmod fchmodat fchown fchownat \ + fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \ + futimens futimes gai_strerror getentropy \ + getgrgid_r getgrnam_r \ +diff --git a/configure.ac b/configure.ac +index 18a044629a..fbb051d49c 100644 +--- configure.ac ++++ configure.ac +@@ -3560,8 +3560,8 @@ fi + + # checks for library functions + AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ +- clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \ +- faccessat fchmod fchmodat fchown fchownat \ ++ clock confstr close_range copy_file_range ctermid dup3 execv explicit_bzero \ ++ explicit_memset faccessat fchmod fchmodat fchown fchownat \ + fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \ + futimens futimes gai_strerror getentropy \ + getgrgid_r getgrnam_r \ +diff --git a/pyconfig.h.in b/pyconfig.h.in +index 4263a712c3..166d278f7c 100644 +--- pyconfig.h.in ++++ pyconfig.h.in +@@ -136,6 +136,9 @@ + /* Define to 1 if you have the `clock_settime' function. */ + #undef HAVE_CLOCK_SETTIME + ++/* Define to 1 if you have the `close_range' function. */ ++#undef HAVE_CLOSE_RANGE ++ + /* Define if the C compiler supports computed gotos. */ + #undef HAVE_COMPUTED_GOTOS + Modified: head/lang/python39/Makefile ============================================================================== --- head/lang/python39/Makefile Wed Oct 14 17:54:50 2020 (r552350) +++ head/lang/python39/Makefile Wed Oct 14 18:09:41 2020 (r552351) @@ -3,6 +3,7 @@ PORTNAME= python PORTVERSION= ${PYTHON_PORTVERSION} +PORTREVISION= 1 CATEGORIES= lang python MASTER_SITES= PYTHON/ftp/python/${PORTVERSION} PKGNAMESUFFIX= ${PYTHON_SUFFIX} Added: head/lang/python39/files/patch-issue40422_issue40423 ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/lang/python39/files/patch-issue40422_issue40423 Wed Oct 14 18:09:41 2020 (r552351) @@ -0,0 +1,206 @@ +diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c +index 5d1691ace4..ed046fc5c1 100644 +--- Modules/_posixsubprocess.c ++++ Modules/_posixsubprocess.c +@@ -250,7 +250,6 @@ _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) + long end_fd = safe_get_max_fd(); + Py_ssize_t num_fds_to_keep = PyTuple_GET_SIZE(py_fds_to_keep); + Py_ssize_t keep_seq_idx; +- int fd_num; + /* As py_fds_to_keep is sorted we can loop through the list closing + * fds in between any in the keep list falling within our range. */ + for (keep_seq_idx = 0; keep_seq_idx < num_fds_to_keep; ++keep_seq_idx) { +@@ -258,21 +257,11 @@ _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) + int keep_fd = PyLong_AsLong(py_keep_fd); + if (keep_fd < start_fd) + continue; +- for (fd_num = start_fd; fd_num < keep_fd; ++fd_num) { +- close(fd_num); +- } ++ _Py_closerange(start_fd, keep_fd - 1); + start_fd = keep_fd + 1; + } + if (start_fd <= end_fd) { +-#if defined(__FreeBSD__) +- /* Any errors encountered while closing file descriptors are ignored */ +- closefrom(start_fd); +-#else +- for (fd_num = start_fd; fd_num < end_fd; ++fd_num) { +- /* Ignore errors */ +- (void)close(fd_num); +- } +-#endif ++ _Py_closerange(start_fd, end_fd); + } + } + +diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c +index 01e8bcbd29..f6aad2e02e 100644 +--- Modules/posixmodule.c ++++ Modules/posixmodule.c +@@ -8691,8 +8691,26 @@ os_close_impl(PyObject *module, int fd) + Py_RETURN_NONE; + } + ++/* Our selection logic for which function to use is as follows: ++ * 1. If close_range(2) is available, always prefer that; it's better for ++ * contiguous ranges like this than fdwalk(3) which entails iterating over ++ * the entire fd space and simply doing nothing for those outside the range. ++ * 2. If closefrom(2) is available, we'll attempt to use that next if we're ++ * closing up to sysconf(_SC_OPEN_MAX). ++ * 2a. Fallback to fdwalk(3) if we're not closing up to sysconf(_SC_OPEN_MAX), ++ * as that will be more performant if the range happens to have any chunk of ++ * non-opened fd in the middle. ++ * 2b. If fdwalk(3) isn't available, just do a plain close(2) loop. ++ */ ++#ifdef __FreeBSD__ ++#define USE_CLOSEFROM ++#endif /* __FreeBSD__ */ + + #ifdef HAVE_FDWALK ++#define USE_FDWALK ++#endif /* HAVE_FDWALK */ ++ ++#ifdef USE_FDWALK + static int + _fdwalk_close_func(void *lohi, int fd) + { +@@ -8708,7 +8726,46 @@ _fdwalk_close_func(void *lohi, int fd) + } + return 0; + } +-#endif /* HAVE_FDWALK */ ++#endif /* USE_FDWALK */ ++ ++/* Closes all file descriptors in [first, last], ignoring errors. */ ++void ++_Py_closerange(int first, int last) ++{ ++ first = Py_MAX(first, 0); ++ _Py_BEGIN_SUPPRESS_IPH ++#ifdef HAVE_CLOSE_RANGE ++ if (close_range(first, last, 0) == 0 || errno != ENOSYS) { ++ /* Any errors encountered while closing file descriptors are ignored; ++ * ENOSYS means no kernel support, though, ++ * so we'll fallback to the other methods. */ ++ } ++ else ++#endif /* HAVE_CLOSE_RANGE */ ++#ifdef USE_CLOSEFROM ++ if (last >= sysconf(_SC_OPEN_MAX)) { ++ /* Any errors encountered while closing file descriptors are ignored */ ++ closefrom(first); ++ } ++ else ++#endif /* USE_CLOSEFROM */ ++#ifdef USE_FDWALK ++ { ++ int lohi[2]; ++ lohi[0] = first; ++ lohi[1] = last + 1; ++ fdwalk(_fdwalk_close_func, lohi); ++ } ++#else ++ { ++ for (int i = first; i <= last; i++) { ++ /* Ignore errors */ ++ (void)close(i); ++ } ++ } ++#endif /* USE_FDWALK */ ++ _Py_END_SUPPRESS_IPH ++} + + /*[clinic input] + os.closerange +@@ -8724,32 +8781,8 @@ static PyObject * + os_closerange_impl(PyObject *module, int fd_low, int fd_high) + /*[clinic end generated code: output=0ce5c20fcda681c2 input=5855a3d053ebd4ec]*/ + { +-#ifdef HAVE_FDWALK +- int lohi[2]; +-#endif + Py_BEGIN_ALLOW_THREADS +- _Py_BEGIN_SUPPRESS_IPH +-#ifdef HAVE_FDWALK +- lohi[0] = Py_MAX(fd_low, 0); +- lohi[1] = fd_high; +- fdwalk(_fdwalk_close_func, lohi); +-#else +- fd_low = Py_MAX(fd_low, 0); +-#ifdef __FreeBSD__ +- if (fd_high >= sysconf(_SC_OPEN_MAX)) { +- /* Any errors encountered while closing file descriptors are ignored */ +- closefrom(fd_low); +- } +- else +-#endif +- { +- for (int i = fd_low; i < fd_high; i++) { +- /* Ignore errors */ +- (void)close(i); +- } +- } +-#endif +- _Py_END_SUPPRESS_IPH ++ _Py_closerange(fd_low, fd_high - 1); + Py_END_ALLOW_THREADS + Py_RETURN_NONE; + } +diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h +index 1e00562abc..749833f71c 100644 +--- Modules/posixmodule.h ++++ Modules/posixmodule.h +@@ -28,6 +28,8 @@ PyAPI_FUNC(int) _Py_Sigset_Converter(PyObject *, void *); + #endif /* HAVE_SIGSET_T */ + #endif /* Py_LIMITED_API */ + ++PyAPI_FUNC(void) _Py_closerange(int first, int last); ++ + #ifdef __cplusplus + } + #endif +diff --git a/configure b/configure +index 9e6fd46583..de517223f6 100755 +--- configure ++++ configure +@@ -11668,8 +11668,8 @@ fi + + # checks for library functions + for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ +- clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \ +- faccessat fchmod fchmodat fchown fchownat \ ++ clock confstr close_range copy_file_range ctermid dup3 execv explicit_bzero \ ++ explicit_memset faccessat fchmod fchmodat fchown fchownat \ + fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \ + futimens futimes gai_strerror getentropy \ + getgrgid_r getgrnam_r \ +diff --git a/configure.ac b/configure.ac +index d60f05251a..faa187af69 100644 +--- configure.ac ++++ configure.ac +@@ -3664,8 +3664,8 @@ fi + + # checks for library functions + AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ +- clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \ +- faccessat fchmod fchmodat fchown fchownat \ ++ clock confstr close_range copy_file_range ctermid dup3 execv explicit_bzero \ ++ explicit_memset faccessat fchmod fchmodat fchown fchownat \ + fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \ + futimens futimes gai_strerror getentropy \ + getgrgid_r getgrnam_r \ +diff --git a/pyconfig.h.in b/pyconfig.h.in +index c9589cd102..449b25f551 100644 +--- pyconfig.h.in ++++ pyconfig.h.in +@@ -136,6 +136,9 @@ + /* Define to 1 if you have the `clock_settime' function. */ + #undef HAVE_CLOCK_SETTIME + ++/* Define to 1 if you have the `close_range' function. */ ++#undef HAVE_CLOSE_RANGE ++ + /* Define if the C compiler supports computed gotos. */ + #undef HAVE_COMPUTED_GOTOS +
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202010141809.09EI9ghd033448>