Date: Thu, 12 Mar 2009 15:40:09 +0000 From: Bruce Simpson <bms@incunabulum.net> To: python@FreeBSD.org Subject: Python 2.6 + PTH + FreeBSD = multiprocessing works Message-ID: <49B92CD9.4050505@incunabulum.net> In-Reply-To: <49B8F18E.8020704@incunabulum.net> References: <49B8F030.8050602@incunabulum.net> <9B2FF54C-1C16-4749-B036-1628213E50BF@gmail.com> <49B8F18E.8020704@incunabulum.net>
next in thread | previous in thread | raw e-mail | index | archive | help
This is a multi-part message in MIME format.
--------------040908060302090801020603
Content-Type: text/plain; charset=ISO-8859-1; format=flowed
Content-Transfer-Encoding: 7bit
I managed to get Python 2.6 passing most of the threading and
multiprocessing regression tests on FreeBSD,
by patching the port build to use GNU Pth as the threading library.
The problems with FreeBSD and multiprocessing I traced back to the rtld
+ malloc. There are folk trying to fix this in -CURRENT at the moment.
The POSIX semaphore bug fix I committed to FreeBSD-CURRENT this morning
is only the tip of the iceberg.
I noticed that it flapped out on thread synchronization when patched to
use POSIX semaphores for that, with PTH there isn't really any need, but
this was probably because I still had the kernel printf()'s in at that
point.
thanks again,
BMS
--------------040908060302090801020603
Content-Type: text/plain;
name="python26-fbsd-pth.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="python26-fbsd-pth.patch"
? .swp
? files/__orig__patch-Python_thread__pthread.h
Index: Makefile
===================================================================
RCS file: /home/pcvs/ports/lang/python26/Makefile,v
retrieving revision 1.156
diff -u -p -r1.156 Makefile
--- Makefile 27 Feb 2009 01:25:31 -0000 1.156
+++ Makefile 12 Mar 2009 15:35:51 -0000
@@ -49,6 +49,7 @@ BINLINKS_SUB= -e 's,smtpd,smtpd${PYTHON_
-e 's,(idle|pydoc|python-shared|python),\1${PYTHON_VER},'
OPTIONS= THREADS "Enable thread support" on \
+ PTH "Force the use of GNU portable threads" off \
HUGE_STACK_SIZE "Use a larger thread stack" off \
UCS4 "Use UCS4 for unicode support" on \
PYMALLOC "Use python's internal malloc" on \
@@ -57,6 +58,11 @@ OPTIONS= THREADS "Enable thread support"
.include <bsd.port.pre.mk>
+# Use POSIX semaphores on 7-STABLE or later.
+#.if ${OSVERSION} >= 700000
+#EXTRA_PATCHES+= ${PATCHDIR}/extra-patch-setup.py
+#.endif
+
.if ${PYTHON_VERSION} == ${PYTHON_DEFAULT_VERSION}
MLINKS= ${PYTHON_VERSION}.1 python.1
PLIST_SUB+= IF_DEFAULT=""
@@ -68,14 +74,24 @@ PLIST_SUB+= IF_DEFAULT="@comment "
CFLAGS+= -D__wchar_t=wchar_t
.if !defined(WITHOUT_THREADS)
+.if defined(WITH_PTH)
+CONFIGURE_ARGS+= --with-pth
+EXTRA_PATCHES+= ${PATCHDIR}/extra-patch-configure-pth
+LIB_DEPENDS+= pth:${PORTSDIR}/devel/pth
+PTH_CPPFLAGS= "-I${LOCALBASE}/include/pth"
+PTH_LDFLAGS= "-L${LOCALBASE}/lib/pth"
+CONFIGURE_ENV+= CPPFLAGS="${PTH_CPPFLAGS} ${CPPFLAGS}"
+CONFIGURE_ENV+= LDFLAGS="${PTH_LDFLAGS} ${LDFLAGS}"
+.else # !defined(WITH_PTH)
CONFIGURE_ARGS+= --with-threads
CFLAGS+= ${PTHREAD_CFLAGS}
+CONFIGURE_ENV+= LDFLAGS="${PTHREAD_LIBS} ${LDFLAGS}"
+.endif # defined(WITH_PTH)
.if defined(WITHOUT_HUGE_STACK_SIZE)
CFLAGS+= -DTHREAD_STACK_SIZE=0x20000
.else
CFLAGS+= -DTHREAD_STACK_SIZE=0x100000
.endif # defined(WITHOUT_HUGE_STACK_SIZE)
-CONFIGURE_ENV+= LDFLAGS="${PTHREAD_LIBS} ${LDFLAGS}"
.else
CONFIGURE_ARGS+= --without-threads
.if defined(LDFLAGS)
Index: files/extra-patch-configure-pth
===================================================================
RCS file: files/extra-patch-configure-pth
diff -N files/extra-patch-configure-pth
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ files/extra-patch-configure-pth 12 Mar 2009 15:35:51 -0000
@@ -0,0 +1,12 @@
+--- configure.orig 2009-03-12 14:16:24.000000000 +0000
++++ configure 2009-03-12 14:17:16.000000000 +0000
+@@ -5230,6 +5230,9 @@
+ CC="$ac_save_cc"
+ fi
+
++# XXX Explicitly disable system pthread libraries.
++ac_cv_pthread=no
++
+ { echo "$as_me:$LINENO: result: $ac_cv_pthread" >&5
+ echo "${ECHO_T}$ac_cv_pthread" >&6; }
+ fi
Index: files/extra-patch-setup.py
===================================================================
RCS file: files/extra-patch-setup.py
diff -N files/extra-patch-setup.py
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ files/extra-patch-setup.py 12 Mar 2009 15:35:51 -0000
@@ -0,0 +1,26 @@
+--- setup.py.1 2009-03-12 04:07:36.000000000 +0000
++++ setup.py 2009-03-12 04:08:36.000000000 +0000
+@@ -1265,13 +1265,21 @@
+ )
+ libraries = []
+
+- elif platform in ('freebsd4', 'freebsd5', 'freebsd6', 'freebsd7', 'freebsd8'):
++ elif platform in ('freebsd4', 'freebsd5', 'freebsd6'):
+ # FreeBSD's P1003.1b semaphore support is very experimental
+ # and has many known problems. (as of June 2008)
+- macros = dict( # FreeBSD
++ macros = dict( # FreeBSD 4-6
+ HAVE_SEM_OPEN=0,
+ HAVE_SEM_TIMEDWAIT=0,
+ HAVE_FD_TRANSFER=1,
++ )
++ libraries = []
++
++ elif platform in ('freebsd7', 'freebsd8'):
++ macros = dict( # FreeBSD 7+
++ HAVE_SEM_OPEN=1,
++ HAVE_SEM_TIMEDWAIT=1,
++ HAVE_FD_TRANSFER=1,
+ )
+ libraries = []
+
Index: files/patch-Lib_test_test_threading.py
===================================================================
RCS file: files/patch-Lib_test_test_threading.py
diff -N files/patch-Lib_test_test_threading.py
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ files/patch-Lib_test_test_threading.py 12 Mar 2009 15:35:51 -0000
@@ -0,0 +1,11 @@
+--- Lib/test/test_threading.py.bak 2009-03-12 07:59:17.000000000 +0000
++++ Lib/test/test_threading.py 2009-03-12 10:48:04.000000000 +0000
+@@ -382,7 +382,7 @@
+ return
+ # Skip platforms with known problems forking from a worker thread.
+ # See http://bugs.python.org/issue3863.
+- if sys.platform in ('freebsd4', 'freebsd5', 'freebsd6', 'os2emx'):
++ if sys.platform in ('freebsd4', 'freebsd5', 'freebsd6', 'freebsd7', 'freebsd8', 'os2emx'):
+ print >>sys.stderr, ('Skipping test_3_join_in_forked_from_thread'
+ ' due to known OS bugs on'), sys.platform
+ return
Index: files/patch-Python_thread__pthread.h
===================================================================
RCS file: /home/pcvs/ports/lang/python26/files/patch-Python_thread__pthread.h,v
retrieving revision 1.1
diff -u -p -r1.1 patch-Python_thread__pthread.h
--- files/patch-Python_thread__pthread.h 27 Feb 2009 01:25:32 -0000 1.1
+++ files/patch-Python_thread__pthread.h 12 Mar 2009 15:35:51 -0000
@@ -1,9 +1,34 @@
-
-$FreeBSD: ports/lang/python26/files/patch-Python_thread__pthread.h,v 1.1 2009/02/27 01:25:32 sobomax Exp $
-
---- Python/thread_pthread.h
-+++ Python/thread_pthread.h
-@@ -149,6 +149,7 @@
+--- Python/thread_pthread.h.orig 2006-06-13 16:04:24.000000000 +0100
++++ Python/thread_pthread.h 2009-03-12 10:55:49.000000000 +0000
+@@ -26,13 +26,18 @@
+ #endif
+ #endif
+
++#ifdef __FreeBSD__
++#include <osreldate.h>
++#endif
++
+ /* The POSIX spec says that implementations supporting the sem_*
+ family of functions must indicate this by defining
+ _POSIX_SEMAPHORES. */
+ #ifdef _POSIX_SEMAPHORES
+ /* On FreeBSD 4.x, _POSIX_SEMAPHORES is defined empty, so
+ we need to add 0 to make it work there as well. */
+-#if (_POSIX_SEMAPHORES+0) == -1
++#if defined(__FreeBSD__) && __FreeBSD_version < 701104 && \
++ (_POSIX_SEMAPHORES+0) == -1
+ #define HAVE_BROKEN_POSIX_SEMAPHORES
+ #else
+ #include <semaphore.h>
+@@ -44,7 +49,6 @@
+ in default setting. So the process scope is preferred to get
+ enough number of threads to work. */
+ #ifdef __FreeBSD__
+-#include <osreldate.h>
+ #if __FreeBSD_version >= 500000 && __FreeBSD_version < 504101
+ #undef PTHREAD_SYSTEM_SCHED_SUPPORTED
+ #endif
+@@ -149,6 +153,7 @@
{
pthread_t th;
int status;
@@ -11,7 +36,7 @@ $FreeBSD: ports/lang/python26/files/patc
#if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
pthread_attr_t attrs;
#endif
-@@ -177,6 +178,8 @@
+@@ -177,6 +182,8 @@
#if defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
pthread_attr_setscope(&attrs, PTHREAD_SCOPE_SYSTEM);
#endif
@@ -20,7 +45,7 @@ $FreeBSD: ports/lang/python26/files/patc
status = pthread_create(&th,
#if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
-@@ -188,6 +191,7 @@
+@@ -188,6 +195,7 @@
(void *)arg
);
--------------040908060302090801020603--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?49B92CD9.4050505>