powerel7
/
base
2
0
Fork 0
Code Issues Pull Requests Releases Activity

python3 update to patches 

Signed-off-by: basebuilder_pel7ppc64lebuilder0 <basebuilder@powerel.org>
master
basebuilder_pel7ppc64lebuilder0 2019-09-08 12:07:53 +02:00
parent 8297b4139a
commit e0ddc9d8de
9 changed files with 1404 additions and 827 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

940
SOURCES/00055-systemtap.patch
View File

@ -1,198 +1,822 @@
diff -up Python-2.7rc1/configure.ac.systemtap Python-2.7rc1/configure.ac
--- Python-2.7rc1/configure.ac.systemtap 2010-06-06 10:53:15.514975012 -0400
+++ Python-2.7rc1/configure.ac 2010-06-06 10:53:15.520974361 -0400
@@ -2616,6 +2616,38 @@ if test "$with_valgrind" != no; then
)
diff -up Python-3.3.0rc2/configure.ac.systemtap Python-3.3.0rc2/configure.ac
--- Python-3.3.0rc2/configure.ac.systemtap 2012-09-09 05:11:14.000000000 -0400
+++ Python-3.3.0rc2/configure.ac 2012-09-10 09:17:21.114511781 -0400
@@ -2678,6 +2678,23 @@ if test "$with_valgrind" != no; then
OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT"
fi
+# Check for dtrace support
+AC_MSG_CHECKING(for --with-dtrace)
+AC_ARG_WITH(dtrace,
+ AC_HELP_STRING(--with(out)-dtrace, disable/enable dtrace support))
+
+if test ! -z "$with_dtrace"
+then
+ if dtrace -G -o /dev/null -s $srcdir/Include/pydtrace.d 2>/dev/null
+ then
+ AC_DEFINE(WITH_DTRACE, 1,
+ [Define if you want to compile in Dtrace support])
+ with_dtrace="Sun"
+ DTRACEOBJS="Python/dtrace.o"
+ DTRADEHDRS=""
+ elif dtrace -h -o /dev/null -s $srcdir/Include/pydtrace.d
+ then
+ AC_DEFINE(WITH_DTRACE, 1,
+ [Define if you want to compile in Dtrace support])
+ with_dtrace="Apple"
+ DTRACEOBJS=""
+ DTRADEHDRS="pydtrace.h"
+ else
+ with_dtrace="no"
+ fi
+else
+ with_dtrace="no"
+# Check for systemtap support
+# On Linux, /usr/bin/dtrace is in fact a shim to SystemTap
+AC_MSG_CHECKING([for --with-systemtap])
+AC_ARG_WITH([systemtap],
+ AC_HELP_STRING([--with(out)-systemtap], [disable/enable SystemTap support]),,
+ with_systemtap=no)
+AC_MSG_RESULT([$with_systemtap])
+if test "$with_systemtap" != no; then
+ AC_DEFINE(WITH_SYSTEMTAP, 1,
+ [Define if you want to compile in SystemTap support])
+ SYSTEMTAPOBJS="Python/pysystemtap.o"
+ SYSTEMTAPDEPS="\$(srcdir)/Python/pysystemtap.h"
+fi
+
+AC_MSG_RESULT($with_dtrace)
+AC_SUBST(DTRACEOBJS)
+AC_SUBST(DTRACEHDRS)
+AC_SUBST(SYSTEMTAPOBJS)
+AC_SUBST(SYSTEMTAPDEPS)
+
# Check for --with-wctype-functions
AC_MSG_CHECKING(for --with-wctype-functions)
AC_ARG_WITH(wctype-functions,
diff -up Python-2.7rc1/Include/pydtrace.d.systemtap Python-2.7rc1/Include/pydtrace.d
--- Python-2.7rc1/Include/pydtrace.d.systemtap 2010-06-06 10:53:15.520974361 -0400
+++ Python-2.7rc1/Include/pydtrace.d 2010-06-06 10:53:15.520974361 -0400
@@ -0,0 +1,10 @@
+provider python {
+ probe function__entry(const char *, const char *, int);
+ probe function__return(const char *, const char *, int);
+};
# -I${DLINCLDIR} is added to the compile rule for importdl.o
AC_SUBST(DLINCLDIR)
DLINCLDIR=.
diff -up Python-3.3.0rc2/configure.systemtap Python-3.3.0rc2/configure
--- Python-3.3.0rc2/configure.systemtap 2012-09-09 05:11:14.000000000 -0400
+++ Python-3.3.0rc2/configure 2012-09-10 09:17:21.116511780 -0400
@@ -618,6 +618,8 @@ TRUE
MACHDEP_OBJS
DYNLOADFILE
DLINCLDIR
+SYSTEMTAPDEPS
+SYSTEMTAPOBJS
THREADOBJ
LDLAST
USE_THREAD_MODULE
@@ -779,6 +781,7 @@ with_doc_strings
with_tsc
with_pymalloc
with_valgrind
+with_systemtap
with_fpectl
with_libm
with_libc
@@ -1456,6 +1459,7 @@ Optional Packages:
--with(out)-tsc enable/disable timestamp counter profile
--with(out)-pymalloc disable/enable specialized mallocs
--with-valgrind Enable Valgrind support
+ --with(out)-systemtap disable/enable SystemTap support
--with-fpectl enable SIGFPE catching
--with-libm=STRING math library
--with-libc=STRING C library
@@ -10065,6 +10069,31 @@ fi
OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT"
fi
+# Check for systemtap support
+# On Linux, /usr/bin/dtrace is in fact a shim to SystemTap
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-systemtap" >&5
+$as_echo_n "checking for --with-systemtap... " >&6; }
+
+#pragma D attributes Evolving/Evolving/Common provider python provider
+#pragma D attributes Private/Private/Common provider python module
+#pragma D attributes Private/Private/Common provider python function
+#pragma D attributes Evolving/Evolving/Common provider python name
+#pragma D attributes Evolving/Evolving/Common provider python args
diff -up Python-2.7rc1/Makefile.pre.in.systemtap Python-2.7rc1/Makefile.pre.in
--- Python-2.7rc1/Makefile.pre.in.systemtap 2010-06-06 10:53:15.488978775 -0400
+++ Python-2.7rc1/Makefile.pre.in 2010-06-06 11:05:30.411100568 -0400
@@ -298,6 +298,7 @@ PYTHON_OBJS= \
+# Check whether --with-systemtap was given.
+if test "${with_systemtap+set}" = set; then :
+ withval=$with_systemtap;
+else
+ with_systemtap=no
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_systemtap" >&5
+$as_echo "$with_systemtap" >&6; }
+if test "$with_systemtap" != no; then
+
+$as_echo "#define WITH_SYSTEMTAP 1" >>confdefs.h
+
+ SYSTEMTAPOBJS="Python/pysystemtap.o"
+ SYSTEMTAPDEPS="\$(srcdir)/Python/pysystemtap.h"
+fi
+
+
+
+
# -I${DLINCLDIR} is added to the compile rule for importdl.o
DLINCLDIR=.
diff -up Python-3.3.0rc2/Doc/howto/index.rst.systemtap Python-3.3.0rc2/Doc/howto/index.rst
--- Python-3.3.0rc2/Doc/howto/index.rst.systemtap 2012-09-09 05:10:51.000000000 -0400
+++ Python-3.3.0rc2/Doc/howto/index.rst 2012-09-10 09:17:21.117511779 -0400
@@ -29,4 +29,5 @@ Currently, the HOWTOs are:
argparse.rst
ipaddress.rst
clinic.rst
+ instrumentation.rst
diff -up Python-3.3.0rc2/Doc/howto/instrumentation.rst.systemtap Python-3.3.0rc2/Doc/howto/instrumentation.rst
--- Python-3.3.0rc2/Doc/howto/instrumentation.rst.systemtap 2012-09-10 09:17:21.117511779 -0400
+++ Python-3.3.0rc2/Doc/howto/instrumentation.rst 2012-09-10 09:17:21.117511779 -0400
@@ -0,0 +1,295 @@
+.. _instrumentation:
+
+====================================
+Instrumenting CPython with SystemTap
+====================================
+
+:author: David Malcolm <dmalcolm@redhat.com>
+
+DTrace and SystemTap are monitoring tools, each providing a way to inspect
+what the processes on a computer system are doing. They both use
+domain-specific languages allowing a user to write scripts which:
+
+ - filter which processes are to be observed
+ - gather data from the processes of interest
+ - generate reports on the data
+
+As of Python 3.3, CPython can be built with embedded "markers" that can be
+observed by a SystemTap script, making it easier to monitor what the CPython
+processes on a system are doing.
+
+.. Potentially this document could be expanded to also cover DTrace markers.
+ However, I'm not a DTrace expert.
+
+.. I'm using ".. code-block:: c" for SystemTap scripts, as "c" is syntactically
+ the closest match that Sphinx supports
+
+
+Enabling the static markers
+---------------------------
+
+In order to build CPython with the embedded markers for SystemTap, the
+SystemTap development tools must be installed.
+
+On a Fedora or Red Hat Enterprise Linux machine, this can be done via::
+
+ yum install systemtap-sdt-devel
+
+CPython must then be configured `--with-systemtap`::
+
+ checking for --with-systemtap... yes
+
+You can verify if the SystemTap static markers are present in the built
+binary by seeing if it contains a ".note.stapsdt" section.
+
+.. code-block:: bash
+
+ $ eu-readelf -S ./python | grep .note.stapsdt
+ [29] .note.stapsdt NOTE 0000000000000000 00308d78 000000b8 0 0 0 4
+
+If you've built python as a shared library (with --enable-shared), you need
+to look instead within the shared library. For example:
+
+.. code-block:: bash
+
+ $ eu-readelf -S libpython3.3dm.so.1.0 | grep .note.stapsdt
+ [28] .note.stapsdt NOTE 0000000000000000 00365b68 000000b8 0 0 0 4
+
+Earlier versions of SystemTap stored the markers in a ".probes" section.
+
+For the curious, you can see the metadata for the static markers using this
+invocation.
+
+.. code-block:: bash
+
+ $ eu-readelf -x .note.stapsdt ./python
+
+ Hex dump of section [29] '.note.stapsdt', 184 bytes at offset 0x308d78:
+ 0x00000000 08000000 45000000 03000000 73746170 ....E.......stap
+ 0x00000010 73647400 d4664b00 00000000 4fc36600 sdt..fK.....O.f.
+ 0x00000020 00000000 488d9000 00000000 70797468 ....H.......pyth
+ 0x00000030 6f6e0066 756e6374 696f6e5f 5f656e74 on.function__ent
+ 0x00000040 72790038 40257261 78203840 25726478 ry.8@%rax 8@%rdx
+ 0x00000050 202d3440 25656378 00000000 08000000 -4@%ecx........
+ 0x00000060 46000000 03000000 73746170 73647400 F.......stapsdt.
+ 0x00000070 0d674b00 00000000 4fc36600 00000000 .gK.....O.f.....
+ 0x00000080 4a8d9000 00000000 70797468 6f6e0066 J.......python.f
+ 0x00000090 756e6374 696f6e5f 5f726574 75726e00 unction__return.
+ 0x000000a0 38402572 61782038 40257264 78202d34 8@%rax 8@%rdx -4
+ 0x000000b0 40256563 78000000 @%ecx...
+
+and a sufficiently modern eu-readelf can print the metadata:
+
+.. code-block:: bash
+
+ $ eu-readelf -n ./python
+
+ Note section [ 1] '.note.gnu.build-id' of 36 bytes at offset 0x190:
+ Owner Data size Type
+ GNU 20 GNU_BUILD_ID
+ Build ID: a28f8db1b224530b0d38ad7b82a249cf7c3f18d6
+
+ Note section [27] '.note.stapsdt' of 184 bytes at offset 0x1ae884:
+ Owner Data size Type
+ stapsdt 70 Version: 3
+ PC: 0xe0d3a, Base: 0x14b150, Semaphore: 0x3ae882
+ Provider: python, Name: function__return, Args: '8@%rbx 8@%r13 -4@%eax'
+ stapsdt 69 Version: 3
+ PC: 0xe0f37, Base: 0x14b150, Semaphore: 0x3ae880
+ Provider: python, Name: function__entry, Args: '8@%rbx 8@%r13 -4@%eax'
+
+The above metadata contains information for SystemTap describing how it can
+patch strategically-placed machine code instructions to enable the tracing
+hooks used by a SystemTap script.
+
+
+Static markers
+--------------
+
+The low-level way to use the SystemTap integration is to use the static
+markers directly. This requires you to explicitly state the binary file
+containing them.
+
+For example, this script can be used to show the call/return hierarchy of a
+Python script:
+
+.. code-block:: c
+
+ probe process('python').mark("function__entry") {
+ filename = user_string($arg1);
+ funcname = user_string($arg2);
+ lineno = $arg3;
+
+ printf("%s => %s in %s:%d\\n",
+ thread_indent(1), funcname, filename, lineno);
+ }
+
+ probe process('python').mark("function__return") {
+ filename = user_string($arg1);
+ funcname = user_string($arg2);
+ lineno = $arg3;
+
+ printf("%s <= %s in %s:%d\\n",
+ thread_indent(-1), funcname, filename, lineno);
+ }
+
+It can be invoked like this:
+
+.. code-block:: bash
+
+ $ stap \
+ show-call-hierarchy.stp \
+ -c ./python test.py
+
+The output looks like this::
+
+ 11408 python(8274): => __contains__ in Lib/_abcoll.py:362
+ 11414 python(8274): => __getitem__ in Lib/os.py:425
+ 11418 python(8274): => encode in Lib/os.py:490
+ 11424 python(8274): <= encode in Lib/os.py:493
+ 11428 python(8274): <= __getitem__ in Lib/os.py:426
+ 11433 python(8274): <= __contains__ in Lib/_abcoll.py:366
+
+where the columns are:
+
+ - time in microseconds since start of script
+
+ - name of executable
+
+ - PID of process
+
+and the remainder indicates the call/return hierarchy as the script executes.
+
+For a `--enable-shared` build of CPython, the markers are contained within the
+libpython shared library, and the probe's dotted path needs to reflect this. For
+example, this line from the above example::
+
+ probe process('python').mark("function__entry") {
+
+should instead read::
+
+ probe process('python').library("libpython3.3dm.so.1.0").mark("function__entry") {
+
+(assuming a debug build of CPython 3.3)
+
+.. I'm reusing the "c:function" type for markers
+
+.. c:function:: function__entry(str filename, str funcname, int lineno)
+
+ This marker indicates that execution of a Python function has begun. It is
+ only triggered for pure-python (bytecode) functions.
+
+ The filename, function name, and line number are provided back to the
+ tracing script as positional arguments, which must be accessed using
+ `$arg1`, `$arg2`:
+
+ * `$arg1` : `(const char *)` filename, accessible using `user_string($arg1)`
+
+ * `$arg2` : `(const char *)` function name, accessible using
+ `user_string($arg2)`
+
+ * `$arg3` : `int` line number
+
+ * `$arg4` : `(PyFrameObject *)`, the frame being executed
+
+.. c:function:: function__return(str filename, str funcname, int lineno)
+
+ This marker is the converse of `function__entry`, and indicates that
+ execution of a Python function has ended (either via ``return``, or via an
+ exception). It is only triggered for pure-python (bytecode) functions.
+
+ The arguments are the same as for `function__entry`
+
+
+Tapsets
+-------
+
+The higher-level way to use the SystemTap integration is to use a "tapset":
+SystemTap's equivalent of a library, which hides some of the lower-level
+details of the static markers.
+
+Here is a tapset file, based on a non-shared build of CPython:
+
+.. code-block:: c
+
+ /*
+ Provide a higher-level wrapping around the function__entry and
+ function__return markers:
+ */
+ probe python.function.entry = process("python").mark("function__entry")
+ {
+ filename = user_string($arg1);
+ funcname = user_string($arg2);
+ lineno = $arg3;
+ frameptr = $arg4
+ }
+ probe python.function.return = process("python").mark("function__return")
+ {
+ filename = user_string($arg1);
+ funcname = user_string($arg2);
+ lineno = $arg3;
+ frameptr = $arg4
+ }
+
+If this file is installed in SystemTap's tapset directory (e.g.
+`/usr/share/systemtap/tapset`), then these additional probepoints become
+available:
+
+.. c:function:: python.function.entry(str filename, str funcname, int lineno, frameptr)
+
+ This probe point indicates that execution of a Python function has begun.
+ It is only triggered for pure-python (bytecode) functions.
+
+.. c:function:: python.function.return(str filename, str funcname, int lineno, frameptr)
+
+ This probe point is the converse of `python.function.return`, and indicates
+ that execution of a Python function has ended (either via ``return``, or
+ via an exception). It is only triggered for pure-python (bytecode) functions.
+
+
+Examples
+--------
+This SystemTap script uses the tapset above to more cleanly implement the
+example given above of tracing the Python function-call hierarchy, without
+needing to directly name the static markers:
+
+.. code-block:: c
+
+ probe python.function.entry
+ {
+ printf("%s => %s in %s:%d\n",
+ thread_indent(1), funcname, filename, lineno);
+ }
+
+ probe python.function.return
+ {
+ printf("%s <= %s in %s:%d\n",
+ thread_indent(-1), funcname, filename, lineno);
+ }
+
+
+The following script uses the tapset above to provide a top-like view of all
+running CPython code, showing the top 20 most frequently-entered bytecode
+frames, each second, across the whole system:
+
+.. code-block:: c
+
+ global fn_calls;
+
+ probe python.function.entry
+ {
+ fn_calls[pid(), filename, funcname, lineno] += 1;
+ }
+
+ probe timer.ms(1000) {
+ printf("\033[2J\033[1;1H") /* clear screen */
+ printf("%6s %80s %6s %30s %6s\n",
+ "PID", "FILENAME", "LINE", "FUNCTION", "CALLS")
+ foreach ([pid, filename, funcname, lineno] in fn_calls- limit 20) {
+ printf("%6d %80s %6d %30s %6d\n",
+ pid, filename, lineno, funcname,
+ fn_calls[pid, filename, funcname, lineno]);
+ }
+ delete fn_calls;
+ }
+
diff -up Python-3.3.0rc2/Lib/test/test_systemtap.py.systemtap Python-3.3.0rc2/Lib/test/test_systemtap.py
--- Python-3.3.0rc2/Lib/test/test_systemtap.py.systemtap 2012-09-10 09:17:21.117511779 -0400
+++ Python-3.3.0rc2/Lib/test/test_systemtap.py 2012-09-10 09:17:21.117511779 -0400
@@ -0,0 +1,234 @@
+# Verify that systemtap static probes work
+#
+import subprocess
+import sys
+import sysconfig
+import os
+import unittest
+
+from test.support import run_unittest, TESTFN, unlink
+
+if '--with-systemtap' not in sysconfig.get_config_var('CONFIG_ARGS'):
+ raise unittest.SkipTest("Python was not configured --with-systemtap")
+
+try:
+ _, stap_version = subprocess.Popen(["stap", "-V"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ ).communicate()
+except OSError:
+ # This is what "no stap" looks like. There may, however, be other
+ # errors that manifest this way too.
+ raise unittest.SkipTest("Couldn't find stap on the path")
+
+def invoke_systemtap_script(script, cmd):
+ # Start a child process, probing with the given systemtap script
+ # (passed as stdin to the "stap" tool)
+ # The script should be a bytes instance
+ # Return (stdout, stderr) pair
+
+ p = subprocess.Popen(["stap", "-", '-vv', '-c', cmd],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = p.communicate(input=script)
+ return out, err
+
+# Verify that stap can run a simple "hello world"-style script
+# This can fail for various reasons:
+# - missing kernel headers
+# - permissions (a non-root user needs to be in the "stapdev" group)
+TRIVIAL_STAP_SCRIPT = b'probe begin { println("hello world") exit () }'
+
+out, err = invoke_systemtap_script(TRIVIAL_STAP_SCRIPT, 'true')
+if out != b'hello world\n':
+ raise unittest.SkipTest("Test systemtap script did not run; stderr was: %s" % err)
+
+# We don't expect stderr to be empty, since we're invoking stap with "-vv": stap
+# will (we hope) generate debugging output on stderr.
+
+def invoke_python_under_systemtap(script, pythoncode=None, pythonfile=None):
+ # Start a child python process, probing with the given systemtap script
+ # (passed as stdin to the "stap" tool)
+ # The script should be a bytes instance
+ # Return (stdout, stderr) pair
+
+ if pythonfile:
+ pythoncmd = '%s %s' % (sys.executable, pythonfile)
+ else:
+ pythoncmd = '%s -c %r' % (sys.executable, pythoncode)
+
+ # The process tree of a stap invocation of a command goes through
+ # something like this:
+ # stap ->fork/exec(staprun; exec stapio ->f/e(-c cmd); exec staprun -r)
+ # and this trip through setuid leads to LD_LIBRARY_PATH being dropped,
+ # which would lead to an --enable-shared build of python failing to be
+ # find its libpython, with an error like:
+ # error while loading shared libraries: libpython3.3dm.so.1.0: cannot
+ # open shared object file: No such file or directory
+ # Hence we need to jump through some hoops to expose LD_LIBRARY_PATH to
+ # the invoked python process:
+ LD_LIBRARY_PATH = os.environ.get('LD_LIBRARY_PATH', '')
+ if LD_LIBRARY_PATH:
+ pythoncmd = 'env LD_LIBRARY_PATH=%s ' % LD_LIBRARY_PATH + pythoncmd
+
+ return invoke_systemtap_script(script, pythoncmd)
+
+# When using the static markers, we need to supply the prefix of a systemtap
+# dotted probe point that containing the marker.
+# See http://sourceware.org/systemtap/langref/Probe_points.html
+#
+# We need to determine if this is a shared-library build
+#
+# Note that sysconfig can get this wrong; see:
+# http://bugs.python.org/issue14774
+#
+if '--enable-shared' in sysconfig.get_config_var('CONFIG_ARGS'):
+ # For a shared-library build, the markers are in library(INSTSONAME):
+ INSTSONAME = sysconfig.get_config_var('INSTSONAME')
+ probe_prefix = 'process("%s").library("%s")' % (sys.executable, INSTSONAME)
+else:
+ # For a non-shared-library build, we can simply use sys.executable:
+ probe_prefix = 'process("%s")' % sys.executable
+
+# The following script ought to generate lots of lines showing recursive
+# function entry and return, of the form:
+# 11408 python(8274): => __contains__ in Lib/_abcoll.py:362
+# 11414 python(8274): => __getitem__ in Lib/os.py:425
+# 11418 python(8274): => encode in Lib/os.py:490
+# 11424 python(8274): <= encode in Lib/os.py:493
+# 11428 python(8274): <= __getitem__ in Lib/os.py:426
+# 11433 python(8274): <= __contains__ in Lib/_abcoll.py:366
+# where the column are:
+# - time in microseconds since start of script
+# - name of executable
+# - PID of process
+# and the remainder indicates the call/return hierarchy
+
+hierarchy_script = ('''
+probe %s.mark("function__entry") {
+ filename = user_string($arg1);
+ funcname = user_string($arg2);
+ lineno = $arg3;
+
+ printf("%%s => %%s in %%s:%%d\\n", thread_indent(1), funcname, filename, lineno);
+}
+
+probe %s.mark("function__return") {
+ filename = user_string($arg1);
+ funcname = user_string($arg2);
+ lineno = $arg3;
+
+ printf("%%s <= %%s in %%s:%%d\\n", thread_indent(-1), funcname, filename, lineno);
+}
+''' % (probe_prefix, probe_prefix)).encode('utf-8')
+
+
+class ErrorDumper:
+ # A context manager that dumps extra information if an exception is raised,
+ # to help track down why the problem occurred
+ def __init__(self, out, err):
+ self.out = out
+ self.err = err
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, type_, value, traceback):
+ if type_:
+ # an exception is being raised:
+ print('stdout: %s' % out.decode())
+ print('stderr: %s' % err.decode())
+
+class SystemtapTests(unittest.TestCase):
+
+ def test_invoking_python(self):
+ # Ensure that we can invoke python under stap, with a trivial stap
+ # script:
+ out, err = invoke_python_under_systemtap(
+ b'probe begin { println("hello from stap") exit () }',
+ pythoncode="print('hello from python')")
+ with ErrorDumper(out, err):
+ self.assertIn(b'hello from stap', out)
+ self.assertIn(b'hello from python', out)
+
+ def test_function_entry(self):
+ # Ensure that the function_entry static marker works
+ out, err = invoke_python_under_systemtap(hierarchy_script)
+ # stdout ought to contain various lines showing recursive function
+ # entry and return (see above)
+
+ # Uncomment this for debugging purposes:
+ # print(out.decode('utf-8'))
+
+ # Executing the cmdline-supplied "pass":
+ # 0 python(8274): => <module> in <string>:1
+ # 5 python(8274): <= <module> in <string>:1
+ with ErrorDumper(out, err):
+ self.assertIn(b'=> <module> in <string>:1', out,
+ msg="stdout: %s\nstderr: %s\n" % (out, err))
+
+ def test_function_encoding(self):
+ # Ensure that function names containing non-Latin 1 code
+ # points are handled:
+ pythonfile = TESTFN
+ try:
+ unlink(pythonfile)
+ f = open(pythonfile, "wb")
+ f.write("""
+# Sample script with non-ASCII filename, for use by test_systemtap.py
+# Implicitly UTF-8
+
+def 文字化け():
+ '''Function with non-ASCII identifier; I believe this reads "mojibake"'''
+ print("hello world!")
+
+文字化け()
+""".encode('utf-8'))
+ f.close()
+
+ out, err = invoke_python_under_systemtap(hierarchy_script,
+ pythonfile=pythonfile)
+ out_utf8 = out.decode('utf-8')
+ with ErrorDumper(out, err):
+ self.assertIn('=> <module> in %s:5' % pythonfile, out_utf8)
+ self.assertIn(' => 文字化け in %s:5' % pythonfile, out_utf8)
+ self.assertIn(' <= 文字化け in %s:7' % pythonfile, out_utf8)
+ self.assertIn('<= <module> in %s:9' % pythonfile, out_utf8)
+ finally:
+ unlink(pythonfile)
+
+ @unittest.skipIf(sys.getfilesystemencoding() == 'ascii',
+ 'the test filename is not encodable with ASCII')
+ def test_filename_encoding(self):
+ # Ensure that scripts names containing non-Latin 1 code
+ # points are handled:
+ pythonfile = TESTFN + '_☠.py'
+ try:
+ unlink(pythonfile)
+ f = open(pythonfile, "wb")
+ f.write("""
+def foo():
+ '''Function with non-ASCII identifier; I believe this reads "mojibake"'''
+ print("hello world!")
+
+foo()
+""".encode('utf-8'))
+ f.close()
+
+ out, err = invoke_python_under_systemtap(hierarchy_script,
+ pythonfile=pythonfile)
+ out_utf8 = out.decode('utf-8')
+ with ErrorDumper(out, err):
+ self.assertIn('=> <module> in %s:2' % pythonfile, out_utf8)
+ self.assertIn(' => foo in %s:2' % pythonfile, out_utf8)
+ self.assertIn(' <= foo in %s:4' % pythonfile, out_utf8)
+ self.assertIn('<= <module> in %s:6' % pythonfile, out_utf8)
+ finally:
+ unlink(pythonfile)
+
+def test_main():
+ run_unittest(SystemtapTests)
+
+if __name__ == "__main__":
+ test_main()
diff -up Python-3.3.0rc2/Makefile.pre.in.systemtap Python-3.3.0rc2/Makefile.pre.in
--- Python-3.3.0rc2/Makefile.pre.in.systemtap 2012-09-09 05:11:05.000000000 -0400
+++ Python-3.3.0rc2/Makefile.pre.in 2012-09-10 09:19:51.195501518 -0400
@@ -363,6 +363,7 @@ PYTHON_OBJS= \
Python/formatter_unicode.o \
Python/formatter_string.o \
Python/fileutils.o \
Python/$(DYNLOADFILE) \
+ @DTRACEOBJS@ \
+ @SYSTEMTAPOBJS@ \
$(LIBOBJS) \
$(MACHDEP_OBJS) \
$(THREADOBJ)
@@ -599,6 +600,18 @@ Python/formatter_unicode.o: $(srcdir)/Py
Python/formatter_string.o: $(srcdir)/Python/formatter_string.c \
$(STRINGLIB_HEADERS)
@@ -713,7 +714,8 @@ Objects/setobject.o: $(srcdir)/Objects/s
$(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES)
$(OPCODETARGETGEN) $(OPCODETARGETS_H)
+# Only needed with --with-dtrace
+buildinclude:
+ mkdir -p Include
-Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h
+Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h \
+ $(srcdir)/Python/ceval_systemtap.h @SYSTEMTAPDEPS@
Python/frozen.o: Python/importlib.h Python/importlib_external.h
@@ -724,6 +726,13 @@ Objects/typeobject.o: $(srcdir)/Objects/
Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py
$(PYTHON) $(srcdir)/Objects/typeslots.py < $(srcdir)/Include/typeslots.h > Objects/typeslots.inc
+# Only needed with --with-systemtap; not a public header:
+$(srcdir)/Python/pysystemtap.h: $(srcdir)/Python/pysystemtap.d
+ dtrace -o $@ $(DFLAGS) -C -h -s $(srcdir)/Python/pysystemtap.d
+
+Include/pydtrace.h: buildinclude $(srcdir)/Include/pydtrace.d
+ dtrace -o $@ $(DFLAGS) -C -h -s $(srcdir)/Include/pydtrace.d
+
+Python/ceval.o: Include/pydtrace.h
+
+Python/dtrace.o: buildinclude $(srcdir)/Include/pydtrace.d Python/ceval.o
+ dtrace -o $@ $(DFLAGS) -C -G -s $(srcdir)/Include/pydtrace.d Python/ceval.o
+Python/pysystemtap.o: $(srcdir)/Python/pysystemtap.d Python/ceval.o
+ dtrace -o $@ $(DFLAGS) -C -G -s $(srcdir)/Python/pysystemtap.d Python/ceval.o
+
############################################################################
# Header files
@@ -1251,7 +1264,7 @@ Python/thread.o: @THREADHEADERS@
.PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure
.PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools
.PHONY: frameworkaltinstallunixtools recheck autoconf clean clobber distclean
-.PHONY: smelly funny patchcheck touch altmaninstall
+.PHONY: smelly funny patchcheck touch altmaninstall buildinclude
.PHONY: gdbhooks
@@ -1345,6 +1354,7 @@ clean: pycremoval
-rm -f Lib/lib2to3/*Grammar*.pickle
-rm -f Programs/_testembed Programs/_freeze_importlib
-rm -rf build
+ -rm -f $(srcdir)/Python/pysystemtap.h
# IF YOU PUT ANYTHING HERE IT WILL GO AWAY
diff -up Python-2.7rc1/pyconfig.h.in.systemtap Python-2.7rc1/pyconfig.h.in
--- Python-2.7rc1/pyconfig.h.in.systemtap 2010-05-08 07:04:18.000000000 -0400
+++ Python-2.7rc1/pyconfig.h.in 2010-06-06 10:53:15.521974070 -0400
@@ -1074,6 +1074,9 @@
/* Define if you want documentation strings in extension modules */
#undef WITH_DOC_STRINGS
profile-removal:
find . -name '*.gc??' -exec rm -f {} ';'
diff -up Python-3.3.0rc2/pyconfig.h.in.systemtap Python-3.3.0rc2/pyconfig.h.in
--- Python-3.3.0rc2/pyconfig.h.in.systemtap 2012-09-09 05:11:14.000000000 -0400
+++ Python-3.3.0rc2/pyconfig.h.in 2012-09-10 09:17:21.120511781 -0400
@@ -1306,6 +1306,9 @@
/* Define if you want to compile in Python-specific mallocs */
#undef WITH_PYMALLOC
+/* Define if you want to compile in Dtrace support */
+#undef WITH_DTRACE
+/* Define if you want to compile in SystemTap support */
+#undef WITH_SYSTEMTAP
+
/* Define if you want to use the new-style (Openstep, Rhapsody, MacOS) dynamic
linker (dyld) instead of the old-style (NextStep) dynamic linker (rld).
Dyld is necessary to support frameworks. */
diff -up Python-2.7rc1/Python/ceval.c.systemtap Python-2.7rc1/Python/ceval.c
--- Python-2.7rc1/Python/ceval.c.systemtap 2010-05-09 10:46:46.000000000 -0400
+++ Python-2.7rc1/Python/ceval.c 2010-06-06 11:08:40.683100500 -0400
@@ -19,6 +19,10 @@
/* Define if you want to compile in rudimentary thread support */
#undef WITH_THREAD
diff -up Python-3.3.0rc2/Python/ceval.c.systemtap Python-3.3.0rc2/Python/ceval.c
--- Python-3.3.0rc2/Python/ceval.c.systemtap 2012-09-09 05:11:12.000000000 -0400
+++ Python-3.3.0rc2/Python/ceval.c 2012-09-10 09:17:21.122511781 -0400
@@ -18,6 +18,8 @@
#include <ctype.h>
+#ifdef WITH_DTRACE
+#include "pydtrace.h"
+#endif
+#include "ceval_systemtap.h"
+
#ifndef WITH_TSC
#define READ_TIMESTAMP(var)
@@ -671,6 +675,55 @@ PyEval_EvalCode(PyCodeObject *co, PyObje
NULL);
}
+#ifdef WITH_DTRACE
+static void
+dtrace_entry(PyFrameObject *f)
+{
+ const char *filename;
+ const char *fname;
+ int lineno;
+
+ filename = PyString_AsString(f->f_code->co_filename);
+ fname = PyString_AsString(f->f_code->co_name);
+ lineno = PyCode_Addr2Line(f->f_code, f->f_lasti);
+
+ PYTHON_FUNCTION_ENTRY((char *)filename, (char *)fname, lineno);
+
+ /*
+ * Currently a USDT tail-call will not receive the correct arguments.
+ * Disable the tail call here.
+ */
+#if defined(__sparc)
+ asm("nop");
+#endif
+}
+
+static void
+dtrace_return(PyFrameObject *f)
+{
+ const char *filename;
+ const char *fname;
+ int lineno;
+
+ filename = PyString_AsString(f->f_code->co_filename);
+ fname = PyString_AsString(f->f_code->co_name);
+ lineno = PyCode_Addr2Line(f->f_code, f->f_lasti);
+ PYTHON_FUNCTION_RETURN((char *)filename, (char *)fname, lineno);
+
+ /*
+ * Currently a USDT tail-call will not receive the correct arguments.
+ * Disable the tail call here.
+ */
+#if defined(__sparc)
+ asm("nop");
+#endif
+}
+#else
+#define PYTHON_FUNCTION_ENTRY_ENABLED() 0
+#define PYTHON_FUNCTION_RETURN_ENABLED() 0
+#define dtrace_entry(f)
+#define dtrace_return(f)
+#endif
/* Interpreter main loop */
@@ -909,6 +962,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int
@@ -1160,6 +1162,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int
}
}
+ if (PYTHON_FUNCTION_ENTRY_ENABLED())
+ dtrace_entry(f);
+ if (PYTHON_FUNCTION_ENTRY_ENABLED()) {
+ systemtap_function_entry(f);
+ }
+
co = f->f_code;
names = co->co_names;
consts = co->co_consts;
@@ -3000,6 +3056,9 @@ fast_yield:
@@ -3077,6 +3083,11 @@ fast_yield:
/* pop frame */
exit_eval_frame:
+ if (PYTHON_FUNCTION_RETURN_ENABLED())
+ dtrace_return(f);
+
+ if (PYTHON_FUNCTION_RETURN_ENABLED()) {
+ systemtap_function_return(f);
+ }
+
Py_LeaveRecursiveCall();
f->f_executing = 0;
tstate->frame = f->f_back;
diff -up Python-3.3.0rc2/Python/ceval_systemtap.h.systemtap Python-3.3.0rc2/Python/ceval_systemtap.h
--- Python-3.3.0rc2/Python/ceval_systemtap.h.systemtap 2012-09-10 09:17:21.122511781 -0400
+++ Python-3.3.0rc2/Python/ceval_systemtap.h 2012-09-10 09:17:21.122511781 -0400
@@ -0,0 +1,86 @@
+/*
+ Support for SystemTap static markers
+*/
+
+#ifdef WITH_SYSTEMTAP
+
+#include "pysystemtap.h"
+
+/*
+ A struct to hold all of the information gathered when one of the traceable
+ markers is triggered
+*/
+struct frame_marker_info
+{
+ PyObject *filename_obj;
+ PyObject *funcname_obj;
+ const char *filename;
+ const char *funcname;
+ int lineno;
+};
+
+static void
+get_frame_marker_info(PyFrameObject *f, struct frame_marker_info *fmi)
+{
+ PyObject *ptype;
+ PyObject *pvalue;
+ PyObject *ptraceback;
+
+ PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+
+ fmi->filename_obj = PyUnicode_EncodeFSDefault(f->f_code->co_filename);
+ if (fmi->filename_obj) {
+ fmi->filename = PyBytes_AsString(fmi->filename_obj);
+ } else {
+ fmi->filename = NULL;
+ }
+
+ fmi->funcname_obj = PyUnicode_AsUTF8String(f->f_code->co_name);
+ if (fmi->funcname_obj) {
+ fmi->funcname = PyBytes_AsString(fmi->funcname_obj);
+ } else {
+ fmi->funcname = NULL;
+ }
+
+ fmi->lineno = PyCode_Addr2Line(f->f_code, f->f_lasti);
+
+ PyErr_Restore(ptype, pvalue, ptraceback);
+
+}
+
+static void
+release_frame_marker_info(struct frame_marker_info *fmi)
+{
+ Py_XDECREF(fmi->filename_obj);
+ Py_XDECREF(fmi->funcname_obj);
+}
+
+static void
+systemtap_function_entry(PyFrameObject *f)
+{
+ struct frame_marker_info fmi;
+ get_frame_marker_info(f, &fmi);
+ PYTHON_FUNCTION_ENTRY(fmi.filename, fmi.funcname, fmi.lineno, f);
+ release_frame_marker_info(&fmi);
+}
+
+static void
+systemtap_function_return(PyFrameObject *f)
+{
+ struct frame_marker_info fmi;
+ get_frame_marker_info(f, &fmi);
+ PYTHON_FUNCTION_RETURN(fmi.filename, fmi.funcname, fmi.lineno, f);
+ release_frame_marker_info(&fmi);
+}
+
+#else /* #ifdef WITH_SYSTEMTAP */
+
+/*
+ When configured --without-systemtap, everything compiles away to nothing:
+*/
+#define PYTHON_FUNCTION_ENTRY_ENABLED() 0
+#define PYTHON_FUNCTION_RETURN_ENABLED() 0
+#define systemtap_function_entry(f)
+#define systemtap_function_return(f)
+
+#endif
diff -up Python-3.3.0rc2/Python/pysystemtap.d.systemtap Python-3.3.0rc2/Python/pysystemtap.d
--- Python-3.3.0rc2/Python/pysystemtap.d.systemtap 2012-09-10 09:17:21.122511781 -0400
+++ Python-3.3.0rc2/Python/pysystemtap.d 2012-09-10 09:17:21.122511781 -0400
@@ -0,0 +1,4 @@
+provider python {
+ probe function__entry(const char *, const char *, int, PyFrameObject *);
+ probe function__return(const char *, const char *, int, PyFrameObject *);
+};

46
SOURCES/00111-no-static-lib.patch
View File

@ -1,19 +1,20 @@
diff -up Python-2.7.3/Makefile.pre.in.no-static-lib Python-2.7.3/Makefile.pre.in
--- Python-2.7.3/Makefile.pre.in.no-static-lib 2013-02-19 14:03:40.801993224 -0500
+++ Python-2.7.3/Makefile.pre.in 2013-02-19 14:04:44.070988898 -0500
@@ -397,7 +397,7 @@ coverage:
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 4b093e3..1088435 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -543,7 +543,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c
$(PYTHON_FOR_REGEN) ./Tools/clinic/clinic.py --make
# Build the interpreter
-$(BUILDPYTHON): Modules/python.o $(LIBRARY) $(LDLIBRARY)
+$(BUILDPYTHON): Modules/python.o $(LDLIBRARY)
$(LINKCC) $(CFLAGS) $(LDFLAGS) $(LINKFORSHARED) -o $@ \
Modules/python.o \
$(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
@@ -413,18 +413,6 @@ sharedmods: $(BUILDPYTHON)
$(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \
-$(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY)
+$(BUILDPYTHON): Programs/python.o $(LDLIBRARY) $(PY3LIBRARY)
$(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
platform: $(BUILDPYTHON) pybuilddir.txt
@@ -588,18 +588,6 @@ sharedmods: $(BUILDPYTHON) pybuilddir.txt Modules/_math.o
$(PYTHON_FOR_BUILD) $(srcdir)/setup.py $$quiet build
-# Build static library
-# avoid long command lines, same as LIBRARY_OBJS
-$(LIBRARY): $(LIBRARY_OBJS)
@ -21,21 +22,30 @@ diff -up Python-2.7.3/Makefile.pre.in.no-static-lib Python-2.7.3/Makefile.pre.in
- $(AR) $(ARFLAGS) $@ Modules/getbuildinfo.o
- $(AR) $(ARFLAGS) $@ $(PARSER_OBJS)
- $(AR) $(ARFLAGS) $@ $(OBJECT_OBJS)
- $(AR) $(ARFLAGS) $@ $(PYTHON_OBJS)
- $(AR) $(ARFLAGS) $@ $(MODULE_OBJS) $(SIGNAL_OBJS)
- $(AR) $(ARFLAGS) $@ $(PYTHON_OBJS) Python/frozen.o
- $(AR) $(ARFLAGS) $@ $(MODULE_OBJS)
- $(AR) $(ARFLAGS) $@ $(MODOBJS)
- $(RANLIB) $@
-
libpython$(VERSION).so: $(LIBRARY_OBJS)
libpython$(LDVERSION).so: $(LIBRARY_OBJS)
if test $(INSTSONAME) != $(LDLIBRARY); then \
$(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) $(LDLAST); \
@@ -1021,18 +1009,6 @@ libainstall: all python-config
@@ -689,7 +677,7 @@ Modules/Setup: $(srcdir)/Modules/Setup.dist
echo "-----------------------------------------------"; \
fi
-Programs/_testembed: Programs/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY)
+Programs/_testembed: Programs/_testembed.o $(LDLIBRARY) $(PY3LIBRARY)
$(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
############################################################################
@@ -1425,18 +1413,6 @@ libainstall: @DEF_MAKE_RULE@ python-config
else true; \
fi; \
done
- @if test -d $(LIBRARY); then :; else \
- if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \
- if test "$(SO)" = .dll; then \
- if test "$(SHLIB_SUFFIX)" = .dll; then \
- $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBPL) ; \
- else \
- $(INSTALL_DATA) $(LIBRARY) $(DESTDIR)$(LIBPL)/$(LIBRARY) ; \
@ -46,5 +56,5 @@ diff -up Python-2.7.3/Makefile.pre.in.no-static-lib Python-2.7.3/Makefile.pre.in
- fi; \
- fi
$(INSTALL_DATA) Modules/config.c $(DESTDIR)$(LIBPL)/config.c
$(INSTALL_DATA) Modules/python.o $(DESTDIR)$(LIBPL)/python.o
$(INSTALL_DATA) Programs/python.o $(DESTDIR)$(LIBPL)/python.o
$(INSTALL_DATA) $(srcdir)/Modules/config.c.in $(DESTDIR)$(LIBPL)/config.c.in

54
SOURCES/00132-add-rpmbuild-hooks-to-unittest.patch
View File

@ -1,17 +1,16 @@
diff -up Python-2.7.2/Lib/unittest/case.py.add-rpmbuild-hooks-to-unittest Python-2.7.2/Lib/unittest/case.py
--- Python-2.7.2/Lib/unittest/case.py.add-rpmbuild-hooks-to-unittest 2011-09-08 14:45:47.677169191 -0400
+++ Python-2.7.2/Lib/unittest/case.py 2011-09-08 16:01:36.287858159 -0400
@@ -1,6 +1,7 @@
"""Test case implementation"""
import collections
+import os
diff -up Python-3.2.2/Lib/unittest/case.py.add-rpmbuild-hooks-to-unittest Python-3.2.2/Lib/unittest/case.py
--- Python-3.2.2/Lib/unittest/case.py.add-rpmbuild-hooks-to-unittest 2011-09-03 12:16:44.000000000 -0400
+++ Python-3.2.2/Lib/unittest/case.py 2011-09-09 06:35:16.365568382 -0400
@@ -3,6 +3,7 @@
import sys
import functools
import difflib
@@ -94,6 +95,43 @@ def expectedFailure(func):
return wrapper
+import os
import logging
import pprint
import re
@@ -101,5 +102,21 @@ def expectedFailure(func):
raise self.test_case.failureException(msg)
+# Non-standard/downstream-only hooks for handling issues with specific test
+# cases:
@ -29,40 +28,19 @@ diff -up Python-2.7.2/Lib/unittest/case.py.add-rpmbuild-hooks-to-unittest Python
+ else:
+ return _id
+
+def _expectedFailureInRpmBuild(func):
+ """
+ Non-standard/downstream-only decorator for marking a specific unit test
+ as expected to fail within the %check of an rpmbuild.
+
+ Specifically, this takes effect when WITHIN_PYTHON_RPM_BUILD is set within
+ the environment, and has no effect otherwise.
+ """
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ if 'WITHIN_PYTHON_RPM_BUILD' in os.environ:
+ try:
+ func(*args, **kwargs)
+ except Exception:
+ raise _ExpectedFailure(sys.exc_info())
+ raise _UnexpectedSuccess
+ else:
+ # Call directly:
+ func(*args, **kwargs)
+ return wrapper
+
class _AssertRaisesContext(object):
"""A context manager used to implement TestCase.assertRaises* methods."""
class _AssertRaisesBaseContext(_BaseTestCaseContext):
diff -up Python-2.7.2/Lib/unittest/__init__.py.add-rpmbuild-hooks-to-unittest Python-2.7.2/Lib/unittest/__init__.py
--- Python-2.7.2/Lib/unittest/__init__.py.add-rpmbuild-hooks-to-unittest 2011-09-08 14:59:39.534112310 -0400
+++ Python-2.7.2/Lib/unittest/__init__.py 2011-09-08 15:07:09.191081562 -0400
def __init__(self, expected, test_case, expected_regex=None):
diff -up Python-3.2.2/Lib/unittest/__init__.py.add-rpmbuild-hooks-to-unittest Python-3.2.2/Lib/unittest/__init__.py
--- Python-3.2.2/Lib/unittest/__init__.py.add-rpmbuild-hooks-to-unittest 2011-09-03 12:16:44.000000000 -0400
+++ Python-3.2.2/Lib/unittest/__init__.py 2011-09-09 06:35:16.366568382 -0400
@@ -57,7 +57,8 @@ __unittest = True
from .result import TestResult
from .case import (TestCase, FunctionTestCase, SkipTest, skip, skipIf,
- skipUnless, expectedFailure)
+ skipUnless, expectedFailure,
+ _skipInRpmBuild, _expectedFailureInRpmBuild)
+ _skipInRpmBuild)
from .suite import BaseTestSuite, TestSuite
from .loader import (TestLoader, defaultTestLoader, makeSuite, getTestCaseNames,
findTestCases)

12
SOURCES/00137-skip-distutils-tests-that-fail-in-rpmbuild.patch
View File

@ -1,12 +1,12 @@
diff -up Python-2.7.3/Lib/distutils/tests/test_bdist_rpm.py.mark-tests-that-fail-in-rpmbuild Python-2.7.3/Lib/distutils/tests/test_bdist_rpm.py
--- Python-2.7.3/Lib/distutils/tests/test_bdist_rpm.py.mark-tests-that-fail-in-rpmbuild 2012-04-09 19:07:29.000000000 -0400
+++ Python-2.7.3/Lib/distutils/tests/test_bdist_rpm.py 2012-04-13 00:20:08.223819263 -0400
@@ -24,6 +24,7 @@ setup(name='foo', version='0.1', py_modu
diff -up Python-3.2.2/Lib/distutils/tests/test_bdist_rpm.py.skip-distutils-tests-that-fail-in-rpmbuild Python-3.2.2/Lib/distutils/tests/test_bdist_rpm.py
--- Python-3.2.2/Lib/distutils/tests/test_bdist_rpm.py.skip-distutils-tests-that-fail-in-rpmbuild 2011-09-03 12:16:40.000000000 -0400
+++ Python-3.2.2/Lib/distutils/tests/test_bdist_rpm.py 2011-09-10 05:04:56.328852558 -0400
@@ -23,6 +23,7 @@ setup(name='foo', version='0.1', py_modu
"""
+@unittest._skipInRpmBuild("don't try to nest one rpm build inside another rpm build")
class BuildRpmTestCase(support.TempdirManager,
support.EnvironGuard,
support.LoggingSilencer,
unittest.TestCase):
diff -up Python-2.7.3/Lib/distutils/tests/test_build_ext.py.mark-tests-that-fail-in-rpmbuild Python-2.7.3/Lib/distutils/tests/test_build_ext.py
diff -up Python-3.2.2/Lib/distutils/tests/test_build_ext.py.skip-distutils-tests-that-fail-in-rpmbuild Python-3.2.2/Lib/distutils/tests/test_build_ext.py

547
SOURCES/00146-hashlib-fips.patch
View File

@ -1,82 +1,48 @@
diff -up Python-2.7.2/Lib/hashlib.py.hashlib-fips Python-2.7.2/Lib/hashlib.py
--- Python-2.7.2/Lib/hashlib.py.hashlib-fips 2011-06-11 11:46:24.000000000 -0400
+++ Python-2.7.2/Lib/hashlib.py 2011-09-14 00:21:26.194252001 -0400
@@ -6,9 +6,12 @@
__doc__ = """hashlib module - A common interface to many hash functions.
-new(name, string='') - returns a new hash object implementing the
- given hash function; initializing the hash
- using the given string data.
+new(name, string='', usedforsecurity=True)
+ - returns a new hash object implementing the given hash function;
+ initializing the hash using the given string data.
+
+ "usedforsecurity" is a non-standard extension for better supporting
+ FIPS-compliant environments (see below)
Named constructor functions are also available, these are much faster
than using new():
@@ -24,6 +27,20 @@ the zlib module.
diff --git a/Lib/hashlib.py b/Lib/hashlib.py
index 316cece..b7ad879 100644
--- a/Lib/hashlib.py
+++ b/Lib/hashlib.py
@@ -23,6 +23,16 @@ the zlib module.
Choose your hash function wisely. Some have known collision weaknesses.
sha384 and sha512 will be slow on 32 bit platforms.
+Our implementation of hashlib uses OpenSSL.
+
+OpenSSL has a "FIPS mode", which, if enabled, may restrict the available hashes
+to only those that are compliant with FIPS regulations. For example, it may
+deny the use of MD5, on the grounds that this is not secure for uses such as
+authentication, system integrity checking, or digital signatures.
+
+If you need to use such a hash for non-security purposes (such as indexing into
+a data structure for speed), you can override the keyword argument
+"usedforsecurity" from True to False to signify that your code is not relying
+on the hash for security purposes, and this will allow the hash to be usable
+even in FIPS mode. This is not a standard feature of Python 2.7's hashlib, and
+is included here to better support FIPS mode.
+If the underlying implementation supports "FIPS mode", and this is enabled, it
+may restrict the available hashes to only those that are compliant with FIPS
+regulations. For example, it may deny the use of MD5, on the grounds that this
+is not secure for uses such as authentication, system integrity checking, or
+digital signatures. If you need to use such a hash for non-security purposes
+(such as indexing into a data structure for speed), you can override the keyword
+argument "usedforsecurity" from True to False to signify that your code is not
+relying on the hash for security purposes, and this will allow the hash to be
+usable even in FIPS mode.
+
Hash objects have these methods:
- update(arg): Update the hash object with the string arg. Repeated calls
- update(arg): Update the hash object with the bytes in arg. Repeated calls
are equivalent to a single call with the concatenation of all
@@ -63,74 +80,39 @@ algorithms = __always_supported
__all__ = __always_supported + ('new', 'algorithms')
@@ -62,6 +72,18 @@ algorithms_available = set(__always_supported)
__all__ = __always_supported + ('new', 'algorithms_guaranteed',
'algorithms_available', 'pbkdf2_hmac')
+import functools
+def __ignore_usedforsecurity(func):
+ """Used for sha3_* functions. Until OpenSSL implements them, we want
+ to use them from Python _sha3 module, but we want them to accept
+ usedforsecurity argument too."""
+ # TODO: remove this function when OpenSSL implements sha3
+ @functools.wraps(func)
+ def inner(*args, **kwargs):
+ if 'usedforsecurity' in kwargs:
+ kwargs.pop('usedforsecurity')
+ return func(*args, **kwargs)
+ return inner
-def __get_builtin_constructor(name):
- try:
- if name in ('SHA1', 'sha1'):
- import _sha
- return _sha.new
- elif name in ('MD5', 'md5'):
- import _md5
- return _md5.new
- elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'):
- import _sha256
- bs = name[3:]
- if bs == '256':
- return _sha256.sha256
- elif bs == '224':
- return _sha256.sha224
- elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'):
- import _sha512
- bs = name[3:]
- if bs == '512':
- return _sha512.sha512
- elif bs == '384':
- return _sha512.sha384
- except ImportError:
- pass # no extension module, this hash is unsupported.
-
- raise ValueError('unsupported hash type ' + name)
-
-
def __get_openssl_constructor(name):
try:
__builtin_constructor_cache = {}
@@ -100,31 +122,39 @@ def __get_openssl_constructor(name):
f = getattr(_hashlib, 'openssl_' + name)
# Allow the C module to raise ValueError. The function will be
# defined but the hash not actually available thanks to OpenSSL.
- f()
+ #
+ # We pass "usedforsecurity=False" to disable FIPS-based restrictions:
+ # at this stage we're merely seeing if the function is callable,
+ # rather than using it for actual work.
@ -84,61 +50,76 @@ diff -up Python-2.7.2/Lib/hashlib.py.hashlib-fips Python-2.7.2/Lib/hashlib.py
# Use the C function directly (very fast)
return f
except (AttributeError, ValueError):
- return __get_builtin_constructor(name)
+ raise
+ # TODO: We want to just raise here when OpenSSL implements sha3
+ # because we want to make sure that Fedora uses everything from OpenSSL
return __get_builtin_constructor(name)
-
-def __py_new(name, string=''):
- """new(name, string='') - Return a new hashing object using the named algorithm;
- optionally initialized with a string.
- """
- return __get_builtin_constructor(name)(string)
-
-
-def __hash_new(name, string=''):
+def __hash_new(name, string='', usedforsecurity=True):
"""new(name, string='') - Return a new hashing object using the named algorithm;
optionally initialized with a string.
-def __py_new(name, data=b''):
- """new(name, data=b'') - Return a new hashing object using the named algorithm;
- optionally initialized with data (which must be bytes).
+def __py_new(name, data=b'', usedforsecurity=True):
+ """new(name, data=b'', usedforsecurity=True) - Return a new hashing object using
+ the named algorithm; optionally initialized with data (which must be bytes).
+ The 'usedforsecurity' keyword argument does nothing, and is for compatibilty
+ with the OpenSSL implementation
"""
return __get_builtin_constructor(name)(data)
-def __hash_new(name, data=b''):
- """new(name, data=b'') - Return a new hashing object using the named algorithm;
- optionally initialized with data (which must be bytes).
+def __hash_new(name, data=b'', usedforsecurity=True):
+ """new(name, data=b'', usedforsecurity=True) - Return a new hashing object using
+ the named algorithm; optionally initialized with data (which must be bytes).
+
+ Override 'usedforsecurity' to False when using for non-security purposes in
+ a FIPS environment
"""
try:
- return _hashlib.new(name, string)
+ return _hashlib.new(name, string, usedforsecurity)
- return _hashlib.new(name, data)
+ return _hashlib.new(name, data, usedforsecurity)
except ValueError:
- # If the _hashlib module (OpenSSL) doesn't support the named
- # hash, try using our builtin implementations.
- # This allows for SHA224/256 and SHA384/512 support even though
- # the OpenSSL library prior to 0.9.8 doesn't provide them.
- return __get_builtin_constructor(name)(string)
-
+ raise
+ # TODO: We want to just raise here when OpenSSL implements sha3
+ # because we want to make sure that Fedora uses everything from OpenSSL
return __get_builtin_constructor(name)(data)
try:
import _hashlib
new = __hash_new
__get_hash = __get_openssl_constructor
except ImportError:
- new = __py_new
- __get_hash = __get_builtin_constructor
+ # We don't build the legacy modules
+ raise
for __func_name in __always_supported:
@@ -207,7 +237,10 @@ for __func_name in __always_supported:
# try them all, some may not work due to the OpenSSL
@@ -143,4 +125,4 @@ for __func_name in __always_supported:
# version not supporting that algorithm.
try:
- globals()[__func_name] = __get_hash(__func_name)
+ func = __get_hash(__func_name)
+ if 'sha3_' in __func_name:
+ func = __ignore_usedforsecurity(func)
+ globals()[__func_name] = func
except ValueError:
import logging
logging.exception('code for hash %s was not found.', __func_name)
@@ -215,3 +248,4 @@ for __func_name in __always_supported:
# Cleanup locals()
del __always_supported, __func_name, __get_hash
-del __py_new, __hash_new, __get_openssl_constructor
+del __hash_new, __get_openssl_constructor
diff -up Python-2.7.2/Lib/test/test_hashlib.py.hashlib-fips Python-2.7.2/Lib/test/test_hashlib.py
--- Python-2.7.2/Lib/test/test_hashlib.py.hashlib-fips 2011-06-11 11:46:25.000000000 -0400
+++ Python-2.7.2/Lib/test/test_hashlib.py 2011-09-14 01:08:55.525254195 -0400
@@ -32,6 +32,19 @@ def hexstr(s):
r = r + h[(i >> 4) & 0xF] + h[i & 0xF]
return r
del __py_new, __hash_new, __get_openssl_constructor
+del __ignore_usedforsecurity
\ No newline at end of file
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index c9b113e..60e2392 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -24,7 +24,22 @@ from test.support import _4G, bigmemtest, import_fresh_module
COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount')
c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib'])
-py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib'])
+# skipped on Fedora, since we always use OpenSSL implementation
+# py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib'])
+
+def openssl_enforces_fips():
+ # Use the "openssl" command (if present) to try to determine if the local
+ # OpenSSL is configured to enforce FIPS
@ -153,122 +134,80 @@ diff -up Python-2.7.2/Lib/test/test_hashlib.py.hashlib-fips Python-2.7.2/Lib/tes
+ return b'unknown cipher' in stderr
+OPENSSL_ENFORCES_FIPS = openssl_enforces_fips()
def hexstr(s):
assert isinstance(s, bytes), repr(s)
@@ -34,6 +49,16 @@ def hexstr(s):
r += h[(i >> 4) & 0xF] + h[i & 0xF]
return r
+# hashlib and _hashlib-based functions support a "usedforsecurity" keyword
+# argument, and FIPS mode requires that it be used overridden with a False
+# value for these selftests to work. Other cryptographic code within Python
+# doesn't support this keyword.
+# Modify a function to one in which "usedforsecurity=False" is added to the
+# keyword arguments:
+def suppress_fips(f):
+ def g(*args, **kwargs):
+ return f(*args, usedforsecurity=False, **kwargs)
+ return g
class HashLibTestCase(unittest.TestCase):
supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1',
@@ -61,10 +74,10 @@ class HashLibTestCase(unittest.TestCase)
@@ -63,11 +88,11 @@ class HashLibTestCase(unittest.TestCase):
# For each algorithm, test the direct constructor and the use
# of hashlib.new given the algorithm name.
for algorithm, constructors in self.constructors_to_test.items():
constructors.add(getattr(hashlib, algorithm))
- def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm):
+ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, usedforsecurity=True):
- constructors.add(getattr(hashlib, algorithm))
+ constructors.add(suppress_fips(getattr(hashlib, algorithm)))
def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm):
if data is None:
- return hashlib.new(_alg)
- return hashlib.new(_alg, data)
+ return hashlib.new(_alg, usedforsecurity=usedforsecurity)
+ return hashlib.new(_alg, data, usedforsecurity=usedforsecurity)
+ return suppress_fips(hashlib.new)(_alg)
+ return suppress_fips(hashlib.new)(_alg, data)
constructors.add(_test_algorithm_via_hashlib_new)
_hashlib = self._conditional_import_module('_hashlib')
@@ -78,28 +91,13 @@ class HashLibTestCase(unittest.TestCase)
@@ -79,27 +104,12 @@ class HashLibTestCase(unittest.TestCase):
for algorithm, constructors in self.constructors_to_test.items():
constructor = getattr(_hashlib, 'openssl_'+algorithm, None)
if constructor:
constructors.add(constructor)
- constructors.add(constructor)
+ constructors.add(suppress_fips(constructor))
def add_builtin_constructor(name):
constructor = getattr(hashlib, "__get_builtin_constructor")(name)
self.constructors_to_test[name].add(constructor)
- _md5 = self._conditional_import_module('_md5')
- if _md5:
- self.constructors_to_test['md5'].add(_md5.new)
- _sha = self._conditional_import_module('_sha')
- if _sha:
- self.constructors_to_test['sha1'].add(_sha.new)
- add_builtin_constructor('md5')
- _sha1 = self._conditional_import_module('_sha1')
- if _sha1:
- add_builtin_constructor('sha1')
- _sha256 = self._conditional_import_module('_sha256')
- if _sha256:
- self.constructors_to_test['sha224'].add(_sha256.sha224)
- self.constructors_to_test['sha256'].add(_sha256.sha256)
- add_builtin_constructor('sha224')
- add_builtin_constructor('sha256')
- _sha512 = self._conditional_import_module('_sha512')
- if _sha512:
- self.constructors_to_test['sha384'].add(_sha512.sha384)
- self.constructors_to_test['sha512'].add(_sha512.sha512)
- add_builtin_constructor('sha384')
- add_builtin_constructor('sha512')
-
super(HashLibTestCase, self).__init__(*args, **kwargs)
def test_hash_array(self):
a = array.array("b", range(10))
constructors = self.constructors_to_test.itervalues()
for cons in itertools.chain.from_iterable(constructors):
- c = cons(a)
+ c = cons(a, usedforsecurity=False)
c.hexdigest()
@property
@@ -148,9 +158,6 @@ class HashLibTestCase(unittest.TestCase):
else:
del sys.modules['_md5']
self.assertRaises(TypeError, get_builtin_constructor, 3)
- constructor = get_builtin_constructor('md5')
- self.assertIs(constructor, _md5.md5)
- self.assertEqual(sorted(builtin_constructor_cache), ['MD5', 'md5'])
def test_algorithms_attribute(self):
@@ -115,28 +113,9 @@ class HashLibTestCase(unittest.TestCase)
self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam')
self.assertRaises(TypeError, hashlib.new, 1)
- def test_get_builtin_constructor(self):
- get_builtin_constructor = hashlib.__dict__[
- '__get_builtin_constructor']
- self.assertRaises(ValueError, get_builtin_constructor, 'test')
- try:
- import _md5
- except ImportError:
- pass
- # This forces an ImportError for "import _md5" statements
- sys.modules['_md5'] = None
- try:
- self.assertRaises(ValueError, get_builtin_constructor, 'md5')
- finally:
- if '_md5' in locals():
- sys.modules['_md5'] = _md5
- else:
- del sys.modules['_md5']
- self.assertRaises(TypeError, get_builtin_constructor, 3)
-
def test_hexdigest(self):
for name in self.supported_hash_names:
- h = hashlib.new(name)
+ h = hashlib.new(name, usedforsecurity=False)
self.assertTrue(hexstr(h.digest()) == h.hexdigest())
def test_large_update(self):
@@ -145,16 +125,16 @@ class HashLibTestCase(unittest.TestCase)
abcs = aas + bees + cees
for name in self.supported_hash_names:
- m1 = hashlib.new(name)
+ m1 = hashlib.new(name, usedforsecurity=False)
m1.update(aas)
m1.update(bees)
m1.update(cees)
- m2 = hashlib.new(name)
+ m2 = hashlib.new(name, usedforsecurity=False)
m2.update(abcs)
self.assertEqual(m1.digest(), m2.digest(), name+' update problem.')
- m3 = hashlib.new(name, abcs)
+ m3 = hashlib.new(name, abcs, usedforsecurity=False)
self.assertEqual(m1.digest(), m3.digest(), name+' new problem.')
def check(self, name, data, digest):
@@ -162,7 +142,7 @@ class HashLibTestCase(unittest.TestCase)
# 2 is for hashlib.name(...) and hashlib.new(name, ...)
self.assertGreaterEqual(len(constructors), 2)
for hash_object_constructor in constructors:
- computed = hash_object_constructor(data).hexdigest()
+ computed = hash_object_constructor(data, usedforsecurity=False).hexdigest()
self.assertEqual(
computed, digest,
"Hash algorithm %s constructed using %s returned hexdigest"
@@ -172,7 +152,8 @@ class HashLibTestCase(unittest.TestCase)
def check_unicode(self, algorithm_name):
# Unicode objects are not allowed as input.
- expected = hashlib.new(algorithm_name, str(u'spam')).hexdigest()
+ expected = hashlib.new(algorithm_name, str(u'spam'),
+ usedforsecurity=False).hexdigest()
self.check(algorithm_name, u'spam', expected)
def test_unicode(self):
@@ -354,6 +335,70 @@ class HashLibTestCase(unittest.TestCase)
for cons in self.hash_constructors:
@@ -433,6 +440,64 @@ class HashLibTestCase(unittest.TestCase):
self.assertEqual(expected_hash, hasher.hexdigest())
@ -288,70 +227,74 @@ diff -up Python-2.7.2/Lib/test/test_hashlib.py.hashlib-fips Python-2.7.2/Lib/tes
+ m = hashlib.new('md5', b'abc\n', usedforsecurity=False)
+ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+ def assertRaisesUnknownCipher(self, callable_obj=None, *args, **kwargs):
+ try:
+ callable_obj(*args, **kwargs)
+ except ValueError, e:
+ if not e.args[0].endswith('unknown cipher'):
+ self.fail('Incorrect exception raised')
+ else:
+ self.fail('Exception was not raised')
+
+ @unittest.skipUnless(OPENSSL_ENFORCES_FIPS,
+ 'FIPS enforcement required for this test.')
+ def test_hashlib_fips_mode(self):
+ # Ensure that we raise a ValueError on vanilla attempts to use MD5
+ # in hashlib in a FIPS-enforced setting:
+ self.assertRaisesUnknownCipher(hashlib.md5)
+ self.assertRaisesUnknownCipher(hashlib.new, 'md5')
+ with self.assertRaisesRegexp(ValueError, '.*unknown cipher'):
+ m = hashlib.md5()
+
+ if not self._conditional_import_module('_md5'):
+ with self.assertRaisesRegexp(ValueError, '.*unknown cipher'):
+ m = hashlib.new('md5')
+
+ @unittest.skipUnless(OPENSSL_ENFORCES_FIPS,
+ 'FIPS enforcement required for this test.')
+ def test_hashopenssl_fips_mode(self):
+ # Verify the _hashlib module's handling of md5:
+ import _hashlib
+
+ _hashlib = self._conditional_import_module('_hashlib')
+ if _hashlib:
+ assert hasattr(_hashlib, 'openssl_md5')
+
+ # Ensure that _hashlib raises a ValueError on vanilla attempts to
+ # use MD5 in a FIPS-enforced setting:
+ self.assertRaisesUnknownCipher(_hashlib.openssl_md5)
+ self.assertRaisesUnknownCipher(_hashlib.new, 'md5')
+ with self.assertRaisesRegexp(ValueError, '.*unknown cipher'):
+ m = _hashlib.openssl_md5()
+ with self.assertRaisesRegexp(ValueError, '.*unknown cipher'):
+ m = _hashlib.new('md5')
+
+ # Ensure that in such a setting we can whitelist a callsite with
+ # usedforsecurity=False and have it succeed:
+ m = _hashlib.openssl_md5(usedforsecurity=False)
+ m.update('abc\n')
+ m.update(b'abc\n')
+ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+ m = _hashlib.new('md5', usedforsecurity=False)
+ m.update('abc\n')
+ m.update(b'abc\n')
+ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+ m = _hashlib.openssl_md5('abc\n', usedforsecurity=False)
+ m = _hashlib.openssl_md5(b'abc\n', usedforsecurity=False)
+ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+ m = _hashlib.new('md5', 'abc\n', usedforsecurity=False)
+ m = _hashlib.new('md5', b'abc\n', usedforsecurity=False)
+ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1")
+
+
+
def test_main():
test_support.run_unittest(HashLibTestCase)
diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_hashopenssl.c
--- Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips 2011-06-11 11:46:26.000000000 -0400
+++ Python-2.7.2/Modules/_hashopenssl.c 2011-09-14 00:21:26.199252001 -0400
@@ -36,6 +36,8 @@
#endif
class KDFTests(unittest.TestCase):
@@ -516,7 +581,7 @@ class KDFTests(unittest.TestCase):
out = pbkdf2(hash_name='sha1', password=b'password', salt=b'salt',
iterations=1, dklen=None)
self.assertEqual(out, self.pbkdf2_results['sha1'][0][0])
-
+ @unittest.skip('skipped on Fedora, as we always use OpenSSL pbkdf2_hmac')
def test_pbkdf2_hmac_py(self):
self._test_pbkdf2_hmac(py_hashlib.pbkdf2_hmac)
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 44765ac..b8cf490 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -20,6 +20,8 @@
/* EVP is the preferred interface to hashing in OpenSSL */
+#include <openssl/ssl.h>
+#include <openssl/err.h>
#include <openssl/evp.h>
#define MUNCH_SIZE INT_MAX
@@ -65,11 +67,19 @@ typedef struct {
#include <openssl/hmac.h>
/* We use the object interface to discover what hashes OpenSSL supports. */
@@ -45,11 +47,19 @@ typedef struct {
static PyTypeObject EVPtype;
@ -375,7 +318,7 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
DEFINE_CONSTS_FOR_NEW(md5)
DEFINE_CONSTS_FOR_NEW(sha1)
@@ -115,6 +125,48 @@ EVP_hash(EVPobject *self, const void *vp
@@ -92,6 +102,48 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len)
}
}
@ -406,7 +349,7 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
+ errstr = ERR_error_string(ERR_peek_last_error(), NULL);
+ ERR_clear_error();
+
+ return PyString_FromString(errstr); /* Can be NULL */
+ return PyUnicode_FromString(errstr); /* Can be NULL */
+}
+
+static void
@ -424,26 +367,27 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
/* Internal methods for a hash object */
static void
@@ -313,14 +365,15 @@ EVP_repr(PyObject *self)
@@ -259,15 +311,16 @@ EVP_repr(EVPobject *self)
static int
EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds)
{
- static char *kwlist[] = {"name", "string", NULL};
+ static char *kwlist[] = {"name", "string", "usedforsecurity", NULL};
PyObject *name_obj = NULL;
PyObject *data_obj = NULL;
+ int usedforsecurity = 1;
Py_buffer view = { 0 };
Py_buffer view;
char *nameStr;
const EVP_MD *digest;
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s*:HASH", kwlist,
- &name_obj, &view)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s*i:HASH", kwlist,
+ &name_obj, &view, &usedforsecurity)) {
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:HASH", kwlist,
- &name_obj, &data_obj)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Oi:HASH", kwlist,
+ &name_obj, &data_obj, &usedforsecurity)) {
return -1;
}
@@ -336,7 +389,12 @@ EVP_tp_init(EVPobject *self, PyObject *a
@@ -288,7 +341,12 @@ EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds)
PyBuffer_Release(&view);
return -1;
}
@ -457,7 +401,7 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
self->name = name_obj;
Py_INCREF(self->name);
@@ -420,7 +478,8 @@ static PyTypeObject EVPtype = {
@@ -372,7 +430,8 @@ static PyTypeObject EVPtype = {
static PyObject *
EVPnew(PyObject *name_obj,
const EVP_MD *digest, const EVP_MD_CTX *initial_ctx,
@ -467,7 +411,7 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
{
EVPobject *self;
@@ -435,7 +494,12 @@ EVPnew(PyObject *name_obj,
@@ -387,7 +446,12 @@ EVPnew(PyObject *name_obj,
if (initial_ctx) {
EVP_MD_CTX_copy(&self->ctx, initial_ctx);
} else {
@ -481,7 +425,7 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
}
if (cp && len) {
@@ -459,20 +523,28 @@ PyDoc_STRVAR(EVP_new__doc__,
@@ -411,21 +475,29 @@ PyDoc_STRVAR(EVP_new__doc__,
An optional string argument may be provided and will be\n\
automatically hashed.\n\
\n\
@ -501,29 +445,32 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
- static char *kwlist[] = {"name", "string", NULL};
+ static char *kwlist[] = {"name", "string", "usedforsecurity", NULL};
PyObject *name_obj = NULL;
PyObject *data_obj = NULL;
+ int usedforsecurity = 1;
Py_buffer view = { 0 };
PyObject *ret_obj;
char *name;
const EVP_MD *digest;
+ int usedforsecurity = 1;
- if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|s*:new", kwlist,
- &name_obj, &view)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|s*i:new", kwlist,
+ &name_obj, &view, &usedforsecurity)) {
- if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|O:new", kwlist,
- &name_obj, &data_obj)) {
+ if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|Oi:new", kwlist,
+ &name_obj, &data_obj, &usedforsecurity)) {
return NULL;
}
@@ -484,58 +556,118 @@ EVP_new(PyObject *self, PyObject *args,
@@ -439,7 +511,8 @@ EVP_new(PyObject *self, PyObject *args, PyObject *kwdict)
digest = EVP_get_digestbyname(name);
ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf,
- view.len);
+ view.len, usedforsecurity);
PyBuffer_Release(&view);
- ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, view.len);
+ ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, view.len,
+ usedforsecurity);
if (data_obj)
PyBuffer_Release(&view);
@@ -722,57 +795,114 @@ generate_hash_name_list(void)
return ret_obj;
}
/*
- * This macro generates constructor function definitions for specific
@ -541,22 +488,29 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
- EVP_new_ ## NAME (PyObject *self, PyObject *args) \
+ EVP_new_ ## NAME (PyObject *self, PyObject *args, PyObject *kwdict) \
{ \
- PyObject *data_obj = NULL; \
- Py_buffer view = { 0 }; \
- PyObject *ret_obj; \
- \
- if (!PyArg_ParseTuple(args, "|s*:" #NAME , &view)) { \
- if (!PyArg_ParseTuple(args, "|O:" #NAME , &data_obj)) { \
- return NULL; \
- } \
- \
- if (data_obj) \
- GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); \
- \
- ret_obj = EVPnew( \
- CONST_ ## NAME ## _name_obj, \
- NULL, \
- CONST_new_ ## NAME ## _ctx_p, \
- (unsigned char*)view.buf, view.len); \
- (unsigned char*)view.buf, \
- view.len); \
- \
- if (data_obj) \
- PyBuffer_Release(&view); \
- return ret_obj; \
+ return implement_specific_EVP_new(self, args, kwdict, \
+ "|s*i:" #NAME, \
+ "|Oi:" #NAME, \
+ &cached_info_ ## NAME ); \
}
@ -566,6 +520,7 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
+ EVPCachedInfo *cached_info)
+{
+ static char *kwlist[] = {"string", "usedforsecurity", NULL};
+ PyObject *data_obj = NULL;
+ Py_buffer view = { 0 };
+ int usedforsecurity = 1;
+ int idx;
@ -574,10 +529,13 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
+ assert(cached_info);
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwdict, format, kwlist,
+ &view, &usedforsecurity)) {
+ &data_obj, &usedforsecurity)) {
+ return NULL;
+ }
+
+ if (data_obj)
+ GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view);
+
+ idx = usedforsecurity ? 1 : 0;
+
+ /*
@ -602,6 +560,7 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
+ }
+ }
+
+ if (data_obj)
+ PyBuffer_Release(&view);
+
+ return ret_obj;
@ -611,7 +570,7 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
#define CONSTRUCTOR_METH_DEF(NAME) \
- {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, METH_VARARGS, \
+ {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, \
+ METH_VARARGS |METH_KEYWORDS, \
+ METH_VARARGS|METH_KEYWORDS, \
PyDoc_STR("Returns a " #NAME \
" hash object; optionally initialized with a string") \
}
@ -620,7 +579,7 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
- constructor constants if they haven't been initialized already. */
-#define INIT_CONSTRUCTOR_CONSTANTS(NAME) do { \
- if (CONST_ ## NAME ## _name_obj == NULL) { \
- CONST_ ## NAME ## _name_obj = PyString_FromString(#NAME); \
- CONST_ ## NAME ## _name_obj = PyUnicode_FromString(#NAME); \
- if (EVP_get_digestbyname(#NAME)) { \
- CONST_new_ ## NAME ## _ctx_p = &CONST_new_ ## NAME ## _ctx; \
- EVP_DigestInit(CONST_new_ ## NAME ## _ctx_p, EVP_get_digestbyname(#NAME)); \
@ -632,18 +591,17 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
+ Try to initialize a context for each hash twice, once with
+ EVP_MD_CTX_FLAG_NON_FIPS_ALLOW and once without.
+
+ Any that have errors during initialization will end up wit a NULL ctx_ptrs
+ Any that have errors during initialization will end up with a NULL ctx_ptrs
+ entry, and err_msgs will be set (unless we're very low on memory)
+*/
+#define INIT_CONSTRUCTOR_CONSTANTS(NAME) do { \
+ init_constructor_constant(&cached_info_ ## NAME, #NAME); \
} while (0);
+static void
+init_constructor_constant(EVPCachedInfo *cached_info, const char *name)
+{
+ assert(cached_info);
+ cached_info->name_obj = PyString_FromString(name);
+ cached_info->name_obj = PyUnicode_FromString(name);
+ if (EVP_get_digestbyname(name)) {
+ int i;
+ for (i=0; i<2; i++) {
@ -660,70 +618,23 @@ diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_
+ }
+ }
+}
+
GEN_CONSTRUCTOR(md5)
GEN_CONSTRUCTOR(sha1)
#ifdef _OPENSSL_SUPPORTS_SHA2
@@ -565,13 +700,10 @@ init_hashlib(void)
@@ -819,13 +949,10 @@ PyInit__hashlib(void)
{
PyObject *m;
PyObject *m, *openssl_md_meth_names;
- OpenSSL_add_all_digests();
- ERR_load_crypto_strings();
+ SSL_load_error_strings();
+ SSL_library_init();
OpenSSL_add_all_digests();
- /* TODO build EVP_functions openssl_* entries dynamically based
- * on what hashes are supported rather than listing many
- * but having some be unsupported. Only init appropriate
- * constants. */
-
+ OpenSSL_add_all_digests();
Py_TYPE(&EVPtype) = &PyType_Type;
if (PyType_Ready(&EVPtype) < 0)
return;
diff -up Python-2.7.2/Modules/Setup.dist.hashlib-fips Python-2.7.2/Modules/Setup.dist
--- Python-2.7.2/Modules/Setup.dist.hashlib-fips 2011-09-14 00:21:26.163252001 -0400
+++ Python-2.7.2/Modules/Setup.dist 2011-09-14 00:21:26.201252001 -0400
@@ -248,14 +248,14 @@ imageop imageop.c # Operations on images
# Message-Digest Algorithm, described in RFC 1321. The necessary files
# md5.c and md5.h are included here.
-_md5 md5module.c md5.c
+#_md5 md5module.c md5.c
# The _sha module implements the SHA checksum algorithms.
# (NIST's Secure Hash Algorithms.)
-_sha shamodule.c
-_sha256 sha256module.c
-_sha512 sha512module.c
+#_sha shamodule.c
+#_sha256 sha256module.c
+#_sha512 sha512module.c
# SGI IRIX specific modules -- off by default.
diff -up Python-2.7.2/setup.py.hashlib-fips Python-2.7.2/setup.py
--- Python-2.7.2/setup.py.hashlib-fips 2011-09-14 00:21:25.722252001 -0400
+++ Python-2.7.2/setup.py 2011-09-14 00:21:26.203252001 -0400
@@ -768,21 +768,6 @@ class PyBuildExt(build_ext):
print ("warning: openssl 0x%08x is too old for _hashlib" %
openssl_ver)
missing.append('_hashlib')
- if COMPILED_WITH_PYDEBUG or not have_usable_openssl:
- # The _sha module implements the SHA1 hash algorithm.
- exts.append( Extension('_sha', ['shamodule.c']) )
- # The _md5 module implements the RSA Data Security, Inc. MD5
- # Message-Digest Algorithm, described in RFC 1321. The
- # necessary files md5.c and md5.h are included here.
- exts.append( Extension('_md5',
- sources = ['md5module.c', 'md5.c'],
- depends = ['md5.h']) )
-
- min_sha2_openssl_ver = 0x00908000
- if COMPILED_WITH_PYDEBUG or openssl_ver < min_sha2_openssl_ver:
- # OpenSSL doesn't do these until 0.9.8 so we'll bring our own hash
- exts.append( Extension('_sha256', ['sha256module.c']) )
- exts.append( Extension('_sha512', ['sha512module.c']) )
# Modules that provide persistent dictionary-like semantics. You will
# probably want to arrange for at least one of them to be available on

12
SOURCES/00155-avoid-ctypes-thunks.patch
View File

@ -1,7 +1,7 @@
diff -up Python-2.7.3/Lib/ctypes/__init__.py.rhbz814391 Python-2.7.3/Lib/ctypes/__init__.py
--- Python-2.7.3/Lib/ctypes/__init__.py.rhbz814391 2012-04-20 14:51:19.390990244 -0400
+++ Python-2.7.3/Lib/ctypes/__init__.py 2012-04-20 14:51:45.141668316 -0400
@@ -272,11 +272,6 @@ def _reset_cache():
diff -up Python-3.2.3/Lib/ctypes/__init__.py.rhbz814391 Python-3.2.3/Lib/ctypes/__init__.py
--- Python-3.2.3/Lib/ctypes/__init__.py.rhbz814391 2012-04-20 15:12:49.017867692 -0400
+++ Python-3.2.3/Lib/ctypes/__init__.py 2012-04-20 15:15:09.501111408 -0400
@@ -275,11 +275,6 @@ def _reset_cache():
# _SimpleCData.c_char_p_from_param
POINTER(c_char).from_param = c_char_p.from_param
_pointer_type_cache[None] = c_void_p
@ -11,5 +11,5 @@ diff -up Python-2.7.3/Lib/ctypes/__init__.py.rhbz814391 Python-2.7.3/Lib/ctypes/
- # compiled with the MS SDK compiler. Or an uninitialized variable?
- CFUNCTYPE(c_int)(lambda: None)
try:
from _ctypes import set_conversion_mode
def create_unicode_buffer(init, size=None):
"""create_unicode_buffer(aString) -> character array

55
SOURCES/00157-uid-gid-overflows.patch
View File

@ -1,49 +1,68 @@
diff -up Python-2.7.3/Lib/test/test_os.py.uid-gid-overflows Python-2.7.3/Lib/test/test_os.py
--- Python-2.7.3/Lib/test/test_os.py.uid-gid-overflows 2012-04-09 19:07:32.000000000 -0400
+++ Python-2.7.3/Lib/test/test_os.py 2012-06-26 14:51:36.000817929 -0400
@@ -677,30 +677,36 @@ if sys.platform != 'win32':
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index e9fdb07..ea60e6e 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1723,30 +1723,36 @@ class PosixUidGidTests(unittest.TestCase):
def test_setuid(self):
if os.getuid() != 0:
self.assertRaises(os.error, os.setuid, 0)
self.assertRaises(OSError, os.setuid, 0)
+ self.assertRaises(TypeError, os.setuid, 'not an int')
self.assertRaises(OverflowError, os.setuid, 1<<32)
if hasattr(os, 'setgid'):
@unittest.skipUnless(hasattr(os, 'setgid'), 'test needs os.setgid()')
def test_setgid(self):
if os.getuid() != 0:
self.assertRaises(os.error, os.setgid, 0)
if os.getuid() != 0 and not HAVE_WHEEL_GROUP:
self.assertRaises(OSError, os.setgid, 0)
+ self.assertRaises(TypeError, os.setgid, 'not an int')
self.assertRaises(OverflowError, os.setgid, 1<<32)
if hasattr(os, 'seteuid'):
@unittest.skipUnless(hasattr(os, 'seteuid'), 'test needs os.seteuid()')
def test_seteuid(self):
if os.getuid() != 0:
self.assertRaises(os.error, os.seteuid, 0)
self.assertRaises(OSError, os.seteuid, 0)
+ self.assertRaises(TypeError, os.seteuid, 'not an int')
self.assertRaises(OverflowError, os.seteuid, 1<<32)
if hasattr(os, 'setegid'):
@unittest.skipUnless(hasattr(os, 'setegid'), 'test needs os.setegid()')
def test_setegid(self):
if os.getuid() != 0:
self.assertRaises(os.error, os.setegid, 0)
if os.getuid() != 0 and not HAVE_WHEEL_GROUP:
self.assertRaises(OSError, os.setegid, 0)
+ self.assertRaises(TypeError, os.setegid, 'not an int')
self.assertRaises(OverflowError, os.setegid, 1<<32)
if hasattr(os, 'setreuid'):
@unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()')
def test_setreuid(self):
if os.getuid() != 0:
self.assertRaises(os.error, os.setreuid, 0, 0)
self.assertRaises(OSError, os.setreuid, 0, 0)
+ self.assertRaises(TypeError, os.setreuid, 'not an int', 0)
+ self.assertRaises(TypeError, os.setreuid, 0, 'not an int')
self.assertRaises(OverflowError, os.setreuid, 1<<32, 0)
self.assertRaises(OverflowError, os.setreuid, 0, 1<<32)
@@ -715,6 +721,8 @@ if sys.platform != 'win32':
@@ -1762,6 +1768,8 @@ class PosixUidGidTests(unittest.TestCase):
def test_setregid(self):
if os.getuid() != 0:
self.assertRaises(os.error, os.setregid, 0, 0)
if os.getuid() != 0 and not HAVE_WHEEL_GROUP:
self.assertRaises(OSError, os.setregid, 0, 0)
+ self.assertRaises(TypeError, os.setregid, 'not an int', 0)
+ self.assertRaises(TypeError, os.setregid, 0, 'not an int')
self.assertRaises(OverflowError, os.setregid, 1<<32, 0)
self.assertRaises(OverflowError, os.setregid, 0, 1<<32)
diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py
index ac9cff7..db98159 100644
--- a/Lib/test/test_pwd.py
+++ b/Lib/test/test_pwd.py
@@ -104,11 +104,11 @@ class PwdTest(unittest.TestCase):
# In some cases, byuids isn't a complete list of all users in the
# system, so if we try to pick a value not in byuids (via a perturbing
# loop, say), pwd.getpwuid() might still be able to find data for that
- # uid. Using sys.maxint may provoke the same problems, but hopefully
+ # uid. Using 2**32 - 2 may provoke the same problems, but hopefully
# it will be a more repeatable failure.
# Android accepts a very large span of uids including sys.maxsize and
# -1; it raises KeyError with 1 or 2 for example.
- fakeuid = sys.maxsize
+ fakeuid = 2**32 - 2
self.assertNotIn(fakeuid, byuids)
if not support.is_android:
self.assertRaises(KeyError, pwd.getpwuid, fakeuid)

454
SOURCES/00170-gc-assertions.patch
View File

@ -1,80 +1,10 @@
diff -up Python-2.7.3/Lib/test/test_gc.py.gc-assertions Python-2.7.3/Lib/test/test_gc.py
--- Python-2.7.3/Lib/test/test_gc.py.gc-assertions 2013-02-20 16:28:20.890536607 -0500
+++ Python-2.7.3/Lib/test/test_gc.py 2013-02-20 16:39:52.720489297 -0500
@@ -1,6 +1,7 @@
import unittest
-from test.test_support import verbose, run_unittest
+from test.test_support import verbose, run_unittest, import_module
import sys
+import sysconfig
import time
import gc
import weakref
@@ -32,6 +33,8 @@ class GC_Detector(object):
self.wr = weakref.ref(C1055820(666), it_happened)
+BUILT_WITH_NDEBUG = ('-DNDEBUG' in sysconfig.get_config_vars()['PY_CFLAGS'])
+
### Tests
###############################################################################
@@ -476,6 +479,49 @@ class GCTests(unittest.TestCase):
# would be damaged, with an empty __dict__.
self.assertEqual(x, None)
+ @unittest.skipIf(BUILT_WITH_NDEBUG,
+ 'built with -NDEBUG')
+ def test_refcount_errors(self):
+ # Verify the "handling" of objects with broken refcounts
+
+ import_module("ctypes") #skip if not supported
+
+ import subprocess
+ code = '''if 1:
+ a = []
+ b = [a]
+
+ # Simulate the refcount of "a" being too low (compared to the
+ # references held on it by live data), but keeping it above zero
+ # (to avoid deallocating it):
+ import ctypes
+ ctypes.pythonapi.Py_DecRef(ctypes.py_object(a))
+
+ # The garbage collector should now have a fatal error when it reaches
+ # the broken object:
+ import gc
+ gc.collect()
+ '''
+ p = subprocess.Popen([sys.executable, "-c", code],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ p.stdout.close()
+ p.stderr.close()
+ # Verify that stderr has a useful error message:
+ self.assertRegexpMatches(stderr,
+ b'Modules/gcmodule.c:[0-9]+: visit_decref: Assertion "gc->gc.gc_refs != 0" failed.')
+ self.assertRegexpMatches(stderr,
+ b'refcount was too small')
+ self.assertRegexpMatches(stderr,
+ b'object : \[\]')
+ self.assertRegexpMatches(stderr,
+ b'type : list')
+ self.assertRegexpMatches(stderr,
+ b'refcount: 1')
+ self.assertRegexpMatches(stderr,
+ b'address : 0x[0-9a-f]+')
+
class GCTogglingTests(unittest.TestCase):
def setUp(self):
gc.enable()
diff -up Python-2.7.3/Modules/gcmodule.c.gc-assertions Python-2.7.3/Modules/gcmodule.c
--- Python-2.7.3/Modules/gcmodule.c.gc-assertions 2012-04-09 19:07:34.000000000 -0400
+++ Python-2.7.3/Modules/gcmodule.c 2013-02-20 16:28:21.029536600 -0500
@@ -21,6 +21,73 @@
#include "Python.h"
#include "frameobject.h" /* for PyFrame_ClearFreeList */
diff --git a/Include/object.h b/Include/object.h
index 0c88603..e3413e8 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -1059,6 +1059,49 @@ PyAPI_FUNC(void)
_PyObject_DebugTypeStats(FILE *out);
#endif /* ifndef Py_LIMITED_API */
+/*
+ Define a pair of assertion macros.
@ -110,11 +40,243 @@ diff -up Python-2.7.3/Modules/gcmodule.c.gc-assertions Python-2.7.3/Modules/gcmo
+#define PyObject_ASSERT(obj, expr) \
+ PyObject_ASSERT_WITH_MSG(obj, expr, NULL)
+
+static void _PyObject_AssertFailed(PyObject *, const char *,
+/*
+ Declare and define the entrypoint even when NDEBUG is defined, to avoid
+ causing compiler/linker errors when building extensions without NDEBUG
+ against a Python built with NDEBUG defined
+*/
+PyAPI_FUNC(void) _PyObject_AssertFailed(PyObject *, const char *,
+ const char *, const char *, int,
+ const char *);
+
+static void
#ifdef __cplusplus
}
#endif
diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py
index e727499..6efcafb 100644
--- a/Lib/test/test_gc.py
+++ b/Lib/test/test_gc.py
@@ -1,10 +1,11 @@
import unittest
from test.support import (verbose, refcount_test, run_unittest,
strip_python_stderr, cpython_only, start_threads,
- temp_dir, requires_type_collecting)
+ temp_dir, import_module, requires_type_collecting)
from test.support.script_helper import assert_python_ok, make_script
import sys
+import sysconfig
import time
import gc
import weakref
@@ -50,6 +51,8 @@ class GC_Detector(object):
# gc collects it.
self.wr = weakref.ref(C1055820(666), it_happened)
+BUILD_WITH_NDEBUG = ('-DNDEBUG' in sysconfig.get_config_vars()['PY_CFLAGS'])
+
@with_tp_del
class Uncollectable(object):
"""Create a reference cycle with multiple __del__ methods.
@@ -862,6 +865,50 @@ class GCCallbackTests(unittest.TestCase):
self.assertEqual(len(gc.garbage), 0)
+ @unittest.skipIf(BUILD_WITH_NDEBUG,
+ 'built with -NDEBUG')
+ def test_refcount_errors(self):
+ self.preclean()
+ # Verify the "handling" of objects with broken refcounts
+ import_module("ctypes") #skip if not supported
+
+ import subprocess
+ code = '''if 1:
+ a = []
+ b = [a]
+
+ # Simulate the refcount of "a" being too low (compared to the
+ # references held on it by live data), but keeping it above zero
+ # (to avoid deallocating it):
+ import ctypes
+ ctypes.pythonapi.Py_DecRef(ctypes.py_object(a))
+
+ # The garbage collector should now have a fatal error when it reaches
+ # the broken object:
+ import gc
+ gc.collect()
+ '''
+ p = subprocess.Popen([sys.executable, "-c", code],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ p.stdout.close()
+ p.stderr.close()
+ # Verify that stderr has a useful error message:
+ self.assertRegex(stderr,
+ b'Modules/gcmodule.c:[0-9]+: visit_decref: Assertion "\(\(gc\)->gc.gc_refs >> \(1\)\) != 0" failed.')
+ self.assertRegex(stderr,
+ b'refcount was too small')
+ self.assertRegex(stderr,
+ b'object : \[\]')
+ self.assertRegex(stderr,
+ b'type : list')
+ self.assertRegex(stderr,
+ b'refcount: 1')
+ self.assertRegex(stderr,
+ b'address : 0x[0-9a-f]+')
+
+
class GCTogglingTests(unittest.TestCase):
def setUp(self):
gc.enable()
diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
index 0c6f444..87edd5a 100644
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -341,7 +341,8 @@ update_refs(PyGC_Head *containers)
{
PyGC_Head *gc = containers->gc.gc_next;
for (; gc != containers; gc = gc->gc.gc_next) {
- assert(_PyGCHead_REFS(gc) == GC_REACHABLE);
+ PyObject_ASSERT(FROM_GC(gc),
+ _PyGCHead_REFS(gc) == GC_REACHABLE);
_PyGCHead_SET_REFS(gc, Py_REFCNT(FROM_GC(gc)));
/* Python's cyclic gc should never see an incoming refcount
* of 0: if something decref'ed to 0, it should have been
@@ -361,7 +362,8 @@ update_refs(PyGC_Head *containers)
* so serious that maybe this should be a release-build
* check instead of an assert?
*/
- assert(_PyGCHead_REFS(gc) != 0);
+ PyObject_ASSERT(FROM_GC(gc),
+ _PyGCHead_REFS(gc) != 0);
}
}
@@ -376,7 +378,9 @@ visit_decref(PyObject *op, void *data)
* generation being collected, which can be recognized
* because only they have positive gc_refs.
*/
- assert(_PyGCHead_REFS(gc) != 0); /* else refcount was too small */
+ PyObject_ASSERT_WITH_MSG(FROM_GC(gc),
+ _PyGCHead_REFS(gc) != 0,
+ "refcount was too small"); /* else refcount was too small */
if (_PyGCHead_REFS(gc) > 0)
_PyGCHead_DECREF(gc);
}
@@ -436,9 +440,10 @@ visit_reachable(PyObject *op, PyGC_Head *reachable)
* If gc_refs == GC_UNTRACKED, it must be ignored.
*/
else {
- assert(gc_refs > 0
- || gc_refs == GC_REACHABLE
- || gc_refs == GC_UNTRACKED);
+ PyObject_ASSERT(FROM_GC(gc),
+ gc_refs > 0
+ || gc_refs == GC_REACHABLE
+ || gc_refs == GC_UNTRACKED);
}
}
return 0;
@@ -480,7 +485,7 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable)
*/
PyObject *op = FROM_GC(gc);
traverseproc traverse = Py_TYPE(op)->tp_traverse;
- assert(_PyGCHead_REFS(gc) > 0);
+ PyObject_ASSERT(op, _PyGCHead_REFS(gc) > 0);
_PyGCHead_SET_REFS(gc, GC_REACHABLE);
(void) traverse(op,
(visitproc)visit_reachable,
@@ -543,7 +548,7 @@ move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers)
for (gc = unreachable->gc.gc_next; gc != unreachable; gc = next) {
PyObject *op = FROM_GC(gc);
- assert(IS_TENTATIVELY_UNREACHABLE(op));
+ PyObject_ASSERT(op, IS_TENTATIVELY_UNREACHABLE(op));
next = gc->gc.gc_next;
if (has_legacy_finalizer(op)) {
@@ -619,7 +624,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
PyWeakReference **wrlist;
op = FROM_GC(gc);
- assert(IS_TENTATIVELY_UNREACHABLE(op));
+ PyObject_ASSERT(op, IS_TENTATIVELY_UNREACHABLE(op));
next = gc->gc.gc_next;
if (! PyType_SUPPORTS_WEAKREFS(Py_TYPE(op)))
@@ -640,9 +645,9 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
* the callback pointer intact. Obscure: it also
* changes *wrlist.
*/
- assert(wr->wr_object == op);
+ PyObject_ASSERT(wr->wr_object, wr->wr_object == op);
_PyWeakref_ClearRef(wr);
- assert(wr->wr_object == Py_None);
+ PyObject_ASSERT(wr->wr_object, wr->wr_object == Py_None);
if (wr->wr_callback == NULL)
continue; /* no callback */
@@ -676,7 +681,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
*/
if (IS_TENTATIVELY_UNREACHABLE(wr))
continue;
- assert(IS_REACHABLE(wr));
+ PyObject_ASSERT(op, IS_REACHABLE(wr));
/* Create a new reference so that wr can't go away
* before we can process it again.
@@ -685,7 +690,8 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
/* Move wr to wrcb_to_call, for the next pass. */
wrasgc = AS_GC(wr);
- assert(wrasgc != next); /* wrasgc is reachable, but
+ PyObject_ASSERT(op, wrasgc != next);
+ /* wrasgc is reachable, but
next isn't, so they can't
be the same */
gc_list_move(wrasgc, &wrcb_to_call);
@@ -701,11 +707,11 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)
gc = wrcb_to_call.gc.gc_next;
op = FROM_GC(gc);
- assert(IS_REACHABLE(op));
- assert(PyWeakref_Check(op));
+ PyObject_ASSERT(op, IS_REACHABLE(op));
+ PyObject_ASSERT(op, PyWeakref_Check(op));
wr = (PyWeakReference *)op;
callback = wr->wr_callback;
- assert(callback != NULL);
+ PyObject_ASSERT(op, callback != NULL);
/* copy-paste of weakrefobject.c's handle_callback() */
temp = PyObject_CallFunctionObjArgs(callback, wr, NULL);
@@ -822,12 +828,14 @@ check_garbage(PyGC_Head *collectable)
for (gc = collectable->gc.gc_next; gc != collectable;
gc = gc->gc.gc_next) {
_PyGCHead_SET_REFS(gc, Py_REFCNT(FROM_GC(gc)));
- assert(_PyGCHead_REFS(gc) != 0);
+ PyObject_ASSERT(FROM_GC(gc),
+ _PyGCHead_REFS(gc) != 0);
}
subtract_refs(collectable);
for (gc = collectable->gc.gc_next; gc != collectable;
gc = gc->gc.gc_next) {
- assert(_PyGCHead_REFS(gc) >= 0);
+ PyObject_ASSERT(FROM_GC(gc),
+ _PyGCHead_REFS(gc) >= 0);
if (_PyGCHead_REFS(gc) != 0)
return -1;
}
diff --git a/Objects/object.c b/Objects/object.c
index 559794f..a47d47f 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -2022,6 +2022,35 @@ _PyTrash_thread_destroy_chain(void)
}
}
+PyAPI_FUNC(void)
+_PyObject_AssertFailed(PyObject *obj, const char *msg, const char *expr,
+ const char *file, int line, const char *function)
+{
@ -143,134 +305,6 @@ diff -up Python-2.7.3/Modules/gcmodule.c.gc-assertions Python-2.7.3/Modules/gcmo
+ abort();
+}
+
/* Get an object's GC head */
#define AS_GC(o) ((PyGC_Head *)(o)-1)
@@ -288,7 +355,8 @@ update_refs(PyGC_Head *containers)
{
PyGC_Head *gc = containers->gc.gc_next;
for (; gc != containers; gc = gc->gc.gc_next) {
- assert(gc->gc.gc_refs == GC_REACHABLE);
+ PyObject_ASSERT(FROM_GC(gc),
+ gc->gc.gc_refs == GC_REACHABLE);
gc->gc.gc_refs = Py_REFCNT(FROM_GC(gc));
/* Python's cyclic gc should never see an incoming refcount
* of 0: if something decref'ed to 0, it should have been
@@ -308,7 +376,8 @@ update_refs(PyGC_Head *containers)
* so serious that maybe this should be a release-build
* check instead of an assert?
*/
- assert(gc->gc.gc_refs != 0);
+ PyObject_ASSERT(FROM_GC(gc),
+ gc->gc.gc_refs != 0);
}
}
@@ -323,7 +392,9 @@ visit_decref(PyObject *op, void *data)
* generation being collected, which can be recognized
* because only they have positive gc_refs.
*/
- assert(gc->gc.gc_refs != 0); /* else refcount was too small */
+ PyObject_ASSERT_WITH_MSG(FROM_GC(gc),
+ gc->gc.gc_refs != 0,
+ "refcount was too small");
if (gc->gc.gc_refs > 0)
gc->gc.gc_refs--;
}
@@ -383,9 +454,10 @@ visit_reachable(PyObject *op, PyGC_Head
* If gc_refs == GC_UNTRACKED, it must be ignored.
*/
else {
- assert(gc_refs > 0
- || gc_refs == GC_REACHABLE
- || gc_refs == GC_UNTRACKED);
+ PyObject_ASSERT(FROM_GC(gc),
+ gc_refs > 0
+ || gc_refs == GC_REACHABLE
+ || gc_refs == GC_UNTRACKED);
}
}
return 0;
@@ -427,7 +499,7 @@ move_unreachable(PyGC_Head *young, PyGC_
*/
PyObject *op = FROM_GC(gc);
traverseproc traverse = Py_TYPE(op)->tp_traverse;
- assert(gc->gc.gc_refs > 0);
+ PyObject_ASSERT(op, gc->gc.gc_refs > 0);
gc->gc.gc_refs = GC_REACHABLE;
(void) traverse(op,
(visitproc)visit_reachable,
@@ -494,7 +566,8 @@ move_finalizers(PyGC_Head *unreachable,
for (gc = unreachable->gc.gc_next; gc != unreachable; gc = next) {
PyObject *op = FROM_GC(gc);
- assert(IS_TENTATIVELY_UNREACHABLE(op));
+ PyObject_ASSERT(op, IS_TENTATIVELY_UNREACHABLE(op));
+
next = gc->gc.gc_next;
if (has_finalizer(op)) {
@@ -570,7 +643,7 @@ handle_weakrefs(PyGC_Head *unreachable,
PyWeakReference **wrlist;
op = FROM_GC(gc);
- assert(IS_TENTATIVELY_UNREACHABLE(op));
+ PyObject_ASSERT(op, IS_TENTATIVELY_UNREACHABLE(op));
next = gc->gc.gc_next;
if (! PyType_SUPPORTS_WEAKREFS(Py_TYPE(op)))
@@ -591,9 +664,9 @@ handle_weakrefs(PyGC_Head *unreachable,
* the callback pointer intact. Obscure: it also
* changes *wrlist.
*/
- assert(wr->wr_object == op);
+ PyObject_ASSERT(wr->wr_object, wr->wr_object == op);
_PyWeakref_ClearRef(wr);
- assert(wr->wr_object == Py_None);
+ PyObject_ASSERT(wr->wr_object, wr->wr_object == Py_None);
if (wr->wr_callback == NULL)
continue; /* no callback */
@@ -627,7 +700,7 @@ handle_weakrefs(PyGC_Head *unreachable,
*/
if (IS_TENTATIVELY_UNREACHABLE(wr))
continue;
- assert(IS_REACHABLE(wr));
+ PyObject_ASSERT(op, IS_REACHABLE(wr));
/* Create a new reference so that wr can't go away
* before we can process it again.
@@ -636,7 +709,8 @@ handle_weakrefs(PyGC_Head *unreachable,
/* Move wr to wrcb_to_call, for the next pass. */
wrasgc = AS_GC(wr);
- assert(wrasgc != next); /* wrasgc is reachable, but
+ PyObject_ASSERT(op, wrasgc != next);
+ /* wrasgc is reachable, but
next isn't, so they can't
be the same */
gc_list_move(wrasgc, &wrcb_to_call);
@@ -652,11 +726,11 @@ handle_weakrefs(PyGC_Head *unreachable,
gc = wrcb_to_call.gc.gc_next;
op = FROM_GC(gc);
- assert(IS_REACHABLE(op));
- assert(PyWeakref_Check(op));
+ PyObject_ASSERT(op, IS_REACHABLE(op));
+ PyObject_ASSERT(op, PyWeakref_Check(op));
wr = (PyWeakReference *)op;
callback = wr->wr_callback;
- assert(callback != NULL);
+ PyObject_ASSERT(op, callback != NULL);
/* copy-paste of weakrefobject.c's handle_callback() */
temp = PyObject_CallFunctionObjArgs(callback, wr, NULL);
@@ -759,7 +833,7 @@ delete_garbage(PyGC_Head *collectable, P
PyGC_Head *gc = collectable->gc.gc_next;
PyObject *op = FROM_GC(gc);
- assert(IS_TENTATIVELY_UNREACHABLE(op));
+ PyObject_ASSERT(op, IS_TENTATIVELY_UNREACHABLE(op));
if (debug & DEBUG_SAVEALL) {
PyList_Append(garbage, op);
}
#ifndef Py_TRACE_REFS
/* For Py_LIMITED_API, we need an out-of-line version of _Py_Dealloc.
Define this here, so we can undefine the macro. */

11
SOURCES/00180-python-add-support-for-ppc64p7.patch
View File

@ -1,12 +1,13 @@
diff -r de35eae9048a config.sub
--- a/config.sub Wed Apr 24 23:33:20 2013 +0200
+++ b/config.sub Thu Apr 25 08:51:00 2013 +0200
@@ -1008,7 +1008,7 @@
diff --git a/config.sub b/config.sub
index 40ea5df..932128b 100755
--- a/config.sub
+++ b/config.sub
@@ -1045,7 +1045,7 @@ case $basic_machine in
;;
ppc64) basic_machine=powerpc64-unknown
;;
- ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ppc64-* | ppc64p7-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
;;
ppc64le | powerpc64little | ppc64-le | powerpc64-little)
ppc64le | powerpc64little)
basic_machine=powerpc64le-unknown