diff --git a/SOURCES/00001-pydocnogui.patch b/SOURCES/00001-pydocnogui.patch new file mode 100644 index 00000000..3b34842b --- /dev/null +++ b/SOURCES/00001-pydocnogui.patch @@ -0,0 +1,38 @@ +diff -up Python-2.7.3/Lib/pydoc.py.no_gui Python-2.7.3/Lib/pydoc.py +--- Python-2.7.3/Lib/pydoc.py.no_gui 2012-04-09 19:07:31.000000000 -0400 ++++ Python-2.7.3/Lib/pydoc.py 2013-02-19 13:48:44.480054515 -0500 +@@ -19,9 +19,6 @@ of all available modules. + Run "pydoc -p " to start an HTTP server on a given port on the + local machine to generate documentation web pages. + +-For platforms without a command line, "pydoc -g" starts the HTTP server +-and also pops up a little window for controlling it. +- + Run "pydoc -w " to write out the HTML documentation for a module + to a file named ".html". + +@@ -2290,9 +2287,6 @@ def cli(): + writing = 0 + + for opt, val in opts: +- if opt == '-g': +- gui() +- return + if opt == '-k': + apropos(val) + return +@@ -2346,13 +2340,10 @@ def cli(): + %s -p + Start an HTTP server on the given port on the local machine. + +-%s -g +- Pop up a graphical interface for finding and serving documentation. +- + %s -w ... + Write out the HTML documentation for a module to a file in the current + directory. If contains a '%s', it is treated as a filename; if + it names a directory, documentation is written for all the contents. +-""" % (cmd, os.sep, cmd, cmd, cmd, cmd, os.sep) ++""" % (cmd, os.sep, cmd, cmd, cmd, os.sep) + + if __name__ == '__main__': cli() diff --git a/SOURCES/00055-systemtap.patch b/SOURCES/00055-systemtap.patch index 0ab03874..67ec005d 100644 --- a/SOURCES/00055-systemtap.patch +++ b/SOURCES/00055-systemtap.patch @@ -1,822 +1,198 @@ -diff -up Python-3.3.0rc2/configure.ac.systemtap Python-3.3.0rc2/configure.ac ---- Python-3.3.0rc2/configure.ac.systemtap 2012-09-09 05:11:14.000000000 -0400 -+++ Python-3.3.0rc2/configure.ac 2012-09-10 09:17:21.114511781 -0400 -@@ -2678,6 +2678,23 @@ if test "$with_valgrind" != no; then - OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT" +diff -up Python-2.7rc1/configure.ac.systemtap Python-2.7rc1/configure.ac +--- Python-2.7rc1/configure.ac.systemtap 2010-06-06 10:53:15.514975012 -0400 ++++ Python-2.7rc1/configure.ac 2010-06-06 10:53:15.520974361 -0400 +@@ -2616,6 +2616,38 @@ if test "$with_valgrind" != no; then + ) fi -+# Check for systemtap support -+# On Linux, /usr/bin/dtrace is in fact a shim to SystemTap -+AC_MSG_CHECKING([for --with-systemtap]) -+AC_ARG_WITH([systemtap], -+ AC_HELP_STRING([--with(out)-systemtap], [disable/enable SystemTap support]),, -+ with_systemtap=no) -+AC_MSG_RESULT([$with_systemtap]) -+if test "$with_systemtap" != no; then -+ AC_DEFINE(WITH_SYSTEMTAP, 1, -+ [Define if you want to compile in SystemTap support]) -+ SYSTEMTAPOBJS="Python/pysystemtap.o" -+ SYSTEMTAPDEPS="\$(srcdir)/Python/pysystemtap.h" -+fi -+ -+AC_SUBST(SYSTEMTAPOBJS) -+AC_SUBST(SYSTEMTAPDEPS) -+ - # -I${DLINCLDIR} is added to the compile rule for importdl.o - AC_SUBST(DLINCLDIR) - DLINCLDIR=. -diff -up Python-3.3.0rc2/configure.systemtap Python-3.3.0rc2/configure ---- Python-3.3.0rc2/configure.systemtap 2012-09-09 05:11:14.000000000 -0400 -+++ Python-3.3.0rc2/configure 2012-09-10 09:17:21.116511780 -0400 -@@ -618,6 +618,8 @@ TRUE - MACHDEP_OBJS - DYNLOADFILE - DLINCLDIR -+SYSTEMTAPDEPS -+SYSTEMTAPOBJS - THREADOBJ - LDLAST - USE_THREAD_MODULE -@@ -779,6 +781,7 @@ with_doc_strings - with_tsc - with_pymalloc - with_valgrind -+with_systemtap - with_fpectl - with_libm - with_libc -@@ -1456,6 +1459,7 @@ Optional Packages: - --with(out)-tsc enable/disable timestamp counter profile - --with(out)-pymalloc disable/enable specialized mallocs - --with-valgrind Enable Valgrind support -+ --with(out)-systemtap disable/enable SystemTap support - --with-fpectl enable SIGFPE catching - --with-libm=STRING math library - --with-libc=STRING C library -@@ -10065,6 +10069,31 @@ fi - OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT" - fi - -+# Check for systemtap support -+# On Linux, /usr/bin/dtrace is in fact a shim to SystemTap -+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-systemtap" >&5 -+$as_echo_n "checking for --with-systemtap... " >&6; } -+ -+# Check whether --with-systemtap was given. -+if test "${with_systemtap+set}" = set; then : -+ withval=$with_systemtap; ++# Check for dtrace support ++AC_MSG_CHECKING(for --with-dtrace) ++AC_ARG_WITH(dtrace, ++ AC_HELP_STRING(--with(out)-dtrace, disable/enable dtrace support)) ++ ++if test ! -z "$with_dtrace" ++then ++ if dtrace -G -o /dev/null -s $srcdir/Include/pydtrace.d 2>/dev/null ++ then ++ AC_DEFINE(WITH_DTRACE, 1, ++ [Define if you want to compile in Dtrace support]) ++ with_dtrace="Sun" ++ DTRACEOBJS="Python/dtrace.o" ++ DTRADEHDRS="" ++ elif dtrace -h -o /dev/null -s $srcdir/Include/pydtrace.d ++ then ++ AC_DEFINE(WITH_DTRACE, 1, ++ [Define if you want to compile in Dtrace support]) ++ with_dtrace="Apple" ++ DTRACEOBJS="" ++ DTRADEHDRS="pydtrace.h" ++ else ++ with_dtrace="no" ++ fi +else -+ with_systemtap=no -+fi -+ -+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_systemtap" >&5 -+$as_echo "$with_systemtap" >&6; } -+if test "$with_systemtap" != no; then -+ -+$as_echo "#define WITH_SYSTEMTAP 1" >>confdefs.h -+ -+ SYSTEMTAPOBJS="Python/pysystemtap.o" -+ SYSTEMTAPDEPS="\$(srcdir)/Python/pysystemtap.h" ++ with_dtrace="no" +fi + ++AC_MSG_RESULT($with_dtrace) ++AC_SUBST(DTRACEOBJS) ++AC_SUBST(DTRACEHDRS) ++ + # Check for --with-wctype-functions + AC_MSG_CHECKING(for --with-wctype-functions) + AC_ARG_WITH(wctype-functions, +diff -up Python-2.7rc1/Include/pydtrace.d.systemtap Python-2.7rc1/Include/pydtrace.d +--- Python-2.7rc1/Include/pydtrace.d.systemtap 2010-06-06 10:53:15.520974361 -0400 ++++ Python-2.7rc1/Include/pydtrace.d 2010-06-06 10:53:15.520974361 -0400 +@@ -0,0 +1,10 @@ ++provider python { ++ probe function__entry(const char *, const char *, int); ++ probe function__return(const char *, const char *, int); ++}; + -+ -+ - # -I${DLINCLDIR} is added to the compile rule for importdl.o - - DLINCLDIR=. -diff -up Python-3.3.0rc2/Doc/howto/index.rst.systemtap Python-3.3.0rc2/Doc/howto/index.rst ---- Python-3.3.0rc2/Doc/howto/index.rst.systemtap 2012-09-09 05:10:51.000000000 -0400 -+++ Python-3.3.0rc2/Doc/howto/index.rst 2012-09-10 09:17:21.117511779 -0400 -@@ -29,4 +29,5 @@ Currently, the HOWTOs are: - argparse.rst - ipaddress.rst - clinic.rst -+ instrumentation.rst - -diff -up Python-3.3.0rc2/Doc/howto/instrumentation.rst.systemtap Python-3.3.0rc2/Doc/howto/instrumentation.rst ---- Python-3.3.0rc2/Doc/howto/instrumentation.rst.systemtap 2012-09-10 09:17:21.117511779 -0400 -+++ Python-3.3.0rc2/Doc/howto/instrumentation.rst 2012-09-10 09:17:21.117511779 -0400 -@@ -0,0 +1,295 @@ -+.. _instrumentation: -+ -+==================================== -+Instrumenting CPython with SystemTap -+==================================== -+ -+:author: David Malcolm -+ -+DTrace and SystemTap are monitoring tools, each providing a way to inspect -+what the processes on a computer system are doing. They both use -+domain-specific languages allowing a user to write scripts which: -+ -+ - filter which processes are to be observed -+ - gather data from the processes of interest -+ - generate reports on the data -+ -+As of Python 3.3, CPython can be built with embedded "markers" that can be -+observed by a SystemTap script, making it easier to monitor what the CPython -+processes on a system are doing. -+ -+.. Potentially this document could be expanded to also cover DTrace markers. -+ However, I'm not a DTrace expert. -+ -+.. I'm using ".. code-block:: c" for SystemTap scripts, as "c" is syntactically -+ the closest match that Sphinx supports -+ -+ -+Enabling the static markers -+--------------------------- -+ -+In order to build CPython with the embedded markers for SystemTap, the -+SystemTap development tools must be installed. -+ -+On a Fedora or Red Hat Enterprise Linux machine, this can be done via:: -+ -+ yum install systemtap-sdt-devel -+ -+CPython must then be configured `--with-systemtap`:: -+ -+ checking for --with-systemtap... yes -+ -+You can verify if the SystemTap static markers are present in the built -+binary by seeing if it contains a ".note.stapsdt" section. -+ -+.. code-block:: bash -+ -+ $ eu-readelf -S ./python | grep .note.stapsdt -+ [29] .note.stapsdt NOTE 0000000000000000 00308d78 000000b8 0 0 0 4 -+ -+If you've built python as a shared library (with --enable-shared), you need -+to look instead within the shared library. For example: -+ -+.. code-block:: bash -+ -+ $ eu-readelf -S libpython3.3dm.so.1.0 | grep .note.stapsdt -+ [28] .note.stapsdt NOTE 0000000000000000 00365b68 000000b8 0 0 0 4 -+ -+Earlier versions of SystemTap stored the markers in a ".probes" section. -+ -+For the curious, you can see the metadata for the static markers using this -+invocation. -+ -+.. code-block:: bash -+ -+ $ eu-readelf -x .note.stapsdt ./python -+ -+ Hex dump of section [29] '.note.stapsdt', 184 bytes at offset 0x308d78: -+ 0x00000000 08000000 45000000 03000000 73746170 ....E.......stap -+ 0x00000010 73647400 d4664b00 00000000 4fc36600 sdt..fK.....O.f. -+ 0x00000020 00000000 488d9000 00000000 70797468 ....H.......pyth -+ 0x00000030 6f6e0066 756e6374 696f6e5f 5f656e74 on.function__ent -+ 0x00000040 72790038 40257261 78203840 25726478 ry.8@%rax 8@%rdx -+ 0x00000050 202d3440 25656378 00000000 08000000 -4@%ecx........ -+ 0x00000060 46000000 03000000 73746170 73647400 F.......stapsdt. -+ 0x00000070 0d674b00 00000000 4fc36600 00000000 .gK.....O.f..... -+ 0x00000080 4a8d9000 00000000 70797468 6f6e0066 J.......python.f -+ 0x00000090 756e6374 696f6e5f 5f726574 75726e00 unction__return. -+ 0x000000a0 38402572 61782038 40257264 78202d34 8@%rax 8@%rdx -4 -+ 0x000000b0 40256563 78000000 @%ecx... -+ -+and a sufficiently modern eu-readelf can print the metadata: -+ -+.. code-block:: bash -+ -+ $ eu-readelf -n ./python -+ -+ Note section [ 1] '.note.gnu.build-id' of 36 bytes at offset 0x190: -+ Owner Data size Type -+ GNU 20 GNU_BUILD_ID -+ Build ID: a28f8db1b224530b0d38ad7b82a249cf7c3f18d6 -+ -+ Note section [27] '.note.stapsdt' of 184 bytes at offset 0x1ae884: -+ Owner Data size Type -+ stapsdt 70 Version: 3 -+ PC: 0xe0d3a, Base: 0x14b150, Semaphore: 0x3ae882 -+ Provider: python, Name: function__return, Args: '8@%rbx 8@%r13 -4@%eax' -+ stapsdt 69 Version: 3 -+ PC: 0xe0f37, Base: 0x14b150, Semaphore: 0x3ae880 -+ Provider: python, Name: function__entry, Args: '8@%rbx 8@%r13 -4@%eax' -+ -+The above metadata contains information for SystemTap describing how it can -+patch strategically-placed machine code instructions to enable the tracing -+hooks used by a SystemTap script. -+ -+ -+Static markers -+-------------- -+ -+The low-level way to use the SystemTap integration is to use the static -+markers directly. This requires you to explicitly state the binary file -+containing them. -+ -+For example, this script can be used to show the call/return hierarchy of a -+Python script: -+ -+.. code-block:: c -+ -+ probe process('python').mark("function__entry") { -+ filename = user_string($arg1); -+ funcname = user_string($arg2); -+ lineno = $arg3; -+ -+ printf("%s => %s in %s:%d\\n", -+ thread_indent(1), funcname, filename, lineno); -+ } -+ -+ probe process('python').mark("function__return") { -+ filename = user_string($arg1); -+ funcname = user_string($arg2); -+ lineno = $arg3; -+ -+ printf("%s <= %s in %s:%d\\n", -+ thread_indent(-1), funcname, filename, lineno); -+ } -+ -+It can be invoked like this: -+ -+.. code-block:: bash -+ -+ $ stap \ -+ show-call-hierarchy.stp \ -+ -c ./python test.py -+ -+The output looks like this:: -+ -+ 11408 python(8274): => __contains__ in Lib/_abcoll.py:362 -+ 11414 python(8274): => __getitem__ in Lib/os.py:425 -+ 11418 python(8274): => encode in Lib/os.py:490 -+ 11424 python(8274): <= encode in Lib/os.py:493 -+ 11428 python(8274): <= __getitem__ in Lib/os.py:426 -+ 11433 python(8274): <= __contains__ in Lib/_abcoll.py:366 -+ -+where the columns are: -+ -+ - time in microseconds since start of script -+ -+ - name of executable -+ -+ - PID of process -+ -+and the remainder indicates the call/return hierarchy as the script executes. -+ -+For a `--enable-shared` build of CPython, the markers are contained within the -+libpython shared library, and the probe's dotted path needs to reflect this. For -+example, this line from the above example:: -+ -+ probe process('python').mark("function__entry") { -+ -+should instead read:: -+ -+ probe process('python').library("libpython3.3dm.so.1.0").mark("function__entry") { -+ -+(assuming a debug build of CPython 3.3) -+ -+.. I'm reusing the "c:function" type for markers -+ -+.. c:function:: function__entry(str filename, str funcname, int lineno) -+ -+ This marker indicates that execution of a Python function has begun. It is -+ only triggered for pure-python (bytecode) functions. -+ -+ The filename, function name, and line number are provided back to the -+ tracing script as positional arguments, which must be accessed using -+ `$arg1`, `$arg2`: -+ -+ * `$arg1` : `(const char *)` filename, accessible using `user_string($arg1)` -+ -+ * `$arg2` : `(const char *)` function name, accessible using -+ `user_string($arg2)` -+ -+ * `$arg3` : `int` line number -+ -+ * `$arg4` : `(PyFrameObject *)`, the frame being executed -+ -+.. c:function:: function__return(str filename, str funcname, int lineno) -+ -+ This marker is the converse of `function__entry`, and indicates that -+ execution of a Python function has ended (either via ``return``, or via an -+ exception). It is only triggered for pure-python (bytecode) functions. -+ -+ The arguments are the same as for `function__entry` -+ -+ -+Tapsets -+------- -+ -+The higher-level way to use the SystemTap integration is to use a "tapset": -+SystemTap's equivalent of a library, which hides some of the lower-level -+details of the static markers. -+ -+Here is a tapset file, based on a non-shared build of CPython: -+ -+.. code-block:: c -+ -+ /* -+ Provide a higher-level wrapping around the function__entry and -+ function__return markers: -+ */ -+ probe python.function.entry = process("python").mark("function__entry") -+ { -+ filename = user_string($arg1); -+ funcname = user_string($arg2); -+ lineno = $arg3; -+ frameptr = $arg4 -+ } -+ probe python.function.return = process("python").mark("function__return") -+ { -+ filename = user_string($arg1); -+ funcname = user_string($arg2); -+ lineno = $arg3; -+ frameptr = $arg4 -+ } -+ -+If this file is installed in SystemTap's tapset directory (e.g. -+`/usr/share/systemtap/tapset`), then these additional probepoints become -+available: -+ -+.. c:function:: python.function.entry(str filename, str funcname, int lineno, frameptr) -+ -+ This probe point indicates that execution of a Python function has begun. -+ It is only triggered for pure-python (bytecode) functions. -+ -+.. c:function:: python.function.return(str filename, str funcname, int lineno, frameptr) -+ -+ This probe point is the converse of `python.function.return`, and indicates -+ that execution of a Python function has ended (either via ``return``, or -+ via an exception). It is only triggered for pure-python (bytecode) functions. -+ -+ -+Examples -+-------- -+This SystemTap script uses the tapset above to more cleanly implement the -+example given above of tracing the Python function-call hierarchy, without -+needing to directly name the static markers: -+ -+.. code-block:: c -+ -+ probe python.function.entry -+ { -+ printf("%s => %s in %s:%d\n", -+ thread_indent(1), funcname, filename, lineno); -+ } -+ -+ probe python.function.return -+ { -+ printf("%s <= %s in %s:%d\n", -+ thread_indent(-1), funcname, filename, lineno); -+ } -+ -+ -+The following script uses the tapset above to provide a top-like view of all -+running CPython code, showing the top 20 most frequently-entered bytecode -+frames, each second, across the whole system: -+ -+.. code-block:: c -+ -+ global fn_calls; -+ -+ probe python.function.entry -+ { -+ fn_calls[pid(), filename, funcname, lineno] += 1; -+ } -+ -+ probe timer.ms(1000) { -+ printf("\033[2J\033[1;1H") /* clear screen */ -+ printf("%6s %80s %6s %30s %6s\n", -+ "PID", "FILENAME", "LINE", "FUNCTION", "CALLS") -+ foreach ([pid, filename, funcname, lineno] in fn_calls- limit 20) { -+ printf("%6d %80s %6d %30s %6d\n", -+ pid, filename, lineno, funcname, -+ fn_calls[pid, filename, funcname, lineno]); -+ } -+ delete fn_calls; -+ } -+ -diff -up Python-3.3.0rc2/Lib/test/test_systemtap.py.systemtap Python-3.3.0rc2/Lib/test/test_systemtap.py ---- Python-3.3.0rc2/Lib/test/test_systemtap.py.systemtap 2012-09-10 09:17:21.117511779 -0400 -+++ Python-3.3.0rc2/Lib/test/test_systemtap.py 2012-09-10 09:17:21.117511779 -0400 -@@ -0,0 +1,234 @@ -+# Verify that systemtap static probes work -+# -+import subprocess -+import sys -+import sysconfig -+import os -+import unittest -+ -+from test.support import run_unittest, TESTFN, unlink -+ -+if '--with-systemtap' not in sysconfig.get_config_var('CONFIG_ARGS'): -+ raise unittest.SkipTest("Python was not configured --with-systemtap") -+ -+try: -+ _, stap_version = subprocess.Popen(["stap", "-V"], -+ stdout=subprocess.PIPE, -+ stderr=subprocess.PIPE, -+ ).communicate() -+except OSError: -+ # This is what "no stap" looks like. There may, however, be other -+ # errors that manifest this way too. -+ raise unittest.SkipTest("Couldn't find stap on the path") -+ -+def invoke_systemtap_script(script, cmd): -+ # Start a child process, probing with the given systemtap script -+ # (passed as stdin to the "stap" tool) -+ # The script should be a bytes instance -+ # Return (stdout, stderr) pair -+ -+ p = subprocess.Popen(["stap", "-", '-vv', '-c', cmd], -+ stdin=subprocess.PIPE, -+ stdout=subprocess.PIPE, -+ stderr=subprocess.PIPE) -+ out, err = p.communicate(input=script) -+ return out, err -+ -+# Verify that stap can run a simple "hello world"-style script -+# This can fail for various reasons: -+# - missing kernel headers -+# - permissions (a non-root user needs to be in the "stapdev" group) -+TRIVIAL_STAP_SCRIPT = b'probe begin { println("hello world") exit () }' -+ -+out, err = invoke_systemtap_script(TRIVIAL_STAP_SCRIPT, 'true') -+if out != b'hello world\n': -+ raise unittest.SkipTest("Test systemtap script did not run; stderr was: %s" % err) -+ -+# We don't expect stderr to be empty, since we're invoking stap with "-vv": stap -+# will (we hope) generate debugging output on stderr. -+ -+def invoke_python_under_systemtap(script, pythoncode=None, pythonfile=None): -+ # Start a child python process, probing with the given systemtap script -+ # (passed as stdin to the "stap" tool) -+ # The script should be a bytes instance -+ # Return (stdout, stderr) pair -+ -+ if pythonfile: -+ pythoncmd = '%s %s' % (sys.executable, pythonfile) -+ else: -+ pythoncmd = '%s -c %r' % (sys.executable, pythoncode) -+ -+ # The process tree of a stap invocation of a command goes through -+ # something like this: -+ # stap ->fork/exec(staprun; exec stapio ->f/e(-c cmd); exec staprun -r) -+ # and this trip through setuid leads to LD_LIBRARY_PATH being dropped, -+ # which would lead to an --enable-shared build of python failing to be -+ # find its libpython, with an error like: -+ # error while loading shared libraries: libpython3.3dm.so.1.0: cannot -+ # open shared object file: No such file or directory -+ # Hence we need to jump through some hoops to expose LD_LIBRARY_PATH to -+ # the invoked python process: -+ LD_LIBRARY_PATH = os.environ.get('LD_LIBRARY_PATH', '') -+ if LD_LIBRARY_PATH: -+ pythoncmd = 'env LD_LIBRARY_PATH=%s ' % LD_LIBRARY_PATH + pythoncmd -+ -+ return invoke_systemtap_script(script, pythoncmd) -+ -+# When using the static markers, we need to supply the prefix of a systemtap -+# dotted probe point that containing the marker. -+# See http://sourceware.org/systemtap/langref/Probe_points.html -+# -+# We need to determine if this is a shared-library build -+# -+# Note that sysconfig can get this wrong; see: -+# http://bugs.python.org/issue14774 -+# -+if '--enable-shared' in sysconfig.get_config_var('CONFIG_ARGS'): -+ # For a shared-library build, the markers are in library(INSTSONAME): -+ INSTSONAME = sysconfig.get_config_var('INSTSONAME') -+ probe_prefix = 'process("%s").library("%s")' % (sys.executable, INSTSONAME) -+else: -+ # For a non-shared-library build, we can simply use sys.executable: -+ probe_prefix = 'process("%s")' % sys.executable -+ -+# The following script ought to generate lots of lines showing recursive -+# function entry and return, of the form: -+# 11408 python(8274): => __contains__ in Lib/_abcoll.py:362 -+# 11414 python(8274): => __getitem__ in Lib/os.py:425 -+# 11418 python(8274): => encode in Lib/os.py:490 -+# 11424 python(8274): <= encode in Lib/os.py:493 -+# 11428 python(8274): <= __getitem__ in Lib/os.py:426 -+# 11433 python(8274): <= __contains__ in Lib/_abcoll.py:366 -+# where the column are: -+# - time in microseconds since start of script -+# - name of executable -+# - PID of process -+# and the remainder indicates the call/return hierarchy -+ -+hierarchy_script = (''' -+probe %s.mark("function__entry") { -+ filename = user_string($arg1); -+ funcname = user_string($arg2); -+ lineno = $arg3; -+ -+ printf("%%s => %%s in %%s:%%d\\n", thread_indent(1), funcname, filename, lineno); -+} -+ -+probe %s.mark("function__return") { -+ filename = user_string($arg1); -+ funcname = user_string($arg2); -+ lineno = $arg3; -+ -+ printf("%%s <= %%s in %%s:%%d\\n", thread_indent(-1), funcname, filename, lineno); -+} -+''' % (probe_prefix, probe_prefix)).encode('utf-8') -+ -+ -+class ErrorDumper: -+ # A context manager that dumps extra information if an exception is raised, -+ # to help track down why the problem occurred -+ def __init__(self, out, err): -+ self.out = out -+ self.err = err -+ -+ def __enter__(self): -+ pass -+ -+ def __exit__(self, type_, value, traceback): -+ if type_: -+ # an exception is being raised: -+ print('stdout: %s' % out.decode()) -+ print('stderr: %s' % err.decode()) -+ -+class SystemtapTests(unittest.TestCase): -+ -+ def test_invoking_python(self): -+ # Ensure that we can invoke python under stap, with a trivial stap -+ # script: -+ out, err = invoke_python_under_systemtap( -+ b'probe begin { println("hello from stap") exit () }', -+ pythoncode="print('hello from python')") -+ with ErrorDumper(out, err): -+ self.assertIn(b'hello from stap', out) -+ self.assertIn(b'hello from python', out) -+ -+ def test_function_entry(self): -+ # Ensure that the function_entry static marker works -+ out, err = invoke_python_under_systemtap(hierarchy_script) -+ # stdout ought to contain various lines showing recursive function -+ # entry and return (see above) -+ -+ # Uncomment this for debugging purposes: -+ # print(out.decode('utf-8')) -+ -+ # Executing the cmdline-supplied "pass": -+ # 0 python(8274): => in :1 -+ # 5 python(8274): <= in :1 -+ with ErrorDumper(out, err): -+ self.assertIn(b'=> in :1', out, -+ msg="stdout: %s\nstderr: %s\n" % (out, err)) -+ -+ def test_function_encoding(self): -+ # Ensure that function names containing non-Latin 1 code -+ # points are handled: -+ pythonfile = TESTFN -+ try: -+ unlink(pythonfile) -+ f = open(pythonfile, "wb") -+ f.write(""" -+# Sample script with non-ASCII filename, for use by test_systemtap.py -+# Implicitly UTF-8 -+ -+def 文字化け(): -+ '''Function with non-ASCII identifier; I believe this reads "mojibake"''' -+ print("hello world!") -+ -+文字化け() -+""".encode('utf-8')) -+ f.close() -+ -+ out, err = invoke_python_under_systemtap(hierarchy_script, -+ pythonfile=pythonfile) -+ out_utf8 = out.decode('utf-8') -+ with ErrorDumper(out, err): -+ self.assertIn('=> in %s:5' % pythonfile, out_utf8) -+ self.assertIn(' => 文字化け in %s:5' % pythonfile, out_utf8) -+ self.assertIn(' <= 文字化け in %s:7' % pythonfile, out_utf8) -+ self.assertIn('<= in %s:9' % pythonfile, out_utf8) -+ finally: -+ unlink(pythonfile) -+ -+ @unittest.skipIf(sys.getfilesystemencoding() == 'ascii', -+ 'the test filename is not encodable with ASCII') -+ def test_filename_encoding(self): -+ # Ensure that scripts names containing non-Latin 1 code -+ # points are handled: -+ pythonfile = TESTFN + '_☠.py' -+ try: -+ unlink(pythonfile) -+ f = open(pythonfile, "wb") -+ f.write(""" -+def foo(): -+ '''Function with non-ASCII identifier; I believe this reads "mojibake"''' -+ print("hello world!") -+ -+foo() -+""".encode('utf-8')) -+ f.close() -+ -+ out, err = invoke_python_under_systemtap(hierarchy_script, -+ pythonfile=pythonfile) -+ out_utf8 = out.decode('utf-8') -+ with ErrorDumper(out, err): -+ self.assertIn('=> in %s:2' % pythonfile, out_utf8) -+ self.assertIn(' => foo in %s:2' % pythonfile, out_utf8) -+ self.assertIn(' <= foo in %s:4' % pythonfile, out_utf8) -+ self.assertIn('<= in %s:6' % pythonfile, out_utf8) -+ finally: -+ unlink(pythonfile) -+ -+def test_main(): -+ run_unittest(SystemtapTests) -+ -+if __name__ == "__main__": -+ test_main() -diff -up Python-3.3.0rc2/Makefile.pre.in.systemtap Python-3.3.0rc2/Makefile.pre.in ---- Python-3.3.0rc2/Makefile.pre.in.systemtap 2012-09-09 05:11:05.000000000 -0400 -+++ Python-3.3.0rc2/Makefile.pre.in 2012-09-10 09:19:51.195501518 -0400 -@@ -363,6 +363,7 @@ PYTHON_OBJS= \ ++#pragma D attributes Evolving/Evolving/Common provider python provider ++#pragma D attributes Private/Private/Common provider python module ++#pragma D attributes Private/Private/Common provider python function ++#pragma D attributes Evolving/Evolving/Common provider python name ++#pragma D attributes Evolving/Evolving/Common provider python args +diff -up Python-2.7rc1/Makefile.pre.in.systemtap Python-2.7rc1/Makefile.pre.in +--- Python-2.7rc1/Makefile.pre.in.systemtap 2010-06-06 10:53:15.488978775 -0400 ++++ Python-2.7rc1/Makefile.pre.in 2010-06-06 11:05:30.411100568 -0400 +@@ -298,6 +298,7 @@ PYTHON_OBJS= \ Python/formatter_unicode.o \ - Python/fileutils.o \ + Python/formatter_string.o \ Python/$(DYNLOADFILE) \ -+ @SYSTEMTAPOBJS@ \ ++ @DTRACEOBJS@ \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ $(THREADOBJ) -@@ -713,7 +714,8 @@ Objects/setobject.o: $(srcdir)/Objects/s - $(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES) - $(OPCODETARGETGEN) $(OPCODETARGETS_H) - --Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h -+Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h \ -+ $(srcdir)/Python/ceval_systemtap.h @SYSTEMTAPDEPS@ +@@ -599,6 +600,18 @@ Python/formatter_unicode.o: $(srcdir)/Py + Python/formatter_string.o: $(srcdir)/Python/formatter_string.c \ + $(STRINGLIB_HEADERS) - Python/frozen.o: Python/importlib.h Python/importlib_external.h - -@@ -724,6 +726,13 @@ Objects/typeobject.o: $(srcdir)/Objects/ - Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py - $(PYTHON) $(srcdir)/Objects/typeslots.py < $(srcdir)/Include/typeslots.h > Objects/typeslots.inc - -+# Only needed with --with-systemtap; not a public header: -+$(srcdir)/Python/pysystemtap.h: $(srcdir)/Python/pysystemtap.d -+ dtrace -o $@ $(DFLAGS) -C -h -s $(srcdir)/Python/pysystemtap.d ++# Only needed with --with-dtrace ++buildinclude: ++ mkdir -p Include ++ ++Include/pydtrace.h: buildinclude $(srcdir)/Include/pydtrace.d ++ dtrace -o $@ $(DFLAGS) -C -h -s $(srcdir)/Include/pydtrace.d ++ ++Python/ceval.o: Include/pydtrace.h + -+Python/pysystemtap.o: $(srcdir)/Python/pysystemtap.d Python/ceval.o -+ dtrace -o $@ $(DFLAGS) -C -G -s $(srcdir)/Python/pysystemtap.d Python/ceval.o ++Python/dtrace.o: buildinclude $(srcdir)/Include/pydtrace.d Python/ceval.o ++ dtrace -o $@ $(DFLAGS) -C -G -s $(srcdir)/Include/pydtrace.d Python/ceval.o + ############################################################################ # Header files -@@ -1345,6 +1354,7 @@ clean: pycremoval - -rm -f Lib/lib2to3/*Grammar*.pickle - -rm -f Programs/_testembed Programs/_freeze_importlib - -rm -rf build -+ -rm -f $(srcdir)/Python/pysystemtap.h - - profile-removal: - find . -name '*.gc??' -exec rm -f {} ';' -diff -up Python-3.3.0rc2/pyconfig.h.in.systemtap Python-3.3.0rc2/pyconfig.h.in ---- Python-3.3.0rc2/pyconfig.h.in.systemtap 2012-09-09 05:11:14.000000000 -0400 -+++ Python-3.3.0rc2/pyconfig.h.in 2012-09-10 09:17:21.120511781 -0400 -@@ -1306,6 +1306,9 @@ - /* Define if you want to compile in Python-specific mallocs */ - #undef WITH_PYMALLOC - -+/* Define if you want to compile in SystemTap support */ -+#undef WITH_SYSTEMTAP -+ - /* Define if you want to compile in rudimentary thread support */ - #undef WITH_THREAD - -diff -up Python-3.3.0rc2/Python/ceval.c.systemtap Python-3.3.0rc2/Python/ceval.c ---- Python-3.3.0rc2/Python/ceval.c.systemtap 2012-09-09 05:11:12.000000000 -0400 -+++ Python-3.3.0rc2/Python/ceval.c 2012-09-10 09:17:21.122511781 -0400 -@@ -18,6 +18,8 @@ +@@ -1251,7 +1264,7 @@ Python/thread.o: @THREADHEADERS@ + .PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure + .PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools + .PHONY: frameworkaltinstallunixtools recheck autoconf clean clobber distclean +-.PHONY: smelly funny patchcheck touch altmaninstall ++.PHONY: smelly funny patchcheck touch altmaninstall buildinclude + .PHONY: gdbhooks + + # IF YOU PUT ANYTHING HERE IT WILL GO AWAY +diff -up Python-2.7rc1/pyconfig.h.in.systemtap Python-2.7rc1/pyconfig.h.in +--- Python-2.7rc1/pyconfig.h.in.systemtap 2010-05-08 07:04:18.000000000 -0400 ++++ Python-2.7rc1/pyconfig.h.in 2010-06-06 10:53:15.521974070 -0400 +@@ -1074,6 +1074,9 @@ + /* Define if you want documentation strings in extension modules */ + #undef WITH_DOC_STRINGS + ++/* Define if you want to compile in Dtrace support */ ++#undef WITH_DTRACE ++ + /* Define if you want to use the new-style (Openstep, Rhapsody, MacOS) dynamic + linker (dyld) instead of the old-style (NextStep) dynamic linker (rld). + Dyld is necessary to support frameworks. */ +diff -up Python-2.7rc1/Python/ceval.c.systemtap Python-2.7rc1/Python/ceval.c +--- Python-2.7rc1/Python/ceval.c.systemtap 2010-05-09 10:46:46.000000000 -0400 ++++ Python-2.7rc1/Python/ceval.c 2010-06-06 11:08:40.683100500 -0400 +@@ -19,6 +19,10 @@ #include -+#include "ceval_systemtap.h" ++#ifdef WITH_DTRACE ++#include "pydtrace.h" ++#endif + #ifndef WITH_TSC #define READ_TIMESTAMP(var) -@@ -1160,6 +1162,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int - } - } - -+ if (PYTHON_FUNCTION_ENTRY_ENABLED()) { -+ systemtap_function_entry(f); -+ } -+ - co = f->f_code; - names = co->co_names; - consts = co->co_consts; -@@ -3077,6 +3083,11 @@ fast_yield: +@@ -671,6 +675,55 @@ PyEval_EvalCode(PyCodeObject *co, PyObje + NULL); + } - /* pop frame */ - exit_eval_frame: -+ -+ if (PYTHON_FUNCTION_RETURN_ENABLED()) { -+ systemtap_function_return(f); -+ } -+ - Py_LeaveRecursiveCall(); - f->f_executing = 0; - tstate->frame = f->f_back; -diff -up Python-3.3.0rc2/Python/ceval_systemtap.h.systemtap Python-3.3.0rc2/Python/ceval_systemtap.h ---- Python-3.3.0rc2/Python/ceval_systemtap.h.systemtap 2012-09-10 09:17:21.122511781 -0400 -+++ Python-3.3.0rc2/Python/ceval_systemtap.h 2012-09-10 09:17:21.122511781 -0400 -@@ -0,0 +1,86 @@ -+/* -+ Support for SystemTap static markers -+*/ -+ -+#ifdef WITH_SYSTEMTAP -+ -+#include "pysystemtap.h" -+ -+/* -+ A struct to hold all of the information gathered when one of the traceable -+ markers is triggered -+*/ -+struct frame_marker_info ++#ifdef WITH_DTRACE ++static void ++dtrace_entry(PyFrameObject *f) +{ -+ PyObject *filename_obj; -+ PyObject *funcname_obj; + const char *filename; -+ const char *funcname; ++ const char *fname; + int lineno; -+}; + -+static void -+get_frame_marker_info(PyFrameObject *f, struct frame_marker_info *fmi) -+{ -+ PyObject *ptype; -+ PyObject *pvalue; -+ PyObject *ptraceback; -+ -+ PyErr_Fetch(&ptype, &pvalue, &ptraceback); -+ -+ fmi->filename_obj = PyUnicode_EncodeFSDefault(f->f_code->co_filename); -+ if (fmi->filename_obj) { -+ fmi->filename = PyBytes_AsString(fmi->filename_obj); -+ } else { -+ fmi->filename = NULL; -+ } ++ filename = PyString_AsString(f->f_code->co_filename); ++ fname = PyString_AsString(f->f_code->co_name); ++ lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); + -+ fmi->funcname_obj = PyUnicode_AsUTF8String(f->f_code->co_name); -+ if (fmi->funcname_obj) { -+ fmi->funcname = PyBytes_AsString(fmi->funcname_obj); -+ } else { -+ fmi->funcname = NULL; -+ } -+ -+ fmi->lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); -+ -+ PyErr_Restore(ptype, pvalue, ptraceback); ++ PYTHON_FUNCTION_ENTRY((char *)filename, (char *)fname, lineno); + ++ /* ++ * Currently a USDT tail-call will not receive the correct arguments. ++ * Disable the tail call here. ++ */ ++#if defined(__sparc) ++ asm("nop"); ++#endif +} + +static void -+release_frame_marker_info(struct frame_marker_info *fmi) ++dtrace_return(PyFrameObject *f) +{ -+ Py_XDECREF(fmi->filename_obj); -+ Py_XDECREF(fmi->funcname_obj); -+} ++ const char *filename; ++ const char *fname; ++ int lineno; + -+static void -+systemtap_function_entry(PyFrameObject *f) -+{ -+ struct frame_marker_info fmi; -+ get_frame_marker_info(f, &fmi); -+ PYTHON_FUNCTION_ENTRY(fmi.filename, fmi.funcname, fmi.lineno, f); -+ release_frame_marker_info(&fmi); -+} ++ filename = PyString_AsString(f->f_code->co_filename); ++ fname = PyString_AsString(f->f_code->co_name); ++ lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); ++ PYTHON_FUNCTION_RETURN((char *)filename, (char *)fname, lineno); + -+static void -+systemtap_function_return(PyFrameObject *f) -+{ -+ struct frame_marker_info fmi; -+ get_frame_marker_info(f, &fmi); -+ PYTHON_FUNCTION_RETURN(fmi.filename, fmi.funcname, fmi.lineno, f); -+ release_frame_marker_info(&fmi); ++ /* ++ * Currently a USDT tail-call will not receive the correct arguments. ++ * Disable the tail call here. ++ */ ++#if defined(__sparc) ++ asm("nop"); ++#endif +} ++#else ++#define PYTHON_FUNCTION_ENTRY_ENABLED() 0 ++#define PYTHON_FUNCTION_RETURN_ENABLED() 0 ++#define dtrace_entry(f) ++#define dtrace_return(f) ++#endif + + /* Interpreter main loop */ + +@@ -909,6 +962,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int + } + } + ++ if (PYTHON_FUNCTION_ENTRY_ENABLED()) ++ dtrace_entry(f); + -+#else /* #ifdef WITH_SYSTEMTAP */ -+ -+/* -+ When configured --without-systemtap, everything compiles away to nothing: -+*/ -+#define PYTHON_FUNCTION_ENTRY_ENABLED() 0 -+#define PYTHON_FUNCTION_RETURN_ENABLED() 0 -+#define systemtap_function_entry(f) -+#define systemtap_function_return(f) + co = f->f_code; + names = co->co_names; + consts = co->co_consts; +@@ -3000,6 +3056,9 @@ fast_yield: + + /* pop frame */ + exit_eval_frame: ++ if (PYTHON_FUNCTION_RETURN_ENABLED()) ++ dtrace_return(f); + -+#endif -diff -up Python-3.3.0rc2/Python/pysystemtap.d.systemtap Python-3.3.0rc2/Python/pysystemtap.d ---- Python-3.3.0rc2/Python/pysystemtap.d.systemtap 2012-09-10 09:17:21.122511781 -0400 -+++ Python-3.3.0rc2/Python/pysystemtap.d 2012-09-10 09:17:21.122511781 -0400 -@@ -0,0 +1,4 @@ -+provider python { -+ probe function__entry(const char *, const char *, int, PyFrameObject *); -+ probe function__return(const char *, const char *, int, PyFrameObject *); -+}; + Py_LeaveRecursiveCall(); + tstate->frame = f->f_back; + diff --git a/SOURCES/00111-no-static-lib.patch b/SOURCES/00111-no-static-lib.patch index bc4203de..2f4fdd68 100644 --- a/SOURCES/00111-no-static-lib.patch +++ b/SOURCES/00111-no-static-lib.patch @@ -1,20 +1,19 @@ -diff --git a/Makefile.pre.in b/Makefile.pre.in -index 4b093e3..1088435 100644 ---- a/Makefile.pre.in -+++ b/Makefile.pre.in -@@ -543,7 +543,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c - $(PYTHON_FOR_REGEN) ./Tools/clinic/clinic.py --make +diff -up Python-2.7.3/Makefile.pre.in.no-static-lib Python-2.7.3/Makefile.pre.in +--- Python-2.7.3/Makefile.pre.in.no-static-lib 2013-02-19 14:03:40.801993224 -0500 ++++ Python-2.7.3/Makefile.pre.in 2013-02-19 14:04:44.070988898 -0500 +@@ -397,7 +397,7 @@ coverage: - # Build the interpreter --$(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) -+$(BUILDPYTHON): Programs/python.o $(LDLIBRARY) $(PY3LIBRARY) - $(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) - platform: $(BUILDPYTHON) pybuilddir.txt -@@ -588,18 +588,6 @@ sharedmods: $(BUILDPYTHON) pybuilddir.txt Modules/_math.o + # Build the interpreter +-$(BUILDPYTHON): Modules/python.o $(LIBRARY) $(LDLIBRARY) ++$(BUILDPYTHON): Modules/python.o $(LDLIBRARY) + $(LINKCC) $(CFLAGS) $(LDFLAGS) $(LINKFORSHARED) -o $@ \ + Modules/python.o \ + $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) +@@ -413,18 +413,6 @@ sharedmods: $(BUILDPYTHON) + $(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \ $(PYTHON_FOR_BUILD) $(srcdir)/setup.py $$quiet build - -# Build static library -# avoid long command lines, same as LIBRARY_OBJS -$(LIBRARY): $(LIBRARY_OBJS) @@ -22,30 +21,21 @@ index 4b093e3..1088435 100644 - $(AR) $(ARFLAGS) $@ Modules/getbuildinfo.o - $(AR) $(ARFLAGS) $@ $(PARSER_OBJS) - $(AR) $(ARFLAGS) $@ $(OBJECT_OBJS) -- $(AR) $(ARFLAGS) $@ $(PYTHON_OBJS) Python/frozen.o -- $(AR) $(ARFLAGS) $@ $(MODULE_OBJS) +- $(AR) $(ARFLAGS) $@ $(PYTHON_OBJS) +- $(AR) $(ARFLAGS) $@ $(MODULE_OBJS) $(SIGNAL_OBJS) - $(AR) $(ARFLAGS) $@ $(MODOBJS) - $(RANLIB) $@ - - libpython$(LDVERSION).so: $(LIBRARY_OBJS) + libpython$(VERSION).so: $(LIBRARY_OBJS) if test $(INSTSONAME) != $(LDLIBRARY); then \ $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) $(LDLAST); \ -@@ -689,7 +677,7 @@ Modules/Setup: $(srcdir)/Modules/Setup.dist - echo "-----------------------------------------------"; \ - fi - --Programs/_testembed: Programs/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) -+Programs/_testembed: Programs/_testembed.o $(LDLIBRARY) $(PY3LIBRARY) - $(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) - - ############################################################################ -@@ -1425,18 +1413,6 @@ libainstall: @DEF_MAKE_RULE@ python-config +@@ -1021,18 +1009,6 @@ libainstall: all python-config else true; \ fi; \ done - @if test -d $(LIBRARY); then :; else \ - if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ -- if test "$(SHLIB_SUFFIX)" = .dll; then \ +- if test "$(SO)" = .dll; then \ - $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBPL) ; \ - else \ - $(INSTALL_DATA) $(LIBRARY) $(DESTDIR)$(LIBPL)/$(LIBRARY) ; \ @@ -56,5 +46,5 @@ index 4b093e3..1088435 100644 - fi; \ - fi $(INSTALL_DATA) Modules/config.c $(DESTDIR)$(LIBPL)/config.c - $(INSTALL_DATA) Programs/python.o $(DESTDIR)$(LIBPL)/python.o + $(INSTALL_DATA) Modules/python.o $(DESTDIR)$(LIBPL)/python.o $(INSTALL_DATA) $(srcdir)/Modules/config.c.in $(DESTDIR)$(LIBPL)/config.c.in diff --git a/SOURCES/00113-more-configuration-flags.patch b/SOURCES/00113-more-configuration-flags.patch new file mode 100644 index 00000000..2d447b24 --- /dev/null +++ b/SOURCES/00113-more-configuration-flags.patch @@ -0,0 +1,50 @@ +diff -up Python-2.6.5/configure.ac.more-configuration-flags Python-2.6.5/configure.ac +--- Python-2.6.5/configure.ac.more-configuration-flags 2010-05-24 18:51:25.410111792 -0400 ++++ Python-2.6.5/configure.ac 2010-05-24 18:59:23.954986388 -0400 +@@ -2515,6 +2515,30 @@ else AC_MSG_RESULT(no) + fi], + [AC_MSG_RESULT(no)]) + ++AC_MSG_CHECKING(for --with-count-allocs) ++AC_ARG_WITH(count-allocs, ++[ --with(out)count-allocs enable/disable per-type instance accounting], [ ++if test "$withval" != no ++then ++ AC_DEFINE(COUNT_ALLOCS, 1, ++ [Define to keep records of the number of instances of each type]) ++ AC_MSG_RESULT(yes) ++else AC_MSG_RESULT(no) ++fi], ++[AC_MSG_RESULT(no)]) ++ ++AC_MSG_CHECKING(for --with-call-profile) ++AC_ARG_WITH(call-profile, ++[ --with(out)-call-profile enable/disable statistics on function call invocation], [ ++if test "$withval" != no ++then ++ AC_DEFINE(CALL_PROFILE, 1, ++ [Define to keep records on function call invocation]) ++ AC_MSG_RESULT(yes) ++else AC_MSG_RESULT(no) ++fi], ++[AC_MSG_RESULT(no)]) ++ + # Check for Python-specific malloc support + AC_MSG_CHECKING(for --with-pymalloc) + AC_ARG_WITH(pymalloc, +diff -up Python-2.6.5/pyconfig.h.in.more-configuration-flags Python-2.6.5/pyconfig.h.in +--- Python-2.6.5/pyconfig.h.in.more-configuration-flags 2010-05-24 18:51:45.677988086 -0400 ++++ Python-2.6.5/pyconfig.h.in 2010-05-24 19:00:44.163987730 -0400 +@@ -1019,6 +1019,12 @@ + /* Define to profile with the Pentium timestamp counter */ + #undef WITH_TSC + ++/* Define to keep records of the number of instances of each type */ ++#undef COUNT_ALLOCS ++ ++/* Define to keep records on function call invocation */ ++#undef CALL_PROFILE ++ + /* Define if you want pymalloc to be disabled when running under valgrind */ + #undef WITH_VALGRIND + diff --git a/SOURCES/00114-statvfs-f_flag-constants.patch b/SOURCES/00114-statvfs-f_flag-constants.patch new file mode 100644 index 00000000..83e7b598 --- /dev/null +++ b/SOURCES/00114-statvfs-f_flag-constants.patch @@ -0,0 +1,47 @@ +diff -up Python-2.7rc1/Modules/posixmodule.c.statvfs-f-flag-constants Python-2.7rc1/Modules/posixmodule.c +--- Python-2.7rc1/Modules/posixmodule.c.statvfs-f-flag-constants 2010-05-15 17:45:30.000000000 -0400 ++++ Python-2.7rc1/Modules/posixmodule.c 2010-06-07 22:54:16.162068624 -0400 +@@ -9174,6 +9174,43 @@ all_ins(PyObject *d) + #endif + #endif + ++ /* These came from statvfs.h */ ++#ifdef ST_RDONLY ++ if (ins(d, "ST_RDONLY", (long)ST_RDONLY)) return -1; ++#endif /* ST_RDONLY */ ++#ifdef ST_NOSUID ++ if (ins(d, "ST_NOSUID", (long)ST_NOSUID)) return -1; ++#endif /* ST_NOSUID */ ++ ++ /* GNU extensions */ ++#ifdef ST_NODEV ++ if (ins(d, "ST_NODEV", (long)ST_NODEV)) return -1; ++#endif /* ST_NODEV */ ++#ifdef ST_NOEXEC ++ if (ins(d, "ST_NOEXEC", (long)ST_NOEXEC)) return -1; ++#endif /* ST_NOEXEC */ ++#ifdef ST_SYNCHRONOUS ++ if (ins(d, "ST_SYNCHRONOUS", (long)ST_SYNCHRONOUS)) return -1; ++#endif /* ST_SYNCHRONOUS */ ++#ifdef ST_MANDLOCK ++ if (ins(d, "ST_MANDLOCK", (long)ST_MANDLOCK)) return -1; ++#endif /* ST_MANDLOCK */ ++#ifdef ST_WRITE ++ if (ins(d, "ST_WRITE", (long)ST_WRITE)) return -1; ++#endif /* ST_WRITE */ ++#ifdef ST_APPEND ++ if (ins(d, "ST_APPEND", (long)ST_APPEND)) return -1; ++#endif /* ST_APPEND */ ++#ifdef ST_NOATIME ++ if (ins(d, "ST_NOATIME", (long)ST_NOATIME)) return -1; ++#endif /* ST_NOATIME */ ++#ifdef ST_NODIRATIME ++ if (ins(d, "ST_NODIRATIME", (long)ST_NODIRATIME)) return -1; ++#endif /* ST_NODIRATIME */ ++#ifdef ST_RELATIME ++ if (ins(d, "ST_RELATIME", (long)ST_RELATIME)) return -1; ++#endif /* ST_RELATIME */ ++ + #if defined(PYOS_OS2) + if (insertvalues(d)) return -1; + #endif diff --git a/SOURCES/00121-add-Modules-to-build-path.patch b/SOURCES/00121-add-Modules-to-build-path.patch new file mode 100644 index 00000000..6e3294db --- /dev/null +++ b/SOURCES/00121-add-Modules-to-build-path.patch @@ -0,0 +1,13 @@ +--- Python-2.7.5/Lib/site.py.orig 2013-05-16 12:47:55.000000000 +0200 ++++ Python-2.7.5/Lib/site.py 2013-05-16 12:56:20.089058109 +0200 +@@ -529,6 +529,10 @@ def main(): + + abs__file__() + known_paths = removeduppaths() ++ from sysconfig import is_python_build ++ if is_python_build(): ++ from _sysconfigdata import build_time_vars ++ sys.path.append(os.path.join(build_time_vars['abs_builddir'], 'Modules')) + if ENABLE_USER_SITE is None: + ENABLE_USER_SITE = check_enableusersite() + known_paths = addusersitepackages(known_paths) diff --git a/SOURCES/00125-less-verbose-COUNT_ALLOCS.patch b/SOURCES/00125-less-verbose-COUNT_ALLOCS.patch new file mode 100644 index 00000000..8cef0155 --- /dev/null +++ b/SOURCES/00125-less-verbose-COUNT_ALLOCS.patch @@ -0,0 +1,20 @@ +diff -up Python-2.7/Python/pythonrun.c.less-verbose-COUNT_ALLOCS Python-2.7/Python/pythonrun.c +--- Python-2.7/Python/pythonrun.c.less-verbose-COUNT_ALLOCS 2010-08-17 14:49:33.321913909 -0400 ++++ Python-2.7/Python/pythonrun.c 2010-08-17 14:54:48.750910403 -0400 +@@ -470,7 +470,15 @@ Py_Finalize(void) + + /* Debugging stuff */ + #ifdef COUNT_ALLOCS +- dump_counts(stdout); ++ /* This is a downstream Fedora modification. ++ The upstream default with COUNT_ALLOCS is to always dump the counts to ++ stdout on exit. For our debug builds its useful to have the info from ++ COUNT_ALLOCS available, but the stdout info here gets in the way, so ++ we make it optional, wrapping it in an environment variable (modelled ++ on the other PYTHONDUMP* env variables): ++ */ ++ if (Py_GETENV("PYTHONDUMPCOUNTS")) ++ dump_counts(stdout); + #endif + + PRINT_TOTAL_REFS(); diff --git a/SOURCES/00131-disable-tests-in-test_io.patch b/SOURCES/00131-disable-tests-in-test_io.patch new file mode 100644 index 00000000..d81a2d0c --- /dev/null +++ b/SOURCES/00131-disable-tests-in-test_io.patch @@ -0,0 +1,11 @@ +diff -up Python-2.7.2/Lib/test/test_io.py.disable-tests-in-test_io Python-2.7.2/Lib/test/test_io.py +--- Python-2.7.2/Lib/test/test_io.py.disable-tests-in-test_io 2011-09-01 14:18:45.963304089 -0400 ++++ Python-2.7.2/Lib/test/test_io.py 2011-09-01 15:08:53.796098413 -0400 +@@ -2669,6 +2669,7 @@ class SignalsTest(unittest.TestCase): + self.check_interrupted_read_retry(lambda x: x, + mode="r") + ++ @unittest.skip('rhbz#732998') + @unittest.skipUnless(threading, 'Threading required for this test.') + def check_interrupted_write_retry(self, item, **fdopen_kwargs): + """Check that a buffered write, when it gets interrupted (either diff --git a/SOURCES/00132-add-rpmbuild-hooks-to-unittest.patch b/SOURCES/00132-add-rpmbuild-hooks-to-unittest.patch index 77dc6ecb..e63395fb 100644 --- a/SOURCES/00132-add-rpmbuild-hooks-to-unittest.patch +++ b/SOURCES/00132-add-rpmbuild-hooks-to-unittest.patch @@ -1,16 +1,17 @@ -diff -up Python-3.2.2/Lib/unittest/case.py.add-rpmbuild-hooks-to-unittest Python-3.2.2/Lib/unittest/case.py ---- Python-3.2.2/Lib/unittest/case.py.add-rpmbuild-hooks-to-unittest 2011-09-03 12:16:44.000000000 -0400 -+++ Python-3.2.2/Lib/unittest/case.py 2011-09-09 06:35:16.365568382 -0400 -@@ -3,6 +3,7 @@ +diff -up Python-2.7.2/Lib/unittest/case.py.add-rpmbuild-hooks-to-unittest Python-2.7.2/Lib/unittest/case.py +--- Python-2.7.2/Lib/unittest/case.py.add-rpmbuild-hooks-to-unittest 2011-09-08 14:45:47.677169191 -0400 ++++ Python-2.7.2/Lib/unittest/case.py 2011-09-08 16:01:36.287858159 -0400 +@@ -1,6 +1,7 @@ + """Test case implementation""" + + import collections ++import os import sys import functools import difflib -+import os - import logging - import pprint - import re -@@ -101,5 +102,21 @@ def expectedFailure(func): - raise self.test_case.failureException(msg) +@@ -94,6 +95,43 @@ def expectedFailure(func): + return wrapper + +# Non-standard/downstream-only hooks for handling issues with specific test +# cases: @@ -28,19 +29,40 @@ diff -up Python-3.2.2/Lib/unittest/case.py.add-rpmbuild-hooks-to-unittest Python + else: + return _id + - class _AssertRaisesBaseContext(_BaseTestCaseContext): ++def _expectedFailureInRpmBuild(func): ++ """ ++ Non-standard/downstream-only decorator for marking a specific unit test ++ as expected to fail within the %check of an rpmbuild. ++ ++ Specifically, this takes effect when WITHIN_PYTHON_RPM_BUILD is set within ++ the environment, and has no effect otherwise. ++ """ ++ @functools.wraps(func) ++ def wrapper(*args, **kwargs): ++ if 'WITHIN_PYTHON_RPM_BUILD' in os.environ: ++ try: ++ func(*args, **kwargs) ++ except Exception: ++ raise _ExpectedFailure(sys.exc_info()) ++ raise _UnexpectedSuccess ++ else: ++ # Call directly: ++ func(*args, **kwargs) ++ return wrapper ++ + class _AssertRaisesContext(object): + """A context manager used to implement TestCase.assertRaises* methods.""" - def __init__(self, expected, test_case, expected_regex=None): -diff -up Python-3.2.2/Lib/unittest/__init__.py.add-rpmbuild-hooks-to-unittest Python-3.2.2/Lib/unittest/__init__.py ---- Python-3.2.2/Lib/unittest/__init__.py.add-rpmbuild-hooks-to-unittest 2011-09-03 12:16:44.000000000 -0400 -+++ Python-3.2.2/Lib/unittest/__init__.py 2011-09-09 06:35:16.366568382 -0400 +diff -up Python-2.7.2/Lib/unittest/__init__.py.add-rpmbuild-hooks-to-unittest Python-2.7.2/Lib/unittest/__init__.py +--- Python-2.7.2/Lib/unittest/__init__.py.add-rpmbuild-hooks-to-unittest 2011-09-08 14:59:39.534112310 -0400 ++++ Python-2.7.2/Lib/unittest/__init__.py 2011-09-08 15:07:09.191081562 -0400 @@ -57,7 +57,8 @@ __unittest = True from .result import TestResult from .case import (TestCase, FunctionTestCase, SkipTest, skip, skipIf, - skipUnless, expectedFailure) + skipUnless, expectedFailure, -+ _skipInRpmBuild) ++ _skipInRpmBuild, _expectedFailureInRpmBuild) from .suite import BaseTestSuite, TestSuite from .loader import (TestLoader, defaultTestLoader, makeSuite, getTestCaseNames, findTestCases) diff --git a/SOURCES/00133-skip-test_dl.patch b/SOURCES/00133-skip-test_dl.patch new file mode 100644 index 00000000..04ad05b9 --- /dev/null +++ b/SOURCES/00133-skip-test_dl.patch @@ -0,0 +1,13 @@ +diff -up Python-2.7.2/Lib/test/test_dl.py.skip-test_dl Python-2.7.2/Lib/test/test_dl.py +--- Python-2.7.2/Lib/test/test_dl.py.skip-test_dl 2011-09-08 15:18:40.529034289 -0400 ++++ Python-2.7.2/Lib/test/test_dl.py 2011-09-08 16:29:45.184742670 -0400 +@@ -13,6 +13,9 @@ sharedlibs = [ + ('/usr/lib/libc.dylib', 'getpid'), + ] + ++# (also, "dl" is deprecated in favor of ctypes) ++@unittest._skipInRpmBuild('fails on 64-bit builds: ' ++ 'module dl requires sizeof(int) == sizeof(long) == sizeof(char*)') + def test_main(): + for s, func in sharedlibs: + try: diff --git a/SOURCES/00134-fix-COUNT_ALLOCS-failure-in-test_sys.patch b/SOURCES/00134-fix-COUNT_ALLOCS-failure-in-test_sys.patch new file mode 100644 index 00000000..38381ef2 --- /dev/null +++ b/SOURCES/00134-fix-COUNT_ALLOCS-failure-in-test_sys.patch @@ -0,0 +1,14 @@ +--- Python-2.7.2/Lib/test/test_sys.py.mark-tests-that-fail-in-rpmbuild 2011-09-08 18:02:31.627362039 -0400 ++++ Python-2.7.2/Lib/test/test_sys.py 2011-09-08 18:15:29.450308851 -0400 +@@ -734,6 +734,11 @@ class SizeofTest(unittest.TestCase): + # (PyTypeObject + PyNumberMethods + PyMappingMethods + + # PySequenceMethods + PyBufferProcs) + s = vsize('P2P15Pl4PP9PP11PI') + struct.calcsize('41P 10P 3P 6P') ++ ++ # COUNT_ALLOCS adds further fields to the end of a PyTypeObject: ++ if hasattr(sys, 'getcounts'): ++ s += size('P') ++ + class newstyleclass(object): + pass + check(newstyleclass, s) diff --git a/SOURCES/00135-skip-test-within-test_weakref-in-debug-build.patch b/SOURCES/00135-skip-test-within-test_weakref-in-debug-build.patch new file mode 100644 index 00000000..e464aa98 --- /dev/null +++ b/SOURCES/00135-skip-test-within-test_weakref-in-debug-build.patch @@ -0,0 +1,18 @@ +diff -up Python-2.7.2/Lib/test/test_weakref.py.skip-test-within-test_weakref-in-debug-build Python-2.7.2/Lib/test/test_weakref.py +--- Python-2.7.2/Lib/test/test_weakref.py.skip-test-within-test_weakref-in-debug-build 2011-09-08 17:55:09.675392260 -0400 ++++ Python-2.7.2/Lib/test/test_weakref.py 2011-09-08 17:59:08.857375903 -0400 +@@ -550,6 +550,14 @@ class ReferencesTestCase(TestBase): + del c1, c2, C, D + gc.collect() + ++ # In a debug build, this fails with: ++ # AssertionError: Lists differ: [] != ['C went away'] ++ # Second list contains 1 additional elements. ++ # First extra element 0: ++ # C went away ++ # - [] ++ # + ['C went away'] ++ @unittest.skipIf(hasattr(sys, 'getobjects'), 'debug build') + def test_callback_in_cycle_resurrection(self): + import gc + diff --git a/SOURCES/00136-skip-tests-of-seeking-stdin-in-rpmbuild.patch b/SOURCES/00136-skip-tests-of-seeking-stdin-in-rpmbuild.patch new file mode 100644 index 00000000..845fb2a9 --- /dev/null +++ b/SOURCES/00136-skip-tests-of-seeking-stdin-in-rpmbuild.patch @@ -0,0 +1,22 @@ +diff -up Python-2.7.2/Lib/test/test_file2k.py.skip-tests-of-seeking-stdin-in-rpmbuild Python-2.7.2/Lib/test/test_file2k.py +--- Python-2.7.2/Lib/test/test_file2k.py.skip-tests-of-seeking-stdin-in-rpmbuild 2011-09-08 17:23:50.922520729 -0400 ++++ Python-2.7.2/Lib/test/test_file2k.py 2011-09-08 17:24:41.368517277 -0400 +@@ -213,6 +213,7 @@ class OtherFileTests(unittest.TestCase): + else: + f.close() + ++ @unittest._skipInRpmBuild('seems not to raise the exception when run in Koji') + def testStdin(self): + # This causes the interpreter to exit on OSF1 v5.1. + if sys.platform != 'osf1V5': +diff -up Python-2.7.2/Lib/test/test_file.py.skip-tests-of-seeking-stdin-in-rpmbuild Python-2.7.2/Lib/test/test_file.py +--- Python-2.7.2/Lib/test/test_file.py.skip-tests-of-seeking-stdin-in-rpmbuild 2011-09-08 17:20:31.146534389 -0400 ++++ Python-2.7.2/Lib/test/test_file.py 2011-09-08 17:24:45.016517030 -0400 +@@ -154,6 +154,7 @@ class OtherFileTests(unittest.TestCase): + f.close() + self.fail('%r is an invalid file mode' % mode) + ++ @unittest._skipInRpmBuild('seems not to raise the exception when run in Koji') + def testStdin(self): + # This causes the interpreter to exit on OSF1 v5.1. + if sys.platform != 'osf1V5': diff --git a/SOURCES/00137-skip-distutils-tests-that-fail-in-rpmbuild.patch b/SOURCES/00137-skip-distutils-tests-that-fail-in-rpmbuild.patch index 04570930..86537722 100644 --- a/SOURCES/00137-skip-distutils-tests-that-fail-in-rpmbuild.patch +++ b/SOURCES/00137-skip-distutils-tests-that-fail-in-rpmbuild.patch @@ -1,12 +1,12 @@ -diff -up Python-3.2.2/Lib/distutils/tests/test_bdist_rpm.py.skip-distutils-tests-that-fail-in-rpmbuild Python-3.2.2/Lib/distutils/tests/test_bdist_rpm.py ---- Python-3.2.2/Lib/distutils/tests/test_bdist_rpm.py.skip-distutils-tests-that-fail-in-rpmbuild 2011-09-03 12:16:40.000000000 -0400 -+++ Python-3.2.2/Lib/distutils/tests/test_bdist_rpm.py 2011-09-10 05:04:56.328852558 -0400 -@@ -23,6 +23,7 @@ setup(name='foo', version='0.1', py_modu +diff -up Python-2.7.3/Lib/distutils/tests/test_bdist_rpm.py.mark-tests-that-fail-in-rpmbuild Python-2.7.3/Lib/distutils/tests/test_bdist_rpm.py +--- Python-2.7.3/Lib/distutils/tests/test_bdist_rpm.py.mark-tests-that-fail-in-rpmbuild 2012-04-09 19:07:29.000000000 -0400 ++++ Python-2.7.3/Lib/distutils/tests/test_bdist_rpm.py 2012-04-13 00:20:08.223819263 -0400 +@@ -24,6 +24,7 @@ setup(name='foo', version='0.1', py_modu """ +@unittest._skipInRpmBuild("don't try to nest one rpm build inside another rpm build") class BuildRpmTestCase(support.TempdirManager, - support.EnvironGuard, support.LoggingSilencer, -diff -up Python-3.2.2/Lib/distutils/tests/test_build_ext.py.skip-distutils-tests-that-fail-in-rpmbuild Python-3.2.2/Lib/distutils/tests/test_build_ext.py + unittest.TestCase): +diff -up Python-2.7.3/Lib/distutils/tests/test_build_ext.py.mark-tests-that-fail-in-rpmbuild Python-2.7.3/Lib/distutils/tests/test_build_ext.py diff --git a/SOURCES/00138-fix-distutils-tests-in-debug-build.patch b/SOURCES/00138-fix-distutils-tests-in-debug-build.patch new file mode 100644 index 00000000..0bfda909 --- /dev/null +++ b/SOURCES/00138-fix-distutils-tests-in-debug-build.patch @@ -0,0 +1,68 @@ +diff -up Python-2.7.2/Lib/distutils/tests/test_build_ext.py.mark-tests-that-fail-in-rpmbuild Python-2.7.2/Lib/distutils/tests/test_build_ext.py +--- Python-2.7.2/Lib/distutils/tests/test_build_ext.py.mark-tests-that-fail-in-rpmbuild 2011-09-08 16:07:25.033834312 -0400 ++++ Python-2.7.2/Lib/distutils/tests/test_build_ext.py 2011-09-08 17:43:15.656441082 -0400 +@@ -330,6 +332,7 @@ class BuildExtTestCase(support.TempdirMa + self.assertEqual(lastdir, 'bar') + + def test_ext_fullpath(self): ++ debug_ext = sysconfig.get_config_var("DEBUG_EXT") + ext = sysconfig.get_config_vars()['SO'] + dist = Distribution() + cmd = build_ext(dist) +@@ -337,14 +340,14 @@ class BuildExtTestCase(support.TempdirMa + cmd.distribution.package_dir = {'': 'src'} + cmd.distribution.packages = ['lxml', 'lxml.html'] + curdir = os.getcwd() +- wanted = os.path.join(curdir, 'src', 'lxml', 'etree' + ext) ++ wanted = os.path.join(curdir, 'src', 'lxml', 'etree' + debug_ext + ext) + path = cmd.get_ext_fullpath('lxml.etree') + self.assertEqual(wanted, path) + + # building lxml.etree not inplace + cmd.inplace = 0 + cmd.build_lib = os.path.join(curdir, 'tmpdir') +- wanted = os.path.join(curdir, 'tmpdir', 'lxml', 'etree' + ext) ++ wanted = os.path.join(curdir, 'tmpdir', 'lxml', 'etree' + debug_ext + ext) + path = cmd.get_ext_fullpath('lxml.etree') + self.assertEqual(wanted, path) + +@@ -354,13 +357,13 @@ class BuildExtTestCase(support.TempdirMa + cmd.distribution.packages = ['twisted', 'twisted.runner.portmap'] + path = cmd.get_ext_fullpath('twisted.runner.portmap') + wanted = os.path.join(curdir, 'tmpdir', 'twisted', 'runner', +- 'portmap' + ext) ++ 'portmap' + debug_ext + ext) + self.assertEqual(wanted, path) + + # building twisted.runner.portmap inplace + cmd.inplace = 1 + path = cmd.get_ext_fullpath('twisted.runner.portmap') +- wanted = os.path.join(curdir, 'twisted', 'runner', 'portmap' + ext) ++ wanted = os.path.join(curdir, 'twisted', 'runner', 'portmap' + debug_ext + ext) + self.assertEqual(wanted, path) + + def test_build_ext_inplace(self): +@@ -373,8 +376,9 @@ class BuildExtTestCase(support.TempdirMa + cmd.distribution.package_dir = {'': 'src'} + cmd.distribution.packages = ['lxml', 'lxml.html'] + curdir = os.getcwd() ++ debug_ext = sysconfig.get_config_var("DEBUG_EXT") + ext = sysconfig.get_config_var("SO") +- wanted = os.path.join(curdir, 'src', 'lxml', 'etree' + ext) ++ wanted = os.path.join(curdir, 'src', 'lxml', 'etree' + debug_ext + ext) + path = cmd.get_ext_fullpath('lxml.etree') + self.assertEqual(wanted, path) + +@@ -412,10 +416,11 @@ class BuildExtTestCase(support.TempdirMa + dist = Distribution({'name': 'UpdateManager'}) + cmd = build_ext(dist) + cmd.ensure_finalized() ++ debug_ext = sysconfig.get_config_var("DEBUG_EXT") + ext = sysconfig.get_config_var("SO") + ext_name = os.path.join('UpdateManager', 'fdsend') + ext_path = cmd.get_ext_fullpath(ext_name) +- wanted = os.path.join(cmd.build_lib, 'UpdateManager', 'fdsend' + ext) ++ wanted = os.path.join(cmd.build_lib, 'UpdateManager', 'fdsend' + debug_ext + ext) + self.assertEqual(ext_path, wanted) + + def test_build_ext_path_cross_platform(self): diff --git a/SOURCES/00139-skip-test_float-known-failure-on-arm.patch b/SOURCES/00139-skip-test_float-known-failure-on-arm.patch new file mode 100644 index 00000000..9d0bfad7 --- /dev/null +++ b/SOURCES/00139-skip-test_float-known-failure-on-arm.patch @@ -0,0 +1,11 @@ +diff -up Python-2.7.2/Lib/test/test_float.py.skip-test_float-known-failure-on-arm Python-2.7.2/Lib/test/test_float.py +--- Python-2.7.2/Lib/test/test_float.py.skip-test_float-known-failure-on-arm 2011-09-08 19:34:09.000986128 -0400 ++++ Python-2.7.2/Lib/test/test_float.py 2011-09-08 19:34:57.969982779 -0400 +@@ -1072,6 +1072,7 @@ class HexFloatTestCase(unittest.TestCase + self.identical(got, expected) + + ++ @unittest.skip('Known failure on ARM: http://bugs.python.org/issue8265') + def test_from_hex(self): + MIN = self.MIN; + MAX = self.MAX; diff --git a/SOURCES/00140-skip-test_ctypes-known-failure-on-sparc.patch b/SOURCES/00140-skip-test_ctypes-known-failure-on-sparc.patch new file mode 100644 index 00000000..95aa41e5 --- /dev/null +++ b/SOURCES/00140-skip-test_ctypes-known-failure-on-sparc.patch @@ -0,0 +1,11 @@ +diff -up Python-2.7.2/Lib/ctypes/test/test_callbacks.py.skip-test_ctypes-known-failure-on-sparc Python-2.7.2/Lib/ctypes/test/test_callbacks.py +--- Python-2.7.2/Lib/ctypes/test/test_callbacks.py.skip-test_ctypes-known-failure-on-sparc 2011-09-08 19:42:35.541951490 -0400 ++++ Python-2.7.2/Lib/ctypes/test/test_callbacks.py 2011-09-08 19:43:40.676947036 -0400 +@@ -67,6 +67,7 @@ class Callbacks(unittest.TestCase): + self.check_type(c_longlong, 42) + self.check_type(c_longlong, -42) + ++ @unittest.skip('Known failure on Sparc: http://bugs.python.org/issue8314') + def test_ulonglong(self): + # test some 64-bit values, with and without msb set. + self.check_type(c_ulonglong, 10955412242170339782) diff --git a/SOURCES/00141-fix-test_gc_with_COUNT_ALLOCS.patch b/SOURCES/00141-fix-test_gc_with_COUNT_ALLOCS.patch new file mode 100644 index 00000000..d5bf3c93 --- /dev/null +++ b/SOURCES/00141-fix-test_gc_with_COUNT_ALLOCS.patch @@ -0,0 +1,24 @@ +diff -up Python-2.7.2/Lib/test/test_gc.py.fix-test_gc_with_COUNT_ALLOCS Python-2.7.2/Lib/test/test_gc.py +--- Python-2.7.2/Lib/test/test_gc.py.fix-test_gc_with_COUNT_ALLOCS 2011-09-08 19:49:13.045924309 -0400 ++++ Python-2.7.2/Lib/test/test_gc.py 2011-09-08 19:50:07.035920617 -0400 +@@ -102,11 +102,17 @@ class GCTests(unittest.TestCase): + del a + self.assertNotEqual(gc.collect(), 0) + del B, C +- self.assertNotEqual(gc.collect(), 0) ++ if hasattr(sys, 'getcounts'): ++ self.assertEqual(gc.collect(), 0) ++ else: ++ self.assertNotEqual(gc.collect(), 0) + A.a = A() + del A +- self.assertNotEqual(gc.collect(), 0) +- self.assertEqual(gc.collect(), 0) ++ if hasattr(sys, 'getcounts'): ++ self.assertEqual(gc.collect(), 0) ++ else: ++ self.assertNotEqual(gc.collect(), 0) ++ self.assertEqual(gc.collect(), 0) + + def test_method(self): + # Tricky: self.__init__ is a bound method, it references the instance. diff --git a/SOURCES/00142-skip-failing-pty-tests-in-rpmbuild.patch b/SOURCES/00142-skip-failing-pty-tests-in-rpmbuild.patch new file mode 100644 index 00000000..414ffcd9 --- /dev/null +++ b/SOURCES/00142-skip-failing-pty-tests-in-rpmbuild.patch @@ -0,0 +1,22 @@ +diff -up Python-2.7.2/Lib/test/test_openpty.py.skip-failing-pty-tests-in-rpmbuild Python-2.7.2/Lib/test/test_openpty.py +--- Python-2.7.2/Lib/test/test_openpty.py.skip-failing-pty-tests-in-rpmbuild 2011-09-09 05:09:28.698920379 -0400 ++++ Python-2.7.2/Lib/test/test_openpty.py 2011-09-09 05:10:54.805914490 -0400 +@@ -8,6 +8,7 @@ if not hasattr(os, "openpty"): + + + class OpenptyTest(unittest.TestCase): ++ @unittest._skipInRpmBuild('sometimes fails in Koji, possibly due to a mock issue (rhbz#714627)') + def test(self): + master, slave = os.openpty() + if not os.isatty(slave): +diff -up Python-2.7.2/Lib/test/test_pty.py.skip-failing-pty-tests-in-rpmbuild Python-2.7.2/Lib/test/test_pty.py +--- Python-2.7.2/Lib/test/test_pty.py.skip-failing-pty-tests-in-rpmbuild 2011-09-09 05:09:36.781919825 -0400 ++++ Python-2.7.2/Lib/test/test_pty.py 2011-09-09 05:11:14.741913127 -0400 +@@ -109,6 +109,7 @@ class PtyTest(unittest.TestCase): + os.close(master_fd) + + ++ @unittest._skipInRpmBuild('sometimes fails in Koji, possibly due to a mock issue (rhbz#714627)') + def test_fork(self): + debug("calling pty.fork()") + pid, master_fd = pty.fork() diff --git a/SOURCES/00143-tsc-on-ppc.patch b/SOURCES/00143-tsc-on-ppc.patch new file mode 100644 index 00000000..447c6e3a --- /dev/null +++ b/SOURCES/00143-tsc-on-ppc.patch @@ -0,0 +1,58 @@ +diff -up Python-2.7.2/Python/ceval.c.tsc-on-ppc Python-2.7.2/Python/ceval.c +--- Python-2.7.2/Python/ceval.c.tsc-on-ppc 2011-08-23 14:59:48.051300849 -0400 ++++ Python-2.7.2/Python/ceval.c 2011-08-23 15:33:25.412162902 -0400 +@@ -37,24 +37,42 @@ typedef unsigned long long uint64; + */ + #if defined(__ppc__) || defined (__powerpc__) + +-#define READ_TIMESTAMP(var) ppc_getcounter(&var) ++#if defined( __powerpc64__) || defined(__LP64__) ++/* 64-bit PowerPC */ ++#define READ_TIMESTAMP(var) ppc64_getcounter(&var) ++static void ++ppc64_getcounter(uint64 *v) ++{ ++ /* On 64-bit PowerPC we can read the 64-bit timebase directly into a ++ 64-bit register */ ++ uint64 timebase; ++#ifdef _ARCH_PWR4 ++ asm volatile ("mfspr %0,268" : "=r" (timebase)); ++#else ++ asm volatile ("mftb %0" : "=r" (timebase)); ++#endif ++ *v = timebase; ++} ++ ++#else ++/* 32-bit PowerPC */ ++#define READ_TIMESTAMP(var) ppc32_getcounter(&var) + + static void +-ppc_getcounter(uint64 *v) ++ppc32_getcounter(uint64 *v) + { +- register unsigned long tbu, tb, tbu2; ++ union { long long ll; long ii[2]; } u; ++ long tmp; + + loop: +- asm volatile ("mftbu %0" : "=r" (tbu) ); +- asm volatile ("mftb %0" : "=r" (tb) ); +- asm volatile ("mftbu %0" : "=r" (tbu2)); +- if (__builtin_expect(tbu != tbu2, 0)) goto loop; +- +- /* The slightly peculiar way of writing the next lines is +- compiled better by GCC than any other way I tried. */ +- ((long*)(v))[0] = tbu; +- ((long*)(v))[1] = tb; ++ asm volatile ("mftbu %0" : "=r" (u.ii[0]) ); ++ asm volatile ("mftb %0" : "=r" (u.ii[1]) ); ++ asm volatile ("mftbu %0" : "=r" (tmp)); ++ if (__builtin_expect(u.ii[0] != tmp, 0)) goto loop; ++ ++ *v = u.ll; + } ++#endif /* powerpc 32/64 bit */ + + #elif defined(__i386__) + diff --git a/SOURCES/00144-no-gdbm.patch b/SOURCES/00144-no-gdbm.patch new file mode 100644 index 00000000..0378d447 --- /dev/null +++ b/SOURCES/00144-no-gdbm.patch @@ -0,0 +1,12 @@ +diff -up Python-2.7.2/Modules/Setup.dist.no-gdbm Python-2.7.2/Modules/Setup.dist +--- Python-2.7.2/Modules/Setup.dist.no-gdbm 2011-09-13 14:25:43.496095926 -0400 ++++ Python-2.7.2/Modules/Setup.dist 2011-09-13 14:25:46.491095724 -0400 +@@ -396,7 +396,7 @@ dl dlmodule.c + # + # First, look at Setup.config; configure may have set this for you. + +-gdbm gdbmmodule.c -lgdbm ++# gdbm gdbmmodule.c -lgdbm + + + # Sleepycat Berkeley DB interface. diff --git a/SOURCES/00146-hashlib-fips.patch b/SOURCES/00146-hashlib-fips.patch index e0cdce0a..c67eb3b5 100644 --- a/SOURCES/00146-hashlib-fips.patch +++ b/SOURCES/00146-hashlib-fips.patch @@ -1,48 +1,82 @@ -diff --git a/Lib/hashlib.py b/Lib/hashlib.py -index 316cece..b7ad879 100644 ---- a/Lib/hashlib.py -+++ b/Lib/hashlib.py -@@ -23,6 +23,16 @@ the zlib module. +diff -up Python-2.7.2/Lib/hashlib.py.hashlib-fips Python-2.7.2/Lib/hashlib.py +--- Python-2.7.2/Lib/hashlib.py.hashlib-fips 2011-06-11 11:46:24.000000000 -0400 ++++ Python-2.7.2/Lib/hashlib.py 2011-09-14 00:21:26.194252001 -0400 +@@ -6,9 +6,12 @@ + + __doc__ = """hashlib module - A common interface to many hash functions. + +-new(name, string='') - returns a new hash object implementing the +- given hash function; initializing the hash +- using the given string data. ++new(name, string='', usedforsecurity=True) ++ - returns a new hash object implementing the given hash function; ++ initializing the hash using the given string data. ++ ++ "usedforsecurity" is a non-standard extension for better supporting ++ FIPS-compliant environments (see below) + + Named constructor functions are also available, these are much faster + than using new(): +@@ -24,6 +27,20 @@ the zlib module. Choose your hash function wisely. Some have known collision weaknesses. sha384 and sha512 will be slow on 32 bit platforms. -+If the underlying implementation supports "FIPS mode", and this is enabled, it -+may restrict the available hashes to only those that are compliant with FIPS -+regulations. For example, it may deny the use of MD5, on the grounds that this -+is not secure for uses such as authentication, system integrity checking, or -+digital signatures. If you need to use such a hash for non-security purposes -+(such as indexing into a data structure for speed), you can override the keyword -+argument "usedforsecurity" from True to False to signify that your code is not -+relying on the hash for security purposes, and this will allow the hash to be -+usable even in FIPS mode. ++Our implementation of hashlib uses OpenSSL. ++ ++OpenSSL has a "FIPS mode", which, if enabled, may restrict the available hashes ++to only those that are compliant with FIPS regulations. For example, it may ++deny the use of MD5, on the grounds that this is not secure for uses such as ++authentication, system integrity checking, or digital signatures. ++ ++If you need to use such a hash for non-security purposes (such as indexing into ++a data structure for speed), you can override the keyword argument ++"usedforsecurity" from True to False to signify that your code is not relying ++on the hash for security purposes, and this will allow the hash to be usable ++even in FIPS mode. This is not a standard feature of Python 2.7's hashlib, and ++is included here to better support FIPS mode. + Hash objects have these methods: - - update(arg): Update the hash object with the bytes in arg. Repeated calls + - update(arg): Update the hash object with the string arg. Repeated calls are equivalent to a single call with the concatenation of all -@@ -62,6 +72,18 @@ algorithms_available = set(__always_supported) - __all__ = __always_supported + ('new', 'algorithms_guaranteed', - 'algorithms_available', 'pbkdf2_hmac') - -+import functools -+def __ignore_usedforsecurity(func): -+ """Used for sha3_* functions. Until OpenSSL implements them, we want -+ to use them from Python _sha3 module, but we want them to accept -+ usedforsecurity argument too.""" -+ # TODO: remove this function when OpenSSL implements sha3 -+ @functools.wraps(func) -+ def inner(*args, **kwargs): -+ if 'usedforsecurity' in kwargs: -+ kwargs.pop('usedforsecurity') -+ return func(*args, **kwargs) -+ return inner - - __builtin_constructor_cache = {} - -@@ -100,31 +122,39 @@ def __get_openssl_constructor(name): +@@ -63,74 +80,39 @@ algorithms = __always_supported + __all__ = __always_supported + ('new', 'algorithms') + + +-def __get_builtin_constructor(name): +- try: +- if name in ('SHA1', 'sha1'): +- import _sha +- return _sha.new +- elif name in ('MD5', 'md5'): +- import _md5 +- return _md5.new +- elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'): +- import _sha256 +- bs = name[3:] +- if bs == '256': +- return _sha256.sha256 +- elif bs == '224': +- return _sha256.sha224 +- elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'): +- import _sha512 +- bs = name[3:] +- if bs == '512': +- return _sha512.sha512 +- elif bs == '384': +- return _sha512.sha384 +- except ImportError: +- pass # no extension module, this hash is unsupported. +- +- raise ValueError('unsupported hash type ' + name) +- +- + def __get_openssl_constructor(name): + try: f = getattr(_hashlib, 'openssl_' + name) # Allow the C module to raise ValueError. The function will be # defined but the hash not actually available thanks to OpenSSL. - f() ++ # + # We pass "usedforsecurity=False" to disable FIPS-based restrictions: + # at this stage we're merely seeing if the function is callable, + # rather than using it for actual work. @@ -50,76 +84,61 @@ index 316cece..b7ad879 100644 # Use the C function directly (very fast) return f except (AttributeError, ValueError): -+ # TODO: We want to just raise here when OpenSSL implements sha3 -+ # because we want to make sure that Fedora uses everything from OpenSSL - return __get_builtin_constructor(name) - - --def __py_new(name, data=b''): -- """new(name, data=b'') - Return a new hashing object using the named algorithm; -- optionally initialized with data (which must be bytes). -+def __py_new(name, data=b'', usedforsecurity=True): -+ """new(name, data=b'', usedforsecurity=True) - Return a new hashing object using -+ the named algorithm; optionally initialized with data (which must be bytes). -+ The 'usedforsecurity' keyword argument does nothing, and is for compatibilty -+ with the OpenSSL implementation - """ - return __get_builtin_constructor(name)(data) - +- return __get_builtin_constructor(name) ++ raise --def __hash_new(name, data=b''): -- """new(name, data=b'') - Return a new hashing object using the named algorithm; -- optionally initialized with data (which must be bytes). -+def __hash_new(name, data=b'', usedforsecurity=True): -+ """new(name, data=b'', usedforsecurity=True) - Return a new hashing object using -+ the named algorithm; optionally initialized with data (which must be bytes). -+ +- +-def __py_new(name, string=''): +- """new(name, string='') - Return a new hashing object using the named algorithm; +- optionally initialized with a string. +- """ +- return __get_builtin_constructor(name)(string) +- +- +-def __hash_new(name, string=''): ++def __hash_new(name, string='', usedforsecurity=True): + """new(name, string='') - Return a new hashing object using the named algorithm; + optionally initialized with a string. + Override 'usedforsecurity' to False when using for non-security purposes in + a FIPS environment """ try: -- return _hashlib.new(name, data) -+ return _hashlib.new(name, data, usedforsecurity) +- return _hashlib.new(name, string) ++ return _hashlib.new(name, string, usedforsecurity) except ValueError: - # If the _hashlib module (OpenSSL) doesn't support the named - # hash, try using our builtin implementations. - # This allows for SHA224/256 and SHA384/512 support even though - # the OpenSSL library prior to 0.9.8 doesn't provide them. -+ # TODO: We want to just raise here when OpenSSL implements sha3 -+ # because we want to make sure that Fedora uses everything from OpenSSL - return __get_builtin_constructor(name)(data) - - -@@ -207,7 +237,10 @@ for __func_name in __always_supported: +- return __get_builtin_constructor(name)(string) +- ++ raise + + try: + import _hashlib + new = __hash_new + __get_hash = __get_openssl_constructor + except ImportError: +- new = __py_new +- __get_hash = __get_builtin_constructor ++ # We don't build the legacy modules ++ raise + + for __func_name in __always_supported: # try them all, some may not work due to the OpenSSL - # version not supporting that algorithm. - try: -- globals()[__func_name] = __get_hash(__func_name) -+ func = __get_hash(__func_name) -+ if 'sha3_' in __func_name: -+ func = __ignore_usedforsecurity(func) -+ globals()[__func_name] = func - except ValueError: - import logging - logging.exception('code for hash %s was not found.', __func_name) -@@ -215,3 +248,4 @@ for __func_name in __always_supported: +@@ -143,4 +125,4 @@ for __func_name in __always_supported: + # Cleanup locals() del __always_supported, __func_name, __get_hash - del __py_new, __hash_new, __get_openssl_constructor -+del __ignore_usedforsecurity -\ No newline at end of file -diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py -index c9b113e..60e2392 100644 ---- a/Lib/test/test_hashlib.py -+++ b/Lib/test/test_hashlib.py -@@ -24,7 +24,22 @@ from test.support import _4G, bigmemtest, import_fresh_module - COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount') - - c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) --py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) -+# skipped on Fedora, since we always use OpenSSL implementation -+# py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) -+ +-del __py_new, __hash_new, __get_openssl_constructor ++del __hash_new, __get_openssl_constructor +diff -up Python-2.7.2/Lib/test/test_hashlib.py.hashlib-fips Python-2.7.2/Lib/test/test_hashlib.py +--- Python-2.7.2/Lib/test/test_hashlib.py.hashlib-fips 2011-06-11 11:46:25.000000000 -0400 ++++ Python-2.7.2/Lib/test/test_hashlib.py 2011-09-14 01:08:55.525254195 -0400 +@@ -32,6 +32,19 @@ def hexstr(s): + r = r + h[(i >> 4) & 0xF] + h[i & 0xF] + return r + +def openssl_enforces_fips(): + # Use the "openssl" command (if present) to try to determine if the local + # OpenSSL is configured to enforce FIPS @@ -134,80 +153,122 @@ index c9b113e..60e2392 100644 + return b'unknown cipher' in stderr +OPENSSL_ENFORCES_FIPS = openssl_enforces_fips() - def hexstr(s): - assert isinstance(s, bytes), repr(s) -@@ -34,6 +49,16 @@ def hexstr(s): - r += h[(i >> 4) & 0xF] + h[i & 0xF] - return r - -+# hashlib and _hashlib-based functions support a "usedforsecurity" keyword -+# argument, and FIPS mode requires that it be used overridden with a False -+# value for these selftests to work. Other cryptographic code within Python -+# doesn't support this keyword. -+# Modify a function to one in which "usedforsecurity=False" is added to the -+# keyword arguments: -+def suppress_fips(f): -+ def g(*args, **kwargs): -+ return f(*args, usedforsecurity=False, **kwargs) -+ return g - class HashLibTestCase(unittest.TestCase): supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1', -@@ -63,11 +88,11 @@ class HashLibTestCase(unittest.TestCase): - # For each algorithm, test the direct constructor and the use +@@ -61,10 +74,10 @@ class HashLibTestCase(unittest.TestCase) # of hashlib.new given the algorithm name. for algorithm, constructors in self.constructors_to_test.items(): -- constructors.add(getattr(hashlib, algorithm)) -+ constructors.add(suppress_fips(getattr(hashlib, algorithm))) - def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm): + constructors.add(getattr(hashlib, algorithm)) +- def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm): ++ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, usedforsecurity=True): if data is None: - return hashlib.new(_alg) - return hashlib.new(_alg, data) -+ return suppress_fips(hashlib.new)(_alg) -+ return suppress_fips(hashlib.new)(_alg, data) ++ return hashlib.new(_alg, usedforsecurity=usedforsecurity) ++ return hashlib.new(_alg, data, usedforsecurity=usedforsecurity) constructors.add(_test_algorithm_via_hashlib_new) _hashlib = self._conditional_import_module('_hashlib') -@@ -79,27 +104,12 @@ class HashLibTestCase(unittest.TestCase): - for algorithm, constructors in self.constructors_to_test.items(): - constructor = getattr(_hashlib, 'openssl_'+algorithm, None) +@@ -78,28 +91,13 @@ class HashLibTestCase(unittest.TestCase) if constructor: -- constructors.add(constructor) -+ constructors.add(suppress_fips(constructor)) - - def add_builtin_constructor(name): - constructor = getattr(hashlib, "__get_builtin_constructor")(name) - self.constructors_to_test[name].add(constructor) + constructors.add(constructor) - _md5 = self._conditional_import_module('_md5') - if _md5: -- add_builtin_constructor('md5') -- _sha1 = self._conditional_import_module('_sha1') -- if _sha1: -- add_builtin_constructor('sha1') +- self.constructors_to_test['md5'].add(_md5.new) +- _sha = self._conditional_import_module('_sha') +- if _sha: +- self.constructors_to_test['sha1'].add(_sha.new) - _sha256 = self._conditional_import_module('_sha256') - if _sha256: -- add_builtin_constructor('sha224') -- add_builtin_constructor('sha256') +- self.constructors_to_test['sha224'].add(_sha256.sha224) +- self.constructors_to_test['sha256'].add(_sha256.sha256) - _sha512 = self._conditional_import_module('_sha512') - if _sha512: -- add_builtin_constructor('sha384') -- add_builtin_constructor('sha512') +- self.constructors_to_test['sha384'].add(_sha512.sha384) +- self.constructors_to_test['sha512'].add(_sha512.sha512) - super(HashLibTestCase, self).__init__(*args, **kwargs) - @property -@@ -148,9 +158,6 @@ class HashLibTestCase(unittest.TestCase): - else: - del sys.modules['_md5'] - self.assertRaises(TypeError, get_builtin_constructor, 3) -- constructor = get_builtin_constructor('md5') -- self.assertIs(constructor, _md5.md5) -- self.assertEqual(sorted(builtin_constructor_cache), ['MD5', 'md5']) - + def test_hash_array(self): + a = array.array("b", range(10)) + constructors = self.constructors_to_test.itervalues() + for cons in itertools.chain.from_iterable(constructors): +- c = cons(a) ++ c = cons(a, usedforsecurity=False) + c.hexdigest() + + def test_algorithms_attribute(self): +@@ -115,28 +113,9 @@ class HashLibTestCase(unittest.TestCase) + self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') + self.assertRaises(TypeError, hashlib.new, 1) + +- def test_get_builtin_constructor(self): +- get_builtin_constructor = hashlib.__dict__[ +- '__get_builtin_constructor'] +- self.assertRaises(ValueError, get_builtin_constructor, 'test') +- try: +- import _md5 +- except ImportError: +- pass +- # This forces an ImportError for "import _md5" statements +- sys.modules['_md5'] = None +- try: +- self.assertRaises(ValueError, get_builtin_constructor, 'md5') +- finally: +- if '_md5' in locals(): +- sys.modules['_md5'] = _md5 +- else: +- del sys.modules['_md5'] +- self.assertRaises(TypeError, get_builtin_constructor, 3) +- def test_hexdigest(self): - for cons in self.hash_constructors: -@@ -433,6 +440,64 @@ class HashLibTestCase(unittest.TestCase): + for name in self.supported_hash_names: +- h = hashlib.new(name) ++ h = hashlib.new(name, usedforsecurity=False) + self.assertTrue(hexstr(h.digest()) == h.hexdigest()) + + def test_large_update(self): +@@ -145,16 +125,16 @@ class HashLibTestCase(unittest.TestCase) + abcs = aas + bees + cees + + for name in self.supported_hash_names: +- m1 = hashlib.new(name) ++ m1 = hashlib.new(name, usedforsecurity=False) + m1.update(aas) + m1.update(bees) + m1.update(cees) + +- m2 = hashlib.new(name) ++ m2 = hashlib.new(name, usedforsecurity=False) + m2.update(abcs) + self.assertEqual(m1.digest(), m2.digest(), name+' update problem.') + +- m3 = hashlib.new(name, abcs) ++ m3 = hashlib.new(name, abcs, usedforsecurity=False) + self.assertEqual(m1.digest(), m3.digest(), name+' new problem.') + + def check(self, name, data, digest): +@@ -162,7 +142,7 @@ class HashLibTestCase(unittest.TestCase) + # 2 is for hashlib.name(...) and hashlib.new(name, ...) + self.assertGreaterEqual(len(constructors), 2) + for hash_object_constructor in constructors: +- computed = hash_object_constructor(data).hexdigest() ++ computed = hash_object_constructor(data, usedforsecurity=False).hexdigest() + self.assertEqual( + computed, digest, + "Hash algorithm %s constructed using %s returned hexdigest" +@@ -172,7 +152,8 @@ class HashLibTestCase(unittest.TestCase) + + def check_unicode(self, algorithm_name): + # Unicode objects are not allowed as input. +- expected = hashlib.new(algorithm_name, str(u'spam')).hexdigest() ++ expected = hashlib.new(algorithm_name, str(u'spam'), ++ usedforsecurity=False).hexdigest() + self.check(algorithm_name, u'spam', expected) + + def test_unicode(self): +@@ -354,6 +335,70 @@ class HashLibTestCase(unittest.TestCase) self.assertEqual(expected_hash, hasher.hexdigest()) @@ -227,74 +288,70 @@ index c9b113e..60e2392 100644 + m = hashlib.new('md5', b'abc\n', usedforsecurity=False) + self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1") + ++ def assertRaisesUnknownCipher(self, callable_obj=None, *args, **kwargs): ++ try: ++ callable_obj(*args, **kwargs) ++ except ValueError, e: ++ if not e.args[0].endswith('unknown cipher'): ++ self.fail('Incorrect exception raised') ++ else: ++ self.fail('Exception was not raised') ++ + @unittest.skipUnless(OPENSSL_ENFORCES_FIPS, + 'FIPS enforcement required for this test.') + def test_hashlib_fips_mode(self): + # Ensure that we raise a ValueError on vanilla attempts to use MD5 + # in hashlib in a FIPS-enforced setting: -+ with self.assertRaisesRegexp(ValueError, '.*unknown cipher'): -+ m = hashlib.md5() -+ -+ if not self._conditional_import_module('_md5'): -+ with self.assertRaisesRegexp(ValueError, '.*unknown cipher'): -+ m = hashlib.new('md5') ++ self.assertRaisesUnknownCipher(hashlib.md5) ++ self.assertRaisesUnknownCipher(hashlib.new, 'md5') + + @unittest.skipUnless(OPENSSL_ENFORCES_FIPS, + 'FIPS enforcement required for this test.') + def test_hashopenssl_fips_mode(self): + # Verify the _hashlib module's handling of md5: -+ _hashlib = self._conditional_import_module('_hashlib') -+ if _hashlib: -+ assert hasattr(_hashlib, 'openssl_md5') ++ import _hashlib ++ ++ assert hasattr(_hashlib, 'openssl_md5') + -+ # Ensure that _hashlib raises a ValueError on vanilla attempts to -+ # use MD5 in a FIPS-enforced setting: -+ with self.assertRaisesRegexp(ValueError, '.*unknown cipher'): -+ m = _hashlib.openssl_md5() -+ with self.assertRaisesRegexp(ValueError, '.*unknown cipher'): -+ m = _hashlib.new('md5') ++ # Ensure that _hashlib raises a ValueError on vanilla attempts to ++ # use MD5 in a FIPS-enforced setting: ++ self.assertRaisesUnknownCipher(_hashlib.openssl_md5) ++ self.assertRaisesUnknownCipher(_hashlib.new, 'md5') + -+ # Ensure that in such a setting we can whitelist a callsite with -+ # usedforsecurity=False and have it succeed: -+ m = _hashlib.openssl_md5(usedforsecurity=False) -+ m.update(b'abc\n') -+ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1") ++ # Ensure that in such a setting we can whitelist a callsite with ++ # usedforsecurity=False and have it succeed: ++ m = _hashlib.openssl_md5(usedforsecurity=False) ++ m.update('abc\n') ++ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1") + -+ m = _hashlib.new('md5', usedforsecurity=False) -+ m.update(b'abc\n') -+ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1") -+ -+ m = _hashlib.openssl_md5(b'abc\n', usedforsecurity=False) -+ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1") ++ m = _hashlib.new('md5', usedforsecurity=False) ++ m.update('abc\n') ++ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1") + -+ m = _hashlib.new('md5', b'abc\n', usedforsecurity=False) -+ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1") - - class KDFTests(unittest.TestCase): - -@@ -516,7 +581,7 @@ class KDFTests(unittest.TestCase): - out = pbkdf2(hash_name='sha1', password=b'password', salt=b'salt', - iterations=1, dklen=None) - self.assertEqual(out, self.pbkdf2_results['sha1'][0][0]) -- -+ @unittest.skip('skipped on Fedora, as we always use OpenSSL pbkdf2_hmac') - def test_pbkdf2_hmac_py(self): - self._test_pbkdf2_hmac(py_hashlib.pbkdf2_hmac) - -diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c -index 44765ac..b8cf490 100644 ---- a/Modules/_hashopenssl.c -+++ b/Modules/_hashopenssl.c -@@ -20,6 +20,8 @@ ++ m = _hashlib.openssl_md5('abc\n', usedforsecurity=False) ++ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1") ++ ++ m = _hashlib.new('md5', 'abc\n', usedforsecurity=False) ++ self.assertEquals(m.hexdigest(), "0bee89b07a248e27c83fc3d5951213c1") ++ ++ ++ + def test_main(): + test_support.run_unittest(HashLibTestCase) +diff -up Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips Python-2.7.2/Modules/_hashopenssl.c +--- Python-2.7.2/Modules/_hashopenssl.c.hashlib-fips 2011-06-11 11:46:26.000000000 -0400 ++++ Python-2.7.2/Modules/_hashopenssl.c 2011-09-14 00:21:26.199252001 -0400 +@@ -36,6 +36,8 @@ + #endif /* EVP is the preferred interface to hashing in OpenSSL */ +#include +#include #include - #include - /* We use the object interface to discover what hashes OpenSSL supports. */ -@@ -45,11 +47,19 @@ typedef struct { + + #define MUNCH_SIZE INT_MAX +@@ -65,11 +67,19 @@ typedef struct { static PyTypeObject EVPtype; @@ -318,7 +375,7 @@ index 44765ac..b8cf490 100644 DEFINE_CONSTS_FOR_NEW(md5) DEFINE_CONSTS_FOR_NEW(sha1) -@@ -92,6 +102,48 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len) +@@ -115,6 +125,48 @@ EVP_hash(EVPobject *self, const void *vp } } @@ -349,7 +406,7 @@ index 44765ac..b8cf490 100644 + errstr = ERR_error_string(ERR_peek_last_error(), NULL); + ERR_clear_error(); + -+ return PyUnicode_FromString(errstr); /* Can be NULL */ ++ return PyString_FromString(errstr); /* Can be NULL */ +} + +static void @@ -367,28 +424,27 @@ index 44765ac..b8cf490 100644 /* Internal methods for a hash object */ static void -@@ -259,15 +311,16 @@ EVP_repr(EVPobject *self) +@@ -313,14 +365,15 @@ EVP_repr(PyObject *self) static int EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"name", "string", NULL}; + static char *kwlist[] = {"name", "string", "usedforsecurity", NULL}; PyObject *name_obj = NULL; - PyObject *data_obj = NULL; + int usedforsecurity = 1; - Py_buffer view; + Py_buffer view = { 0 }; char *nameStr; const EVP_MD *digest; -- if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:HASH", kwlist, -- &name_obj, &data_obj)) { -+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Oi:HASH", kwlist, -+ &name_obj, &data_obj, &usedforsecurity)) { +- if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s*:HASH", kwlist, +- &name_obj, &view)) { ++ if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s*i:HASH", kwlist, ++ &name_obj, &view, &usedforsecurity)) { return -1; } -@@ -288,7 +341,12 @@ EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds) - PyBuffer_Release(&view); +@@ -336,7 +389,12 @@ EVP_tp_init(EVPobject *self, PyObject *a + PyBuffer_Release(&view); return -1; } - EVP_DigestInit(&self->ctx, digest); @@ -401,7 +457,7 @@ index 44765ac..b8cf490 100644 self->name = name_obj; Py_INCREF(self->name); -@@ -372,7 +430,8 @@ static PyTypeObject EVPtype = { +@@ -420,7 +478,8 @@ static PyTypeObject EVPtype = { static PyObject * EVPnew(PyObject *name_obj, const EVP_MD *digest, const EVP_MD_CTX *initial_ctx, @@ -411,7 +467,7 @@ index 44765ac..b8cf490 100644 { EVPobject *self; -@@ -387,7 +446,12 @@ EVPnew(PyObject *name_obj, +@@ -435,7 +494,12 @@ EVPnew(PyObject *name_obj, if (initial_ctx) { EVP_MD_CTX_copy(&self->ctx, initial_ctx); } else { @@ -425,7 +481,7 @@ index 44765ac..b8cf490 100644 } if (cp && len) { -@@ -411,21 +475,29 @@ PyDoc_STRVAR(EVP_new__doc__, +@@ -459,20 +523,28 @@ PyDoc_STRVAR(EVP_new__doc__, An optional string argument may be provided and will be\n\ automatically hashed.\n\ \n\ @@ -445,32 +501,29 @@ index 44765ac..b8cf490 100644 - static char *kwlist[] = {"name", "string", NULL}; + static char *kwlist[] = {"name", "string", "usedforsecurity", NULL}; PyObject *name_obj = NULL; - PyObject *data_obj = NULL; -+ int usedforsecurity = 1; Py_buffer view = { 0 }; PyObject *ret_obj; char *name; const EVP_MD *digest; ++ int usedforsecurity = 1; -- if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|O:new", kwlist, -- &name_obj, &data_obj)) { -+ if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|Oi:new", kwlist, -+ &name_obj, &data_obj, &usedforsecurity)) { +- if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|s*:new", kwlist, +- &name_obj, &view)) { ++ if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|s*i:new", kwlist, ++ &name_obj, &view, &usedforsecurity)) { return NULL; } -@@ -439,7 +511,8 @@ EVP_new(PyObject *self, PyObject *args, PyObject *kwdict) - +@@ -484,58 +556,118 @@ EVP_new(PyObject *self, PyObject *args, digest = EVP_get_digestbyname(name); -- ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, view.len); -+ ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, view.len, -+ usedforsecurity); - - if (data_obj) - PyBuffer_Release(&view); -@@ -722,57 +795,114 @@ generate_hash_name_list(void) + ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf, +- view.len); ++ view.len, usedforsecurity); + PyBuffer_Release(&view); + return ret_obj; + } /* - * This macro generates constructor function definitions for specific @@ -486,32 +539,25 @@ index 44765ac..b8cf490 100644 #define GEN_CONSTRUCTOR(NAME) \ static PyObject * \ - EVP_new_ ## NAME (PyObject *self, PyObject *args) \ -+ EVP_new_ ## NAME (PyObject *self, PyObject *args, PyObject *kwdict) \ ++ EVP_new_ ## NAME (PyObject *self, PyObject *args, PyObject *kwdict) \ { \ -- PyObject *data_obj = NULL; \ - Py_buffer view = { 0 }; \ - PyObject *ret_obj; \ - \ -- if (!PyArg_ParseTuple(args, "|O:" #NAME , &data_obj)) { \ +- if (!PyArg_ParseTuple(args, "|s*:" #NAME , &view)) { \ - return NULL; \ - } \ - \ -- if (data_obj) \ -- GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); \ -- \ - ret_obj = EVPnew( \ - CONST_ ## NAME ## _name_obj, \ - NULL, \ - CONST_new_ ## NAME ## _ctx_p, \ -- (unsigned char*)view.buf, \ -- view.len); \ -- \ -- if (data_obj) \ -- PyBuffer_Release(&view); \ +- (unsigned char*)view.buf, view.len); \ +- PyBuffer_Release(&view); \ - return ret_obj; \ -+ return implement_specific_EVP_new(self, args, kwdict, \ -+ "|Oi:" #NAME, \ -+ &cached_info_ ## NAME ); \ ++ return implement_specific_EVP_new(self, args, kwdict, \ ++ "|s*i:" #NAME, \ ++ &cached_info_ ## NAME ); \ } +static PyObject * @@ -520,7 +566,6 @@ index 44765ac..b8cf490 100644 + EVPCachedInfo *cached_info) +{ + static char *kwlist[] = {"string", "usedforsecurity", NULL}; -+ PyObject *data_obj = NULL; + Py_buffer view = { 0 }; + int usedforsecurity = 1; + int idx; @@ -529,13 +574,10 @@ index 44765ac..b8cf490 100644 + assert(cached_info); + + if (!PyArg_ParseTupleAndKeywords(args, kwdict, format, kwlist, -+ &data_obj, &usedforsecurity)) { ++ &view, &usedforsecurity)) { + return NULL; + } + -+ if (data_obj) -+ GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); -+ + idx = usedforsecurity ? 1 : 0; + + /* @@ -558,10 +600,9 @@ index 44765ac..b8cf490 100644 + } else { + PyErr_SetString(PyExc_ValueError, "Error initializing hash"); + } -+ } -+ -+ if (data_obj) -+ PyBuffer_Release(&view); ++ } ++ ++ PyBuffer_Release(&view); + + return ret_obj; +} @@ -570,7 +611,7 @@ index 44765ac..b8cf490 100644 #define CONSTRUCTOR_METH_DEF(NAME) \ - {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, METH_VARARGS, \ + {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, \ -+ METH_VARARGS|METH_KEYWORDS, \ ++ METH_VARARGS |METH_KEYWORDS, \ PyDoc_STR("Returns a " #NAME \ " hash object; optionally initialized with a string") \ } @@ -579,7 +620,7 @@ index 44765ac..b8cf490 100644 - constructor constants if they haven't been initialized already. */ -#define INIT_CONSTRUCTOR_CONSTANTS(NAME) do { \ - if (CONST_ ## NAME ## _name_obj == NULL) { \ -- CONST_ ## NAME ## _name_obj = PyUnicode_FromString(#NAME); \ +- CONST_ ## NAME ## _name_obj = PyString_FromString(#NAME); \ - if (EVP_get_digestbyname(#NAME)) { \ - CONST_new_ ## NAME ## _ctx_p = &CONST_new_ ## NAME ## _ctx; \ - EVP_DigestInit(CONST_new_ ## NAME ## _ctx_p, EVP_get_digestbyname(#NAME)); \ @@ -591,17 +632,18 @@ index 44765ac..b8cf490 100644 + Try to initialize a context for each hash twice, once with + EVP_MD_CTX_FLAG_NON_FIPS_ALLOW and once without. + -+ Any that have errors during initialization will end up with a NULL ctx_ptrs ++ Any that have errors during initialization will end up wit a NULL ctx_ptrs + entry, and err_msgs will be set (unless we're very low on memory) +*/ +#define INIT_CONSTRUCTOR_CONSTANTS(NAME) do { \ + init_constructor_constant(&cached_info_ ## NAME, #NAME); \ } while (0); + +static void +init_constructor_constant(EVPCachedInfo *cached_info, const char *name) +{ + assert(cached_info); -+ cached_info->name_obj = PyUnicode_FromString(name); ++ cached_info->name_obj = PyString_FromString(name); + if (EVP_get_digestbyname(name)) { + int i; + for (i=0; i<2; i++) { @@ -612,29 +654,76 @@ index 44765ac..b8cf490 100644 + cached_info->ctx_ptrs[i] = &cached_info->ctxs[i]; + } else { + /* Failure: */ -+ cached_info->ctx_ptrs[i] = NULL; -+ cached_info->error_msgs[i] = error_msg_for_last_error(); ++ cached_info->ctx_ptrs[i] = NULL; ++ cached_info->error_msgs[i] = error_msg_for_last_error(); + } + } + } +} - ++ GEN_CONSTRUCTOR(md5) GEN_CONSTRUCTOR(sha1) -@@ -819,13 +949,10 @@ PyInit__hashlib(void) + #ifdef _OPENSSL_SUPPORTS_SHA2 +@@ -565,13 +700,10 @@ init_hashlib(void) { - PyObject *m, *openssl_md_meth_names; + PyObject *m; -- OpenSSL_add_all_digests(); -- ERR_load_crypto_strings(); + SSL_load_error_strings(); + SSL_library_init(); + OpenSSL_add_all_digests(); - /* TODO build EVP_functions openssl_* entries dynamically based - * on what hashes are supported rather than listing many - * but having some be unsupported. Only init appropriate - * constants. */ -+ OpenSSL_add_all_digests(); - +- Py_TYPE(&EVPtype) = &PyType_Type; if (PyType_Ready(&EVPtype) < 0) + return; +diff -up Python-2.7.2/Modules/Setup.dist.hashlib-fips Python-2.7.2/Modules/Setup.dist +--- Python-2.7.2/Modules/Setup.dist.hashlib-fips 2011-09-14 00:21:26.163252001 -0400 ++++ Python-2.7.2/Modules/Setup.dist 2011-09-14 00:21:26.201252001 -0400 +@@ -248,14 +248,14 @@ imageop imageop.c # Operations on images + # Message-Digest Algorithm, described in RFC 1321. The necessary files + # md5.c and md5.h are included here. + +-_md5 md5module.c md5.c ++#_md5 md5module.c md5.c + + + # The _sha module implements the SHA checksum algorithms. + # (NIST's Secure Hash Algorithms.) +-_sha shamodule.c +-_sha256 sha256module.c +-_sha512 sha512module.c ++#_sha shamodule.c ++#_sha256 sha256module.c ++#_sha512 sha512module.c + + + # SGI IRIX specific modules -- off by default. +diff -up Python-2.7.2/setup.py.hashlib-fips Python-2.7.2/setup.py +--- Python-2.7.2/setup.py.hashlib-fips 2011-09-14 00:21:25.722252001 -0400 ++++ Python-2.7.2/setup.py 2011-09-14 00:21:26.203252001 -0400 +@@ -768,21 +768,6 @@ class PyBuildExt(build_ext): + print ("warning: openssl 0x%08x is too old for _hashlib" % + openssl_ver) + missing.append('_hashlib') +- if COMPILED_WITH_PYDEBUG or not have_usable_openssl: +- # The _sha module implements the SHA1 hash algorithm. +- exts.append( Extension('_sha', ['shamodule.c']) ) +- # The _md5 module implements the RSA Data Security, Inc. MD5 +- # Message-Digest Algorithm, described in RFC 1321. The +- # necessary files md5.c and md5.h are included here. +- exts.append( Extension('_md5', +- sources = ['md5module.c', 'md5.c'], +- depends = ['md5.h']) ) +- +- min_sha2_openssl_ver = 0x00908000 +- if COMPILED_WITH_PYDEBUG or openssl_ver < min_sha2_openssl_ver: +- # OpenSSL doesn't do these until 0.9.8 so we'll bring our own hash +- exts.append( Extension('_sha256', ['sha256module.c']) ) +- exts.append( Extension('_sha512', ['sha512module.c']) ) + + # Modules that provide persistent dictionary-like semantics. You will + # probably want to arrange for at least one of them to be available on diff --git a/SOURCES/00147-add-debug-malloc-stats.patch b/SOURCES/00147-add-debug-malloc-stats.patch new file mode 100644 index 00000000..48952f0f --- /dev/null +++ b/SOURCES/00147-add-debug-malloc-stats.patch @@ -0,0 +1,762 @@ +diff --git a/Include/dictobject.h b/Include/dictobject.h +index ece01c6..acc1df0 100644 +--- a/Include/dictobject.h ++++ b/Include/dictobject.h +@@ -150,6 +150,8 @@ PyAPI_FUNC(PyObject *) PyDict_GetItemString(PyObject *dp, const char *key); + PyAPI_FUNC(int) PyDict_SetItemString(PyObject *dp, const char *key, PyObject *item); + PyAPI_FUNC(int) PyDict_DelItemString(PyObject *dp, const char *key); + ++PyAPI_FUNC(void) _PyDict_DebugMallocStats(FILE *out); ++ + #ifdef __cplusplus + } + #endif +diff --git a/Include/floatobject.h b/Include/floatobject.h +index 54e8825..33c6ac0 100644 +--- a/Include/floatobject.h ++++ b/Include/floatobject.h +@@ -132,6 +132,7 @@ PyAPI_FUNC(PyObject *) _PyFloat_FormatAdvanced(PyObject *obj, + failure. Used in builtin_round in bltinmodule.c. */ + PyAPI_FUNC(PyObject *) _Py_double_round(double x, int ndigits); + ++PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); + + + #ifdef __cplusplus +diff --git a/Include/frameobject.h b/Include/frameobject.h +index 17e7679..66d9d8b 100644 +--- a/Include/frameobject.h ++++ b/Include/frameobject.h +@@ -80,6 +80,8 @@ PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + + PyAPI_FUNC(int) PyFrame_ClearFreeList(void); + ++PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out); ++ + /* Return the line of code the frame is currently executing. */ + PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); + +diff --git a/Include/intobject.h b/Include/intobject.h +index 252eea9..4003736 100644 +--- a/Include/intobject.h ++++ b/Include/intobject.h +@@ -75,6 +75,8 @@ PyAPI_FUNC(PyObject *) _PyInt_FormatAdvanced(PyObject *obj, + char *format_spec, + Py_ssize_t format_spec_len); + ++PyAPI_FUNC(void) _PyInt_DebugMallocStats(FILE *out); ++ + #ifdef __cplusplus + } + #endif +diff --git a/Include/listobject.h b/Include/listobject.h +index c445873..04664d7 100644 +--- a/Include/listobject.h ++++ b/Include/listobject.h +@@ -62,6 +62,8 @@ PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *); + #define PyList_SET_ITEM(op, i, v) (((PyListObject *)(op))->ob_item[i] = (v)) + #define PyList_GET_SIZE(op) Py_SIZE(op) + ++PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); ++ + #ifdef __cplusplus + } + #endif +diff --git a/Include/methodobject.h b/Include/methodobject.h +index 6e160b6..1944517 100644 +--- a/Include/methodobject.h ++++ b/Include/methodobject.h +@@ -87,6 +87,10 @@ typedef struct { + + PyAPI_FUNC(int) PyCFunction_ClearFreeList(void); + ++PyAPI_FUNC(void) _PyCFunction_DebugMallocStats(FILE *out); ++PyAPI_FUNC(void) _PyMethod_DebugMallocStats(FILE *out); ++ ++ + #ifdef __cplusplus + } + #endif +diff --git a/Include/object.h b/Include/object.h +index afbc68d..ce5febf 100644 +--- a/Include/object.h ++++ b/Include/object.h +@@ -1005,6 +1005,13 @@ PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(void); + _PyTrash_thread_deposit_object((PyObject*)op); \ + } while (0); + ++PyAPI_FUNC(void) ++_PyDebugAllocatorStats(FILE *out, const char *block_name, int num_blocks, ++ size_t sizeof_block); ++ ++PyAPI_FUNC(void) ++_PyObject_DebugTypeStats(FILE *out); ++ + #ifdef __cplusplus + } + #endif +diff --git a/Include/objimpl.h b/Include/objimpl.h +index 55e83ec..331b456 100644 +--- a/Include/objimpl.h ++++ b/Include/objimpl.h +@@ -101,13 +101,13 @@ PyAPI_FUNC(void) PyObject_Free(void *); + + /* Macros */ + #ifdef WITH_PYMALLOC ++PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out); + #ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ + PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes); + PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes); + PyAPI_FUNC(void) _PyObject_DebugFree(void *p); + PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p); + PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p); +-PyAPI_FUNC(void) _PyObject_DebugMallocStats(void); + PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes); + PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes); + PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p); +diff --git a/Include/setobject.h b/Include/setobject.h +index 52b07d5..143b175 100644 +--- a/Include/setobject.h ++++ b/Include/setobject.h +@@ -93,6 +93,7 @@ PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, + PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set); + PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); + ++PyAPI_FUNC(void) _PySet_DebugMallocStats(FILE *out); + #ifdef __cplusplus + } + #endif +diff --git a/Include/stringobject.h b/Include/stringobject.h +index 18b5b41..de78d76 100644 +--- a/Include/stringobject.h ++++ b/Include/stringobject.h +@@ -204,6 +204,8 @@ PyAPI_FUNC(PyObject *) _PyBytes_FormatAdvanced(PyObject *obj, + char *format_spec, + Py_ssize_t format_spec_len); + ++PyAPI_FUNC(void) _PyString_DebugMallocStats(FILE *out); ++ + #ifdef __cplusplus + } + #endif +diff --git a/Include/tupleobject.h b/Include/tupleobject.h +index a5ab733..e233f47 100644 +--- a/Include/tupleobject.h ++++ b/Include/tupleobject.h +@@ -54,7 +54,7 @@ PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *); + #define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v) + + PyAPI_FUNC(int) PyTuple_ClearFreeList(void); +- ++PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); + #ifdef __cplusplus + } + #endif +diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h +index 9ab724a..b91250a 100644 +--- a/Include/unicodeobject.h ++++ b/Include/unicodeobject.h +@@ -1406,6 +1406,8 @@ PyAPI_FUNC(int) _PyUnicode_IsAlpha( + Py_UNICODE ch /* Unicode character */ + ); + ++PyAPI_FUNC(void) _PyUnicode_DebugMallocStats(FILE *out); ++ + #ifdef __cplusplus + } + #endif +diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py +index 82243f3..8f1e1a0 100644 +--- a/Lib/test/test_sys.py ++++ b/Lib/test/test_sys.py +@@ -488,6 +488,32 @@ class SysModuleTest(unittest.TestCase): + p.wait() + self.assertIn(executable, ["''", repr(sys.executable)]) + ++ def test_debugmallocstats(self): ++ # Test sys._debugmallocstats() ++ ++ import subprocess ++ ++ # Verify the default of writing to stderr: ++ p = subprocess.Popen([sys.executable, ++ '-c', 'import sys; sys._debugmallocstats()'], ++ stderr=subprocess.PIPE) ++ out, err = p.communicate() ++ p.wait() ++ self.assertIn("arenas allocated current", err) ++ ++ # Verify that we can redirect the output to a file (not a file-like ++ # object, though): ++ with open('mallocstats.txt', 'w') as out: ++ sys._debugmallocstats(out) ++ result = open('mallocstats.txt').read() ++ self.assertIn("arenas allocated current", result) ++ os.unlink('mallocstats.txt') ++ ++ # Verify that the destination must be a file: ++ with self.assertRaises(TypeError): ++ sys._debugmallocstats(42) ++ ++ + class SizeofTest(unittest.TestCase): + + def setUp(self): +diff --git a/Objects/classobject.c b/Objects/classobject.c +index 2c9c216..2ba7077 100644 +--- a/Objects/classobject.c ++++ b/Objects/classobject.c +@@ -2694,3 +2694,12 @@ PyMethod_Fini(void) + { + (void)PyMethod_ClearFreeList(); + } ++ ++/* Print summary info about the state of the optimized allocator */ ++void ++_PyMethod_DebugMallocStats(FILE *out) ++{ ++ _PyDebugAllocatorStats(out, ++ "free PyMethodObject", ++ numfree, sizeof(PyMethodObject)); ++} +diff --git a/Objects/dictobject.c b/Objects/dictobject.c +index ba36b18..b8a5c7f 100644 +--- a/Objects/dictobject.c ++++ b/Objects/dictobject.c +@@ -225,6 +225,15 @@ show_track(void) + static PyDictObject *free_list[PyDict_MAXFREELIST]; + static int numfree = 0; + ++/* Print summary info about the state of the optimized allocator */ ++void ++_PyDict_DebugMallocStats(FILE *out) ++{ ++ _PyDebugAllocatorStats(out, ++ "free PyDictObject", numfree, sizeof(PyDictObject)); ++} ++ ++ + void + PyDict_Fini(void) + { +diff --git a/Objects/floatobject.c b/Objects/floatobject.c +index ba867ef..533511d 100644 +--- a/Objects/floatobject.c ++++ b/Objects/floatobject.c +@@ -35,6 +35,22 @@ typedef struct _floatblock PyFloatBlock; + static PyFloatBlock *block_list = NULL; + static PyFloatObject *free_list = NULL; + ++/* Print summary info about the state of the optimized allocator */ ++void ++_PyFloat_DebugMallocStats(FILE *out) ++{ ++ int num_blocks = 0; ++ PyFloatBlock *block; ++ ++ /* Walk the block list, counting */ ++ for (block = block_list; block ; block = block->next) { ++ num_blocks++; ++ } ++ ++ _PyDebugAllocatorStats(out, ++ "PyFloatBlock", num_blocks, sizeof(PyFloatBlock)); ++} ++ + static PyFloatObject * + fill_free_list(void) + { +diff --git a/Objects/frameobject.c b/Objects/frameobject.c +index f9e4a0e..337fc58 100644 +--- a/Objects/frameobject.c ++++ b/Objects/frameobject.c +@@ -982,3 +982,13 @@ PyFrame_Fini(void) + Py_XDECREF(builtin_object); + builtin_object = NULL; + } ++ ++/* Print summary info about the state of the optimized allocator */ ++void ++_PyFrame_DebugMallocStats(FILE *out) ++{ ++ _PyDebugAllocatorStats(out, ++ "free PyFrameObject", ++ numfree, sizeof(PyFrameObject)); ++} ++ +diff --git a/Objects/intobject.c b/Objects/intobject.c +index 28182f9..f442ea0 100644 +--- a/Objects/intobject.c ++++ b/Objects/intobject.c +@@ -44,6 +44,23 @@ typedef struct _intblock PyIntBlock; + static PyIntBlock *block_list = NULL; + static PyIntObject *free_list = NULL; + ++ ++/* Print summary info about the state of the optimized allocator */ ++void ++_PyInt_DebugMallocStats(FILE *out) ++{ ++ int num_blocks = 0; ++ PyIntBlock *block; ++ ++ /* Walk the block list, counting */ ++ for (block = block_list; block ; block = block->next) { ++ num_blocks++; ++ } ++ ++ _PyDebugAllocatorStats(out, ++ "PyIntBlock", num_blocks, sizeof(PyIntBlock)); ++} ++ + static PyIntObject * + fill_free_list(void) + { +diff --git a/Objects/listobject.c b/Objects/listobject.c +index f753643..e6fa17d 100644 +--- a/Objects/listobject.c ++++ b/Objects/listobject.c +@@ -109,6 +109,15 @@ PyList_Fini(void) + } + } + ++/* Print summary info about the state of the optimized allocator */ ++void ++_PyList_DebugMallocStats(FILE *out) ++{ ++ _PyDebugAllocatorStats(out, ++ "free PyListObject", ++ numfree, sizeof(PyListObject)); ++} ++ + PyObject * + PyList_New(Py_ssize_t size) + { +diff --git a/Objects/methodobject.c b/Objects/methodobject.c +index 0b60ca3..3193135 100644 +--- a/Objects/methodobject.c ++++ b/Objects/methodobject.c +@@ -412,6 +412,15 @@ PyCFunction_Fini(void) + (void)PyCFunction_ClearFreeList(); + } + ++/* Print summary info about the state of the optimized allocator */ ++void ++_PyCFunction_DebugMallocStats(FILE *out) ++{ ++ _PyDebugAllocatorStats(out, ++ "free PyCFunction", ++ numfree, sizeof(PyCFunction)); ++} ++ + /* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(), + but it's part of the API so we need to keep a function around that + existing C extensions can call. +diff --git a/Objects/object.c b/Objects/object.c +index 14f4e9f..68aedcd 100644 +--- a/Objects/object.c ++++ b/Objects/object.c +@@ -2355,6 +2355,23 @@ PyMem_Free(void *p) + PyMem_FREE(p); + } + ++void ++_PyObject_DebugTypeStats(FILE *out) ++{ ++ _PyString_DebugMallocStats(out); ++ _PyCFunction_DebugMallocStats(out); ++ _PyDict_DebugMallocStats(out); ++ _PyFloat_DebugMallocStats(out); ++ _PyFrame_DebugMallocStats(out); ++ _PyInt_DebugMallocStats(out); ++ _PyList_DebugMallocStats(out); ++ _PyMethod_DebugMallocStats(out); ++ _PySet_DebugMallocStats(out); ++ _PyTuple_DebugMallocStats(out); ++#if Py_USING_UNICODE ++ _PyUnicode_DebugMallocStats(out); ++#endif ++} + + /* These methods are used to control infinite recursion in repr, str, print, + etc. Container objects that may recursively contain themselves, +diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c +index 38ebc37..2c05359 100644 +--- a/Objects/obmalloc.c ++++ b/Objects/obmalloc.c +@@ -508,12 +508,10 @@ static struct arena_object* usable_arenas = NULL; + /* Number of arenas allocated that haven't been free()'d. */ + static size_t narenas_currently_allocated = 0; + +-#ifdef PYMALLOC_DEBUG + /* Total number of times malloc() called to allocate an arena. */ + static size_t ntimes_arena_allocated = 0; + /* High water mark (max value ever seen) for narenas_currently_allocated. */ + static size_t narenas_highwater = 0; +-#endif + + /* Allocate a new arena. If we run out of memory, return NULL. Else + * allocate a new arena, and return the address of an arena_object +@@ -528,7 +526,7 @@ new_arena(void) + + #ifdef PYMALLOC_DEBUG + if (Py_GETENV("PYTHONMALLOCSTATS")) +- _PyObject_DebugMallocStats(); ++ _PyObject_DebugMallocStats(stderr); + #endif + if (unused_arena_objects == NULL) { + uint i; +@@ -588,11 +586,9 @@ new_arena(void) + } + + ++narenas_currently_allocated; +-#ifdef PYMALLOC_DEBUG + ++ntimes_arena_allocated; + if (narenas_currently_allocated > narenas_highwater) + narenas_highwater = narenas_currently_allocated; +-#endif + arenaobj->freepools = NULL; + /* pool_address <- first pool-aligned address in the arena + nfreepools <- number of whole pools that fit after alignment */ +@@ -1694,17 +1690,19 @@ _PyObject_DebugDumpAddress(const void *p) + } + } + ++#endif /* PYMALLOC_DEBUG */ ++ + static size_t +-printone(const char* msg, size_t value) ++printone(FILE *out, const char* msg, size_t value) + { + int i, k; + char buf[100]; + size_t origvalue = value; + +- fputs(msg, stderr); ++ fputs(msg, out); + for (i = (int)strlen(msg); i < 35; ++i) +- fputc(' ', stderr); +- fputc('=', stderr); ++ fputc(' ', out); ++ fputc('=', out); + + /* Write the value with commas. */ + i = 22; +@@ -1725,17 +1723,32 @@ printone(const char* msg, size_t value) + + while (i >= 0) + buf[i--] = ' '; +- fputs(buf, stderr); ++ fputs(buf, out); + + return origvalue; + } + +-/* Print summary info to stderr about the state of pymalloc's structures. ++void ++_PyDebugAllocatorStats(FILE *out, ++ const char *block_name, int num_blocks, size_t sizeof_block) ++{ ++ char buf1[128]; ++ char buf2[128]; ++ PyOS_snprintf(buf1, sizeof(buf1), ++ "%d %ss * %zd bytes each", ++ num_blocks, block_name, sizeof_block); ++ PyOS_snprintf(buf2, sizeof(buf2), ++ "%48s ", buf1); ++ (void)printone(out, buf2, num_blocks * sizeof_block); ++} ++ ++ ++/* Print summary info to "out" about the state of pymalloc's structures. + * In Py_DEBUG mode, also perform some expensive internal consistency + * checks. + */ + void +-_PyObject_DebugMallocStats(void) ++_PyObject_DebugMallocStats(FILE *out) + { + uint i; + const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT; +@@ -1764,7 +1777,7 @@ _PyObject_DebugMallocStats(void) + size_t total; + char buf[128]; + +- fprintf(stderr, "Small block threshold = %d, in %u size classes.\n", ++ fprintf(out, "Small block threshold = %d, in %u size classes.\n", + SMALL_REQUEST_THRESHOLD, numclasses); + + for (i = 0; i < numclasses; ++i) +@@ -1818,10 +1831,10 @@ _PyObject_DebugMallocStats(void) + } + assert(narenas == narenas_currently_allocated); + +- fputc('\n', stderr); ++ fputc('\n', out); + fputs("class size num pools blocks in use avail blocks\n" + "----- ---- --------- ------------- ------------\n", +- stderr); ++ out); + + for (i = 0; i < numclasses; ++i) { + size_t p = numpools[i]; +@@ -1832,7 +1845,7 @@ _PyObject_DebugMallocStats(void) + assert(b == 0 && f == 0); + continue; + } +- fprintf(stderr, "%5u %6u " ++ fprintf(out, "%5u %6u " + "%11" PY_FORMAT_SIZE_T "u " + "%15" PY_FORMAT_SIZE_T "u " + "%13" PY_FORMAT_SIZE_T "u\n", +@@ -1842,36 +1855,35 @@ _PyObject_DebugMallocStats(void) + pool_header_bytes += p * POOL_OVERHEAD; + quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); + } +- fputc('\n', stderr); +- (void)printone("# times object malloc called", serialno); +- +- (void)printone("# arenas allocated total", ntimes_arena_allocated); +- (void)printone("# arenas reclaimed", ntimes_arena_allocated - narenas); +- (void)printone("# arenas highwater mark", narenas_highwater); +- (void)printone("# arenas allocated current", narenas); ++ fputc('\n', out); ++#ifdef PYMALLOC_DEBUG ++ (void)printone(out, "# times object malloc called", serialno); ++#endif ++ (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); ++ (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); ++ (void)printone(out, "# arenas highwater mark", narenas_highwater); ++ (void)printone(out, "# arenas allocated current", narenas); + + PyOS_snprintf(buf, sizeof(buf), + "%" PY_FORMAT_SIZE_T "u arenas * %d bytes/arena", + narenas, ARENA_SIZE); +- (void)printone(buf, narenas * ARENA_SIZE); ++ (void)printone(out, buf, narenas * ARENA_SIZE); + +- fputc('\n', stderr); ++ fputc('\n', out); + +- total = printone("# bytes in allocated blocks", allocated_bytes); +- total += printone("# bytes in available blocks", available_bytes); ++ total = printone(out, "# bytes in allocated blocks", allocated_bytes); ++ total += printone(out, "# bytes in available blocks", available_bytes); + + PyOS_snprintf(buf, sizeof(buf), + "%u unused pools * %d bytes", numfreepools, POOL_SIZE); +- total += printone(buf, (size_t)numfreepools * POOL_SIZE); ++ total += printone(out, buf, (size_t)numfreepools * POOL_SIZE); + +- total += printone("# bytes lost to pool headers", pool_header_bytes); +- total += printone("# bytes lost to quantization", quantization); +- total += printone("# bytes lost to arena alignment", arena_alignment); +- (void)printone("Total", total); ++ total += printone(out, "# bytes lost to pool headers", pool_header_bytes); ++ total += printone(out, "# bytes lost to quantization", quantization); ++ total += printone(out, "# bytes lost to arena alignment", arena_alignment); ++ (void)printone(out, "Total", total); + } + +-#endif /* PYMALLOC_DEBUG */ +- + #ifdef Py_USING_MEMORY_DEBUGGER + /* Make this function last so gcc won't inline it since the definition is + * after the reference. +diff --git a/Objects/setobject.c b/Objects/setobject.c +index af1ce16..3439b7c 100644 +--- a/Objects/setobject.c ++++ b/Objects/setobject.c +@@ -1088,6 +1088,16 @@ PySet_Fini(void) + Py_CLEAR(emptyfrozenset); + } + ++/* Print summary info about the state of the optimized allocator */ ++void ++_PySet_DebugMallocStats(FILE *out) ++{ ++ _PyDebugAllocatorStats(out, ++ "free PySetObject", ++ numfree, sizeof(PySetObject)); ++} ++ ++ + static PyObject * + set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) + { +diff --git a/Objects/stringobject.c b/Objects/stringobject.c +index 1209197..b8646dd 100644 +--- a/Objects/stringobject.c ++++ b/Objects/stringobject.c +@@ -4843,3 +4843,43 @@ void _Py_ReleaseInternedStrings(void) + PyDict_Clear(interned); + Py_CLEAR(interned); + } ++ ++void _PyString_DebugMallocStats(FILE *out) ++{ ++ ssize_t i; ++ int num_immortal = 0, num_mortal = 0; ++ ssize_t immortal_size = 0, mortal_size = 0; ++ ++ if (interned == NULL || !PyDict_Check(interned)) ++ return; ++ ++ for (i = 0; i <= ((PyDictObject*)interned)->ma_mask; i++) { ++ PyDictEntry *ep = ((PyDictObject*)interned)->ma_table + i; ++ PyObject *pvalue = ep->me_value; ++ if (pvalue != NULL) { ++ PyStringObject *s = (PyStringObject *)ep->me_key; ++ ++ switch (s->ob_sstate) { ++ case SSTATE_NOT_INTERNED: ++ /* XXX Shouldn't happen */ ++ break; ++ case SSTATE_INTERNED_IMMORTAL: ++ num_immortal ++; ++ immortal_size += s->ob_size; ++ break; ++ case SSTATE_INTERNED_MORTAL: ++ num_mortal ++; ++ mortal_size += s->ob_size; ++ break; ++ default: ++ Py_FatalError("Inconsistent interned string state."); ++ } ++ } ++ } ++ ++ fprintf(out, "%d mortal interned strings\n", num_mortal); ++ fprintf(out, "%d immortal interned strings\n", num_immortal); ++ fprintf(out, "total size of all interned strings: " ++ "%zi/%zi " ++ "mortal/immortal\n", mortal_size, immortal_size); ++} +diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c +index 00f2e47..7682d81 100644 +--- a/Objects/tupleobject.c ++++ b/Objects/tupleobject.c +@@ -44,6 +44,22 @@ show_track(void) + } + #endif + ++/* Print summary info about the state of the optimized allocator */ ++void ++_PyTuple_DebugMallocStats(FILE *out) ++{ ++#if PyTuple_MAXSAVESIZE > 0 ++ int i; ++ char buf[128]; ++ for (i = 1; i < PyTuple_MAXSAVESIZE; i++) { ++ PyOS_snprintf(buf, sizeof(buf), ++ "free %d-sized PyTupleObject", i); ++ _PyDebugAllocatorStats(out, ++ buf, ++ numfree[i], _PyObject_VAR_SIZE(&PyTuple_Type, i)); ++ } ++#endif ++} + + PyObject * + PyTuple_New(register Py_ssize_t size) +diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c +index 6bea370..ced9acf 100644 +--- a/Objects/unicodeobject.c ++++ b/Objects/unicodeobject.c +@@ -8920,6 +8920,12 @@ _PyUnicode_Fini(void) + (void)PyUnicode_ClearFreeList(); + } + ++void _PyUnicode_DebugMallocStats(FILE *out) ++{ ++ _PyDebugAllocatorStats(out, "free PyUnicodeObject", numfree, ++ sizeof(PyUnicodeObject)); ++} ++ + #ifdef __cplusplus + } + #endif +diff --git a/Python/pythonrun.c b/Python/pythonrun.c +index f0fbd74..0b73f3a 100644 +--- a/Python/pythonrun.c ++++ b/Python/pythonrun.c +@@ -557,7 +557,7 @@ Py_Finalize(void) + #endif /* Py_TRACE_REFS */ + #ifdef PYMALLOC_DEBUG + if (Py_GETENV("PYTHONMALLOCSTATS")) +- _PyObject_DebugMallocStats(); ++ _PyObject_DebugMallocStats(stderr); + #endif + + call_ll_exitfuncs(); +diff --git a/Python/sysmodule.c b/Python/sysmodule.c +index 2a7c207..fbb637b 100644 +--- a/Python/sysmodule.c ++++ b/Python/sysmodule.c +@@ -873,6 +873,57 @@ a 11-tuple where the entries in the tuple are counts of:\n\ + extern "C" { + #endif + ++static PyObject * ++sys_debugmallocstats(PyObject *self, PyObject *args) ++{ ++ PyObject *file = NULL; ++ FILE *fp; ++ ++ if (!PyArg_ParseTuple(args, "|O!", ++ &PyFile_Type, &file)) { ++ return NULL; ++ } ++ if (!file) { ++ /* Default to sys.stderr: */ ++ file = PySys_GetObject("stderr"); ++ if (!file) { ++ PyErr_SetString(PyExc_ValueError, "sys.stderr not set"); ++ return NULL; ++ } ++ if (!PyFile_Check(file)) { ++ PyErr_SetString(PyExc_TypeError, "sys.stderr is not a file"); ++ return NULL; ++ } ++ } ++ ++ Py_INCREF(file); ++ /* OK, we now own a ref on non-NULL "file" */ ++ ++ fp = PyFile_AsFile(file); ++ if (!fp) { ++ PyErr_SetString(PyExc_ValueError, "file is closed"); ++ Py_DECREF(file); ++ return NULL; ++ } ++ ++ _PyObject_DebugMallocStats(fp); ++ fputc('\n', fp); ++ _PyObject_DebugTypeStats(fp); ++ ++ Py_DECREF(file); ++ ++ Py_RETURN_NONE; ++} ++PyDoc_STRVAR(debugmallocstats_doc, ++"_debugmallocstats([file])\n\ ++\n\ ++Print summary info to the given file (or sys.stderr) about the state of\n\ ++pymalloc's structures.\n\ ++\n\ ++In Py_DEBUG mode, also perform some expensive internal consistency\n\ ++checks.\n\ ++"); ++ + #ifdef Py_TRACE_REFS + /* Defined in objects.c because it uses static globals if that file */ + extern PyObject *_Py_GetObjects(PyObject *, PyObject *); +@@ -971,6 +1022,8 @@ static PyMethodDef sys_methods[] = { + {"settrace", sys_settrace, METH_O, settrace_doc}, + {"gettrace", sys_gettrace, METH_NOARGS, gettrace_doc}, + {"call_tracing", sys_call_tracing, METH_VARARGS, call_tracing_doc}, ++ {"_debugmallocstats", sys_debugmallocstats, METH_VARARGS, ++ debugmallocstats_doc}, + {NULL, NULL} /* sentinel */ + }; + diff --git a/SOURCES/00153-fix-test_gdb-noise.patch b/SOURCES/00153-fix-test_gdb-noise.patch new file mode 100644 index 00000000..50a0917e --- /dev/null +++ b/SOURCES/00153-fix-test_gdb-noise.patch @@ -0,0 +1,31 @@ +--- Lib/test/test_gdb.py.old 2012-04-11 21:04:01.367073855 -0400 ++++ Lib/test/test_gdb.py 2012-04-12 08:52:58.320288761 -0400 +@@ -96,6 +96,15 @@ class DebuggerTests(unittest.TestCase): + # Generate a list of commands in gdb's language: + commands = ['set breakpoint pending yes', + 'break %s' % breakpoint, ++ ++ # GDB as of Fedora 17 onwards can distinguish between the ++ # value of a variable at entry vs current value: ++ # http://sourceware.org/gdb/onlinedocs/gdb/Variables.html ++ # which leads to the selftests failing with errors like this: ++ # AssertionError: 'v@entry=()' != '()' ++ # Disable this: ++ 'set print entry-values no', ++ + 'run'] + if cmds_after_breakpoint: + commands += cmds_after_breakpoint +--- Lib/test/test_gdb.py.old 2012-04-11 21:04:01.367073855 -0400 ++++ Lib/test/test_gdb.py 2012-04-12 08:52:58.320288761 -0400 +@@ -144,6 +153,10 @@ + 'Do you need "set solib-search-path" or ' + '"set sysroot"?', + ) ++ ignore_patterns += ('warning: Unable to open', ++ 'Missing separate debuginfo for', ++ 'Try: yum --disablerepo=', ++ 'Undefined set print command') + for line in errlines: + if not line.startswith(ignore_patterns): + unexpected_errlines.append(line) diff --git a/SOURCES/00155-avoid-ctypes-thunks.patch b/SOURCES/00155-avoid-ctypes-thunks.patch index f03890ee..92dd6685 100644 --- a/SOURCES/00155-avoid-ctypes-thunks.patch +++ b/SOURCES/00155-avoid-ctypes-thunks.patch @@ -1,7 +1,7 @@ -diff -up Python-3.2.3/Lib/ctypes/__init__.py.rhbz814391 Python-3.2.3/Lib/ctypes/__init__.py ---- Python-3.2.3/Lib/ctypes/__init__.py.rhbz814391 2012-04-20 15:12:49.017867692 -0400 -+++ Python-3.2.3/Lib/ctypes/__init__.py 2012-04-20 15:15:09.501111408 -0400 -@@ -275,11 +275,6 @@ def _reset_cache(): +diff -up Python-2.7.3/Lib/ctypes/__init__.py.rhbz814391 Python-2.7.3/Lib/ctypes/__init__.py +--- Python-2.7.3/Lib/ctypes/__init__.py.rhbz814391 2012-04-20 14:51:19.390990244 -0400 ++++ Python-2.7.3/Lib/ctypes/__init__.py 2012-04-20 14:51:45.141668316 -0400 +@@ -272,11 +272,6 @@ def _reset_cache(): # _SimpleCData.c_char_p_from_param POINTER(c_char).from_param = c_char_p.from_param _pointer_type_cache[None] = c_void_p @@ -11,5 +11,5 @@ diff -up Python-3.2.3/Lib/ctypes/__init__.py.rhbz814391 Python-3.2.3/Lib/ctypes/ - # compiled with the MS SDK compiler. Or an uninitialized variable? - CFUNCTYPE(c_int)(lambda: None) - def create_unicode_buffer(init, size=None): - """create_unicode_buffer(aString) -> character array + try: + from _ctypes import set_conversion_mode diff --git a/SOURCES/00156-gdb-autoload-safepath.patch b/SOURCES/00156-gdb-autoload-safepath.patch new file mode 100644 index 00000000..54a5a6e7 --- /dev/null +++ b/SOURCES/00156-gdb-autoload-safepath.patch @@ -0,0 +1,52 @@ +diff -up Python-2.7.3/Lib/test/test_gdb.py.gdb-autoload-safepath Python-2.7.3/Lib/test/test_gdb.py +--- Python-2.7.3/Lib/test/test_gdb.py.gdb-autoload-safepath 2012-04-30 15:53:57.254045220 -0400 ++++ Python-2.7.3/Lib/test/test_gdb.py 2012-04-30 16:19:19.569941124 -0400 +@@ -54,6 +54,19 @@ def gdb_has_frame_select(): + + HAS_PYUP_PYDOWN = gdb_has_frame_select() + ++def gdb_has_autoload_safepath(): ++ # Recent GDBs will only auto-load scripts from certain safe ++ # locations, so we will need to turn off this protection. ++ # However, if the GDB doesn't have it, then the following ++ # command will generate noise on stderr (rhbz#817072): ++ cmd = "--eval-command=set auto-load safe-path /" ++ p = subprocess.Popen(["gdb", "--batch", cmd], ++ stderr=subprocess.PIPE) ++ _, stderr = p.communicate() ++ return '"on" or "off" expected.' not in stderr ++ ++HAS_AUTOLOAD_SAFEPATH = gdb_has_autoload_safepath() ++ + class DebuggerTests(unittest.TestCase): + + """Test that the debugger can debug Python.""" +@@ -112,15 +125,28 @@ class DebuggerTests(unittest.TestCase): + 'set print entry-values no', + + 'run'] ++ ++ if HAS_AUTOLOAD_SAFEPATH: ++ # Recent GDBs will only auto-load scripts from certain safe ++ # locations. ++ # Where necessary, turn off this protection to ensure that ++ # our -gdb.py script can be loaded - but not on earlier gdb builds ++ # as this would generate noise on stderr (rhbz#817072): ++ init_commands = ['set auto-load safe-path /'] ++ else: ++ init_commands = [] ++ + if cmds_after_breakpoint: + commands += cmds_after_breakpoint + else: + commands += ['backtrace'] + ++ # print init_commands + # print commands + + # Use "commands" to generate the arguments with which to invoke "gdb": + args = ["gdb", "--batch"] ++ args += ['--init-eval-command=%s' % cmd for cmd in init_commands] + args += ['--eval-command=%s' % cmd for cmd in commands] + args += ["--args", + sys.executable] diff --git a/SOURCES/00157-uid-gid-overflows.patch b/SOURCES/00157-uid-gid-overflows.patch index 03f3e021..13546bbf 100644 --- a/SOURCES/00157-uid-gid-overflows.patch +++ b/SOURCES/00157-uid-gid-overflows.patch @@ -1,68 +1,49 @@ -diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py -index e9fdb07..ea60e6e 100644 ---- a/Lib/test/test_os.py -+++ b/Lib/test/test_os.py -@@ -1723,30 +1723,36 @@ class PosixUidGidTests(unittest.TestCase): - def test_setuid(self): - if os.getuid() != 0: - self.assertRaises(OSError, os.setuid, 0) -+ self.assertRaises(TypeError, os.setuid, 'not an int') - self.assertRaises(OverflowError, os.setuid, 1<<32) +diff -up Python-2.7.3/Lib/test/test_os.py.uid-gid-overflows Python-2.7.3/Lib/test/test_os.py +--- Python-2.7.3/Lib/test/test_os.py.uid-gid-overflows 2012-04-09 19:07:32.000000000 -0400 ++++ Python-2.7.3/Lib/test/test_os.py 2012-06-26 14:51:36.000817929 -0400 +@@ -677,30 +677,36 @@ if sys.platform != 'win32': + def test_setuid(self): + if os.getuid() != 0: + self.assertRaises(os.error, os.setuid, 0) ++ self.assertRaises(TypeError, os.setuid, 'not an int') + self.assertRaises(OverflowError, os.setuid, 1<<32) - @unittest.skipUnless(hasattr(os, 'setgid'), 'test needs os.setgid()') - def test_setgid(self): - if os.getuid() != 0 and not HAVE_WHEEL_GROUP: - self.assertRaises(OSError, os.setgid, 0) -+ self.assertRaises(TypeError, os.setgid, 'not an int') - self.assertRaises(OverflowError, os.setgid, 1<<32) + if hasattr(os, 'setgid'): + def test_setgid(self): + if os.getuid() != 0: + self.assertRaises(os.error, os.setgid, 0) ++ self.assertRaises(TypeError, os.setgid, 'not an int') + self.assertRaises(OverflowError, os.setgid, 1<<32) - @unittest.skipUnless(hasattr(os, 'seteuid'), 'test needs os.seteuid()') - def test_seteuid(self): - if os.getuid() != 0: - self.assertRaises(OSError, os.seteuid, 0) -+ self.assertRaises(TypeError, os.seteuid, 'not an int') - self.assertRaises(OverflowError, os.seteuid, 1<<32) + if hasattr(os, 'seteuid'): + def test_seteuid(self): + if os.getuid() != 0: + self.assertRaises(os.error, os.seteuid, 0) ++ self.assertRaises(TypeError, os.seteuid, 'not an int') + self.assertRaises(OverflowError, os.seteuid, 1<<32) - @unittest.skipUnless(hasattr(os, 'setegid'), 'test needs os.setegid()') - def test_setegid(self): - if os.getuid() != 0 and not HAVE_WHEEL_GROUP: - self.assertRaises(OSError, os.setegid, 0) -+ self.assertRaises(TypeError, os.setegid, 'not an int') - self.assertRaises(OverflowError, os.setegid, 1<<32) + if hasattr(os, 'setegid'): + def test_setegid(self): + if os.getuid() != 0: + self.assertRaises(os.error, os.setegid, 0) ++ self.assertRaises(TypeError, os.setegid, 'not an int') + self.assertRaises(OverflowError, os.setegid, 1<<32) - @unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()') - def test_setreuid(self): - if os.getuid() != 0: - self.assertRaises(OSError, os.setreuid, 0, 0) -+ self.assertRaises(TypeError, os.setreuid, 'not an int', 0) -+ self.assertRaises(TypeError, os.setreuid, 0, 'not an int') - self.assertRaises(OverflowError, os.setreuid, 1<<32, 0) - self.assertRaises(OverflowError, os.setreuid, 0, 1<<32) + if hasattr(os, 'setreuid'): + def test_setreuid(self): + if os.getuid() != 0: + self.assertRaises(os.error, os.setreuid, 0, 0) ++ self.assertRaises(TypeError, os.setreuid, 'not an int', 0) ++ self.assertRaises(TypeError, os.setreuid, 0, 'not an int') + self.assertRaises(OverflowError, os.setreuid, 1<<32, 0) + self.assertRaises(OverflowError, os.setreuid, 0, 1<<32) -@@ -1762,6 +1768,8 @@ class PosixUidGidTests(unittest.TestCase): - def test_setregid(self): - if os.getuid() != 0 and not HAVE_WHEEL_GROUP: - self.assertRaises(OSError, os.setregid, 0, 0) -+ self.assertRaises(TypeError, os.setregid, 'not an int', 0) -+ self.assertRaises(TypeError, os.setregid, 0, 'not an int') - self.assertRaises(OverflowError, os.setregid, 1<<32, 0) - self.assertRaises(OverflowError, os.setregid, 0, 1<<32) +@@ -715,6 +721,8 @@ if sys.platform != 'win32': + def test_setregid(self): + if os.getuid() != 0: + self.assertRaises(os.error, os.setregid, 0, 0) ++ self.assertRaises(TypeError, os.setregid, 'not an int', 0) ++ self.assertRaises(TypeError, os.setregid, 0, 'not an int') + self.assertRaises(OverflowError, os.setregid, 1<<32, 0) + self.assertRaises(OverflowError, os.setregid, 0, 1<<32) -diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py -index ac9cff7..db98159 100644 ---- a/Lib/test/test_pwd.py -+++ b/Lib/test/test_pwd.py -@@ -104,11 +104,11 @@ class PwdTest(unittest.TestCase): - # In some cases, byuids isn't a complete list of all users in the - # system, so if we try to pick a value not in byuids (via a perturbing - # loop, say), pwd.getpwuid() might still be able to find data for that -- # uid. Using sys.maxint may provoke the same problems, but hopefully -+ # uid. Using 2**32 - 2 may provoke the same problems, but hopefully - # it will be a more repeatable failure. - # Android accepts a very large span of uids including sys.maxsize and - # -1; it raises KeyError with 1 or 2 for example. -- fakeuid = sys.maxsize -+ fakeuid = 2**32 - 2 - self.assertNotIn(fakeuid, byuids) - if not support.is_android: - self.assertRaises(KeyError, pwd.getpwuid, fakeuid) diff --git a/SOURCES/00165-crypt-module-salt-backport.patch b/SOURCES/00165-crypt-module-salt-backport.patch new file mode 100644 index 00000000..4308b4c2 --- /dev/null +++ b/SOURCES/00165-crypt-module-salt-backport.patch @@ -0,0 +1,285 @@ +diff -up Python-2.7.3/Doc/library/crypt.rst.crypt-module-salt-backport Python-2.7.3/Doc/library/crypt.rst +--- Python-2.7.3/Doc/library/crypt.rst.crypt-module-salt-backport 2012-04-09 19:07:28.000000000 -0400 ++++ Python-2.7.3/Doc/library/crypt.rst 2013-02-19 16:44:20.465334062 -0500 +@@ -16,9 +16,9 @@ + + This module implements an interface to the :manpage:`crypt(3)` routine, which is + a one-way hash function based upon a modified DES algorithm; see the Unix man +-page for further details. Possible uses include allowing Python scripts to +-accept typed passwords from the user, or attempting to crack Unix passwords with +-a dictionary. ++page for further details. Possible uses include storing hashed passwords ++so you can check passwords without storing the actual password, or attempting ++to crack Unix passwords with a dictionary. + + .. index:: single: crypt(3) + +@@ -27,15 +27,81 @@ the :manpage:`crypt(3)` routine in the r + extensions available on the current implementation will also be available on + this module. + ++Hashing Methods ++--------------- + +-.. function:: crypt(word, salt) ++The :mod:`crypt` module defines the list of hashing methods (not all methods ++are available on all platforms): ++ ++.. data:: METHOD_SHA512 ++ ++ A Modular Crypt Format method with 16 character salt and 86 character ++ hash. This is the strongest method. ++ ++.. versionadded:: 3.3 ++ ++.. data:: METHOD_SHA256 ++ ++ Another Modular Crypt Format method with 16 character salt and 43 ++ character hash. ++ ++.. versionadded:: 3.3 ++ ++.. data:: METHOD_MD5 ++ ++ Another Modular Crypt Format method with 8 character salt and 22 ++ character hash. ++ ++.. versionadded:: 3.3 ++ ++.. data:: METHOD_CRYPT ++ ++ The traditional method with a 2 character salt and 13 characters of ++ hash. This is the weakest method. ++ ++.. versionadded:: 3.3 ++ ++ ++Module Attributes ++----------------- ++ ++ ++.. attribute:: methods ++ ++ A list of available password hashing algorithms, as ++ ``crypt.METHOD_*`` objects. This list is sorted from strongest to ++ weakest, and is guaranteed to have at least ``crypt.METHOD_CRYPT``. ++ ++.. versionadded:: 3.3 ++ ++ ++Module Functions ++---------------- ++ ++The :mod:`crypt` module defines the following functions: ++ ++.. function:: crypt(word, salt=None) + + *word* will usually be a user's password as typed at a prompt or in a graphical +- interface. *salt* is usually a random two-character string which will be used +- to perturb the DES algorithm in one of 4096 ways. The characters in *salt* must +- be in the set ``[./a-zA-Z0-9]``. Returns the hashed password as a string, which +- will be composed of characters from the same alphabet as the salt (the first two +- characters represent the salt itself). ++ interface. The optional *salt* is either a string as returned from ++ :func:`mksalt`, one of the ``crypt.METHOD_*`` values (though not all ++ may be available on all platforms), or a full encrypted password ++ including salt, as returned by this function. If *salt* is not ++ provided, the strongest method will be used (as returned by ++ :func:`methods`. ++ ++ Checking a password is usually done by passing the plain-text password ++ as *word* and the full results of a previous :func:`crypt` call, ++ which should be the same as the results of this call. ++ ++ *salt* (either a random 2 or 16 character string, possibly prefixed with ++ ``$digit$`` to indicate the method) which will be used to perturb the ++ encryption algorithm. The characters in *salt* must be in the set ++ ``[./a-zA-Z0-9]``, with the exception of Modular Crypt Format which ++ prefixes a ``$digit$``. ++ ++ Returns the hashed password as a string, which will be composed of ++ characters from the same alphabet as the salt. + + .. index:: single: crypt(3) + +@@ -43,6 +109,27 @@ this module. + different sizes in the *salt*, it is recommended to use the full crypted + password as salt when checking for a password. + ++.. versionchanged:: 3.3 ++ Before version 3.3, *salt* must be specified as a string and cannot ++ accept ``crypt.METHOD_*`` values (which don't exist anyway). ++ ++ ++.. function:: mksalt(method=None) ++ ++ Return a randomly generated salt of the specified method. If no ++ *method* is given, the strongest method available as returned by ++ :func:`methods` is used. ++ ++ The return value is a string either of 2 characters in length for ++ ``crypt.METHOD_CRYPT``, or 19 characters starting with ``$digit$`` and ++ 16 random characters from the set ``[./a-zA-Z0-9]``, suitable for ++ passing as the *salt* argument to :func:`crypt`. ++ ++.. versionadded:: 3.3 ++ ++Examples ++-------- ++ + A simple example illustrating typical use:: + + import crypt, getpass, pwd +@@ -59,3 +146,11 @@ A simple example illustrating typical us + else: + return 1 + ++To generate a hash of a password using the strongest available method and ++check it against the original:: ++ ++ import crypt ++ ++ hashed = crypt.crypt(plaintext) ++ if hashed != crypt.crypt(plaintext, hashed): ++ raise "Hashed version doesn't validate against original" +diff -up Python-2.7.3/Lib/crypt.py.crypt-module-salt-backport Python-2.7.3/Lib/crypt.py +--- Python-2.7.3/Lib/crypt.py.crypt-module-salt-backport 2013-02-19 16:44:20.465334062 -0500 ++++ Python-2.7.3/Lib/crypt.py 2013-02-19 16:49:56.425311089 -0500 +@@ -0,0 +1,71 @@ ++"""Wrapper to the POSIX crypt library call and associated functionality. ++ ++Note that the ``methods`` and ``METHOD_*`` attributes are non-standard ++extensions to Python 2.7, backported from 3.3""" ++ ++import _crypt ++import string as _string ++from random import SystemRandom as _SystemRandom ++from collections import namedtuple as _namedtuple ++ ++ ++_saltchars = _string.ascii_letters + _string.digits + './' ++_sr = _SystemRandom() ++ ++ ++class _Method(_namedtuple('_Method', 'name ident salt_chars total_size')): ++ ++ """Class representing a salt method per the Modular Crypt Format or the ++ legacy 2-character crypt method.""" ++ ++ def __repr__(self): ++ return '' % self.name ++ ++ ++def mksalt(method=None): ++ """Generate a salt for the specified method. ++ ++ If not specified, the strongest available method will be used. ++ ++ This is a non-standard extension to Python 2.7, backported from 3.3 ++ """ ++ if method is None: ++ method = methods[0] ++ s = '$%s$' % method.ident if method.ident else '' ++ s += ''.join(_sr.sample(_saltchars, method.salt_chars)) ++ return s ++ ++ ++def crypt(word, salt=None): ++ """Return a string representing the one-way hash of a password, with a salt ++ prepended. ++ ++ If ``salt`` is not specified or is ``None``, the strongest ++ available method will be selected and a salt generated. Otherwise, ++ ``salt`` may be one of the ``crypt.METHOD_*`` values, or a string as ++ returned by ``crypt.mksalt()``. ++ ++ Note that these are non-standard extensions to Python 2.7's crypt.crypt() ++ entrypoint, backported from 3.3: the standard Python 2.7 crypt.crypt() ++ entrypoint requires two strings as the parameters, and does not support ++ keyword arguments. ++ """ ++ if salt is None or isinstance(salt, _Method): ++ salt = mksalt(salt) ++ return _crypt.crypt(word, salt) ++ ++ ++# available salting/crypto methods ++METHOD_CRYPT = _Method('CRYPT', None, 2, 13) ++METHOD_MD5 = _Method('MD5', '1', 8, 34) ++METHOD_SHA256 = _Method('SHA256', '5', 16, 63) ++METHOD_SHA512 = _Method('SHA512', '6', 16, 106) ++ ++methods = [] ++for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5): ++ _result = crypt('', _method) ++ if _result and len(_result) == _method.total_size: ++ methods.append(_method) ++methods.append(METHOD_CRYPT) ++del _result, _method ++ +diff -up Python-2.7.3/Lib/test/test_crypt.py.crypt-module-salt-backport Python-2.7.3/Lib/test/test_crypt.py +--- Python-2.7.3/Lib/test/test_crypt.py.crypt-module-salt-backport 2012-04-09 19:07:31.000000000 -0400 ++++ Python-2.7.3/Lib/test/test_crypt.py 2013-02-19 16:44:20.465334062 -0500 +@@ -10,6 +10,25 @@ class CryptTestCase(unittest.TestCase): + if test_support.verbose: + print 'Test encryption: ', c + ++ def test_salt(self): ++ self.assertEqual(len(crypt._saltchars), 64) ++ for method in crypt.methods: ++ salt = crypt.mksalt(method) ++ self.assertEqual(len(salt), ++ method.salt_chars + (3 if method.ident else 0)) ++ ++ def test_saltedcrypt(self): ++ for method in crypt.methods: ++ pw = crypt.crypt('assword', method) ++ self.assertEqual(len(pw), method.total_size) ++ pw = crypt.crypt('assword', crypt.mksalt(method)) ++ self.assertEqual(len(pw), method.total_size) ++ ++ def test_methods(self): ++ # Gurantee that METHOD_CRYPT is the last method in crypt.methods. ++ self.assertTrue(len(crypt.methods) >= 1) ++ self.assertEqual(crypt.METHOD_CRYPT, crypt.methods[-1]) ++ + def test_main(): + test_support.run_unittest(CryptTestCase) + +diff -up Python-2.7.3/Modules/cryptmodule.c.crypt-module-salt-backport Python-2.7.3/Modules/cryptmodule.c +--- Python-2.7.3/Modules/cryptmodule.c.crypt-module-salt-backport 2012-04-09 19:07:34.000000000 -0400 ++++ Python-2.7.3/Modules/cryptmodule.c 2013-02-19 16:44:20.466334063 -0500 +@@ -43,7 +43,7 @@ static PyMethodDef crypt_methods[] = { + }; + + PyMODINIT_FUNC +-initcrypt(void) ++init_crypt(void) + { +- Py_InitModule("crypt", crypt_methods); ++ Py_InitModule("_crypt", crypt_methods); + } +diff -up Python-2.7.3/Modules/Setup.dist.crypt-module-salt-backport Python-2.7.3/Modules/Setup.dist +--- Python-2.7.3/Modules/Setup.dist.crypt-module-salt-backport 2013-02-19 16:44:20.463334063 -0500 ++++ Python-2.7.3/Modules/Setup.dist 2013-02-19 16:44:20.466334063 -0500 +@@ -221,7 +221,7 @@ _ssl _ssl.c \ + # + # First, look at Setup.config; configure may have set this for you. + +-crypt cryptmodule.c # -lcrypt # crypt(3); needs -lcrypt on some systems ++_crypt _cryptmodule.c -lcrypt # crypt(3); needs -lcrypt on some systems + + + # Some more UNIX dependent modules -- off by default, since these +diff -up Python-2.7.3/setup.py.crypt-module-salt-backport Python-2.7.3/setup.py +--- Python-2.7.3/setup.py.crypt-module-salt-backport 2013-02-19 16:44:20.425334067 -0500 ++++ Python-2.7.3/setup.py 2013-02-19 16:44:20.466334063 -0500 +@@ -693,7 +693,7 @@ class PyBuildExt(build_ext): + libs = ['crypt'] + else: + libs = [] +- exts.append( Extension('crypt', ['cryptmodule.c'], libraries=libs) ) ++ exts.append( Extension('_crypt', ['_cryptmodule.c'], libraries=libs) ) + + # CSV files + exts.append( Extension('_csv', ['_csv.c']) ) diff --git a/SOURCES/00166-fix-fake-repr-in-gdb-hooks.patch b/SOURCES/00166-fix-fake-repr-in-gdb-hooks.patch new file mode 100644 index 00000000..bfd24598 --- /dev/null +++ b/SOURCES/00166-fix-fake-repr-in-gdb-hooks.patch @@ -0,0 +1,125 @@ +diff -up Python-2.7.3/Tools/gdb/libpython.py.fix-fake-repr-in-gdb-hooks Python-2.7.3/Tools/gdb/libpython.py +--- Python-2.7.3/Tools/gdb/libpython.py.fix-fake-repr-in-gdb-hooks 2013-02-19 17:21:33.541181366 -0500 ++++ Python-2.7.3/Tools/gdb/libpython.py 2013-02-19 17:21:42.090180782 -0500 +@@ -105,6 +105,24 @@ class TruncatedStringIO(object): + def getvalue(self): + return self._val + ++class FakeProxy(object): ++ """ ++ Class representing a non-descript PyObject* value in the inferior ++ process for when we don't have a custom scraper, intended to have ++ a sane repr(). ++ """ ++ def __init__(self, tp_name, address): ++ self.tp_name = tp_name ++ self.address = address ++ ++ def __repr__(self): ++ # For the NULL pointer, we have no way of knowing a type, so ++ # special-case it as per ++ # http://bugs.python.org/issue8032#msg100882 ++ if self.address == 0: ++ return '0x0' ++ return '<%s at remote 0x%x>' % (self.tp_name, self.address) ++ + class PyObjectPtr(object): + """ + Class wrapping a gdb.Value that's a either a (PyObject*) within the +@@ -232,28 +250,8 @@ class PyObjectPtr(object): + visiting object graphs with loops). Analogous to Py_ReprEnter and + Py_ReprLeave + ''' +- +- class FakeRepr(object): +- """ +- Class representing a non-descript PyObject* value in the inferior +- process for when we don't have a custom scraper, intended to have +- a sane repr(). +- """ +- +- def __init__(self, tp_name, address): +- self.tp_name = tp_name +- self.address = address +- +- def __repr__(self): +- # For the NULL pointer, we have no way of knowing a type, so +- # special-case it as per +- # http://bugs.python.org/issue8032#msg100882 +- if self.address == 0: +- return '0x0' +- return '<%s at remote 0x%x>' % (self.tp_name, self.address) +- +- return FakeRepr(self.safe_tp_name(), +- long(self._gdbval)) ++ return FakeProxy(self.safe_tp_name(), ++ long(self._gdbval)) + + def write_repr(self, out, visited): + ''' +@@ -384,7 +382,7 @@ def _write_instance_repr(out, visited, n + if not first: + out.write(', ') + first = False +- out.write(pyop_arg.proxyval(visited)) ++ out.write(str(pyop_arg.proxyval(visited))) + out.write('=') + pyop_val.write_repr(out, visited) + out.write(')') +@@ -785,6 +783,8 @@ class PyNoneStructPtr(PyObjectPtr): + def proxyval(self, visited): + return None + ++class CantReadFilename(ValueError): ++ pass + + class PyFrameObjectPtr(PyObjectPtr): + _typename = 'PyFrameObject' +@@ -861,7 +861,10 @@ class PyFrameObjectPtr(PyObjectPtr): + '''Get the path of the current Python source file, as a string''' + if self.is_optimized_out(): + return '(frame information optimized out)' +- return self.co_filename.proxyval(set()) ++ value = self.co_filename.proxyval(set()) ++ if isinstance(value, FakeProxy): ++ raise CantReadFilename('unable to extract filename)') ++ return value + + def current_line_num(self): + '''Get current line number as an integer (1-based) +@@ -907,7 +910,7 @@ class PyFrameObjectPtr(PyObjectPtr): + out.write(', ') + first = False + +- out.write(pyop_name.proxyval(visited)) ++ out.write(str(pyop_name.proxyval(visited))) + out.write('=') + pyop_value.write_repr(out, visited) + +@@ -1252,8 +1255,11 @@ class Frame(object): + if pyop: + sys.stdout.write('#%i %s\n' % (self.get_index(), pyop.get_truncated_repr(MAX_OUTPUT_LEN))) + if not pyop.is_optimized_out(): +- line = pyop.current_line() +- sys.stdout.write(' %s\n' % line.strip()) ++ try: ++ line = pyop.current_line() ++ sys.stdout.write(' %s\n' % line.strip()) ++ except CantReadFilename: ++ sys.stdout.write(' %s\n' % '(unable to read filename)') + else: + sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index()) + else: +@@ -1303,7 +1309,11 @@ class PyList(gdb.Command): + print 'Unable to read information on python frame' + return + +- filename = pyop.filename() ++ try: ++ filename = pyop.filename() ++ except CantReadFilename: ++ print "Unable to extract filename from python frame" ++ return + lineno = pyop.current_line_num() + + if start is None: diff --git a/SOURCES/00167-disable-stack-navigation-tests-when-optimized-in-test_gdb.patch b/SOURCES/00167-disable-stack-navigation-tests-when-optimized-in-test_gdb.patch new file mode 100644 index 00000000..3fa94fb8 --- /dev/null +++ b/SOURCES/00167-disable-stack-navigation-tests-when-optimized-in-test_gdb.patch @@ -0,0 +1,43 @@ +diff -up Python-2.7.3/Lib/test/test_gdb.py.disable-stack-navigation-tests-when-optimized-in-test_gdb Python-2.7.3/Lib/test/test_gdb.py +--- Python-2.7.3/Lib/test/test_gdb.py.disable-stack-navigation-tests-when-optimized-in-test_gdb 2013-02-20 12:27:05.669526425 -0500 ++++ Python-2.7.3/Lib/test/test_gdb.py 2013-02-20 12:27:05.715526422 -0500 +@@ -653,10 +653,10 @@ class PyListTests(DebuggerTests): + ' 3 def foo(a, b, c):\n', + bt) + ++@unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") ++@unittest.skipIf(python_is_optimized(), ++ "Python was compiled with optimizations") + class StackNavigationTests(DebuggerTests): +- @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") +- @unittest.skipIf(python_is_optimized(), +- "Python was compiled with optimizations") + def test_pyup_command(self): + 'Verify that the "py-up" command works' + bt = self.get_stack_trace(script=self.get_sample_script(), +@@ -667,7 +667,6 @@ class StackNavigationTests(DebuggerTests + baz\(a, b, c\) + $''') + +- @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + def test_down_at_bottom(self): + 'Verify handling of "py-down" at the bottom of the stack' + bt = self.get_stack_trace(script=self.get_sample_script(), +@@ -675,7 +674,6 @@ $''') + self.assertEndsWith(bt, + 'Unable to find a newer python frame\n') + +- @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + def test_up_at_top(self): + 'Verify handling of "py-up" at the top of the stack' + bt = self.get_stack_trace(script=self.get_sample_script(), +@@ -683,9 +681,6 @@ $''') + self.assertEndsWith(bt, + 'Unable to find an older python frame\n') + +- @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") +- @unittest.skipIf(python_is_optimized(), +- "Python was compiled with optimizations") + def test_up_then_down(self): + 'Verify "py-up" followed by "py-down"' + bt = self.get_stack_trace(script=self.get_sample_script(), diff --git a/SOURCES/00168-distutils-cflags.patch b/SOURCES/00168-distutils-cflags.patch new file mode 100644 index 00000000..0c4a8df3 --- /dev/null +++ b/SOURCES/00168-distutils-cflags.patch @@ -0,0 +1,12 @@ +diff -up Python-2.6.6/Lib/distutils/sysconfig.py.distutils-cflags Python-2.6.6/Lib/distutils/sysconfig.py +--- Python-2.6.6/Lib/distutils/sysconfig.py.distutils-cflags 2011-08-12 17:18:17.833091153 -0400 ++++ Python-2.6.6/Lib/distutils/sysconfig.py 2011-08-12 17:18:27.449106938 -0400 +@@ -187,7 +187,7 @@ def customize_compiler(compiler): + if 'LDFLAGS' in os.environ: + ldshared = ldshared + ' ' + os.environ['LDFLAGS'] + if 'CFLAGS' in os.environ: +- cflags = opt + ' ' + os.environ['CFLAGS'] ++ cflags = cflags + ' ' + os.environ['CFLAGS'] + ldshared = ldshared + ' ' + os.environ['CFLAGS'] + if 'CPPFLAGS' in os.environ: + cpp = cpp + ' ' + os.environ['CPPFLAGS'] diff --git a/SOURCES/00169-avoid-implicit-usage-of-md5-in-multiprocessing.patch b/SOURCES/00169-avoid-implicit-usage-of-md5-in-multiprocessing.patch new file mode 100644 index 00000000..debf92f1 --- /dev/null +++ b/SOURCES/00169-avoid-implicit-usage-of-md5-in-multiprocessing.patch @@ -0,0 +1,41 @@ +diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py +--- a/Lib/multiprocessing/connection.py ++++ b/Lib/multiprocessing/connection.py +@@ -41,6 +41,10 @@ + # A very generous timeout when it comes to local connections... + CONNECTION_TIMEOUT = 20. + ++# The hmac module implicitly defaults to using MD5. ++# Support using a stronger algorithm for the challenge/response code: ++HMAC_DIGEST_NAME='sha256' ++ + _mmap_counter = itertools.count() + + default_family = 'AF_INET' +@@ -700,12 +704,16 @@ + WELCOME = b'#WELCOME#' + FAILURE = b'#FAILURE#' + ++def get_digestmod_for_hmac(): ++ import hashlib ++ return getattr(hashlib, HMAC_DIGEST_NAME) ++ + def deliver_challenge(connection, authkey): + import hmac + assert isinstance(authkey, bytes) + message = os.urandom(MESSAGE_LENGTH) + connection.send_bytes(CHALLENGE + message) +- digest = hmac.new(authkey, message).digest() ++ digest = hmac.new(authkey, message, get_digestmod_for_hmac()).digest() + response = connection.recv_bytes(256) # reject large message + if response == digest: + connection.send_bytes(WELCOME) +@@ -719,7 +727,7 @@ + message = connection.recv_bytes(256) # reject large message + assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message + message = message[len(CHALLENGE):] +- digest = hmac.new(authkey, message).digest() ++ digest = hmac.new(authkey, message, get_digestmod_for_hmac()).digest() + connection.send_bytes(digest) + response = connection.recv_bytes(256) # reject large message + if response != WELCOME: diff --git a/SOURCES/00170-gc-assertions.patch b/SOURCES/00170-gc-assertions.patch index f4917334..3fb37ff3 100644 --- a/SOURCES/00170-gc-assertions.patch +++ b/SOURCES/00170-gc-assertions.patch @@ -1,92 +1,33 @@ -diff --git a/Include/object.h b/Include/object.h -index 0c88603..e3413e8 100644 ---- a/Include/object.h -+++ b/Include/object.h -@@ -1059,6 +1059,49 @@ PyAPI_FUNC(void) - _PyObject_DebugTypeStats(FILE *out); - #endif /* ifndef Py_LIMITED_API */ - -+/* -+ Define a pair of assertion macros. -+ -+ These work like the regular C assert(), in that they will abort the -+ process with a message on stderr if the given condition fails to hold, -+ but compile away to nothing if NDEBUG is defined. -+ -+ However, before aborting, Python will also try to call _PyObject_Dump() on -+ the given object. This may be of use when investigating bugs in which a -+ particular object is corrupt (e.g. buggy a tp_visit method in an extension -+ module breaking the garbage collector), to help locate the broken objects. -+ -+ The WITH_MSG variant allows you to supply an additional message that Python -+ will attempt to print to stderr, after the object dump. -+*/ -+#ifdef NDEBUG -+/* No debugging: compile away the assertions: */ -+#define PyObject_ASSERT_WITH_MSG(obj, expr, msg) ((void)0) -+#else -+/* With debugging: generate checks: */ -+#define PyObject_ASSERT_WITH_MSG(obj, expr, msg) \ -+ ((expr) \ -+ ? (void)(0) \ -+ : _PyObject_AssertFailed((obj), \ -+ (msg), \ -+ (__STRING(expr)), \ -+ (__FILE__), \ -+ (__LINE__), \ -+ (__PRETTY_FUNCTION__))) -+#endif -+ -+#define PyObject_ASSERT(obj, expr) \ -+ PyObject_ASSERT_WITH_MSG(obj, expr, NULL) -+ -+/* -+ Declare and define the entrypoint even when NDEBUG is defined, to avoid -+ causing compiler/linker errors when building extensions without NDEBUG -+ against a Python built with NDEBUG defined -+*/ -+PyAPI_FUNC(void) _PyObject_AssertFailed(PyObject *, const char *, -+ const char *, const char *, int, -+ const char *); -+ - #ifdef __cplusplus - } - #endif -diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py -index e727499..6efcafb 100644 ---- a/Lib/test/test_gc.py -+++ b/Lib/test/test_gc.py -@@ -1,10 +1,11 @@ +diff -up Python-2.7.3/Lib/test/test_gc.py.gc-assertions Python-2.7.3/Lib/test/test_gc.py +--- Python-2.7.3/Lib/test/test_gc.py.gc-assertions 2013-02-20 16:28:20.890536607 -0500 ++++ Python-2.7.3/Lib/test/test_gc.py 2013-02-20 16:39:52.720489297 -0500 +@@ -1,6 +1,7 @@ import unittest - from test.support import (verbose, refcount_test, run_unittest, - strip_python_stderr, cpython_only, start_threads, -- temp_dir, requires_type_collecting) -+ temp_dir, import_module, requires_type_collecting) - from test.support.script_helper import assert_python_ok, make_script - +-from test.test_support import verbose, run_unittest ++from test.test_support import verbose, run_unittest, import_module import sys +import sysconfig import time import gc import weakref -@@ -50,6 +51,8 @@ class GC_Detector(object): - # gc collects it. +@@ -32,6 +33,8 @@ class GC_Detector(object): self.wr = weakref.ref(C1055820(666), it_happened) -+BUILD_WITH_NDEBUG = ('-DNDEBUG' in sysconfig.get_config_vars()['PY_CFLAGS']) + ++BUILT_WITH_NDEBUG = ('-DNDEBUG' in sysconfig.get_config_vars()['PY_CFLAGS']) + - @with_tp_del - class Uncollectable(object): - """Create a reference cycle with multiple __del__ methods. -@@ -862,6 +865,50 @@ class GCCallbackTests(unittest.TestCase): - self.assertEqual(len(gc.garbage), 0) + ### Tests + ############################################################################### +@@ -476,6 +479,49 @@ class GCTests(unittest.TestCase): + # would be damaged, with an empty __dict__. + self.assertEqual(x, None) -+ @unittest.skipIf(BUILD_WITH_NDEBUG, ++ @unittest.skipIf(BUILT_WITH_NDEBUG, + 'built with -NDEBUG') + def test_refcount_errors(self): -+ self.preclean() + # Verify the "handling" of objects with broken refcounts ++ + import_module("ctypes") #skip if not supported + + import subprocess @@ -112,59 +53,131 @@ index e727499..6efcafb 100644 + p.stdout.close() + p.stderr.close() + # Verify that stderr has a useful error message: -+ self.assertRegex(stderr, -+ b'Modules/gcmodule.c:[0-9]+: visit_decref: Assertion "\(\(gc\)->gc.gc_refs >> \(1\)\) != 0" failed.') -+ self.assertRegex(stderr, ++ self.assertRegexpMatches(stderr, ++ b'Modules/gcmodule.c:[0-9]+: visit_decref: Assertion "gc->gc.gc_refs != 0" failed.') ++ self.assertRegexpMatches(stderr, + b'refcount was too small') -+ self.assertRegex(stderr, ++ self.assertRegexpMatches(stderr, + b'object : \[\]') -+ self.assertRegex(stderr, ++ self.assertRegexpMatches(stderr, + b'type : list') -+ self.assertRegex(stderr, ++ self.assertRegexpMatches(stderr, + b'refcount: 1') -+ self.assertRegex(stderr, ++ self.assertRegexpMatches(stderr, + b'address : 0x[0-9a-f]+') -+ + class GCTogglingTests(unittest.TestCase): def setUp(self): gc.enable() -diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c -index 0c6f444..87edd5a 100644 ---- a/Modules/gcmodule.c -+++ b/Modules/gcmodule.c -@@ -341,7 +341,8 @@ update_refs(PyGC_Head *containers) +diff -up Python-2.7.3/Modules/gcmodule.c.gc-assertions Python-2.7.3/Modules/gcmodule.c +--- Python-2.7.3/Modules/gcmodule.c.gc-assertions 2012-04-09 19:07:34.000000000 -0400 ++++ Python-2.7.3/Modules/gcmodule.c 2013-02-20 16:28:21.029536600 -0500 +@@ -21,6 +21,73 @@ + #include "Python.h" + #include "frameobject.h" /* for PyFrame_ClearFreeList */ + ++/* ++ Define a pair of assertion macros. ++ ++ These work like the regular C assert(), in that they will abort the ++ process with a message on stderr if the given condition fails to hold, ++ but compile away to nothing if NDEBUG is defined. ++ ++ However, before aborting, Python will also try to call _PyObject_Dump() on ++ the given object. This may be of use when investigating bugs in which a ++ particular object is corrupt (e.g. buggy a tp_visit method in an extension ++ module breaking the garbage collector), to help locate the broken objects. ++ ++ The WITH_MSG variant allows you to supply an additional message that Python ++ will attempt to print to stderr, after the object dump. ++*/ ++#ifdef NDEBUG ++/* No debugging: compile away the assertions: */ ++#define PyObject_ASSERT_WITH_MSG(obj, expr, msg) ((void)0) ++#else ++/* With debugging: generate checks: */ ++#define PyObject_ASSERT_WITH_MSG(obj, expr, msg) \ ++ ((expr) \ ++ ? (void)(0) \ ++ : _PyObject_AssertFailed((obj), \ ++ (msg), \ ++ (__STRING(expr)), \ ++ (__FILE__), \ ++ (__LINE__), \ ++ (__PRETTY_FUNCTION__))) ++#endif ++ ++#define PyObject_ASSERT(obj, expr) \ ++ PyObject_ASSERT_WITH_MSG(obj, expr, NULL) ++ ++static void _PyObject_AssertFailed(PyObject *, const char *, ++ const char *, const char *, int, ++ const char *); ++ ++static void ++_PyObject_AssertFailed(PyObject *obj, const char *msg, const char *expr, ++ const char *file, int line, const char *function) ++{ ++ fprintf(stderr, ++ "%s:%d: %s: Assertion \"%s\" failed.\n", ++ file, line, function, expr); ++ if (msg) { ++ fprintf(stderr, "%s\n", msg); ++ } ++ ++ fflush(stderr); ++ ++ if (obj) { ++ /* This might succeed or fail, but we're about to abort, so at least ++ try to provide any extra info we can: */ ++ _PyObject_Dump(obj); ++ } ++ else { ++ fprintf(stderr, "NULL object\n"); ++ } ++ ++ fflush(stdout); ++ fflush(stderr); ++ ++ /* Terminate the process: */ ++ abort(); ++} ++ + /* Get an object's GC head */ + #define AS_GC(o) ((PyGC_Head *)(o)-1) + +@@ -288,7 +355,8 @@ update_refs(PyGC_Head *containers) { PyGC_Head *gc = containers->gc.gc_next; for (; gc != containers; gc = gc->gc.gc_next) { -- assert(_PyGCHead_REFS(gc) == GC_REACHABLE); +- assert(gc->gc.gc_refs == GC_REACHABLE); + PyObject_ASSERT(FROM_GC(gc), -+ _PyGCHead_REFS(gc) == GC_REACHABLE); - _PyGCHead_SET_REFS(gc, Py_REFCNT(FROM_GC(gc))); ++ gc->gc.gc_refs == GC_REACHABLE); + gc->gc.gc_refs = Py_REFCNT(FROM_GC(gc)); /* Python's cyclic gc should never see an incoming refcount * of 0: if something decref'ed to 0, it should have been -@@ -361,7 +362,8 @@ update_refs(PyGC_Head *containers) +@@ -308,7 +376,8 @@ update_refs(PyGC_Head *containers) * so serious that maybe this should be a release-build * check instead of an assert? */ -- assert(_PyGCHead_REFS(gc) != 0); +- assert(gc->gc.gc_refs != 0); + PyObject_ASSERT(FROM_GC(gc), -+ _PyGCHead_REFS(gc) != 0); ++ gc->gc.gc_refs != 0); } } -@@ -376,7 +378,9 @@ visit_decref(PyObject *op, void *data) +@@ -323,7 +392,9 @@ visit_decref(PyObject *op, void *data) * generation being collected, which can be recognized * because only they have positive gc_refs. */ -- assert(_PyGCHead_REFS(gc) != 0); /* else refcount was too small */ +- assert(gc->gc.gc_refs != 0); /* else refcount was too small */ + PyObject_ASSERT_WITH_MSG(FROM_GC(gc), -+ _PyGCHead_REFS(gc) != 0, -+ "refcount was too small"); /* else refcount was too small */ - if (_PyGCHead_REFS(gc) > 0) - _PyGCHead_DECREF(gc); ++ gc->gc.gc_refs != 0, ++ "refcount was too small"); + if (gc->gc.gc_refs > 0) + gc->gc.gc_refs--; } -@@ -436,9 +440,10 @@ visit_reachable(PyObject *op, PyGC_Head *reachable) +@@ -383,9 +454,10 @@ visit_reachable(PyObject *op, PyGC_Head * If gc_refs == GC_UNTRACKED, it must be ignored. */ else { @@ -178,25 +191,26 @@ index 0c6f444..87edd5a 100644 } } return 0; -@@ -480,7 +485,7 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) +@@ -427,7 +499,7 @@ move_unreachable(PyGC_Head *young, PyGC_ */ PyObject *op = FROM_GC(gc); traverseproc traverse = Py_TYPE(op)->tp_traverse; -- assert(_PyGCHead_REFS(gc) > 0); -+ PyObject_ASSERT(op, _PyGCHead_REFS(gc) > 0); - _PyGCHead_SET_REFS(gc, GC_REACHABLE); +- assert(gc->gc.gc_refs > 0); ++ PyObject_ASSERT(op, gc->gc.gc_refs > 0); + gc->gc.gc_refs = GC_REACHABLE; (void) traverse(op, (visitproc)visit_reachable, -@@ -543,7 +548,7 @@ move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) +@@ -494,7 +566,8 @@ move_finalizers(PyGC_Head *unreachable, for (gc = unreachable->gc.gc_next; gc != unreachable; gc = next) { PyObject *op = FROM_GC(gc); - assert(IS_TENTATIVELY_UNREACHABLE(op)); + PyObject_ASSERT(op, IS_TENTATIVELY_UNREACHABLE(op)); ++ next = gc->gc.gc_next; - if (has_legacy_finalizer(op)) { -@@ -619,7 +624,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) + if (has_finalizer(op)) { +@@ -570,7 +643,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyWeakReference **wrlist; op = FROM_GC(gc); @@ -205,7 +219,7 @@ index 0c6f444..87edd5a 100644 next = gc->gc.gc_next; if (! PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) -@@ -640,9 +645,9 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) +@@ -591,9 +664,9 @@ handle_weakrefs(PyGC_Head *unreachable, * the callback pointer intact. Obscure: it also * changes *wrlist. */ @@ -217,7 +231,7 @@ index 0c6f444..87edd5a 100644 if (wr->wr_callback == NULL) continue; /* no callback */ -@@ -676,7 +681,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) +@@ -627,7 +700,7 @@ handle_weakrefs(PyGC_Head *unreachable, */ if (IS_TENTATIVELY_UNREACHABLE(wr)) continue; @@ -226,7 +240,7 @@ index 0c6f444..87edd5a 100644 /* Create a new reference so that wr can't go away * before we can process it again. -@@ -685,7 +690,8 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) +@@ -636,7 +709,8 @@ handle_weakrefs(PyGC_Head *unreachable, /* Move wr to wrcb_to_call, for the next pass. */ wrasgc = AS_GC(wr); @@ -236,7 +250,7 @@ index 0c6f444..87edd5a 100644 next isn't, so they can't be the same */ gc_list_move(wrasgc, &wrcb_to_call); -@@ -701,11 +707,11 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) +@@ -652,11 +726,11 @@ handle_weakrefs(PyGC_Head *unreachable, gc = wrcb_to_call.gc.gc_next; op = FROM_GC(gc); @@ -251,60 +265,12 @@ index 0c6f444..87edd5a 100644 /* copy-paste of weakrefobject.c's handle_callback() */ temp = PyObject_CallFunctionObjArgs(callback, wr, NULL); -@@ -822,12 +828,14 @@ check_garbage(PyGC_Head *collectable) - for (gc = collectable->gc.gc_next; gc != collectable; - gc = gc->gc.gc_next) { - _PyGCHead_SET_REFS(gc, Py_REFCNT(FROM_GC(gc))); -- assert(_PyGCHead_REFS(gc) != 0); -+ PyObject_ASSERT(FROM_GC(gc), -+ _PyGCHead_REFS(gc) != 0); - } - subtract_refs(collectable); - for (gc = collectable->gc.gc_next; gc != collectable; - gc = gc->gc.gc_next) { -- assert(_PyGCHead_REFS(gc) >= 0); -+ PyObject_ASSERT(FROM_GC(gc), -+ _PyGCHead_REFS(gc) >= 0); - if (_PyGCHead_REFS(gc) != 0) - return -1; - } -diff --git a/Objects/object.c b/Objects/object.c -index 559794f..a47d47f 100644 ---- a/Objects/object.c -+++ b/Objects/object.c -@@ -2022,6 +2022,35 @@ _PyTrash_thread_destroy_chain(void) - } - } +@@ -759,7 +833,7 @@ delete_garbage(PyGC_Head *collectable, P + PyGC_Head *gc = collectable->gc.gc_next; + PyObject *op = FROM_GC(gc); -+PyAPI_FUNC(void) -+_PyObject_AssertFailed(PyObject *obj, const char *msg, const char *expr, -+ const char *file, int line, const char *function) -+{ -+ fprintf(stderr, -+ "%s:%d: %s: Assertion \"%s\" failed.\n", -+ file, line, function, expr); -+ if (msg) { -+ fprintf(stderr, "%s\n", msg); -+ } -+ -+ fflush(stderr); -+ -+ if (obj) { -+ /* This might succeed or fail, but we're about to abort, so at least -+ try to provide any extra info we can: */ -+ _PyObject_Dump(obj); -+ } -+ else { -+ fprintf(stderr, "NULL object\n"); -+ } -+ -+ fflush(stdout); -+ fflush(stderr); -+ -+ /* Terminate the process: */ -+ abort(); -+} -+ - #ifndef Py_TRACE_REFS - /* For Py_LIMITED_API, we need an out-of-line version of _Py_Dealloc. - Define this here, so we can undefine the macro. */ +- assert(IS_TENTATIVELY_UNREACHABLE(op)); ++ PyObject_ASSERT(op, IS_TENTATIVELY_UNREACHABLE(op)); + if (debug & DEBUG_SAVEALL) { + PyList_Append(garbage, op); + } diff --git a/SOURCES/00173-workaround-ENOPROTOOPT-in-bind_port.patch b/SOURCES/00173-workaround-ENOPROTOOPT-in-bind_port.patch new file mode 100644 index 00000000..eb346105 --- /dev/null +++ b/SOURCES/00173-workaround-ENOPROTOOPT-in-bind_port.patch @@ -0,0 +1,13 @@ +diff -up Python-2.7.3/Lib/test/test_support.py.rhbz913732 Python-2.7.3/Lib/test/test_support.py +--- Python-2.7.3/Lib/test/test_support.py.rhbz913732 2013-03-04 16:11:53.757315921 -0500 ++++ Python-2.7.3/Lib/test/test_support.py 2013-03-04 16:12:11.331314722 -0500 +@@ -304,7 +304,8 @@ def bind_port(sock, host=HOST): + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1: + raise TestFailed("tests should never set the SO_REUSEADDR " \ + "socket option on TCP/IP sockets!") +- if hasattr(socket, 'SO_REUSEPORT'): ++ if hasattr(socket, 'SO_REUSEPORT') \ ++ and 'WITHIN_PYTHON_RPM_BUILD' not in os.environ: # rhbz#913732 + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1: + raise TestFailed("tests should never set the SO_REUSEPORT " \ + "socket option on TCP/IP sockets!") diff --git a/SOURCES/00174-fix-for-usr-move.patch b/SOURCES/00174-fix-for-usr-move.patch new file mode 100644 index 00000000..b48dc5cf --- /dev/null +++ b/SOURCES/00174-fix-for-usr-move.patch @@ -0,0 +1,28 @@ +diff -up Python-2.7.3/Modules/getpath.c.fix-for-usr-move Python-2.7.3/Modules/getpath.c +--- Python-2.7.3/Modules/getpath.c.fix-for-usr-move 2013-03-06 14:25:32.801828698 -0500 ++++ Python-2.7.3/Modules/getpath.c 2013-03-06 15:59:30.872443168 -0500 +@@ -510,6 +510,24 @@ calculate_path(void) + MAXPATHLEN bytes long. + */ + ++ /* ++ Workaround for rhbz#817554, where an empty argv0_path erroneously ++ locates "prefix" as "/lib[64]/python2.7" due to it finding ++ "/lib[64]/python2.7/os.py" via the /lib -> /usr/lib symlink for ++ https://fedoraproject.org/wiki/Features/UsrMove ++ */ ++ if (argv0_path[0] == '\0' && 0 == strcmp(prog, "cmpi_swig")) { ++ /* ++ We have an empty argv0_path, presumably because prog aka ++ Py_GetProgramName() was not found on $PATH. ++ ++ Set argv0_path to "/usr/" so that search_for_prefix() and ++ search_for_exec_prefix() don't erroneously pick up ++ on /lib/ via the UsrMove symlink: ++ */ ++ strcpy(argv0_path, "/usr/"); ++ } ++ + if (!(pfound = search_for_prefix(argv0_path, home))) { + if (!Py_FrozenFlag) + fprintf(stderr, diff --git a/SOURCES/00180-python-add-support-for-ppc64p7.patch b/SOURCES/00180-python-add-support-for-ppc64p7.patch index 054f9f39..022944a5 100644 --- a/SOURCES/00180-python-add-support-for-ppc64p7.patch +++ b/SOURCES/00180-python-add-support-for-ppc64p7.patch @@ -1,13 +1,12 @@ -diff --git a/config.sub b/config.sub -index 40ea5df..932128b 100755 ---- a/config.sub -+++ b/config.sub -@@ -1045,7 +1045,7 @@ case $basic_machine in +diff -r de35eae9048a config.sub +--- a/config.sub Wed Apr 24 23:33:20 2013 +0200 ++++ b/config.sub Thu Apr 25 08:51:00 2013 +0200 +@@ -1008,7 +1008,7 @@ ;; ppc64) basic_machine=powerpc64-unknown ;; - ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` + ppc64-* | ppc64p7-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` ;; - ppc64le | powerpc64little) + ppc64le | powerpc64little | ppc64-le | powerpc64-little) basic_machine=powerpc64le-unknown diff --git a/SOURCES/00181-allow-arbitrary-timeout-in-condition-wait.patch b/SOURCES/00181-allow-arbitrary-timeout-in-condition-wait.patch new file mode 100644 index 00000000..665965d0 --- /dev/null +++ b/SOURCES/00181-allow-arbitrary-timeout-in-condition-wait.patch @@ -0,0 +1,70 @@ +diff --git a/Lib/threading.py b/Lib/threading.py +index cb49c4a..c9795a5 100644 +--- a/Lib/threading.py ++++ b/Lib/threading.py +@@ -305,7 +305,7 @@ class _Condition(_Verbose): + else: + return True + +- def wait(self, timeout=None): ++ def wait(self, timeout=None, balancing=True): + """Wait until notified or until a timeout occurs. + + If the calling thread has not acquired the lock when this method is +@@ -354,7 +354,10 @@ class _Condition(_Verbose): + remaining = endtime - _time() + if remaining <= 0: + break +- delay = min(delay * 2, remaining, .05) ++ if balancing: ++ delay = min(delay * 2, remaining, 0.05) ++ else: ++ delay = remaining + _sleep(delay) + if not gotit: + if __debug__: +@@ -599,7 +602,7 @@ class _Event(_Verbose): + finally: + self.__cond.release() + +- def wait(self, timeout=None): ++ def wait(self, timeout=None, balancing=True): + """Block until the internal flag is true. + + If the internal flag is true on entry, return immediately. Otherwise, +@@ -617,7 +620,7 @@ class _Event(_Verbose): + self.__cond.acquire() + try: + if not self.__flag: +- self.__cond.wait(timeout) ++ self.__cond.wait(timeout, balancing) + return self.__flag + finally: + self.__cond.release() +@@ -908,7 +911,7 @@ class Thread(_Verbose): + if 'dummy_threading' not in _sys.modules: + raise + +- def join(self, timeout=None): ++ def join(self, timeout=None, balancing=True): + """Wait until the thread terminates. + + This blocks the calling thread until the thread whose join() method is +@@ -957,7 +960,7 @@ class Thread(_Verbose): + if __debug__: + self._note("%s.join(): timed out", self) + break +- self.__block.wait(delay) ++ self.__block.wait(delay, balancing) + else: + if __debug__: + self._note("%s.join(): thread stopped", self) +@@ -1143,7 +1146,7 @@ class _DummyThread(Thread): + def _set_daemon(self): + return True + +- def join(self, timeout=None): ++ def join(self, timeout=None, balancing=True): + assert False, "cannot join a dummy thread" + + diff --git a/SOURCES/00184-ctypes-should-build-with-libffi-multilib-wrapper.patch b/SOURCES/00184-ctypes-should-build-with-libffi-multilib-wrapper.patch new file mode 100644 index 00000000..82309863 --- /dev/null +++ b/SOURCES/00184-ctypes-should-build-with-libffi-multilib-wrapper.patch @@ -0,0 +1,13 @@ +diff -up Python-2.7.5/setup.py.orig Python-2.7.5/setup.py +--- Python-2.7.5/setup.py.orig 2013-07-17 15:20:12.086820082 +0200 ++++ Python-2.7.5/setup.py 2013-07-17 15:21:28.490023903 +0200 +@@ -2050,7 +2050,8 @@ class PyBuildExt(build_ext): + if not line: + ffi_inc = None + break +- if line.startswith('#define LIBFFI_H'): ++ if line.startswith('#define LIBFFI_H') or \ ++ line.startswith('#define ffi_wrapper_h'): + break + ffi_lib = None + if ffi_inc is not None: diff --git a/SOURCES/00185-urllib2-honors-noproxy-for-ftp.patch b/SOURCES/00185-urllib2-honors-noproxy-for-ftp.patch new file mode 100644 index 00000000..b26c4d49 --- /dev/null +++ b/SOURCES/00185-urllib2-honors-noproxy-for-ftp.patch @@ -0,0 +1,12 @@ +diff -up Python-2.7.5/Lib/urllib2.py.orig Python-2.7.5/Lib/urllib2.py +--- Python-2.7.5/Lib/urllib2.py.orig 2013-07-17 12:22:58.595525622 +0200 ++++ Python-2.7.5/Lib/urllib2.py 2013-07-17 12:19:59.875898030 +0200 +@@ -728,6 +728,8 @@ class ProxyHandler(BaseHandler): + if proxy_type is None: + proxy_type = orig_type + ++ req.get_host() ++ + if req.host and proxy_bypass(req.host): + return None + diff --git a/SOURCES/00186-memory-leak-marshalc.patch b/SOURCES/00186-memory-leak-marshalc.patch new file mode 100644 index 00000000..19fb175f --- /dev/null +++ b/SOURCES/00186-memory-leak-marshalc.patch @@ -0,0 +1,57 @@ +--- Python-2.7.5/Python/marshal.c 2013-05-12 05:32:53.000000000 +0200 ++++ /home/rkuska/hg/cpython/Python/marshal.c 2013-07-18 10:33:26.392486235 +0200 +@@ -88,7 +88,7 @@ + } + + static void +-w_string(char *s, Py_ssize_t n, WFILE *p) ++w_string(const char *s, Py_ssize_t n, WFILE *p) + { + if (p->fp != NULL) { + fwrite(s, 1, n, p->fp); +@@ -141,6 +141,13 @@ + # define W_SIZE w_long + #endif + ++static void ++w_pstring(const char *s, Py_ssize_t n, WFILE *p) ++{ ++ W_SIZE(n, p); ++ w_string(s, n, p); ++} ++ + /* We assume that Python longs are stored internally in base some power of + 2**15; for the sake of portability we'll always read and write them in base + exactly 2**15. */ +@@ -338,9 +345,7 @@ + else { + w_byte(TYPE_STRING, p); + } +- n = PyString_GET_SIZE(v); +- W_SIZE(n, p); +- w_string(PyString_AS_STRING(v), n, p); ++ w_pstring(PyBytes_AS_STRING(v), PyString_GET_SIZE(v), p); + } + #ifdef Py_USING_UNICODE + else if (PyUnicode_CheckExact(v)) { +@@ -352,9 +357,7 @@ + return; + } + w_byte(TYPE_UNICODE, p); +- n = PyString_GET_SIZE(utf8); +- W_SIZE(n, p); +- w_string(PyString_AS_STRING(utf8), n, p); ++ w_pstring(PyString_AS_STRING(utf8), PyString_GET_SIZE(utf8), p); + Py_DECREF(utf8); + } + #endif +@@ -441,8 +444,7 @@ + PyBufferProcs *pb = v->ob_type->tp_as_buffer; + w_byte(TYPE_STRING, p); + n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s); +- W_SIZE(n, p); +- w_string(s, n, p); ++ w_pstring(s, n, p); + } + else { + w_byte(TYPE_UNKNOWN, p); diff --git a/SOURCES/00187-add-RPATH-to-pyexpat.patch b/SOURCES/00187-add-RPATH-to-pyexpat.patch new file mode 100644 index 00000000..0ac52278 --- /dev/null +++ b/SOURCES/00187-add-RPATH-to-pyexpat.patch @@ -0,0 +1,25 @@ +diff -r e8b8279ca118 setup.py +--- a/setup.py Sun Jul 21 21:57:52 2013 -0400 ++++ b/setup.py Tue Aug 20 09:45:31 2013 +0200 +@@ -1480,12 +1480,21 @@ + 'expat/xmltok_impl.h' + ] + ++ # Add an explicit RPATH to pyexpat.so pointing at the directory ++ # containing the system expat (which has the extra XML_SetHashSalt ++ # symbol), to avoid an ImportError with a link error if there's an ++ # LD_LIBRARY_PATH containing a "vanilla" build of expat (without the ++ # symbol) (rhbz#833271): ++ EXPAT_RPATH = '/usr/lib64' if sys.maxint == 0x7fffffffffffffff else '/usr/lib' ++ ++ + exts.append(Extension('pyexpat', + define_macros = define_macros, + include_dirs = expat_inc, + libraries = expat_lib, + sources = ['pyexpat.c'] + expat_sources, + depends = expat_depends, ++ extra_link_args = ['-Wl,-rpath,%s' % EXPAT_RPATH] + )) + + # Fredrik Lundh's cElementTree module. Note that this also diff --git a/SOURCES/00188-CVE-2013-4238-hostname-check-bypass-in-SSL-module.patch b/SOURCES/00188-CVE-2013-4238-hostname-check-bypass-in-SSL-module.patch new file mode 100644 index 00000000..e215589d --- /dev/null +++ b/SOURCES/00188-CVE-2013-4238-hostname-check-bypass-in-SSL-module.patch @@ -0,0 +1,247 @@ +diff -r 9ddc63c039ba Lib/test/nullbytecert.pem +--- /dev/null Thu Jan 01 00:00:00 1970 +0000 ++++ b/Lib/test/nullbytecert.pem Sun Aug 11 18:13:17 2013 +0200 +@@ -0,0 +1,90 @@ ++Certificate: ++ Data: ++ Version: 3 (0x2) ++ Serial Number: 0 (0x0) ++ Signature Algorithm: sha1WithRSAEncryption ++ Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org ++ Validity ++ Not Before: Aug 7 13:11:52 2013 GMT ++ Not After : Aug 7 13:12:52 2013 GMT ++ Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org ++ Subject Public Key Info: ++ Public Key Algorithm: rsaEncryption ++ Public-Key: (2048 bit) ++ Modulus: ++ 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: ++ 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: ++ 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: ++ 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: ++ 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: ++ 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: ++ a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: ++ 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: ++ ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: ++ 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: ++ 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: ++ 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: ++ f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: ++ f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: ++ ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: ++ d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: ++ 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: ++ 2f:85 ++ Exponent: 65537 (0x10001) ++ X509v3 extensions: ++ X509v3 Basic Constraints: critical ++ CA:FALSE ++ X509v3 Subject Key Identifier: ++ 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C ++ X509v3 Key Usage: ++ Digital Signature, Non Repudiation, Key Encipherment ++ X509v3 Subject Alternative Name: ++ ************************************************************* ++ WARNING: The values for DNS, email and URI are WRONG. OpenSSL ++ doesn't print the text after a NULL byte. ++ ************************************************************* ++ DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 ++ Signature Algorithm: sha1WithRSAEncryption ++ ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: ++ a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: ++ 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: ++ 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: ++ 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: ++ de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: ++ 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: ++ 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: ++ d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: ++ 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: ++ 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: ++ 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: ++ 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: ++ 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: ++ c1:ca:a9:94 ++-----BEGIN CERTIFICATE----- ++MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx ++DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ ++eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg ++RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y ++ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw ++NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI ++DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv ++ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt ++ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq ++hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB ++BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j ++pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P ++vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv ++KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA ++oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL ++08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV ++HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E ++BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu ++Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 ++bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA ++AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 ++i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j ++HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk ++kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx ++VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW ++RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= ++-----END CERTIFICATE----- +diff -r 9ddc63c039ba Lib/test/test_ssl.py +--- a/Lib/test/test_ssl.py Sun Aug 11 13:04:50 2013 +0300 ++++ b/Lib/test/test_ssl.py Sun Aug 11 18:13:17 2013 +0200 +@@ -25,6 +25,7 @@ + HOST = test_support.HOST + CERTFILE = None + SVN_PYTHON_ORG_ROOT_CERT = None ++NULLBYTECERT = None + + def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) +@@ -123,6 +124,27 @@ + ('DNS', 'projects.forum.nokia.com')) + ) + ++ def test_parse_cert_CVE_2013_4073(self): ++ p = ssl._ssl._test_decode_cert(NULLBYTECERT) ++ if test_support.verbose: ++ sys.stdout.write("\n" + pprint.pformat(p) + "\n") ++ subject = ((('countryName', 'US'),), ++ (('stateOrProvinceName', 'Oregon'),), ++ (('localityName', 'Beaverton'),), ++ (('organizationName', 'Python Software Foundation'),), ++ (('organizationalUnitName', 'Python Core Development'),), ++ (('commonName', 'null.python.org\x00example.org'),), ++ (('emailAddress', 'python-dev@python.org'),)) ++ self.assertEqual(p['subject'], subject) ++ self.assertEqual(p['issuer'], subject) ++ self.assertEqual(p['subjectAltName'], ++ (('DNS', 'altnull.python.org\x00example.com'), ++ ('email', 'null@python.org\x00user@example.org'), ++ ('URI', 'http://null.python.org\x00http://example.org'), ++ ('IP Address', '192.0.2.1'), ++ ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) ++ ) ++ + def test_DER_to_PEM(self): + with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f: + pem = f.read() +@@ -1360,7 +1382,7 @@ + + + def test_main(verbose=False): +- global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, NOKIACERT ++ global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, NOKIACERT, NULLBYTECERT + CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, + "keycert.pem") + SVN_PYTHON_ORG_ROOT_CERT = os.path.join( +@@ -1368,10 +1390,13 @@ + "https_svn_python_org_root.pem") + NOKIACERT = os.path.join(os.path.dirname(__file__) or os.curdir, + "nokia.pem") ++ NULLBYTECERT = os.path.join(os.path.dirname(__file__) or os.curdir, ++ "nullbytecert.pem") + + if (not os.path.exists(CERTFILE) or + not os.path.exists(SVN_PYTHON_ORG_ROOT_CERT) or +- not os.path.exists(NOKIACERT)): ++ not os.path.exists(NOKIACERT) or ++ not os.path.exists(NULLBYTECERT)): + raise test_support.TestFailed("Can't read certificate files!") + + tests = [BasicTests, BasicSocketTests] +diff -r 9ddc63c039ba Modules/_ssl.c +--- a/Modules/_ssl.c Sun Aug 11 13:04:50 2013 +0300 ++++ b/Modules/_ssl.c Sun Aug 11 18:13:17 2013 +0200 +@@ -741,8 +741,13 @@ + + /* get a rendering of each name in the set of names */ + ++ int gntype; ++ ASN1_STRING *as = NULL; ++ + name = sk_GENERAL_NAME_value(names, j); +- if (name->type == GEN_DIRNAME) { ++ gntype = name-> type; ++ switch (gntype) { ++ case GEN_DIRNAME: + + /* we special-case DirName as a tuple of tuples of attributes */ + +@@ -764,11 +769,61 @@ + goto fail; + } + PyTuple_SET_ITEM(t, 1, v); ++ break; + +- } else { ++ case GEN_EMAIL: ++ case GEN_DNS: ++ case GEN_URI: ++ /* GENERAL_NAME_print() doesn't handle NUL bytes in ASN1_string ++ correctly. */ ++ t = PyTuple_New(2); ++ if (t == NULL) ++ goto fail; ++ switch (gntype) { ++ case GEN_EMAIL: ++ v = PyUnicode_FromString("email"); ++ as = name->d.rfc822Name; ++ break; ++ case GEN_DNS: ++ v = PyUnicode_FromString("DNS"); ++ as = name->d.dNSName; ++ break; ++ case GEN_URI: ++ v = PyUnicode_FromString("URI"); ++ as = name->d.uniformResourceIdentifier; ++ break; ++ } ++ if (v == NULL) { ++ Py_DECREF(t); ++ goto fail; ++ } ++ PyTuple_SET_ITEM(t, 0, v); ++ v = PyString_FromStringAndSize((char *)ASN1_STRING_data(as), ++ ASN1_STRING_length(as)); ++ if (v == NULL) { ++ Py_DECREF(t); ++ goto fail; ++ } ++ PyTuple_SET_ITEM(t, 1, v); ++ break; + ++ default: + /* for everything else, we use the OpenSSL print form */ +- ++ switch (gntype) { ++ /* check for new general name type */ ++ case GEN_OTHERNAME: ++ case GEN_X400: ++ case GEN_EDIPARTY: ++ case GEN_IPADD: ++ case GEN_RID: ++ break; ++ default: ++ if (PyErr_Warn(PyExc_RuntimeWarning, ++ "Unknown general name type") == -1) { ++ goto fail; ++ } ++ break; ++ } + (void) BIO_reset(biobuf); + GENERAL_NAME_print(biobuf, name); + len = BIO_gets(biobuf, buf, sizeof(buf)-1); +@@ -794,6 +849,7 @@ + goto fail; + } + PyTuple_SET_ITEM(t, 1, v); ++ break; + } + + /* and add that rendering to the list */ diff --git a/SOURCES/00189-gdb-py-bt-dont-raise-exception-from-eval.patch b/SOURCES/00189-gdb-py-bt-dont-raise-exception-from-eval.patch new file mode 100644 index 00000000..4e828593 --- /dev/null +++ b/SOURCES/00189-gdb-py-bt-dont-raise-exception-from-eval.patch @@ -0,0 +1,11 @@ +--- Python-2.7.5-orig/Tools/gdb/libpython.py 2013-05-12 03:32:54.000000000 +0000 ++++ Python-2.7.5-orig/Tools/gdb/libpython.py 2013-09-15 09:56:25.494000000 +0000 +@@ -887,6 +887,8 @@ + newline character''' + if self.is_optimized_out(): + return '(frame information optimized out)' ++ if self.filename() == '': ++ return '(in an eval block)' + with open(self.filename(), 'r') as f: + all_lines = f.readlines() + # Convert from 1-based current_line_num to 0-based list offset: diff --git a/SOURCES/00190-gdb-fix-ppc64-failures.patch b/SOURCES/00190-gdb-fix-ppc64-failures.patch new file mode 100644 index 00000000..9bb723fa --- /dev/null +++ b/SOURCES/00190-gdb-fix-ppc64-failures.patch @@ -0,0 +1,207 @@ +--- Tools/gdb/libpython.py.orig 2013-10-09 10:54:59.894701668 +0200 ++++ Tools/gdb/libpython.py 2013-10-09 11:09:30.278703290 +0200 +@@ -1194,39 +1194,113 @@ + iter_frame = iter_frame.newer() + return index + ++ # We divide frames into: ++ # - "python frames": ++ # - "bytecode frames" i.e. PyEval_EvalFrameEx ++ # - "other python frames": things that are of interest from a python ++ # POV, but aren't bytecode (e.g. GC, GIL) ++ # - everything else ++ ++ def is_python_frame(self): ++ '''Is this a PyEval_EvalFrameEx frame, or some other important ++ frame? (see is_other_python_frame for what "important" means in this ++ context)''' ++ if self.is_evalframeex(): ++ return True ++ if self.is_other_python_frame(): ++ return True ++ return False ++ + def is_evalframeex(self): +- '''Is this a PyEval_EvalFrameEx frame?''' +- if self._gdbframe.name() == 'PyEval_EvalFrameEx': +- ''' +- I believe we also need to filter on the inline +- struct frame_id.inline_depth, only regarding frames with +- an inline depth of 0 as actually being this function +- +- So we reject those with type gdb.INLINE_FRAME +- ''' +- if self._gdbframe.type() == gdb.NORMAL_FRAME: +- # We have a PyEval_EvalFrameEx frame: +- return True ++ if self._gdbframe.function(): ++ if self._gdbframe.function().name == 'PyEval_EvalFrameEx': ++ ''' ++ I believe we also need to filter on the inline ++ struct frame_id.inline_depth, only regarding frames with ++ an inline depth of 0 as actually being this function ++ ++ So we reject those with type gdb.INLINE_FRAME ++ ''' ++ if self._gdbframe.type() == gdb.NORMAL_FRAME: ++ # We have a PyEval_EvalFrameEx frame: ++ return True ++ ++ return False ++ ++ def is_other_python_frame(self): ++ '''Is this frame worth displaying in python backtraces? ++ Examples: ++ - waiting on the GIL ++ - garbage-collecting ++ - within a CFunction ++ If it is, return a descriptive string ++ For other frames, return False ++ ''' ++ if self.is_waiting_for_gil(): ++ return 'Waiting for a lock (e.g. GIL)' ++ elif self.is_gc_collect(): ++ return 'Garbage-collecting' ++ else: ++ # Detect invocations of PyCFunction instances: ++ if self._gdbframe.name() == 'PyCFunction_Call': ++ try: ++ func = self._gdbframe.read_var('func') ++ # Use the prettyprinter for the func: ++ return str(func) ++ except RuntimeError: ++ return 'PyCFunction invocation (unable to read "func")' ++ older = self.older() ++ if older and older._gdbframe.name() == 'call_function': ++ # Within that frame: ++ # 'call_function' contains, amongst other things, a ++ # hand-inlined copy of PyCFunction_Call. ++ # "func" is the local containing the PyObject* of the ++ # callable instance ++ # Report it, but only if it's a PyCFunction (since otherwise ++ # we'd be reporting an implementation detail of every other ++ # function invocation) ++ try: ++ func = older._gdbframe.read_var('func') ++ funcobj = PyObjectPtr.from_pyobject_ptr(func) ++ if isinstance(funcobj, PyCFunctionObjectPtr): ++ # Use the prettyprinter for the func: ++ return str(func) ++ except RuntimeError: ++ return False + ++ # This frame isn't worth reporting: + return False + ++ def is_waiting_for_gil(self): ++ '''Is this frame waiting for a lock?''' ++ framename = self._gdbframe.name() ++ if framename: ++ return 'pthread_cond_timedwait' in framename or \ ++ 'PyThread_acquire_lock' in framename ++ ++ def is_gc_collect(self): ++ '''Is this frame "collect" within the the garbage-collector?''' ++ return self._gdbframe.name() == 'collect' ++ + def get_pyop(self): + try: + f = self._gdbframe.read_var('f') +- frame = PyFrameObjectPtr.from_pyobject_ptr(f) +- if not frame.is_optimized_out(): +- return frame +- # gdb is unable to get the "f" argument of PyEval_EvalFrameEx() +- # because it was "optimized out". Try to get "f" from the frame +- # of the caller, PyEval_EvalCodeEx(). +- orig_frame = frame +- caller = self._gdbframe.older() +- if caller: +- f = caller.read_var('f') +- frame = PyFrameObjectPtr.from_pyobject_ptr(f) +- if not frame.is_optimized_out(): +- return frame +- return orig_frame ++ obj = PyFrameObjectPtr.from_pyobject_ptr(f) ++ if isinstance(obj, PyFrameObjectPtr): ++ return obj ++ else: ++ return None ++ except ValueError: ++ return None ++ ++ def get_py_co(self): ++ try: ++ co = self._gdbframe.read_var('co') ++ obj = PyCodeObjectPtr.from_pyobject_ptr(co) ++ if isinstance(obj, PyCodeObjectPtr): ++ return obj ++ else: ++ return None + except ValueError: + return None + +@@ -1239,8 +1313,22 @@ + + @classmethod + def get_selected_python_frame(cls): +- '''Try to obtain the Frame for the python code in the selected frame, +- or None''' ++ '''Try to obtain the Frame for the python-related code in the selected ++ frame, or None''' ++ frame = cls.get_selected_frame() ++ ++ while frame: ++ if frame.is_python_frame(): ++ return frame ++ frame = frame.older() ++ ++ # Not found: ++ return None ++ ++ @classmethod ++ def get_selected_bytecode_frame(cls): ++ '''Try to obtain the Frame for the python bytecode interpreter in the ++ selected GDB frame, or None''' + frame = cls.get_selected_frame() + + while frame: +@@ -1265,7 +1353,11 @@ + else: + sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index()) + else: +- sys.stdout.write('#%i\n' % self.get_index()) ++ info = self.is_other_python_frame() ++ if info: ++ sys.stdout.write('#%i %s\n' % (self.get_index(), info)) ++ else: ++ sys.stdout.write('#%i\n' % self.get_index()) + + class PyList(gdb.Command): + '''List the current Python source code, if any +@@ -1301,7 +1393,7 @@ + if m: + start, end = map(int, m.groups()) + +- frame = Frame.get_selected_python_frame() ++ frame = Frame.get_selected_bytecode_frame() + if not frame: + print 'Unable to locate python frame' + return +@@ -1353,7 +1445,7 @@ + if not iter_frame: + break + +- if iter_frame.is_evalframeex(): ++ if iter_frame.is_python_frame(): + # Result: + if iter_frame.select(): + iter_frame.print_summary() +@@ -1407,7 +1499,7 @@ + def invoke(self, args, from_tty): + frame = Frame.get_selected_python_frame() + while frame: +- if frame.is_evalframeex(): ++ if frame.is_python_frame(): + frame.print_summary() + frame = frame.older() + diff --git a/SOURCES/00191-add-RPATH-to-elementtree.patch b/SOURCES/00191-add-RPATH-to-elementtree.patch new file mode 100644 index 00000000..10cd5858 --- /dev/null +++ b/SOURCES/00191-add-RPATH-to-elementtree.patch @@ -0,0 +1,21 @@ +diff -up Python-2.7.5/setup.py.orig Python-2.7.5/setup.py +--- Python-2.7.5/setup.py.orig 2013-11-07 01:36:18.853604232 +0100 ++++ Python-2.7.5/setup.py 2013-11-07 01:39:22.163305821 +0100 +@@ -1483,6 +1483,9 @@ class PyBuildExt(build_ext): + # Fredrik Lundh's cElementTree module. Note that this also + # uses expat (via the CAPI hook in pyexpat). + ++ # Add an explicit RPATH to _elementtree.so (rhbz#1019345) ++ EXPAT_RPATH = '/usr/lib64' if sys.maxint == 0x7fffffffffffffff else '/usr/lib' ++ + if os.path.isfile(os.path.join(srcdir, 'Modules', '_elementtree.c')): + define_macros.append(('USE_PYEXPAT_CAPI', None)) + exts.append(Extension('_elementtree', +@@ -1492,6 +1495,7 @@ class PyBuildExt(build_ext): + sources = ['_elementtree.c'], + depends = ['pyexpat.c'] + expat_sources + + expat_depends, ++ extra_link_args = ['-Wl,-rpath,%s' % EXPAT_RPATH] + )) + else: + missing.append('_elementtree') diff --git a/SOURCES/00192-Fix-missing-documentation-for-some-keywords.patch b/SOURCES/00192-Fix-missing-documentation-for-some-keywords.patch new file mode 100644 index 00000000..d40f0b0c --- /dev/null +++ b/SOURCES/00192-Fix-missing-documentation-for-some-keywords.patch @@ -0,0 +1,42 @@ +diff --git a/Doc/tools/sphinxext/pyspecific.py b/Doc/tools/sphinxext/pyspecific.py +--- a/Doc/tools/sphinxext/pyspecific.py ++++ b/Doc/tools/sphinxext/pyspecific.py +@@ -184,11 +184,11 @@ + 'bltin-null-object', 'bltin-type-objects', 'booleans', + 'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound', + 'context-managers', 'continue', 'conversions', 'customization', 'debugger', +- 'del', 'dict', 'dynamic-features', 'else', 'exceptions', 'execmodel', ++ 'del', 'dict', 'dynamic-features', 'else', 'exceptions', 'exec', 'execmodel', + 'exprlists', 'floating', 'for', 'formatstrings', 'function', 'global', + 'id-classes', 'identifiers', 'if', 'imaginary', 'import', 'in', 'integers', + 'lambda', 'lists', 'naming', 'numbers', 'numeric-types', +- 'objects', 'operator-summary', 'pass', 'power', 'raise', 'return', ++ 'objects', 'operator-summary', 'pass', 'power', 'print', 'raise', 'return', + 'sequence-types', 'shifting', 'slicings', 'specialattrs', 'specialnames', + 'string-methods', 'strings', 'subscriptions', 'truth', 'try', 'types', + 'typesfunctions', 'typesmapping', 'typesmethods', 'typesmodules', +diff -up Python-2.7.5/Lib/pydoc_data/topics.py.orig Python-2.7.5/Lib/pydoc_data/topics.py +--- Python-2.7.5/Lib/pydoc_data/topics.py.orig 2014-01-14 12:29:32.511756259 +0100 ++++ Python-2.7.5/Lib/pydoc_data/topics.py 2014-01-14 12:29:40.396795516 +0100 +@@ -29,11 +29,12 @@ topics = {'assert': '\nThe ``assert`` st + 'dynamic-features': '\nInteraction with dynamic features\n*********************************\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n', + 'else': '\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n', + 'exceptions': '\nExceptions\n**********\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', ++'exec': '\nThe ``exec`` statement\n**********************\n\n exec_stmt ::= "exec" or_expr ["in" expression ["," expression]]\n\nThis statement supports dynamic execution of Python code. The first\nexpression should evaluate to either a Unicode string, a *Latin-1*\nencoded string, an open file object, a code object, or a tuple. If it\nis a string, the string is parsed as a suite of Python statements\nwhich is then executed (unless a syntax error occurs). [1] If it is an\nopen file, the file is parsed until EOF and executed. If it is a code\nobject, it is simply executed. For the interpretation of a tuple, see\nbelow. In all cases, the code that\'s executed is expected to be valid\nas file input (see section *File input*). Be aware that the\n``return`` and ``yield`` statements may not be used outside of\nfunction definitions even within the context of code passed to the\n``exec`` statement.\n\nIn all cases, if the optional parts are omitted, the code is executed\nin the current scope. If only the first expression after ``in`` is\nspecified, it should be a dictionary, which will be used for both the\nglobal and the local variables. If two expressions are given, they\nare used for the global and local variables, respectively. If\nprovided, *locals* can be any mapping object. Remember that at module\nlevel, globals and locals are the same dictionary. If two separate\nobjects are given as *globals* and *locals*, the code will be executed\nas if it were embedded in a class definition.\n\nThe first expression may also be a tuple of length 2 or 3. In this\ncase, the optional parts must be omitted. The form ``exec(expr,\nglobals)`` is equivalent to ``exec expr in globals``, while the form\n``exec(expr, globals, locals)`` is equivalent to ``exec expr in\nglobals, locals``. The tuple form of ``exec`` provides compatibility\nwith Python 3, where ``exec`` is a function rather than a statement.\n\nChanged in version 2.4: Formerly, *locals* was required to be a\ndictionary.\n\nAs a side effect, an implementation may insert additional keys into\nthe dictionaries given besides those corresponding to variable names\nset by the executed code. For example, the current implementation may\nadd a reference to the dictionary of the built-in module\n``__builtin__`` under the key ``__builtins__`` (!).\n\n**Programmer\'s hints:** dynamic evaluation of expressions is supported\nby the built-in function ``eval()``. The built-in functions\n``globals()`` and ``locals()`` return the current global and local\ndictionary, respectively, which may be useful to pass around for use\nby ``exec``.\n\n-[ Footnotes ]-\n\n[1] Note that the parser only accepts the Unix-style end of line\n convention. If you are reading the code from a file, make sure to\n use *universal newlines* mode to convert Windows or Mac-style\n newlines.\n', + 'execmodel': '\nExecution model\n***************\n\n\nNaming and binding\n==================\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block. The file read by the\nbuilt-in function ``execfile()`` is a code block. The string argument\npassed to the built-in function ``eval()`` and to the ``exec``\nstatement is a code block. The expression read and evaluated by the\nbuilt-in function ``input()`` is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes generator expressions since\nthey are implemented using a function scope. This means that the\nfollowing will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block.\nIf a name is bound at the module level, it is a global variable. (The\nvariables of the module code block are local and global.) If a\nvariable is used in a code block but not defined there, it is a *free\nvariable*.\n\nWhen a name is not found at all, a ``NameError`` exception is raised.\nIf the name refers to a local variable that has not been bound, a\n``UnboundLocalError`` exception is raised. ``UnboundLocalError`` is a\nsubclass of ``NameError``.\n\nThe following constructs bind names: formal parameters to functions,\n``import`` statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, ``for`` loop header, in the\nsecond position of an ``except`` clause header or after ``as`` in a\n``with`` statement. The ``import`` statement of the form ``from ...\nimport *`` binds all names defined in the imported module, except\nthose beginning with an underscore. This form may only be used at the\nmodule level.\n\nA target occurring in a ``del`` statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name). It\nis illegal to unbind a name that is referenced by an enclosing scope;\nthe compiler will report a ``SyntaxError``.\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the global statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module ``__builtin__``. The global namespace is searched\nfirst. If the name is not found there, the builtins namespace is\nsearched. The global statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name ``__builtins__`` in its\nglobal namespace; this should be a dictionary or a module (in the\nlatter case the module\'s dictionary is used). By default, when in the\n``__main__`` module, ``__builtins__`` is the built-in module\n``__builtin__`` (note: no \'s\'); when in any other module,\n``__builtins__`` is an alias for the dictionary of the ``__builtin__``\nmodule itself. ``__builtins__`` can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n``__builtins__``; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should ``import``\nthe ``__builtin__`` (no \'s\') module and modify its attributes\nappropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n``__main__``.\n\nThe ``global`` statement has the same scope as a name binding\noperation in the same block. If the nearest enclosing scope for a\nfree variable contains a global statement, the free variable is\ntreated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n---------------------------------\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- ``import *`` --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a ``SyntaxError``.\n\nIf ``exec`` is used in a function and the function contains or is a\nnested block with free variables, the compiler will raise a\n``SyntaxError`` unless the exec explicitly specifies the local\nnamespace for the ``exec``. (In other words, ``exec obj`` would be\nillegal, but ``exec obj in ns`` would be legal.)\n\nThe ``eval()``, ``execfile()``, and ``input()`` functions and the\n``exec`` statement do not have access to the full environment for\nresolving names. Names may be resolved in the local and global\nnamespaces of the caller. Free variables are not resolved in the\nnearest enclosing namespace, but in the global namespace. [1] The\n``exec`` statement and the ``eval()`` and ``execfile()`` functions\nhave optional arguments to override the global and local namespace.\nIf only one namespace is specified, it is used for both.\n\n\nExceptions\n==========\n\nExceptions are a means of breaking out of the normal flow of control\nof a code block in order to handle errors or other exceptional\nconditions. An exception is *raised* at the point where the error is\ndetected; it may be *handled* by the surrounding code block or by any\ncode block that directly or indirectly invoked the code block where\nthe error occurred.\n\nThe Python interpreter raises an exception when it detects a run-time\nerror (such as division by zero). A Python program can also\nexplicitly raise an exception with the ``raise`` statement. Exception\nhandlers are specified with the ``try`` ... ``except`` statement. The\n``finally`` clause of such a statement can be used to specify cleanup\ncode which does not handle the exception, but is executed whether an\nexception occurred or not in the preceding code.\n\nPython uses the "termination" model of error handling: an exception\nhandler can find out what happened and continue execution at an outer\nlevel, but it cannot repair the cause of the error and retry the\nfailing operation (except by re-entering the offending piece of code\nfrom the top).\n\nWhen an exception is not handled at all, the interpreter terminates\nexecution of the program, or returns to its interactive main loop. In\neither case, it prints a stack backtrace, except when the exception is\n``SystemExit``.\n\nExceptions are identified by class instances. The ``except`` clause\nis selected depending on the class of the instance: it must reference\nthe class of the instance or a base class thereof. The instance can\nbe received by the handler and can carry additional information about\nthe exceptional condition.\n\nExceptions can also be identified by strings, in which case the\n``except`` clause is selected by object identity. An arbitrary value\ncan be raised along with the identifying string which can be passed to\nthe handler.\n\nNote: Messages to exceptions are not part of the Python API. Their\n contents may change from one version of Python to the next without\n warning and should not be relied on by code which will run under\n multiple versions of the interpreter.\n\nSee also the description of the ``try`` statement in section *The try\nstatement* and ``raise`` statement in section *The raise statement*.\n\n-[ Footnotes ]-\n\n[1] This limitation occurs because the code that is executed by these\n operations is not available at the time the module is compiled.\n', + 'exprlists': '\nExpression lists\n****************\n\n expression_list ::= expression ( "," expression )* [","]\n\nAn expression list containing at least one comma yields a tuple. The\nlength of the tuple is the number of expressions in the list. The\nexpressions are evaluated from left to right.\n\nThe trailing comma is required only to create a single tuple (a.k.a. a\n*singleton*); it is optional in all other cases. A single expression\nwithout a trailing comma doesn\'t create a tuple, but rather yields the\nvalue of that expression. (To create an empty tuple, use an empty pair\nof parentheses: ``()``.)\n', + 'floating': '\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n floatnumber ::= pointfloat | exponentfloat\n pointfloat ::= [intpart] fraction | intpart "."\n exponentfloat ::= (intpart | pointfloat) exponent\n intpart ::= digit+\n fraction ::= "." digit+\n exponent ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts of floating point numbers can\nlook like octal integers, but are interpreted using radix 10. For\nexample, ``077e010`` is legal, and denotes the same number as\n``77e10``. The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n 3.14 10. .001 1e100 3.14e-10 0e0\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator ``-`` and the\nliteral ``1``.\n', + 'for': '\nThe ``for`` statement\n*********************\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n', +- 'formatstrings': '\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= +\n conversion ::= "r" | "s"\n format_spec ::= \n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword. If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument. If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings ``\'10\'`` or\n``\':-]\'``) within a format string. The *arg_name* can be followed by\nany number of index or attribute expressions. An expression of the\nform ``\'.name\'`` selects the named attribute using ``getattr()``,\nwhile an expression of the form ``\'[index]\'`` does an index lookup\nusing ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= \n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'. The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options. If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default for most objects). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space (this is the default for numbers). |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nPreceding the *width* field by a zero (``\'0\'``) character enables\nsign-aware zero-padding for numeric types. This is equivalent to a\n*fill* character of ``\'0\'`` with an *alignment* type of ``\'=\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n | | The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n | | The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Positive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', ++ 'formatstrings': '\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= +\n conversion ::= "r" | "s"\n format_spec ::= \n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword. If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument. If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings ``\'10\'`` or\n``\':-]\'``) within a format string. The *arg_name* can be followed by\nany number of index or attribute expressions. An expression of the\nform ``\'.name\'`` selects the named attribute using ``getattr()``,\nwhile an expression of the form ``\'[index]\'`` does an index lookup\nusing ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= \n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nIf a valid *align* value is specified, it can be preceded by a *fill*\ncharacter that can be any character and defaults to a space if\nomitted. Note that it is not possible to use ``{`` and ``}`` as *fill*\nchar while using the ``str.format()`` method; this limitation however\ndoesn\'t affect the ``format()`` function.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default for most objects). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space (this is the default for numbers). |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nPreceding the *width* field by a zero (``\'0\'``) character enables\nsign-aware zero-padding for numeric types. This is equivalent to a\n*fill* character of ``\'0\'`` with an *alignment* type of ``\'=\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n | | The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n | | The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Positive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', + 'function': '\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier ["," "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level *parameters* have the form *parameter*\n``=`` *expression*, the function is said to have "default parameter\nvalues." For a parameter with a default value, the corresponding\n*argument* may be omitted from a call, in which case the parameter\'s\ndefault value is substituted. If a parameter has a default value, all\nfollowing parameters must also have a default value --- this is a\nsyntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that the same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n', + 'global': '\nThe ``global`` statement\n************************\n\n global_stmt ::= "global" identifier ("," identifier)*\n\nThe ``global`` statement is a declaration which holds for the entire\ncurrent code block. It means that the listed identifiers are to be\ninterpreted as globals. It would be impossible to assign to a global\nvariable without ``global``, although free variables may refer to\nglobals without being declared global.\n\nNames listed in a ``global`` statement must not be used in the same\ncode block textually preceding that ``global`` statement.\n\nNames listed in a ``global`` statement must not be defined as formal\nparameters or in a ``for`` loop control target, ``class`` definition,\nfunction definition, or ``import`` statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the latter two restrictions, but programs should not abuse\nthis freedom, as future implementations may enforce them or silently\nchange the meaning of the program.\n\n**Programmer\'s note:** the ``global`` is a directive to the parser.\nIt applies only to code parsed at the same time as the ``global``\nstatement. In particular, a ``global`` statement contained in an\n``exec`` statement does not affect the code block *containing* the\n``exec`` statement, and code contained in an ``exec`` statement is\nunaffected by ``global`` statements in the code containing the\n``exec`` statement. The same applies to the ``eval()``,\n``execfile()`` and ``compile()`` functions.\n', + 'id-classes': '\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', +@@ -52,6 +53,7 @@ topics = {'assert': '\nThe ``assert`` st + 'operator-summary': '\nOperator precedence\n*******************\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` ``x`` | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not in``, ``is``, ``is not``, ``<``, | Comparisons, including membership |\n| ``<=``, ``>``, ``>=``, ``<>``, ``!=``, ``==`` | tests and identity tests |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n| | [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [9] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key: value...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. The function\n ``math.fmod()`` returns a result whose sign matches the sign of\n the first argument instead, and so returns ``-1e-100`` in this\n case. Which approach is more appropriate depends on the\n application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The ``%`` operator is also used for string formatting; the same\n precedence applies.\n\n[9] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', + 'pass': '\nThe ``pass`` statement\n**********************\n\n pass_stmt ::= "pass"\n\n``pass`` is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n def f(arg): pass # a function that does nothing (yet)\n\n class C: pass # a class with no methods (yet)\n', + 'power': '\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right. The\nsyntax is:\n\n power ::= primary ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): ``-1**2`` results in ``-1``.\n\nThe power operator has the same semantics as the built-in ``pow()``\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument. The numeric arguments are\nfirst converted to a common type. The result type is that of the\narguments after coercion.\n\nWith mixed operand types, the coercion rules for binary arithmetic\noperators apply. For int and long int operands, the result has the\nsame type as the operands (after coercion) unless the second argument\nis negative; in that case, all arguments are converted to float and a\nfloat result is delivered. For example, ``10**2`` returns ``100``, but\n``10**-2`` returns ``0.01``. (This last feature was added in Python\n2.2. In Python 2.1 and before, if both arguments were of integer types\nand the second argument was negative, an exception was raised).\n\nRaising ``0.0`` to a negative power results in a\n``ZeroDivisionError``. Raising a negative number to a fractional power\nresults in a ``ValueError``.\n', ++ 'print': '\nThe ``print`` statement\n***********************\n\n print_stmt ::= "print" ([expression ("," expression)* [","]]\n | ">>" expression [("," expression)+ [","]])\n\n``print`` evaluates each expression in turn and writes the resulting\nobject to standard output (see below). If an object is not a string,\nit is first converted to a string using the rules for string\nconversions. The (resulting or original) string is then written. A\nspace is written before each object is (converted and) written, unless\nthe output system believes it is positioned at the beginning of a\nline. This is the case (1) when no characters have yet been written\nto standard output, (2) when the last character written to standard\noutput is a whitespace character except ``\' \'``, or (3) when the last\nwrite operation on standard output was not a ``print`` statement. (In\nsome cases it may be functional to write an empty string to standard\noutput for this reason.)\n\nNote: Objects which act like file objects but which are not the built-in\n file objects often do not properly emulate this aspect of the file\n object\'s behavior, so it is best not to rely on this.\n\nA ``\'\\n\'`` character is written at the end, unless the ``print``\nstatement ends with a comma. This is the only action if the statement\ncontains just the keyword ``print``.\n\nStandard output is defined as the file object named ``stdout`` in the\nbuilt-in module ``sys``. If no such object exists, or if it does not\nhave a ``write()`` method, a ``RuntimeError`` exception is raised.\n\n``print`` also has an extended form, defined by the second portion of\nthe syntax described above. This form is sometimes referred to as\n"``print`` chevron." In this form, the first expression after the\n``>>`` must evaluate to a "file-like" object, specifically an object\nthat has a ``write()`` method as described above. With this extended\nform, the subsequent expressions are printed to this file object. If\nthe first expression evaluates to ``None``, then ``sys.stdout`` is\nused as the file for output.\n', + 'raise': '\nThe ``raise`` statement\n***********************\n\n raise_stmt ::= "raise" [expression ["," expression ["," expression]]]\n\nIf no expressions are present, ``raise`` re-raises the last exception\nthat was active in the current scope. If no exception is active in\nthe current scope, a ``TypeError`` exception is raised indicating that\nthis is an error (if running under IDLE, a ``Queue.Empty`` exception\nis raised instead).\n\nOtherwise, ``raise`` evaluates the expressions to get three objects,\nusing ``None`` as the value of omitted expressions. The first two\nobjects are used to determine the *type* and *value* of the exception.\n\nIf the first object is an instance, the type of the exception is the\nclass of the instance, the instance itself is the value, and the\nsecond object must be ``None``.\n\nIf the first object is a class, it becomes the type of the exception.\nThe second object is used to determine the exception value: If it is\nan instance of the class, the instance becomes the exception value. If\nthe second object is a tuple, it is used as the argument list for the\nclass constructor; if it is ``None``, an empty argument list is used,\nand any other object is treated as a single argument to the\nconstructor. The instance so created by calling the constructor is\nused as the exception value.\n\nIf a third object is present and not ``None``, it must be a traceback\nobject (see section *The standard type hierarchy*), and it is\nsubstituted instead of the current location as the place where the\nexception occurred. If the third object is present and not a\ntraceback object or ``None``, a ``TypeError`` exception is raised.\nThe three-expression form of ``raise`` is useful to re-raise an\nexception transparently in an except clause, but ``raise`` with no\nexpressions should be preferred if the exception to be re-raised was\nthe most recently active exception in the current scope.\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information about handling exceptions is in section\n*The try statement*.\n', + 'return': '\nThe ``return`` statement\n************************\n\n return_stmt ::= "return" [expression_list]\n\n``return`` may only occur syntactically nested in a function\ndefinition, not within a nested class definition.\n\nIf an expression list is present, it is evaluated, else ``None`` is\nsubstituted.\n\n``return`` leaves the current function call with the expression list\n(or ``None``) as return value.\n\nWhen ``return`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nleaving the function.\n\nIn a generator function, the ``return`` statement is not allowed to\ninclude an ``expression_list``. In that context, a bare ``return``\nindicates that the generator is done and will cause ``StopIteration``\nto be raised.\n', + 'sequence-types': "\nEmulating container types\n*************************\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python's\nstandard dictionary objects. The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects. Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators. It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn't define a ``__nonzero__()`` method and whose\n ``__len__()`` method returns zero is considered to be false in a\n Boolean context.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``iterkeys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\n New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don't define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n", diff --git a/SOURCES/00193-buffer-overflow.patch b/SOURCES/00193-buffer-overflow.patch new file mode 100644 index 00000000..164b4627 --- /dev/null +++ b/SOURCES/00193-buffer-overflow.patch @@ -0,0 +1,43 @@ + +# HG changeset patch +# User Benjamin Peterson +# Date 1389671978 18000 +# Node ID 87673659d8f7ba1623cd4914f09ad3d2ade034e9 +# Parent 2631d33ee7fbd5f0288931ef37872218d511d2e8 +complain when nbytes > buflen to fix possible buffer overflow (closes #20246) + +diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py +--- a/Lib/test/test_socket.py ++++ b/Lib/test/test_socket.py +@@ -1620,6 +1620,16 @@ class BufferIOTest(SocketConnectedTest): + + _testRecvFromIntoMemoryview = _testRecvFromIntoArray + ++ def testRecvFromIntoSmallBuffer(self): ++ # See issue #20246. ++ buf = bytearray(8) ++ self.assertRaises(ValueError, self.cli_conn.recvfrom_into, buf, 1024) ++ ++ def _testRecvFromIntoSmallBuffer(self): ++ with test_support.check_py3k_warnings(): ++ buf = buffer(MSG*2048) ++ self.serv_conn.send(buf) ++ + + TIPC_STYPE = 2000 + TIPC_LOWER = 200 +diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c +--- a/Modules/socketmodule.c ++++ b/Modules/socketmodule.c +@@ -2742,6 +2742,10 @@ sock_recvfrom_into(PySocketSockObject *s + if (recvlen == 0) { + /* If nbytes was not specified, use the buffer's length */ + recvlen = buflen; ++ } else if (recvlen > buflen) { ++ PyErr_SetString(PyExc_ValueError, ++ "nbytes is greater than the length of the buffer"); ++ goto error; + } + + readlen = sock_recvfrom_guts(s, buf.buf, recvlen, flags, &addr); + diff --git a/SOURCES/00194-gdb-dont-fail-on-frame-with-address.patch b/SOURCES/00194-gdb-dont-fail-on-frame-with-address.patch new file mode 100644 index 00000000..501b5dba --- /dev/null +++ b/SOURCES/00194-gdb-dont-fail-on-frame-with-address.patch @@ -0,0 +1,19 @@ +--- Lib/test/test_gdb.py.orig 2014-08-01 14:30:43.397473152 +0200 ++++ Lib/test/test_gdb.py 2014-08-01 14:34:50.907325691 +0200 +@@ -135,6 +135,16 @@ + # Disable this: + 'set print entry-values no', + ++ # The tests assume that the first frame of printed ++ # backtrace will not contain program counter, ++ # that is however not guaranteed by gdb (rhbz#1125657) ++ # therefore we need to use 'set print address off' to ++ # make sure the counter is not there. For example: ++ # #0 in PyObject_Print ... ++ # is assumed, but sometimes this can be e.g. ++ # #0 0x00003fffb7dd1798 in PyObject_Print ... ++ 'set print address off', ++ + 'run'] + + if HAS_AUTOLOAD_SAFEPATH: diff --git a/SOURCES/00195-make-multiproc-ignore-EINTR.patch b/SOURCES/00195-make-multiproc-ignore-EINTR.patch new file mode 100644 index 00000000..050320a0 --- /dev/null +++ b/SOURCES/00195-make-multiproc-ignore-EINTR.patch @@ -0,0 +1,216 @@ + +# HG changeset patch +# User Richard Oudkerk +# Date 1372700728 -3600 +# Node ID bc34fe4a0d58a047509798acb0b4b2a21ce1e375 +# Parent 26ef5d5d5c3ea76ab411f2984d507aadce0ce8d7 +Issue #17097: Make multiprocessing ignore EINTR. + +diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py +--- a/Lib/multiprocessing/connection.py ++++ b/Lib/multiprocessing/connection.py +@@ -270,7 +270,14 @@ class SocketListener(object): + self._unlink = None + + def accept(self): +- s, self._last_accepted = self._socket.accept() ++ while True: ++ try: ++ s, self._last_accepted = self._socket.accept() ++ except socket.error as e: ++ if e.args[0] != errno.EINTR: ++ raise ++ else: ++ break + s.setblocking(True) + fd = duplicate(s.fileno()) + conn = _multiprocessing.Connection(fd) +diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py +--- a/Lib/test/test_multiprocessing.py ++++ b/Lib/test/test_multiprocessing.py +@@ -2461,12 +2461,80 @@ class TestForkAwareThreadLock(unittest.T + self.assertLessEqual(new_size, old_size) + + # ++# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc ++# ++ ++class TestIgnoreEINTR(unittest.TestCase): ++ ++ @classmethod ++ def _test_ignore(cls, conn): ++ def handler(signum, frame): ++ pass ++ signal.signal(signal.SIGUSR1, handler) ++ conn.send('ready') ++ x = conn.recv() ++ conn.send(x) ++ conn.send_bytes(b'x'*(1024*1024)) # sending 1 MB should block ++ ++ @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') ++ def test_ignore(self): ++ conn, child_conn = multiprocessing.Pipe() ++ try: ++ p = multiprocessing.Process(target=self._test_ignore, ++ args=(child_conn,)) ++ p.daemon = True ++ p.start() ++ child_conn.close() ++ self.assertEqual(conn.recv(), 'ready') ++ time.sleep(0.1) ++ os.kill(p.pid, signal.SIGUSR1) ++ time.sleep(0.1) ++ conn.send(1234) ++ self.assertEqual(conn.recv(), 1234) ++ time.sleep(0.1) ++ os.kill(p.pid, signal.SIGUSR1) ++ self.assertEqual(conn.recv_bytes(), b'x'*(1024*1024)) ++ time.sleep(0.1) ++ p.join() ++ finally: ++ conn.close() ++ ++ @classmethod ++ def _test_ignore_listener(cls, conn): ++ def handler(signum, frame): ++ pass ++ signal.signal(signal.SIGUSR1, handler) ++ l = multiprocessing.connection.Listener() ++ conn.send(l.address) ++ a = l.accept() ++ a.send('welcome') ++ ++ @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') ++ def test_ignore_listener(self): ++ conn, child_conn = multiprocessing.Pipe() ++ try: ++ p = multiprocessing.Process(target=self._test_ignore_listener, ++ args=(child_conn,)) ++ p.daemon = True ++ p.start() ++ child_conn.close() ++ address = conn.recv() ++ time.sleep(0.1) ++ os.kill(p.pid, signal.SIGUSR1) ++ time.sleep(0.1) ++ client = multiprocessing.connection.Client(address) ++ self.assertEqual(client.recv(), 'welcome') ++ p.join() ++ finally: ++ conn.close() ++ ++# + # + # + + testcases_other = [OtherTest, TestInvalidHandle, TestInitializers, + TestStdinBadfiledescriptor, TestTimeouts, TestNoForkBomb, +- TestFlags, TestForkAwareThreadLock] ++ TestFlags, TestForkAwareThreadLock, TestIgnoreEINTR] + + # + # +diff --git a/Modules/_multiprocessing/socket_connection.c b/Modules/_multiprocessing/socket_connection.c +--- a/Modules/_multiprocessing/socket_connection.c ++++ b/Modules/_multiprocessing/socket_connection.c +@@ -23,6 +23,21 @@ + #endif + + /* ++ * Wrapper for PyErr_CheckSignals() which can be called without the GIL ++ */ ++ ++static int ++check_signals(void) ++{ ++ PyGILState_STATE state; ++ int res; ++ state = PyGILState_Ensure(); ++ res = PyErr_CheckSignals(); ++ PyGILState_Release(state); ++ return res; ++} ++ ++/* + * Send string to file descriptor + */ + +@@ -34,8 +49,14 @@ static Py_ssize_t + + while (length > 0) { + res = WRITE(h, p, length); +- if (res < 0) ++ if (res < 0) { ++ if (errno == EINTR) { ++ if (check_signals() < 0) ++ return MP_EXCEPTION_HAS_BEEN_SET; ++ continue; ++ } + return MP_SOCKET_ERROR; ++ } + length -= res; + p += res; + } +@@ -56,12 +77,16 @@ static Py_ssize_t + + while (remaining > 0) { + temp = READ(h, p, remaining); +- if (temp <= 0) { +- if (temp == 0) +- return remaining == length ? +- MP_END_OF_FILE : MP_EARLY_END_OF_FILE; +- else +- return temp; ++ if (temp < 0) { ++ if (errno == EINTR) { ++ if (check_signals() < 0) ++ return MP_EXCEPTION_HAS_BEEN_SET; ++ continue; ++ } ++ return temp; ++ } ++ else if (temp == 0) { ++ return remaining == length ? MP_END_OF_FILE : MP_EARLY_END_OF_FILE; + } + remaining -= temp; + p += temp; +@@ -171,9 +196,16 @@ conn_poll(ConnectionObject *conn, double + p.revents = 0; + + if (timeout < 0) { +- res = poll(&p, 1, -1); ++ do { ++ res = poll(&p, 1, -1); ++ } while (res < 0 && errno == EINTR); + } else { + res = poll(&p, 1, (int)(timeout * 1000 + 0.5)); ++ if (res < 0 && errno == EINTR) { ++ /* We were interrupted by a signal. Just indicate a ++ timeout even though we are early. */ ++ return FALSE; ++ } + } + + if (res < 0) { +@@ -209,12 +241,19 @@ conn_poll(ConnectionObject *conn, double + FD_SET((SOCKET)conn->handle, &rfds); + + if (timeout < 0.0) { +- res = select((int)conn->handle+1, &rfds, NULL, NULL, NULL); ++ do { ++ res = select((int)conn->handle+1, &rfds, NULL, NULL, NULL); ++ } while (res < 0 && errno == EINTR); + } else { + struct timeval tv; + tv.tv_sec = (long)timeout; + tv.tv_usec = (long)((timeout - tv.tv_sec) * 1e6 + 0.5); + res = select((int)conn->handle+1, &rfds, NULL, NULL, &tv); ++ if (res < 0 && errno == EINTR) { ++ /* We were interrupted by a signal. Just indicate a ++ timeout even though we are early. */ ++ return FALSE; ++ } + } + + if (res < 0) { + diff --git a/SOURCES/00196-avoid-double-close-of-pipes.patch b/SOURCES/00196-avoid-double-close-of-pipes.patch new file mode 100644 index 00000000..bd53bbce --- /dev/null +++ b/SOURCES/00196-avoid-double-close-of-pipes.patch @@ -0,0 +1,288 @@ + +# HG changeset patch +# User Antoine Pitrou +# Date 1377898693 -7200 +# Node ID 43749cb6bdbd0fdab70f76cd171c3c02a3f600dd +# Parent ba54011aa295004ad87438211fe3bb1568dd69ab +Issue #18851: Avoid a double close of subprocess pipes when the child process fails starting. + +diff --git a/Lib/subprocess.py b/Lib/subprocess.py +--- a/Lib/subprocess.py ++++ b/Lib/subprocess.py +@@ -698,12 +698,12 @@ class Popen(object): + + (p2cread, p2cwrite, + c2pread, c2pwrite, +- errread, errwrite) = self._get_handles(stdin, stdout, stderr) ++ errread, errwrite), to_close = self._get_handles(stdin, stdout, stderr) + + try: + self._execute_child(args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, +- startupinfo, creationflags, shell, ++ startupinfo, creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) +@@ -711,18 +711,12 @@ class Popen(object): + # Preserve original exception in case os.close raises. + exc_type, exc_value, exc_trace = sys.exc_info() + +- to_close = [] +- # Only close the pipes we created. +- if stdin == PIPE: +- to_close.extend((p2cread, p2cwrite)) +- if stdout == PIPE: +- to_close.extend((c2pread, c2pwrite)) +- if stderr == PIPE: +- to_close.extend((errread, errwrite)) +- + for fd in to_close: + try: +- os.close(fd) ++ if mswindows: ++ fd.Close() ++ else: ++ os.close(fd) + except EnvironmentError: + pass + +@@ -816,8 +810,9 @@ class Popen(object): + """Construct and return tuple with IO objects: + p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite + """ ++ to_close = set() + if stdin is None and stdout is None and stderr is None: +- return (None, None, None, None, None, None) ++ return (None, None, None, None, None, None), to_close + + p2cread, p2cwrite = None, None + c2pread, c2pwrite = None, None +@@ -835,6 +830,10 @@ class Popen(object): + # Assuming file-like object + p2cread = msvcrt.get_osfhandle(stdin.fileno()) + p2cread = self._make_inheritable(p2cread) ++ # We just duplicated the handle, it has to be closed at the end ++ to_close.add(p2cread) ++ if stdin == PIPE: ++ to_close.add(p2cwrite) + + if stdout is None: + c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE) +@@ -848,6 +847,10 @@ class Popen(object): + # Assuming file-like object + c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) + c2pwrite = self._make_inheritable(c2pwrite) ++ # We just duplicated the handle, it has to be closed at the end ++ to_close.add(c2pwrite) ++ if stdout == PIPE: ++ to_close.add(c2pread) + + if stderr is None: + errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE) +@@ -863,10 +866,14 @@ class Popen(object): + # Assuming file-like object + errwrite = msvcrt.get_osfhandle(stderr.fileno()) + errwrite = self._make_inheritable(errwrite) ++ # We just duplicated the handle, it has to be closed at the end ++ to_close.add(errwrite) ++ if stderr == PIPE: ++ to_close.add(errread) + + return (p2cread, p2cwrite, + c2pread, c2pwrite, +- errread, errwrite) ++ errread, errwrite), to_close + + + def _make_inheritable(self, handle): +@@ -895,7 +902,7 @@ class Popen(object): + + def _execute_child(self, args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, +- startupinfo, creationflags, shell, ++ startupinfo, creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite): +@@ -934,6 +941,10 @@ class Popen(object): + # kill children. + creationflags |= _subprocess.CREATE_NEW_CONSOLE + ++ def _close_in_parent(fd): ++ fd.Close() ++ to_close.remove(fd) ++ + # Start the process + try: + hp, ht, pid, tid = _subprocess.CreateProcess(executable, args, +@@ -958,11 +969,11 @@ class Popen(object): + # pipe will not close when the child process exits and the + # ReadFile will hang. + if p2cread is not None: +- p2cread.Close() ++ _close_in_parent(p2cread) + if c2pwrite is not None: +- c2pwrite.Close() ++ _close_in_parent(c2pwrite) + if errwrite is not None: +- errwrite.Close() ++ _close_in_parent(errwrite) + + # Retain the process handle, but close the thread handle + self._child_created = True +@@ -1088,6 +1099,7 @@ class Popen(object): + """Construct and return tuple with IO objects: + p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite + """ ++ to_close = set() + p2cread, p2cwrite = None, None + c2pread, c2pwrite = None, None + errread, errwrite = None, None +@@ -1096,6 +1108,7 @@ class Popen(object): + pass + elif stdin == PIPE: + p2cread, p2cwrite = self.pipe_cloexec() ++ to_close.update((p2cread, p2cwrite)) + elif isinstance(stdin, int): + p2cread = stdin + else: +@@ -1106,6 +1119,7 @@ class Popen(object): + pass + elif stdout == PIPE: + c2pread, c2pwrite = self.pipe_cloexec() ++ to_close.update((c2pread, c2pwrite)) + elif isinstance(stdout, int): + c2pwrite = stdout + else: +@@ -1116,6 +1130,7 @@ class Popen(object): + pass + elif stderr == PIPE: + errread, errwrite = self.pipe_cloexec() ++ to_close.update((errread, errwrite)) + elif stderr == STDOUT: + errwrite = c2pwrite + elif isinstance(stderr, int): +@@ -1126,7 +1141,7 @@ class Popen(object): + + return (p2cread, p2cwrite, + c2pread, c2pwrite, +- errread, errwrite) ++ errread, errwrite), to_close + + + def _set_cloexec_flag(self, fd, cloexec=True): +@@ -1170,7 +1185,7 @@ class Popen(object): + + def _execute_child(self, args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, +- startupinfo, creationflags, shell, ++ startupinfo, creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite): +@@ -1189,6 +1204,10 @@ class Popen(object): + if executable is None: + executable = args[0] + ++ def _close_in_parent(fd): ++ os.close(fd) ++ to_close.remove(fd) ++ + # For transferring possible exec failure from child to parent + # The first char specifies the exception type: 0 means + # OSError, 1 means some other error. +@@ -1283,17 +1302,17 @@ class Popen(object): + # be sure the FD is closed no matter what + os.close(errpipe_write) + +- if p2cread is not None and p2cwrite is not None: +- os.close(p2cread) +- if c2pwrite is not None and c2pread is not None: +- os.close(c2pwrite) +- if errwrite is not None and errread is not None: +- os.close(errwrite) +- + # Wait for exec to fail or succeed; possibly raising exception + # Exception limited to 1M + data = _eintr_retry_call(os.read, errpipe_read, 1048576) + finally: ++ if p2cread is not None and p2cwrite is not None: ++ _close_in_parent(p2cread) ++ if c2pwrite is not None and c2pread is not None: ++ _close_in_parent(c2pwrite) ++ if errwrite is not None and errread is not None: ++ _close_in_parent(errwrite) ++ + # be sure the FD is closed no matter what + os.close(errpipe_read) + +diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py +--- a/Lib/test/test_subprocess.py ++++ b/Lib/test/test_subprocess.py +@@ -14,6 +14,10 @@ try: + import resource + except ImportError: + resource = None ++try: ++ import threading ++except ImportError: ++ threading = None + + mswindows = (sys.platform == "win32") + +@@ -629,6 +633,36 @@ class ProcessTestCase(BaseTestCase): + if c.exception.errno not in (errno.ENOENT, errno.EACCES): + raise c.exception + ++ @unittest.skipIf(threading is None, "threading required") ++ def test_double_close_on_error(self): ++ # Issue #18851 ++ fds = [] ++ def open_fds(): ++ for i in range(20): ++ fds.extend(os.pipe()) ++ time.sleep(0.001) ++ t = threading.Thread(target=open_fds) ++ t.start() ++ try: ++ with self.assertRaises(EnvironmentError): ++ subprocess.Popen(['nonexisting_i_hope'], ++ stdin=subprocess.PIPE, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.PIPE) ++ finally: ++ t.join() ++ exc = None ++ for fd in fds: ++ # If a double close occurred, some of those fds will ++ # already have been closed by mistake, and os.close() ++ # here will raise. ++ try: ++ os.close(fd) ++ except OSError as e: ++ exc = e ++ if exc is not None: ++ raise exc ++ + def test_handles_closed_on_exception(self): + # If CreateProcess exits with an error, ensure the + # duplicate output handles are released +@@ -783,7 +817,7 @@ class POSIXProcessTestCase(BaseTestCase) + + def _execute_child( + self, args, executable, preexec_fn, close_fds, cwd, env, +- universal_newlines, startupinfo, creationflags, shell, ++ universal_newlines, startupinfo, creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite): +@@ -791,7 +825,7 @@ class POSIXProcessTestCase(BaseTestCase) + subprocess.Popen._execute_child( + self, args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, +- startupinfo, creationflags, shell, ++ startupinfo, creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) diff --git a/SOURCES/00197-add-missing-import-in-bdist_rpm.patch b/SOURCES/00197-add-missing-import-in-bdist_rpm.patch new file mode 100644 index 00000000..79ad5f41 --- /dev/null +++ b/SOURCES/00197-add-missing-import-in-bdist_rpm.patch @@ -0,0 +1,20 @@ + +# HG changeset patch +# User Éric Araujo +# Date 1394614885 14400 +# Node ID 677327810121891704491bafa6209af5b60ebc91 +# Parent 0f1237b61f58a77a159ab6e452782a8924ff2966 +Fix missing import in bdist_rpm (#18045) + +diff --git a/Lib/distutils/command/bdist_rpm.py b/Lib/distutils/command/bdist_rpm.py +--- a/Lib/distutils/command/bdist_rpm.py ++++ b/Lib/distutils/command/bdist_rpm.py +@@ -12,6 +12,7 @@ import string + from distutils.core import Command + from distutils.debug import DEBUG + from distutils.file_util import write_file ++from distutils.sysconfig import get_python_version + from distutils.errors import (DistutilsOptionError, DistutilsPlatformError, + DistutilsFileError, DistutilsExecError) + from distutils import log + diff --git a/SOURCES/00198-fix-readline-erroneous-output.patch b/SOURCES/00198-fix-readline-erroneous-output.patch new file mode 100644 index 00000000..63224467 --- /dev/null +++ b/SOURCES/00198-fix-readline-erroneous-output.patch @@ -0,0 +1,166 @@ + +# HG changeset patch +# User Victor Stinner +# Date 1406197344 -7200 +# Node ID 0177d8a4e82a613de0c64e747656c1d0b63e49b3 +# Parent e70ab72286b470b7209b91d3aa8a21953aafb78f +Issue #19884: readline: Disable the meta modifier key if stdout is not a +terminal to not write the ANSI sequence "\033[1034h" into stdout. This sequence +is used on some terminal (ex: TERM=xterm-256color") to enable support of 8 bit +characters. + +diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py +--- a/Lib/test/test_readline.py ++++ b/Lib/test/test_readline.py +@@ -1,17 +1,19 @@ + """ + Very minimal unittests for parts of the readline module. +- +-These tests were added to check that the libedit emulation on OSX and +-the "real" readline have the same interface for history manipulation. That's +-why the tests cover only a small subset of the interface. + """ ++import os + import unittest + from test.test_support import run_unittest, import_module ++from test.script_helper import assert_python_ok + + # Skip tests if there is no readline module + readline = import_module('readline') + + class TestHistoryManipulation (unittest.TestCase): ++ """These tests were added to check that the libedit emulation on OSX and ++ the "real" readline have the same interface for history manipulation. ++ That's why the tests cover only a small subset of the interface. ++ """ + + @unittest.skipIf(not hasattr(readline, 'clear_history'), + "The history update test cannot be run because the " +@@ -40,8 +42,18 @@ class TestHistoryManipulation (unittest. + self.assertEqual(readline.get_current_history_length(), 1) + + ++class TestReadline(unittest.TestCase): ++ def test_init(self): ++ # Issue #19884: Ensure that the ANSI sequence "\033[1034h" is not ++ # written into stdout when the readline module is imported and stdout ++ # is redirected to a pipe. ++ rc, stdout, stderr = assert_python_ok('-c', 'import readline', ++ TERM='xterm-256color') ++ self.assertEqual(stdout, b'') ++ ++ + def test_main(): +- run_unittest(TestHistoryManipulation) ++ run_unittest(TestHistoryManipulation, TestReadline) + + if __name__ == "__main__": + test_main() +diff --git a/Modules/readline.c b/Modules/readline.c +--- a/Modules/readline.c ++++ b/Modules/readline.c +@@ -887,7 +887,7 @@ setup_readline(void) + #endif + + #ifdef __APPLE__ +- /* the libedit readline emulation resets key bindings etc ++ /* the libedit readline emulation resets key bindings etc + * when calling rl_initialize. So call it upfront + */ + if (using_libedit_emulation) +@@ -932,6 +932,17 @@ setup_readline(void) + + begidx = PyInt_FromLong(0L); + endidx = PyInt_FromLong(0L); ++ ++ if (!isatty(STDOUT_FILENO)) { ++ /* Issue #19884: stdout is no a terminal. Disable meta modifier ++ keys to not write the ANSI sequence "\033[1034h" into stdout. On ++ terminals supporting 8 bit characters like TERM=xterm-256color ++ (which is now the default Fedora since Fedora 18), the meta key is ++ used to enable support of 8 bit characters (ANSI sequence ++ "\033[1034h"). */ ++ rl_variable_bind ("enable-meta-key", "off"); ++ } ++ + /* Initialize (allows .inputrc to override) + * + * XXX: A bug in the readline-2.2 library causes a memory leak +@@ -943,7 +954,7 @@ setup_readline(void) + else + #endif /* __APPLE__ */ + rl_initialize(); +- ++ + RESTORE_LOCALE(saved_locale) + } + + + +# HG changeset patch +# User Victor Stinner +# Date 1406232681 -7200 +# Node ID f0ab6f9f06036dfacff09f22f86464840b50eb0a +# Parent d422062d7d366386acdb81851b0f2ec3a6f6750c +Issue #19884, readline: calling rl_variable_bind ("enable-meta-key", "off") +does crash on Mac OS X which uses libedit instead of readline. + +diff --git a/Modules/readline.c b/Modules/readline.c +--- a/Modules/readline.c ++++ b/Modules/readline.c +@@ -933,15 +933,19 @@ setup_readline(void) + begidx = PyInt_FromLong(0L); + endidx = PyInt_FromLong(0L); + ++#ifndef __APPLE__ + if (!isatty(STDOUT_FILENO)) { + /* Issue #19884: stdout is no a terminal. Disable meta modifier + keys to not write the ANSI sequence "\033[1034h" into stdout. On + terminals supporting 8 bit characters like TERM=xterm-256color + (which is now the default Fedora since Fedora 18), the meta key is + used to enable support of 8 bit characters (ANSI sequence +- "\033[1034h"). */ ++ "\033[1034h"). ++ ++ With libedit, this call makes readline() crash. */ + rl_variable_bind ("enable-meta-key", "off"); + } ++#endif + + /* Initialize (allows .inputrc to override) + * + + +# HG changeset patch +# User Antoine Pitrou +# Date 1415109130 -3600 +# Node ID eba6e68e818c694e499dfc4b22dde095d2557ab1 +# Parent e54d0b197c8245bd29ea09f421e2f1da47370f41 +Issue #22773: fix failing test with old readline versions due to issue #19884. + +diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py +--- a/Lib/test/test_readline.py ++++ b/Lib/test/test_readline.py +@@ -43,6 +43,10 @@ class TestHistoryManipulation (unittest. + + + class TestReadline(unittest.TestCase): ++ ++ @unittest.skipIf(readline._READLINE_VERSION < 0x0600 ++ and "libedit" not in readline.__doc__, ++ "not supported in this library version") + def test_init(self): + # Issue #19884: Ensure that the ANSI sequence "\033[1034h" is not + # written into stdout when the readline module is imported and stdout +diff --git a/Modules/readline.c b/Modules/readline.c +--- a/Modules/readline.c ++++ b/Modules/readline.c +@@ -1184,4 +1184,7 @@ initreadline(void) + + PyOS_ReadlineFunctionPointer = call_readline; + setup_readline(); ++ ++ PyModule_AddIntConstant(m, "_READLINE_VERSION", RL_READLINE_VERSION); ++ PyModule_AddIntConstant(m, "_READLINE_RUNTIME_VERSION", rl_readline_version); + } + diff --git a/SOURCES/00199-CVE-2013-1753.patch b/SOURCES/00199-CVE-2013-1753.patch new file mode 100644 index 00000000..a838c1f6 --- /dev/null +++ b/SOURCES/00199-CVE-2013-1753.patch @@ -0,0 +1,88 @@ + +# HG changeset patch +# User Benjamin Peterson +# Date 1417828515 18000 +# Node ID d50096708b2d701937e78f525446d729fc28db88 +# Parent 923aac88a3cc76a95d5a04d9d3ece245147a8064 +add a default limit for the amount of data xmlrpclib.gzip_decode will return (closes #16043) + +diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py +--- a/Lib/test/test_xmlrpc.py ++++ b/Lib/test/test_xmlrpc.py +@@ -737,7 +737,7 @@ class GzipServerTestCase(BaseServerTestC + with cm: + p.pow(6, 8) + +- def test_gsip_response(self): ++ def test_gzip_response(self): + t = self.Transport() + p = xmlrpclib.ServerProxy(URL, transport=t) + old = self.requestHandler.encode_threshold +@@ -750,6 +750,23 @@ class GzipServerTestCase(BaseServerTestC + self.requestHandler.encode_threshold = old + self.assertTrue(a>b) + ++ def test_gzip_decode_limit(self): ++ max_gzip_decode = 20 * 1024 * 1024 ++ data = '\0' * max_gzip_decode ++ encoded = xmlrpclib.gzip_encode(data) ++ decoded = xmlrpclib.gzip_decode(encoded) ++ self.assertEqual(len(decoded), max_gzip_decode) ++ ++ data = '\0' * (max_gzip_decode + 1) ++ encoded = xmlrpclib.gzip_encode(data) ++ ++ with self.assertRaisesRegexp(ValueError, ++ "max gzipped payload length exceeded"): ++ xmlrpclib.gzip_decode(encoded) ++ ++ xmlrpclib.gzip_decode(encoded, max_decode=-1) ++ ++ + #Test special attributes of the ServerProxy object + class ServerProxyTestCase(unittest.TestCase): + def setUp(self): +diff --git a/Lib/xmlrpclib.py b/Lib/xmlrpclib.py +--- a/Lib/xmlrpclib.py ++++ b/Lib/xmlrpclib.py +@@ -49,6 +49,7 @@ + # 2003-07-12 gp Correct marshalling of Faults + # 2003-10-31 mvl Add multicall support + # 2004-08-20 mvl Bump minimum supported Python version to 2.1 ++# 2014-12-02 ch/doko Add workaround for gzip bomb vulnerability + # + # Copyright (c) 1999-2002 by Secret Labs AB. + # Copyright (c) 1999-2002 by Fredrik Lundh. +@@ -1165,10 +1166,13 @@ def gzip_encode(data): + # in the HTTP header, as described in RFC 1952 + # + # @param data The encoded data ++# @keyparam max_decode Maximum bytes to decode (20MB default), use negative ++# values for unlimited decoding + # @return the unencoded data + # @raises ValueError if data is not correctly coded. ++# @raises ValueError if max gzipped payload length exceeded + +-def gzip_decode(data): ++def gzip_decode(data, max_decode=20971520): + """gzip encoded data -> unencoded data + + Decode data using the gzip content encoding as described in RFC 1952 +@@ -1178,11 +1182,16 @@ def gzip_decode(data): + f = StringIO.StringIO(data) + gzf = gzip.GzipFile(mode="rb", fileobj=f) + try: +- decoded = gzf.read() ++ if max_decode < 0: # no limit ++ decoded = gzf.read() ++ else: ++ decoded = gzf.read(max_decode + 1) + except IOError: + raise ValueError("invalid data") + f.close() + gzf.close() ++ if max_decode >= 0 and len(decoded) > max_decode: ++ raise ValueError("max gzipped payload length exceeded") + return decoded + + ## diff --git a/SOURCES/00200-CVE-2014-4616.patch b/SOURCES/00200-CVE-2014-4616.patch new file mode 100644 index 00000000..c60831e2 --- /dev/null +++ b/SOURCES/00200-CVE-2014-4616.patch @@ -0,0 +1,52 @@ + +# HG changeset patch +# User Benjamin Peterson +# Date 1397441438 14400 +# Node ID 50c07ed1743da9cd4540d83de0c30bd17aeb41b0 +# Parent 218e28a935ab4494d05215c243e2129625a71893 +in scan_once, prevent the reading of arbitrary memory when passed a negative index + +Bug reported by Guido Vranken. + +diff --git a/Lib/json/tests/test_decode.py b/Lib/json/tests/test_decode.py +--- a/Lib/json/tests/test_decode.py ++++ b/Lib/json/tests/test_decode.py +@@ -60,5 +60,10 @@ class TestDecode(object): + msg = 'escape' + self.assertRaisesRegexp(ValueError, msg, self.loads, s) + ++ def test_negative_index(self): ++ d = self.json.JSONDecoder() ++ self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000) ++ self.assertRaises(ValueError, d.raw_decode, u'a'*42, -50000) ++ + class TestPyDecode(TestDecode, PyTest): pass + class TestCDecode(TestDecode, CTest): pass +diff --git a/Modules/_json.c b/Modules/_json.c +--- a/Modules/_json.c ++++ b/Modules/_json.c +@@ -1468,7 +1468,10 @@ scan_once_str(PyScannerObject *s, PyObje + PyObject *res; + char *str = PyString_AS_STRING(pystr); + Py_ssize_t length = PyString_GET_SIZE(pystr); +- if (idx >= length) { ++ if (idx < 0) ++ /* Compatibility with the Python version. */ ++ idx += length; ++ if (idx < 0 || idx >= length) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } +@@ -1555,7 +1558,10 @@ scan_once_unicode(PyScannerObject *s, Py + PyObject *res; + Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); + Py_ssize_t length = PyUnicode_GET_SIZE(pystr); +- if (idx >= length) { ++ if (idx < 0) ++ /* Compatibility with Python version. */ ++ idx += length; ++ if (idx < 0 || idx >= length) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + diff --git a/SOURCES/00201-CVE-2014-4650.patch b/SOURCES/00201-CVE-2014-4650.patch new file mode 100644 index 00000000..031c8592 --- /dev/null +++ b/SOURCES/00201-CVE-2014-4650.patch @@ -0,0 +1,35 @@ + +# HG changeset patch +# User Benjamin Peterson +# Date 1402796189 25200 +# Node ID b4bab078876811c7d95231d08aa6fa7142fdda66 +# Parent bb8b0c7fefd0c5ed99b3f336178a4f9554a1d0ef +url unquote the path before checking if it refers to a CGI script (closes #21766) + +diff --git a/Lib/CGIHTTPServer.py b/Lib/CGIHTTPServer.py +--- a/Lib/CGIHTTPServer.py ++++ b/Lib/CGIHTTPServer.py +@@ -84,7 +84,7 @@ class CGIHTTPRequestHandler(SimpleHTTPSe + path begins with one of the strings in self.cgi_directories + (and the next character is a '/' or the end of the string). + """ +- collapsed_path = _url_collapse_path(self.path) ++ collapsed_path = _url_collapse_path(urllib.unquote(self.path)) + dir_sep = collapsed_path.find('/', 1) + head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:] + if head in self.cgi_directories: +diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py +--- a/Lib/test/test_httpservers.py ++++ b/Lib/test/test_httpservers.py +@@ -510,6 +510,11 @@ class CGIHTTPServerTestCase(BaseTestCase + (res.read(), res.getheader('Content-type'), res.status)) + self.assertEqual(os.environ['SERVER_SOFTWARE'], signature) + ++ def test_urlquote_decoding_in_cgi_check(self): ++ res = self.request('/cgi-bin%2ffile1.py') ++ self.assertEqual((b'Hello World\n', 'text/html', 200), ++ (res.read(), res.getheader('Content-type'), res.status)) ++ + + class SimpleHTTPRequestHandlerTestCase(unittest.TestCase): + """ Test url parsing """ diff --git a/SOURCES/00202-CVE-2014-7185.patch b/SOURCES/00202-CVE-2014-7185.patch new file mode 100644 index 00000000..8ddb798c --- /dev/null +++ b/SOURCES/00202-CVE-2014-7185.patch @@ -0,0 +1,51 @@ + +# HG changeset patch +# User Benjamin Peterson +# Date 1403579547 25200 +# Node ID 8d963c7db507be561e26bbbb852e3a2be3327c3f +# Parent 8e0b7393e921fb5e05c40265f9272dec90512ef6 +avoid overflow with large buffer sizes and/or offsets (closes #21831) + +diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py +--- a/Lib/test/test_buffer.py ++++ b/Lib/test/test_buffer.py +@@ -4,6 +4,7 @@ For now, tests just new or changed funct + + """ + ++import sys + import unittest + from test import test_support + +@@ -29,6 +30,11 @@ class BufferTests(unittest.TestCase): + m = memoryview(b) # Should not raise an exception + self.assertEqual(m.tobytes(), s) + ++ def test_large_buffer_size_and_offset(self): ++ data = bytearray('hola mundo') ++ buf = buffer(data, sys.maxsize, sys.maxsize) ++ self.assertEqual(buf[:4096], "") ++ + + def test_main(): + with test_support.check_py3k_warnings(("buffer.. not supported", +diff --git a/Objects/bufferobject.c b/Objects/bufferobject.c +--- a/Objects/bufferobject.c ++++ b/Objects/bufferobject.c +@@ -88,7 +88,7 @@ get_buf(PyBufferObject *self, void **ptr + *size = count; + else + *size = self->b_size; +- if (offset + *size > count) ++ if (*size > count - offset) + *size = count - offset; + } + return 1; +@@ -875,4 +875,4 @@ PyTypeObject PyBuffer_Type = { + 0, /* tp_init */ + 0, /* tp_alloc */ + buffer_new, /* tp_new */ +-}; +\ No newline at end of file ++}; + diff --git a/SOURCES/00203-CVE-2013-1752-nntplib.patch b/SOURCES/00203-CVE-2013-1752-nntplib.patch new file mode 100644 index 00000000..37c6f124 --- /dev/null +++ b/SOURCES/00203-CVE-2013-1752-nntplib.patch @@ -0,0 +1,108 @@ + +# HG changeset patch +# User Barry Warsaw +# Date 1380582569 14400 +# Node ID 36680a7c0e22686df9c338a9ca3cdb2c60e05b27 +# Parent 0f5611bca5a284c0b5f978e83a05818f0907bda8# Parent 731abf7834c43efb321231e65e7dd76ad9e8e661 +- Issue #16040: CVE-2013-1752: nntplib: Limit maximum line lengths to 2048 to + prevent readline() calls from consuming too much memory. Patch by Jyrki + Pulliainen. + +diff --git a/Lib/nntplib.py b/Lib/nntplib.py +--- a/Lib/nntplib.py ++++ b/Lib/nntplib.py +@@ -37,6 +37,13 @@ import socket + "error_reply","error_temp","error_perm","error_proto", + "error_data",] + ++# maximal line length when calling readline(). This is to prevent ++# reading arbitrary lenght lines. RFC 3977 limits NNTP line length to ++# 512 characters, including CRLF. We have selected 2048 just to be on ++# the safe side. ++_MAXLINE = 2048 ++ ++ + # Exceptions raised when an error or invalid response is received + class NNTPError(Exception): + """Base class for all nntplib exceptions""" +@@ -200,7 +207,9 @@ class NNTP: + def getline(self): + """Internal: return one line from the server, stripping CRLF. + Raise EOFError if the connection is closed.""" +- line = self.file.readline() ++ line = self.file.readline(_MAXLINE + 1) ++ if len(line) > _MAXLINE: ++ raise NNTPDataError('line too long') + if self.debugging > 1: + print '*get*', repr(line) + if not line: raise EOFError +diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py +new file mode 100644 +--- /dev/null ++++ b/Lib/test/test_nntplib.py +@@ -0,0 +1,65 @@ ++import socket ++import threading ++import nntplib ++import time ++ ++from unittest import TestCase ++from test import test_support ++ ++HOST = test_support.HOST ++ ++ ++def server(evt, serv, evil=False): ++ serv.listen(5) ++ try: ++ conn, addr = serv.accept() ++ except socket.timeout: ++ pass ++ else: ++ if evil: ++ conn.send("1 I'm too long response" * 3000 + "\n") ++ else: ++ conn.send("1 I'm OK response\n") ++ conn.close() ++ finally: ++ serv.close() ++ evt.set() ++ ++ ++class BaseServerTest(TestCase): ++ def setUp(self): ++ self.evt = threading.Event() ++ self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) ++ self.sock.settimeout(3) ++ self.port = test_support.bind_port(self.sock) ++ threading.Thread( ++ target=server, ++ args=(self.evt, self.sock, self.evil)).start() ++ time.sleep(.1) ++ ++ def tearDown(self): ++ self.evt.wait() ++ ++ ++class ServerTests(BaseServerTest): ++ evil = False ++ ++ def test_basic_connect(self): ++ nntp = nntplib.NNTP('localhost', self.port) ++ nntp.sock.close() ++ ++ ++class EvilServerTests(BaseServerTest): ++ evil = True ++ ++ def test_too_long_line(self): ++ self.assertRaises(nntplib.NNTPDataError, ++ nntplib.NNTP, 'localhost', self.port) ++ ++ ++def test_main(verbose=None): ++ test_support.run_unittest(EvilServerTests) ++ test_support.run_unittest(ServerTests) ++ ++if __name__ == '__main__': ++ test_main() diff --git a/SOURCES/00204-CVE-2013-1752-ftplib.patch b/SOURCES/00204-CVE-2013-1752-ftplib.patch new file mode 100644 index 00000000..97c890e7 --- /dev/null +++ b/SOURCES/00204-CVE-2013-1752-ftplib.patch @@ -0,0 +1,149 @@ + +# HG changeset patch +# User Serhiy Storchaka +# Date 1382277427 -10800 +# Node ID 44ac81e6d584758ee56a865a7c18d82505be0643 +# Parent 625ece68d79a27d376889579c414ed4b2d8a2649 +Issue #16038: CVE-2013-1752: ftplib: Limit amount of data read by +limiting the call to readline(). Original patch by Michał +Jastrzębski and Giampaolo Rodola. + +diff --git a/Lib/ftplib.py b/Lib/ftplib.py +--- a/Lib/ftplib.py ++++ b/Lib/ftplib.py +@@ -55,6 +55,8 @@ MSG_OOB = 0x1 + + # The standard FTP server control port + FTP_PORT = 21 ++# The sizehint parameter passed to readline() calls ++MAXLINE = 8192 + + + # Exception raised when an error or invalid response is received +@@ -101,6 +103,7 @@ class FTP: + debugging = 0 + host = '' + port = FTP_PORT ++ maxline = MAXLINE + sock = None + file = None + welcome = None +@@ -180,7 +183,9 @@ class FTP: + # Internal: return one line from the server, stripping CRLF. + # Raise EOFError if the connection is closed + def getline(self): +- line = self.file.readline() ++ line = self.file.readline(self.maxline + 1) ++ if len(line) > self.maxline: ++ raise Error("got more than %d bytes" % self.maxline) + if self.debugging > 1: + print '*get*', self.sanitize(line) + if not line: raise EOFError +@@ -432,7 +437,9 @@ class FTP: + conn = self.transfercmd(cmd) + fp = conn.makefile('rb') + while 1: +- line = fp.readline() ++ line = fp.readline(self.maxline + 1) ++ if len(line) > self.maxline: ++ raise Error("got more than %d bytes" % self.maxline) + if self.debugging > 2: print '*retr*', repr(line) + if not line: + break +@@ -485,7 +492,9 @@ class FTP: + self.voidcmd('TYPE A') + conn = self.transfercmd(cmd) + while 1: +- buf = fp.readline() ++ buf = fp.readline(self.maxline + 1) ++ if len(buf) > self.maxline: ++ raise Error("got more than %d bytes" % self.maxline) + if not buf: break + if buf[-2:] != CRLF: + if buf[-1] in CRLF: buf = buf[:-1] +@@ -710,7 +719,9 @@ else: + fp = conn.makefile('rb') + try: + while 1: +- line = fp.readline() ++ line = fp.readline(self.maxline + 1) ++ if len(line) > self.maxline: ++ raise Error("got more than %d bytes" % self.maxline) + if self.debugging > 2: print '*retr*', repr(line) + if not line: + break +@@ -748,7 +759,9 @@ else: + conn = self.transfercmd(cmd) + try: + while 1: +- buf = fp.readline() ++ buf = fp.readline(self.maxline + 1) ++ if len(buf) > self.maxline: ++ raise Error("got more than %d bytes" % self.maxline) + if not buf: break + if buf[-2:] != CRLF: + if buf[-1] in CRLF: buf = buf[:-1] +@@ -905,7 +918,9 @@ class Netrc: + fp = open(filename, "r") + in_macro = 0 + while 1: +- line = fp.readline() ++ line = fp.readline(self.maxline + 1) ++ if len(line) > self.maxline: ++ raise Error("got more than %d bytes" % self.maxline) + if not line: break + if in_macro and line.strip(): + macro_lines.append(line) +diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py +--- a/Lib/test/test_ftplib.py ++++ b/Lib/test/test_ftplib.py +@@ -65,6 +65,7 @@ class DummyFTPHandler(asynchat.async_cha + self.last_received_data = '' + self.next_response = '' + self.rest = None ++ self.next_retr_data = RETR_DATA + self.push('220 welcome') + + def collect_incoming_data(self, data): +@@ -189,7 +190,7 @@ class DummyFTPHandler(asynchat.async_cha + offset = int(self.rest) + else: + offset = 0 +- self.dtp.push(RETR_DATA[offset:]) ++ self.dtp.push(self.next_retr_data[offset:]) + self.dtp.close_when_done() + self.rest = None + +@@ -203,6 +204,11 @@ class DummyFTPHandler(asynchat.async_cha + self.dtp.push(NLST_DATA) + self.dtp.close_when_done() + ++ def cmd_setlongretr(self, arg): ++ # For testing. Next RETR will return long line. ++ self.next_retr_data = 'x' * int(arg) ++ self.push('125 setlongretr ok') ++ + + class DummyFTPServer(asyncore.dispatcher, threading.Thread): + +@@ -558,6 +564,20 @@ class TestFTPClass(TestCase): + # IPv4 is in use, just make sure send_epsv has not been used + self.assertEqual(self.server.handler.last_received_cmd, 'pasv') + ++ def test_line_too_long(self): ++ self.assertRaises(ftplib.Error, self.client.sendcmd, ++ 'x' * self.client.maxline * 2) ++ ++ def test_retrlines_too_long(self): ++ self.client.sendcmd('SETLONGRETR %d' % (self.client.maxline * 2)) ++ received = [] ++ self.assertRaises(ftplib.Error, ++ self.client.retrlines, 'retr', received.append) ++ ++ def test_storlines_too_long(self): ++ f = StringIO.StringIO('x' * self.client.maxline * 2) ++ self.assertRaises(ftplib.Error, self.client.storlines, 'stor', f) ++ + + class TestIPv6Environment(TestCase): + diff --git a/SOURCES/00205-CVE-2013-1752-httplib-headers.patch b/SOURCES/00205-CVE-2013-1752-httplib-headers.patch new file mode 100644 index 00000000..b6e01474 --- /dev/null +++ b/SOURCES/00205-CVE-2013-1752-httplib-headers.patch @@ -0,0 +1,51 @@ + +# HG changeset patch +# User Berker Peksag +# Date 1407212157 -10800 +# Node ID 5e310c6a8520603bca8bc4b40eaf4f074db47c0d +# Parent 46c7a724b487295257423a69478392cb01ce74e6 +Issue #16037: HTTPMessage.readheaders() raises an HTTPException when more +than 100 headers are read. + +Patch by Jyrki Pulliainen and Daniel Eriksson. + +diff --git a/Lib/httplib.py b/Lib/httplib.py +--- a/Lib/httplib.py ++++ b/Lib/httplib.py +@@ -215,6 +215,10 @@ MAXAMOUNT = 1048576 + # maximal line length when calling readline(). + _MAXLINE = 65536 + ++# maximum amount of headers accepted ++_MAXHEADERS = 100 ++ ++ + class HTTPMessage(mimetools.Message): + + def addheader(self, key, value): +@@ -271,6 +275,8 @@ class HTTPMessage(mimetools.Message): + elif self.seekable: + tell = self.fp.tell + while True: ++ if len(hlist) > _MAXHEADERS: ++ raise HTTPException("got more than %d headers" % _MAXHEADERS) + if tell: + try: + startofline = tell() +diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py +--- a/Lib/test/test_httplib.py ++++ b/Lib/test/test_httplib.py +@@ -262,6 +262,13 @@ class BasicTest(TestCase): + if resp.read() != "": + self.fail("Did not expect response from HEAD request") + ++ def test_too_many_headers(self): ++ headers = '\r\n'.join('Header%d: foo' % i for i in xrange(200)) + '\r\n' ++ text = ('HTTP/1.1 200 OK\r\n' + headers) ++ s = FakeSocket(text) ++ r = httplib.HTTPResponse(s) ++ self.assertRaises(httplib.HTTPException, r.begin) ++ + def test_send_file(self): + expected = 'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \ + 'Accept-Encoding: identity\r\nContent-Length:' diff --git a/SOURCES/00206-CVE-2013-1752-poplib.patch b/SOURCES/00206-CVE-2013-1752-poplib.patch new file mode 100644 index 00000000..f20c1111 --- /dev/null +++ b/SOURCES/00206-CVE-2013-1752-poplib.patch @@ -0,0 +1,60 @@ + +# HG changeset patch +# User Benjamin Peterson +# Date 1417827758 18000 +# Node ID 339f877cca115c1901f5dd93d7bc066031d2a669 +# Parent 54af094087953f4997a4ead63e949d845c4b4412 +in poplib, limit maximum line length that we read from the network (closes #16041) + +Patch from Berker Peksag. + +diff --git a/Lib/poplib.py b/Lib/poplib.py +--- a/Lib/poplib.py ++++ b/Lib/poplib.py +@@ -32,6 +32,12 @@ CR = '\r' + LF = '\n' + CRLF = CR+LF + ++# maximal line length when calling readline(). This is to prevent ++# reading arbitrary length lines. RFC 1939 limits POP3 line length to ++# 512 characters, including CRLF. We have selected 2048 just to be on ++# the safe side. ++_MAXLINE = 2048 ++ + + class POP3: + +@@ -103,7 +109,9 @@ class POP3: + # Raise error_proto('-ERR EOF') if the connection is closed. + + def _getline(self): +- line = self.file.readline() ++ line = self.file.readline(_MAXLINE + 1) ++ if len(line) > _MAXLINE: ++ raise error_proto('line too long') + if self._debugging > 1: print '*get*', repr(line) + if not line: raise error_proto('-ERR EOF') + octets = len(line) +@@ -365,6 +373,8 @@ else: + match = renewline.match(self.buffer) + while not match: + self._fillBuffer() ++ if len(self.buffer) > _MAXLINE: ++ raise error_proto('line too long') + match = renewline.match(self.buffer) + line = match.group(0) + self.buffer = renewline.sub('' ,self.buffer, 1) +diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py +--- a/Lib/test/test_poplib.py ++++ b/Lib/test/test_poplib.py +@@ -198,6 +198,10 @@ class TestPOP3Class(TestCase): + 113) + self.assertEqual(self.client.retr('foo'), expected) + ++ def test_too_long_lines(self): ++ self.assertRaises(poplib.error_proto, self.client._shortcmd, ++ 'echo +%s' % ((poplib._MAXLINE + 10) * 'a')) ++ + def test_dele(self): + self.assertOK(self.client.dele('foo')) + diff --git a/SOURCES/00207-CVE-2013-1752-smtplib.patch b/SOURCES/00207-CVE-2013-1752-smtplib.patch new file mode 100644 index 00000000..11b08195 --- /dev/null +++ b/SOURCES/00207-CVE-2013-1752-smtplib.patch @@ -0,0 +1,100 @@ + +# HG changeset patch +# User Benjamin Peterson +# Date 1417827918 18000 +# Node ID 923aac88a3cc76a95d5a04d9d3ece245147a8064 +# Parent 339f877cca115c1901f5dd93d7bc066031d2a669 +smtplib: limit amount read from the network (closes #16042) + +diff --git a/Lib/smtplib.py b/Lib/smtplib.py +--- a/Lib/smtplib.py ++++ b/Lib/smtplib.py +@@ -57,6 +57,7 @@ from sys import stderr + SMTP_PORT = 25 + SMTP_SSL_PORT = 465 + CRLF = "\r\n" ++_MAXLINE = 8192 # more than 8 times larger than RFC 821, 4.5.3 + + OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I) + +@@ -179,10 +180,14 @@ else: + def __init__(self, sslobj): + self.sslobj = sslobj + +- def readline(self): ++ def readline(self, size=-1): ++ if size < 0: ++ size = None + str = "" + chr = None + while chr != "\n": ++ if size is not None and len(str) >= size: ++ break + chr = self.sslobj.read(1) + if not chr: + break +@@ -353,7 +358,7 @@ class SMTP: + self.file = self.sock.makefile('rb') + while 1: + try: +- line = self.file.readline() ++ line = self.file.readline(_MAXLINE + 1) + except socket.error as e: + self.close() + raise SMTPServerDisconnected("Connection unexpectedly closed: " +@@ -363,6 +368,8 @@ class SMTP: + raise SMTPServerDisconnected("Connection unexpectedly closed") + if self.debuglevel > 0: + print>>stderr, 'reply:', repr(line) ++ if len(line) > _MAXLINE: ++ raise SMTPResponseException(500, "Line too long.") + resp.append(line[4:].strip()) + code = line[:3] + # Check that the error code is syntactically correct. +diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py +--- a/Lib/test/test_smtplib.py ++++ b/Lib/test/test_smtplib.py +@@ -292,6 +292,33 @@ class BadHELOServerTests(unittest.TestCa + HOST, self.port, 'localhost', 3) + + ++@unittest.skipUnless(threading, 'Threading required for this test.') ++class TooLongLineTests(unittest.TestCase): ++ respdata = '250 OK' + ('.' * smtplib._MAXLINE * 2) + '\n' ++ ++ def setUp(self): ++ self.old_stdout = sys.stdout ++ self.output = StringIO.StringIO() ++ sys.stdout = self.output ++ ++ self.evt = threading.Event() ++ self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) ++ self.sock.settimeout(15) ++ self.port = test_support.bind_port(self.sock) ++ servargs = (self.evt, self.respdata, self.sock) ++ threading.Thread(target=server, args=servargs).start() ++ self.evt.wait() ++ self.evt.clear() ++ ++ def tearDown(self): ++ self.evt.wait() ++ sys.stdout = self.old_stdout ++ ++ def testLineTooLong(self): ++ self.assertRaises(smtplib.SMTPResponseException, smtplib.SMTP, ++ HOST, self.port, 'localhost', 3) ++ ++ + sim_users = {'Mr.A@somewhere.com':'John A', + 'Ms.B@somewhere.com':'Sally B', + 'Mrs.C@somewhereesle.com':'Ruth C', +@@ -526,7 +553,8 @@ class SMTPSimTests(unittest.TestCase): + def test_main(verbose=None): + test_support.run_unittest(GeneralTests, DebuggingServerTests, + NonConnectingTests, +- BadHELOServerTests, SMTPSimTests) ++ BadHELOServerTests, SMTPSimTests, ++ TooLongLineTests) + + if __name__ == '__main__': + test_main() diff --git a/SOURCES/00208-CVE-2013-1752-imaplib.patch b/SOURCES/00208-CVE-2013-1752-imaplib.patch new file mode 100644 index 00000000..91395d6d --- /dev/null +++ b/SOURCES/00208-CVE-2013-1752-imaplib.patch @@ -0,0 +1,59 @@ + +# HG changeset patch +# User R David Murray +# Date 1388775562 18000 +# Node ID dd906f4ab9237020a7a275c2d361fa288e553481 +# Parent 69b5f692455306c98aa27ecea17e6290787ebd3f +closes 16039: CVE-2013-1752: limit line length in imaplib readline calls. + +diff --git a/Lib/imaplib.py b/Lib/imaplib.py +--- a/Lib/imaplib.py ++++ b/Lib/imaplib.py +@@ -35,6 +35,15 @@ IMAP4_PORT = 143 + IMAP4_SSL_PORT = 993 + AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first + ++# Maximal line length when calling readline(). This is to prevent ++# reading arbitrary length lines. RFC 3501 and 2060 (IMAP 4rev1) ++# don't specify a line length. RFC 2683 however suggests limiting client ++# command lines to 1000 octets and server command lines to 8000 octets. ++# We have selected 10000 for some extra margin and since that is supposedly ++# also what UW and Panda IMAP does. ++_MAXLINE = 10000 ++ ++ + # Commands + + Commands = { +@@ -237,7 +246,10 @@ class IMAP4: + + def readline(self): + """Read line from remote.""" +- return self.file.readline() ++ line = self.file.readline(_MAXLINE + 1) ++ if len(line) > _MAXLINE: ++ raise self.error("got more than %d bytes" % _MAXLINE) ++ return line + + + def send(self, data): +diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py +--- a/Lib/test/test_imaplib.py ++++ b/Lib/test/test_imaplib.py +@@ -165,6 +165,16 @@ class BaseThreadedNetworkedTests(unittes + self.imap_class, *server.server_address) + + ++ def test_linetoolong(self): ++ class TooLongHandler(SimpleIMAPHandler): ++ def handle(self): ++ # Send a very long response line ++ self.wfile.write('* OK ' + imaplib._MAXLINE*'x' + '\r\n') ++ ++ with self.reaped_server(TooLongHandler) as server: ++ self.assertRaises(imaplib.IMAP4.error, ++ self.imap_class, *server.server_address) ++ + class ThreadedNetworkedTests(BaseThreadedNetworkedTests): + + server_class = SocketServer.TCPServer diff --git a/SOURCES/00256-fix-incorrect-parsing-of-regular-expressions.patch b/SOURCES/00256-fix-incorrect-parsing-of-regular-expressions.patch new file mode 100644 index 00000000..9c467b81 --- /dev/null +++ b/SOURCES/00256-fix-incorrect-parsing-of-regular-expressions.patch @@ -0,0 +1,80 @@ +diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py +index 7cda2b6..15d2324 100644 +--- a/Lib/sre_compile.py ++++ b/Lib/sre_compile.py +@@ -355,8 +355,6 @@ def _optimize_unicode(charset, fixup): + def _simple(av): + # check if av is a "simple" operator + lo, hi = av[2].getwidth() +- if lo == 0 and hi == MAXREPEAT: +- raise error, "nothing to repeat" + return lo == hi == 1 and av[2][0][0] != SUBPATTERN + + def _compile_info(code, pattern, flags): +diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py +index 75f8c96..644441d 100644 +--- a/Lib/sre_parse.py ++++ b/Lib/sre_parse.py +@@ -147,7 +147,7 @@ class SubPattern: + REPEATCODES = (MIN_REPEAT, MAX_REPEAT) + for op, av in self.data: + if op is BRANCH: +- i = sys.maxint ++ i = MAXREPEAT - 1 + j = 0 + for av in av[1]: + l, h = av.getwidth() +@@ -165,14 +165,14 @@ class SubPattern: + hi = hi + j + elif op in REPEATCODES: + i, j = av[2].getwidth() +- lo = lo + long(i) * av[0] +- hi = hi + long(j) * av[1] ++ lo = lo + i * av[0] ++ hi = hi + j * av[1] + elif op in UNITCODES: + lo = lo + 1 + hi = hi + 1 + elif op == SUCCESS: + break +- self.width = int(min(lo, sys.maxint)), int(min(hi, sys.maxint)) ++ self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT) + return self.width + + class Tokenizer: +diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py +index 18a81a2..f0827d8 100644 +--- a/Lib/test/test_re.py ++++ b/Lib/test/test_re.py +@@ -897,6 +897,17 @@ class ReTests(unittest.TestCase): + with self.assertRaisesRegexp(sre_constants.error, '\?foo'): + re.compile('(?P)') + ++ def test_bug_2537(self): ++ # issue 2537: empty submatches ++ for outer_op in ('{0,}', '*', '+', '{1,187}'): ++ for inner_op in ('{0,}', '*', '?'): ++ r = re.compile("^((x|y)%s)%s" % (inner_op, outer_op)) ++ m = r.match("xyyzy") ++ self.assertEqual(m.group(0), "xyy") ++ self.assertEqual(m.group(1), "") ++ self.assertEqual(m.group(2), "y") ++ ++ + + def run_re_tests(): + from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR +diff --git a/Lib/doctest.py b/Lib/doctest.py +index 90bcca1..0ee40a2 100644 +--- a/Lib/doctest.py ++++ b/Lib/doctest.py +@@ -564,7 +564,7 @@ class DocTestParser: + # Want consists of any non-blank lines that do not start with PS1. + (?P (?:(?![ ]*$) # Not a blank line + (?![ ]*>>>) # Not a line starting with PS1 +- .*$\n? # But any other line ++ .+$\n? # But any other line + )*) + ''', re.MULTILINE | re.VERBOSE) + + diff --git a/SOURCES/00310-use-xml-sethashsalt-in-elementtree.patch b/SOURCES/00310-use-xml-sethashsalt-in-elementtree.patch new file mode 100644 index 00000000..d3363610 --- /dev/null +++ b/SOURCES/00310-use-xml-sethashsalt-in-elementtree.patch @@ -0,0 +1,54 @@ +diff --git a/Include/pyexpat.h b/Include/pyexpat.h +index 5340ef5..3fc5fa5 100644 +--- a/Include/pyexpat.h ++++ b/Include/pyexpat.h +@@ -3,7 +3,7 @@ + + /* note: you must import expat.h before importing this module! */ + +-#define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.0" ++#define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.1" + #define PyExpat_CAPSULE_NAME "pyexpat.expat_CAPI" + + struct PyExpat_CAPI +@@ -43,6 +43,8 @@ struct PyExpat_CAPI + XML_Parser parser, XML_UnknownEncodingHandler handler, + void *encodingHandlerData); + void (*SetUserData)(XML_Parser parser, void *userData); ++ /* might be none for expat < 2.1.0 */ ++ int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); + /* always add new stuff to the end! */ + }; + +diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c +index 379aa01..ce62081 100644 +--- a/Modules/_elementtree.c ++++ b/Modules/_elementtree.c +@@ -2500,6 +2500,11 @@ xmlparser(PyObject* self_, PyObject* args, PyObject* kw) + PyErr_NoMemory(); + return NULL; + } ++ /* expat < 2.1.0 has no XML_SetHashSalt() */ ++ if (EXPAT(SetHashSalt) != NULL) { ++ EXPAT(SetHashSalt)(self->parser, ++ (unsigned long)_Py_HashSecret.prefix); ++ } + + /* setup target handlers */ + if (!target) { +diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c +index f269113..147b8a9 100644 +--- a/Modules/pyexpat.c ++++ b/Modules/pyexpat.c +@@ -2037,6 +2037,11 @@ MODULE_INITFUNC(void) + capi.SetProcessingInstructionHandler = XML_SetProcessingInstructionHandler; + capi.SetUnknownEncodingHandler = XML_SetUnknownEncodingHandler; + capi.SetUserData = XML_SetUserData; ++#if XML_COMBINED_VERSION >= 20100 ++ capi.SetHashSalt = XML_SetHashSalt; ++#else ++ capi.SetHashSalt = NULL; ++#endif + + /* export using capsule */ + capi_object = PyCapsule_New(&capi, PyExpat_CAPSULE_NAME, NULL); diff --git a/SOURCES/00314-parser-check-e_io.patch b/SOURCES/00314-parser-check-e_io.patch new file mode 100644 index 00000000..2119f584 --- /dev/null +++ b/SOURCES/00314-parser-check-e_io.patch @@ -0,0 +1,56 @@ +commit bcd39b7b9bd3a7f8a6a34410169794a6264a6fed +Author: Victor Stinner +Date: Wed Nov 7 00:45:13 2018 +0100 + + bpo-25083: Python can sometimes create incorrect .pyc files + + Python 2 never checked for I/O error when reading .py files and + thus could mistake an I/O error for EOF and create incorrect .pyc + files. This adds an check for this and aborts on an error. + + Patch by tzickel, commit f64c813de84011a84ca21d75a294861a9cc2dfdc. + + Resolves: rhbz#1629982 + +diff --git a/Include/errcode.h b/Include/errcode.h +index becec80..5c5a0f7 100644 +--- a/Include/errcode.h ++++ b/Include/errcode.h +@@ -29,6 +29,7 @@ extern "C" { + #define E_EOFS 23 /* EOF in triple-quoted string */ + #define E_EOLS 24 /* EOL in single-quoted string */ + #define E_LINECONT 25 /* Unexpected characters after a line continuation */ ++#define E_IO 26 /* I/O error */ + + #ifdef __cplusplus + } +diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c +index ee6313b..0217f2b 100644 +--- a/Parser/tokenizer.c ++++ b/Parser/tokenizer.c +@@ -1644,6 +1644,11 @@ int + PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end) + { + int result = tok_get(tok, p_start, p_end); ++ if (tok->fp && ferror(tok->fp)) { ++ clearerr(tok->fp); ++ result = ERRORTOKEN; ++ tok->done = E_IO; ++ } + if (tok->decoding_erred) { + result = ERRORTOKEN; + tok->done = E_DECODE; +diff --git a/Python/pythonrun.c b/Python/pythonrun.c +index 0b73f3a..9f06236 100644 +--- a/Python/pythonrun.c ++++ b/Python/pythonrun.c +@@ -1643,6 +1643,9 @@ err_input(perrdetail *err) + Py_XDECREF(tb); + break; + } ++ case E_IO: ++ msg = "I/O error while reading"; ++ break; + case E_LINECONT: + msg = "unexpected character after line continuation character"; + break; diff --git a/SOURCES/00317-CVE-2019-5010-ssl-crl.patch b/SOURCES/00317-CVE-2019-5010-ssl-crl.patch new file mode 100644 index 00000000..bd53e2e6 --- /dev/null +++ b/SOURCES/00317-CVE-2019-5010-ssl-crl.patch @@ -0,0 +1,100 @@ +commit 88a31ffeccce13192a474f4981b9cf6cfdfe065e +Author: Victor Stinner +Date: Wed Mar 20 17:43:20 2019 +0100 + + bpo-35746: Fix segfault in ssl's cert parser (GH-11569) + + Fix a NULL pointer deref in ssl module. The cert parser did not handle CRL + distribution points with empty DP or URI correctly. A malicious or buggy + certificate can result into segfault. + + Signed-off-by: Christian Heimes + + https://bugs.python.org/issue35746 + (cherry picked from commit a37f52436f9aa4b9292878b72f3ff1480e2606c3) + + Co-authored-by: Christian Heimes + +diff --git a/Lib/test/talos-2019-0758.pem b/Lib/test/talos-2019-0758.pem +new file mode 100644 +index 0000000..13b95a7 +--- /dev/null ++++ b/Lib/test/talos-2019-0758.pem +@@ -0,0 +1,22 @@ ++-----BEGIN CERTIFICATE----- ++MIIDqDCCApKgAwIBAgIBAjALBgkqhkiG9w0BAQswHzELMAkGA1UEBhMCVUsxEDAO ++BgNVBAMTB2NvZHktY2EwHhcNMTgwNjE4MTgwMDU4WhcNMjgwNjE0MTgwMDU4WjA7 ++MQswCQYDVQQGEwJVSzEsMCoGA1UEAxMjY29kZW5vbWljb24tdm0tMi50ZXN0Lmxh ++bC5jaXNjby5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC63fGB ++J80A9Av1GB0bptslKRIUtJm8EeEu34HkDWbL6AJY0P8WfDtlXjlPaLqFa6sqH6ES ++V48prSm1ZUbDSVL8R6BYVYpOlK8/48xk4pGTgRzv69gf5SGtQLwHy8UPBKgjSZoD ++5a5k5wJXGswhKFFNqyyxqCvWmMnJWxXTt2XDCiWc4g4YAWi4O4+6SeeHVAV9rV7C ++1wxqjzKovVe2uZOHjKEzJbbIU6JBPb6TRfMdRdYOw98n1VXDcKVgdX2DuuqjCzHP ++WhU4Tw050M9NaK3eXp4Mh69VuiKoBGOLSOcS8reqHIU46Reg0hqeL8LIL6OhFHIF ++j7HR6V1X6F+BfRS/AgMBAAGjgdYwgdMwCQYDVR0TBAIwADAdBgNVHQ4EFgQUOktp ++HQjxDXXUg8prleY9jeLKeQ4wTwYDVR0jBEgwRoAUx6zgPygZ0ZErF9sPC4+5e2Io ++UU+hI6QhMB8xCzAJBgNVBAYTAlVLMRAwDgYDVQQDEwdjb2R5LWNhggkA1QEAuwb7 ++2s0wCQYDVR0SBAIwADAuBgNVHREEJzAlgiNjb2Rlbm9taWNvbi12bS0yLnRlc3Qu ++bGFsLmNpc2NvLmNvbTAOBgNVHQ8BAf8EBAMCBaAwCwYDVR0fBAQwAjAAMAsGCSqG ++SIb3DQEBCwOCAQEAvqantx2yBlM11RoFiCfi+AfSblXPdrIrHvccepV4pYc/yO6p ++t1f2dxHQb8rWH3i6cWag/EgIZx+HJQvo0rgPY1BFJsX1WnYf1/znZpkUBGbVmlJr ++t/dW1gSkNS6sPsM0Q+7HPgEv8CPDNK5eo7vU2seE0iWOkxSyVUuiCEY9ZVGaLVit ++p0C78nZ35Pdv4I+1cosmHl28+es1WI22rrnmdBpH8J1eY6WvUw2xuZHLeNVN0TzV ++Q3qq53AaCWuLOD1AjESWuUCxMZTK9DPS4JKXTK8RLyDeqOvJGjsSWp3kL0y3GaQ+ ++10T1rfkKJub2+m9A9duin1fn6tHc2wSvB7m3DA== ++-----END CERTIFICATE----- +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index f7a6746..31af578 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -68,6 +68,7 @@ WRONGCERT = data_file("XXXnonexisting.pem") + BADKEY = data_file("badkey.pem") + NOKIACERT = data_file("nokia.pem") + NULLBYTECERT = data_file("nullbytecert.pem") ++TALOS_INVALID_CRLDP = data_file("talos-2019-0758.pem") + + DHFILE = data_file("dh1024.pem") + BYTES_DHFILE = DHFILE.encode(sys.getfilesystemencoding()) +@@ -238,6 +239,27 @@ class BasicSocketTests(unittest.TestCase): + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + ) + ++ def test_parse_cert_CVE_2019_5010(self): ++ p = ssl._ssl._test_decode_cert(TALOS_INVALID_CRLDP) ++ if support.verbose: ++ sys.stdout.write("\n" + pprint.pformat(p) + "\n") ++ self.assertEqual( ++ p, ++ { ++ 'issuer': ( ++ (('countryName', 'UK'),), (('commonName', 'cody-ca'),)), ++ 'notAfter': 'Jun 14 18:00:58 2028 GMT', ++ 'notBefore': 'Jun 18 18:00:58 2018 GMT', ++ 'serialNumber': '02', ++ 'subject': ((('countryName', 'UK'),), ++ (('commonName', ++ 'codenomicon-vm-2.test.lal.cisco.com'),)), ++ 'subjectAltName': ( ++ ('DNS', 'codenomicon-vm-2.test.lal.cisco.com'),), ++ 'version': 3 ++ } ++ ) ++ + def test_parse_all_sans(self): + p = ssl._ssl._test_decode_cert(ALLSANFILE) + self.assertEqual(p['subjectAltName'], +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index 6220bea..baea6e1 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -1103,6 +1103,10 @@ _get_crl_dp(X509 *certificate) { + STACK_OF(GENERAL_NAME) *gns; + + dp = sk_DIST_POINT_value(dps, i); ++ if (dp->distpoint == NULL) { ++ /* Ignore empty DP value, CVE-2019-5010 */ ++ continue; ++ } + gns = dp->distpoint->name.fullname; + + for (j=0; j < sk_GENERAL_NAME_num(gns); j++) { diff --git a/SOURCES/00320-CVE-2019-9636-and-CVE-2019-10160.patch b/SOURCES/00320-CVE-2019-9636-and-CVE-2019-10160.patch new file mode 100644 index 00000000..60674972 --- /dev/null +++ b/SOURCES/00320-CVE-2019-9636-and-CVE-2019-10160.patch @@ -0,0 +1,156 @@ +diff --git a/Doc/library/urlparse.rst b/Doc/library/urlparse.rst +index efd112d..61022f7 100644 +--- a/Doc/library/urlparse.rst ++++ b/Doc/library/urlparse.rst +@@ -118,6 +118,12 @@ The :mod:`urlparse` module defines the following functions: + See section :ref:`urlparse-result-object` for more information on the result + object. + ++ Characters in the :attr:`netloc` attribute that decompose under NFKC ++ normalization (as used by the IDNA encoding) into any of ``/``, ``?``, ++ ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is ++ decomposed before parsing, or is not a Unicode string, no error will be ++ raised. ++ + .. versionchanged:: 2.5 + Added attributes to return value. + +@@ -125,6 +131,11 @@ The :mod:`urlparse` module defines the following functions: + Added IPv6 URL parsing capabilities. + + ++ .. versionchanged:: 2.7.17 ++ Characters that affect netloc parsing under NFKC normalization will ++ now raise :exc:`ValueError`. ++ ++ + .. function:: parse_qs(qs[, keep_blank_values[, strict_parsing]]) + + Parse a query string given as a string argument (data of type +@@ -219,11 +230,21 @@ The :mod:`urlparse` module defines the following functions: + See section :ref:`urlparse-result-object` for more information on the result + object. + ++ Characters in the :attr:`netloc` attribute that decompose under NFKC ++ normalization (as used by the IDNA encoding) into any of ``/``, ``?``, ++ ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is ++ decomposed before parsing, or is not a Unicode string, no error will be ++ raised. ++ + .. versionadded:: 2.2 + + .. versionchanged:: 2.5 + Added attributes to return value. + ++ .. versionchanged:: 2.7.17 ++ Characters that affect netloc parsing under NFKC normalization will ++ now raise :exc:`ValueError`. ++ + + .. function:: urlunsplit(parts) + +diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py +index 72ebfaa..2717163 100644 +--- a/Lib/test/test_urlparse.py ++++ b/Lib/test/test_urlparse.py +@@ -1,6 +1,8 @@ + #! /usr/bin/env python + + from test import test_support ++import sys ++import unicodedata + import unittest + import urlparse + +@@ -564,6 +566,45 @@ class UrlParseTestCase(unittest.TestCase): + self.assertEqual(urlparse.urlparse("http://www.python.org:80"), + ('http','www.python.org:80','','','','')) + ++ def test_urlsplit_normalization(self): ++ # Certain characters should never occur in the netloc, ++ # including under normalization. ++ # Ensure that ALL of them are detected and cause an error ++ illegal_chars = u'/:#?@' ++ hex_chars = {'{:04X}'.format(ord(c)) for c in illegal_chars} ++ denorm_chars = [ ++ c for c in map(unichr, range(128, sys.maxunicode)) ++ if (hex_chars & set(unicodedata.decomposition(c).split())) ++ and c not in illegal_chars ++ ] ++ # Sanity check that we found at least one such character ++ self.assertIn(u'\u2100', denorm_chars) ++ self.assertIn(u'\uFF03', denorm_chars) ++ ++ # bpo-36742: Verify port separators are ignored when they ++ # existed prior to decomposition ++ urlparse.urlsplit(u'http://\u30d5\u309a:80') ++ with self.assertRaises(ValueError): ++ urlparse.urlsplit(u'http://\u30d5\u309a\ufe1380') ++ ++ for scheme in [u"http", u"https", u"ftp"]: ++ for netloc in [u"netloc{}false.netloc", u"n{}user@netloc"]: ++ for c in denorm_chars: ++ url = u"{}://{}/path".format(scheme, netloc.format(c)) ++ if test_support.verbose: ++ print "Checking %r" % url ++ with self.assertRaises(ValueError): ++ urlparse.urlsplit(url) ++ ++ # check error message: invalid netloc must be formated with repr() ++ # to get an ASCII error message ++ with self.assertRaises(ValueError) as cm: ++ urlparse.urlsplit(u'http://example.com\uFF03@bing.com') ++ self.assertEqual(str(cm.exception), ++ "netloc u'example.com\\uff03@bing.com' contains invalid characters " ++ "under NFKC normalization") ++ self.assertIsInstance(cm.exception.args[0], str) ++ + def test_main(): + test_support.run_unittest(UrlParseTestCase) + +diff --git a/Lib/urlparse.py b/Lib/urlparse.py +index 4ce982e..9a1df74 100644 +--- a/Lib/urlparse.py ++++ b/Lib/urlparse.py +@@ -164,6 +164,25 @@ def _splitnetloc(url, start=0): + delim = min(delim, wdelim) # use earliest delim position + return url[start:delim], url[delim:] # return (domain, rest) + ++def _checknetloc(netloc): ++ if not netloc or not isinstance(netloc, unicode): ++ return ++ # looking for characters like \u2100 that expand to 'a/c' ++ # IDNA uses NFKC equivalence, so normalize for this check ++ import unicodedata ++ n = netloc.replace(u'@', u'') # ignore characters already included ++ n = n.replace(u':', u'') # but not the surrounding text ++ n = n.replace(u'#', u'') ++ n = n.replace(u'?', u'') ++ netloc2 = unicodedata.normalize('NFKC', n) ++ if n == netloc2: ++ return ++ for c in '/?#@:': ++ if c in netloc2: ++ raise ValueError("netloc %r contains invalid characters " ++ "under NFKC normalization" ++ % netloc) ++ + def urlsplit(url, scheme='', allow_fragments=True): + """Parse a URL into 5 components: + :///?# +@@ -192,6 +211,7 @@ def urlsplit(url, scheme='', allow_fragments=True): + url, fragment = url.split('#', 1) + if '?' in url: + url, query = url.split('?', 1) ++ _checknetloc(netloc) + v = SplitResult(scheme, netloc, url, query, fragment) + _parse_cache[key] = v + return v +@@ -215,6 +235,7 @@ def urlsplit(url, scheme='', allow_fragments=True): + url, fragment = url.split('#', 1) + if '?' in url: + url, query = url.split('?', 1) ++ _checknetloc(netloc) + v = SplitResult(scheme, netloc, url, query, fragment) + _parse_cache[key] = v + return v diff --git a/SOURCES/00324-disallow-control-chars-in-http-urls.patch b/SOURCES/00324-disallow-control-chars-in-http-urls.patch new file mode 100644 index 00000000..e12aeb1c --- /dev/null +++ b/SOURCES/00324-disallow-control-chars-in-http-urls.patch @@ -0,0 +1,222 @@ +diff --git a/Lib/httplib.py b/Lib/httplib.py +index da2f346..fc8e895 100644 +--- a/Lib/httplib.py ++++ b/Lib/httplib.py +@@ -247,6 +247,15 @@ _MAXHEADERS = 100 + _is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match + _is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search + ++# These characters are not allowed within HTTP URL paths. ++# See https://tools.ietf.org/html/rfc3986#section-3.3 and the ++# https://tools.ietf.org/html/rfc3986#appendix-A pchar definition. ++# Prevents CVE-2019-9740. Includes control characters such as \r\n. ++# Restrict non-ASCII characters above \x7f (0x80-0xff). ++_contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f-\xff]') ++# Arguably only these _should_ allowed: ++# _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$") ++# We are more lenient for assumed real world compatibility purposes. + + class HTTPMessage(mimetools.Message): + +@@ -926,6 +935,12 @@ class HTTPConnection: + self._method = method + if not url: + url = '/' ++ # Prevent CVE-2019-9740. ++ match = _contains_disallowed_url_pchar_re.search(url) ++ if match: ++ raise InvalidURL("URL can't contain control characters. %r " ++ "(found at least %r)" ++ % (url, match.group())) + hdr = '%s %s %s' % (method, url, self._http_vsn_str) + + self._output(hdr) +diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py +index 3845012..d2da0f8 100644 +--- a/Lib/test/test_urllib.py ++++ b/Lib/test/test_urllib.py +@@ -198,6 +198,31 @@ class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin): + finally: + self.unfakehttp() + ++ def test_url_with_control_char_rejected(self): ++ for char_no in range(0, 0x21) + range(0x7f, 0x100): ++ char = chr(char_no) ++ schemeless_url = "//localhost:7777/test%s/" % char ++ self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.") ++ try: ++ # urllib quotes the URL so there is no injection. ++ resp = urllib.urlopen("http:" + schemeless_url) ++ self.assertNotIn(char, resp.geturl()) ++ finally: ++ self.unfakehttp() ++ ++ def test_url_with_newline_header_injection_rejected(self): ++ self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.") ++ host = "localhost:7777?a=1 HTTP/1.1\r\nX-injected: header\r\nTEST: 123" ++ schemeless_url = "//" + host + ":8080/test/?test=a" ++ try: ++ # urllib quotes the URL so there is no injection. ++ resp = urllib.urlopen("http:" + schemeless_url) ++ self.assertNotIn(' ', resp.geturl()) ++ self.assertNotIn('\r', resp.geturl()) ++ self.assertNotIn('\n', resp.geturl()) ++ finally: ++ self.unfakehttp() ++ + def test_read_bogus(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp('''HTTP/1.1 401 Authentication Required +@@ -786,6 +811,35 @@ class Pathname_Tests(unittest.TestCase): + class Utility_Tests(unittest.TestCase): + """Testcase to test the various utility functions in the urllib.""" + ++ def test_splithost(self): ++ splithost = urllib.splithost ++ self.assertEqual(splithost('//www.example.org:80/foo/bar/baz.html'), ++ ('www.example.org:80', '/foo/bar/baz.html')) ++ self.assertEqual(splithost('//www.example.org:80'), ++ ('www.example.org:80', '')) ++ self.assertEqual(splithost('/foo/bar/baz.html'), ++ (None, '/foo/bar/baz.html')) ++ ++ # bpo-30500: # starts a fragment. ++ self.assertEqual(splithost('//127.0.0.1#@host.com'), ++ ('127.0.0.1', '/#@host.com')) ++ self.assertEqual(splithost('//127.0.0.1#@host.com:80'), ++ ('127.0.0.1', '/#@host.com:80')) ++ self.assertEqual(splithost('//127.0.0.1:80#@host.com'), ++ ('127.0.0.1:80', '/#@host.com')) ++ ++ # Empty host is returned as empty string. ++ self.assertEqual(splithost("///file"), ++ ('', '/file')) ++ ++ # Trailing semicolon, question mark and hash symbol are kept. ++ self.assertEqual(splithost("//example.net/file;"), ++ ('example.net', '/file;')) ++ self.assertEqual(splithost("//example.net/file?"), ++ ('example.net', '/file?')) ++ self.assertEqual(splithost("//example.net/file#"), ++ ('example.net', '/file#')) ++ + def test_splitpasswd(self): + """Some of the password examples are not sensible, but it is added to + confirming to RFC2617 and addressing issue4675. +diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py +index c317b8d..63fefd6 100644 +--- a/Lib/test/test_urllib2.py ++++ b/Lib/test/test_urllib2.py +@@ -7,12 +7,16 @@ import StringIO + + import urllib2 + from urllib2 import Request, OpenerDirector ++import httplib + + try: + import ssl + except ImportError: + ssl = None + ++from test.test_urllib import FakeHTTPMixin ++ ++ + # XXX + # Request + # CacheFTPHandler (hard to write) +@@ -1243,7 +1247,7 @@ class HandlerTests(unittest.TestCase): + self.assertEqual(len(http_handler.requests), 1) + self.assertFalse(http_handler.requests[0].has_header(auth_header)) + +-class MiscTests(unittest.TestCase): ++class MiscTests(unittest.TestCase, FakeHTTPMixin): + + def test_build_opener(self): + class MyHTTPHandler(urllib2.HTTPHandler): pass +@@ -1289,6 +1293,53 @@ class MiscTests(unittest.TestCase): + else: + self.assertTrue(False) + ++ @unittest.skipUnless(ssl, "ssl module required") ++ def test_url_with_control_char_rejected(self): ++ for char_no in range(0, 0x21) + range(0x7f, 0x100): ++ char = chr(char_no) ++ schemeless_url = "//localhost:7777/test%s/" % char ++ self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.") ++ try: ++ # We explicitly test urllib.request.urlopen() instead of the top ++ # level 'def urlopen()' function defined in this... (quite ugly) ++ # test suite. They use different url opening codepaths. Plain ++ # urlopen uses FancyURLOpener which goes via a codepath that ++ # calls urllib.parse.quote() on the URL which makes all of the ++ # above attempts at injection within the url _path_ safe. ++ escaped_char_repr = repr(char).replace('\\', r'\\') ++ InvalidURL = httplib.InvalidURL ++ with self.assertRaisesRegexp( ++ InvalidURL, "contain control.*" + escaped_char_repr): ++ urllib2.urlopen("http:" + schemeless_url) ++ with self.assertRaisesRegexp( ++ InvalidURL, "contain control.*" + escaped_char_repr): ++ urllib2.urlopen("https:" + schemeless_url) ++ finally: ++ self.unfakehttp() ++ ++ @unittest.skipUnless(ssl, "ssl module required") ++ def test_url_with_newline_header_injection_rejected(self): ++ self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.") ++ host = "localhost:7777?a=1 HTTP/1.1\r\nX-injected: header\r\nTEST: 123" ++ schemeless_url = "//" + host + ":8080/test/?test=a" ++ try: ++ # We explicitly test urllib2.urlopen() instead of the top ++ # level 'def urlopen()' function defined in this... (quite ugly) ++ # test suite. They use different url opening codepaths. Plain ++ # urlopen uses FancyURLOpener which goes via a codepath that ++ # calls urllib.parse.quote() on the URL which makes all of the ++ # above attempts at injection within the url _path_ safe. ++ InvalidURL = httplib.InvalidURL ++ with self.assertRaisesRegexp( ++ InvalidURL, r"contain control.*\\r.*(found at least . .)"): ++ urllib2.urlopen("http:" + schemeless_url) ++ with self.assertRaisesRegexp(InvalidURL, r"contain control.*\\n"): ++ urllib2.urlopen("https:" + schemeless_url) ++ finally: ++ self.unfakehttp() ++ ++ ++ + class RequestTests(unittest.TestCase): + + def setUp(self): +diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py +index 79e862a..347b494 100644 +--- a/Lib/test/test_xmlrpc.py ++++ b/Lib/test/test_xmlrpc.py +@@ -592,7 +592,13 @@ class SimpleServerTestCase(BaseServerTestCase): + def test_partial_post(self): + # Check that a partial POST doesn't make the server loop: issue #14001. + conn = httplib.HTTPConnection(ADDR, PORT) +- conn.request('POST', '/RPC2 HTTP/1.0\r\nContent-Length: 100\r\n\r\nbye') ++ conn.send('POST /RPC2 HTTP/1.0\r\n' ++ 'Content-Length: 100\r\n\r\n' ++ 'bye HTTP/1.1\r\n' ++ 'Host: %s:%s\r\n' ++ 'Accept-Encoding: identity\r\n' ++ 'Content-Length: 0\r\n\r\n' ++ % (ADDR, PORT)) + conn.close() + + class MultiPathServerTestCase(BaseServerTestCase): +diff --git a/Lib/urllib.py b/Lib/urllib.py +index 9b31df1..2201e3e 100644 +--- a/Lib/urllib.py ++++ b/Lib/urllib.py +@@ -1079,8 +1079,7 @@ def splithost(url): + """splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" + global _hostprog + if _hostprog is None: +- import re +- _hostprog = re.compile('^//([^/?]*)(.*)$') ++ _hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL) + + match = _hostprog.match(url) + if match: diff --git a/SOURCES/00325-CVE-2019-9948.patch b/SOURCES/00325-CVE-2019-9948.patch new file mode 100644 index 00000000..890bf711 --- /dev/null +++ b/SOURCES/00325-CVE-2019-9948.patch @@ -0,0 +1,37 @@ +diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py +index d2da0f8..7813b9f 100644 +--- a/Lib/test/test_urllib.py ++++ b/Lib/test/test_urllib.py +@@ -872,6 +872,17 @@ class URLopener_Tests(unittest.TestCase): + "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"), + "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/") + ++ def test_local_file_open(self): ++ # bpo-35907, CVE-2019-9948: urllib must reject local_file:// scheme ++ class DummyURLopener(urllib.URLopener): ++ def open_local_file(self, url): ++ return url ++ for url in ('local_file://example', 'local-file://example'): ++ self.assertRaises(IOError, urllib.urlopen, url) ++ self.assertRaises(IOError, urllib.URLopener().open, url) ++ self.assertRaises(IOError, urllib.URLopener().retrieve, url) ++ self.assertRaises(IOError, DummyURLopener().open, url) ++ self.assertRaises(IOError, DummyURLopener().retrieve, url) + + # Just commented them out. + # Can't really tell why keep failing in windows and sparc. +diff --git a/Lib/urllib.py b/Lib/urllib.py +index 2201e3e..71e3637 100644 +--- a/Lib/urllib.py ++++ b/Lib/urllib.py +@@ -198,7 +198,9 @@ class URLopener: + name = 'open_' + urltype + self.type = urltype + name = name.replace('-', '_') +- if not hasattr(self, name): ++ ++ # bpo-35907: disallow the file reading with the type not allowed ++ if not hasattr(self, name) or name == 'open_local_file': + if proxy: + return self.open_unknown_proxy(proxy, fullurl, data) + else: diff --git a/SOURCES/00330-CVE-2018-20852.patch b/SOURCES/00330-CVE-2018-20852.patch new file mode 100644 index 00000000..97ff719a --- /dev/null +++ b/SOURCES/00330-CVE-2018-20852.patch @@ -0,0 +1,93 @@ +diff --git a/Lib/cookielib.py b/Lib/cookielib.py +index f9c8d2f..9144e1f 100644 +--- a/Lib/cookielib.py ++++ b/Lib/cookielib.py +@@ -1123,6 +1123,11 @@ class DefaultCookiePolicy(CookiePolicy): + req_host, erhn = eff_request_host(request) + domain = cookie.domain + ++ if domain and not domain.startswith("."): ++ dotdomain = "." + domain ++ else: ++ dotdomain = domain ++ + # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't + if (cookie.version == 0 and + (self.strict_ns_domain & self.DomainStrictNonDomain) and +@@ -1135,7 +1140,7 @@ class DefaultCookiePolicy(CookiePolicy): + _debug(" effective request-host name %s does not domain-match " + "RFC 2965 cookie domain %s", erhn, domain) + return False +- if cookie.version == 0 and not ("."+erhn).endswith(domain): ++ if cookie.version == 0 and not ("."+erhn).endswith(dotdomain): + _debug(" request-host %s does not match Netscape cookie domain " + "%s", req_host, domain) + return False +@@ -1149,7 +1154,11 @@ class DefaultCookiePolicy(CookiePolicy): + req_host = "."+req_host + if not erhn.startswith("."): + erhn = "."+erhn +- if not (req_host.endswith(domain) or erhn.endswith(domain)): ++ if domain and not domain.startswith("."): ++ dotdomain = "." + domain ++ else: ++ dotdomain = domain ++ if not (req_host.endswith(dotdomain) or erhn.endswith(dotdomain)): + #_debug(" request domain %s does not match cookie domain %s", + # req_host, domain) + return False +diff --git a/Lib/test/test_cookielib.py b/Lib/test/test_cookielib.py +index dd0ad32..b4f5ea0 100644 +--- a/Lib/test/test_cookielib.py ++++ b/Lib/test/test_cookielib.py +@@ -353,6 +353,7 @@ class CookieTests(TestCase): + ("http://foo.bar.com/", ".foo.bar.com", True), + ("http://foo.bar.com/", "foo.bar.com", True), + ("http://foo.bar.com/", ".bar.com", True), ++ ("http://foo.bar.com/", "bar.com", True), + ("http://foo.bar.com/", "com", True), + ("http://foo.com/", "rhubarb.foo.com", False), + ("http://foo.com/", ".foo.com", True), +@@ -363,6 +364,8 @@ class CookieTests(TestCase): + ("http://foo/", "foo", True), + ("http://foo/", "foo.local", True), + ("http://foo/", ".local", True), ++ ("http://barfoo.com", ".foo.com", False), ++ ("http://barfoo.com", "foo.com", False), + ]: + request = urllib2.Request(url) + r = pol.domain_return_ok(domain, request) +@@ -910,6 +913,33 @@ class CookieTests(TestCase): + c.add_cookie_header(req) + self.assertTrue(not req.has_header("Cookie")) + ++ c.clear() ++ ++ pol.set_blocked_domains([]) ++ req = Request("http://acme.com/") ++ res = FakeResponse(headers, "http://acme.com/") ++ cookies = c.make_cookies(res, req) ++ c.extract_cookies(res, req) ++ self.assertEqual(len(c), 1) ++ ++ req = Request("http://acme.com/") ++ c.add_cookie_header(req) ++ self.assertTrue(req.has_header("Cookie")) ++ ++ req = Request("http://badacme.com/") ++ c.add_cookie_header(req) ++ self.assertFalse(pol.return_ok(cookies[0], req)) ++ self.assertFalse(req.has_header("Cookie")) ++ ++ p = pol.set_blocked_domains(["acme.com"]) ++ req = Request("http://acme.com/") ++ c.add_cookie_header(req) ++ self.assertFalse(req.has_header("Cookie")) ++ ++ req = Request("http://badacme.com/") ++ c.add_cookie_header(req) ++ self.assertFalse(req.has_header("Cookie")) ++ + def test_secure(self): + from cookielib import CookieJar, DefaultCookiePolicy + diff --git a/SOURCES/00332-CVE-2019-16056.patch b/SOURCES/00332-CVE-2019-16056.patch new file mode 100644 index 00000000..aff257fc --- /dev/null +++ b/SOURCES/00332-CVE-2019-16056.patch @@ -0,0 +1,54 @@ +diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py +index 690db2c..dc49d2e 100644 +--- a/Lib/email/_parseaddr.py ++++ b/Lib/email/_parseaddr.py +@@ -336,7 +336,12 @@ class AddrlistClass: + aslist.append('@') + self.pos += 1 + self.gotonext() +- return EMPTYSTRING.join(aslist) + self.getdomain() ++ domain = self.getdomain() ++ if not domain: ++ # Invalid domain, return an empty address instead of returning a ++ # local part to denote failed parsing. ++ return EMPTYSTRING ++ return EMPTYSTRING.join(aslist) + domain + + def getdomain(self): + """Get the complete domain name from an address.""" +@@ -351,6 +356,10 @@ class AddrlistClass: + elif self.field[self.pos] == '.': + self.pos += 1 + sdlist.append('.') ++ elif self.field[self.pos] == '@': ++ # bpo-34155: Don't parse domains with two `@` like ++ # `a@malicious.org@important.com`. ++ return EMPTYSTRING + elif self.field[self.pos] in self.atomends: + break + else: +diff --git a/Lib/email/test/test_email.py b/Lib/email/test/test_email.py +index b32da9d..1739033 100644 +--- a/Lib/email/test/test_email.py ++++ b/Lib/email/test/test_email.py +@@ -2308,6 +2308,20 @@ class TestMiscellaneous(TestEmailBase): + self.assertEqual(Utils.parseaddr('<>'), ('', '')) + self.assertEqual(Utils.formataddr(Utils.parseaddr('<>')), '') + ++ def test_parseaddr_multiple_domains(self): ++ self.assertEqual( ++ Utils.parseaddr('a@b@c'), ++ ('', '') ++ ) ++ self.assertEqual( ++ Utils.parseaddr('a@b.c@c'), ++ ('', '') ++ ) ++ self.assertEqual( ++ Utils.parseaddr('a@172.17.0.1@c'), ++ ('', '') ++ ) ++ + def test_noquote_dump(self): + self.assertEqual( + Utils.formataddr(('A Silly Person', 'person@dom.ain')), diff --git a/SOURCES/00344-CVE-2019-16935.patch b/SOURCES/00344-CVE-2019-16935.patch new file mode 100644 index 00000000..86d192f2 --- /dev/null +++ b/SOURCES/00344-CVE-2019-16935.patch @@ -0,0 +1,144 @@ +diff --git a/Lib/DocXMLRPCServer.py b/Lib/DocXMLRPCServer.py +index 4064ec2..90b037d 100644 +--- a/Lib/DocXMLRPCServer.py ++++ b/Lib/DocXMLRPCServer.py +@@ -20,6 +20,16 @@ from SimpleXMLRPCServer import (SimpleXMLRPCServer, + CGIXMLRPCRequestHandler, + resolve_dotted_attribute) + ++ ++def _html_escape_quote(s): ++ s = s.replace("&", "&") # Must be done first! ++ s = s.replace("<", "<") ++ s = s.replace(">", ">") ++ s = s.replace('"', """) ++ s = s.replace('\'', "'") ++ return s ++ ++ + class ServerHTMLDoc(pydoc.HTMLDoc): + """Class used to generate pydoc HTML document for a server""" + +@@ -210,7 +220,8 @@ class XMLRPCDocGenerator: + methods + ) + +- return documenter.page(self.server_title, documentation) ++ title = _html_escape_quote(self.server_title) ++ return documenter.page(title, documentation) + + class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): + """XML-RPC and documentation request handler class. +diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py +index 80d1803..d464ef8 100644 +--- a/Lib/test/test_docxmlrpc.py ++++ b/Lib/test/test_docxmlrpc.py +@@ -1,13 +1,11 @@ + from DocXMLRPCServer import DocXMLRPCServer + import httplib ++import re + import sys + from test import test_support + threading = test_support.import_module('threading') +-import time +-import socket + import unittest + +-PORT = None + + def make_request_and_skipIf(condition, reason): + # If we skip the test, we have to make a request because the +@@ -23,13 +21,10 @@ def make_request_and_skipIf(condition, reason): + return decorator + + +-def server(evt, numrequests): ++def make_server(): + serv = DocXMLRPCServer(("localhost", 0), logRequests=False) + + try: +- global PORT +- PORT = serv.socket.getsockname()[1] +- + # Add some documentation + serv.set_server_title("DocXMLRPCServer Test Documentation") + serv.set_server_name("DocXMLRPCServer Test Docs") +@@ -56,42 +51,31 @@ def server(evt, numrequests): + + serv.register_function(add) + serv.register_function(lambda x, y: x-y) +- +- while numrequests > 0: +- serv.handle_request() +- numrequests -= 1 +- except socket.timeout: +- pass +- finally: ++ return serv ++ except: + serv.server_close() +- PORT = None +- evt.set() ++ raise + + class DocXMLRPCHTTPGETServer(unittest.TestCase): + def setUp(self): +- self._threads = test_support.threading_setup() + # Enable server feedback + DocXMLRPCServer._send_traceback_header = True + +- self.evt = threading.Event() +- threading.Thread(target=server, args=(self.evt, 1)).start() +- +- # wait for port to be assigned +- n = 1000 +- while n > 0 and PORT is None: +- time.sleep(0.001) +- n -= 1 ++ self.serv = make_server() ++ self.thread = threading.Thread(target=self.serv.serve_forever) ++ self.thread.start() + ++ PORT = self.serv.server_address[1] + self.client = httplib.HTTPConnection("localhost:%d" % PORT) + + def tearDown(self): + self.client.close() + +- self.evt.wait() +- + # Disable server feedback + DocXMLRPCServer._send_traceback_header = False +- test_support.threading_cleanup(*self._threads) ++ self.serv.shutdown() ++ self.thread.join() ++ self.serv.server_close() + + def test_valid_get_response(self): + self.client.request("GET", "/") +@@ -194,6 +178,25 @@ class DocXMLRPCHTTPGETServer(unittest.TestCase): + self.assertIn("""Try self.add, too.""", + response.read()) + ++ def test_server_title_escape(self): ++ """Test that the server title and documentation ++ are escaped for HTML. ++ """ ++ self.serv.set_server_title('test_title