Merge commit 'f0d9ebc41e51c4c4c9990b1eed02d297fd1b20d8' into merge-fr…

…om-master-2019-07-26

I tried and fail to understand how we should adapt this commit to our
work. Given that the current situation is likely to change (see
python-trio#123), I did not merge the
actual functionality, but only the test of the default behavior (and the
CHANGES/docs as we're not currently touching those for easier merges).

CHANGES.rst

@@ -37,6 +37,9 @@ dev (master)
 * Fix issue where URLs containing invalid characters within ``Url.auth`` would
   raise an exception instead of percent-encoding those characters.
 
+* Add support for ``HTTPResponse.auto_close = False`` which makes HTTP responses
+  work well with BufferedReaders and other ``io`` module features. (Pull #1652)
+
 
 1.25.3 (2019-05-23)
 -------------------

docs/user-guide.rst

@@ -77,6 +77,20 @@ to a byte string representing the response content::
 .. note:: For larger responses, it's sometimes better to :ref:`stream <stream>`
     the response.
 
+Using io Wrappers with Response content
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes you want to use :class:`io.TextIOWrapper` or similar objects like a CSV reader
+directly with :class:`~response.HTTPResponse` data. Making these two interfaces play nice
+together requires using the :attr:`~response.HTTPResponse.auto_close` attribute by setting it
+to ``False``. By default HTTP responses are closed after reading all bytes, this disables that behavior::
+
+    >>> import io
+    >>> r = http.request('GET', 'https://example.com', preload_content=False)
+    >>> r.auto_close = False
+    >>> for line in io.TextIOWrapper(r):
+    >>>     print(line)
+
 .. _request_data:
 
 Request data

test/test_response.py

@@ -1,8 +1,11 @@
+# -*- coding: utf-8 -*-
+
 import zlib
 
-from io import BytesIO, BufferedReader
+from io import BytesIO, BufferedReader, TextIOWrapper
 
 import pytest
+import six
 
 from urllib3.base import Response
 from urllib3.response import HTTPResponse, brotli
@@ -305,6 +308,14 @@ def test_io_bufferedreader(self):
         br.close()
         assert resp.closed
 
+        # HTTPResponse.read() by default closes the response
+        # https://github.com/urllib3/urllib3/issues/1305
+        fp = BytesIO(b"hello\nworld")
+        resp = HTTPResponse(fp, preload_content=False)
+        with pytest.raises(ValueError) as ctx:
+            list(BufferedReader(resp))
+        assert str(ctx.value) == "readline of closed file"
+
         b = b"!tenbytes!"
         fp = BytesIO(b)
         resp = HTTPResponse(fp, preload_content=False)
@@ -316,6 +327,30 @@ def test_io_bufferedreader(self):
         assert len(br.read(5)) == 5
         assert len(br.read(5)) == 0
 
+    def test_io_textiowrapper(self):
+        fp = BytesIO(b"\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f")
+        resp = HTTPResponse(fp, preload_content=False)
+        br = TextIOWrapper(resp, encoding="utf8")
+
+        assert br.read() == u"äöüß"
+
+        br.close()
+        assert resp.closed
+
+        # HTTPResponse.read() by default closes the response
+        # https://github.com/urllib3/urllib3/issues/1305
+        fp = BytesIO(
+            b"\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f\n\xce\xb1\xce\xb2\xce\xb3\xce\xb4"
+        )
+        resp = HTTPResponse(fp, preload_content=False)
+        with pytest.raises(ValueError) as ctx:
+            if six.PY2:
+                # py2's implementation of TextIOWrapper requires `read1`
+                # method which is provided by `BufferedReader` wrapper
+                resp = BufferedReader(resp)
+            list(TextIOWrapper(resp))
+        assert str(ctx.value) == "I/O operation on closed file."
+
     def test_streaming(self):
         fp = [b"fo", b"o"]
         resp = HTTPResponse(fp, preload_content=False)