Update

httplib/retriever.py

@@ -42,6 +42,28 @@ class Retriever(ABC):
             return ContentLengthRetriever(client, int(content_length))
 
 
+class PreambleRetriever(Retriever):
+    client: HTTPSocket
+    buffer: []
+
+    def __init__(self, client: HTTPSocket):
+        super().__init__(client)
+        self.client = client
+        self.buffer = []
+
+    def retrieve(self):
+
+        line = self.client.read_line()
+        while True:
+            self.buffer.append(line)
+
+            if line in ("\r\n", "\n", " "):
+                break
+
+            yield line
+            line = self.client.read_line()
+
+
 class ContentLengthRetriever(Retriever):
     length: int
 
@@ -63,21 +85,16 @@ class ContentLengthRetriever(Retriever):
                 buffer = self.client.read(remaining)
             except TimeoutError:
                 logging.error("Timed out before receiving complete payload")
-                self.client.close()
                 raise IncompleteResponse("Timed out before receiving complete payload")
             except ConnectionError:
                 logging.error("Timed out before receiving complete payload")
-                self.client.close()
                 raise IncompleteResponse("Connection closed before receiving complete payload")
 
-            logging.debug("Received payload length: %s", len(buffer))
-
             if len(buffer) == 0:
                 logging.warning("Received payload length %s less than expected %s", cur_payload_size, self.length)
                 break
 
             cur_payload_size += len(buffer)
-            logging.debug("Processed payload: %r", cur_payload_size)
             yield buffer
 
         return b""
@@ -108,7 +125,6 @@ class ChunkedRetriever(Retriever):
             yield buffer
 
             self.client.read_line()  # remove CRLF
-        return b""
 
     def __get_chunk_size(self):
         line = self.client.read_line()