Fix some issues, improve documentation

httplib/exceptions.py

@@ -34,8 +34,10 @@ class HTTPServerException(Exception):
     status_code: str
     message: str
     body: str
+    arg: str
 
-    def __init__(self, body=""):
+    def __init__(self, arg, body=""):
+        self.arg = arg
         self.body = body
 
 

httplib/retriever.py

@@ -7,6 +7,9 @@ from httplib.httpsocket import HTTPSocket, BUFSIZE
 
 
 class Retriever(ABC):
+    """
+    This is a helper class for retrieving HTTP messages.
+    """
     client: HTTPSocket
 
     def __init__(self, client: HTTPSocket):
@@ -14,10 +17,23 @@ class Retriever(ABC):
 
     @abstractmethod
     def retrieve(self):
+        """
+        Creates an iterator of the retrieved message content.
+        """
         pass
 
     @staticmethod
     def create(client: HTTPSocket, headers: Dict[str, str]):
+        """
+        Creates a Retriever instance depending on the give headers.
+
+        @param client: the socket to retrieve from
+        @param headers: the message headers for choosing the retriever instance
+        @return: ChunkedRetriever if the message uses chunked encoding, ContentLengthRetriever if the message
+        specifies a content-length, RawRetriever if none of the above is True.
+        @raise UnsupportedEncoding: if the `transfer-encoding` is not supported or if the `content-encoding` is not
+        supported.
+        """
 
         # only chunked transfer-encoding is supported
         transfer_encoding = headers.get("transfer-encoding")
@@ -32,17 +48,20 @@ class Retriever(ABC):
 
         if chunked:
             return ChunkedRetriever(client)
-        else:
-            content_length = headers.get("content-length")
 
-            if not content_length:
-                logging.warning("Transfer-encoding and content-length not specified, trying without")
-                return RawRetriever(client)
+        content_length = headers.get("content-length")
 
-            return ContentLengthRetriever(client, int(content_length))
+        if not content_length:
+            logging.warning("Transfer-encoding and content-length not specified, trying without")
+            return RawRetriever(client)
+
+        return ContentLengthRetriever(client, int(content_length))
 
 
 class PreambleRetriever(Retriever):
+    """
+    Retriever instance for retrieving the start-line and headers of an HTTP message.
+    """
     client: HTTPSocket
     _buffer: []
 
@@ -59,6 +78,10 @@ class PreambleRetriever(Retriever):
         self._buffer = []
 
     def retrieve(self):
+        """
+        Returns an iterator of the retrieved lines.
+        @return:
+        """
 
         line = self.client.read_line()
         while True:
@@ -76,6 +99,9 @@ class PreambleRetriever(Retriever):
 
 
 class ContentLengthRetriever(Retriever):
+    """
+    Retriever instance for retrieving a message body with a given content-length.
+    """
     length: int
 
     def __init__(self, client: HTTPSocket, length: int):
@@ -83,6 +109,11 @@ class ContentLengthRetriever(Retriever):
         self.length = length
 
     def retrieve(self):
+        """
+        Returns an iterator of the received message bytes.
+        The size of each iteration is not necessarily constant.
+        @raise IncompleteResponse: if the connection is closed or timed out before receiving the complete payload.
+        """
 
         cur_payload_size = 0
         read_size = BUFSIZE
@@ -95,10 +126,10 @@ class ContentLengthRetriever(Retriever):
             try:
                 buffer = self.client.read(remaining)
             except TimeoutError:
-                logging.error("Timed out before receiving complete payload")
+                logging.error("Timed out before receiving the complete payload")
                 raise IncompleteResponse("Timed out before receiving complete payload")
             except ConnectionError:
-                logging.error("Timed out before receiving complete payload")
+                logging.error("Connection closed before receiving the complete payload")
                 raise IncompleteResponse("Connection closed before receiving complete payload")
 
             if len(buffer) == 0:
@@ -110,6 +141,10 @@ class ContentLengthRetriever(Retriever):
 
 
 class RawRetriever(Retriever):
+    """
+    Retriever instance for retrieve a message body without any length specifier or encoding.
+    This retriever will keep waiting until a timeout occurs or the connection is disconnected.
+    """
 
     def retrieve(self):
         while True:
@@ -120,20 +155,28 @@ class RawRetriever(Retriever):
 
 
 class ChunkedRetriever(Retriever):
+    """
+    Retriever instance for retrieving a message body with chunked encoding.
+    """
 
     def retrieve(self):
+        """
+        Returns an iterator of the received message bytes.
+        The size of each iteration is not necessarily constant.
+        @raise IncompleteResponse: if the connection is closed or timed out before receiving the complete payload.
+        """
         while True:
             chunk_size = self.__get_chunk_size()
             logging.debug("chunk-size: %s", chunk_size)
             if chunk_size == 0:
+                # remove all trailing lines
                 self.client.reset_request()
                 break
 
             buffer = self.client.read(chunk_size)
-            logging.debug("chunk: %r", buffer)
             yield buffer
 
-            self.client.read_line()  # remove CRLF
+            self.client.read_line()  # remove trailing CRLF
 
     def __get_chunk_size(self):
         line = self.client.read_line()