From fdbd8658892f08e45f60314717b395c8a6a01db3 Mon Sep 17 00:00:00 2001 From: Arthur Bols Date: Fri, 26 Mar 2021 18:25:03 +0100 Subject: [PATCH] Update --- client/command.py | 6 +- client/response_handler.py | 2 +- httplib/exceptions.py | 29 +++ httplib/httpsocket.py | 19 +- httplib/message.py | 29 ++- httplib/parser.py | 46 +++-- httplib/retriever.py | 19 +- server/command.py | 176 +++++++++++++----- server/httpserver.py | 3 +- .../{RequestHandler.py => requesthandler.py} | 95 +++++----- server/worker.py | 9 +- 11 files changed, 297 insertions(+), 136 deletions(-) rename server/{RequestHandler.py => requesthandler.py} (53%) diff --git a/client/command.py b/client/command.py index 41dddba..f525c2d 100644 --- a/client/command.py +++ b/client/command.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse from client.httpclient import FORMAT, HTTPClient from httplib import parser from httplib.exceptions import InvalidResponse, InvalidStatusLine, UnsupportedEncoding -from httplib.message import Message +from httplib.message import ClientMessage as Message from httplib.retriever import PreambleRetriever sockets: Dict[str, HTTPClient] = {} @@ -29,12 +29,12 @@ class AbstractCommand(ABC): uri: str host: str path: str - port: Tuple[str, int] + port: int def __init__(self, uri: str, port): self.uri = uri self.host, _, self.path = parser.parse_uri(uri) - self.port = port + self.port = int(port) @property @abstractmethod diff --git a/client/response_handler.py b/client/response_handler.py index be1a976..c163a56 100644 --- a/client/response_handler.py +++ b/client/response_handler.py @@ -10,7 +10,7 @@ from client.command import AbstractCommand, GetCommand from client.httpclient import HTTPClient, FORMAT from httplib import parser from httplib.exceptions import InvalidResponse -from httplib.message import Message +from httplib.message import ClientMessage as Message from httplib.retriever import Retriever diff --git a/httplib/exceptions.py b/httplib/exceptions.py index bbe62be..55bd1fc 100644 --- a/httplib/exceptions.py +++ b/httplib/exceptions.py @@ -33,6 +33,10 @@ class HTTPServerException(Exception): """ Base class for HTTP Server exceptions """ status_code: str message: str + body: str + + def __init__(self, body=""): + self.body = body class BadRequest(HTTPServerException): @@ -66,3 +70,28 @@ class NotFound(HTTPServerException): """ Resource not found """ status_code = 404 message = "Not Found" + + +class Forbidden(HTTPServerException): + """ Request not allowed """ + status_code = 403 + message = "Forbidden" + + +class Conflict(HTTPServerException): + """ Conflict in the current state of the target resource """ + status_code = 409 + message = "Conflict" + + +class HTTPVersionNotSupported(HTTPServerException): + """ The server does not support the major version HTTP used in the request message """ + status_code = 505 + message = "HTTP Version Not Supported" + + +class InvalidRequestLine(BadRequest): + """ Request start-line is invalid """ + + def __init__(self, line): + self.request_line = line diff --git a/httplib/httpsocket.py b/httplib/httpsocket.py index a881760..2dc1372 100644 --- a/httplib/httpsocket.py +++ b/httplib/httpsocket.py @@ -3,6 +3,8 @@ import socket from io import BufferedReader from typing import Tuple +from httplib.exceptions import BadRequest + BUFSIZE = 4096 TIMEOUT = 3 FORMAT = "UTF-8" @@ -61,17 +63,28 @@ class HTTPSocket: def read(self, size=BUFSIZE, blocking=True) -> bytes: if blocking: - return self.file.read(size) + buffer = self.file.read(size) + else: + buffer = self.file.read1(size) - return self.file.read1(size) + if len(buffer) == 0: + raise ConnectionAbortedError + return buffer def read_line(self): - return str(self.read_bytes_line(), FORMAT) + try: + line = str(self.read_bytes_line(), FORMAT) + except UnicodeDecodeError: + # Expected UTF-8 + raise BadRequest() + return line def read_bytes_line(self) -> bytes: line = self.file.readline(MAXLINE + 1) if len(line) > MAXLINE: raise InvalidResponse("Line too long") + elif len(line) == 0: + raise ConnectionAbortedError return line diff --git a/httplib/message.py b/httplib/message.py index 80fcb14..16a9545 100644 --- a/httplib/message.py +++ b/httplib/message.py @@ -1,18 +1,35 @@ +from abc import ABC from typing import Dict +from urllib.parse import SplitResult -class Message: +class Message(ABC): version: str - status: int - msg: str headers: Dict[str, str] raw: str body: bytes - def __init__(self, version: str, status: int, msg: str, headers: Dict[str, str], raw=None, body: bytes = None): + def __init__(self, version: str, headers: Dict[str, str], raw=None, body: bytes = None): self.version = version - self.status = status - self.msg = msg self.headers = headers self.raw = raw self.body = body + + +class ClientMessage(Message): + status: int + msg: str + + def __init__(self, version: str, status: int, msg: str, headers: Dict[str, str], raw=None, body: bytes = None): + super().__init__(version, headers, raw, body) + self.status = status + + +class ServerMessage(Message): + method: str + target: SplitResult + + def __init__(self, version: str, method: str, target, headers: Dict[str, str], raw=None, body: bytes = None): + super().__init__(version, headers, raw, body) + self.method = method + self.target = target diff --git a/httplib/parser.py b/httplib/parser.py index c62268b..32b04c1 100644 --- a/httplib/parser.py +++ b/httplib/parser.py @@ -3,7 +3,7 @@ import os.path import re from urllib.parse import urlparse, urlsplit -from httplib.exceptions import InvalidStatusLine, InvalidResponse, BadRequest +from httplib.exceptions import InvalidStatusLine, InvalidResponse, BadRequest, InvalidRequestLine from httplib.httpsocket import HTTPSocket @@ -48,7 +48,7 @@ def parse_status_line(line: str): if len(split) < 3: raise InvalidStatusLine(line) # TODO fix exception - (http_version, status, reason) = split + http_version, status, reason = split if not _is_valid_http_version(http_version): raise InvalidStatusLine(line) @@ -63,11 +63,12 @@ def parse_status_line(line: str): return version, status, reason -def parse_request_line(client: HTTPSocket): - line, (method, target, version) = _get_start_line(client) - - logging.debug("Parsed request-line=%r, method=%r, target=%r, version=%r", line, method, target, version) +def parse_request_line(line: str): + split = list(filter(None, line.rstrip().split(" ", 2))) + if len(split) < 3: + raise InvalidRequestLine(line) + method, target, version = split if method not in ("CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "POST", "PUT", "TRACE"): raise BadRequest() @@ -146,7 +147,7 @@ def parse_request_headers(client: HTTPSocket): def get_headers(client: HTTPSocket): headers = [] - # first header after the status-line may not contain a space + # first header after the status-line may not start with a space while True: line = client.read_line() if line[0].isspace(): @@ -181,21 +182,28 @@ def get_headers(client: HTTPSocket): def parse_headers(lines): headers = [] - # first header after the status-line may not contain a space - for line in lines: - if line[0].isspace(): - continue - else: - break - for line in lines: - if line in ("\r\n", "\n", " "): - break + try: + # first header after the start-line may not start with a space + line = next(lines) + while True: + if line[0].isspace(): + continue + else: + break - if line[0].isspace(): - headers[-1] = headers[-1].rstrip("\r\n") + while True: + if line in ("\r\n", "\n", ""): + break - headers.append(line.lstrip()) + if line[0].isspace(): + headers[-1] = headers[-1].rstrip("\r\n") + + headers.append(line.lstrip()) + line = next(lines) + except StopIteration: + # No more lines to be parsed + pass result = {} header_str = "".join(headers) diff --git a/httplib/retriever.py b/httplib/retriever.py index 2b9be76..fbd4328 100644 --- a/httplib/retriever.py +++ b/httplib/retriever.py @@ -44,25 +44,36 @@ class Retriever(ABC): class PreambleRetriever(Retriever): client: HTTPSocket - buffer: [] + _buffer: [] + + @property + def buffer(self): + tmp_buffer = self._buffer + self._buffer = [] + + return tmp_buffer def __init__(self, client: HTTPSocket): super().__init__(client) self.client = client - self.buffer = [] + self._buffer = [] def retrieve(self): line = self.client.read_line() while True: - self.buffer.append(line) + self._buffer.append(line) if line in ("\r\n", "\n", ""): - break + return line yield line line = self.client.read_line() + def reset_buffer(self, line): + self._buffer.clear() + self._buffer.append(line) + class ContentLengthRetriever(Retriever): length: int diff --git a/server/command.py b/server/command.py index adbfa3c..31ab58c 100644 --- a/server/command.py +++ b/server/command.py @@ -1,25 +1,39 @@ -import logging +import mimetypes +import os +import sys from abc import ABC, abstractmethod -from typing import Dict, Tuple -from urllib.parse import urlparse +from datetime import datetime +from time import mktime +from typing import Dict +from wsgiref.handlers import format_date_time -from client.httpclient import FORMAT, HTTPClient -from httplib import parser -from httplib.exceptions import InvalidResponse, InvalidStatusLine, UnsupportedEncoding -from httplib.message import Message -from httplib.retriever import PreambleRetriever +from client.httpclient import FORMAT +from httplib.exceptions import NotFound, Conflict, Forbidden +from httplib.message import ServerMessage as Message + +root = os.path.join(os.path.dirname(sys.argv[0]), "public") + +status_message = { + 200: "OK", + 201: "Created", + 202: "Accepted", + 304: "Not Modified", + 400: "Bad Request", + 404: "Not Found", + 500: "Internal Server Error", + +} -def create(method: str, message: Message): - - if method == "GET": - return GetCommand(url, port) - elif method == "HEAD": - return HeadCommand(url, port) - elif method == "POST": - return PostCommand(url, port) - elif method == "PUT": - return PutCommand(url, port) +def create(message: Message): + if message.method == "GET": + return GetCommand(message) + elif message.method == "HEAD": + return HeadCommand(message) + elif message.method == "POST": + return PostCommand(message) + elif message.method == "PUT": + return PutCommand(message) else: raise ValueError() @@ -27,8 +41,10 @@ def create(method: str, message: Message): class AbstractCommand(ABC): path: str headers: Dict[str, str] + msg: Message - def __init(self): + def __init__(self, message: Message): + self.msg = message pass @property @@ -36,63 +52,133 @@ class AbstractCommand(ABC): def command(self): pass + def _get_date(self): + now = datetime.now() + stamp = mktime(now.timetuple()) + return format_date_time(stamp) -class AbstractWithBodyCommand(AbstractCommand, ABC): + @abstractmethod + def execute(self): + pass - def _build_message(self, message: str) -> bytes: - body = input(f"Enter {self.command} data: ").encode(FORMAT) - print() + def _build_message(self, status: int, content_type: str, body: bytes): + message = f"HTTP/1.1 {status} {status_message[status]}\r\n" + message += self._get_date() + "\r\n" + + content_length = len(body) + message += f"Content-Length: {content_length}\r\n" + + if content_type: + message += f"Content-Type: {content_type}" + if content_type.startswith("text"): + message += "; charset=UTF-8" + message += "\r\n" + elif content_length > 0: + message += f"Content-Type: application/octet-stream" - message += "Content-Type: text/plain\r\n" - message += f"Content-Length: {len(body)}\r\n" message += "\r\n" message = message.encode(FORMAT) - message += body - message += b"\r\n" + if content_length > 0: + message += body + message += b"\r\n" return message + def _get_path(self, check=True): + norm_path = os.path.normpath(self.msg.target.path) + + if norm_path == "/": + path = root + "/index.html" + else: + path = root + norm_path + + if check and not os.path.exists(path): + raise NotFound() + + return path + + +class AbstractModifyCommand(AbstractCommand, ABC): + + @property + @abstractmethod + def _file_mode(self): + pass + + def execute(self): + path = self._get_path(False) + dir = os.path.dirname(path) + + if not os.path.exists(dir): + raise Forbidden("Target directory does not exists!") + if os.path.exists(dir) and not os.path.isdir(dir): + raise Forbidden("Target directory is an existing file!") + + try: + with open(path, mode=f"{self._file_mode}b") as file: + file.write(self.msg.body) + except IsADirectoryError: + raise Forbidden("The target resource is a directory!") + + class HeadCommand(AbstractCommand): + def execute(self): + path = self._get_path() + + mime = mimetypes.guess_type(path)[0] + return self._build_message(200, mime, b"") + @property def command(self): return "HEAD" class GetCommand(AbstractCommand): - - def __init__(self, uri: str, port, dir=None): - super().__init__(uri, port) - self.dir = dir - self.filename = None - @property def command(self): return "GET" - def _get_preamble(self, retriever): - lines = retriever.retrieve() - (version, status, msg) = parser.parse_status_line(next(lines)) - headers = parser.parse_headers(lines) + def get_mimetype(self, path): + mime = mimetypes.guess_type(path)[0] - logging.debug("---response begin---\r\n%s--- response end---", "".join(retriever.buffer)) + if mime: + return mime - return Message(version, status, msg, headers, retriever.buffer) + try: + file = open(path, "r", encoding="utf-8") + file.readline() + file.close() + return "text/plain" + except UnicodeDecodeError: + return "application/octet-stream" - def _await_response(self, client, retriever): - msg = self._get_preamble(retriever) + def execute(self): + path = self._get_path() + mime = self.get_mimetype(path) - from client import response_handler - self.filename = response_handler.handle(client, msg, self, self.dir) + file = open(path, "rb") + buffer = file.read() + file.close() + + return self._build_message(200, mime, buffer) -class PostCommand(AbstractWithBodyCommand): +class PostCommand(AbstractModifyCommand): @property def command(self): return "POST" + @property + def _file_mode(self): + return "a" -class PutCommand(AbstractWithBodyCommand): + +class PutCommand(AbstractModifyCommand): @property def command(self): return "PUT" + + @property + def _file_mode(self): + return "w" diff --git a/server/httpserver.py b/server/httpserver.py index 3a5a24e..b9f141e 100644 --- a/server/httpserver.py +++ b/server/httpserver.py @@ -37,7 +37,6 @@ class HTTPServer: def __do_start(self): # Create socket - self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.bind((self.address, self.port)) @@ -56,6 +55,8 @@ class HTTPServer: continue conn, addr = self.server.accept() + conn.settimeout(5) + logging.info("New connection: %s", addr[0]) self._dispatch_queue.put((conn, addr)) logging.debug("Dispatched connection %s", addr) diff --git a/server/RequestHandler.py b/server/requesthandler.py similarity index 53% rename from server/RequestHandler.py rename to server/requesthandler.py index f02f9a6..311ca1c 100644 --- a/server/RequestHandler.py +++ b/server/requesthandler.py @@ -1,7 +1,7 @@ import logging -import mimetypes import os import sys +import time from datetime import datetime from socket import socket from time import mktime @@ -10,9 +10,12 @@ from urllib.parse import ParseResultBytes, ParseResult from wsgiref.handlers import format_date_time from httplib import parser -from httplib.exceptions import MethodNotAllowed, BadRequest, UnsupportedEncoding, NotImplemented, NotFound +from httplib.exceptions import MethodNotAllowed, BadRequest, UnsupportedEncoding, NotImplemented, NotFound, \ + HTTPVersionNotSupported from httplib.httpsocket import HTTPSocket, FORMAT -from httplib.retriever import Retriever +from httplib.message import ServerMessage as Message +from httplib.retriever import Retriever, PreambleRetriever +from server import command METHODS = ("GET", "HEAD", "PUT", "POST") @@ -25,13 +28,28 @@ class RequestHandler: self.conn = HTTPSocket(conn, host) def listen(self): - logging.debug("Parsing request line") - (method, target, version) = parser.parse_request_line(self.conn) - headers = parser.parse_request_headers(self.conn) - self._validate_request(method, target, version, headers) + retriever = PreambleRetriever(self.conn) - logging.debug("Parsed request-line: method: %s, target: %r", method, target) + while True: + line = self.conn.read_line() + + if line in ("\r\n", "\r", "\n"): + continue + + retriever.reset_buffer(line) + self._handle_message(retriever, line) + + def _handle_message(self, retriever, line): + lines = retriever.retrieve() + (method, target, version) = parser.parse_request_line(line) + headers = parser.parse_headers(lines) + + message = Message(version, method, target, headers, retriever.buffer) + + logging.debug("---request begin---\r\n%s---request end---", "".join(message.raw)) + + self._validate_request(message) body = b"" if self._has_body(headers): @@ -44,8 +62,13 @@ class RequestHandler: for buffer in retriever.retrieve(): body += buffer + message.body = body + # completed message - self._handle_message(method, target.path, body) + + cmd = command.create(message) + msg = cmd.execute() + self.conn.conn.sendall(msg) def _check_request_line(self, method: str, target: Union[ParseResultBytes, ParseResult], version): @@ -53,30 +76,34 @@ class RequestHandler: raise MethodNotAllowed(METHODS) if version not in ("1.0", "1.1"): - raise BadRequest() + raise HTTPVersionNotSupported() # only origin-form and absolute-form are allowed if target.scheme not in ("", "http"): # Only http is supported... raise BadRequest() + if target.netloc != "" and target.netloc != self.conn.host and target.netloc != self.conn.host.split(":")[0]: raise NotFound() if target.path == "" or target.path[0] != "/": raise NotFound() - norm_path = os.path.normpath(target.path) - if not os.path.exists(self.root + norm_path): - raise NotFound() - - def _validate_request(self, method, target, version, headers): - if version == "1.1" and "host" not in headers: + def _validate_request(self, msg): + if msg.version == "1.1" and "host" not in msg.headers: raise BadRequest() - self._check_request_line(method, target, version) + self._check_request_line(msg.method, msg.target, msg.version) def _has_body(self, headers): - return "transfer-encoding" in headers or "content-encoding" in headers + + if "transfer-encoding" in headers: + return True + + if "content-length" in headers and int(headers["content-length"]) > 0: + return True + + return False @staticmethod def _get_date(): @@ -84,38 +111,6 @@ class RequestHandler: stamp = mktime(now.timetuple()) return format_date_time(stamp) - def _handle_message(self, method: str, target, body: bytes): - date = self._get_date() - - if method == "GET": - if target == "/": - path = self.root + "/index.html" - else: - path = self.root + target - mime = mimetypes.guess_type(path)[0] - if mime.startswith("text"): - file = open(path, "rb", FORMAT) - else: - file = open(path, "rb") - buffer = file.read() - file.close() - - message = "HTTP/1.1 200 OK\r\n" - message += date + "\r\n" - if mime: - message += f"Content-Type: {mime}" - if mime.startswith("text"): - message += "; charset=UTF-8" - message += "\r\n" - message += f"Content-Length: {len(buffer)}\r\n" - message += "\r\n" - message = message.encode(FORMAT) - message += buffer - message += b"\r\n" - - logging.debug("Sending: %r", message) - self.conn.conn.sendall(message) - @staticmethod def send_error(client: socket, code, message): message = f"HTTP/1.1 {code} {message}\r\n" diff --git a/server/worker.py b/server/worker.py index 361536b..28f6ec9 100644 --- a/server/worker.py +++ b/server/worker.py @@ -5,7 +5,7 @@ import threading from concurrent.futures import ThreadPoolExecutor from httplib.exceptions import HTTPServerException, InternalServerError -from server.RequestHandler import RequestHandler +from server.requesthandler import RequestHandler THREAD_LIMIT = 128 @@ -62,15 +62,16 @@ class Worker: def _handle_client(self, conn: socket.socket, addr): try: - logging.debug("Handling client: %s", addr) - handler = RequestHandler(conn, self.host) handler.listen() except HTTPServerException as e: + logging.debug("HTTP Exception:", exc_info=e) RequestHandler.send_error(conn, e.status_code, e.message) + except socket.timeout: + logging.debug("Socket for client %s timed out", addr) except Exception as e: - RequestHandler.send_error(conn, InternalServerError.status_code, InternalServerError.message) logging.debug("Internal error", exc_info=e) + RequestHandler.send_error(conn, InternalServerError.status_code, InternalServerError.message) conn.shutdown(socket.SHUT_RDWR) conn.close()