Principis/CN2021
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

client: fix image url parsing

Browse Source
This commit is contained in:
Arthur Bols
2021-03-25 17:56:21 +01:00
parent b37aa33131
commit f15ff38f69
7 changed files with 24 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
client/command.py
View File

@@ -148,9 +148,9 @@ class GetCommand(AbstractCommand):
(version, status, msg) = parser.parse_status_line(next(lines))
headers = parser.parse_headers(lines)
logging.debug("---response begin---\r\n%s--- response end---", "".join(retriever.buffer))
logging.debug("---response begin---\r\n%s---response end---", "".join(retriever.buffer))
return Message(version, status, msg, headers)
return Message(version, status, msg, headers, retriever.buffer)
def _await_response(self, client, retriever):
msg = self._get_preamble(retriever)

23
client/response_handler.py
View File

@@ -88,8 +88,7 @@ class BasicResponseHandler(ResponseHandler):
if self.msg.status == 101:
# Switching protocols is not supported
print(f"{self.msg.version} {self.msg.status} {self.msg.msg}")
print(self.msg.headers)
print("".join(self.msg.raw), end="")
return
if 200 <= self.msg.status < 300:
@@ -100,8 +99,7 @@ class BasicResponseHandler(ResponseHandler):
return self._do_handle_redirect()
if 400 <= self.msg.status < 500:
# Dump headers and exit with error
print(f"{self.msg.version} {self.msg.status} {self.msg.msg}")
print(self.msg.headers)
print("".join(self.msg.raw), end="")
return None
def _do_handle_redirect(self):
@@ -216,15 +214,14 @@ class HTMLDownloadHandler(DownloadHandler):
def _download_images(self, tmp_filename, target_filename):
(host, path) = ResponseHandler.parse_uri(self.cmd.uri)
with open(tmp_filename, "rb") as fp:
soup = BeautifulSoup(fp, 'lxml')
base_url = self.cmd.uri
base_url = parser.base_url(self.cmd.uri)
base_element = soup.find("base")
if base_element:
base_url = base_element["href"]
base_url = f"http://{self.cmd.host}" + base_element["href"]
processed = {}
tag: Tag
@@ -236,7 +233,7 @@ class HTMLDownloadHandler(DownloadHandler):
if tag["src"] in processed:
new_url = processed.get(tag["src"])
else:
new_url = self.__download_image(tag["src"], host, base_url)
new_url = self.__download_image(tag["src"], base_url)
processed[tag["src"]] = new_url
if new_url:
tag["src"] = os.path.basename(new_url)
@@ -246,8 +243,8 @@ class HTMLDownloadHandler(DownloadHandler):
with open(target_filename, 'w') as file:
file.write(str(soup))
def __download_image(self, img_src, host, base_url):
logging.debug("Downloading image: %s", img_src)
def __download_image(self, img_src, base_url):
logging.info("Downloading image: %s", img_src)
parsed = urlsplit(img_src)
@@ -257,11 +254,11 @@ class HTMLDownloadHandler(DownloadHandler):
if parsed.hostname is None:
if img_src[0] == "/":
img_src = host + img_src
img_src = f"http://{self.cmd.host}{img_src}"
else:
img_src = os.path.join(os.path.dirname(base_url), img_src)
img_src = os.path.join(base_url, img_src)
if parsed.hostname is None or parsed.hostname == host:
if parsed.hostname is None or parsed.hostname == self.cmd.host:
port = self.cmd.port
elif ":" in parsed.netloc:
port = parsed.netloc.split(":", 1)[1]