I had a blast to get a awesome code reviewer yesterday who taught me very much and I do hope I have done a pretty good job on improving my knowledge both with exceptions and in general.
Currently I have been working on to get a nice exception/more correctly way to do the exceptions as well.
For now I have done something like this:
from typing import Iterable
import requests
from requests.exceptions import ConnectionError, ConnectTimeout, HTTPError, ProxyError, ReadTimeout, RequestException,
Timeout
# ----------------------------------------------------
# Exceptions
# ----------------------------------------------------
class RequestsException(Exception):
"""
Base exception class for Requests
"""
class TooManyFailedRequests(RequestsException):
"""
Raise an exception for FailedRequests
"""
class TooManyTimedOut(RequestsException):
"""
Raise an exception for TimedOut
"""
# ----------------------------------------------------
# Counter Exception
# ----------------------------------------------------
class ExceptionCounter:
"""
Counter to check if we get exceptions x times in a row.
"""
def __init__(self, limits):
self.limits = limits
self.exception_counts: DefaultDict(Type(Exception), int) = defaultdict(int)
def check(self, exception):
allowed_count = self.limits.get(type(exception), 3)
self.exception_counts(type(exception)) += 1
if self.exception_counts(type(exception)) >= allowed_count:
raise
def reset(self):
self.exception_counts.clear()
simple_exception = ExceptionCounter({})
# ----------------------------------------------------
# Run requests
# ----------------------------------------------------
def run_request(site_url: str, session: requests) -> Iterable(requests.Response):
try:
return get(
site_url=site_url,
session=session
)
except HTTPError as err:
raise TooManyFailedRequests("Too many response status err", err)
except (ReadTimeout, Timeout, ConnectionError, ProxyError) as err:
raise TooManyTimedOut("Msg", err)
except Exception as err:
print("We dont know")
raise Exception("We dont know")
# ----------------------------------------------------
# Do the request
# ----------------------------------------------------
def get(site_url, session) -> Iterable(requests.Response):
while True:
try:
response = session.get(site_url, timeout=10)
if response.ok or response.status_code == 404:
return response
response.raise_for_status()
except HTTPError as http_error:
# In the event of the rare invalid HTTP response, Requests will raise an HTTPError exception. Response.raise_for_status() will raise an HTTPError if the HTTP request returned an unsuccessful status code.
print(f'HTTP Error | URL -> {site_url} | Exception -> {http_error}')
simple_exception.check(exception=http_error)
except ProxyError as proxy_error:
print(f'Proxy Error | URL -> {site_url} | Exception -> {proxy_error}')
# Manage Proxy Error. Re-try if proxy returns 503
if "503 Service Unavailable" not in str(proxy_error):
print("Unknown Proxy error")
simple_exception.check(exception=proxy_error)
except (ReadTimeout, Timeout, ConnectTimeout, ConnectionError) as err:
# Manage Read timeouts, Timeouts, Connection error
print(f'Timeout/Connection | URL -> {site_url} | Exception -> {err}')
simple_exception.check(exception=err)
except RequestException as uncaught_req_error:
# All exceptions that Requests explicitly raises inherit from requests.exceptions.RequestException. So a base handler can look like,
print(f'RequestException | URL -> {site_url} | Exception -> {uncaught_req_error}')
simple_exception.check(exception=uncaught_req_error)
except Exception as unexpected_error:
print(f'Unexpected caught | URL -> {site_url} | Exception -> {unexpected_error}')
simple_exception.check(exception=unexpected_error)
def main():
session = requests.session()
test = run_request("https://stackoverflow.com/testing123", session=session)
print(test)
if __name__ == '__main__':
main()
I wonder if i’m doing it correct or if there is anything I can improve 🙂