diff options
| author | Jordan Cook <jordan.cook@pioneer.com> | 2021-05-10 19:25:38 -0500 |
|---|---|---|
| committer | Jordan Cook <jordan.cook@pioneer.com> | 2021-05-26 20:48:16 -0500 |
| commit | 2f3cfbf926e2c99919e7a5b02f4926d5dbd67f24 (patch) | |
| tree | 04dc4232a31aaea8239d382ebdfec7efbad1d5ee | |
| parent | 865d9825b78d4f4571d5b20990c0c498f2d0ab99 (diff) | |
| download | requests-cache-2f3cfbf926e2c99919e7a5b02f4926d5dbd67f24.tar.gz | |
Split out serializers and models into separate classes
| -rw-r--r-- | pyproject.toml | 2 | ||||
| -rw-r--r-- | requests_cache/__init__.py | 3 | ||||
| -rw-r--r-- | requests_cache/backends/base.py | 19 | ||||
| -rw-r--r-- | requests_cache/models/__init__.py | 16 | ||||
| -rw-r--r-- | requests_cache/models/raw_response.py | 83 | ||||
| -rw-r--r-- | requests_cache/models/request.py | 51 | ||||
| -rwxr-xr-x | requests_cache/models/response.py | 154 | ||||
| -rw-r--r-- | requests_cache/patcher.py | 2 | ||||
| -rwxr-xr-x | requests_cache/response.py | 306 | ||||
| -rw-r--r-- | requests_cache/serializers/__init__.py | 3 | ||||
| -rw-r--r-- | requests_cache/serializers/base.py | 64 | ||||
| -rw-r--r-- | requests_cache/serializers/pickle.py | 34 | ||||
| -rw-r--r-- | requests_cache/session.py | 4 |
13 files changed, 422 insertions, 319 deletions
diff --git a/pyproject.toml b/pyproject.toml index 1f28015..6219c4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,8 @@ source = ['requests_cache'] omit = [ 'requests_cache/__init__.py', 'requests_cache/backends/__init__.py', + 'requests_cache/models/__init__.py', + 'requests_cache/serializers/__init__.py', ] [tool.isort] diff --git a/requests_cache/__init__.py b/requests_cache/__init__.py index 66de2c7..dbd4b85 100644 --- a/requests_cache/__init__.py +++ b/requests_cache/__init__.py @@ -10,7 +10,8 @@ logger = getLogger(__name__) try: from .backends import * from .patcher import * - from .response import CachedHTTPResponse, CachedResponse + from .models import * + from .serializers import * from .session import ALL_METHODS, CachedSession, CacheMixin # Log and ignore ImportErrors, if setup.py is invoked outside a virtualenv except ImportError as e: diff --git a/requests_cache/backends/base.py b/requests_cache/backends/base.py index ebc0b93..9c4c95f 100644 --- a/requests_cache/backends/base.py +++ b/requests_cache/backends/base.py @@ -9,8 +9,10 @@ from typing import Iterable, Iterator, Tuple, Union import requests from requests.models import PreparedRequest +from ..cache_control import ExpirationTime from ..cache_keys import create_key, url_to_key -from ..response import AnyResponse, CachedResponse, ExpirationTime +from ..models.response import AnyResponse, CachedResponse +from ..serializers import PickleSerializer, SafePickleSerializer # Specific exceptions that may be raised during deserialization DESERIALIZE_ERRORS = (AttributeError, TypeError, ValueError, pickle.PickleError) @@ -83,6 +85,7 @@ class BaseCache: return default except DESERIALIZE_ERRORS as e: logger.error(f'Unable to deserialize response with key {key}: {str(e)}') + logger.debug(e, exc_info=True) return default def delete(self, key: str): @@ -214,8 +217,8 @@ class BaseStorage(MutableMapping, ABC): serializer=None, **kwargs, ): - self._serializer = serializer or self._get_serializer(secret_key, salt) - logger.debug(f'Initializing {type(self).__name__} with serializer: {self._serializer}') + self.serializer = serializer or self._get_serializer(secret_key, salt) + logger.debug(f'Initializing {type(self).__name__} with serializer: {self.serializer}') if not secret_key: level = DEBUG if suppress_warnings else WARNING @@ -223,11 +226,11 @@ class BaseStorage(MutableMapping, ABC): def serialize(self, item: ResponseOrKey) -> bytes: """Serialize a URL or response into bytes""" - return self._serializer.dumps(item) + return self.serializer.dumps(item) def deserialize(self, item: Union[ResponseOrKey, bytes]) -> ResponseOrKey: """Deserialize a cached URL or response""" - return self._serializer.loads(bytes(item)) + return self.serializer.loads(item) @staticmethod def _get_serializer(secret_key, salt): @@ -236,11 +239,9 @@ class BaseStorage(MutableMapping, ABC): """ # Import in function scope to make itsdangerous an optional dependency if secret_key: - from itsdangerous.serializer import Serializer - - return Serializer(secret_key, salt=salt, serializer=pickle) + return SafePickleSerializer(secret_key=secret_key, salt=salt) else: - return pickle + return PickleSerializer() def bulk_delete(self, keys: Iterable[str]): """Delete multiple keys from the cache. Does not raise errors for missing keys. This is a diff --git a/requests_cache/models/__init__.py b/requests_cache/models/__init__.py new file mode 100644 index 0000000..4a9d00d --- /dev/null +++ b/requests_cache/models/__init__.py @@ -0,0 +1,16 @@ +# flake8: noqa: F401 +import attr + +dataclass = attr.s( + auto_attribs=False, + auto_detect=True, + collect_by_mro=True, + kw_only=True, + slots=True, + weakref_slot=False, +) + + +from .raw_response import CachedHTTPResponse +from .request import CachedRequest +from .response import CachedResponse diff --git a/requests_cache/models/raw_response.py b/requests_cache/models/raw_response.py new file mode 100644 index 0000000..fd1875f --- /dev/null +++ b/requests_cache/models/raw_response.py @@ -0,0 +1,83 @@ +from io import BytesIO +from logging import getLogger + +import attr +from requests import Response +from requests.structures import CaseInsensitiveDict +from urllib3.response import HTTPResponse, is_fp_closed + +logger = getLogger(__name__) + + +@attr.s(auto_attribs=False, auto_detect=True, init=False, kw_only=True) +class CachedHTTPResponse(HTTPResponse): + """A serializable dataclass that emulates :py:class:`~urllib3.response.HTTPResponse`. + Supports streaming requests and generator usage. + + The only action this doesn't support is explicitly calling :py:meth:`.read` with + ``decode_content=False``, but a use case for this has not come up yet. + """ + + decode_content: bool = attr.ib(default=None) + headers: CaseInsensitiveDict = attr.ib(factory=dict) + reason: str = attr.ib(default=None) + request_url: str = attr.ib(default=None) + status: int = attr.ib(default=0) + strict: int = attr.ib(default=0) + version: int = attr.ib(default=0) + + def __init__(self, *args, body: bytes = None, **kwargs): + """First initialize via HTTPResponse, then via attrs""" + super().__init__(body=BytesIO(body or b''), preload_content=False, **kwargs) + self._body = body + self.__attrs_init__(*args, **kwargs) + + @classmethod + def from_response(cls, original_response: Response): + """Create a CachedHTTPResponse based on an original response""" + # Copy basic attributes + raw = original_response.raw + kwargs = {k: getattr(raw, k, None) for k in attr.fields_dict(cls).keys()} + # TODO: Better means of handling naming differences between class attrs and method kwargs + kwargs['request_url'] = raw._request_url + + # Copy response data and restore response object to its original state + if hasattr(raw, '_fp') and not is_fp_closed(raw._fp): + body = raw.read(decode_content=False) + kwargs['body'] = body + raw._fp = BytesIO(body) + original_response.content # This property reads, decodes, and stores response content + + # After reading, reset file pointer on original raw response + raw._fp = BytesIO(body) + raw._fp_bytes_read = 0 + raw.length_remaining = len(body) + + return cls(**kwargs) + + def release_conn(self): + """No-op for compatibility""" + + def read(self, amt=None, decode_content=None, **kwargs): + """Simplified reader for cached content that emulates + :py:meth:`urllib3.response.HTTPResponse.read()` + """ + if 'content-encoding' in self.headers and decode_content is False: + logger.warning('read() returns decoded data, even with decode_content=False') + + data = self._fp.read(amt) + # "close" the file to inform consumers to stop reading from it + if not data: + self._fp.close() + return data + + def reset(self): + """Reset raw response file pointer""" + self._fp = BytesIO(self._body) + + def stream(self, amt=None, **kwargs): + """Simplified generator over cached content that emulates + :py:meth:`urllib3.response.HTTPResponse.stream()` + """ + while not self._fp.closed: + yield self.read(amt=amt, **kwargs) diff --git a/requests_cache/models/request.py b/requests_cache/models/request.py new file mode 100644 index 0000000..d5c1ba1 --- /dev/null +++ b/requests_cache/models/request.py @@ -0,0 +1,51 @@ +"""Classes to wrap cached response objects""" +from logging import getLogger +from typing import Any + +import attr +from requests import PreparedRequest +from requests.cookies import RequestsCookieJar +from requests.structures import CaseInsensitiveDict + +from . import dataclass + +logger = getLogger(__name__) + + +@dataclass +class CachedRequest: + """A serializable dataclass that emulates :py:class:`requests.PreparedResponse`""" + + body: Any = attr.ib(default=None) + cookies: RequestsCookieJar = attr.ib(factory=dict) + headers: CaseInsensitiveDict = attr.ib(factory=CaseInsensitiveDict) + method: str = attr.ib(default=None) + url: str = attr.ib(default=None) + + @classmethod + def from_request(cls, original_request: PreparedRequest): + """Create a CachedRequest based on an original request object""" + kwargs = {k: getattr(original_request, k, None) for k in attr.fields_dict(cls).keys()} + # TODO: Better means of handling naming differences between class attrs and method kwargs + kwargs['cookies'] = original_request._cookies + return cls(**kwargs) + + # TODO: Is this necessary, or will cattr.structure() be sufficient? + @classmethod + def prepare(self, obj) -> PreparedRequest: + """Turn a CachedRequest object back into a PreparedRequest. This lets PreparedRequest do the + work of normalizing any values that may have changed during (de)serialization. + """ + req = PreparedRequest() + kwargs = attr.asdict(obj) + # TODO: Better means of handling naming differences between class attrs and method kwargs + kwargs['data'] = kwargs.pop('body') + req.prepare(**kwargs) + return req + + @property + def _cookies(self): + return self.cookies + + def __str__(self): + return f'{self.method} {self.url}' diff --git a/requests_cache/models/response.py b/requests_cache/models/response.py new file mode 100755 index 0000000..eddffeb --- /dev/null +++ b/requests_cache/models/response.py @@ -0,0 +1,154 @@ +"""Classes to wrap cached response objects""" +from datetime import datetime, timedelta, timezone +from logging import getLogger +from typing import List, Optional, Tuple, Union + +import attr +from requests import Response +from requests.cookies import RequestsCookieJar +from requests.structures import CaseInsensitiveDict + +from ..cache_control import ExpirationTime, get_expiration_datetime +from . import CachedHTTPResponse, CachedRequest, dataclass + +DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' # Format used for __str__ only +DO_NOT_CACHE = 0 + +HeaderList = List[Tuple[str, str]] + +logger = getLogger(__name__) + + +# TODO: Make this fully take advantage of slots +# Make a slotted copy of Response to subclass; we don't need its attrs, only its methods +# from requests import Response as OriginalResponse +# Response = attr.s(slots=True)(OriginalResponse) +@dataclass +class CachedResponse(Response): + """A serializable dataclass that emulates :py:class:`requests.Response`. Public attributes and + methods on CachedResponse objects will behave the same as those from the original response, but + with different internals optimized for serialization. + + This means doing some pre- and post-initialization steps common to all serializers, such as + breaking nested objects down into their basic attributes and lazily re-initializing them, which + saves a bit of memory and deserialization steps when those objects aren't accessed. + """ + + _content: bytes = attr.ib(default=b'', repr=False, converter=lambda x: x or b'') + url: str = attr.ib(default=None) + status_code: int = attr.ib(default=0) + cookies: RequestsCookieJar = attr.ib(factory=dict) + created_at: datetime = attr.ib(factory=datetime.utcnow) + elapsed: timedelta = attr.ib(factory=timedelta) + expires: datetime = attr.ib(default=None) + encoding: str = attr.ib(default=None) + headers: CaseInsensitiveDict = attr.ib(factory=dict) + history: List = attr.ib(factory=list) + reason: str = attr.ib(default=None) + request: CachedRequest = attr.ib(factory=CachedRequest) + raw: CachedHTTPResponse = attr.ib(factory=CachedHTTPResponse, repr=False) + + @classmethod + def from_response(cls, original_response: Response, **kwargs): + """Create a CachedResponse based on an original response object""" + obj = cls(**kwargs) + + # Copy basic attributes + for k in Response.__attrs__: + setattr(obj, k, getattr(original_response, k, None)) + + # Store request and raw response + obj.request = CachedRequest.from_request(original_response.request) + obj.raw = CachedHTTPResponse.from_response(original_response) + + # Store response body, which will have been read & decoded by requests.Response by now + obj._content = original_response.content + + # Copy redirect history, if any; avoid recursion by not copying redirects of redirects + obj.history = [] + if not obj.is_redirect: + for redirect in original_response.history: + obj.history.append(cls.from_response(redirect)) + + return obj + + @property + def _content_consumed(self): + """For compatibility with Response; will always be True for a cached response""" + return True + + @_content_consumed.setter + def _content_consumed(self, value): + pass + + @property + def from_cache(self) -> bool: + return True + + @property + def is_expired(self) -> bool: + """Determine if this cached response is expired""" + return self.expires is not None and datetime.utcnow() >= self.expires + + def revalidate(self, expire_after: ExpirationTime) -> bool: + """Set a new expiration for this response, and determine if it is now expired""" + self.expires = get_expiration_datetime(expire_after) + return self.is_expired + + def reset(self): + if self.raw: + self.raw.reset() + + @property + def size(self) -> int: + """Get the size of the response body in bytes""" + return len(self.content) if self.content else 0 + + # TODO: Behavior will be different for slotted classes + # def __getstate__(self): + # """Override pickling behavior in ``requests.Response.__getstate__``""" + # return self.__dict__ + + def __str__(self): + return ( + f'request: {self.request}, response: {self.status_code} ' + f'({format_file_size(self.size)}), created: {format_datetime(self.created_at)}, ' + f'expires: {format_datetime(self.expires)} ({"stale" if self.is_expired else "fresh"})' + ) + + +AnyResponse = Union[Response, CachedResponse] + + +def format_datetime(value: Optional[datetime]) -> str: + """Get a formatted datetime string in the local time zone""" + if not value: + return "N/A" + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone().strftime(DATETIME_FORMAT) + + +def format_file_size(n_bytes: int) -> str: + """Convert a file size in bytes into a human-readable format""" + filesize = float(n_bytes or 0) + + def _format(unit): + return f'{int(filesize)} {unit}' if unit == 'bytes' else f'{filesize:.2f} {unit}' + + for unit in ['bytes', 'KiB', 'MiB', 'GiB']: + if filesize < 1024 or unit == 'GiB': + return _format(unit) + filesize /= 1024 + + +def set_response_defaults(response: AnyResponse) -> AnyResponse: + """Set some default CachedResponse values on a requests.Response object, so they can be + expected to always be present + """ + if not isinstance(response, CachedResponse): + response.created_at = None + response.expires = None + response.from_cache = False + response.is_expired = False + return response diff --git a/requests_cache/patcher.py b/requests_cache/patcher.py index 7d89acc..d947e94 100644 --- a/requests_cache/patcher.py +++ b/requests_cache/patcher.py @@ -10,7 +10,7 @@ from typing import Callable, Dict, Iterable, Optional, Type import requests from .backends import BackendSpecifier, BaseCache -from .response import ExpirationTime +from .cache_control import ExpirationTime from .session import CachedSession, OriginalSession logger = getLogger(__name__) diff --git a/requests_cache/response.py b/requests_cache/response.py deleted file mode 100755 index 687d884..0000000 --- a/requests_cache/response.py +++ /dev/null @@ -1,306 +0,0 @@ -# TODO: Maybe split this into separate modules -"""Classes to wrap cached response objects""" -from datetime import datetime, timedelta, timezone -from io import BytesIO -from logging import getLogger -from typing import Dict, List, Optional, Tuple, Union - -import attr -import cattr -from requests import PreparedRequest, Response -from requests.cookies import RequestsCookieJar, cookiejar_from_dict -from requests.structures import CaseInsensitiveDict -from urllib3.response import HTTPResponse, is_fp_closed - -from .cache_control import get_expiration_datetime - -logger = getLogger(__name__) - -DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' # Format used for __str__ only -DO_NOT_CACHE = 0 - -ExpirationTime = Union[None, int, float, datetime, timedelta] -HeaderList = List[Tuple[str, str]] - -# Aliases for the most common attr options -dataclass = attr.s( - auto_attribs=False, - auto_detect=True, - collect_by_mro=True, - kw_only=True, - slots=True, - weakref_slot=False, -) -public_attr = attr.ib(default=None) -bytes_attr = attr.ib(default=b'', repr=False, converter=lambda x: x or b'') - - -@dataclass -class CachedHTTPResponse(HTTPResponse): - """A serializable dataclass that emulates :py:class:`~urllib3.response.HTTPResponse`. - Supports streaming requests and generator usage. - - The only action this doesn't support is explicitly calling :py:meth:`.read` with - ``decode_content=False``, but a use case for this has not come up yet. - """ - - decode_content: bool = public_attr - headers: CaseInsensitiveDict = attr.ib(factory=dict) - reason: str = public_attr - request_url: str = public_attr - status: int = public_attr - strict: int = public_attr - version: int = public_attr - - def __attrs_post_init__(self, body: bytes = None, **kwargs): - kwargs.setdefault('preload_content', False) - super().__init__(body=BytesIO(body or b''), **kwargs) - self._body = body - - @classmethod - def from_response(cls, original_response: Response): - """Create a CachedHTTPResponse based on an original response object's raw response""" - # Copy basic attributes - raw = original_response.raw - kwargs = {k: getattr(raw, k, None) for k in attr.fields_dict(cls).keys()} - # TODO: Better means of handling naming differences between class attrs and method kwargs - kwargs['request_url'] = raw._request_url - - # Copy response data and restore response object to its original state - if not is_fp_closed(getattr(original_response.raw, '_fp', None)): - body = raw.read(decode_content=False) - kwargs['body'] = body - raw._fp = BytesIO(body) - original_response.content # This property reads, decodes, and stores response content - - # After reading, reset file pointer on original raw response - raw._fp = BytesIO(body) - raw._fp_bytes_read = 0 - raw.length_remaining = len(body) - - return cls(**kwargs) - - def release_conn(self): - """No-op for compatibility""" - - def read(self, amt=None, decode_content=None, **kwargs): - """Simplified reader for cached content that emulates - :py:meth:`urllib3.response.HTTPResponse.read()` - """ - if 'content-encoding' in self.headers and decode_content is False: - logger.warning('read() returns decoded data, even with decode_content=False') - - data = self._fp.read(amt) - # "close" the file to inform consumers to stop reading from it - if not data: - self._fp.close() - return data - - def reset(self): - """Reset raw response file pointer""" - self._fp = BytesIO(self._body) - - def stream(self, amt=None, **kwargs): - """Simplified generator over cached content that emulates - :py:meth:`urllib3.response.HTTPResponse.stream()` - """ - while not self._fp.closed: - yield self.read(amt=amt, **kwargs) - - -@dataclass -class CachedRequest: - """A serializable dataclass that emulates :py:class:`requests.PreparedResponse`""" - - body: bytes = bytes_attr - cookies: RequestsCookieJar = public_attr - headers: CaseInsensitiveDict = attr.ib(factory=CaseInsensitiveDict) - method: str = public_attr - url: str = public_attr - - @classmethod - def from_request(cls, original_request: PreparedRequest): - """Create a CachedRequest based on an original request object""" - kwargs = {k: getattr(original_request, k, None) for k in attr.fields_dict(cls).keys()} - # TODO: Better means of handling naming differences between class attrs and method kwargs - kwargs['cookies'] = original_request._cookies - return cls(**kwargs) - - # TODO: Is this necessary, or will cattr.structure() be sufficient? - @classmethod - def prepare(self, obj) -> PreparedRequest: - """Turn a CachedRequest object back into a PreparedRequest. This lets PreparedRequest do the - work of normalizing any values that may have changed during (de)serialization. - """ - req = PreparedRequest() - kwargs = attr.asdict(obj) - # TODO: Better means of handling naming differences between class attrs and method kwargs - kwargs['_cookies'] = kwargs.pop('cookies') - kwargs['body'] = kwargs.pop('data') - req.prepare(**kwargs) - return req - - @property - def _cookies(self): - return self.cookies - - -# TODO: Make this fully take advantage of slots -# Make a slotted copy of Response to subclass; we don't need its attrs, only its methods - -# from requests import Response as OriginalResponse -# Response = attr.s(slots=True)(OriginalResponse) -# @attr.s(kw_only=True, slots=True) -@dataclass -class CachedResponse(Response): - """A serializable dataclass that emulates :py:class:`requests.Response`. Public attributes and - methods on CachedResponse objects will behave the same as those from the original response, but - with different internals optimized for serialization. - - This means doing some pre- and post-initialization steps common to all serializers, such as - breaking nested objects down into their basic attributes and lazily re-initializing them, which - saves a bit of memory and deserialization steps when those objects aren't accessed. - """ - - _content: bytes = bytes_attr - url: str = public_attr - status_code: int = public_attr - cookies: RequestsCookieJar = public_attr - created_at: datetime = attr.ib(factory=datetime.utcnow) - elapsed: timedelta = attr.ib(factory=timedelta) - expires: datetime = public_attr - encoding: str = public_attr - headers: CaseInsensitiveDict = attr.ib(factory=dict) - history: List = attr.ib(factory=list) - reason: str = public_attr - request: CachedRequest = public_attr - raw: CachedHTTPResponse = attr.ib(default=None, repr=False) - - @classmethod - def from_response(cls, original_response: Response, **kwargs): - """Create a CachedResponse based on an original response object""" - obj = cls(**kwargs) - - # Copy basic attributes - for k in Response.__attrs__: - setattr(obj, k, getattr(original_response, k, None)) - - # Store request and raw response - obj.request = CachedRequest.from_request(original_response.request) - obj.raw = CachedHTTPResponse.from_response(original_response) - - # Store response body, which will have been read & decoded by requests.Response by now - obj._content = original_response.content - - # Copy redirect history, if any; avoid recursion by not copying redirects of redirects - obj.history = [] - if not obj.is_redirect: - for redirect in original_response.history: - obj.history.append(cls.from_response(redirect)) - - return obj - - @property - def from_cache(self) -> bool: - return True - - @property - def is_expired(self) -> bool: - """Determine if this cached response is expired""" - return self.expires is not None and datetime.utcnow() >= self.expires - - def revalidate(self, expire_after: ExpirationTime) -> bool: - """Set a new expiration for this response, and determine if it is now expired""" - self.expires = get_expiration_datetime(expire_after) - return self.is_expired - - def reset(self): - if self.raw: - self.raw.reset() - - @property - def size(self) -> int: - """Get the size of the response body in bytes""" - return len(self.content) if self.content else 0 - - # TODO: Behavior will be different for slotted classes - # def __getstate__(self): - # """Override pickling behavior in ``requests.Response.__getstate__``""" - # return self.__dict__ - - def __str__(self): - return ( - f'request: {self.request.method} {self.request.url}, response: {self.status_code} ' - f'({format_file_size(self.size)}), created: {format_datetime(self.created_at)}, ' - f'expires: {format_datetime(self.expires)} ({"stale" if self.is_expired else "fresh"})' - ) - - -# TODO: Should this go in a base serializer class instead? -def get_converter(): - """Make a converter to structure and unstructure some of the nested objects within a response""" - converter = cattr.Converter() - - # Convert datetimes to and from iso-formatted strings - converter.register_unstructure_hook(datetime, lambda obj: obj.isoformat() if obj else None) - converter.register_structure_hook( - datetime, lambda obj, cls: datetime.fromisoformat(obj) if obj else None - ) - - # Convert timedeltas to and from float values in seconds - converter.register_unstructure_hook(timedelta, lambda obj: obj.total_seconds() if obj else None) - converter.register_structure_hook( - timedelta, lambda obj, cls: timedelta(seconds=obj) if obj else None - ) - - # Convert dict-like objects to and from plain dicts - converter.register_unstructure_hook(RequestsCookieJar, lambda obj: dict(obj.items())) - converter.register_structure_hook(RequestsCookieJar, lambda obj, cls: cookiejar_from_dict(obj)) - converter.register_unstructure_hook(CaseInsensitiveDict, dict) - converter.register_structure_hook(CaseInsensitiveDict, lambda obj, cls: CaseInsensitiveDict(obj)) - - # Not sure yet if this will be needed - # converter.register_unstructure_hook(PreparedRequest, CachedRequest.from_request) - # converter.register_structure_hook(PreparedRequest, CachedRequest.prepare) - # converter.register_unstructure_hook(HTTPResponse, lambda obj, cls: CachedHTTPResponse.from_response(obj)) - # converter.register_structure_hook(HTTPResponse, lambda obj, cls: CachedHTTPResponse(obj)) - - return converter - - -AnyResponse = Union[Response, CachedResponse] -ResponseConverter = get_converter() - - -def format_datetime(value: Optional[datetime]) -> str: - """Get a formatted datetime string in the local time zone""" - if not value: - return "N/A" - if value.tzinfo is None: - value = value.replace(tzinfo=timezone.utc) - return value.astimezone().strftime(DATETIME_FORMAT) - - -def format_file_size(n_bytes: int) -> str: - """Convert a file size in bytes into a human-readable format""" - filesize = float(n_bytes or 0) - - def _format(unit): - return f'{int(filesize)} {unit}' if unit == 'bytes' else f'{filesize:.2f} {unit}' - - for unit in ['bytes', 'KiB', 'MiB', 'GiB']: - if filesize < 1024 or unit == 'GiB': - return _format(unit) - filesize /= 1024 - - -def set_response_defaults(response: AnyResponse) -> AnyResponse: - """Set some default CachedResponse values on a requests.Response object, so they can be - expected to always be present - """ - if not isinstance(response, CachedResponse): - response.created_at = None - response.expires = None - response.from_cache = False - response.is_expired = False - return response diff --git a/requests_cache/serializers/__init__.py b/requests_cache/serializers/__init__.py new file mode 100644 index 0000000..c2cd5b2 --- /dev/null +++ b/requests_cache/serializers/__init__.py @@ -0,0 +1,3 @@ +# flake8: noqa: F401 +from .base import BaseSerializer +from .pickle import PickleSerializer, SafePickleSerializer diff --git a/requests_cache/serializers/base.py b/requests_cache/serializers/base.py new file mode 100644 index 0000000..bc2b732 --- /dev/null +++ b/requests_cache/serializers/base.py @@ -0,0 +1,64 @@ +from abc import abstractmethod +from datetime import datetime, timedelta +from typing import Dict + +import cattr +from requests.cookies import RequestsCookieJar, cookiejar_from_dict +from requests.structures import CaseInsensitiveDict + +from ..models import CachedResponse + + +class BaseSerializer: + """Base serializer class for :py:class:`.CachedResponse` that does pre/post-processing with cattrs. + This does the majority of the work to break objects down into builtin types and reassemble them + without data loss. Subclasses just need to provide ``dumps`` and ``loads`` methods. + """ + + is_binary = True # TODO: This may or may not be needed to determine return type in backends + + def __init__(self, *args, **kwargs): + """Make a converter to structure and unstructure some of the nested objects within a response""" + super().__init__(*args, **kwargs) + converter = cattr.Converter() + + # Convert datetimes to and from iso-formatted strings + converter.register_unstructure_hook(datetime, lambda obj: obj.isoformat() if obj else None) + converter.register_structure_hook( + datetime, lambda obj, cls: datetime.fromisoformat(obj) if obj else None + ) + + # Convert timedeltas to and from float values in seconds + converter.register_unstructure_hook(timedelta, lambda obj: obj.total_seconds() if obj else None) + converter.register_structure_hook( + timedelta, lambda obj, cls: timedelta(seconds=obj) if obj else None + ) + + # Convert dict-like objects to and from plain dicts + converter.register_unstructure_hook(RequestsCookieJar, lambda obj: dict(obj.items())) + converter.register_structure_hook(RequestsCookieJar, lambda obj, cls: cookiejar_from_dict(obj)) + converter.register_unstructure_hook(CaseInsensitiveDict, dict) + converter.register_structure_hook(CaseInsensitiveDict, lambda obj, cls: CaseInsensitiveDict(obj)) + + # Not sure yet if this will be needed + # converter.register_unstructure_hook(PreparedRequest, CachedRequest.from_request) + # converter.register_structure_hook(PreparedRequest, lambda obj, cls: CachedRequest.prepare(obj)) + # converter.register_unstructure_hook(HTTPResponse, lambda obj, cls: CachedHTTPResponse.from_response(obj)) + # converter.register_structure_hook(HTTPResponse, lambda obj, cls: CachedHTTPResponse(obj)) + # converter.register_structure_hook(CachedRequest, lambda obj, cls: cls.prepare(obj)) + + self.converter = converter + + def unstructure(self, response: CachedResponse) -> Dict: + return self.converter.unstructure(response) + + def structure(self, obj: Dict) -> CachedResponse: + return self.converter.structure(obj, CachedResponse) + + @abstractmethod + def dumps(self, response: CachedResponse): + pass + + @abstractmethod + def loads(self, obj) -> CachedResponse: + pass diff --git a/requests_cache/serializers/pickle.py b/requests_cache/serializers/pickle.py new file mode 100644 index 0000000..c1a5726 --- /dev/null +++ b/requests_cache/serializers/pickle.py @@ -0,0 +1,34 @@ +import pickle + +from itsdangerous.serializer import Serializer as SafeSerializer + +from ..models import CachedResponse +from .base import BaseSerializer + + +class PickleSerializer(BaseSerializer): + """Wrapper for pickle that pre/post-processes with cattrs""" + + def dumps(self, response: CachedResponse) -> bytes: + return pickle.dumps(super().unstructure(response)) + + def loads(self, obj: bytes) -> CachedResponse: + return super().structure(pickle.loads(obj)) + + +class SafePickleSerializer(BaseSerializer, SafeSerializer): + """Wrapper for itsdangerous + pickle that pre/post-processes with cattrs""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs, serializer=PickleSerializer()) + + def dumps(self, response: CachedResponse) -> bytes: + x = super().unstructure(response) + # breakpoint() + return SafeSerializer.dumps(self, x) + + # TODO: Something weird is going on here + def loads(self, obj: bytes) -> CachedResponse: + return SafeSerializer.loads(self, obj) + # breakpoint() + return super().structure(SafeSerializer.loads(self, obj)) diff --git a/requests_cache/session.py b/requests_cache/session.py index 91d0a5d..93e3608 100644 --- a/requests_cache/session.py +++ b/requests_cache/session.py @@ -10,9 +10,9 @@ from requests.hooks import dispatch_hook from urllib3 import filepost from .backends import BackendSpecifier, get_valid_kwargs, init_backend -from .cache_control import CacheActions +from .cache_control import CacheActions, ExpirationTime from .cache_keys import normalize_dict -from .response import AnyResponse, ExpirationTime, set_response_defaults +from .models.response import AnyResponse, set_response_defaults ALL_METHODS = ['GET', 'HEAD', 'OPTIONS', 'POST', 'PUT', 'PATCH', 'DELETE'] logger = getLogger(__name__) |
