summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJordan Cook <jordan.cook@pioneer.com>2021-05-10 19:25:38 -0500
committerJordan Cook <jordan.cook@pioneer.com>2021-05-26 20:48:16 -0500
commit2f3cfbf926e2c99919e7a5b02f4926d5dbd67f24 (patch)
tree04dc4232a31aaea8239d382ebdfec7efbad1d5ee
parent865d9825b78d4f4571d5b20990c0c498f2d0ab99 (diff)
downloadrequests-cache-2f3cfbf926e2c99919e7a5b02f4926d5dbd67f24.tar.gz
Split out serializers and models into separate classes
-rw-r--r--pyproject.toml2
-rw-r--r--requests_cache/__init__.py3
-rw-r--r--requests_cache/backends/base.py19
-rw-r--r--requests_cache/models/__init__.py16
-rw-r--r--requests_cache/models/raw_response.py83
-rw-r--r--requests_cache/models/request.py51
-rwxr-xr-xrequests_cache/models/response.py154
-rw-r--r--requests_cache/patcher.py2
-rwxr-xr-xrequests_cache/response.py306
-rw-r--r--requests_cache/serializers/__init__.py3
-rw-r--r--requests_cache/serializers/base.py64
-rw-r--r--requests_cache/serializers/pickle.py34
-rw-r--r--requests_cache/session.py4
13 files changed, 422 insertions, 319 deletions
diff --git a/pyproject.toml b/pyproject.toml
index 1f28015..6219c4a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -79,6 +79,8 @@ source = ['requests_cache']
omit = [
'requests_cache/__init__.py',
'requests_cache/backends/__init__.py',
+ 'requests_cache/models/__init__.py',
+ 'requests_cache/serializers/__init__.py',
]
[tool.isort]
diff --git a/requests_cache/__init__.py b/requests_cache/__init__.py
index 66de2c7..dbd4b85 100644
--- a/requests_cache/__init__.py
+++ b/requests_cache/__init__.py
@@ -10,7 +10,8 @@ logger = getLogger(__name__)
try:
from .backends import *
from .patcher import *
- from .response import CachedHTTPResponse, CachedResponse
+ from .models import *
+ from .serializers import *
from .session import ALL_METHODS, CachedSession, CacheMixin
# Log and ignore ImportErrors, if setup.py is invoked outside a virtualenv
except ImportError as e:
diff --git a/requests_cache/backends/base.py b/requests_cache/backends/base.py
index ebc0b93..9c4c95f 100644
--- a/requests_cache/backends/base.py
+++ b/requests_cache/backends/base.py
@@ -9,8 +9,10 @@ from typing import Iterable, Iterator, Tuple, Union
import requests
from requests.models import PreparedRequest
+from ..cache_control import ExpirationTime
from ..cache_keys import create_key, url_to_key
-from ..response import AnyResponse, CachedResponse, ExpirationTime
+from ..models.response import AnyResponse, CachedResponse
+from ..serializers import PickleSerializer, SafePickleSerializer
# Specific exceptions that may be raised during deserialization
DESERIALIZE_ERRORS = (AttributeError, TypeError, ValueError, pickle.PickleError)
@@ -83,6 +85,7 @@ class BaseCache:
return default
except DESERIALIZE_ERRORS as e:
logger.error(f'Unable to deserialize response with key {key}: {str(e)}')
+ logger.debug(e, exc_info=True)
return default
def delete(self, key: str):
@@ -214,8 +217,8 @@ class BaseStorage(MutableMapping, ABC):
serializer=None,
**kwargs,
):
- self._serializer = serializer or self._get_serializer(secret_key, salt)
- logger.debug(f'Initializing {type(self).__name__} with serializer: {self._serializer}')
+ self.serializer = serializer or self._get_serializer(secret_key, salt)
+ logger.debug(f'Initializing {type(self).__name__} with serializer: {self.serializer}')
if not secret_key:
level = DEBUG if suppress_warnings else WARNING
@@ -223,11 +226,11 @@ class BaseStorage(MutableMapping, ABC):
def serialize(self, item: ResponseOrKey) -> bytes:
"""Serialize a URL or response into bytes"""
- return self._serializer.dumps(item)
+ return self.serializer.dumps(item)
def deserialize(self, item: Union[ResponseOrKey, bytes]) -> ResponseOrKey:
"""Deserialize a cached URL or response"""
- return self._serializer.loads(bytes(item))
+ return self.serializer.loads(item)
@staticmethod
def _get_serializer(secret_key, salt):
@@ -236,11 +239,9 @@ class BaseStorage(MutableMapping, ABC):
"""
# Import in function scope to make itsdangerous an optional dependency
if secret_key:
- from itsdangerous.serializer import Serializer
-
- return Serializer(secret_key, salt=salt, serializer=pickle)
+ return SafePickleSerializer(secret_key=secret_key, salt=salt)
else:
- return pickle
+ return PickleSerializer()
def bulk_delete(self, keys: Iterable[str]):
"""Delete multiple keys from the cache. Does not raise errors for missing keys. This is a
diff --git a/requests_cache/models/__init__.py b/requests_cache/models/__init__.py
new file mode 100644
index 0000000..4a9d00d
--- /dev/null
+++ b/requests_cache/models/__init__.py
@@ -0,0 +1,16 @@
+# flake8: noqa: F401
+import attr
+
+dataclass = attr.s(
+ auto_attribs=False,
+ auto_detect=True,
+ collect_by_mro=True,
+ kw_only=True,
+ slots=True,
+ weakref_slot=False,
+)
+
+
+from .raw_response import CachedHTTPResponse
+from .request import CachedRequest
+from .response import CachedResponse
diff --git a/requests_cache/models/raw_response.py b/requests_cache/models/raw_response.py
new file mode 100644
index 0000000..fd1875f
--- /dev/null
+++ b/requests_cache/models/raw_response.py
@@ -0,0 +1,83 @@
+from io import BytesIO
+from logging import getLogger
+
+import attr
+from requests import Response
+from requests.structures import CaseInsensitiveDict
+from urllib3.response import HTTPResponse, is_fp_closed
+
+logger = getLogger(__name__)
+
+
+@attr.s(auto_attribs=False, auto_detect=True, init=False, kw_only=True)
+class CachedHTTPResponse(HTTPResponse):
+ """A serializable dataclass that emulates :py:class:`~urllib3.response.HTTPResponse`.
+ Supports streaming requests and generator usage.
+
+ The only action this doesn't support is explicitly calling :py:meth:`.read` with
+ ``decode_content=False``, but a use case for this has not come up yet.
+ """
+
+ decode_content: bool = attr.ib(default=None)
+ headers: CaseInsensitiveDict = attr.ib(factory=dict)
+ reason: str = attr.ib(default=None)
+ request_url: str = attr.ib(default=None)
+ status: int = attr.ib(default=0)
+ strict: int = attr.ib(default=0)
+ version: int = attr.ib(default=0)
+
+ def __init__(self, *args, body: bytes = None, **kwargs):
+ """First initialize via HTTPResponse, then via attrs"""
+ super().__init__(body=BytesIO(body or b''), preload_content=False, **kwargs)
+ self._body = body
+ self.__attrs_init__(*args, **kwargs)
+
+ @classmethod
+ def from_response(cls, original_response: Response):
+ """Create a CachedHTTPResponse based on an original response"""
+ # Copy basic attributes
+ raw = original_response.raw
+ kwargs = {k: getattr(raw, k, None) for k in attr.fields_dict(cls).keys()}
+ # TODO: Better means of handling naming differences between class attrs and method kwargs
+ kwargs['request_url'] = raw._request_url
+
+ # Copy response data and restore response object to its original state
+ if hasattr(raw, '_fp') and not is_fp_closed(raw._fp):
+ body = raw.read(decode_content=False)
+ kwargs['body'] = body
+ raw._fp = BytesIO(body)
+ original_response.content # This property reads, decodes, and stores response content
+
+ # After reading, reset file pointer on original raw response
+ raw._fp = BytesIO(body)
+ raw._fp_bytes_read = 0
+ raw.length_remaining = len(body)
+
+ return cls(**kwargs)
+
+ def release_conn(self):
+ """No-op for compatibility"""
+
+ def read(self, amt=None, decode_content=None, **kwargs):
+ """Simplified reader for cached content that emulates
+ :py:meth:`urllib3.response.HTTPResponse.read()`
+ """
+ if 'content-encoding' in self.headers and decode_content is False:
+ logger.warning('read() returns decoded data, even with decode_content=False')
+
+ data = self._fp.read(amt)
+ # "close" the file to inform consumers to stop reading from it
+ if not data:
+ self._fp.close()
+ return data
+
+ def reset(self):
+ """Reset raw response file pointer"""
+ self._fp = BytesIO(self._body)
+
+ def stream(self, amt=None, **kwargs):
+ """Simplified generator over cached content that emulates
+ :py:meth:`urllib3.response.HTTPResponse.stream()`
+ """
+ while not self._fp.closed:
+ yield self.read(amt=amt, **kwargs)
diff --git a/requests_cache/models/request.py b/requests_cache/models/request.py
new file mode 100644
index 0000000..d5c1ba1
--- /dev/null
+++ b/requests_cache/models/request.py
@@ -0,0 +1,51 @@
+"""Classes to wrap cached response objects"""
+from logging import getLogger
+from typing import Any
+
+import attr
+from requests import PreparedRequest
+from requests.cookies import RequestsCookieJar
+from requests.structures import CaseInsensitiveDict
+
+from . import dataclass
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class CachedRequest:
+ """A serializable dataclass that emulates :py:class:`requests.PreparedResponse`"""
+
+ body: Any = attr.ib(default=None)
+ cookies: RequestsCookieJar = attr.ib(factory=dict)
+ headers: CaseInsensitiveDict = attr.ib(factory=CaseInsensitiveDict)
+ method: str = attr.ib(default=None)
+ url: str = attr.ib(default=None)
+
+ @classmethod
+ def from_request(cls, original_request: PreparedRequest):
+ """Create a CachedRequest based on an original request object"""
+ kwargs = {k: getattr(original_request, k, None) for k in attr.fields_dict(cls).keys()}
+ # TODO: Better means of handling naming differences between class attrs and method kwargs
+ kwargs['cookies'] = original_request._cookies
+ return cls(**kwargs)
+
+ # TODO: Is this necessary, or will cattr.structure() be sufficient?
+ @classmethod
+ def prepare(self, obj) -> PreparedRequest:
+ """Turn a CachedRequest object back into a PreparedRequest. This lets PreparedRequest do the
+ work of normalizing any values that may have changed during (de)serialization.
+ """
+ req = PreparedRequest()
+ kwargs = attr.asdict(obj)
+ # TODO: Better means of handling naming differences between class attrs and method kwargs
+ kwargs['data'] = kwargs.pop('body')
+ req.prepare(**kwargs)
+ return req
+
+ @property
+ def _cookies(self):
+ return self.cookies
+
+ def __str__(self):
+ return f'{self.method} {self.url}'
diff --git a/requests_cache/models/response.py b/requests_cache/models/response.py
new file mode 100755
index 0000000..eddffeb
--- /dev/null
+++ b/requests_cache/models/response.py
@@ -0,0 +1,154 @@
+"""Classes to wrap cached response objects"""
+from datetime import datetime, timedelta, timezone
+from logging import getLogger
+from typing import List, Optional, Tuple, Union
+
+import attr
+from requests import Response
+from requests.cookies import RequestsCookieJar
+from requests.structures import CaseInsensitiveDict
+
+from ..cache_control import ExpirationTime, get_expiration_datetime
+from . import CachedHTTPResponse, CachedRequest, dataclass
+
+DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' # Format used for __str__ only
+DO_NOT_CACHE = 0
+
+HeaderList = List[Tuple[str, str]]
+
+logger = getLogger(__name__)
+
+
+# TODO: Make this fully take advantage of slots
+# Make a slotted copy of Response to subclass; we don't need its attrs, only its methods
+# from requests import Response as OriginalResponse
+# Response = attr.s(slots=True)(OriginalResponse)
+@dataclass
+class CachedResponse(Response):
+ """A serializable dataclass that emulates :py:class:`requests.Response`. Public attributes and
+ methods on CachedResponse objects will behave the same as those from the original response, but
+ with different internals optimized for serialization.
+
+ This means doing some pre- and post-initialization steps common to all serializers, such as
+ breaking nested objects down into their basic attributes and lazily re-initializing them, which
+ saves a bit of memory and deserialization steps when those objects aren't accessed.
+ """
+
+ _content: bytes = attr.ib(default=b'', repr=False, converter=lambda x: x or b'')
+ url: str = attr.ib(default=None)
+ status_code: int = attr.ib(default=0)
+ cookies: RequestsCookieJar = attr.ib(factory=dict)
+ created_at: datetime = attr.ib(factory=datetime.utcnow)
+ elapsed: timedelta = attr.ib(factory=timedelta)
+ expires: datetime = attr.ib(default=None)
+ encoding: str = attr.ib(default=None)
+ headers: CaseInsensitiveDict = attr.ib(factory=dict)
+ history: List = attr.ib(factory=list)
+ reason: str = attr.ib(default=None)
+ request: CachedRequest = attr.ib(factory=CachedRequest)
+ raw: CachedHTTPResponse = attr.ib(factory=CachedHTTPResponse, repr=False)
+
+ @classmethod
+ def from_response(cls, original_response: Response, **kwargs):
+ """Create a CachedResponse based on an original response object"""
+ obj = cls(**kwargs)
+
+ # Copy basic attributes
+ for k in Response.__attrs__:
+ setattr(obj, k, getattr(original_response, k, None))
+
+ # Store request and raw response
+ obj.request = CachedRequest.from_request(original_response.request)
+ obj.raw = CachedHTTPResponse.from_response(original_response)
+
+ # Store response body, which will have been read & decoded by requests.Response by now
+ obj._content = original_response.content
+
+ # Copy redirect history, if any; avoid recursion by not copying redirects of redirects
+ obj.history = []
+ if not obj.is_redirect:
+ for redirect in original_response.history:
+ obj.history.append(cls.from_response(redirect))
+
+ return obj
+
+ @property
+ def _content_consumed(self):
+ """For compatibility with Response; will always be True for a cached response"""
+ return True
+
+ @_content_consumed.setter
+ def _content_consumed(self, value):
+ pass
+
+ @property
+ def from_cache(self) -> bool:
+ return True
+
+ @property
+ def is_expired(self) -> bool:
+ """Determine if this cached response is expired"""
+ return self.expires is not None and datetime.utcnow() >= self.expires
+
+ def revalidate(self, expire_after: ExpirationTime) -> bool:
+ """Set a new expiration for this response, and determine if it is now expired"""
+ self.expires = get_expiration_datetime(expire_after)
+ return self.is_expired
+
+ def reset(self):
+ if self.raw:
+ self.raw.reset()
+
+ @property
+ def size(self) -> int:
+ """Get the size of the response body in bytes"""
+ return len(self.content) if self.content else 0
+
+ # TODO: Behavior will be different for slotted classes
+ # def __getstate__(self):
+ # """Override pickling behavior in ``requests.Response.__getstate__``"""
+ # return self.__dict__
+
+ def __str__(self):
+ return (
+ f'request: {self.request}, response: {self.status_code} '
+ f'({format_file_size(self.size)}), created: {format_datetime(self.created_at)}, '
+ f'expires: {format_datetime(self.expires)} ({"stale" if self.is_expired else "fresh"})'
+ )
+
+
+AnyResponse = Union[Response, CachedResponse]
+
+
+def format_datetime(value: Optional[datetime]) -> str:
+ """Get a formatted datetime string in the local time zone"""
+ if not value:
+ return "N/A"
+ if value.tzinfo is None:
+ value = value.replace(tzinfo=timezone.utc)
+ return value.astimezone().strftime(DATETIME_FORMAT)
+
+
+def format_file_size(n_bytes: int) -> str:
+ """Convert a file size in bytes into a human-readable format"""
+ filesize = float(n_bytes or 0)
+
+ def _format(unit):
+ return f'{int(filesize)} {unit}' if unit == 'bytes' else f'{filesize:.2f} {unit}'
+
+ for unit in ['bytes', 'KiB', 'MiB', 'GiB']:
+ if filesize < 1024 or unit == 'GiB':
+ return _format(unit)
+ filesize /= 1024
+
+
+def set_response_defaults(response: AnyResponse) -> AnyResponse:
+ """Set some default CachedResponse values on a requests.Response object, so they can be
+ expected to always be present
+ """
+ if not isinstance(response, CachedResponse):
+ response.created_at = None
+ response.expires = None
+ response.from_cache = False
+ response.is_expired = False
+ return response
diff --git a/requests_cache/patcher.py b/requests_cache/patcher.py
index 7d89acc..d947e94 100644
--- a/requests_cache/patcher.py
+++ b/requests_cache/patcher.py
@@ -10,7 +10,7 @@ from typing import Callable, Dict, Iterable, Optional, Type
import requests
from .backends import BackendSpecifier, BaseCache
-from .response import ExpirationTime
+from .cache_control import ExpirationTime
from .session import CachedSession, OriginalSession
logger = getLogger(__name__)
diff --git a/requests_cache/response.py b/requests_cache/response.py
deleted file mode 100755
index 687d884..0000000
--- a/requests_cache/response.py
+++ /dev/null
@@ -1,306 +0,0 @@
-# TODO: Maybe split this into separate modules
-"""Classes to wrap cached response objects"""
-from datetime import datetime, timedelta, timezone
-from io import BytesIO
-from logging import getLogger
-from typing import Dict, List, Optional, Tuple, Union
-
-import attr
-import cattr
-from requests import PreparedRequest, Response
-from requests.cookies import RequestsCookieJar, cookiejar_from_dict
-from requests.structures import CaseInsensitiveDict
-from urllib3.response import HTTPResponse, is_fp_closed
-
-from .cache_control import get_expiration_datetime
-
-logger = getLogger(__name__)
-
-DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' # Format used for __str__ only
-DO_NOT_CACHE = 0
-
-ExpirationTime = Union[None, int, float, datetime, timedelta]
-HeaderList = List[Tuple[str, str]]
-
-# Aliases for the most common attr options
-dataclass = attr.s(
- auto_attribs=False,
- auto_detect=True,
- collect_by_mro=True,
- kw_only=True,
- slots=True,
- weakref_slot=False,
-)
-public_attr = attr.ib(default=None)
-bytes_attr = attr.ib(default=b'', repr=False, converter=lambda x: x or b'')
-
-
-@dataclass
-class CachedHTTPResponse(HTTPResponse):
- """A serializable dataclass that emulates :py:class:`~urllib3.response.HTTPResponse`.
- Supports streaming requests and generator usage.
-
- The only action this doesn't support is explicitly calling :py:meth:`.read` with
- ``decode_content=False``, but a use case for this has not come up yet.
- """
-
- decode_content: bool = public_attr
- headers: CaseInsensitiveDict = attr.ib(factory=dict)
- reason: str = public_attr
- request_url: str = public_attr
- status: int = public_attr
- strict: int = public_attr
- version: int = public_attr
-
- def __attrs_post_init__(self, body: bytes = None, **kwargs):
- kwargs.setdefault('preload_content', False)
- super().__init__(body=BytesIO(body or b''), **kwargs)
- self._body = body
-
- @classmethod
- def from_response(cls, original_response: Response):
- """Create a CachedHTTPResponse based on an original response object's raw response"""
- # Copy basic attributes
- raw = original_response.raw
- kwargs = {k: getattr(raw, k, None) for k in attr.fields_dict(cls).keys()}
- # TODO: Better means of handling naming differences between class attrs and method kwargs
- kwargs['request_url'] = raw._request_url
-
- # Copy response data and restore response object to its original state
- if not is_fp_closed(getattr(original_response.raw, '_fp', None)):
- body = raw.read(decode_content=False)
- kwargs['body'] = body
- raw._fp = BytesIO(body)
- original_response.content # This property reads, decodes, and stores response content
-
- # After reading, reset file pointer on original raw response
- raw._fp = BytesIO(body)
- raw._fp_bytes_read = 0
- raw.length_remaining = len(body)
-
- return cls(**kwargs)
-
- def release_conn(self):
- """No-op for compatibility"""
-
- def read(self, amt=None, decode_content=None, **kwargs):
- """Simplified reader for cached content that emulates
- :py:meth:`urllib3.response.HTTPResponse.read()`
- """
- if 'content-encoding' in self.headers and decode_content is False:
- logger.warning('read() returns decoded data, even with decode_content=False')
-
- data = self._fp.read(amt)
- # "close" the file to inform consumers to stop reading from it
- if not data:
- self._fp.close()
- return data
-
- def reset(self):
- """Reset raw response file pointer"""
- self._fp = BytesIO(self._body)
-
- def stream(self, amt=None, **kwargs):
- """Simplified generator over cached content that emulates
- :py:meth:`urllib3.response.HTTPResponse.stream()`
- """
- while not self._fp.closed:
- yield self.read(amt=amt, **kwargs)
-
-
-@dataclass
-class CachedRequest:
- """A serializable dataclass that emulates :py:class:`requests.PreparedResponse`"""
-
- body: bytes = bytes_attr
- cookies: RequestsCookieJar = public_attr
- headers: CaseInsensitiveDict = attr.ib(factory=CaseInsensitiveDict)
- method: str = public_attr
- url: str = public_attr
-
- @classmethod
- def from_request(cls, original_request: PreparedRequest):
- """Create a CachedRequest based on an original request object"""
- kwargs = {k: getattr(original_request, k, None) for k in attr.fields_dict(cls).keys()}
- # TODO: Better means of handling naming differences between class attrs and method kwargs
- kwargs['cookies'] = original_request._cookies
- return cls(**kwargs)
-
- # TODO: Is this necessary, or will cattr.structure() be sufficient?
- @classmethod
- def prepare(self, obj) -> PreparedRequest:
- """Turn a CachedRequest object back into a PreparedRequest. This lets PreparedRequest do the
- work of normalizing any values that may have changed during (de)serialization.
- """
- req = PreparedRequest()
- kwargs = attr.asdict(obj)
- # TODO: Better means of handling naming differences between class attrs and method kwargs
- kwargs['_cookies'] = kwargs.pop('cookies')
- kwargs['body'] = kwargs.pop('data')
- req.prepare(**kwargs)
- return req
-
- @property
- def _cookies(self):
- return self.cookies
-
-
-# TODO: Make this fully take advantage of slots
-# Make a slotted copy of Response to subclass; we don't need its attrs, only its methods
-
-# from requests import Response as OriginalResponse
-# Response = attr.s(slots=True)(OriginalResponse)
-# @attr.s(kw_only=True, slots=True)
-@dataclass
-class CachedResponse(Response):
- """A serializable dataclass that emulates :py:class:`requests.Response`. Public attributes and
- methods on CachedResponse objects will behave the same as those from the original response, but
- with different internals optimized for serialization.
-
- This means doing some pre- and post-initialization steps common to all serializers, such as
- breaking nested objects down into their basic attributes and lazily re-initializing them, which
- saves a bit of memory and deserialization steps when those objects aren't accessed.
- """
-
- _content: bytes = bytes_attr
- url: str = public_attr
- status_code: int = public_attr
- cookies: RequestsCookieJar = public_attr
- created_at: datetime = attr.ib(factory=datetime.utcnow)
- elapsed: timedelta = attr.ib(factory=timedelta)
- expires: datetime = public_attr
- encoding: str = public_attr
- headers: CaseInsensitiveDict = attr.ib(factory=dict)
- history: List = attr.ib(factory=list)
- reason: str = public_attr
- request: CachedRequest = public_attr
- raw: CachedHTTPResponse = attr.ib(default=None, repr=False)
-
- @classmethod
- def from_response(cls, original_response: Response, **kwargs):
- """Create a CachedResponse based on an original response object"""
- obj = cls(**kwargs)
-
- # Copy basic attributes
- for k in Response.__attrs__:
- setattr(obj, k, getattr(original_response, k, None))
-
- # Store request and raw response
- obj.request = CachedRequest.from_request(original_response.request)
- obj.raw = CachedHTTPResponse.from_response(original_response)
-
- # Store response body, which will have been read & decoded by requests.Response by now
- obj._content = original_response.content
-
- # Copy redirect history, if any; avoid recursion by not copying redirects of redirects
- obj.history = []
- if not obj.is_redirect:
- for redirect in original_response.history:
- obj.history.append(cls.from_response(redirect))
-
- return obj
-
- @property
- def from_cache(self) -> bool:
- return True
-
- @property
- def is_expired(self) -> bool:
- """Determine if this cached response is expired"""
- return self.expires is not None and datetime.utcnow() >= self.expires
-
- def revalidate(self, expire_after: ExpirationTime) -> bool:
- """Set a new expiration for this response, and determine if it is now expired"""
- self.expires = get_expiration_datetime(expire_after)
- return self.is_expired
-
- def reset(self):
- if self.raw:
- self.raw.reset()
-
- @property
- def size(self) -> int:
- """Get the size of the response body in bytes"""
- return len(self.content) if self.content else 0
-
- # TODO: Behavior will be different for slotted classes
- # def __getstate__(self):
- # """Override pickling behavior in ``requests.Response.__getstate__``"""
- # return self.__dict__
-
- def __str__(self):
- return (
- f'request: {self.request.method} {self.request.url}, response: {self.status_code} '
- f'({format_file_size(self.size)}), created: {format_datetime(self.created_at)}, '
- f'expires: {format_datetime(self.expires)} ({"stale" if self.is_expired else "fresh"})'
- )
-
-
-# TODO: Should this go in a base serializer class instead?
-def get_converter():
- """Make a converter to structure and unstructure some of the nested objects within a response"""
- converter = cattr.Converter()
-
- # Convert datetimes to and from iso-formatted strings
- converter.register_unstructure_hook(datetime, lambda obj: obj.isoformat() if obj else None)
- converter.register_structure_hook(
- datetime, lambda obj, cls: datetime.fromisoformat(obj) if obj else None
- )
-
- # Convert timedeltas to and from float values in seconds
- converter.register_unstructure_hook(timedelta, lambda obj: obj.total_seconds() if obj else None)
- converter.register_structure_hook(
- timedelta, lambda obj, cls: timedelta(seconds=obj) if obj else None
- )
-
- # Convert dict-like objects to and from plain dicts
- converter.register_unstructure_hook(RequestsCookieJar, lambda obj: dict(obj.items()))
- converter.register_structure_hook(RequestsCookieJar, lambda obj, cls: cookiejar_from_dict(obj))
- converter.register_unstructure_hook(CaseInsensitiveDict, dict)
- converter.register_structure_hook(CaseInsensitiveDict, lambda obj, cls: CaseInsensitiveDict(obj))
-
- # Not sure yet if this will be needed
- # converter.register_unstructure_hook(PreparedRequest, CachedRequest.from_request)
- # converter.register_structure_hook(PreparedRequest, CachedRequest.prepare)
- # converter.register_unstructure_hook(HTTPResponse, lambda obj, cls: CachedHTTPResponse.from_response(obj))
- # converter.register_structure_hook(HTTPResponse, lambda obj, cls: CachedHTTPResponse(obj))
-
- return converter
-
-
-AnyResponse = Union[Response, CachedResponse]
-ResponseConverter = get_converter()
-
-
-def format_datetime(value: Optional[datetime]) -> str:
- """Get a formatted datetime string in the local time zone"""
- if not value:
- return "N/A"
- if value.tzinfo is None:
- value = value.replace(tzinfo=timezone.utc)
- return value.astimezone().strftime(DATETIME_FORMAT)
-
-
-def format_file_size(n_bytes: int) -> str:
- """Convert a file size in bytes into a human-readable format"""
- filesize = float(n_bytes or 0)
-
- def _format(unit):
- return f'{int(filesize)} {unit}' if unit == 'bytes' else f'{filesize:.2f} {unit}'
-
- for unit in ['bytes', 'KiB', 'MiB', 'GiB']:
- if filesize < 1024 or unit == 'GiB':
- return _format(unit)
- filesize /= 1024
-
-
-def set_response_defaults(response: AnyResponse) -> AnyResponse:
- """Set some default CachedResponse values on a requests.Response object, so they can be
- expected to always be present
- """
- if not isinstance(response, CachedResponse):
- response.created_at = None
- response.expires = None
- response.from_cache = False
- response.is_expired = False
- return response
diff --git a/requests_cache/serializers/__init__.py b/requests_cache/serializers/__init__.py
new file mode 100644
index 0000000..c2cd5b2
--- /dev/null
+++ b/requests_cache/serializers/__init__.py
@@ -0,0 +1,3 @@
+# flake8: noqa: F401
+from .base import BaseSerializer
+from .pickle import PickleSerializer, SafePickleSerializer
diff --git a/requests_cache/serializers/base.py b/requests_cache/serializers/base.py
new file mode 100644
index 0000000..bc2b732
--- /dev/null
+++ b/requests_cache/serializers/base.py
@@ -0,0 +1,64 @@
+from abc import abstractmethod
+from datetime import datetime, timedelta
+from typing import Dict
+
+import cattr
+from requests.cookies import RequestsCookieJar, cookiejar_from_dict
+from requests.structures import CaseInsensitiveDict
+
+from ..models import CachedResponse
+
+
+class BaseSerializer:
+ """Base serializer class for :py:class:`.CachedResponse` that does pre/post-processing with cattrs.
+ This does the majority of the work to break objects down into builtin types and reassemble them
+ without data loss. Subclasses just need to provide ``dumps`` and ``loads`` methods.
+ """
+
+ is_binary = True # TODO: This may or may not be needed to determine return type in backends
+
+ def __init__(self, *args, **kwargs):
+ """Make a converter to structure and unstructure some of the nested objects within a response"""
+ super().__init__(*args, **kwargs)
+ converter = cattr.Converter()
+
+ # Convert datetimes to and from iso-formatted strings
+ converter.register_unstructure_hook(datetime, lambda obj: obj.isoformat() if obj else None)
+ converter.register_structure_hook(
+ datetime, lambda obj, cls: datetime.fromisoformat(obj) if obj else None
+ )
+
+ # Convert timedeltas to and from float values in seconds
+ converter.register_unstructure_hook(timedelta, lambda obj: obj.total_seconds() if obj else None)
+ converter.register_structure_hook(
+ timedelta, lambda obj, cls: timedelta(seconds=obj) if obj else None
+ )
+
+ # Convert dict-like objects to and from plain dicts
+ converter.register_unstructure_hook(RequestsCookieJar, lambda obj: dict(obj.items()))
+ converter.register_structure_hook(RequestsCookieJar, lambda obj, cls: cookiejar_from_dict(obj))
+ converter.register_unstructure_hook(CaseInsensitiveDict, dict)
+ converter.register_structure_hook(CaseInsensitiveDict, lambda obj, cls: CaseInsensitiveDict(obj))
+
+ # Not sure yet if this will be needed
+ # converter.register_unstructure_hook(PreparedRequest, CachedRequest.from_request)
+ # converter.register_structure_hook(PreparedRequest, lambda obj, cls: CachedRequest.prepare(obj))
+ # converter.register_unstructure_hook(HTTPResponse, lambda obj, cls: CachedHTTPResponse.from_response(obj))
+ # converter.register_structure_hook(HTTPResponse, lambda obj, cls: CachedHTTPResponse(obj))
+ # converter.register_structure_hook(CachedRequest, lambda obj, cls: cls.prepare(obj))
+
+ self.converter = converter
+
+ def unstructure(self, response: CachedResponse) -> Dict:
+ return self.converter.unstructure(response)
+
+ def structure(self, obj: Dict) -> CachedResponse:
+ return self.converter.structure(obj, CachedResponse)
+
+ @abstractmethod
+ def dumps(self, response: CachedResponse):
+ pass
+
+ @abstractmethod
+ def loads(self, obj) -> CachedResponse:
+ pass
diff --git a/requests_cache/serializers/pickle.py b/requests_cache/serializers/pickle.py
new file mode 100644
index 0000000..c1a5726
--- /dev/null
+++ b/requests_cache/serializers/pickle.py
@@ -0,0 +1,34 @@
+import pickle
+
+from itsdangerous.serializer import Serializer as SafeSerializer
+
+from ..models import CachedResponse
+from .base import BaseSerializer
+
+
+class PickleSerializer(BaseSerializer):
+ """Wrapper for pickle that pre/post-processes with cattrs"""
+
+ def dumps(self, response: CachedResponse) -> bytes:
+ return pickle.dumps(super().unstructure(response))
+
+ def loads(self, obj: bytes) -> CachedResponse:
+ return super().structure(pickle.loads(obj))
+
+
+class SafePickleSerializer(BaseSerializer, SafeSerializer):
+ """Wrapper for itsdangerous + pickle that pre/post-processes with cattrs"""
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs, serializer=PickleSerializer())
+
+ def dumps(self, response: CachedResponse) -> bytes:
+ x = super().unstructure(response)
+ # breakpoint()
+ return SafeSerializer.dumps(self, x)
+
+ # TODO: Something weird is going on here
+ def loads(self, obj: bytes) -> CachedResponse:
+ return SafeSerializer.loads(self, obj)
+ # breakpoint()
+ return super().structure(SafeSerializer.loads(self, obj))
diff --git a/requests_cache/session.py b/requests_cache/session.py
index 91d0a5d..93e3608 100644
--- a/requests_cache/session.py
+++ b/requests_cache/session.py
@@ -10,9 +10,9 @@ from requests.hooks import dispatch_hook
from urllib3 import filepost
from .backends import BackendSpecifier, get_valid_kwargs, init_backend
-from .cache_control import CacheActions
+from .cache_control import CacheActions, ExpirationTime
from .cache_keys import normalize_dict
-from .response import AnyResponse, ExpirationTime, set_response_defaults
+from .models.response import AnyResponse, set_response_defaults
ALL_METHODS = ['GET', 'HEAD', 'OPTIONS', 'POST', 'PUT', 'PATCH', 'DELETE']
logger = getLogger(__name__)