diff options
| author | Jordan Cook <jordan.cook@pioneer.com> | 2021-05-26 20:29:13 -0500 |
|---|---|---|
| committer | Jordan Cook <jordan.cook@pioneer.com> | 2021-05-26 20:55:45 -0500 |
| commit | 6edf2e1c5e3712b2083a5daff142db65232c6e1c (patch) | |
| tree | 7252457e69d910496abd3bf0f5710d1f20166eb2 /requests_cache | |
| parent | 7c7126475d741ab32b14637343dc604dae9cd65c (diff) | |
| download | requests-cache-6edf2e1c5e3712b2083a5daff142db65232c6e1c.tar.gz | |
Make cattrs optional, and other cleanup
Diffstat (limited to 'requests_cache')
| -rw-r--r-- | requests_cache/backends/filesystem.py | 1 | ||||
| -rw-r--r-- | requests_cache/models/__init__.py | 12 | ||||
| -rw-r--r-- | requests_cache/models/raw_response.py | 21 | ||||
| -rw-r--r-- | requests_cache/models/request.py | 35 | ||||
| -rwxr-xr-x | requests_cache/models/response.py | 41 | ||||
| -rw-r--r-- | requests_cache/serializers/base.py | 79 |
6 files changed, 79 insertions, 110 deletions
diff --git a/requests_cache/backends/filesystem.py b/requests_cache/backends/filesystem.py index bf3d23c..8cf5332 100644 --- a/requests_cache/backends/filesystem.py +++ b/requests_cache/backends/filesystem.py @@ -1,4 +1,3 @@ -# TODO: Add option for compression? from contextlib import contextmanager from os import listdir, makedirs, unlink from os.path import abspath, dirname, expanduser, isabs, join diff --git a/requests_cache/models/__init__.py b/requests_cache/models/__init__.py index 4a9d00d..78a340f 100644 --- a/requests_cache/models/__init__.py +++ b/requests_cache/models/__init__.py @@ -1,16 +1,4 @@ # flake8: noqa: F401 -import attr - -dataclass = attr.s( - auto_attribs=False, - auto_detect=True, - collect_by_mro=True, - kw_only=True, - slots=True, - weakref_slot=False, -) - - from .raw_response import CachedHTTPResponse from .request import CachedRequest from .response import CachedResponse diff --git a/requests_cache/models/raw_response.py b/requests_cache/models/raw_response.py index 63c6d8b..3cdc385 100644 --- a/requests_cache/models/raw_response.py +++ b/requests_cache/models/raw_response.py @@ -1,14 +1,14 @@ from io import BytesIO from logging import getLogger -import attr +from attr import define, field, fields_dict from requests import Response from urllib3.response import HTTPHeaderDict, HTTPResponse, is_fp_closed logger = getLogger(__name__) -@attr.s(auto_attribs=False, auto_detect=True, kw_only=True) +@define(auto_attribs=False, slots=False) class CachedHTTPResponse(HTTPResponse): """A serializable dataclass that extends/emulates :py:class:`~urllib3.response.HTTPResponse`. Supports streaming requests and generator usage. @@ -17,13 +17,13 @@ class CachedHTTPResponse(HTTPResponse): ``decode_content=False``, but a use case for this has not come up yet. """ - decode_content: bool = attr.ib(default=None) - headers: HTTPHeaderDict = attr.ib(factory=dict) - reason: str = attr.ib(default=None) - request_url: str = attr.ib(default=None) # TODO: Not available in urllib <=1.21. Is this needed? - status: int = attr.ib(default=0) - strict: int = attr.ib(default=0) - version: int = attr.ib(default=0) + decode_content: bool = field(default=None) + headers: HTTPHeaderDict = field(factory=dict) + reason: str = field(default=None) + request_url: str = field(default=None) # Note: Not available in urllib <=1.21 + status: int = field(default=0) + strict: int = field(default=0) + version: int = field(default=0) def __init__(self, *args, body: bytes = None, **kwargs): """First initialize via HTTPResponse, then via attrs""" @@ -36,8 +36,7 @@ class CachedHTTPResponse(HTTPResponse): """Create a CachedHTTPResponse based on an original response""" # Copy basic attributes raw = original_response.raw - kwargs = {k: getattr(raw, k, None) for k in attr.fields_dict(cls).keys()} - # TODO: Better means of handling naming differences between class attrs and method kwargs + kwargs = {k: getattr(raw, k, None) for k in fields_dict(cls).keys()} kwargs['request_url'] = raw._request_url # Copy response data and restore response object to its original state diff --git a/requests_cache/models/request.py b/requests_cache/models/request.py index d5c1ba1..f8f5a54 100644 --- a/requests_cache/models/request.py +++ b/requests_cache/models/request.py @@ -2,49 +2,36 @@ from logging import getLogger from typing import Any -import attr +from attr import define, field, fields_dict from requests import PreparedRequest from requests.cookies import RequestsCookieJar from requests.structures import CaseInsensitiveDict -from . import dataclass - logger = getLogger(__name__) -@dataclass +@define(auto_attribs=False) class CachedRequest: """A serializable dataclass that emulates :py:class:`requests.PreparedResponse`""" - body: Any = attr.ib(default=None) - cookies: RequestsCookieJar = attr.ib(factory=dict) - headers: CaseInsensitiveDict = attr.ib(factory=CaseInsensitiveDict) - method: str = attr.ib(default=None) - url: str = attr.ib(default=None) + body: Any = field(default=None) + cookies: RequestsCookieJar = field(factory=dict) + headers: CaseInsensitiveDict = field(factory=CaseInsensitiveDict) + method: str = field(default=None) + url: str = field(default=None) @classmethod def from_request(cls, original_request: PreparedRequest): """Create a CachedRequest based on an original request object""" - kwargs = {k: getattr(original_request, k, None) for k in attr.fields_dict(cls).keys()} - # TODO: Better means of handling naming differences between class attrs and method kwargs + kwargs = {k: getattr(original_request, k, None) for k in fields_dict(cls).keys()} kwargs['cookies'] = original_request._cookies return cls(**kwargs) - # TODO: Is this necessary, or will cattr.structure() be sufficient? - @classmethod - def prepare(self, obj) -> PreparedRequest: - """Turn a CachedRequest object back into a PreparedRequest. This lets PreparedRequest do the - work of normalizing any values that may have changed during (de)serialization. - """ - req = PreparedRequest() - kwargs = attr.asdict(obj) - # TODO: Better means of handling naming differences between class attrs and method kwargs - kwargs['data'] = kwargs.pop('body') - req.prepare(**kwargs) - return req - @property def _cookies(self): + """For compatibility with PreparedRequest, which has an attribute named '_cookies', and a + keyword argument named 'cookies'. + """ return self.cookies def __str__(self): diff --git a/requests_cache/models/response.py b/requests_cache/models/response.py index 86b21b9..627a753 100755 --- a/requests_cache/models/response.py +++ b/requests_cache/models/response.py @@ -3,27 +3,25 @@ from datetime import datetime, timedelta, timezone from logging import getLogger from typing import List, Optional, Tuple, Union -import attr -from requests import Response +from attr import define, field +from requests import Response as OriginalResponse from requests.cookies import RequestsCookieJar from requests.structures import CaseInsensitiveDict from ..cache_control import ExpirationTime, get_expiration_datetime -from . import CachedHTTPResponse, CachedRequest, dataclass +from . import CachedHTTPResponse, CachedRequest DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' # Format used for __str__ only DO_NOT_CACHE = 0 +# Make a slotted copy of requests.Response to subclass +Response = define(slots=True)(OriginalResponse) HeaderList = List[Tuple[str, str]] logger = getLogger(__name__) -# TODO: Make this fully take advantage of slots -# Make a slotted copy of Response to subclass; we don't need its attrs, only its methods -# from requests import Response as OriginalResponse -# Response = attr.s(slots=True)(OriginalResponse) -@dataclass +@define(auto_attribs=False) class CachedResponse(Response): """A serializable dataclass that emulates :py:class:`requests.Response`. Public attributes and methods on CachedResponse objects will behave the same as those from the original response, but @@ -34,20 +32,19 @@ class CachedResponse(Response): saves a bit of memory and deserialization steps when those objects aren't accessed. """ - # _content: bytes = attr.ib(default=b'', repr=False, converter=lambda x: x or b'') - _content: bytes = attr.ib(default=None) - url: str = attr.ib(default=None) - status_code: int = attr.ib(default=0) - cookies: RequestsCookieJar = attr.ib(factory=dict) - created_at: datetime = attr.ib(factory=datetime.utcnow) - elapsed: timedelta = attr.ib(factory=timedelta) - expires: datetime = attr.ib(default=None) - encoding: str = attr.ib(default=None) - headers: CaseInsensitiveDict = attr.ib(factory=dict) - history: List = attr.ib(factory=list) - reason: str = attr.ib(default=None) - request: CachedRequest = attr.ib(factory=CachedRequest) - raw: CachedHTTPResponse = attr.ib(factory=CachedHTTPResponse, repr=False) + _content: bytes = field(default=None) + url: str = field(default=None) + status_code: int = field(default=0) + cookies: RequestsCookieJar = field(factory=dict) + created_at: datetime = field(factory=datetime.utcnow) + elapsed: timedelta = field(factory=timedelta) + expires: datetime = field(default=None) + encoding: str = field(default=None) + headers: CaseInsensitiveDict = field(factory=dict) + history: List = field(factory=list) + reason: str = field(default=None) + request: CachedRequest = field(factory=CachedRequest) + raw: CachedHTTPResponse = field(factory=CachedHTTPResponse, repr=False) def __attrs_post_init__(self): """Re-initialize raw response body after deserialization""" diff --git a/requests_cache/serializers/base.py b/requests_cache/serializers/base.py index 144f075..bb3bb83 100644 --- a/requests_cache/serializers/base.py +++ b/requests_cache/serializers/base.py @@ -2,7 +2,6 @@ from abc import abstractmethod from datetime import datetime, timedelta from typing import Any -import cattr from requests.cookies import RequestsCookieJar, cookiejar_from_dict from requests.structures import CaseInsensitiveDict from urllib3.response import HTTPHeaderDict @@ -10,56 +9,26 @@ from urllib3.response import HTTPHeaderDict from ..models import CachedResponse +# TODO: Document this more thoroughly class BaseSerializer: - """Base serializer class for :py:class:`.CachedResponse` that does pre/post-processing with cattrs. - This does the majority of the work to break objects down into builtin types and reassemble them - without data loss. Subclasses just need to provide ``dumps`` and ``loads`` methods. - """ + """Base serializer class for :py:class:`.CachedResponse` that optionally does + pre/post-processing with cattrs. This provides an easy starting point for alternative + serialization formats, and potential for some backend-specific optimizations. - is_binary = True # TODO: This may or may not be needed to determine return type in backends + Subclasses must provide ``dumps`` and ``loads`` methods. + """ def __init__(self, *args, **kwargs): - """Make a converter to structure and unstructure some of the nested objects within a response""" super().__init__(*args, **kwargs) - try: - # raise AttributeError - converter = cattr.GenConverter(omit_if_default=True) - # Python 3.6 compatibility - except AttributeError: - converter = cattr.Converter() - - # Convert datetimes to and from iso-formatted strings - converter.register_unstructure_hook(datetime, lambda obj: obj.isoformat() if obj else None) - converter.register_structure_hook(datetime, to_datetime) - - # Convert timedeltas to and from float values in seconds - converter.register_unstructure_hook(timedelta, lambda obj: obj.total_seconds() if obj else None) - converter.register_structure_hook(timedelta, to_timedelta) - - # Convert dict-like objects to and from plain dicts - converter.register_unstructure_hook(RequestsCookieJar, lambda obj: dict(obj.items())) - converter.register_structure_hook(RequestsCookieJar, lambda obj, cls: cookiejar_from_dict(obj)) - converter.register_unstructure_hook(CaseInsensitiveDict, dict) - converter.register_structure_hook(CaseInsensitiveDict, lambda obj, cls: CaseInsensitiveDict(obj)) - converter.register_unstructure_hook(HTTPHeaderDict, dict) - converter.register_structure_hook(HTTPHeaderDict, lambda obj, cls: HTTPHeaderDict(obj)) - - # Not sure yet if this will be needed - # converter.register_unstructure_hook(PreparedRequest, CachedRequest.from_request) - # converter.register_structure_hook(PreparedRequest, lambda obj, cls: CachedRequest.prepare(obj)) - # converter.register_unstructure_hook(HTTPResponse, lambda obj, cls: CachedHTTPResponse.from_response(obj)) - # converter.register_structure_hook(HTTPResponse, lambda obj, cls: CachedHTTPResponse(obj)) - # converter.register_structure_hook(CachedRequest, lambda obj, cls: cls.prepare(obj)) - - self.converter = converter + self.converter = init_converter() def unstructure(self, obj: Any) -> Any: - if not isinstance(obj, CachedResponse): + if not isinstance(obj, CachedResponse) or not self.converter: return obj return self.converter.unstructure(obj) def structure(self, obj: Any) -> Any: - if not isinstance(obj, dict): + if not isinstance(obj, dict) or not self.converter: return obj return self.converter.structure(obj, CachedResponse) @@ -72,6 +41,36 @@ class BaseSerializer: pass +def init_converter(): + """Make a converter to structure and unstructure some of the nested objects within a response, + if cattrs is installed. + """ + try: + from cattr import GenConverter + except ImportError: + return None + + converter = GenConverter(omit_if_default=True) + + # Convert datetimes to and from iso-formatted strings + converter.register_unstructure_hook(datetime, lambda obj: obj.isoformat() if obj else None) + converter.register_structure_hook(datetime, to_datetime) + + # Convert timedeltas to and from float values in seconds + converter.register_unstructure_hook(timedelta, lambda obj: obj.total_seconds() if obj else None) + converter.register_structure_hook(timedelta, to_timedelta) + + # Convert dict-like objects to and from plain dicts + converter.register_unstructure_hook(RequestsCookieJar, lambda obj: dict(obj.items())) + converter.register_structure_hook(RequestsCookieJar, lambda obj, cls: cookiejar_from_dict(obj)) + converter.register_unstructure_hook(CaseInsensitiveDict, dict) + converter.register_structure_hook(CaseInsensitiveDict, lambda obj, cls: CaseInsensitiveDict(obj)) + converter.register_unstructure_hook(HTTPHeaderDict, dict) + converter.register_structure_hook(HTTPHeaderDict, lambda obj, cls: HTTPHeaderDict(obj)) + + return converter + + def to_datetime(obj, cls) -> datetime: if isinstance(obj, str): obj = datetime.fromisoformat(obj) |
