Store responses in DynamoDB as JSON documents instead of serialized binaries

6 files changed, 74 insertions, 25 deletions

diff --git a/requests_cache/backends/__init__.py b/requests_cache/backends/__init__.py
index 250be7e..9f87908 100644
--- a/requests_cache/backends/__init__.py
+++ b/requests_cache/backends/__init__.py
@@ -15,17 +15,17 @@ logger = getLogger(__name__)
 
 # Import all backend classes for which dependencies are installed
 try:
-    from .dynamodb import DynamoDbCache, DynamoDbDict
+    from .dynamodb import DynamoDbCache, DynamoDbDict, DynamoDocumentDict
 except ImportError as e:
-    DynamoDbCache = DynamoDbDict = get_placeholder_class(e)  # type: ignore
+    DynamoDbCache = DynamoDbDict = DynamoDocumentDict = get_placeholder_class(e)  # type: ignore
 try:
     from .gridfs import GridFSCache, GridFSPickleDict
 except ImportError as e:
     GridFSCache = GridFSPickleDict = get_placeholder_class(e)  # type: ignore
 try:
-    from .mongodb import MongoCache, MongoDict, MongoPickleDict
+    from .mongodb import MongoCache, MongoDict, MongoDocumentDict
 except ImportError as e:
-    MongoCache = MongoDict = MongoPickleDict = get_placeholder_class(e)  # type: ignore
+    MongoCache = MongoDict = MongoDocumentDict = get_placeholder_class(e)  # type: ignore
 try:
     from .redis import RedisCache, RedisDict, RedisHashDict
 except ImportError as e:
diff --git a/requests_cache/backends/dynamodb.py b/requests_cache/backends/dynamodb.py
index 17721f6..17f4661 100644
--- a/requests_cache/backends/dynamodb.py
+++ b/requests_cache/backends/dynamodb.py
@@ -12,6 +12,7 @@ from boto3.resources.base import ServiceResource
 from botocore.exceptions import ClientError
 
 from .._utils import get_valid_kwargs
+from ..serializers import dynamodb_document_serializer
 from . import BaseCache, BaseStorage
 
 
@@ -30,18 +31,16 @@ class DynamoDbCache(BaseCache):
         self, table_name: str = 'http_cache', connection: ServiceResource = None, **kwargs
     ):
         super().__init__(cache_name=table_name, **kwargs)
-        self.responses = DynamoDbDict(table_name, 'responses', connection=connection, **kwargs)
+        self.responses = DynamoDocumentDict(
+            table_name, 'responses', connection=connection, **kwargs
+        )
         self.redirects = DynamoDbDict(
             table_name, 'redirects', connection=self.responses.connection, **kwargs
         )
 
 
 class DynamoDbDict(BaseStorage):
-    """A dictionary-like interface for DynamoDB key-value store
-
-    **Notes:**
-        * The actual table name on the Dynamodb server will be ``namespace:table_name``
-        * In order to deal with how DynamoDB stores data, all values are serialized.
+    """A dictionary-like interface for DynamoDB table
 
     Args:
         table_name: DynamoDB table name
@@ -54,7 +53,7 @@ class DynamoDbDict(BaseStorage):
     def __init__(
         self,
         table_name: str,
-        namespace: str = 'http_cache',
+        namespace: str,
         connection: ServiceResource = None,
         **kwargs,
     ):
@@ -111,12 +110,10 @@ class DynamoDbDict(BaseStorage):
 
         # Depending on the serializer, the value may be either a string or Binary object
         raw_value = result['Item']['value']
-        return self.serializer.loads(
-            raw_value.value if isinstance(raw_value, Binary) else raw_value
-        )
+        return raw_value.value if isinstance(raw_value, Binary) else raw_value
 
     def __setitem__(self, key, value):
-        item = {**self.composite_key(key), 'value': self.serializer.dumps(value)}
+        item = {**self.composite_key(key), 'value': value}
         self._table.put_item(Item=item)
 
     def __delitem__(self, key):
@@ -145,3 +142,19 @@ class DynamoDbDict(BaseStorage):
 
     def clear(self):
         self.bulk_delete((k for k in self))
+
+
+class DynamoDocumentDict(DynamoDbDict):
+    """Same as :class:`DynamoDbDict`, but serializes values before saving.
+
+    By default, responses are only partially serialized into a DynamoDB-compatible document format.
+    """
+
+    def __init__(self, *args, serializer=None, **kwargs):
+        super().__init__(*args, serializer=serializer or dynamodb_document_serializer, **kwargs)
+
+    def __getitem__(self, key):
+        return self.serializer.loads(super().__getitem__(key))
+
+    def __setitem__(self, key, item):
+        super().__setitem__(key, self.serializer.dumps(item))
diff --git a/requests_cache/backends/mongodb.py b/requests_cache/backends/mongodb.py
index b48f2da..056d658 100644
--- a/requests_cache/backends/mongodb.py
+++ b/requests_cache/backends/mongodb.py
@@ -30,7 +30,7 @@ class MongoCache(BaseCache):
 
     def __init__(self, db_name: str = 'http_cache', connection: MongoClient = None, **kwargs):
         super().__init__(cache_name=db_name, **kwargs)
-        self.responses: MongoDict = MongoPickleDict(
+        self.responses: MongoDict = MongoDocumentDict(
             db_name,
             collection_name='responses',
             connection=connection,
@@ -140,10 +140,10 @@ class MongoDict(BaseStorage):
         self.connection.close()
 
 
-class MongoPickleDict(MongoDict):
+class MongoDocumentDict(MongoDict):
     """Same as :class:`MongoDict`, but serializes values before saving.
 
-    By default, responses are only partially serialized into a MongoDB-compatible document mapping.
+    By default, responses are only partially serialized into a MongoDB-compatible document format.
     """
 
     def __init__(self, *args, serializer=None, **kwargs):
diff --git a/requests_cache/serializers/__init__.py b/requests_cache/serializers/__init__.py
index 72420b7..d49545a 100644
--- a/requests_cache/serializers/__init__.py
+++ b/requests_cache/serializers/__init__.py
@@ -7,6 +7,7 @@ from .preconf import (
     bson_document_serializer,
     bson_serializer,
     dict_serializer,
+    dynamodb_document_serializer,
     json_serializer,
     pickle_serializer,
     safe_pickle_serializer,
@@ -21,6 +22,7 @@ __all__ = [
     'Stage',
     'bson_serializer',
     'bson_document_serializer',
+    'dynamodb_document_serializer',
     'dict_serializer',
     'json_serializer',
     'pickle_serializer',
diff --git a/requests_cache/serializers/cattrs.py b/requests_cache/serializers/cattrs.py
index 0aa62c2..3de7079 100644
--- a/requests_cache/serializers/cattrs.py
+++ b/requests_cache/serializers/cattrs.py
@@ -12,6 +12,7 @@ serialization format.
    :nosignatures:
 """
 from datetime import datetime, timedelta
+from decimal import Decimal
 from typing import Callable, Dict, ForwardRef, MutableMapping
 
 from cattr import GenConverter
@@ -42,7 +43,11 @@ class CattrStage(Stage):
         return self.converter.structure(value, cl=CachedResponse)
 
 
-def init_converter(factory: Callable[..., GenConverter] = None, convert_datetime: bool = True):
+def init_converter(
+    factory: Callable[..., GenConverter] = None,
+    convert_datetime: bool = True,
+    convert_timedelta: bool = True,
+) -> GenConverter:
     """Make a converter to structure and unstructure nested objects within a
     :py:class:`.CachedResponse`
 
@@ -56,15 +61,18 @@ def init_converter(factory: Callable[..., GenConverter] = None, convert_datetime
 
     # Convert datetimes to and from iso-formatted strings
     if convert_datetime:
-        converter.register_unstructure_hook(datetime, lambda obj: obj.isoformat() if obj else None)  # type: ignore
+        converter.register_unstructure_hook(datetime, lambda obj: obj.isoformat() if obj else None)
         converter.register_structure_hook(datetime, _to_datetime)
 
     # Convert timedeltas to and from float values in seconds
-    converter.register_unstructure_hook(timedelta, lambda obj: obj.total_seconds() if obj else None)  # type: ignore
-    converter.register_structure_hook(timedelta, _to_timedelta)
+    if convert_timedelta:
+        converter.register_unstructure_hook(
+            timedelta, lambda obj: obj.total_seconds() if obj else None
+        )
+        converter.register_structure_hook(timedelta, _to_timedelta)
 
     # Convert dict-like objects to and from plain dicts
-    converter.register_unstructure_hook(RequestsCookieJar, lambda obj: dict(obj.items()))  # type: ignore
+    converter.register_unstructure_hook(RequestsCookieJar, lambda obj: dict(obj.items()))
     converter.register_structure_hook(RequestsCookieJar, lambda obj, cls: cookiejar_from_dict(obj))
     converter.register_unstructure_hook(CaseInsensitiveDict, dict)
     converter.register_structure_hook(
@@ -85,6 +93,16 @@ def init_converter(factory: Callable[..., GenConverter] = None, convert_datetime
     return converter
 
 
+def make_decimal_timedelta_converter(**kwargs) -> GenConverter:
+    """Make a converter that uses Decimals instead of floats to represent timedelta objects"""
+    converter = GenConverter(**kwargs)
+    converter.register_unstructure_hook(
+        timedelta, lambda obj: Decimal(str(obj.total_seconds())) if obj else None
+    )
+    converter.register_structure_hook(timedelta, _to_timedelta)
+    return converter
+
+
 def _to_datetime(obj, cls) -> datetime:
     if isinstance(obj, str):
         obj = datetime.fromisoformat(obj)
@@ -94,4 +112,6 @@ def _to_datetime(obj, cls) -> datetime:
 def _to_timedelta(obj, cls) -> timedelta:
     if isinstance(obj, (int, float)):
         obj = timedelta(seconds=obj)
+    elif isinstance(obj, Decimal):
+        obj = timedelta(seconds=float(obj))
     return obj
diff --git a/requests_cache/serializers/preconf.py b/requests_cache/serializers/preconf.py
index 557c1a4..1cf6816 100644
--- a/requests_cache/serializers/preconf.py
+++ b/requests_cache/serializers/preconf.py
@@ -4,7 +4,7 @@
 required for specific serialization formats.
 
 This module wraps those converters as serializer :py:class:`.Stage` objects. These are then used as
-a stage in a :py:class:`.SerializerPipeline`, which runs after the base converter and before the
+stages in a :py:class:`.SerializerPipeline`, which runs after the base converter and before the
 format's ``dumps()`` (or equivalent) method.
 
 For any optional libraries that aren't installed, the corresponding serializer will be a placeholder
@@ -14,11 +14,15 @@ class that raises an ``ImportError`` at initialization time instead of at import
    :nosignatures:
 """
 import pickle
+from datetime import timedelta
+from decimal import Decimal
 from functools import partial
 from importlib import import_module
 
+from cattr import GenConverter
+
 from .._utils import get_placeholder_class
-from .cattrs import CattrStage
+from .cattrs import CattrStage, make_decimal_timedelta_converter
 from .pipeline import SerializerPipeline, Stage
 
 
@@ -144,3 +148,13 @@ try:
     )  #: Complete YAML serializer
 except ImportError as e:
     yaml_serializer = get_placeholder_class(e)
+
+
+dynamodb_preconf_stage = CattrStage(
+    factory=make_decimal_timedelta_converter, convert_timedelta=False
+)
+dynamodb_document_serializer = SerializerPipeline(
+    [dynamodb_preconf_stage],
+    name='dynamodb_document',
+    is_binary=False,
+)