diff options
| author | Jordan Cook <jordan.cook@pioneer.com> | 2021-06-03 22:39:40 -0500 |
|---|---|---|
| committer | Jordan Cook <jordan.cook@pioneer.com> | 2021-06-11 16:43:35 -0500 |
| commit | 2392cdf9d7ad847da43ee30ba86c65e9739d8122 (patch) | |
| tree | 91f344c750b978f20c130c6825d2bc08df0afbcf /requests_cache/backends | |
| parent | 4b4746f647240d277675f51849ba8b4fb83a7b6d (diff) | |
| download | requests-cache-2392cdf9d7ad847da43ee30ba86c65e9739d8122.tar.gz | |
Integrate serializers with backends
Diffstat (limited to 'requests_cache/backends')
| -rw-r--r-- | requests_cache/backends/dynamodb.py | 8 | ||||
| -rw-r--r-- | requests_cache/backends/filesystem.py | 10 | ||||
| -rw-r--r-- | requests_cache/backends/gridfs.py | 5 | ||||
| -rw-r--r-- | requests_cache/backends/mongo.py | 4 | ||||
| -rw-r--r-- | requests_cache/backends/redis.py | 15 | ||||
| -rw-r--r-- | requests_cache/backends/sqlite.py | 15 |
6 files changed, 34 insertions, 23 deletions
diff --git a/requests_cache/backends/dynamodb.py b/requests_cache/backends/dynamodb.py index 3cfcf32..977e425 100644 --- a/requests_cache/backends/dynamodb.py +++ b/requests_cache/backends/dynamodb.py @@ -1,6 +1,7 @@ from typing import Dict, Iterable import boto3 +from boto3.dynamodb.types import Binary from boto3.resources.base import ServiceResource from botocore.exceptions import ClientError @@ -90,10 +91,13 @@ class DynamoDbDict(BaseStorage): result = self._table.get_item(Key=self.composite_key(key)) if 'Item' not in result: raise KeyError - return self.deserialize(result['Item']['value'].value) + + # Depending on the serializer, the value may be either a string or Binary object + raw_value = result['Item']['value'] + return self.serializer.loads(raw_value.value if isinstance(raw_value, Binary) else raw_value) def __setitem__(self, key, value): - item = {**self.composite_key(key), 'value': self.serialize(value)} + item = {**self.composite_key(key), 'value': self.serializer.dumps(value)} self._table.put_item(Item=item) def __delitem__(self, key): diff --git a/requests_cache/backends/filesystem.py b/requests_cache/backends/filesystem.py index 8cf5332..3c49625 100644 --- a/requests_cache/backends/filesystem.py +++ b/requests_cache/backends/filesystem.py @@ -48,18 +48,20 @@ class FileDict(BaseStorage): raise KeyError(e) def __getitem__(self, key): + mode = 'rb' if self.serializer.is_binary else 'r' with self._try_io(): - with open(join(self.cache_dir, str(key)), 'rb') as f: - return self.deserialize(f.read()) + with open(join(self.cache_dir, str(key)), mode) as f: + return self.serializer.loads(f.read()) def __delitem__(self, key): with self._try_io(): unlink(join(self.cache_dir, str(key))) def __setitem__(self, key, value): + mode = 'wb' if self.serializer.is_binary else 'w' with self._try_io(): - with open(join(self.cache_dir, str(key)), 'wb') as f: - f.write(self.serialize(value)) + with open(join(self.cache_dir, str(key)), mode) as f: + f.write(self.serializer.dumps(value)) def __iter__(self): for filename in listdir(self.cache_dir): diff --git a/requests_cache/backends/gridfs.py b/requests_cache/backends/gridfs.py index 82ca05c..0b1a8d2 100644 --- a/requests_cache/backends/gridfs.py +++ b/requests_cache/backends/gridfs.py @@ -52,14 +52,15 @@ class GridFSPickleDict(BaseStorage): result = self.fs.find_one({'_id': key}) if result is None: raise KeyError - return self.deserialize(result.read()) + return self.serializer.loads(result.read()) def __setitem__(self, key, item): try: self.__delitem__(key) except KeyError: pass - self.fs.put(self.serialize(item), **{'_id': key}) + encoding = None if self.serializer.is_binary else 'utf-8' + self.fs.put(self.serializer.dumps(item), encoding=encoding, **{'_id': key}) def __delitem__(self, key): res = self.fs.find_one({'_id': key}) diff --git a/requests_cache/backends/mongo.py b/requests_cache/backends/mongo.py index 878f46d..e487823 100644 --- a/requests_cache/backends/mongo.py +++ b/requests_cache/backends/mongo.py @@ -75,7 +75,7 @@ class MongoPickleDict(MongoDict): """Same as :class:`MongoDict`, but pickles values before saving""" def __setitem__(self, key, item): - super().__setitem__(key, self.serialize(item)) + super().__setitem__(key, self.serializer.dumps(item)) def __getitem__(self, key): - return self.deserialize(super().__getitem__(key)) + return self.serializer.loads(super().__getitem__(key)) diff --git a/requests_cache/backends/redis.py b/requests_cache/backends/redis.py index 57ee2b4..51fbae8 100644 --- a/requests_cache/backends/redis.py +++ b/requests_cache/backends/redis.py @@ -2,6 +2,7 @@ from typing import Iterable from redis import Redis, StrictRedis +from ..cache_keys import decode, encode from . import BaseCache, BaseStorage, get_valid_kwargs @@ -43,29 +44,29 @@ class RedisDict(BaseStorage): self._self_key = ':'.join([namespace, collection_name]) def __getitem__(self, key): - result = self.connection.hget(self._self_key, self.serialize(key)) + result = self.connection.hget(self._self_key, encode(key)) if result is None: raise KeyError - return self.deserialize(result) + return self.serializer.loads(result) def __setitem__(self, key, item): - self.connection.hset(self._self_key, self.serialize(key), self.serialize(item)) + self.connection.hset(self._self_key, encode(key), self.serializer.dumps(item)) def __delitem__(self, key): - if not self.connection.hdel(self._self_key, self.serialize(key)): + if not self.connection.hdel(self._self_key, encode(key)): raise KeyError def __len__(self): return self.connection.hlen(self._self_key) def __iter__(self): - for v in self.connection.hkeys(self._self_key): - yield self.deserialize(v) + for key in self.connection.hkeys(self._self_key): + yield decode(key) def bulk_delete(self, keys: Iterable[str]): """Delete multiple keys from the cache. Does not raise errors for missing keys.""" if keys: - self.connection.hdel(self._self_key, *[self.serialize(key) for key in keys]) + self.connection.hdel(self._self_key, *[encode(key) for key in keys]) def clear(self): self.connection.delete(self._self_key) diff --git a/requests_cache/backends/sqlite.py b/requests_cache/backends/sqlite.py index 0efbc7f..9b894b5 100644 --- a/requests_cache/backends/sqlite.py +++ b/requests_cache/backends/sqlite.py @@ -138,11 +138,11 @@ class DbDict(BaseStorage): raise KeyError return row[0] - def __setitem__(self, key, item): + def __setitem__(self, key, value): with self.connection(commit=True) as con: con.execute( f'INSERT OR REPLACE INTO {self.table_name} (key,value) VALUES (?,?)', - (key, item), + (key, value), ) def __iter__(self): @@ -180,13 +180,16 @@ class DbDict(BaseStorage): class DbPickleDict(DbDict): - """Same as :class:`DbDict`, but pickles values before saving""" + """Same as :class:`DbDict`, but serializes values before saving""" - def __setitem__(self, key, item): - super().__setitem__(key, sqlite3.Binary(self.serialize(item))) + def __setitem__(self, key, value): + serialized_value = self.serializer.dumps(value) + if self.serializer.is_binary: + serialized_value = sqlite3.Binary(serialized_value) + super().__setitem__(key, serialized_value) def __getitem__(self, key): - return self.deserialize(super().__getitem__(key)) + return self.serializer.loads(super().__getitem__(key)) def _format_sequence(values: Iterable) -> Tuple[str, List]: |
