From 40dc1e79d1c75d0007fe74becf75a1dc3e84d6b9 Mon Sep 17 00:00:00 2001 From: Jordan Cook Date: Wed, 21 Apr 2021 21:12:46 -0500 Subject: For SQLite and filesystem backends, resolve file paths in BaseStorage class rather than BaseCache class --- requests_cache/backends/filesystem.py | 11 +++++------ requests_cache/backends/sqlite.py | 3 +-- 2 files changed, 6 insertions(+), 8 deletions(-) (limited to 'requests_cache') diff --git a/requests_cache/backends/filesystem.py b/requests_cache/backends/filesystem.py index c85751f..b7f3e6a 100644 --- a/requests_cache/backends/filesystem.py +++ b/requests_cache/backends/filesystem.py @@ -24,18 +24,17 @@ class FileCache(BaseCache): def __init__(self, cache_name: Union[Path, str] = 'http_cache', use_temp: bool = False, **kwargs): super().__init__(**kwargs) - cache_dir = _get_cache_dir(cache_name, use_temp) - self.responses = FileDict(cache_dir, **kwargs) - self.redirects = DbDict(join(cache_dir, 'redirects.sqlite'), 'redirects', **kwargs) + self.responses = FileDict(cache_name, use_temp=use_temp, **kwargs) + self.redirects = DbDict(join(self.responses.cache_dir, 'redirects.sqlite'), 'redirects', **kwargs) class FileDict(BaseStorage): """A dictionary-like interface to files on the local filesystem""" - def __init__(self, cache_dir, **kwargs): + def __init__(self, cache_name, use_temp: bool = False, **kwargs): kwargs.setdefault('suppress_warnings', True) super().__init__(**kwargs) - self.cache_dir = cache_dir + self.cache_dir = _get_cache_dir(cache_name, use_temp) makedirs(self.cache_dir, exist_ok=True) @contextmanager @@ -70,7 +69,7 @@ class FileDict(BaseStorage): def clear(self): with self._try_io(ignore_errors=True): - rmtree(self.cache_dir) + rmtree(self.cache_dir, ignore_errors=True) makedirs(self.cache_dir) def paths(self): diff --git a/requests_cache/backends/sqlite.py b/requests_cache/backends/sqlite.py index 470d00c..face65e 100644 --- a/requests_cache/backends/sqlite.py +++ b/requests_cache/backends/sqlite.py @@ -25,7 +25,6 @@ class DbCache(BaseCache): def __init__(self, db_path: Union[Path, str] = 'http_cache', fast_save: bool = False, **kwargs): super().__init__(**kwargs) - db_path = _get_db_path(db_path) self.responses = DbPickleDict(db_path, table_name='responses', fast_save=fast_save, **kwargs) self.redirects = DbDict(db_path, table_name='redirects', **kwargs) @@ -60,7 +59,7 @@ class DbDict(BaseStorage): kwargs.setdefault('suppress_warnings', True) super().__init__(**kwargs) self.connection_kwargs = get_valid_kwargs(sqlite_template, kwargs) - self.db_path = db_path + self.db_path = _get_db_path(db_path) self.fast_save = fast_save self.table_name = table_name -- cgit v1.2.1 From e78ae33b552f6549b86c4c5355d2b8aae13ee369 Mon Sep 17 00:00:00 2001 From: Jordan Cook Date: Thu, 22 Apr 2021 00:00:26 -0500 Subject: Store responses for filesystem backend in a 'responses' subdirectory, to avoid deleting redirects.sqlite with clear() --- requests_cache/backends/filesystem.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'requests_cache') diff --git a/requests_cache/backends/filesystem.py b/requests_cache/backends/filesystem.py index b7f3e6a..c9fc515 100644 --- a/requests_cache/backends/filesystem.py +++ b/requests_cache/backends/filesystem.py @@ -1,7 +1,7 @@ # TODO: Add option for compression? from contextlib import contextmanager from os import listdir, makedirs, unlink -from os.path import abspath, expanduser, isabs, join +from os.path import abspath, dirname, expanduser, isabs, join from pathlib import Path from pickle import PickleError from shutil import rmtree @@ -13,8 +13,9 @@ from .sqlite import DbDict class FileCache(BaseCache): - """Backend that stores cached responses as files on the local filesystem. Response paths will be - in the format ``/``. Redirects are stored in a SQLite database. + """Backend that stores cached responses as files on the local filesystem. + Response paths will be in the format ``/responses/``. + Redirects are stored in a SQLite database, located at ``/redirects.sqlite``. Args: cache_name: Base directory for cache files @@ -25,7 +26,8 @@ class FileCache(BaseCache): def __init__(self, cache_name: Union[Path, str] = 'http_cache', use_temp: bool = False, **kwargs): super().__init__(**kwargs) self.responses = FileDict(cache_name, use_temp=use_temp, **kwargs) - self.redirects = DbDict(join(self.responses.cache_dir, 'redirects.sqlite'), 'redirects', **kwargs) + db_path = join(dirname(self.responses.cache_dir), 'redirects.sqlite') + self.redirects = DbDict(db_path, 'redirects', **kwargs) class FileDict(BaseStorage): @@ -80,7 +82,7 @@ class FileDict(BaseStorage): def _get_cache_dir(cache_dir: Union[Path, str], use_temp: bool) -> str: if use_temp and not isabs(cache_dir): - cache_dir = join(gettempdir(), cache_dir) + cache_dir = join(gettempdir(), cache_dir, 'responses') cache_dir = abspath(expanduser(str(cache_dir))) makedirs(cache_dir, exist_ok=True) return cache_dir -- cgit v1.2.1 From dc204a8d7df9c627fcfb2b098a28f2f27fbb31c8 Mon Sep 17 00:00:00 2001 From: Jordan Cook Date: Thu, 22 Apr 2021 00:21:33 -0500 Subject: Move storage class __str__ methods to base class, and just show keys instead of items --- requests_cache/backends/base.py | 3 +++ requests_cache/backends/dynamodb.py | 3 --- requests_cache/backends/gridfs.py | 3 --- requests_cache/backends/mongo.py | 3 --- requests_cache/backends/redis.py | 3 --- requests_cache/backends/sqlite.py | 3 --- 6 files changed, 3 insertions(+), 15 deletions(-) (limited to 'requests_cache') diff --git a/requests_cache/backends/base.py b/requests_cache/backends/base.py index e912879..def4dc1 100644 --- a/requests_cache/backends/base.py +++ b/requests_cache/backends/base.py @@ -201,3 +201,6 @@ class BaseStorage(MutableMapping, ABC): return Serializer(secret_key, salt=salt, serializer=pickle) else: return pickle + + def __str__(self): + return str(list(self.keys())) diff --git a/requests_cache/backends/dynamodb.py b/requests_cache/backends/dynamodb.py index ffb610c..bcd1e26 100644 --- a/requests_cache/backends/dynamodb.py +++ b/requests_cache/backends/dynamodb.py @@ -110,9 +110,6 @@ class DynamoDbDict(BaseStorage): composite_key = {'namespace': v['namespace'], 'key': v['key']} self._table.delete_item(Key=composite_key) - def __str__(self): - return str(dict(self.items())) - def __scan_table(self): expression_attribute_values = {':Namespace': self._self_key} expression_attribute_names = {'#N': 'namespace'} diff --git a/requests_cache/backends/gridfs.py b/requests_cache/backends/gridfs.py index 73f258e..82ca05c 100644 --- a/requests_cache/backends/gridfs.py +++ b/requests_cache/backends/gridfs.py @@ -77,6 +77,3 @@ class GridFSPickleDict(BaseStorage): def clear(self): self.db['fs.files'].drop() self.db['fs.chunks'].drop() - - def __str__(self): - return str(dict(self.items())) diff --git a/requests_cache/backends/mongo.py b/requests_cache/backends/mongo.py index b479a47..9ebadf4 100644 --- a/requests_cache/backends/mongo.py +++ b/requests_cache/backends/mongo.py @@ -70,9 +70,6 @@ class MongoDict(BaseStorage): def clear(self): self.collection.drop() - def __str__(self): - return str(dict(self.items())) - class MongoPickleDict(MongoDict): """Same as :class:`MongoDict`, but pickles values before saving""" diff --git a/requests_cache/backends/redis.py b/requests_cache/backends/redis.py index 33236d5..5209aa6 100644 --- a/requests_cache/backends/redis.py +++ b/requests_cache/backends/redis.py @@ -60,6 +60,3 @@ class RedisDict(BaseStorage): def clear(self): self.connection.delete(self._self_key) - - def __str__(self): - return str(dict(self.items())) diff --git a/requests_cache/backends/sqlite.py b/requests_cache/backends/sqlite.py index face65e..9c98fcd 100644 --- a/requests_cache/backends/sqlite.py +++ b/requests_cache/backends/sqlite.py @@ -143,9 +143,6 @@ class DbDict(BaseStorage): with self.connection(True) as con: con.execute("vacuum") - def __str__(self): - return str(dict(self.items())) - class DbPickleDict(DbDict): """Same as :class:`DbDict`, but pickles values before saving""" -- cgit v1.2.1