Merge pull request #238 from JWCook/filesystem-backend

Add a filesystem backend

4 files changed, 94 insertions, 2 deletions

diff --git a/requests_cache/__init__.py b/requests_cache/__init__.py
index cc8cd3d..f5b61a8 100644
--- a/requests_cache/__init__.py
+++ b/requests_cache/__init__.py
@@ -2,7 +2,7 @@
 from logging import getLogger
 from os import getenv
 
-__version__ = '0.6.3'
+__version__ = '0.7.0'
 
 try:
     from .response import AnyResponse, CachedHTTPResponse, CachedResponse, ExpirationTime
diff --git a/requests_cache/backends/__init__.py b/requests_cache/backends/__init__.py
index f64f051..2b165a2 100644
--- a/requests_cache/backends/__init__.py
+++ b/requests_cache/backends/__init__.py
@@ -72,10 +72,15 @@ try:
     from .sqlite import DbCache, DbDict, DbPickleDict
 except ImportError as e:
     DbCache = DbDict = DbPickleDict = get_placeholder_backend(e)  # type: ignore
+try:
+    from .filesystem import FileCache, FileDict
+except ImportError as e:
+    FileCache = FileDict = get_placeholder_backend(e)  # type: ignore
 
 
 BACKEND_CLASSES = {
     'dynamodb': DynamoDbCache,
+    'filesystem': FileCache,
     'gridfs': GridFSCache,
     'memory': BaseCache,
     'mongo': MongoCache,
diff --git a/requests_cache/backends/filesystem.py b/requests_cache/backends/filesystem.py
new file mode 100644
index 0000000..c85751f
--- /dev/null
+++ b/requests_cache/backends/filesystem.py
@@ -0,0 +1,87 @@
+# TODO: Add option for compression?
+from contextlib import contextmanager
+from os import listdir, makedirs, unlink
+from os.path import abspath, expanduser, isabs, join
+from pathlib import Path
+from pickle import PickleError
+from shutil import rmtree
+from tempfile import gettempdir
+from typing import Union
+
+from . import BaseCache, BaseStorage
+from .sqlite import DbDict
+
+
+class FileCache(BaseCache):
+    """Backend that stores cached responses as files on the local filesystem. Response paths will be
+    in the format ``<cache_name>/<cache_key>``. Redirects are stored in a SQLite database.
+
+    Args:
+        cache_name: Base directory for cache files
+        use_temp: Store cache files in a temp directory (e.g., ``/tmp/http_cache/``).
+            Note: if ``cache_name`` is an absolute path, this option will be ignored.
+    """
+
+    def __init__(self, cache_name: Union[Path, str] = 'http_cache', use_temp: bool = False, **kwargs):
+        super().__init__(**kwargs)
+        cache_dir = _get_cache_dir(cache_name, use_temp)
+        self.responses = FileDict(cache_dir, **kwargs)
+        self.redirects = DbDict(join(cache_dir, 'redirects.sqlite'), 'redirects', **kwargs)
+
+
+class FileDict(BaseStorage):
+    """A dictionary-like interface to files on the local filesystem"""
+
+    def __init__(self, cache_dir, **kwargs):
+        kwargs.setdefault('suppress_warnings', True)
+        super().__init__(**kwargs)
+        self.cache_dir = cache_dir
+        makedirs(self.cache_dir, exist_ok=True)
+
+    @contextmanager
+    def _try_io(self, ignore_errors: bool = False):
+        """Attempt an I/O operation, and either ignore errors or re-raise them as KeyErrors"""
+        try:
+            yield
+        except (IOError, OSError, PickleError) as e:
+            if not ignore_errors:
+                raise KeyError(e)
+
+    def __getitem__(self, key):
+        with self._try_io():
+            with open(join(self.cache_dir, str(key)), 'rb') as f:
+                return self.deserialize(f.read())
+
+    def __delitem__(self, key):
+        with self._try_io():
+            unlink(join(self.cache_dir, str(key)))
+
+    def __setitem__(self, key, value):
+        with self._try_io():
+            with open(join(self.cache_dir, str(key)), 'wb') as f:
+                f.write(self.serialize(value))
+
+    def __iter__(self):
+        for filename in listdir(self.cache_dir):
+            yield filename
+
+    def __len__(self):
+        return len(listdir(self.cache_dir))
+
+    def clear(self):
+        with self._try_io(ignore_errors=True):
+            rmtree(self.cache_dir)
+            makedirs(self.cache_dir)
+
+    def paths(self):
+        """Get file paths to all cached responses"""
+        for key in self:
+            yield join(self.cache_dir, key)
+
+
+def _get_cache_dir(cache_dir: Union[Path, str], use_temp: bool) -> str:
+    if use_temp and not isabs(cache_dir):
+        cache_dir = join(gettempdir(), cache_dir)
+    cache_dir = abspath(expanduser(str(cache_dir)))
+    makedirs(cache_dir, exist_ok=True)
+    return cache_dir
diff --git a/requests_cache/backends/sqlite.py b/requests_cache/backends/sqlite.py
index df08cf8..470d00c 100644
--- a/requests_cache/backends/sqlite.py
+++ b/requests_cache/backends/sqlite.py
@@ -136,7 +136,7 @@ class DbDict(BaseStorage):
 
     def clear(self):
         with self.connection(True) as con:
-            con.execute("drop table `%s`" % self.table_name)
+            con.execute("drop table if exists `%s`" % self.table_name)
             con.execute("create table `%s` (key PRIMARY KEY, value)" % self.table_name)
             con.execute("vacuum")