Merge pull request #1407 from iafork/iwana-20210910T2303

Fix mypy type errors and add mypy to .drone.yml

23 files changed, 134 insertions, 41 deletions

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000..c1024fc5
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,4 @@
+.tox
+.venv
+.mypy_cache
+.git
diff --git a/.drone.yml b/.drone.yml
index 55d1c81e..4abdb59c 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -26,8 +26,9 @@ steps:
   - pip install --default-timeout 60 -r requirements.dev.txt
   - pip install --default-timeout 60 coveralls && export HAS_COVERALLS=1
   - python setup.py install
-  - black --config black.toml --check ./rdflib | true
+  - black --config black.toml --check ./rdflib || true
   - flake8 --exit-zero rdflib
+  - mypy --show-error-context --show-error-codes rdflib
   - PYTHONWARNINGS=default nosetests --with-timer --timer-top-n 42 --with-coverage --cover-tests --cover-package=rdflib
   - coverage report --skip-covered
   - coveralls
diff --git a/.flake8 b/.flake8
new file mode 100644
index 00000000..177f450c
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,6 @@
+# https://flake8.pycqa.org/en/latest/user/configuration.html
+[flake8]
+extend-ignore =
+    # E501: line too long
+    # Disabled so that black can control line length.
+    E501,
diff --git a/Makefile b/Makefile
index 171f76cf..3b9b73f5 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,7 @@ tests:
 	docker-compose -f docker-compose.tests.yml up test-runner
 	docker-compose -f docker-compose.tests.yml down
 
+.PHONY: build
 build:
 	docker-compose -f docker-compose.tests.yml build
 
@@ -14,3 +15,6 @@ reformat:
 
 check-format:
 	black --config ./black.toml --check .
+
+check-types:
+	docker-compose -f docker-compose.tests.yml up check-types
diff --git a/docker-compose.tests.yml b/docker-compose.tests.yml
index c71e3143..00c61d28 100644
--- a/docker-compose.tests.yml
+++ b/docker-compose.tests.yml
@@ -13,4 +13,12 @@ services:
       dockerfile: test/Dockerfile
     volumes:
       - .:/rdflib
-    command: ["/rdflib/run_tests_with_coverage_report.sh"]
\ No newline at end of file
+    command: ["/rdflib/run_tests_with_coverage_report.sh"]
+
+  check-types:
+    build:
+      context: .
+      dockerfile: test/Dockerfile
+    volumes:
+      - .:/rdflib
+    command: ["python", "-m", "mypy", "--show-error-context", "--show-error-codes" ,"/rdflib/rdflib"]
diff --git a/docs/developers.rst b/docs/developers.rst
index 52d8ea89..fed80505 100644
--- a/docs/developers.rst
+++ b/docs/developers.rst
@@ -15,6 +15,9 @@ Code should be formatted using `black <https://github.com/psf/black>`_.
 While not yet mandatory, it will be required in the future  (6.0.0+).1
 Use Black v21.6b1, with the black.toml config file provided.
 
+Code should also pass `flake8 <https://github.com/psf/black>`_ linting
+and `mypy <http://mypy-lang.org/>`_ type checking.
+
 Any new functionality being added to RDFLib should have doc tests and
 unit tests. Tests should be added for any functionality being changed
 that currently does not have any doc tests or unit tests. And all the
@@ -28,7 +31,7 @@ Running tests
 -------------
 Run tests with `nose <https://nose.readthedocs.org/en/latest/>`_:
 
-.. code-block: bash
+.. code-block:: bash
 
    $ pip install nose
    $ python run_tests.py
@@ -42,10 +45,31 @@ Specific tests can either be run by module name or file name. For example::
   $ python run_tests.py --tests rdflib.graph
   $ python run_tests.py --tests test/test_graph.py
 
+Running static checks
+---------------------
+
+Check formatting with `black <https://github.com/psf/black>`_:
+
+.. code-block:: bash
+
+    python -m black --config black.toml --check ./rdflib
+
+Check style and conventions with `flake8 <https://github.com/psf/black>`_:
+
+.. code-block:: bash
+
+    python -m flake8 rdflib
+
+Check types with `mypy <http://mypy-lang.org/>`_:
+
+.. code-block:: bash
+
+    python -m mypy --show-error-context --show-error-codes rdflib
+
 Writing documentation
 ---------------------
 
-We use sphinx for generating HTML docs, see :ref:`docs`
+We use sphinx for generating HTML docs, see :ref:`docs`.
 
 Continuous Integration
 ----------------------
diff --git a/rdflib/compare.py b/rdflib/compare.py
index d3620276..1349db8e 100644
--- a/rdflib/compare.py
+++ b/rdflib/compare.py
@@ -454,15 +454,15 @@ class _TripleCanonicalizer(object):
             experimental = self._experimental_path(coloring_copy)
             experimental_score = set([c.key() for c in experimental])
             if last_coloring:
-                generator = self._create_generator(
+                generator = self._create_generator(  # type: ignore[unreachable]
                     [last_coloring, experimental], generator
                 )
             last_coloring = experimental
-            if best_score is None or best_score < color_score:
+            if best_score is None or best_score < color_score:  # type: ignore[unreachable]
                 best = [refined_coloring]
                 best_score = color_score
                 best_experimental_score = experimental_score
-            elif best_score > color_score:
+            elif best_score > color_score:  # type: ignore[unreachable]
                 # prune this branch.
                 if stats is not None:
                     stats["prunings"] += 1
@@ -480,7 +480,7 @@ class _TripleCanonicalizer(object):
                 d = [depth[0]]
                 new_color = self._traces(coloring, stats=stats, depth=d)
                 color_score = tuple([c.key() for c in refined_coloring])
-                if best_score is None or color_score > best_score:
+                if best_score is None or color_score > best_score:  # type: ignore[unreachable]
                     discrete = [new_color]
                     best_score = color_score
                     best_depth = d[0]
diff --git a/rdflib/extras/infixowl.py b/rdflib/extras/infixowl.py
index d1de5d80..fbb28314 100644
--- a/rdflib/extras/infixowl.py
+++ b/rdflib/extras/infixowl.py
@@ -331,9 +331,6 @@ def manchesterSyntax(thing, store, boolean=None, transientList=False):
             if isinstance(thing, BNode):
                 return thing.n3()
             return "<" + thing + ">"
-            logger.debug(list(store.objects(subject=thing, predicate=RDF.type)))
-            raise
-            return "[]"  # +thing._id.encode('utf-8')+'</em>'
         label = first(Class(thing, graph=store).label)
         if label:
             return label
diff --git a/rdflib/graph.py b/rdflib/graph.py
index d416ac1f..805bb7c6 100644
--- a/rdflib/graph.py
+++ b/rdflib/graph.py
@@ -1057,34 +1057,34 @@ class Graph(Node):
     @overload
     def serialize(
         self,
-        destination: Union[str, BufferedIOBase],
+        destination: Union[str, BufferedIOBase, pathlib.PurePath],
         format: str = ...,
         base: Optional[str] = ...,
         encoding: Optional[str] = ...,
         **args,
-    ) -> None:
+    ) -> "Graph":
         ...
 
     # fallback
     @overload
     def serialize(
         self,
-        destination: Union[str, BufferedIOBase, None] = None,
+        destination: Union[str, BufferedIOBase, pathlib.PurePath, None] = None,
         format: str = "turtle",
         base: Optional[str] = None,
         encoding: Optional[str] = None,
         **args,
-    ) -> Optional[Union[bytes, str]]:
+    ) -> Union[bytes, str, "Graph"]:
         ...
 
     def serialize(
         self,
-        destination: Union[str, BufferedIOBase, None] = None,
+        destination: Union[str, BufferedIOBase, pathlib.PurePath, None] = None,
         format: str = "turtle",
         base: Optional[str] = None,
         encoding: Optional[str] = None,
         **args,
-    ) -> Optional[Union[bytes, str]]:
+    ) -> Union[bytes, str, "Graph"]:
         """Serialize the Graph to destination
 
         If destination is None serialize method returns the serialization as
@@ -1123,10 +1123,9 @@ class Graph(Node):
                 location = cast(str, destination)
             scheme, netloc, path, params, _query, fragment = urlparse(location)
             if netloc != "":
-                print(
-                    "WARNING: not saving as location" + "is not a local file reference"
+                raise ValueError(
+                    f"destination {destination} is not a local file reference"
                 )
-                return None
             fd, name = tempfile.mkstemp()
             stream = os.fdopen(fd, "wb")
             serializer.serialize(stream, base=base, encoding=encoding, **args)
diff --git a/rdflib/plugins/parsers/jsonld.py b/rdflib/plugins/parsers/jsonld.py
index dcb5eca3..77fa8b4d 100644
--- a/rdflib/plugins/parsers/jsonld.py
+++ b/rdflib/plugins/parsers/jsonld.py
@@ -35,7 +35,8 @@ Example usage::
 
 import warnings
 from rdflib.graph import ConjunctiveGraph
-from rdflib.parser import Parser, URLInputSource
+from rdflib.parser import URLInputSource
+import rdflib.parser
 from rdflib.namespace import RDF, XSD
 from rdflib.term import URIRef, BNode, Literal
 
@@ -78,12 +79,12 @@ except ImportError:
     pass
 
 
-TYPE_TERM = Term(str(RDF.type), TYPE, VOCAB)
+TYPE_TERM = Term(str(RDF.type), TYPE, VOCAB)  # type: ignore[call-arg]
 
 ALLOW_LISTS_OF_LISTS = True  # NOTE: Not allowed in JSON-LD 1.0
 
 
-class JsonLDParser(Parser):
+class JsonLDParser(rdflib.parser.Parser):
     def __init__(self):
         super(JsonLDParser, self).__init__()
 
diff --git a/rdflib/plugins/parsers/notation3.py b/rdflib/plugins/parsers/notation3.py
index dd64f582..3f8fdbac 100755
--- a/rdflib/plugins/parsers/notation3.py
+++ b/rdflib/plugins/parsers/notation3.py
@@ -1168,7 +1168,7 @@ class SinkParser:
             pfx, ln = qn[0]
             if pfx is None:
                 assert 0, "not used?"
-                ns = self._baseURI + ADDED_HASH
+                ns = self._baseURI + ADDED_HASH  # type: ignore[unreachable]
             else:
                 try:
                     ns = self._bindings[pfx]
diff --git a/rdflib/plugins/parsers/rdfxml.py b/rdflib/plugins/parsers/rdfxml.py
index 2571f771..35986889 100644
--- a/rdflib/plugins/parsers/rdfxml.py
+++ b/rdflib/plugins/parsers/rdfxml.py
@@ -2,7 +2,7 @@
 An RDF/XML parser for RDFLib
 """
 
-from xml.sax import make_parser, handler
+from xml.sax import make_parser, handler, xmlreader
 from xml.sax.handler import ErrorHandler
 from xml.sax.saxutils import quoteattr, escape
 
@@ -266,7 +266,7 @@ class RDFXMLHandler(handler.ContentHandler):
                 pass
             elif att in UNQUALIFIED:
                 # if not RDFNS[att] in atts:
-                atts[RDFNS[att]] = v
+                atts[RDFNS[att]] = v  # type: ignore[misc]
             else:
                 atts[URIRef(att)] = v
         return name, atts
@@ -474,7 +474,7 @@ class RDFXMLHandler(handler.ContentHandler):
                     o = URIRef(atts[att])
                 else:
                     if datatype is not None:
-                        language = None
+                        language = None  # type: ignore[unreachable]
                     o = Literal(atts[att], language, datatype)
 
                 if object is None:
@@ -575,12 +575,12 @@ class RDFXMLHandler(handler.ContentHandler):
         self.parent.object += self.current.object + end
 
 
-def create_parser(target, store):
+def create_parser(target, store) -> xmlreader.XMLReader:
     parser = make_parser()
     try:
         # Workaround for bug in expatreader.py. Needed when
         # expatreader is trying to guess a prefix.
-        parser.start_namespace_decl("xml", "http://www.w3.org/XML/1998/namespace")
+        parser.start_namespace_decl("xml", "http://www.w3.org/XML/1998/namespace")  # type: ignore[attr-defined]
     except AttributeError:
         pass  # Not present in Jython (at least)
     parser.setFeature(handler.feature_namespaces, 1)
diff --git a/rdflib/plugins/shared/jsonld/context.py b/rdflib/plugins/shared/jsonld/context.py
index bfd41e23..30bb02ea 100644
--- a/rdflib/plugins/shared/jsonld/context.py
+++ b/rdflib/plugins/shared/jsonld/context.py
@@ -544,4 +544,4 @@ Term = namedtuple(
     "Term",
     "id, name, type, container, index, language, reverse, context," "prefix, protected",
 )
-Term.__new__.__defaults__ = (UNDEF, UNDEF, UNDEF, UNDEF, False, UNDEF, False, False)
+Term.__new__.__defaults__ = (UNDEF, UNDEF, UNDEF, UNDEF, False, UNDEF, False, False)  # type: ignore[attr-defined]
diff --git a/rdflib/plugins/shared/jsonld/util.py b/rdflib/plugins/shared/jsonld/util.py
index 50b34b54..49fbdab4 100644
--- a/rdflib/plugins/shared/jsonld/util.py
+++ b/rdflib/plugins/shared/jsonld/util.py
@@ -1,12 +1,16 @@
 # -*- coding: utf-8 -*-
 # https://github.com/RDFLib/rdflib-jsonld/blob/feature/json-ld-1.1/rdflib_jsonld/util.py
+import typing as t
 
-try:
+if t.TYPE_CHECKING:
     import json
+else:
+    try:
+        import json
 
-    assert json  # workaround for pyflakes issue #13
-except ImportError:
-    import simplejson as json
+        assert json  # workaround for pyflakes issue #13
+    except ImportError:
+        import simplejson as json
 
 from os import sep
 from os.path import normpath
diff --git a/rdflib/plugins/sparql/algebra.py b/rdflib/plugins/sparql/algebra.py
index dd5216bb..3177d47d 100644
--- a/rdflib/plugins/sparql/algebra.py
+++ b/rdflib/plugins/sparql/algebra.py
@@ -801,7 +801,7 @@ class ExpressionNotCoveredException(Exception):
     pass
 
 
-def translateAlgebra(query_algebra: Query = None):
+def translateAlgebra(query_algebra: Query):
     """
 
     :param query_algebra: An algebra returned by the function call algebra.translateQuery(parse_tree).
diff --git a/rdflib/plugins/sparql/sparql.py b/rdflib/plugins/sparql/sparql.py
index 70f803d0..eedc9e74 100644
--- a/rdflib/plugins/sparql/sparql.py
+++ b/rdflib/plugins/sparql/sparql.py
@@ -78,7 +78,7 @@ class Bindings(MutableMapping):
         while d is not None:
             i += len(d._d)
             d = d.outer
-        return i
+        return i  # type: ignore[unreachable]
 
     def __iter__(self):
         d = self
diff --git a/rdflib/py.typed b/rdflib/py.typed
new file mode 100644
index 00000000..cff72238
--- /dev/null
+++ b/rdflib/py.typed
@@ -0,0 +1,2 @@
+# PEP 561 marker file
+# https://www.python.org/dev/peps/pep-0561/
diff --git a/rdflib/store.py b/rdflib/store.py
index 9461eecf..915ca57a 100644
--- a/rdflib/store.py
+++ b/rdflib/store.py
@@ -364,8 +364,6 @@ class Store(object):
 
     def namespaces(self):
         """ """
-        if False:
-            yield None
 
     # Optional Transactional methods
 
diff --git a/requirements.dev.txt b/requirements.dev.txt
index e0b9cf7a..fda1b49e 100644
--- a/requirements.dev.txt
+++ b/requirements.dev.txt
@@ -7,3 +7,6 @@ flake8
 doctest-ignore-unicode==0.1.2
 berkeleydb
 black==21.6b0
+flake8-black
+mypy
+types-setuptools
diff --git a/setup.cfg b/setup.cfg
index a334b051..798121d6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,7 @@
-[nosetests]
+[options.package_data]
+rdflib = py.typed
 
+[nosetests]
 attr=!known_issue,!non_core,!performancetest
 verbosity=1
 with-doctest=1
@@ -33,3 +35,4 @@ python_version = 3.6
 warn_unused_configs = True
 ignore_missing_imports = True
 disallow_subclassing_any = False
+warn_unreachable = True
diff --git a/test/test_csv2rdf.py b/test/test_csv2rdf.py
index 16cf678c..c3e9b92f 100644
--- a/test/test_csv2rdf.py
+++ b/test/test_csv2rdf.py
@@ -1,5 +1,6 @@
 import subprocess
 import unittest
+import sys
 from os import remove
 from tempfile import mkstemp
 from pathlib import Path
@@ -11,7 +12,12 @@ class CSV2RDFTest(unittest.TestCase):
 
     def test_csv2rdf_cli(self):
         completed = subprocess.run(
-            ["csv2rdf", str(self.REALESTATE_FILE_PATH)],
+            [
+                sys.executable,
+                "-m",
+                "rdflib.tools.csv2rdf",
+                str(self.REALESTATE_FILE_PATH),
+            ],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             universal_newlines=True,
@@ -25,7 +31,14 @@ class CSV2RDFTest(unittest.TestCase):
     def test_csv2rdf_cli_fileout(self):
         _, fname = mkstemp()
         completed = subprocess.run(
-            ["csv2rdf", "-o", fname, str(self.REALESTATE_FILE_PATH)],
+            [
+                sys.executable,
+                "-m",
+                "rdflib.tools.csv2rdf",
+                "-o",
+                fname,
+                str(self.REALESTATE_FILE_PATH),
+            ],
         )
         self.assertEqual(completed.returncode, 0)
         with open(fname) as f:
diff --git a/test/test_serialize.py b/test/test_serialize.py
index cfc8a9d7..90fe14df 100644
--- a/test/test_serialize.py
+++ b/test/test_serialize.py
@@ -38,6 +38,23 @@ class TestSerialize(unittest.TestCase):
 
         self.assertEqual(self.triple, next(iter(graph_check)))
 
+    def test_serialize_to_neturl(self):
+        with self.assertRaises(ValueError) as raised:
+            self.graph.serialize(destination="http://example.com/", format="nt")
+        self.assertIn("destination", f"{raised.exception}")
+
+    def test_serialize_to_fileurl(self):
+        with TemporaryDirectory() as td:
+            tfpath = Path(td) / "out.nt"
+            tfurl = tfpath.as_uri()
+            self.assertRegex(tfurl, r"^file:")
+            self.assertFalse(tfpath.exists())
+            self.graph.serialize(destination=tfurl, format="nt")
+            self.assertTrue(tfpath.exists())
+            graph_check = Graph()
+            graph_check.parse(source=tfpath, format="nt")
+        self.assertEqual(self.triple, next(iter(graph_check)))
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tox.ini b/tox.ini
index 2fd97716..0615a0b3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -24,3 +24,12 @@ commands =
 deps =
 	-rrequirements.txt
 	-rrequirements.dev.txt
+
+[testenv:mypy]
+basepython =
+    python3.7
+commands =
+    {envpython} -m mypy rdflib --show-error-context --show-error-codes
+deps =
+	-rrequirements.txt
+	-rrequirements.dev.txt