replace requests with urllib for SPARQLStore/SPARQLUpdateStore, update relevant e.g.s

4 files changed, 163 insertions, 83 deletions

diff --git a/examples/sparqlstore_example.py b/examples/sparqlstore_example.py
index 936f6540..1d2ed958 100644
--- a/examples/sparqlstore_example.py
+++ b/examples/sparqlstore_example.py
@@ -1,8 +1,7 @@
 """
-A simple example showing how to use the SPARQLStore
+Simple examples showing how to use the SPARQLStore
 """
 
-import locale
 from rdflib import Graph, URIRef, Namespace
 from rdflib.plugins.stores.sparqlstore import SPARQLStore
 
@@ -10,7 +9,7 @@ if __name__ == "__main__":
 
     dbo = Namespace("http://dbpedia.org/ontology/")
 
-    # using a Graph with the Store type string set to "SPARQLStore"
+    # EXAMPLE 1: using a Graph with the Store type string set to "SPARQLStore"
     graph = Graph("SPARQLStore", identifier="http://dbpedia.org")
     graph.open("http://dbpedia.org/sparql")
 
@@ -22,13 +21,28 @@ if __name__ == "__main__":
         ).replace(",", ".")
     )
 
-    # using a SPARQLStore object directly
-    s = SPARQLStore(endpoint="http://dbpedia.org/sparql")
-    s.open(None)
-    pop = graph.value(
-        URIRef("http://dbpedia.org/resource/Brisbane"), dbo.populationTotal
-    )
-    print(
-        "According to DBPedia, Brisbane has a population of "
-        "{0:,}".format(int(pop), ",d")
-    )
+    # EXAMPLE 2: using a SPARQLStore object directly
+    st = SPARQLStore(query_endpoint="http://dbpedia.org/sparql")
+
+    for p in st.objects(URIRef("http://dbpedia.org/resource/Brisbane"), dbo.populationTotal):
+        print(
+            "According to DBPedia, Brisbane has a population of "
+            "{0:,}".format(int(pop), ",d")
+        )
+
+    # EXAMPLE 3: doing RDFlib triple navigation using SPARQLStore as a Graph()
+    graph = Graph("SPARQLStore", identifier="http://dbpedia.org")
+    graph.open("http://dbpedia.org/sparql")
+    # we are asking DBPedia for 3 skos:Concept instances
+    count = 0
+    from rdflib.namespace import RDF, SKOS
+    for s in graph.subjects(predicate=RDF.type, object=SKOS.Concept):
+        count += 1
+        print(s)
+        if count >= 3:
+            break
+
+    # EXAMPLE 4: using a SPARQL endpoint that requires Basic HTTP authentication
+    # NOTE: this example won't run since the endpoint isn't live (or real)
+    s = SPARQLStore(query_endpoint="http://fake-sparql-endpoint.com/repository/x", auth=("my_username", "my_password"))
+    # do normal Graph things
diff --git a/rdflib/plugins/stores/sparqlconnector.py b/rdflib/plugins/stores/sparqlconnector.py
index abec85a8..4e667297 100644
--- a/rdflib/plugins/stores/sparqlconnector.py
+++ b/rdflib/plugins/stores/sparqlconnector.py
@@ -1,6 +1,9 @@
 import logging
 import threading
-import requests
+from urllib.request import urlopen, Request
+from urllib.parse import urlencode
+from urllib.error import HTTPError, URLError
+import base64
 
 import os
 
@@ -26,7 +29,6 @@ _response_mime_types = {
 
 
 class SPARQLConnector(object):
-
     """
     this class deals with nitty gritty details of talking to a SPARQL server
     """
@@ -37,10 +39,13 @@ class SPARQLConnector(object):
         update_endpoint=None,
         returnFormat="xml",
         method="GET",
+        auth=None,
         **kwargs
     ):
         """
-        Any additional keyword arguments will be passed to requests, and can be used to setup timesouts, basic auth, etc.
+        auth, if present, must be a tuple of (username, password) used for Basic Authentication
+
+        Any additional keyword arguments will be passed to to the request, and can be used to setup timesouts etc.
         """
 
         self.returnFormat = returnFormat
@@ -48,18 +53,12 @@ class SPARQLConnector(object):
         self.update_endpoint = update_endpoint
         self.kwargs = kwargs
         self.method = method
-
-        # it is recommended to have one session object per thread/process. This assures that is the case.
-        # https://github.com/kennethreitz/requests/issues/1871
-
-        self._session = threading.local()
-
-    @property
-    def session(self):
-        k = "session_%d" % os.getpid()
-        self._session.__dict__.setdefault(k, requests.Session())
-        log.debug("Session %s %s", os.getpid(), id(self._session.__dict__[k]))
-        return self._session.__dict__[k]
+        if auth is not None:
+            assert type(auth) == tuple, "auth must be a tuple"
+            assert len(auth) == 2, "auth must be a tuple (user, password)"
+            base64string = base64.b64encode(bytes('%s:%s' % auth, 'ascii'))
+            self.kwargs.setdefault("headers", {})
+            self.kwargs["headers"].update({"Authorization": "Basic %s" % base64string.decode('utf-8')})
 
     @property
     def method(self):
@@ -72,7 +71,7 @@ class SPARQLConnector(object):
 
         self._method = method
 
-    def query(self, query, default_graph=None):
+    def query(self, query, default_graph: str = None, named_graph: str = None):
 
         if not self.query_endpoint:
             raise SPARQLConnectorException("Query endpoint not set!")
@@ -84,7 +83,6 @@ class SPARQLConnector(object):
         headers = {"Accept": _response_mime_types[self.returnFormat]}
 
         args = dict(self.kwargs)
-        args.update(url=self.query_endpoint)
 
         # merge params/headers dicts
         args.setdefault("params", {})
@@ -94,47 +92,44 @@ class SPARQLConnector(object):
 
         if self.method == "GET":
             args["params"].update(params)
+            qsa = "?" + urlencode(args["params"])
+            res = urlopen(Request(self.query_endpoint + qsa, headers=args["headers"]))
         elif self.method == "POST":
             args["headers"].update({"Content-Type": "application/sparql-query"})
-            args["data"] = params
+            try:
+                res = urlopen(Request(self.query_endpoint, data=query.encode(), headers=args["headers"]))
+            except HTTPError as e:
+                return e.code, str(e), None
         else:
             raise SPARQLConnectorException("Unknown method %s" % self.method)
 
-        res = self.session.request(self.method, **args)
-
-        res.raise_for_status()
-
         return Result.parse(
-            BytesIO(res.content), content_type=res.headers["Content-type"]
+            BytesIO(res.read()), content_type=res.headers["Content-Type"].split(";")[0]
         )
 
-    def update(self, update, default_graph=None):
+    def update(self, query, default_graph: str = None, named_graph: str = None):
         if not self.update_endpoint:
             raise SPARQLConnectorException("Query endpoint not set!")
 
         params = {}
 
-        if default_graph:
+        if default_graph is not None:
             params["using-graph-uri"] = default_graph
 
+        if named_graph is not None:
+            params["using-named-graph-uri"] = default_graph
+
         headers = {
             "Accept": _response_mime_types[self.returnFormat],
             "Content-Type": "application/sparql-update",
         }
 
-        args = dict(self.kwargs)
-
-        args.update(url=self.update_endpoint, data=update.encode("utf-8"))
+        args = dict(self.kwargs)  # other QSAs
 
-        # merge params/headers dicts
         args.setdefault("params", {})
         args["params"].update(params)
         args.setdefault("headers", {})
         args["headers"].update(headers)
 
-        res = self.session.post(**args)
-
-        res.raise_for_status()
-
-    def close(self):
-        self.session.close()
+        qsa = "?" + urlencode(args["params"])
+        res = urlopen(Request(self.update_endpoint + qsa, data=query.encode(), headers=args["headers"]))
diff --git a/rdflib/plugins/stores/sparqlstore.py b/rdflib/plugins/stores/sparqlstore.py
index 51c40844..612c3074 100644
--- a/rdflib/plugins/stores/sparqlstore.py
+++ b/rdflib/plugins/stores/sparqlstore.py
@@ -93,17 +93,18 @@ class SPARQLStore(SPARQLConnector, Store):
 
     def __init__(
         self,
-        endpoint=None,
+        query_endpoint=None,
         sparql11=True,
         context_aware=True,
         node_to_sparql=_node_to_sparql,
         returnFormat="xml",
+        auth=None,
         **sparqlconnector_kwargs
     ):
         """
         """
         super(SPARQLStore, self).__init__(
-            endpoint, returnFormat=returnFormat, **sparqlconnector_kwargs
+            query_endpoint=query_endpoint, returnFormat=returnFormat, auth=auth, **sparqlconnector_kwargs
         )
 
         self.node_to_sparql = node_to_sparql
@@ -113,19 +114,20 @@ class SPARQLStore(SPARQLConnector, Store):
         self.graph_aware = context_aware
         self._queries = 0
 
-    # Database Management Methods
-    def create(self, configuration):
-        raise TypeError("The SPARQL store is read only")
-
-    def open(self, configuration, create=False):
-        """
-        sets the endpoint URL for this SPARQLStore
-        if create==True an exception is thrown.
+    def open(self, configuration: str, create=False):
+        """This method is included so that calls to this Store via Graph, e.g. Graph("SPARQLStore"),
+        can set the required parameters
         """
-        if create:
-            raise Exception("Cannot create a SPARQL Endpoint")
+        if type(configuration) == str:
+            self.query_endpoint = configuration
+        else:
+            raise Exception(
+                "configuration must be a string (a single query endpoint URI)"
+            )
 
-        self.query_endpoint = configuration
+    # Database Management Methods
+    def create(self, configuration):
+        raise TypeError("The SPARQL Store is read only. Try SPARQLUpdateStore for read/write.")
 
     def destroy(self, configuration):
         raise TypeError("The SPARQL store is read only")
@@ -280,6 +282,9 @@ class SPARQLStore(SPARQLConnector, Store):
         )
 
         if vars:
+            if type(result) == tuple:
+                if result[0] == 401:
+                    raise ValueError("It looks like you need to authenticate with this SPARQL Store. HTTP unauthorized")
             for row in result:
                 yield (
                     row.get(s, s),
@@ -381,8 +386,35 @@ class SPARQLStore(SPARQLConnector, Store):
         else:
             return graph.identifier != DATASET_DEFAULT_GRAPH_ID
 
-    def close(self, commit_pending_transaction=None):
-        SPARQLConnector.close(self)
+    def subjects(self, predicate=None, object=None):
+        """A generator of subjects with the given predicate and object"""
+        for t, c in self.triples((None, predicate, object)):
+            yield t[0]
+
+    def predicates(self, subject=None, object=None):
+        """A generator of predicates with the given subject and object"""
+        for t, c in self.triples((subject, None, object)):
+            yield t[1]
+
+    def objects(self, subject=None, predicate=None):
+        """A generator of objects with the given subject and predicate"""
+        for t, c in self.triples((subject, predicate, None)):
+            yield t[2]
+
+    def subject_predicates(self, object=None):
+        """A generator of (subject, predicate) tuples for the given object"""
+        for t, c in self.triples((None, None, object)):
+            yield t[0], t[1]
+
+    def subject_objects(self, predicate=None):
+        """A generator of (subject, object) tuples for the given predicate"""
+        for t, c in self.triples((None, predicate, None)):
+            yield t[0], t[2]
+
+    def predicate_objects(self, subject=None):
+        """A generator of (predicate, object) tuples for the given subject"""
+        for t, c in self.triples((subject, None, None)):
+            yield t[1], t[2]
 
 
 class SPARQLUpdateStore(SPARQLStore):
@@ -460,7 +492,7 @@ class SPARQLUpdateStore(SPARQLStore):
 
     def __init__(
         self,
-        queryEndpoint=None,
+        query_endpoint=None,
         update_endpoint=None,
         sparql11=True,
         context_aware=True,
@@ -481,7 +513,7 @@ class SPARQLUpdateStore(SPARQLStore):
 
         SPARQLStore.__init__(
             self,
-            queryEndpoint,
+            query_endpoint,
             sparql11,
             context_aware,
             update_endpoint=update_endpoint,
@@ -494,6 +526,21 @@ class SPARQLUpdateStore(SPARQLStore):
         self._edits = None
         self._updates = 0
 
+    def open(self, configuration: str or tuple, create=False):
+        """This method is included so that calls to this Store via Graph, e.g. Graph("SPARQLStore"),
+        can set the required parameters
+        """
+        if type(configuration) == str:
+            self.query_endpoint = configuration
+        elif type(configuration) == tuple:
+            self.query_endpoint = configuration[0]
+            self.update_endpoint = configuration[1]
+        else:
+            raise Exception(
+                "configuration must be either a string (a single query endpoint URI) "
+                "or a tuple (a query/update endpoint URI pair)"
+            )
+
     def query(self, *args, **kwargs):
         if not self.autocommit and not self.dirty_reads:
             self.commit()
@@ -556,7 +603,7 @@ class SPARQLUpdateStore(SPARQLStore):
         """ Add a triple to the store of triples. """
 
         if not self.update_endpoint:
-            raise Exception("UpdateEndpoint is not set - call 'open'")
+            raise Exception("UpdateEndpoint is not set")
 
         assert not quoted
         (subject, predicate, obj) = spo
@@ -664,7 +711,7 @@ class SPARQLUpdateStore(SPARQLStore):
 
         """
         if not self.update_endpoint:
-            raise Exception("UpdateEndpoint is not set - call 'open'")
+            raise Exception("Update endpoint is not set!")
 
         self.debug = DEBUG
         assert isinstance(query, str)
@@ -754,13 +801,6 @@ class SPARQLUpdateStore(SPARQLStore):
         elif graph.identifier != DATASET_DEFAULT_GRAPH_ID:
             self.update("CREATE GRAPH %s" % self.node_to_sparql(graph.identifier))
 
-    def close(self, commit_pending_transaction=False):
-
-        if commit_pending_transaction:
-            self.commit()
-
-        super(SPARQLStore, self).close()
-
     def remove_graph(self, graph):
         if not self.graph_aware:
             Store.remove_graph(self, graph)
@@ -768,3 +808,33 @@ class SPARQLUpdateStore(SPARQLStore):
             self.update("DROP DEFAULT")
         else:
             self.update("DROP GRAPH %s" % self.node_to_sparql(graph.identifier))
+
+    def subjects(self, predicate=None, object=None):
+        """A generator of subjects with the given predicate and object"""
+        for t, c in self.triples((None, predicate, object)):
+            yield t[0]
+
+    def predicates(self, subject=None, object=None):
+        """A generator of predicates with the given subject and object"""
+        for t, c in self.triples((subject, None, object)):
+            yield t[1]
+
+    def objects(self, subject=None, predicate=None):
+        """A generator of objects with the given subject and predicate"""
+        for t, c in self.triples((subject, predicate, None)):
+            yield t[2]
+
+    def subject_predicates(self, object=None):
+        """A generator of (subject, predicate) tuples for the given object"""
+        for t, c in self.triples((None, None, object)):
+            yield t[0], t[1]
+
+    def subject_objects(self, predicate=None):
+        """A generator of (subject, object) tuples for the given predicate"""
+        for t, c in self.triples((None, predicate, None)):
+            yield t[0], t[2]
+
+    def predicate_objects(self, subject=None):
+        """A generator of (predicate, object) tuples for the given subject"""
+        for t, c in self.triples((subject, None, None)):
+            yield t[1], t[2]
diff --git a/test/test_sparqlstore.py b/test/test_sparqlstore.py
index 38a8b481..3ca69186 100644
--- a/test/test_sparqlstore.py
+++ b/test/test_sparqlstore.py
@@ -1,12 +1,11 @@
 from rdflib import Graph, URIRef, Literal
 from urllib.request import urlopen
+from urllib.error import HTTPError
 import unittest
 from nose import SkipTest
-from requests import HTTPError
 from http.server import BaseHTTPRequestHandler, HTTPServer
 import socket
 from threading import Thread
-import requests
 
 try:
     assert len(urlopen("http://dbpedia.org/sparql").read()) > 0
@@ -119,7 +118,7 @@ class SPARQL11ProtocolStoreMock(BaseHTTPRequestHandler):
         ```
         """
         contenttype = self.headers.get("Content-Type")
-        if self.path == "/query":
+        if self.path == "/query" or self.path == "/query?":
             if self.headers.get("Content-Type") == "application/sparql-query":
                 pass
             elif (
@@ -127,9 +126,9 @@ class SPARQL11ProtocolStoreMock(BaseHTTPRequestHandler):
             ):
                 pass
             else:
-                self.send_response(requests.codes.not_acceptable)
+                self.send_response(406, "Not Acceptable")
                 self.end_headers()
-        elif self.path == "/update":
+        elif self.path == "/update" or self.path == "/update?":
             if self.headers.get("Content-Type") == "application/sparql-update":
                 pass
             elif (
@@ -137,18 +136,20 @@ class SPARQL11ProtocolStoreMock(BaseHTTPRequestHandler):
             ):
                 pass
             else:
-                self.send_response(requests.codes.not_acceptable)
+                self.send_response(406, "Not Acceptable")
                 self.end_headers()
         else:
-            self.send_response(requests.codes.not_found)
+            print("self.path")
+            print(self.path)
+            self.send_response(404, "Not Found")
             self.end_headers()
-        self.send_response(requests.codes.ok)
+        self.send_response(200, "OK")
         self.end_headers()
         return
 
     def do_GET(self):
         # Process an HTTP GET request and return a response with an HTTP 200 status.
-        self.send_response(requests.codes.ok)
+        self.send_response(200, "OK")
         self.end_headers()
         return