Merge remote-tracking branch 'upstream/master' into mark_ill_typed_literals

9 files changed, 1137 insertions, 0 deletions

diff --git a/test/test_literal/test_datetime.py b/test/test_literal/test_datetime.py
new file mode 100644
index 00000000..d71fc392
--- /dev/null
+++ b/test/test_literal/test_datetime.py
@@ -0,0 +1,77 @@
+import sys
+import unittest
+
+from datetime import datetime
+
+from isodate import datetime_isoformat, UTC
+from isodate.isostrf import DATE_EXT_COMPLETE, TZ_EXT
+
+from rdflib.term import URIRef
+from rdflib.term import Literal
+from rdflib.namespace import XSD
+
+
+class TestRelativeBase(unittest.TestCase):
+    def test_equality(self):
+        x = Literal(
+            "2008-12-01T18:02:00Z",
+            datatype=URIRef("http://www.w3.org/2001/XMLSchema#dateTime"),
+        )
+        self.assertEqual(x == x, True)
+
+    def test_microseconds(self):
+        dt1 = datetime(2009, 6, 15, 23, 37, 6, 522630)
+        l = Literal(dt1)
+
+        # datetime with microseconds should be cast as a literal with using
+        # XML Schema dateTime as the literal datatype
+        self.assertEqual(str(l), "2009-06-15T23:37:06.522630")
+        self.assertEqual(l.datatype, XSD.dateTime)
+
+        dt2 = l.toPython()
+        self.assertEqual(dt2, dt1)
+
+    def test_to_python(self):
+        dt = "2008-12-01T18:02:00"
+        l = Literal(dt, datatype=URIRef("http://www.w3.org/2001/XMLSchema#dateTime"))
+
+        self.assertTrue(isinstance(l.toPython(), datetime))
+        self.assertEqual(l.toPython().isoformat(), dt)
+
+    def test_timezone_z(self):
+        dt = "2008-12-01T18:02:00.522630Z"
+        l = Literal(dt, datatype=URIRef("http://www.w3.org/2001/XMLSchema#dateTime"))
+
+        self.assertTrue(isinstance(l.toPython(), datetime))
+        self.assertEqual(
+            datetime_isoformat(
+                l.toPython(), DATE_EXT_COMPLETE + "T" + "%H:%M:%S.%f" + TZ_EXT
+            ),
+            dt,
+        )
+        self.assertEqual(l.toPython().isoformat(), "2008-12-01T18:02:00.522630+00:00")
+
+    def test_timezone_offset(self):
+        dt = "2010-02-10T12:36:00+03:00"
+        l = Literal(dt, datatype=URIRef("http://www.w3.org/2001/XMLSchema#dateTime"))
+
+        self.assertTrue(isinstance(l.toPython(), datetime))
+        self.assertEqual(l.toPython().isoformat(), dt)
+
+    def test_timezone_offset_to_utc(self):
+        dt = "2010-02-10T12:36:00+03:00"
+        l = Literal(dt, datatype=URIRef("http://www.w3.org/2001/XMLSchema#dateTime"))
+
+        utc_dt = l.toPython().astimezone(UTC)
+        self.assertEqual(datetime_isoformat(utc_dt), "2010-02-10T09:36:00Z")
+
+    def test_timezone_offset_millisecond(self):
+        dt = "2011-01-16T19:39:18.239743+01:00"
+        l = Literal(dt, datatype=URIRef("http://www.w3.org/2001/XMLSchema#dateTime"))
+
+        self.assertTrue(isinstance(l.toPython(), datetime))
+        self.assertEqual(l.toPython().isoformat(), dt)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/test_literal/test_duration.py b/test/test_literal/test_duration.py
new file mode 100644
index 00000000..cdea7ab7
--- /dev/null
+++ b/test/test_literal/test_duration.py
@@ -0,0 +1,46 @@
+import unittest
+from datetime import timedelta
+
+from isodate import Duration, parse_duration
+
+from rdflib.namespace import XSD
+from rdflib.term import Literal
+
+
+class TestDuration(unittest.TestCase):
+    def test_to_python_timedelta(self):
+        l = Literal("P4DT5H6M7S", datatype=XSD.dayTimeDuration)
+        self.assertTrue(isinstance(l.toPython(), timedelta))
+        self.assertEqual(l.toPython(), parse_duration("P4DT5H6M7S"))
+
+    def test_to_python_ym_duration(self):
+        l = Literal("P1Y2M", datatype=XSD.yearMonthDuration)
+        self.assertTrue(isinstance(l.toPython(), Duration))
+        self.assertEqual(l.toPython(), parse_duration("P1Y2M"))
+
+    def test_to_python_ymdhms_duration(self):
+        l = Literal("P1Y2M4DT5H6M7S", datatype=XSD.duration)
+        self.assertTrue(isinstance(l.toPython(), Duration))
+        self.assertEqual(l.toPython(), parse_duration("P1Y2M4DT5H6M7S"))
+
+    def test_equality(self):
+        x = Literal("P1Y2M3W4DT5H6M7S", datatype=XSD.duration)
+        y = Literal("P1Y2M25DT5H6M7S", datatype=XSD.duration)
+        self.assertTrue(x == y)
+
+    def test_duration_le(self):
+        self.assertTrue(
+            Literal("P4DT5H6M7S", datatype=XSD.duration)
+            < Literal("P8DT10H12M14S", datatype=XSD.duration)
+        )
+
+    def test_duration_sum(self):
+        self.assertEqual(
+            Literal("P1Y2M4DT5H6M7S", datatype=XSD.duration)
+            + Literal("P1Y2M4DT5H6M7S", datatype=XSD.duration).toPython(),
+            Literal("P2Y4M8DT10H12M14S", datatype=XSD.duration),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/test_literal/test_hex_binary.py b/test/test_literal/test_hex_binary.py
new file mode 100644
index 00000000..fc152126
--- /dev/null
+++ b/test/test_literal/test_hex_binary.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+import binascii
+from rdflib import Literal, XSD
+
+
+class HexBinaryTestCase(unittest.TestCase):
+    def test_int(self):
+        self._test_integer(5)
+        self._test_integer(3452)
+        self._test_integer(4886)
+
+    def _test_integer(self, i):
+        hex_i = format(i, "x")
+        # Make it has a even-length (Byte)
+        len_hex_i = len(hex_i)
+        hex_i = hex_i.zfill(len_hex_i + len_hex_i % 2)
+
+        l = Literal(hex_i, datatype=XSD.hexBinary)
+        bin_i = l.toPython()
+        self.assertEqual(int(binascii.hexlify(bin_i), 16), i)
+
+        self.assertEqual(str(l), hex_i)
+        self.assertEqual(int(hex_i, 16), i)
+        self.assertEqual(int(l, 16), i)
+        self.assertEqual(int(str(l), 16), i)
+
+    def test_unicode(self):
+        str1 = "Test utf-8 string éàë"
+        # u hexstring
+        hex_str1 = binascii.hexlify(str1.encode("utf-8")).decode()
+        l1 = Literal(hex_str1, datatype=XSD.hexBinary)
+        b_str1 = l1.toPython()
+        self.assertEqual(b_str1.decode("utf-8"), str1)
+        self.assertEqual(str(l1), hex_str1)
+
+        # b hexstring
+        hex_str1b = binascii.hexlify(str1.encode("utf-8"))
+        l1b = Literal(hex_str1b, datatype=XSD.hexBinary)
+        b_str1b = l1b.toPython()
+        self.assertEqual(b_str1, b_str1b)
+        self.assertEqual(b_str1b.decode("utf-8"), str1)
+        self.assertEqual(str(l1b), hex_str1)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/test_literal/test_literal.py b/test/test_literal/test_literal.py
new file mode 100644
index 00000000..5f7c490d
--- /dev/null
+++ b/test/test_literal/test_literal.py
@@ -0,0 +1,371 @@
+# NOTE: The config below enables strict mode for mypy.
+# mypy: no_ignore_errors
+# mypy: warn_unused_configs, disallow_any_generics
+# mypy: disallow_subclassing_any, disallow_untyped_calls
+# mypy: disallow_untyped_defs, disallow_incomplete_defs
+# mypy: check_untyped_defs, disallow_untyped_decorators
+# mypy: no_implicit_optional, warn_redundant_casts, warn_unused_ignores
+# mypy: warn_return_any, no_implicit_reexport, strict_equality
+
+from decimal import Decimal
+from typing import Any, Optional, Sequence, Tuple, Type, Union
+import unittest
+import datetime
+
+import rdflib  # needed for eval(repr(...)) below
+from rdflib.term import Literal, URIRef, _XSD_DOUBLE, bind, _XSD_BOOLEAN
+from rdflib import XSD
+from rdflib.namespace import RDF, Namespace
+
+import pytest
+
+EGNS = Namespace("http://example.com/")
+
+
+class TestLiteral(unittest.TestCase):
+    def setUp(self) -> None:
+        pass
+
+    def test_repr_apostrophe(self) -> None:
+        a = rdflib.Literal("'")
+        b = eval(repr(a))
+        self.assertEqual(a, b)
+
+    def test_repr_quote(self) -> None:
+        a = rdflib.Literal('"')
+        b = eval(repr(a))
+        self.assertEqual(a, b)
+
+    def test_backslash(self) -> None:
+        d = r"""
+<rdf:RDF
+  xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+  xmlns:foo="http://example.org/foo#">
+    <rdf:Description>
+      <foo:bar>a\b</foo:bar>
+    </rdf:Description>
+</rdf:RDF>
+"""
+        g = rdflib.Graph()
+        g.parse(data=d, format="xml")
+        a = rdflib.Literal("a\\b")
+        b = list(g.objects())[0]
+        self.assertEqual(a, b)
+
+    def test_literal_from_bool(self) -> None:
+        l = rdflib.Literal(True)
+        self.assertEqual(l.datatype, rdflib.XSD["boolean"])
+
+
+class TestNewPT:
+    # NOTE: TestNewPT is written for pytest so that pytest features like
+    # parametrize can be used.
+    # New tests should be added here instead of in TestNew.
+    @pytest.mark.parametrize(
+        "lang, exception_type",
+        [
+            ({}, TypeError),
+            ([], TypeError),
+            (1, TypeError),
+            (b"en", TypeError),
+            ("999", ValueError),
+            ("-", ValueError),
+        ],
+    )
+    def test_cant_pass_invalid_lang(
+        self,
+        lang: Any,
+        exception_type: Type[Exception],
+    ) -> None:
+        """
+        Construction of Literal fails if the language tag is invalid.
+        """
+        with pytest.raises(exception_type):
+            Literal("foo", lang=lang)
+
+    @pytest.mark.parametrize(
+        "lexical, datatype, is_ill_formed",
+        [
+            ("true", XSD.boolean, False),
+            ("1", XSD.boolean, False),
+            (b"false", XSD.boolean, False),
+            (b"0", XSD.boolean, False),
+            ("yes", XSD.boolean, True),
+            ("200", XSD.byte, True),
+            (b"-128", XSD.byte, False),
+            ("127", XSD.byte, False),
+            ("255", XSD.unsignedByte, False),
+            ("-100", XSD.unsignedByte, True),
+            (b"200", XSD.unsignedByte, False),
+            (b"64300", XSD.short, True),
+            ("-6000", XSD.short, False),
+            ("1000000", XSD.nonNegativeInteger, False),
+            ("-100", XSD.nonNegativeInteger, True),
+            ("a", XSD.double, True),
+            ("0", XSD.double, False),
+            ("0.1", XSD.double, False),
+            ("0.1", XSD.decimal, False),
+            ("0.g", XSD.decimal, True),
+            ("b", XSD.integer, True),
+            ("2147483647", XSD.int, False),
+            ("2147483648", XSD.int, True),
+            ("2147483648", XSD.integer, False),
+            ("valid ASCII", XSD.string, False),
+            ("Not a valid time", XSD.time, True),
+            ("Not a valid date", XSD.date, True),
+            ("7264666c6962", XSD.hexBinary, False),
+
+            # RDF.langString is not a recognized datatype IRI as we assing no literal value to it, though this should likely change.
+            ("English string", RDF.langString, None),
+
+            # The datatypes IRIs below should never be recognized.
+            ("[p]", EGNS.unrecognized, None),
+        ],
+    )
+    def test_ill_formed_literals(
+        self,
+        lexical: Union[bytes, str],
+        datatype: URIRef,
+        is_ill_formed: Optional[bool],
+    ) -> None:
+        """
+        Construction of Literal fails if the language tag is invalid.
+        """
+        lit = Literal(lexical, datatype=datatype)
+        assert lit.ill_formed is is_ill_formed
+        if is_ill_formed is False:
+            # If the literal is not ill formed it should have a value associated with it.
+            assert lit.value is not None
+
+
+class TestNew(unittest.TestCase):
+    # NOTE: Please use TestNewPT for new tests instead of this which is written
+    # for unittest.
+    def testCantPassLangAndDatatype(self) -> None:
+        self.assertRaises(
+            TypeError, Literal, "foo", lang="en", datatype=URIRef("http://example.com/")
+        )
+
+    def testCantPassInvalidLang(self) -> None:
+        self.assertRaises(ValueError, Literal, "foo", lang="999")
+
+    def testFromOtherLiteral(self) -> None:
+        l = Literal(1)
+        l2 = Literal(l)
+        self.assertTrue(isinstance(l.value, int))
+        self.assertTrue(isinstance(l2.value, int))
+
+        # change datatype
+        l = Literal("1")
+        l2 = Literal(l, datatype=rdflib.XSD.integer)
+        self.assertTrue(isinstance(l2.value, int))
+
+    def testDatatypeGetsAutoURIRefConversion(self) -> None:
+        # drewp disapproves of this behavior, but it should be
+        # represented in the tests
+        x = Literal("foo", datatype="http://example.com/")
+        self.assertTrue(isinstance(x.datatype, URIRef))
+
+        x = Literal("foo", datatype=Literal("pennies"))
+        self.assertEqual(x.datatype, URIRef("pennies"))
+
+
+class TestRepr(unittest.TestCase):
+    def testOmitsMissingDatatypeAndLang(self) -> None:
+        self.assertEqual(repr(Literal("foo")), "rdflib.term.Literal('foo')")
+
+    def testOmitsMissingDatatype(self) -> None:
+        self.assertEqual(
+            repr(Literal("foo", lang="en")),
+            "rdflib.term.Literal('foo', lang='en')",
+        )
+
+    def testOmitsMissingLang(self) -> None:
+        self.assertEqual(
+            repr(Literal("foo", datatype=URIRef("http://example.com/"))),
+            "rdflib.term.Literal('foo', datatype=rdflib.term.URIRef('http://example.com/'))",
+        )
+
+    def testSubclassNameAppearsInRepr(self) -> None:
+        class MyLiteral(Literal):
+            pass
+
+        x = MyLiteral("foo")
+        self.assertEqual(repr(x), "MyLiteral('foo')")
+
+
+class TestDoubleOutput(unittest.TestCase):
+    def testNoDanglingPoint(self) -> None:
+        """confirms the fix for https://github.com/RDFLib/rdflib/issues/237"""
+        vv = Literal("0.88", datatype=_XSD_DOUBLE)
+        out = vv._literal_n3(use_plain=True)
+        self.assertTrue(out in ["8.8e-01", "0.88"], out)
+
+
+class TestParseBoolean(unittest.TestCase):
+    """confirms the fix for https://github.com/RDFLib/rdflib/issues/913"""
+
+    def testTrueBoolean(self) -> None:
+        test_value = Literal("tRue", datatype=_XSD_BOOLEAN)
+        self.assertTrue(test_value.value)
+        test_value = Literal("1", datatype=_XSD_BOOLEAN)
+        self.assertTrue(test_value.value)
+
+    def testFalseBoolean(self) -> None:
+        test_value = Literal("falsE", datatype=_XSD_BOOLEAN)
+        self.assertFalse(test_value.value)
+        test_value = Literal("0", datatype=_XSD_BOOLEAN)
+        self.assertFalse(test_value.value)
+
+    def testNonFalseBoolean(self) -> None:
+        test_value = Literal("abcd", datatype=_XSD_BOOLEAN)
+        self.assertRaises(UserWarning)
+        self.assertFalse(test_value.value)
+        test_value = Literal("10", datatype=_XSD_BOOLEAN)
+        self.assertRaises(UserWarning)
+        self.assertFalse(test_value.value)
+
+
+class TestBindings(unittest.TestCase):
+    def testBinding(self) -> None:
+        class a:
+            def __init__(self, v: str) -> None:
+                self.v = v[3:-3]
+
+            def __str__(self) -> str:
+                return "<<<%s>>>" % self.v
+
+        dtA = rdflib.URIRef("urn:dt:a")
+        bind(dtA, a)
+
+        va = a("<<<2>>>")
+        la = Literal(va, normalize=True)
+        self.assertEqual(la.value, va)
+        self.assertEqual(la.datatype, dtA)
+
+        la2 = Literal("<<<2>>>", datatype=dtA)
+        self.assertTrue(isinstance(la2.value, a))
+        self.assertEqual(la2.value.v, va.v)
+
+        class b:
+            def __init__(self, v: str) -> None:
+                self.v = v[3:-3]
+
+            def __str__(self) -> str:
+                return "B%s" % self.v
+
+        dtB = rdflib.URIRef("urn:dt:b")
+        bind(dtB, b, None, lambda x: "<<<%s>>>" % x)
+
+        vb = b("<<<3>>>")
+        lb = Literal(vb, normalize=True)
+        self.assertEqual(lb.value, vb)
+        self.assertEqual(lb.datatype, dtB)
+
+    def testSpecificBinding(self) -> None:
+        def lexify(s: str) -> str:
+            return "--%s--" % s
+
+        def unlexify(s: str) -> str:
+            return s[2:-2]
+
+        datatype = rdflib.URIRef("urn:dt:mystring")
+
+        # Datatype-specific rule
+        bind(datatype, str, unlexify, lexify, datatype_specific=True)
+
+        s = "Hello"
+        normal_l = Literal(s)
+        self.assertEqual(str(normal_l), s)
+        self.assertEqual(normal_l.toPython(), s)
+        self.assertEqual(normal_l.datatype, None)
+
+        specific_l = Literal("--%s--" % s, datatype=datatype)
+        self.assertEqual(str(specific_l), lexify(s))
+        self.assertEqual(specific_l.toPython(), s)
+        self.assertEqual(specific_l.datatype, datatype)
+
+
+class TestXsdLiterals(unittest.TestCase):
+    def test_make_literals(self) -> None:
+        """
+        Tests literal construction.
+        """
+        inputs = [
+            # these literals do not get converted to Python types
+            ("ABCD", XSD.integer, None),
+            ("ABCD", XSD.gYear, None),
+            ("-10000", XSD.gYear, None),
+            ("-1921-00", XSD.gYearMonth, None),
+            ("1921-00", XSD.gMonthDay, None),
+            ("1921-13", XSD.gMonthDay, None),
+            ("-1921-00", XSD.gMonthDay, None),
+            ("10", XSD.gDay, None),
+            ("-1", XSD.gDay, None),
+            ("0000", XSD.gYear, None),
+            ("0000-00-00", XSD.date, None),
+            ("NOT A VALID HEX STRING", XSD.hexBinary, None),
+            ("NOT A VALID BASE64 STRING", XSD.base64Binary, None),
+            # these literals get converted to python types
+            ("1921-05-01", XSD.date, datetime.date),
+            ("1921-05-01T00:00:00", XSD.dateTime, datetime.datetime),
+            ("1921-05", XSD.gYearMonth, datetime.date),
+            ("0001-01", XSD.gYearMonth, datetime.date),
+            ("0001-12", XSD.gYearMonth, datetime.date),
+            ("2002-01", XSD.gYearMonth, datetime.date),
+            ("9999-01", XSD.gYearMonth, datetime.date),
+            ("9999-12", XSD.gYearMonth, datetime.date),
+            ("1921", XSD.gYear, datetime.date),
+            ("2000", XSD.gYear, datetime.date),
+            ("0001", XSD.gYear, datetime.date),
+            ("9999", XSD.gYear, datetime.date),
+            ("1982", XSD.gYear, datetime.date),
+            ("2002", XSD.gYear, datetime.date),
+            ("1921-05-01T00:00:00+00:30", XSD.dateTime, datetime.datetime),
+            ("1921-05-01T00:00:00-00:30", XSD.dateTime, datetime.datetime),
+            ("true", XSD.boolean, bool),
+            ("abcdef0123", XSD.hexBinary, bytes),
+            ("", XSD.hexBinary, bytes),
+            ("UkRGTGli", XSD.base64Binary, bytes),
+            ("", XSD.base64Binary, bytes),
+            ("0.0000000000000000000000000000001", XSD.decimal, Decimal),
+            ("0.1", XSD.decimal, Decimal),
+            ("1", XSD.integer, int),
+        ]
+        self.check_make_literals(inputs)
+
+    @unittest.expectedFailure
+    def test_make_literals_ki(self) -> None:
+        """
+        Known issues with literal construction.
+        """
+        inputs = [
+            ("1921-01Z", XSD.gYearMonth, datetime.date),
+            ("1921Z", XSD.gYear, datetime.date),
+            ("1921-00", XSD.gYearMonth, datetime.date),
+            ("1921-05-01Z", XSD.date, datetime.date),
+            ("1921-05-01+00:30", XSD.date, datetime.date),
+            ("1921-05-01+00:30", XSD.date, datetime.date),
+            ("1921-05-01+00:00", XSD.date, datetime.date),
+            ("1921-05-01+00:00", XSD.date, datetime.date),
+            ("1921-05-01T00:00:00Z", XSD.dateTime, datetime.datetime),
+            ("1e-31", XSD.decimal, None),  # This is not a valid decimal value
+        ]
+        self.check_make_literals(inputs)
+
+    def check_make_literals(
+        self, inputs: Sequence[Tuple[str, URIRef, Optional[type]]]
+    ) -> None:
+        for literal_pair in inputs:
+            (lexical, _type, value_cls) = literal_pair
+            with self.subTest(f"testing {literal_pair}"):
+                literal = Literal(lexical, datatype=_type)
+                if value_cls is not None:
+                    self.assertIsInstance(literal.value, value_cls)
+                else:
+                    self.assertIsNone(literal.value)
+                self.assertEqual(lexical, f"{literal}")
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/test_literal/test_normalized_string.py b/test/test_literal/test_normalized_string.py
new file mode 100644
index 00000000..5c11c107
--- /dev/null
+++ b/test/test_literal/test_normalized_string.py
@@ -0,0 +1,36 @@
+import unittest
+
+from rdflib import Literal
+from rdflib.namespace import XSD
+
+
+class NormalizedStringTest(unittest.TestCase):
+    def test1(self):
+        lit2 = Literal("\two\nw", datatype=XSD.normalizedString)
+        lit = Literal("\two\nw", datatype=XSD.string)
+        self.assertFalse(str(lit) == str(lit2))
+
+    def test2(self):
+        lit = Literal(
+            "\tBeing a Doctor Is\n\ta Full-Time Job\r", datatype=XSD.normalizedString
+        )
+        st = Literal(" Being a Doctor Is  a Full-Time Job ", datatype=XSD.string)
+        self.assertFalse(Literal.eq(st, lit))
+        self.assertEqual(str(lit), str(st))
+
+    def test3(self):
+        lit = Literal("hey\nthere", datatype=XSD.normalizedString).n3()
+        self.assertTrue(
+            lit == '"hey there"^^<http://www.w3.org/2001/XMLSchema#normalizedString>'
+        )
+
+    def test4(self):
+        lit = Literal(
+            "hey\nthere\ta tab\rcarriage return", datatype=XSD.normalizedString
+        )
+        expected = Literal("""hey there a tab carriage return""", datatype=XSD.string)
+        self.assertEqual(str(lit), str(expected))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/test_literal/test_term.py b/test/test_literal/test_term.py
new file mode 100644
index 00000000..0f9dbea2
--- /dev/null
+++ b/test/test_literal/test_term.py
@@ -0,0 +1,285 @@
+"""
+some more specific Literal tests are in test_literal.py
+"""
+
+import base64
+import unittest
+import random
+
+from rdflib.term import URIRef, BNode, Literal, _is_valid_unicode
+from rdflib.graph import QuotedGraph, Graph
+from rdflib.namespace import XSD
+
+
+def uformat(s):
+    return s.replace("u'", "'")
+
+
+class TestURIRefRepr(unittest.TestCase):
+    """
+    see also test_literal.TestRepr
+    """
+
+    def testSubclassNameAppearsInRepr(self):
+        class MyURIRef(URIRef):
+            pass
+
+        x = MyURIRef("http://example.com/")
+        self.assertEqual(repr(x), uformat("MyURIRef(u'http://example.com/')"))
+
+    def testGracefulOrdering(self):
+        u = URIRef("cake")
+        g = Graph()
+        a = u > u
+        a = u > BNode()
+        a = u > QuotedGraph(g.store, u)
+        a = u > g
+
+
+class TestBNodeRepr(unittest.TestCase):
+    def testSubclassNameAppearsInRepr(self):
+        class MyBNode(BNode):
+            pass
+
+        x = MyBNode()
+        self.assertTrue(repr(x).startswith("MyBNode("))
+
+
+class TestLiteral(unittest.TestCase):
+    def test_base64_values(self):
+        b64msg = "cmRmbGliIGlzIGNvb2whIGFsc28gaGVyZSdzIHNvbWUgYmluYXJ5IAAR83UC"
+        decoded_b64msg = base64.b64decode(b64msg)
+        lit = Literal(b64msg, datatype=XSD.base64Binary)
+        self.assertEqual(lit.value, decoded_b64msg)
+        self.assertEqual(str(lit), b64msg)
+
+    def test_total_order(self):
+        types = {
+            XSD.dateTime: (
+                "2001-01-01T00:00:00",
+                "2001-01-01T00:00:00Z",
+                "2001-01-01T00:00:00-00:00",
+            ),
+            XSD.date: ("2001-01-01", "2001-01-01Z", "2001-01-01-00:00"),
+            XSD.time: ("00:00:00", "00:00:00Z", "00:00:00-00:00"),
+            XSD.gYear: ("2001", "2001Z", "2001-00:00"),  # interval
+            XSD.gYearMonth: ("2001-01", "2001-01Z", "2001-01-00:00"),
+        }
+        literals = [
+            Literal(literal, datatype=t)
+            for t, literals in types.items()
+            for literal in literals
+        ]
+        try:
+            sorted(literals)
+            orderable = True
+        except TypeError as e:
+            for l_ in literals:
+                print(repr(l_), repr(l_.value))
+            print(e)
+            orderable = False
+        self.assertTrue(orderable)
+
+        # also make sure that within a datetime things are still ordered:
+        l1 = [
+            Literal(l_, datatype=XSD.dateTime)
+            for l_ in [
+                "2001-01-01T00:00:00",
+                "2001-01-01T01:00:00",
+                "2001-01-01T01:00:01",
+                "2001-01-02T01:00:01",
+                "2001-01-01T00:00:00Z",
+                "2001-01-01T00:00:00-00:00",
+                "2001-01-01T01:00:00Z",
+                "2001-01-01T01:00:00-00:00",
+                "2001-01-01T00:00:00-01:30",
+                "2001-01-01T01:00:00-01:30",
+                "2001-01-02T01:00:01Z",
+                "2001-01-02T01:00:01-00:00",
+                "2001-01-02T01:00:01-01:30",
+            ]
+        ]
+        l2 = list(l1)
+        random.shuffle(l2)
+        self.assertListEqual(l1, sorted(l2))
+
+    def test_literal_add(self):
+        from decimal import Decimal
+
+        # compares Python decimals
+        def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
+            a = float(a)
+            b = float(b)
+            return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
+
+        cases = [
+            (1, Literal(1), Literal(1), Literal(2)),
+            (2, Literal(Decimal(1)), Literal(Decimal(1)), Literal(Decimal(2))),
+            (3, Literal(float(1)), Literal(float(1)), Literal(float(2))),
+            (4, Literal(1), Literal(1.1), Literal(2.1, datatype=XSD.decimal)),
+            (5, Literal(1.1), Literal(1.1), Literal(2.2)),
+            (
+                6,
+                Literal(Decimal(1)),
+                Literal(Decimal(1.1)),
+                Literal(Decimal(2.1), datatype=XSD.decimal),
+            ),
+            (7, Literal(Decimal(1.1)), Literal(Decimal(1.1)), Literal(Decimal(2.2))),
+            (8, Literal(float(1)), Literal(float(1.1)), Literal(float(2.1))),
+            (9, Literal(float(1.1)), Literal(float(1.1)), Literal(float(2.2))),
+            (10, Literal(-1), Literal(-1), Literal(-2)),
+            (12, Literal(Decimal(-1)), Literal(Decimal(-1)), Literal(Decimal(-2))),
+            (13, Literal(float(-1)), Literal(float(-1)), Literal(float(-2))),
+            (14, Literal(-1), Literal(-1.1), Literal(-2.1)),
+            (15, Literal(-1.1), Literal(-1.1), Literal(-2.2)),
+            (16, Literal(Decimal(-1)), Literal(Decimal(-1.1)), Literal(Decimal(-2.1))),
+            (
+                17,
+                Literal(Decimal(-1.1)),
+                Literal(Decimal(-1.1)),
+                Literal(Decimal(-2.2)),
+            ),
+            (18, Literal(float(-1)), Literal(float(-1.1)), Literal(float(-2.1))),
+            (19, Literal(float(-1.1)), Literal(float(-1.1)), Literal(float(-2.2))),
+            (20, Literal(1), Literal(1.0), Literal(2.0)),
+            (21, Literal(1.0), Literal(1.0), Literal(2.0)),
+            (22, Literal(Decimal(1)), Literal(Decimal(1.0)), Literal(Decimal(2.0))),
+            (23, Literal(Decimal(1.0)), Literal(Decimal(1.0)), Literal(Decimal(2.0))),
+            (24, Literal(float(1)), Literal(float(1.0)), Literal(float(2.0))),
+            (25, Literal(float(1.0)), Literal(float(1.0)), Literal(float(2.0))),
+            (
+                26,
+                Literal(1, datatype=XSD.integer),
+                Literal(1, datatype=XSD.integer),
+                Literal(2, datatype=XSD.integer),
+            ),
+            (
+                27,
+                Literal(1, datatype=XSD.integer),
+                Literal("1", datatype=XSD.integer),
+                Literal("2", datatype=XSD.integer),
+            ),
+            (
+                28,
+                Literal("1", datatype=XSD.integer),
+                Literal("1", datatype=XSD.integer),
+                Literal("2", datatype=XSD.integer),
+            ),
+            (
+                29,
+                Literal("1"),
+                Literal("1", datatype=XSD.integer),
+                Literal("11", datatype=XSD.string),
+            ),
+            (
+                30,
+                Literal(1),
+                Literal("1", datatype=XSD.integer),
+                Literal("2", datatype=XSD.integer),
+            ),
+            (
+                31,
+                Literal(Decimal(1), datatype=XSD.decimal),
+                Literal(Decimal(1), datatype=XSD.decimal),
+                Literal(Decimal(2), datatype=XSD.decimal),
+            ),
+            (
+                32,
+                Literal(Decimal(1)),
+                Literal(Decimal(1), datatype=XSD.decimal),
+                Literal(Decimal(2), datatype=XSD.decimal),
+            ),
+            (
+                33,
+                Literal(float(1)),
+                Literal(float(1), datatype=XSD.float),
+                Literal(float(2), datatype=XSD.float),
+            ),
+            (
+                34,
+                Literal(float(1), datatype=XSD.float),
+                Literal(float(1), datatype=XSD.float),
+                Literal(float(2), datatype=XSD.float),
+            ),
+            (35, Literal(1), 1, Literal(2)),
+            (36, Literal(1), 1.0, Literal(2, datatype=XSD.decimal)),
+            (37, Literal(1.0), 1, Literal(2, datatype=XSD.decimal)),
+            (38, Literal(1.0), 1.0, Literal(2.0)),
+            (39, Literal(Decimal(1.0)), Decimal(1), Literal(Decimal(2.0))),
+            (40, Literal(Decimal(1.0)), Decimal(1.0), Literal(Decimal(2.0))),
+            (41, Literal(float(1.0)), float(1), Literal(float(2.0))),
+            (42, Literal(float(1.0)), float(1.0), Literal(float(2.0))),
+            (
+                43,
+                Literal(1, datatype=XSD.integer),
+                "+1.1",
+                Literal("1+1.1", datatype=XSD.string),
+            ),
+            (
+                44,
+                Literal(1, datatype=XSD.integer),
+                Literal("+1.1", datatype=XSD.string),
+                Literal("1+1.1", datatype=XSD.string),
+            ),
+            (
+                45,
+                Literal(Decimal(1.0), datatype=XSD.integer),
+                Literal("1", datatype=XSD.string),
+                Literal("11", datatype=XSD.string),
+            ),
+            (
+                46,
+                Literal(1.1, datatype=XSD.integer),
+                Literal("1", datatype=XSD.string),
+                Literal("1.11", datatype=XSD.string),
+            ),
+            (
+                47,
+                Literal(1, datatype=XSD.integer),
+                None,
+                Literal(1, datatype=XSD.integer),
+            ),
+            (
+                48,
+                Literal("1", datatype=XSD.string),
+                None,
+                Literal("1", datatype=XSD.string),
+            ),
+        ]
+
+        for case in cases:
+            # see if the addition exactly matches the expected output
+            case_passed = (case[1] + case[2]) == (case[3])
+            # see if the addition almost matches the expected output, for decimal precision errors
+            if not case_passed:
+                try:
+                    case_passed = isclose((case[1] + case[2].value), case[3].value)
+                except:
+                    pass
+
+            if not case_passed:
+                print(case[1], case[2])
+                print("expected: " + case[3] + ", " + case[3].datatype)
+                print(
+                    "actual: "
+                    + (case[1] + case[2])
+                    + ", "
+                    + (case[1] + case[2]).datatype
+                )
+
+            self.assertTrue(case_passed, "Case " + str(case[0]) + " failed")
+
+
+class TestValidityFunctions(unittest.TestCase):
+    def test_is_valid_unicode(self):
+        testcase_list = (
+            (None, True),
+            (1, True),
+            (["foo"], True),
+            ({"foo": b"bar"}, True),
+            ("foo", True),
+            (b"foo\x00", True),
+            (b"foo\xf3\x02", False),
+        )
+        for val, expected in testcase_list:
+            self.assertEqual(_is_valid_unicode(val), expected)
diff --git a/test/test_literal/test_tokendatatype.py b/test/test_literal/test_tokendatatype.py
new file mode 100644
index 00000000..565f5972
--- /dev/null
+++ b/test/test_literal/test_tokendatatype.py
@@ -0,0 +1,40 @@
+import unittest
+
+from rdflib import Literal, XSD
+
+
+class TokenDatatypeTest(unittest.TestCase):
+    def test1(self):
+        lit2 = Literal("\two\nw", datatype=XSD.normalizedString)
+        lit = Literal("\two\nw", datatype=XSD.string)
+        self.assertFalse(str(lit) == str(lit2))
+
+    def test2(self):
+        lit = Literal("\tBeing a Doctor    Is\n\ta Full-Time Job\r", datatype=XSD.token)
+        st = Literal("Being a Doctor Is a Full-Time Job", datatype=XSD.string)
+        self.assertFalse(Literal.eq(st, lit))
+        self.assertEqual(str(lit), str(st))
+
+    def test3(self):
+        lit = Literal("       hey\nthere      ", datatype=XSD.token).n3()
+        self.assertTrue(lit == '"hey there"^^<http://www.w3.org/2001/XMLSchema#token>')
+
+    def test4(self):
+        lit = Literal("hey\nthere\ta tab\rcarriage return", datatype=XSD.token)
+        expected = Literal("""hey there a tab carriage return""", datatype=XSD.string)
+        self.assertEqual(str(lit), str(expected))
+
+    def test_whitespace_is_collapsed_and_trailing_whitespace_is_stripped(self):
+        lit = Literal(
+            "\n  hey -  white  space is collapsed for xsd:token       and preceding and trailing whitespace is stripped     ",
+            datatype=XSD.token,
+        )
+        expected = Literal(
+            "hey - white space is collapsed for xsd:token and preceding and trailing whitespace is stripped",
+            datatype=XSD.string,
+        )
+        self.assertEqual(str(lit), str(expected))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/test_literal/test_uriref_literal_comparison.py b/test/test_literal/test_uriref_literal_comparison.py
new file mode 100644
index 00000000..f6efa598
--- /dev/null
+++ b/test/test_literal/test_uriref_literal_comparison.py
@@ -0,0 +1,82 @@
+import unittest
+
+from rdflib.namespace import RDF
+
+from rdflib.term import URIRef
+from rdflib.term import BNode
+from rdflib.term import Literal
+
+from rdflib.graph import Graph
+
+from rdflib.plugins.parsers.rdfxml import CORE_SYNTAX_TERMS
+
+
+"""
+Ah... it's coming back to me...
+[6:32p] eikeon: think it's so transitivity holds...
+[6:32p] eikeon: if a==b and b==c then a should == c
+[6:32p] eikeon: "foo"==Literal("foo")
+[6:33p] eikeon: We don't want URIRef("foo")==Literal("foo")
+[6:33p] eikeon: But if we have URIRef("foo")=="foo" then it implies it.
+[6:33p] chimezie: yes, definately not the other RDFLib 'typed' RDF (and N3) terms
+[6:34p] eikeon: Why do you need URIRef("foo")=="foo" ?
+[6:34p] chimezie: i'm just wondering if a URI and a string with the same lexical value, are by definition 'different'
+[6:35p] eikeon: Think so, actually. Think of trying to serialize some triples.
+[6:36p] eikeon: If they are the same you'd serialize them the same, no?
+[6:36p] chimezie: I guess I was thinking of a 'string' in a native datatype sense, not in the RDF sense (where they would be distinctly different)
+[6:37p] eikeon: We should try and brain dump some of this...
+[6:37p] eikeon: it look a fairly long time to work out.
+[6:37p] eikeon: But think we finally landed in the right spot.
+[6:38p] eikeon: I know many of the backends break if URIRef("foo")==Literal("foo")
+[6:39p] eikeon: And if we want "foo"==Literal("foo") --- then we really can't have URIRef("foo") also == "foo"
+"""
+
+
+class IdentifierEquality(unittest.TestCase):
+    def setUp(self):
+        self.uriref = URIRef("http://example.org/")
+        self.bnode = BNode()
+        self.literal = Literal("http://example.org/")
+        self.python_literal = "http://example.org/"
+        self.python_literal_2 = "foo"
+
+    def testA(self):
+        self.assertEqual(self.uriref == self.literal, False)
+
+    def testB(self):
+        self.assertEqual(self.literal == self.uriref, False)
+
+    def testC(self):
+        self.assertEqual(self.uriref == self.python_literal, False)
+
+    def testD(self):
+        self.assertEqual(self.python_literal == self.uriref, False)
+
+    def testE(self):
+        self.assertEqual(self.literal == self.python_literal, False)
+
+    def testE2(self):
+        self.assertTrue(self.literal.eq(self.python_literal), True)
+
+    def testF(self):
+        self.assertEqual(self.python_literal == self.literal, False)
+
+    def testG(self):
+        self.assertEqual("foo" in CORE_SYNTAX_TERMS, False)
+
+    def testH(self):
+        self.assertEqual(
+            URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#RDF")
+            in CORE_SYNTAX_TERMS,
+            True,
+        )
+
+    def testI(self):
+        g = Graph()
+        g.add((self.uriref, RDF.value, self.literal))
+        g.add((self.uriref, RDF.value, self.uriref))
+        self.assertEqual(len(g), 2)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/test_literal/test_xmlliterals.py b/test/test_literal/test_xmlliterals.py
new file mode 100644
index 00000000..69e72705
--- /dev/null
+++ b/test/test_literal/test_xmlliterals.py
@@ -0,0 +1,152 @@
+import logging
+import textwrap
+from typing import Callable, Optional, Sequence
+
+import pytest
+import rdflib
+from rdflib import RDF, Literal
+
+import xml.dom.minidom
+
+
+def testPythonRoundtrip():
+    l1 = Literal("<msg>hello</msg>", datatype=RDF.XMLLiteral)
+    assert l1.value is not None, "xml must have been parsed"
+    assert l1.datatype == RDF.XMLLiteral, "literal must have right datatype"
+
+    l2 = Literal("<msg>good morning</msg>", datatype=RDF.XMLLiteral)
+    assert l2.value is not None, "xml must have been parsed"
+    assert not l1.eq(l2), "literals must NOT be equal"
+
+    l3 = Literal(l1.value)
+    assert l1.eq(l3), "roundtripped literals must be equal"
+    assert l3.datatype == RDF.XMLLiteral, "literal must have right datatype"
+
+    l4 = Literal("<msg >hello</msg>", datatype=RDF.XMLLiteral)
+    assert l1 == l4
+    assert l1.eq(l4)
+
+    rdflib.NORMALIZE_LITERALS = False
+    try:
+        l4 = Literal("<msg >hello</msg>", datatype=RDF.XMLLiteral)
+        assert l1 != l4
+        assert l1.eq(l4)
+    finally:
+        rdflib.NORMALIZE_LITERALS = True
+
+
+def testRDFXMLParse():
+    rdfxml = """\
+<rdf:RDF
+    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    xmlns:dc="http://purl.org/dc/elements/1.1/"
+>
+
+<rdf:Description rdf:about="http://example.org/">
+    <dc:description rdf:parseType="Literal">
+        <p xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"></p>
+    </dc:description>
+</rdf:Description>
+
+</rdf:RDF>"""
+
+    g = rdflib.Graph()
+    g.parse(data=rdfxml, format="xml")
+    l1 = list(g)[0][2]
+    assert l1.datatype == RDF.XMLLiteral
+
+
+def graph():
+    g = rdflib.Graph()
+    g.add(
+        (
+            rdflib.URIRef("http://example.org/a"),
+            rdflib.URIRef("http://example.org/p"),
+            rdflib.Literal("<msg>hei</hei>", datatype=RDF.XMLLiteral),
+        )
+    )
+    return g
+
+
+def roundtrip(fmt):
+    g1 = graph()
+    l1 = list(g1)[0][2]
+    g2 = rdflib.Graph()
+    g2.parse(data=g1.serialize(format=fmt), format=fmt)
+    l2 = list(g2)[0][2]
+    assert l1.eq(l2)
+
+
+def testRoundtrip():
+    roundtrip("xml")
+    roundtrip("n3")
+    roundtrip("nt")
+
+
+def testHTML():
+
+    l1 = Literal("<msg>hello</msg>", datatype=RDF.XMLLiteral)
+    assert l1.value is not None, "xml must have been parsed"
+    assert l1.datatype == RDF.XMLLiteral, "literal must have right datatype"
+
+    l2 = Literal("<msg>hello</msg>", datatype=RDF.HTML)
+    assert l2.value is not None, "xml must have been parsed"
+    assert l2.datatype == RDF.HTML, "literal must have right datatype"
+
+    assert l1 != l2
+    assert not l1.eq(l2)
+
+
+@pytest.mark.parametrize(
+    ("values", "equals"),
+    [
+        pytest.param(
+            [
+                lambda: Literal('<something />', datatype=RDF.XMLLiteral),
+                lambda: Literal('<something/>', datatype=RDF.XMLLiteral),
+            ],
+            True,
+        ),
+        pytest.param(
+            [
+                lambda: Literal(
+                    xml.dom.minidom.parseString(
+                        textwrap.dedent(
+                            """\
+                    <!DOCTYPE example>
+                    <something/>
+                    """
+                        )
+                    ),
+                    datatype=RDF.XMLLiteral,
+                ),
+                lambda: Literal(
+                    xml.dom.minidom.parseString(
+                        textwrap.dedent(
+                            """\
+                    <!DOCTYPE example>
+                    <something />
+                    """
+                        )
+                    ),
+                    datatype=RDF.XMLLiteral,
+                ),
+            ],
+            True,
+        ),
+    ],
+)
+def test_eq(values: Sequence[Callable[[], Literal]], equals: bool) -> None:
+    first_value = values[0]()
+    logging.debug("first_value = \n%r", first_value)
+    for value in values[1:]:
+        current_value = value()
+        logging.debug("current_value = \n%r", current_value)
+        if equals:
+            assert first_value.eq(
+                current_value
+            ), f"{current_value} must be equal to {first_value}"
+        else:
+            assert not first_value.eq(
+                current_value
+            ), f"{current_value} must be not be equal to {first_value}"