- Added support for the :class:`postgresql.JSONB` datatype when

using psycopg2 2.5.4 or greater, which features native conversion
of JSONB data so that SQLAlchemy's converters must be disabled;
additionally, the newly added psycopg2 extension
``extras.register_default_jsonb`` is used to establish a JSON
deserializer passed to the dialect via the ``json_deserializer``
argument.  Also repaired the Postgresql integration tests which
weren't actually round-tripping the JSONB type as opposed to the
JSON type.  Pull request courtesy Mateusz Susik.

- Repaired the use of the "array_oid" flag when registering the
HSTORE type with older psycopg2 versions < 2.4.3, which does not
support this flag, as well as use of the native json serializer
hook "register_default_json" with user-defined ``json_deserializer``
on psycopg2 versions < 2.5, which does not include native json.

5 files changed, 77 insertions, 17 deletions

diff --git a/README.unittests.rst b/README.unittests.rst
index 7ca229df6..5aeba51f6 100644
--- a/README.unittests.rst
+++ b/README.unittests.rst
@@ -198,6 +198,18 @@ expect them to be present will fail.
 
 Additional steps specific to individual databases are as follows::
 
+    POSTGRESQL: To enable unicode testing with JSONB, create the
+    database with UTF8 encoding::
+
+        postgres=# create database test with owner=scott encoding='utf8' template=template0;
+
+    To include tests for HSTORE, create the HSTORE type engine::
+
+        postgres=# \c test;
+        You are now connected to database "test" as user "postgresql".
+        test=# create extension hstore;
+        CREATE EXTENSION
+
     MYSQL: Default storage engine should be "MyISAM".   Tests that require
     "InnoDB" as the engine will specify this explicitly.
 
diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst
index 8d4c8d7f6..2af1cd35f 100644
--- a/doc/build/changelog/changelog_09.rst
+++ b/doc/build/changelog/changelog_09.rst
@@ -15,6 +15,29 @@
     :version: 0.9.9
 
     .. change::
+        :tags: bug, postgresql
+        :pullreq: github:145
+
+        Added support for the :class:`postgresql.JSONB` datatype when
+        using psycopg2 2.5.4 or greater, which features native conversion
+        of JSONB data so that SQLAlchemy's converters must be disabled;
+        additionally, the newly added psycopg2 extension
+        ``extras.register_default_jsonb`` is used to establish a JSON
+        deserializer passed to the dialect via the ``json_deserializer``
+        argument.  Also repaired the Postgresql integration tests which
+        weren't actually round-tripping the JSONB type as opposed to the
+        JSON type.  Pull request courtesy Mateusz Susik.
+
+    .. change::
+        :tags: bug, postgresql
+
+        Repaired the use of the "array_oid" flag when registering the
+        HSTORE type with older psycopg2 versions < 2.4.3, which does not
+        support this flag, as well as use of the native json serializer
+        hook "register_default_json" with user-defined ``json_deserializer``
+        on psycopg2 versions < 2.5, which does not include native json.
+
+    .. change::
         :tags: bug, schema
         :tickets: 3298, 1765
 
diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py
index 21011b68b..26e45fed2 100644
--- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py
+++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py
@@ -263,14 +263,17 @@ HSTORE type
 
 The ``psycopg2`` DBAPI includes an extension to natively handle marshalling of
 the HSTORE type.   The SQLAlchemy psycopg2 dialect will enable this extension
-by default when it is detected that the target database has the HSTORE
-type set up for use.   In other words, when the dialect makes the first
+by default when psycopg2 version 2.4 or greater is used, and
+it is detected that the target database has the HSTORE type set up for use.
+In other words, when the dialect makes the first
 connection, a sequence like the following is performed:
 
 1. Request the available HSTORE oids using
    ``psycopg2.extras.HstoreAdapter.get_oids()``.
    If this function returns a list of HSTORE identifiers, we then determine
    that the ``HSTORE`` extension is present.
+   This function is **skipped** if the version of psycopg2 installed is
+   less than version 2.4.
 
 2. If the ``use_native_hstore`` flag is at its default of ``True``, and
    we've detected that ``HSTORE`` oids are available, the
@@ -583,19 +586,22 @@ class PGDialect_psycopg2(PGDialect):
                 hstore_oids = self._hstore_oids(conn)
                 if hstore_oids is not None:
                     oid, array_oid = hstore_oids
+                    kw = {'oid': oid}
                     if util.py2k:
-                        extras.register_hstore(conn, oid=oid,
-                                               array_oid=array_oid,
-                                               unicode=True)
-                    else:
-                        extras.register_hstore(conn, oid=oid,
-                                               array_oid=array_oid)
+                        kw['unicode'] = True
+                    if self.psycopg2_version >= (2, 4, 3):
+                        kw['array_oid'] = array_oid
+                    extras.register_hstore(conn, **kw)
             fns.append(on_connect)
 
         if self.dbapi and self._json_deserializer:
             def on_connect(conn):
-                extras.register_default_json(
-                    conn, loads=self._json_deserializer)
+                if self._has_native_json:
+                    extras.register_default_json(
+                        conn, loads=self._json_deserializer)
+                if self._has_native_jsonb:
+                    extras.register_default_jsonb(
+                        conn, loads=self._json_deserializer)
             fns.append(on_connect)
 
         if fns:
diff --git a/test/dialect/postgresql/test_types.py b/test/dialect/postgresql/test_types.py
index 1f572c9a1..5e00fd605 100644
--- a/test/dialect/postgresql/test_types.py
+++ b/test/dialect/postgresql/test_types.py
@@ -1948,13 +1948,15 @@ class JSONRoundTripTest(fixtures.TablesTest):
     __only_on__ = ('postgresql >= 9.3',)
     __backend__ = True
 
+    test_type = JSON
+
     @classmethod
     def define_tables(cls, metadata):
         Table('data_table', metadata,
               Column('id', Integer, primary_key=True),
               Column('name', String(30), nullable=False),
-              Column('data', JSON),
-              Column('nulldata', JSON(none_as_null=True))
+              Column('data', cls.test_type),
+              Column('nulldata', cls.test_type(none_as_null=True))
               )
 
     def _fixture_data(self, engine):
@@ -2016,7 +2018,8 @@ class JSONRoundTripTest(fixtures.TablesTest):
         else:
             options = {}
 
-        if testing.against("postgresql+psycopg2"):
+        if testing.against("postgresql+psycopg2") and \
+                testing.db.dialect.psycopg2_version >= (2, 5):
             from psycopg2.extras import register_default_json
             engine = engines.testing_engine(options=options)
 
@@ -2037,7 +2040,7 @@ class JSONRoundTripTest(fixtures.TablesTest):
     def test_reflect(self):
         insp = inspect(testing.db)
         cols = insp.get_columns('data_table')
-        assert isinstance(cols[2]['type'], JSON)
+        assert isinstance(cols[2]['type'], self.test_type)
 
     @testing.only_on("postgresql+psycopg2")
     def test_insert_native(self):
@@ -2096,7 +2099,7 @@ class JSONRoundTripTest(fixtures.TablesTest):
                     "key": "value",
                     "x": "q"
                 },
-                JSON
+                self.test_type
             )
         ])
         eq_(
@@ -2172,7 +2175,7 @@ class JSONRoundTripTest(fixtures.TablesTest):
                     "key": "value",
                     "key2": {"k1": "v1", "k2": "v2"}
                 },
-                JSON
+                self.test_type
             )
         ])
         eq_(
@@ -2199,7 +2202,7 @@ class JSONRoundTripTest(fixtures.TablesTest):
                     util.u('réveillé'): util.u('réveillé'),
                     "data": {"k1": util.u('drôle')}
                 },
-                JSON
+                self.test_type
             )
         ])
         eq_(
@@ -2266,3 +2269,13 @@ class JSONBTest(JSONTest):
 
 class JSONBRoundTripTest(JSONRoundTripTest):
     __only_on__ = ('postgresql >= 9.4',)
+
+    test_type = JSONB
+
+    @testing.requires.postgresql_utf8_server_encoding
+    def test_unicode_round_trip_python(self):
+        super(JSONBRoundTripTest, self).test_unicode_round_trip_python()
+
+    @testing.requires.postgresql_utf8_server_encoding
+    def test_unicode_round_trip_native(self):
+        super(JSONBRoundTripTest, self).test_unicode_round_trip_native()
diff --git a/test/requirements.py b/test/requirements.py
index 89fc108b9..4d5869226 100644
--- a/test/requirements.py
+++ b/test/requirements.py
@@ -816,3 +816,9 @@ class DefaultRequirements(SuiteRequirements):
         return against(config, 'mysql') and \
                 config.db.dialect._detect_casing(config.db) == 0
 
+    @property
+    def postgresql_utf8_server_encoding(self):
+        return only_if(
+            lambda config: against(config, 'postgresql') and
+            config.db.scalar("show server_encoding").lower() == "utf8"
+        )