lazy loading for logbooks and flowdetails

You can now specify a "lazy" argument when asking a
persistence backend for a logbook for flowdetail.
This will return a logbook with no flowdetails in it,
or a flowdetail with no atomdetails in it, respectively.

Change-Id: Ie81a19a2107b44e2c35c2a92507bec03eee55c50

4 files changed, 65 insertions, 15 deletions

diff --git a/taskflow/persistence/backends/impl_sqlalchemy.py b/taskflow/persistence/backends/impl_sqlalchemy.py
index 4368b78..d4f343f 100644
--- a/taskflow/persistence/backends/impl_sqlalchemy.py
+++ b/taskflow/persistence/backends/impl_sqlalchemy.py
@@ -527,7 +527,7 @@ class Connection(base.Connection):
             raise exc.StorageFailure("Failed saving logbook"
                                      " '%s'" % book.uuid, e)
 
-    def get_logbook(self, book_uuid):
+    def get_logbook(self, book_uuid, lazy=False):
         try:
             logbooks = self._tables.logbooks
             with contextlib.closing(self._engine.connect()) as conn:
@@ -538,40 +538,42 @@ class Connection(base.Connection):
                     raise exc.NotFound("No logbook found with"
                                        " uuid '%s'" % book_uuid)
                 book = self._converter.convert_book(row)
-                self._converter.populate_book(conn, book)
+                if not lazy:
+                    self._converter.populate_book(conn, book)
                 return book
         except sa_exc.DBAPIError as e:
             raise exc.StorageFailure(
                 "Failed getting logbook '%s'" % book_uuid, e)
 
-    def get_logbooks(self):
+    def get_logbooks(self, lazy=False):
         gathered = []
         try:
             with contextlib.closing(self._engine.connect()) as conn:
                 q = sql.select([self._tables.logbooks])
                 for row in conn.execute(q):
                     book = self._converter.convert_book(row)
-                    self._converter.populate_book(conn, book)
+                    if not lazy:
+                        self._converter.populate_book(conn, book)
                     gathered.append(book)
         except sa_exc.DBAPIError as e:
             raise exc.StorageFailure("Failed getting logbooks", e)
         for book in gathered:
             yield book
 
-    def get_flows_for_book(self, book_uuid):
+    def get_flows_for_book(self, book_uuid, lazy=False):
         gathered = []
         try:
             with contextlib.closing(self._engine.connect()) as conn:
-                for row in self._converter.flow_query_iter(conn, book_uuid):
-                    flow_details = self._converter.populate_flow_detail(conn,
-                                                                        row)
-                    gathered.append(flow_details)
+                for fd in self._converter.flow_query_iter(conn, book_uuid):
+                    if not lazy:
+                        self._converter.populate_flow_detail(conn, fd)
+                    gathered.append(fd)
         except sa_exc.DBAPIError as e:
             raise exc.StorageFailure("Failed getting flow details", e)
         for flow_details in gathered:
             yield flow_details
 
-    def get_flow_details(self, fd_uuid):
+    def get_flow_details(self, fd_uuid, lazy=False):
         try:
             flowdetails = self._tables.flowdetails
             with self._engine.begin() as conn:
@@ -581,7 +583,10 @@ class Connection(base.Connection):
                 if not row:
                     raise exc.NotFound("No flow details found with uuid"
                                        " '%s'" % fd_uuid)
-                return self._converter.convert_flow_detail(row)
+                fd = self._converter.convert_flow_detail(row)
+                if not lazy:
+                    self._converter.populate_flow_detail(conn, fd)
+                return fd
         except sa_exc.SQLAlchemyError as e:
             raise exc.StorageFailure("Failed getting flow details with"
                                      " uuid '%s'" % fd_uuid, e)
@@ -601,5 +606,16 @@ class Connection(base.Connection):
             raise exc.StorageFailure("Failed getting atom details with"
                                      " uuid '%s'" % ad_uuid, e)
 
+    def get_atoms_for_flow(self, fd_uuid):
+        gathered = []
+        try:
+            with contextlib.closing(self._engine.connect()) as conn:
+                for ad in self._converter.atom_query_iter(conn, fd_uuid):
+                    gathered.append(ad)
+        except sa_exc.DBAPIError as e:
+            raise exc.StorageFailure("Failed getting atom details", e)
+        for atom_details in gathered:
+            yield atom_details
+
     def close(self):
         pass
diff --git a/taskflow/persistence/base.py b/taskflow/persistence/base.py
index a1f120d..aebb7dd 100644
--- a/taskflow/persistence/base.py
+++ b/taskflow/persistence/base.py
@@ -109,12 +109,12 @@ class Connection(object):
         pass
 
     @abc.abstractmethod
-    def get_logbook(self, book_uuid):
+    def get_logbook(self, book_uuid, lazy=False):
         """Fetches a logbook object matching the given uuid."""
         pass
 
     @abc.abstractmethod
-    def get_logbooks(self):
+    def get_logbooks(self, lazy=False):
         """Return an iterable of logbook objects."""
         pass
 
@@ -124,7 +124,7 @@ class Connection(object):
         pass
 
     @abc.abstractmethod
-    def get_flow_details(self, fd_uuid):
+    def get_flow_details(self, fd_uuid, lazy=False):
         """Fetches a flowdetails object matching the given uuid."""
         pass
 
@@ -133,6 +133,11 @@ class Connection(object):
         """Fetches a atomdetails object matching the given uuid."""
         pass
 
+    @abc.abstractmethod
+    def get_atoms_for_flow(self, fd_uuid):
+        """Return an iterable of atomdetails for a given flowdetails uuid."""
+        pass
+
 
 def _format_atom(atom_detail):
     return {
diff --git a/taskflow/persistence/path_based.py b/taskflow/persistence/path_based.py
index ea08025..d3e2bfe 100644
--- a/taskflow/persistence/path_based.py
+++ b/taskflow/persistence/path_based.py
@@ -149,7 +149,7 @@ class PathBasedConnection(base.Connection):
 
     def get_logbooks(self, lazy=False):
         for book_uuid in self._get_children(self.book_path):
-            yield self.get_logbook(book_uuid, lazy)
+            yield self.get_logbook(book_uuid, lazy=lazy)
 
     def get_logbook(self, book_uuid, lazy=False):
         book_path = self._join_path(self.book_path, book_uuid)
diff --git a/taskflow/tests/unit/persistence/base.py b/taskflow/tests/unit/persistence/base.py
index 184cf51..924e62b 100644
--- a/taskflow/tests/unit/persistence/base.py
+++ b/taskflow/tests/unit/persistence/base.py
@@ -130,6 +130,22 @@ class PersistenceTestMixin(object):
         fd2 = lb2.find(fd.uuid)
         self.assertEqual(fd2.meta.get('test'), 43)
 
+    def test_flow_detail_lazy_fetch(self):
+        lb_id = uuidutils.generate_uuid()
+        lb_name = 'lb-%s' % (lb_id)
+        lb = logbook.LogBook(name=lb_name, uuid=lb_id)
+        fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid())
+        td = logbook.TaskDetail("detail-1", uuid=uuidutils.generate_uuid())
+        td.version = '4.2'
+        fd.add(td)
+        lb.add(fd)
+        with contextlib.closing(self._get_connection()) as conn:
+            conn.save_logbook(lb)
+        with contextlib.closing(self._get_connection()) as conn:
+            fd2 = conn.get_flow_details(fd.uuid, lazy=True)
+            self.assertEqual(0, len(fd2))
+            self.assertEqual(1, len(fd))
+
     def test_task_detail_save(self):
         lb_id = uuidutils.generate_uuid()
         lb_name = 'lb-%s' % (lb_id)
@@ -239,6 +255,19 @@ class PersistenceTestMixin(object):
             self.assertEqual(1, len(lb))
             self.assertEqual(fd.name, lb2.find(fd.uuid).name)
 
+    def test_logbook_lazy_fetch(self):
+        lb_id = uuidutils.generate_uuid()
+        lb_name = 'lb-%s' % (lb_id)
+        lb = logbook.LogBook(name=lb_name, uuid=lb_id)
+        fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid())
+        lb.add(fd)
+        with contextlib.closing(self._get_connection()) as conn:
+            conn.save_logbook(lb)
+        with contextlib.closing(self._get_connection()) as conn:
+            lb2 = conn.get_logbook(lb_id, lazy=True)
+            self.assertEqual(0, len(lb2))
+            self.assertEqual(1, len(lb))
+
     def test_logbook_add_task_detail(self):
         lb_id = uuidutils.generate_uuid()
         lb_name = 'lb-%s' % (lb_id)