From e7f8cd3804245104657e41b548a431801f6c1cee Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Tue, 31 Jul 2018 06:09:00 -0400
Subject: Move sqlite into sqldata.py

---
 coverage/sqldata.py | 172 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 172 insertions(+)
 create mode 100644 coverage/sqldata.py

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
new file mode 100644
index 00000000..ee0798e3
--- /dev/null
+++ b/coverage/sqldata.py
@@ -0,0 +1,172 @@
+# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
+# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
+
+"""Sqlite coverage data."""
+
+import os
+import sqlite3
+
+from coverage.backward import iitems
+from coverage.misc import CoverageException, file_be_gone
+
+
+SCHEMA = """
+create table schema (
+    version integer
+);
+
+insert into schema (version) values (1);
+
+create table meta (
+    name text,
+    value text,
+    unique(name)
+);
+
+create table file (
+    id integer primary key,
+    path text,
+    tracer text,
+    unique(path)
+);
+
+create table line (
+    file_id integer,
+    lineno integer,
+    unique(file_id, lineno)
+);
+
+create table arc (
+    file_id integer,
+    fromno integer,
+    tono integer,
+    unique(file_id, fromno, tono)
+);
+"""
+
+def _create_db(filename, schema):
+    con = sqlite3.connect(filename)
+    with con:
+        for stmt in schema.split(';'):
+            con.execute(stmt.strip())
+    con.close()
+
+
+class CoverageDataSqlite(object):
+    def __init__(self, basename=None, warn=None, debug=None):
+        self.filename = os.path.abspath(basename or ".coverage")
+        self._warn = warn
+        self._debug = debug
+
+        self._file_map = {}
+        self._db = None
+        self._have_read = False
+
+    def _reset(self):
+        self._file_map = {}
+        if self._db is not None:
+            self._db.close()
+        self._db = None
+
+    def _connect(self):
+        if self._db is None:
+            if not os.path.exists(self.filename):
+                if self._debug and self._debug.should('dataio'):
+                    self._debug.write("Creating data file %r" % (self.filename,))
+                _create_db(self.filename, SCHEMA)
+            self._db = sqlite3.connect(self.filename)
+            for path, id in self._db.execute("select path, id from file"):
+                self._file_map[path] = id
+        return self._db
+
+    def _file_id(self, filename):
+        self._start_writing()
+        if filename not in self._file_map:
+            with self._connect() as con:
+                cur = con.cursor()
+                cur.execute("insert into file (path) values (?)", (filename,))
+                self._file_map[filename] = cur.lastrowid
+        return self._file_map[filename]
+
+    def add_lines(self, line_data):
+        """Add measured line data.
+
+        `line_data` is a dictionary mapping file names to dictionaries::
+
+            { filename: { lineno: None, ... }, ...}
+
+        """
+        self._start_writing()
+        with self._connect() as con:
+            for filename, linenos in iitems(line_data):
+                file_id = self._file_id(filename)
+                for lineno in linenos:
+                    con.execute(
+                        "insert or ignore into line (file_id, lineno) values (?, ?)",
+                        (file_id, lineno),
+                    )
+
+    def add_file_tracers(self, file_tracers):
+        """Add per-file plugin information.
+
+        `file_tracers` is { filename: plugin_name, ... }
+
+        """
+        self._start_writing()
+        with self._connect() as con:
+            for filename, tracer in iitems(file_tracers):
+                con.execute(
+                    "insert into file (path, tracer) values (?, ?) on duplicate key update",
+                    (filename, tracer),
+                )
+
+    def erase(self, parallel=False):
+        """Erase the data in this object.
+
+        If `parallel` is true, then also deletes data files created from the
+        basename by parallel-mode.
+
+        """
+        self._reset()
+        if self._debug and self._debug.should('dataio'):
+            self._debug.write("Erasing data file %r" % (self.filename,))
+        file_be_gone(self.filename)
+        if parallel:
+            data_dir, local = os.path.split(self.filename)
+            localdot = local + '.*'
+            pattern = os.path.join(os.path.abspath(data_dir), localdot)
+            for filename in glob.glob(pattern):
+                if self._debug and self._debug.should('dataio'):
+                    self._debug.write("Erasing parallel data file %r" % (filename,))
+                file_be_gone(filename)
+
+    def read(self):
+        self._have_read = True
+
+    def write(self, suffix=None):
+        """Write the collected coverage data to a file."""
+        pass
+
+    def _start_writing(self):
+        if not self._have_read:
+            self.erase()
+        self._have_read = True
+
+    def has_arcs(self):
+        return False    # TODO!
+
+    def measured_files(self):
+        """A list of all files that had been measured."""
+        self._connect()
+        return list(self._file_map)
+
+    def file_tracer(self, filename):
+        """Get the plugin name of the file tracer for a file.
+
+        Returns the name of the plugin that handles this file.  If the file was
+        measured, but didn't use a plugin, then "" is returned.  If the file
+        was not measured, then None is returned.
+
+        """
+        with self._connect() as con:
+            pass
-- 
cgit v1.2.1


From 0db04f814339e143422c211fdba351554fcc8f77 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Tue, 31 Jul 2018 06:19:05 -0400
Subject: Report works

---
 coverage/sqldata.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index ee0798e3..db2b0b17 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -141,6 +141,7 @@ class CoverageDataSqlite(object):
                 file_be_gone(filename)
 
     def read(self):
+        self._connect()     # TODO: doesn't look right
         self._have_read = True
 
     def write(self, suffix=None):
@@ -157,7 +158,6 @@ class CoverageDataSqlite(object):
 
     def measured_files(self):
         """A list of all files that had been measured."""
-        self._connect()
         return list(self._file_map)
 
     def file_tracer(self, filename):
@@ -168,5 +168,9 @@ class CoverageDataSqlite(object):
         was not measured, then None is returned.
 
         """
+        return ""    # TODO
+
+    def lines(self, filename):
         with self._connect() as con:
-            pass
+            file_id = self._file_id(filename)
+            return [lineno for lineno, in con.execute("select lineno from line where file_id = ?", (file_id,))]
-- 
cgit v1.2.1


From 48e7984b5c28a6d6a324bdb0fa62ae626be60f8a Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Tue, 31 Jul 2018 07:23:45 -0400
Subject: SQL debugging

---
 coverage/sqldata.py | 44 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 10 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index db2b0b17..05680043 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -44,12 +44,6 @@ create table arc (
 );
 """
 
-def _create_db(filename, schema):
-    con = sqlite3.connect(filename)
-    with con:
-        for stmt in schema.split(';'):
-            con.execute(stmt.strip())
-    con.close()
 
 
 class CoverageDataSqlite(object):
@@ -73,8 +67,14 @@ class CoverageDataSqlite(object):
             if not os.path.exists(self.filename):
                 if self._debug and self._debug.should('dataio'):
                     self._debug.write("Creating data file %r" % (self.filename,))
-                _create_db(self.filename, SCHEMA)
-            self._db = sqlite3.connect(self.filename)
+                self._db = Sqlite(self.filename, self._debug)
+                with self._db:
+                    for stmt in SCHEMA.split(';'):
+                        stmt = stmt.strip()
+                        if stmt:
+                            self._db.execute(stmt)
+            else:
+                self._db = Sqlite(self.filename, self._debug)
             for path, id in self._db.execute("select path, id from file"):
                 self._file_map[path] = id
         return self._db
@@ -83,8 +83,7 @@ class CoverageDataSqlite(object):
         self._start_writing()
         if filename not in self._file_map:
             with self._connect() as con:
-                cur = con.cursor()
-                cur.execute("insert into file (path) values (?)", (filename,))
+                cur = con.execute("insert into file (path) values (?)", (filename,))
                 self._file_map[filename] = cur.lastrowid
         return self._file_map[filename]
 
@@ -174,3 +173,28 @@ class CoverageDataSqlite(object):
         with self._connect() as con:
             file_id = self._file_id(filename)
             return [lineno for lineno, in con.execute("select lineno from line where file_id = ?", (file_id,))]
+
+
+class Sqlite(object):
+    def __init__(self, filename, debug):
+        self.debug = debug if debug.should('sql') else None
+        if self.debug:
+            self.debug.write("Connecting to {!r}".format(filename))
+        self.con = sqlite3.connect(filename)
+
+    def close(self):
+        self.con.close()
+
+    def __enter__(self):
+        self.con.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        return self.con.__exit__(exc_type, exc_value, traceback)
+
+    def execute(self, sql, parameters=()):
+        if self.debug:
+            tail = " with {!r}".format(parameters) if parameters else ""
+            self.debug.write("Executing {!r}{}".format(sql, tail))
+        cur = self.con.execute(sql, parameters)
+        return cur
-- 
cgit v1.2.1


From b457052020ec90fdba964ff8bd5abe6d92032e6b Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Fri, 3 Aug 2018 07:44:56 -0400
Subject: Make writing data faster

---
 coverage/sqldata.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 05680043..39d6268b 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -46,7 +46,7 @@ create table arc (
 
 
 
-class CoverageDataSqlite(object):
+class CoverageSqliteData(object):
     def __init__(self, basename=None, warn=None, debug=None):
         self.filename = os.path.abspath(basename or ".coverage")
         self._warn = warn
@@ -99,11 +99,11 @@ class CoverageDataSqlite(object):
         with self._connect() as con:
             for filename, linenos in iitems(line_data):
                 file_id = self._file_id(filename)
-                for lineno in linenos:
-                    con.execute(
-                        "insert or ignore into line (file_id, lineno) values (?, ?)",
-                        (file_id, lineno),
-                    )
+                data = [(file_id, lineno) for lineno in linenos]
+                con.executemany(
+                    "insert or ignore into line (file_id, lineno) values (?, ?)",
+                    data,
+                )
 
     def add_file_tracers(self, file_tracers):
         """Add per-file plugin information.
@@ -177,11 +177,16 @@ class CoverageDataSqlite(object):
 
 class Sqlite(object):
     def __init__(self, filename, debug):
-        self.debug = debug if debug.should('sql') else None
+        self.debug = debug if (debug and debug.should('sql')) else None
         if self.debug:
             self.debug.write("Connecting to {!r}".format(filename))
         self.con = sqlite3.connect(filename)
 
+        # This pragma makes writing faster. It disables rollbacks, but we never need them.
+        self.con.execute("pragma journal_mode=off")
+        # This pragma makes writing faster.
+        self.con.execute("pragma synchronous=off")
+
     def close(self):
         self.con.close()
 
@@ -196,5 +201,9 @@ class Sqlite(object):
         if self.debug:
             tail = " with {!r}".format(parameters) if parameters else ""
             self.debug.write("Executing {!r}{}".format(sql, tail))
-        cur = self.con.execute(sql, parameters)
-        return cur
+        return self.con.execute(sql, parameters)
+
+    def executemany(self, sql, data):
+        if self.debug:
+            self.debug.write("Executing many {!r}".format(sql))
+        return self.con.executemany(sql, data)
-- 
cgit v1.2.1


From 9244c218d282d6e7542487521d9ea0f17bc0c89d Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Fri, 3 Aug 2018 10:44:19 -0400
Subject: Use a Sqlite application_id to identify the file.

---
 coverage/sqldata.py | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 39d6268b..1e5fb570 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -44,6 +44,7 @@ create table arc (
 );
 """
 
+APP_ID = 0x0c07ea6e     # Kind of looks like "coverage"
 
 
 class CoverageSqliteData(object):
@@ -69,12 +70,18 @@ class CoverageSqliteData(object):
                     self._debug.write("Creating data file %r" % (self.filename,))
                 self._db = Sqlite(self.filename, self._debug)
                 with self._db:
+                    self._db.execute("pragma application_id = {}".format(APP_ID))
                     for stmt in SCHEMA.split(';'):
                         stmt = stmt.strip()
                         if stmt:
                             self._db.execute(stmt)
             else:
                 self._db = Sqlite(self.filename, self._debug)
+                with self._db:
+                    for app_id, in self._db.execute("pragma application_id"):
+                        app_id = int(app_id)
+                        if app_id != APP_ID:
+                            raise Exception("Doesn't look like a coverage data file: 0x{:08x} != 0x{:08x}".format(app_id, APP_ID))
             for path, id in self._db.execute("select path, id from file"):
                 self._file_map[path] = id
         return self._db
-- 
cgit v1.2.1


From 0c5af4612210fa08113ea93372f877ef13aaa007 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Fri, 3 Aug 2018 20:57:36 -0400
Subject: Can measure and report branches

---
 coverage/sqldata.py | 118 +++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 89 insertions(+), 29 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 1e5fb570..cddece31 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -18,9 +18,8 @@ create table schema (
 insert into schema (version) values (1);
 
 create table meta (
-    name text,
-    value text,
-    unique(name)
+    has_lines boolean,
+    has_arcs boolean
 );
 
 create table file (
@@ -44,8 +43,9 @@ create table arc (
 );
 """
 
-APP_ID = 0x0c07ea6e     # Kind of looks like "coverage"
-
+# >>> struct.unpack(">i", b"\xc0\x7e\x8a\x6e")      # "coverage", kind of.
+# (-1065448850,)
+APP_ID = -1065448850
 
 class CoverageSqliteData(object):
     def __init__(self, basename=None, warn=None, debug=None):
@@ -55,35 +55,58 @@ class CoverageSqliteData(object):
 
         self._file_map = {}
         self._db = None
+        # Are we in sync with the data file?
         self._have_read = False
 
+        self._has_lines = False
+        self._has_arcs = False
+
     def _reset(self):
         self._file_map = {}
         if self._db is not None:
             self._db.close()
         self._db = None
+        self._have_read = False
+
+    def _create_db(self):
+        if self._debug and self._debug.should('dataio'):
+            self._debug.write("Creating data file {!r}".format(self.filename))
+        self._db = Sqlite(self.filename, self._debug)
+        with self._db:
+            self._db.execute("pragma application_id = {}".format(APP_ID))
+            for stmt in SCHEMA.split(';'):
+                stmt = stmt.strip()
+                if stmt:
+                    self._db.execute(stmt)
+            self._db.execute(
+                "insert into meta (has_lines, has_arcs) values (?, ?)",
+                (self._has_lines, self._has_arcs)
+            )
+
+    def _open_db(self):
+        if self._debug and self._debug.should('dataio'):
+            self._debug.write("Opening data file {!r}".format(self.filename))
+        self._db = Sqlite(self.filename, self._debug)
+        with self._db:
+            for app_id, in self._db.execute("pragma application_id"):
+                app_id = int(app_id)
+                if app_id != APP_ID:
+                    raise CoverageException(
+                        "File {!r} doesn't look like a coverage data file: "
+                        "0x{:08x} != 0x{:08x}".format(self.filename, app_id, APP_ID)
+                    )
+        for row in self._db.execute("select has_lines, has_arcs from meta"):
+            self._has_lines, self._has_arcs = row
+
+        for path, id in self._db.execute("select path, id from file"):
+            self._file_map[path] = id
 
     def _connect(self):
         if self._db is None:
-            if not os.path.exists(self.filename):
-                if self._debug and self._debug.should('dataio'):
-                    self._debug.write("Creating data file %r" % (self.filename,))
-                self._db = Sqlite(self.filename, self._debug)
-                with self._db:
-                    self._db.execute("pragma application_id = {}".format(APP_ID))
-                    for stmt in SCHEMA.split(';'):
-                        stmt = stmt.strip()
-                        if stmt:
-                            self._db.execute(stmt)
+            if os.path.exists(self.filename):
+                self._open_db()
             else:
-                self._db = Sqlite(self.filename, self._debug)
-                with self._db:
-                    for app_id, in self._db.execute("pragma application_id"):
-                        app_id = int(app_id)
-                        if app_id != APP_ID:
-                            raise Exception("Doesn't look like a coverage data file: 0x{:08x} != 0x{:08x}".format(app_id, APP_ID))
-            for path, id in self._db.execute("select path, id from file"):
-                self._file_map[path] = id
+                self._create_db()
         return self._db
 
     def _file_id(self, filename):
@@ -103,6 +126,7 @@ class CoverageSqliteData(object):
 
         """
         self._start_writing()
+        self._choose_lines_or_arcs(lines=True)
         with self._connect() as con:
             for filename, linenos in iitems(line_data):
                 file_id = self._file_id(filename)
@@ -112,6 +136,36 @@ class CoverageSqliteData(object):
                     data,
                 )
 
+    def add_arcs(self, arc_data):
+        """Add measured arc data.
+
+        `arc_data` is a dictionary mapping file names to dictionaries::
+
+            { filename: { (l1,l2): None, ... }, ...}
+
+        """
+        self._start_writing()
+        self._choose_lines_or_arcs(arcs=True)
+        with self._connect() as con:
+            for filename, arcs in iitems(arc_data):
+                file_id = self._file_id(filename)
+                data = [(file_id, fromno, tono) for fromno, tono in arcs]
+                con.executemany(
+                    "insert or ignore into arc (file_id, fromno, tono) values (?, ?, ?)",
+                    data,
+                )
+
+    def _choose_lines_or_arcs(self, lines=False, arcs=False):
+        if lines and self._has_arcs:
+            raise CoverageException("Can't add lines to existing arc data")
+        if arcs and self._has_lines:
+            raise CoverageException("Can't add arcs to existing line data")
+        if not self._has_arcs and not self._has_lines:
+            self._has_lines = lines
+            self._has_arcs = arcs
+            with self._connect() as con:
+                con.execute("update meta set has_lines = ?, has_arcs = ?", (lines, arcs))
+
     def add_file_tracers(self, file_tracers):
         """Add per-file plugin information.
 
@@ -120,10 +174,11 @@ class CoverageSqliteData(object):
         """
         self._start_writing()
         with self._connect() as con:
-            for filename, tracer in iitems(file_tracers):
-                con.execute(
+            data = list(iitems(file_tracers))
+            if data:
+                con.executemany(
                     "insert into file (path, tracer) values (?, ?) on duplicate key update",
-                    (filename, tracer),
+                    data,
                 )
 
     def erase(self, parallel=False):
@@ -135,7 +190,7 @@ class CoverageSqliteData(object):
         """
         self._reset()
         if self._debug and self._debug.should('dataio'):
-            self._debug.write("Erasing data file %r" % (self.filename,))
+            self._debug.write("Erasing data file {!r}".format(self.filename))
         file_be_gone(self.filename)
         if parallel:
             data_dir, local = os.path.split(self.filename)
@@ -143,7 +198,7 @@ class CoverageSqliteData(object):
             pattern = os.path.join(os.path.abspath(data_dir), localdot)
             for filename in glob.glob(pattern):
                 if self._debug and self._debug.should('dataio'):
-                    self._debug.write("Erasing parallel data file %r" % (filename,))
+                    self._debug.write("Erasing parallel data file {!r}".format(filename))
                 file_be_gone(filename)
 
     def read(self):
@@ -160,7 +215,7 @@ class CoverageSqliteData(object):
         self._have_read = True
 
     def has_arcs(self):
-        return False    # TODO!
+        return self._has_arcs
 
     def measured_files(self):
         """A list of all files that had been measured."""
@@ -181,6 +236,11 @@ class CoverageSqliteData(object):
             file_id = self._file_id(filename)
             return [lineno for lineno, in con.execute("select lineno from line where file_id = ?", (file_id,))]
 
+    def arcs(self, filename):
+        with self._connect() as con:
+            file_id = self._file_id(filename)
+            return [pair for pair in con.execute("select fromno, tono from arc where file_id = ?", (file_id,))]
+
 
 class Sqlite(object):
     def __init__(self, filename, debug):
-- 
cgit v1.2.1


From f1561b04f58fdd04b86d2ed0dc858a1222752b02 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 4 Aug 2018 10:03:47 -0400
Subject: Improved debugging

---
 coverage/sqldata.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index cddece31..296e353e 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -7,6 +7,7 @@ import os
 import sqlite3
 
 from coverage.backward import iitems
+from coverage.debug import SimpleRepr
 from coverage.misc import CoverageException, file_be_gone
 
 
@@ -47,7 +48,7 @@ create table arc (
 # (-1065448850,)
 APP_ID = -1065448850
 
-class CoverageSqliteData(object):
+class CoverageSqliteData(SimpleRepr):
     def __init__(self, basename=None, warn=None, debug=None):
         self.filename = os.path.abspath(basename or ".coverage")
         self._warn = warn
@@ -125,6 +126,10 @@ class CoverageSqliteData(object):
             { filename: { lineno: None, ... }, ...}
 
         """
+        if self._debug and self._debug.should('dataop'):
+            self._debug.write("Adding lines: %d files, %d lines total" % (
+                len(line_data), sum(len(lines) for lines in line_data.values())
+            ))
         self._start_writing()
         self._choose_lines_or_arcs(lines=True)
         with self._connect() as con:
@@ -144,6 +149,10 @@ class CoverageSqliteData(object):
             { filename: { (l1,l2): None, ... }, ...}
 
         """
+        if self._debug and self._debug.should('dataop'):
+            self._debug.write("Adding arcs: %d files, %d arcs total" % (
+                len(arc_data), sum(len(arcs) for arcs in arc_data.values())
+            ))
         self._start_writing()
         self._choose_lines_or_arcs(arcs=True)
         with self._connect() as con:
@@ -242,7 +251,7 @@ class CoverageSqliteData(object):
             return [pair for pair in con.execute("select fromno, tono from arc where file_id = ?", (file_id,))]
 
 
-class Sqlite(object):
+class Sqlite(SimpleRepr):
     def __init__(self, filename, debug):
         self.debug = debug if (debug and debug.should('sql')) else None
         if self.debug:
@@ -250,9 +259,9 @@ class Sqlite(object):
         self.con = sqlite3.connect(filename)
 
         # This pragma makes writing faster. It disables rollbacks, but we never need them.
-        self.con.execute("pragma journal_mode=off")
+        self.execute("pragma journal_mode=off")
         # This pragma makes writing faster.
-        self.con.execute("pragma synchronous=off")
+        self.execute("pragma synchronous=off")
 
     def close(self):
         self.con.close()
@@ -272,5 +281,5 @@ class Sqlite(object):
 
     def executemany(self, sql, data):
         if self.debug:
-            self.debug.write("Executing many {!r}".format(sql))
+            self.debug.write("Executing many {!r} with {} rows".format(sql, len(data)))
         return self.con.executemany(sql, data)
-- 
cgit v1.2.1


From 02b2bd8afe3cd171e4bd454ccf244f788ccded3c Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Thu, 9 Aug 2018 22:22:45 -0400
Subject: Forgot an import

---
 coverage/sqldata.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 296e353e..80188fca 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -3,6 +3,7 @@
 
 """Sqlite coverage data."""
 
+import glob
 import os
 import sqlite3
 
-- 
cgit v1.2.1


From 90bb6a77e02cbac6a23723b5907d5f59d1db1b82 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Fri, 10 Aug 2018 16:15:00 -0400
Subject: Move a common method outside the data classes

---
 coverage/sqldata.py | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 80188fca..25a6d62d 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -242,6 +242,13 @@ class CoverageSqliteData(SimpleRepr):
         return ""    # TODO
 
     def lines(self, filename):
+        if self.has_arcs():
+            arcs = self.arcs(filename)
+            if arcs is not None:
+                import itertools
+                all_lines = itertools.chain.from_iterable(arcs)
+                return list(set(l for l in all_lines if l > 0))
+
         with self._connect() as con:
             file_id = self._file_id(filename)
             return [lineno for lineno, in con.execute("select lineno from line where file_id = ?", (file_id,))]
-- 
cgit v1.2.1


From 420c1b10ddeed1da66a2ffb81d7ac2af32939be5 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 11 Aug 2018 07:40:05 -0400
Subject: Implement more

---
 coverage/sqldata.py | 71 ++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 57 insertions(+), 14 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 25a6d62d..9d25d92c 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -6,6 +6,7 @@
 import glob
 import os
 import sqlite3
+import struct
 
 from coverage.backward import iitems
 from coverage.debug import SimpleRepr
@@ -45,9 +46,13 @@ create table arc (
 );
 """
 
-# >>> struct.unpack(">i", b"\xc0\x7e\x8a\x6e")      # "coverage", kind of.
-# (-1065448850,)
-APP_ID = -1065448850
+APP_ID = 0xc07e8a6e         # "coverage", kind of.
+
+def unsigned_to_signed(val):
+    return struct.unpack('>i', struct.pack('>I', val))[0]
+
+def signed_to_unsigned(val):
+    return struct.unpack('>I', struct.pack('>i', val))[0]
 
 class CoverageSqliteData(SimpleRepr):
     def __init__(self, basename=None, warn=None, debug=None):
@@ -75,7 +80,7 @@ class CoverageSqliteData(SimpleRepr):
             self._debug.write("Creating data file {!r}".format(self.filename))
         self._db = Sqlite(self.filename, self._debug)
         with self._db:
-            self._db.execute("pragma application_id = {}".format(APP_ID))
+            self._db.execute("pragma application_id = {}".format(unsigned_to_signed(APP_ID)))
             for stmt in SCHEMA.split(';'):
                 stmt = stmt.strip()
                 if stmt:
@@ -91,10 +96,10 @@ class CoverageSqliteData(SimpleRepr):
         self._db = Sqlite(self.filename, self._debug)
         with self._db:
             for app_id, in self._db.execute("pragma application_id"):
-                app_id = int(app_id)
+                app_id = signed_to_unsigned(int(app_id))
                 if app_id != APP_ID:
                     raise CoverageException(
-                        "File {!r} doesn't look like a coverage data file: "
+                        "Couldn't use {!r}: wrong application_id: "
                         "0x{:08x} != 0x{:08x}".format(self.filename, app_id, APP_ID)
                     )
         for row in self._db.execute("select has_lines, has_arcs from meta"):
@@ -111,6 +116,19 @@ class CoverageSqliteData(SimpleRepr):
                 self._create_db()
         return self._db
 
+    def __nonzero__(self):
+        try:
+            with self._connect() as con:
+                if self.has_arcs():
+                    rows = con.execute("select * from arc limit 1")
+                else:
+                    rows = con.execute("select * from line limit 1")
+                return bool(list(rows))
+        except CoverageException:
+            return False
+
+    __bool__ = __nonzero__
+
     def _file_id(self, filename):
         self._start_writing()
         if filename not in self._file_map:
@@ -184,13 +202,28 @@ class CoverageSqliteData(SimpleRepr):
         """
         self._start_writing()
         with self._connect() as con:
-            data = list(iitems(file_tracers))
-            if data:
-                con.executemany(
-                    "insert into file (path, tracer) values (?, ?) on duplicate key update",
-                    data,
+            for filename, plugin_name in iitems(file_tracers):
+                con.execute(
+                    "update file set tracer = ? where path = ?",
+                    (plugin_name, filename)
                 )
 
+    def touch_file(self, filename, plugin_name=""):
+        """Ensure that `filename` appears in the data, empty if needed.
+
+        `plugin_name` is the name of the plugin resposible for this file. It is used
+        to associate the right filereporter, etc.
+        """
+        if self._debug and self._debug.should('dataop'):
+            self._debug.write("Touching %r" % (filename,))
+        if not self._has_arcs and not self._has_lines:
+            raise CoverageException("Can't touch files in an empty CoverageSqliteData")
+
+        file_id = self._file_id(filename)
+        if plugin_name:
+            # Set the tracer for this file
+            self.add_file_tracers({filename: plugin_name})
+
     def erase(self, parallel=False):
         """Erase the data in this object.
 
@@ -239,7 +272,10 @@ class CoverageSqliteData(SimpleRepr):
         was not measured, then None is returned.
 
         """
-        return ""    # TODO
+        with self._connect() as con:
+            for tracer, in con.execute("select tracer from file where path = ?", (filename,)):
+                return tracer or ""
+        return None
 
     def lines(self, filename):
         if self.has_arcs():
@@ -258,13 +294,17 @@ class CoverageSqliteData(SimpleRepr):
             file_id = self._file_id(filename)
             return [pair for pair in con.execute("select fromno, tono from arc where file_id = ?", (file_id,))]
 
+    def run_infos(self):
+        return []   # TODO
+
 
 class Sqlite(SimpleRepr):
     def __init__(self, filename, debug):
         self.debug = debug if (debug and debug.should('sql')) else None
         if self.debug:
             self.debug.write("Connecting to {!r}".format(filename))
-        self.con = sqlite3.connect(filename)
+        self.filename = filename
+        self.con = sqlite3.connect(self.filename)
 
         # This pragma makes writing faster. It disables rollbacks, but we never need them.
         self.execute("pragma journal_mode=off")
@@ -285,7 +325,10 @@ class Sqlite(SimpleRepr):
         if self.debug:
             tail = " with {!r}".format(parameters) if parameters else ""
             self.debug.write("Executing {!r}{}".format(sql, tail))
-        return self.con.execute(sql, parameters)
+        try:
+            return self.con.execute(sql, parameters)
+        except sqlite3.Error as exc:
+            raise CoverageException("Couldn't use data file {!r}: {}".format(self.filename, exc))
 
     def executemany(self, sql, data):
         if self.debug:
-- 
cgit v1.2.1


From e70a13b69912591a81dfded0261fa3f847232ba1 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 11 Aug 2018 08:46:56 -0400
Subject: Don't add data by asking about data

---
 coverage/sqldata.py | 37 ++++++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 11 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 9d25d92c..e84c82fc 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -114,6 +114,7 @@ class CoverageSqliteData(SimpleRepr):
                 self._open_db()
             else:
                 self._create_db()
+            self._have_read = True
         return self._db
 
     def __nonzero__(self):
@@ -129,13 +130,19 @@ class CoverageSqliteData(SimpleRepr):
 
     __bool__ = __nonzero__
 
-    def _file_id(self, filename):
-        self._start_writing()
+    def _file_id(self, filename, add=False):
+        """Get the file id for `filename`.
+
+        If filename is not in the database yet, add if it `add` is True.
+        If `add` is not True, return None.
+        """
         if filename not in self._file_map:
-            with self._connect() as con:
-                cur = con.execute("insert into file (path) values (?)", (filename,))
-                self._file_map[filename] = cur.lastrowid
-        return self._file_map[filename]
+            if add:
+                self._start_writing()
+                with self._connect() as con:
+                    cur = con.execute("insert into file (path) values (?)", (filename,))
+                    self._file_map[filename] = cur.lastrowid
+        return self._file_map.get(filename)
 
     def add_lines(self, line_data):
         """Add measured line data.
@@ -153,7 +160,7 @@ class CoverageSqliteData(SimpleRepr):
         self._choose_lines_or_arcs(lines=True)
         with self._connect() as con:
             for filename, linenos in iitems(line_data):
-                file_id = self._file_id(filename)
+                file_id = self._file_id(filename, add=True)
                 data = [(file_id, lineno) for lineno in linenos]
                 con.executemany(
                     "insert or ignore into line (file_id, lineno) values (?, ?)",
@@ -176,7 +183,7 @@ class CoverageSqliteData(SimpleRepr):
         self._choose_lines_or_arcs(arcs=True)
         with self._connect() as con:
             for filename, arcs in iitems(arc_data):
-                file_id = self._file_id(filename)
+                file_id = self._file_id(filename, add=True)
                 data = [(file_id, fromno, tono) for fromno, tono in arcs]
                 con.executemany(
                     "insert or ignore into arc (file_id, fromno, tono) values (?, ?, ?)",
@@ -219,7 +226,7 @@ class CoverageSqliteData(SimpleRepr):
         if not self._has_arcs and not self._has_lines:
             raise CoverageException("Can't touch files in an empty CoverageSqliteData")
 
-        file_id = self._file_id(filename)
+        file_id = self._file_id(filename, add=True)
         if plugin_name:
             # Set the tracer for this file
             self.add_file_tracers({filename: plugin_name})
@@ -287,12 +294,20 @@ class CoverageSqliteData(SimpleRepr):
 
         with self._connect() as con:
             file_id = self._file_id(filename)
-            return [lineno for lineno, in con.execute("select lineno from line where file_id = ?", (file_id,))]
+            if file_id is None:
+                return None
+            else:
+                linenos = con.execute("select lineno from line where file_id = ?", (file_id,))
+                return [lineno for lineno, in linenos]
 
     def arcs(self, filename):
         with self._connect() as con:
             file_id = self._file_id(filename)
-            return [pair for pair in con.execute("select fromno, tono from arc where file_id = ?", (file_id,))]
+            if file_id is None:
+                return None
+            else:
+                arcs = con.execute("select fromno, tono from arc where file_id = ?", (file_id,))
+                return [pair for pair in arcs]
 
     def run_infos(self):
         return []   # TODO
-- 
cgit v1.2.1


From 0812699cab9226a342dd9b914d3e14ceccdf7691 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 11 Aug 2018 16:42:44 -0400
Subject: A little better

---
 coverage/sqldata.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index e84c82fc..ce78c63b 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -226,7 +226,7 @@ class CoverageSqliteData(SimpleRepr):
         if not self._has_arcs and not self._has_lines:
             raise CoverageException("Can't touch files in an empty CoverageSqliteData")
 
-        file_id = self._file_id(filename, add=True)
+        self._file_id(filename, add=True)
         if plugin_name:
             # Set the tracer for this file
             self.add_file_tracers({filename: plugin_name})
-- 
cgit v1.2.1


From c362e44f3ebeda9929c3537df96eecfa218d83c2 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 11 Aug 2018 19:08:19 -0400
Subject: Error handling in add_file_tracers

---
 coverage/sqldata.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index ce78c63b..dbeced84 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -3,6 +3,10 @@
 
 """Sqlite coverage data."""
 
+# TODO: check the schema
+# TODO: factor out dataop debugging to a wrapper class?
+# TODO: make sure all dataop debugging is in place somehow
+
 import glob
 import os
 import sqlite3
@@ -210,6 +214,21 @@ class CoverageSqliteData(SimpleRepr):
         self._start_writing()
         with self._connect() as con:
             for filename, plugin_name in iitems(file_tracers):
+                file_id = self._file_id(filename)
+                if file_id is None:
+                    raise CoverageException(
+                        "Can't add file tracer data for unmeasured file '%s'" % (filename,)
+                    )
+
+                cur = con.execute("select tracer from file where id = ?", (file_id,))
+                [existing_plugin] = cur.fetchone()
+                if existing_plugin is not None and existing_plugin != plugin_name:
+                    raise CoverageException(
+                        "Conflicting file tracer name for '%s': %r vs %r" % (
+                            filename, existing_plugin, plugin_name,
+                        )
+                    )
+
                 con.execute(
                     "update file set tracer = ? where path = ?",
                     (plugin_name, filename)
-- 
cgit v1.2.1


From 3d6b9819921f5be15168631452053c63424fa8d8 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 11 Aug 2018 19:57:05 -0400
Subject: Sqlite update() method

---
 coverage/sqldata.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index dbeced84..996a2ae8 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -14,6 +14,7 @@ import struct
 
 from coverage.backward import iitems
 from coverage.debug import SimpleRepr
+from coverage.files import PathAliases
 from coverage.misc import CoverageException, file_be_gone
 
 
@@ -250,6 +251,37 @@ class CoverageSqliteData(SimpleRepr):
             # Set the tracer for this file
             self.add_file_tracers({filename: plugin_name})
 
+    def update(self, other_data, aliases=None):
+        if self._has_lines and other_data._has_arcs:
+            raise CoverageException("Can't combine arc data with line data")
+        if self._has_arcs and other_data._has_lines:
+            raise CoverageException("Can't combine line data with arc data")
+
+        aliases = aliases or PathAliases()
+
+        # lines
+        if other_data._has_lines:
+            for filename in other_data.measured_files():
+                lines = set(other_data.lines(filename))
+                filename = aliases.map(filename)
+                lines.update(self.lines(filename) or ())
+                self.add_lines({filename: lines})
+
+        # arcs
+        if other_data._has_arcs:
+            for filename in other_data.measured_files():
+                arcs = set(other_data.arcs(filename))
+                filename = aliases.map(filename)
+                arcs.update(self.arcs(filename) or ())
+                self.add_arcs({filename: arcs})
+
+        # file_tracers
+        for filename in other_data.measured_files():
+            other_plugin = other_data.file_tracer(filename)
+            filename = aliases.map(filename)
+            self.add_file_tracers({filename: other_plugin})
+
+
     def erase(self, parallel=False):
         """Erase the data in this object.
 
-- 
cgit v1.2.1


From da37af9a65b144ce6b1f26430bcbc9786e055f8b Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Tue, 14 Aug 2018 08:37:09 -0400
Subject: Move the suffix parameter, but no implementation yet

---
 coverage/sqldata.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 996a2ae8..5ae5e64d 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -60,7 +60,7 @@ def signed_to_unsigned(val):
     return struct.unpack('>I', struct.pack('>i', val))[0]
 
 class CoverageSqliteData(SimpleRepr):
-    def __init__(self, basename=None, warn=None, debug=None):
+    def __init__(self, basename=None, suffix=None, warn=None, debug=None):
         self.filename = os.path.abspath(basename or ".coverage")
         self._warn = warn
         self._debug = debug
@@ -306,7 +306,7 @@ class CoverageSqliteData(SimpleRepr):
         self._connect()     # TODO: doesn't look right
         self._have_read = True
 
-    def write(self, suffix=None):
+    def write(self):
         """Write the collected coverage data to a file."""
         pass
 
-- 
cgit v1.2.1


From 067d0a60384b5f12cfee622381cfb5905efb8e13 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Tue, 14 Aug 2018 20:38:39 -0400
Subject: Use pid-random suffixes for SQL files

---
 coverage/sqldata.py | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 5ae5e64d..f36a9385 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -13,6 +13,7 @@ import sqlite3
 import struct
 
 from coverage.backward import iitems
+from coverage.data import filename_suffix
 from coverage.debug import SimpleRepr
 from coverage.files import PathAliases
 from coverage.misc import CoverageException, file_be_gone
@@ -62,6 +63,9 @@ def signed_to_unsigned(val):
 class CoverageSqliteData(SimpleRepr):
     def __init__(self, basename=None, suffix=None, warn=None, debug=None):
         self.filename = os.path.abspath(basename or ".coverage")
+        suffix = filename_suffix(suffix)
+        if suffix:
+            self.filename += "." + suffix
         self._warn = warn
         self._debug = debug
 
-- 
cgit v1.2.1


From 9b13a1a7d44d991c4c5dd51d5624f5abe84b77f8 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Tue, 14 Aug 2018 20:39:31 -0400
Subject: Skip some tests for SQL for now

---
 coverage/sqldata.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index f36a9385..c79ad175 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -3,9 +3,12 @@
 
 """Sqlite coverage data."""
 
+# TODO: get rid of skip_unless_data_storage_is_json
 # TODO: check the schema
 # TODO: factor out dataop debugging to a wrapper class?
 # TODO: make sure all dataop debugging is in place somehow
+# TODO: should writes be batched?
+# TODO: settle the os.fork question
 
 import glob
 import os
-- 
cgit v1.2.1


From 19ec83bde56b6dfecef4ddae275376fdb4262e3a Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Tue, 14 Aug 2018 20:39:57 -0400
Subject: Be flexible, and accept either json-sourced or sql-source error
 messages in some tests

---
 coverage/sqldata.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index c79ad175..3abc3af3 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -4,11 +4,14 @@
 """Sqlite coverage data."""
 
 # TODO: get rid of skip_unless_data_storage_is_json
+# TODO: get rid of "JSON message" and "SQL message" in the tests
 # TODO: check the schema
+# TODO: get rid of the application_id?
 # TODO: factor out dataop debugging to a wrapper class?
 # TODO: make sure all dataop debugging is in place somehow
 # TODO: should writes be batched?
 # TODO: settle the os.fork question
+# TODO: run_info
 
 import glob
 import os
@@ -323,7 +326,7 @@ class CoverageSqliteData(SimpleRepr):
         self._have_read = True
 
     def has_arcs(self):
-        return self._has_arcs
+        return bool(self._has_arcs)
 
     def measured_files(self):
         """A list of all files that had been measured."""
-- 
cgit v1.2.1


From f30f591be04a88dac2080505c241457d45f0314b Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 18 Aug 2018 11:12:40 -0400
Subject: Get file_tracer semantics right, whew

---
 coverage/sqldata.py | 79 ++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 51 insertions(+), 28 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 3abc3af3..01082a9b 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -40,7 +40,6 @@ create table meta (
 create table file (
     id integer primary key,
     path text,
-    tracer text,
     unique(path)
 );
 
@@ -56,6 +55,11 @@ create table arc (
     tono integer,
     unique(file_id, fromno, tono)
 );
+
+create table tracer (
+    file_id integer primary key,
+    tracer text
+);
 """
 
 APP_ID = 0xc07e8a6e         # "coverage", kind of.
@@ -78,7 +82,7 @@ class CoverageSqliteData(SimpleRepr):
         self._file_map = {}
         self._db = None
         # Are we in sync with the data file?
-        self._have_read = False
+        self._have_used = False
 
         self._has_lines = False
         self._has_arcs = False
@@ -88,7 +92,7 @@ class CoverageSqliteData(SimpleRepr):
         if self._db is not None:
             self._db.close()
         self._db = None
-        self._have_read = False
+        self._have_used = False
 
     def _create_db(self):
         if self._debug and self._debug.should('dataio'):
@@ -129,7 +133,6 @@ class CoverageSqliteData(SimpleRepr):
                 self._open_db()
             else:
                 self._create_db()
-            self._have_read = True
         return self._db
 
     def __nonzero__(self):
@@ -153,7 +156,6 @@ class CoverageSqliteData(SimpleRepr):
         """
         if filename not in self._file_map:
             if add:
-                self._start_writing()
                 with self._connect() as con:
                     cur = con.execute("insert into file (path) values (?)", (filename,))
                     self._file_map[filename] = cur.lastrowid
@@ -171,7 +173,7 @@ class CoverageSqliteData(SimpleRepr):
             self._debug.write("Adding lines: %d files, %d lines total" % (
                 len(line_data), sum(len(lines) for lines in line_data.values())
             ))
-        self._start_writing()
+        self._start_using()
         self._choose_lines_or_arcs(lines=True)
         with self._connect() as con:
             for filename, linenos in iitems(line_data):
@@ -194,7 +196,7 @@ class CoverageSqliteData(SimpleRepr):
             self._debug.write("Adding arcs: %d files, %d arcs total" % (
                 len(arc_data), sum(len(arcs) for arcs in arc_data.values())
             ))
-        self._start_writing()
+        self._start_using()
         self._choose_lines_or_arcs(arcs=True)
         with self._connect() as con:
             for filename, arcs in iitems(arc_data):
@@ -222,7 +224,7 @@ class CoverageSqliteData(SimpleRepr):
         `file_tracers` is { filename: plugin_name, ... }
 
         """
-        self._start_writing()
+        self._start_using()
         with self._connect() as con:
             for filename, plugin_name in iitems(file_tracers):
                 file_id = self._file_id(filename)
@@ -231,26 +233,27 @@ class CoverageSqliteData(SimpleRepr):
                         "Can't add file tracer data for unmeasured file '%s'" % (filename,)
                     )
 
-                cur = con.execute("select tracer from file where id = ?", (file_id,))
-                [existing_plugin] = cur.fetchone()
-                if existing_plugin is not None and existing_plugin != plugin_name:
-                    raise CoverageException(
-                        "Conflicting file tracer name for '%s': %r vs %r" % (
-                            filename, existing_plugin, plugin_name,
+                existing_plugin = self.file_tracer(filename)
+                if existing_plugin:
+                    if existing_plugin != plugin_name:
+                        raise CoverageException(
+                            "Conflicting file tracer name for '%s': %r vs %r" % (
+                                filename, existing_plugin, plugin_name,
+                            )
                         )
+                elif plugin_name:
+                    con.execute(
+                        "insert into tracer (file_id, tracer) values (?, ?)",
+                        (file_id, plugin_name)
                     )
 
-                con.execute(
-                    "update file set tracer = ? where path = ?",
-                    (plugin_name, filename)
-                )
-
     def touch_file(self, filename, plugin_name=""):
         """Ensure that `filename` appears in the data, empty if needed.
 
         `plugin_name` is the name of the plugin resposible for this file. It is used
         to associate the right filereporter, etc.
         """
+        self._start_using()
         if self._debug and self._debug.should('dataop'):
             self._debug.write("Touching %r" % (filename,))
         if not self._has_arcs and not self._has_lines:
@@ -269,6 +272,9 @@ class CoverageSqliteData(SimpleRepr):
 
         aliases = aliases or PathAliases()
 
+        # See what we had already measured, for accurate conflict reporting.
+        this_measured = set(self.measured_files())
+
         # lines
         if other_data._has_lines:
             for filename in other_data.measured_files():
@@ -289,8 +295,18 @@ class CoverageSqliteData(SimpleRepr):
         for filename in other_data.measured_files():
             other_plugin = other_data.file_tracer(filename)
             filename = aliases.map(filename)
-            self.add_file_tracers({filename: other_plugin})
-
+            if filename in this_measured:
+                this_plugin = self.file_tracer(filename)
+            else:
+                this_plugin = None
+            if this_plugin is None:
+                self.add_file_tracers({filename: other_plugin})
+            elif this_plugin != other_plugin:
+                raise CoverageException(
+                    "Conflicting file tracer name for '%s': %r vs %r" % (
+                        filename, this_plugin, other_plugin,
+                    )
+                )
 
     def erase(self, parallel=False):
         """Erase the data in this object.
@@ -314,16 +330,16 @@ class CoverageSqliteData(SimpleRepr):
 
     def read(self):
         self._connect()     # TODO: doesn't look right
-        self._have_read = True
+        self._have_used = True
 
     def write(self):
         """Write the collected coverage data to a file."""
         pass
 
-    def _start_writing(self):
-        if not self._have_read:
+    def _start_using(self):
+        if not self._have_used:
             self.erase()
-        self._have_read = True
+        self._have_used = True
 
     def has_arcs(self):
         return bool(self._has_arcs)
@@ -340,12 +356,18 @@ class CoverageSqliteData(SimpleRepr):
         was not measured, then None is returned.
 
         """
+        self._start_using()
         with self._connect() as con:
-            for tracer, in con.execute("select tracer from file where path = ?", (filename,)):
-                return tracer or ""
-        return None
+            file_id = self._file_id(filename)
+            if file_id is None:
+                return None
+            row = con.execute("select tracer from tracer where file_id = ?", (file_id,)).fetchone()
+            if row is not None:
+                return row[0] or ""
+            return ""   # File was measured, but no tracer associated.
 
     def lines(self, filename):
+        self._start_using()
         if self.has_arcs():
             arcs = self.arcs(filename)
             if arcs is not None:
@@ -362,6 +384,7 @@ class CoverageSqliteData(SimpleRepr):
                 return [lineno for lineno, in linenos]
 
     def arcs(self, filename):
+        self._start_using()
         with self._connect() as con:
             file_id = self._file_id(filename)
             if file_id is None:
-- 
cgit v1.2.1


From 6ee0473a77c8bd8c91681fa86e58acb55a6e44f4 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sat, 18 Aug 2018 11:26:14 -0400
Subject: A better more accurate bool(data)

---
 coverage/sqldata.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 01082a9b..f53561e7 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -138,10 +138,7 @@ class CoverageSqliteData(SimpleRepr):
     def __nonzero__(self):
         try:
             with self._connect() as con:
-                if self.has_arcs():
-                    rows = con.execute("select * from arc limit 1")
-                else:
-                    rows = con.execute("select * from line limit 1")
+                rows = con.execute("select * from file limit 1")
                 return bool(list(rows))
         except CoverageException:
             return False
-- 
cgit v1.2.1


From 948c307c89c9f61256bd96b770fa5b14ee4fe383 Mon Sep 17 00:00:00 2001
From: Ned Batchelder <ned@nedbatchelder.com>
Date: Sun, 19 Aug 2018 07:47:54 -0400
Subject: PyPy needs to close cursors from pragmas

---
 coverage/sqldata.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'coverage/sqldata.py')

diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index f53561e7..f92e245b 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -3,6 +3,7 @@
 
 """Sqlite coverage data."""
 
+# TODO: get sys_info for data class, so we can see sqlite version etc
 # TODO: get rid of skip_unless_data_storage_is_json
 # TODO: get rid of "JSON message" and "SQL message" in the tests
 # TODO: check the schema
@@ -403,9 +404,11 @@ class Sqlite(SimpleRepr):
         self.con = sqlite3.connect(self.filename)
 
         # This pragma makes writing faster. It disables rollbacks, but we never need them.
-        self.execute("pragma journal_mode=off")
+        # PyPy needs the .close() calls here, or sqlite gets twisted up:
+        # https://bitbucket.org/pypy/pypy/issues/2872/default-isolation-mode-is-different-on
+        self.execute("pragma journal_mode=off").close()
         # This pragma makes writing faster.
-        self.execute("pragma synchronous=off")
+        self.execute("pragma synchronous=off").close()
 
     def close(self):
         self.con.close()
-- 
cgit v1.2.1