Merge branch 'master' into bugfix/invalid-ascii-csv

# Conflicts:
#	setup.py
#	tablib/compat.py
#	test_tablib.py

8 files changed, 376 insertions, 15 deletions

diff --git a/tablib/formats/__init__.py b/tablib/formats/__init__.py
index 5cca19f..418e607 100644
--- a/tablib/formats/__init__.py
+++ b/tablib/formats/__init__.py
@@ -13,5 +13,8 @@ from . import _xlsx as xlsx
 from . import _ods as ods
 from . import _dbf as dbf
 from . import _latex as latex
+from . import _df as df
+from . import _rst as rst
+from . import _jira as jira
 
-available = (json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods)
+available = (json, xls, yaml, csv, dbf, tsv, html, jira, latex, xlsx, ods, df, rst)
diff --git a/tablib/formats/_csv.py b/tablib/formats/_csv.py
index b74afd7..8b536a7 100644
--- a/tablib/formats/_csv.py
+++ b/tablib/formats/_csv.py
@@ -39,7 +39,7 @@ def import_set(dset, in_stream, headers=True, **kwargs):
 
         if (i == 0) and (headers):
             dset.headers = row
-        else:
+        elif row:
             dset.append(row)
 
 
diff --git a/tablib/formats/_df.py b/tablib/formats/_df.py
new file mode 100644
index 0000000..44b967f
--- /dev/null
+++ b/tablib/formats/_df.py
@@ -0,0 +1,49 @@
+""" Tablib - DataFrame Support.
+"""
+
+
+import sys
+
+
+if sys.version_info[0] > 2:
+    from io import BytesIO
+else:
+    from cStringIO import StringIO as BytesIO
+
+try:
+    from pandas import DataFrame
+except ImportError:
+    DataFrame = None
+
+import tablib
+
+from tablib.compat import unicode
+
+title = 'df'
+extensions = ('df', )
+
+def detect(stream):
+    """Returns True if given stream is a DataFrame."""
+    if DataFrame is None:
+        return False
+    try:
+        DataFrame(stream)
+        return True
+    except ValueError:
+        return False
+
+
+def export_set(dset, index=None):
+    """Returns DataFrame representation of DataBook."""
+    if DataFrame is None:
+        raise NotImplementedError(
+            'DataFrame Format requires `pandas` to be installed.'
+            ' Try `pip install tablib[pandas]`.')
+    dataframe = DataFrame(dset.dict, columns=dset.headers)
+    return dataframe
+
+
+def import_set(dset, in_stream):
+    """Returns dataset from DataFrame."""
+    dset.wipe()
+    dset.dict = in_stream.to_dict(orient='records')
diff --git a/tablib/formats/_jira.py b/tablib/formats/_jira.py
new file mode 100644
index 0000000..55fce52
--- /dev/null
+++ b/tablib/formats/_jira.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+"""Tablib - Jira table export support.
+
+   Generates a Jira table from the dataset.
+"""
+from tablib.compat import unicode
+
+title = 'jira'
+
+
+def export_set(dataset):
+    """Formats the dataset according to the Jira table syntax:
+
+    ||heading 1||heading 2||heading 3||
+    |col A1|col A2|col A3|
+    |col B1|col B2|col B3|
+
+    :param dataset: dataset to serialize
+    :type dataset: tablib.core.Dataset
+    """
+
+    header = _get_header(dataset.headers) if dataset.headers else ''
+    body = _get_body(dataset)
+    return '%s\n%s' % (header, body) if header else body
+
+
+def _get_body(dataset):
+    return '\n'.join([_serialize_row(row) for row in dataset])
+
+
+def _get_header(headers):
+    return _serialize_row(headers, delimiter='||')
+
+
+def _serialize_row(row, delimiter='|'):
+    return '%s%s%s' % (delimiter,
+                       delimiter.join([unicode(item) if item else ' ' for item in row]),
+                       delimiter)
diff --git a/tablib/formats/_json.py b/tablib/formats/_json.py
index a3d6cc3..bbd2c96 100644
--- a/tablib/formats/_json.py
+++ b/tablib/formats/_json.py
@@ -3,36 +3,33 @@
 """ Tablib - JSON Support
 """
 import decimal
+import json
+from uuid import UUID
 
 import tablib
 
-try:
-    import ujson as json
-except ImportError:
-    import json
 
 title = 'json'
 extensions = ('json', 'jsn')
 
 
-def date_handler(obj):
-    if isinstance(obj, decimal.Decimal):
+def serialize_objects_handler(obj):
+    if isinstance(obj, decimal.Decimal) or isinstance(obj, UUID):
         return str(obj)
     elif hasattr(obj, 'isoformat'):
         return obj.isoformat()
     else:
         return obj
-    # return obj.isoformat() if hasattr(obj, 'isoformat') else obj
 
 
 def export_set(dataset):
     """Returns JSON representation of Dataset."""
-    return json.dumps(dataset.dict, default=date_handler)
+    return json.dumps(dataset.dict, default=serialize_objects_handler)
 
 
 def export_book(databook):
     """Returns JSON representation of Databook."""
-    return json.dumps(databook._package(), default=date_handler)
+    return json.dumps(databook._package(), default=serialize_objects_handler)
 
 
 def import_set(dset, in_stream):
diff --git a/tablib/formats/_rst.py b/tablib/formats/_rst.py
new file mode 100644
index 0000000..4b53ad7
--- /dev/null
+++ b/tablib/formats/_rst.py
@@ -0,0 +1,273 @@
+# -*- coding: utf-8 -*-
+
+""" Tablib - reStructuredText Support
+"""
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from textwrap import TextWrapper
+
+from tablib.compat import (
+    median,
+    unicode,
+    izip_longest,
+)
+
+
+title = 'rst'
+extensions = ('rst',)
+
+
+MAX_TABLE_WIDTH = 80  # Roughly. It may be wider to avoid breaking words.
+
+
+JUSTIFY_LEFT = 'left'
+JUSTIFY_CENTER = 'center'
+JUSTIFY_RIGHT = 'right'
+JUSTIFY_VALUES = (JUSTIFY_LEFT, JUSTIFY_CENTER, JUSTIFY_RIGHT)
+
+
+def to_unicode(value):
+    if isinstance(value, bytes):
+        return value.decode('utf-8')
+    return unicode(value)
+
+
+def _max_word_len(text):
+    """
+    Return the length of the longest word in `text`.
+
+
+    >>> _max_word_len('Python Module for Tabular Datasets')
+    8
+
+    """
+    return max((len(word) for word in text.split()))
+
+
+def _get_column_string_lengths(dataset):
+    """
+    Returns a list of string lengths of each column, and a list of
+    maximum word lengths.
+    """
+    if dataset.headers:
+        column_lengths = [[len(h)] for h in dataset.headers]
+        word_lens = [_max_word_len(h) for h in dataset.headers]
+    else:
+        column_lengths = [[] for _ in range(dataset.width)]
+        word_lens = [0 for _ in range(dataset.width)]
+    for row in dataset.dict:
+        values = iter(row.values() if hasattr(row, 'values') else row)
+        for i, val in enumerate(values):
+            text = to_unicode(val)
+            column_lengths[i].append(len(text))
+            word_lens[i] = max(word_lens[i], _max_word_len(text))
+    return column_lengths, word_lens
+
+
+def _row_to_lines(values, widths, wrapper, sep='|', justify=JUSTIFY_LEFT):
+    """
+    Returns a table row of wrapped values as a list of lines
+    """
+    if justify not in JUSTIFY_VALUES:
+        raise ValueError('Value of "justify" must be one of "{}"'.format(
+            '", "'.join(JUSTIFY_VALUES)
+        ))
+    if justify == JUSTIFY_LEFT:
+        just = lambda text, width: text.ljust(width)
+    elif justify == JUSTIFY_CENTER:
+        just = lambda text, width: text.center(width)
+    else:
+        just = lambda text, width: text.rjust(width)
+    lpad = sep + ' ' if sep else ''
+    rpad = ' ' + sep if sep else ''
+    pad = ' ' + sep + ' '
+    cells = []
+    for value, width in zip(values, widths):
+        wrapper.width = width
+        text = to_unicode(value)
+        cell = wrapper.wrap(text)
+        cells.append(cell)
+    lines = izip_longest(*cells, fillvalue='')
+    lines = (
+        (just(cell_line, widths[i]) for i, cell_line in enumerate(line))
+        for line in lines
+    )
+    lines = [''.join((lpad, pad.join(line), rpad)) for line in lines]
+    return lines
+
+
+def _get_column_widths(dataset, max_table_width=MAX_TABLE_WIDTH, pad_len=3):
+    """
+    Returns a list of column widths proportional to the median length
+    of the text in their cells.
+    """
+    str_lens, word_lens = _get_column_string_lengths(dataset)
+    median_lens = [int(median(lens)) for lens in str_lens]
+    total = sum(median_lens)
+    if total > max_table_width - (pad_len * len(median_lens)):
+        column_widths = (max_table_width * l // total for l in median_lens)
+    else:
+        column_widths = (l for l in median_lens)
+    # Allow for separator and padding:
+    column_widths = (w - pad_len if w > pad_len else w for w in column_widths)
+    # Rather widen table than break words:
+    column_widths = [max(w, l) for w, l in zip(column_widths, word_lens)]
+    return column_widths
+
+
+def export_set_as_simple_table(dataset, column_widths=None):
+    """
+    Returns reStructuredText grid table representation of dataset.
+    """
+    lines = []
+    wrapper = TextWrapper()
+    if column_widths is None:
+        column_widths = _get_column_widths(dataset, pad_len=2)
+    border = '  '.join(['=' * w for w in column_widths])
+
+    lines.append(border)
+    if dataset.headers:
+        lines.extend(_row_to_lines(
+            dataset.headers,
+            column_widths,
+            wrapper,
+            sep='',
+            justify=JUSTIFY_CENTER,
+        ))
+        lines.append(border)
+    for row in dataset.dict:
+        values = iter(row.values() if hasattr(row, 'values') else row)
+        lines.extend(_row_to_lines(values, column_widths, wrapper, ''))
+    lines.append(border)
+    return '\n'.join(lines)
+
+
+def export_set_as_grid_table(dataset, column_widths=None):
+    """
+    Returns reStructuredText grid table representation of dataset.
+
+
+    >>> from tablib import Dataset
+    >>> from tablib.formats import rst
+    >>> bits = ((0, 0), (1, 0), (0, 1), (1, 1))
+    >>> data = Dataset()
+    >>> data.headers = ['A', 'B', 'A and B']
+    >>> for a, b in bits:
+    ...     data.append([bool(a), bool(b), bool(a * b)])
+    >>> print(rst.export_set(data, force_grid=True))
+    +-------+-------+-------+
+    |   A   |   B   | A and |
+    |       |       |   B   |
+    +=======+=======+=======+
+    | False | False | False |
+    +-------+-------+-------+
+    | True  | False | False |
+    +-------+-------+-------+
+    | False | True  | False |
+    +-------+-------+-------+
+    | True  | True  | True  |
+    +-------+-------+-------+
+
+    """
+    lines = []
+    wrapper = TextWrapper()
+    if column_widths is None:
+        column_widths = _get_column_widths(dataset)
+    header_sep = '+=' + '=+='.join(['=' * w for w in column_widths]) + '=+'
+    row_sep = '+-' + '-+-'.join(['-' * w for w in column_widths]) + '-+'
+
+    lines.append(row_sep)
+    if dataset.headers:
+        lines.extend(_row_to_lines(
+            dataset.headers,
+            column_widths,
+            wrapper,
+            justify=JUSTIFY_CENTER,
+        ))
+        lines.append(header_sep)
+    for row in dataset.dict:
+        values = iter(row.values() if hasattr(row, 'values') else row)
+        lines.extend(_row_to_lines(values, column_widths, wrapper))
+        lines.append(row_sep)
+    return '\n'.join(lines)
+
+
+def _use_simple_table(head0, col0, width0):
+    """
+    Use a simple table if the text in the first column is never wrapped
+
+
+    >>> _use_simple_table('menu', ['egg', 'bacon'], 10)
+    True
+    >>> _use_simple_table(None, ['lobster thermidor', 'spam'], 10)
+    False
+
+    """
+    if head0 is not None:
+        head0 = to_unicode(head0)
+        if len(head0) > width0:
+            return False
+    for cell in col0:
+        cell = to_unicode(cell)
+        if len(cell) > width0:
+            return False
+    return True
+
+
+def export_set(dataset, **kwargs):
+    """
+    Returns reStructuredText table representation of dataset.
+
+    Returns a simple table if the text in the first column is never
+    wrapped, otherwise returns a grid table.
+
+
+    >>> from tablib import Dataset
+    >>> bits = ((0, 0), (1, 0), (0, 1), (1, 1))
+    >>> data = Dataset()
+    >>> data.headers = ['A', 'B', 'A and B']
+    >>> for a, b in bits:
+    ...     data.append([bool(a), bool(b), bool(a * b)])
+    >>> table = data.rst
+    >>> table.split('\\n') == [
+    ...     '=====  =====  =====',
+    ...     '  A      B    A and',
+    ...     '                B  ',
+    ...     '=====  =====  =====',
+    ...     'False  False  False',
+    ...     'True   False  False',
+    ...     'False  True   False',
+    ...     'True   True   True ',
+    ...     '=====  =====  =====',
+    ... ]
+    True
+
+    """
+    if not dataset.dict:
+        return ''
+    force_grid = kwargs.get('force_grid', False)
+    max_table_width = kwargs.get('max_table_width', MAX_TABLE_WIDTH)
+    column_widths = _get_column_widths(dataset, max_table_width)
+
+    use_simple_table = _use_simple_table(
+        dataset.headers[0] if dataset.headers else None,
+        dataset.get_col(0),
+        column_widths[0],
+    )
+    if use_simple_table and not force_grid:
+        return export_set_as_simple_table(dataset, column_widths)
+    else:
+        return export_set_as_grid_table(dataset, column_widths)
+
+
+def export_book(databook):
+    """
+    reStructuredText representation of a Databook.
+
+    Tables are separated by a blank line. All tables use the grid
+    format.
+    """
+    return '\n\n'.join(export_set(dataset, force_grid=True)
+                       for dataset in databook._datasets)
diff --git a/tablib/formats/_xlsx.py b/tablib/formats/_xlsx.py
index 20f55df..816fd37 100644
--- a/tablib/formats/_xlsx.py
+++ b/tablib/formats/_xlsx.py
@@ -52,7 +52,7 @@ def export_book(databook, freeze_panes=True):
 
     wb = Workbook()
     for sheet in wb.worksheets:
-        wb.remove_sheet(sheet)
+        wb.remove(sheet)
     for i, dset in enumerate(databook._datasets):
         ws = wb.create_sheet()
         ws.title = dset.title if dset.title else 'Sheet%s' % (i)
@@ -71,7 +71,7 @@ def import_set(dset, in_stream, headers=True):
     dset.wipe()
 
     xls_book = openpyxl.reader.excel.load_workbook(BytesIO(in_stream))
-    sheet = xls_book.get_active_sheet()
+    sheet = xls_book.active
 
     dset.title = sheet.title
 
@@ -119,7 +119,7 @@ def dset_sheet(dataset, ws, freeze_panes=True):
         row_number = i + 1
         for j, col in enumerate(row):
             col_idx = get_column_letter(j + 1)
-            cell = ws.cell('%s%s' % (col_idx, row_number))
+            cell = ws['%s%s' % (col_idx, row_number)]
 
             # bold headers
             if (row_number == 1) and dataset.headers:
diff --git a/tablib/formats/_yaml.py b/tablib/formats/_yaml.py
index 5aecb42..3d17baf 100644
--- a/tablib/formats/_yaml.py
+++ b/tablib/formats/_yaml.py
@@ -33,7 +33,7 @@ def import_book(dbook, in_stream):
 
     dbook.wipe()
 
-    for sheet in yaml.load(in_stream):
+    for sheet in yaml.safe_load(in_stream):
         data = tablib.Dataset()
         data.title = sheet['title']
         data.dict = sheet['data']