diff options
Diffstat (limited to 'tablib/formats')
| -rw-r--r-- | tablib/formats/__init__.py | 4 | ||||
| -rw-r--r-- | tablib/formats/_csv.py | 10 | ||||
| -rw-r--r-- | tablib/formats/_dbf.py | 93 | ||||
| -rw-r--r-- | tablib/formats/_html.py | 52 | ||||
| -rw-r--r-- | tablib/formats/_latex.py | 134 | ||||
| -rw-r--r-- | tablib/formats/_tsv.py | 49 | ||||
| -rw-r--r-- | tablib/formats/_xls.py | 2 | ||||
| -rw-r--r-- | tablib/formats/_xlsx.py | 6 |
8 files changed, 275 insertions, 75 deletions
diff --git a/tablib/formats/__init__.py b/tablib/formats/__init__.py index 5fdf279..5cca19f 100644 --- a/tablib/formats/__init__.py +++ b/tablib/formats/__init__.py @@ -11,5 +11,7 @@ from . import _tsv as tsv from . import _html as html from . import _xlsx as xlsx from . import _ods as ods +from . import _dbf as dbf +from . import _latex as latex -available = (json, xls, yaml, csv, tsv, html, xlsx, ods) +available = (json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods) diff --git a/tablib/formats/_csv.py b/tablib/formats/_csv.py index 7d29318..4c00809 100644 --- a/tablib/formats/_csv.py +++ b/tablib/formats/_csv.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" Tablib - CSV Support. +""" Tablib - *SV Support. """ from tablib.compat import is_py3, csv, StringIO @@ -11,13 +11,14 @@ extensions = ('csv',) DEFAULT_ENCODING = 'utf-8' - +DEFAULT_DELIMITER = ',' def export_set(dataset, **kwargs): """Returns CSV representation of Dataset.""" stream = StringIO() + kwargs.setdefault('delimeter', DEFAULT_DELIMITER) if not is_py3: kwargs.setdefault('encoding', DEFAULT_ENCODING) @@ -34,6 +35,7 @@ def import_set(dset, in_stream, headers=True, **kwargs): dset.wipe() + kwargs.setdefault('delimeter', DEFAULT_DELIMITER) if not is_py3: kwargs.setdefault('encoding', DEFAULT_ENCODING) @@ -46,10 +48,10 @@ def import_set(dset, in_stream, headers=True, **kwargs): dset.append(row) -def detect(stream): +def detect(stream, delimiter=DEFAULT_DELIMITER): """Returns True if given stream is valid CSV.""" try: - csv.Sniffer().sniff(stream, delimiters=',') + csv.Sniffer().sniff(stream, delimiters=delimiter) return True except (csv.Error, TypeError): return False diff --git a/tablib/formats/_dbf.py b/tablib/formats/_dbf.py new file mode 100644 index 0000000..41c2ef4 --- /dev/null +++ b/tablib/formats/_dbf.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +""" Tablib - DBF Support. +""" +import tempfile +import struct +import os + +from tablib.compat import StringIO +from tablib.compat import dbfpy +from tablib.compat import is_py3 + +if is_py3: + from tablib.packages.dbfpy3 import dbf + from tablib.packages.dbfpy3 import dbfnew + from tablib.packages.dbfpy3 import record as dbfrecord + import io +else: + from tablib.packages.dbfpy import dbf + from tablib.packages.dbfpy import dbfnew + from tablib.packages.dbfpy import record as dbfrecord + + +title = 'dbf' +extensions = ('csv',) + +DEFAULT_ENCODING = 'utf-8' + +def export_set(dataset): + """Returns DBF representation of a Dataset""" + new_dbf = dbfnew.dbf_new() + temp_file, temp_uri = tempfile.mkstemp() + + # create the appropriate fields based on the contents of the first row + first_row = dataset[0] + for fieldname, field_value in zip(dataset.headers, first_row): + if type(field_value) in [int, float]: + new_dbf.add_field(fieldname, 'N', 10, 8) + else: + new_dbf.add_field(fieldname, 'C', 80) + + new_dbf.write(temp_uri) + + dbf_file = dbf.Dbf(temp_uri, readOnly=0) + for row in dataset: + record = dbfrecord.DbfRecord(dbf_file) + for fieldname, field_value in zip(dataset.headers, row): + record[fieldname] = field_value + record.store() + + dbf_file.close() + dbf_stream = open(temp_uri, 'rb') + if is_py3: + stream = io.BytesIO(dbf_stream.read()) + else: + stream = StringIO(dbf_stream.read()) + dbf_stream.close() + os.remove(temp_uri) + return stream.getvalue() + +def import_set(dset, in_stream, headers=True): + """Returns a dataset from a DBF stream.""" + + dset.wipe() + if is_py3: + _dbf = dbf.Dbf(io.BytesIO(in_stream)) + else: + _dbf = dbf.Dbf(StringIO(in_stream)) + dset.headers = _dbf.fieldNames + for record in range(_dbf.recordCount): + row = [_dbf[record][f] for f in _dbf.fieldNames] + dset.append(row) + +def detect(stream): + """Returns True if the given stream is valid DBF""" + #_dbf = dbf.Table(StringIO(stream)) + try: + if is_py3: + if type(stream) is not bytes: + stream = bytes(stream, 'utf-8') + _dbf = dbf.Dbf(io.BytesIO(stream), readOnly=True) + else: + _dbf = dbf.Dbf(StringIO(stream), readOnly=True) + return True + except (ValueError, struct.error): + # When we try to open up a file that's not a DBF, dbfpy raises a + # ValueError. + # When unpacking a string argument with less than 8 chars, struct.error is + # raised. + return False + + + diff --git a/tablib/formats/_html.py b/tablib/formats/_html.py index 7bb77f0..0b45f14 100644 --- a/tablib/formats/_html.py +++ b/tablib/formats/_html.py @@ -23,45 +23,45 @@ extensions = ('html', ) def export_set(dataset): - """HTML representation of a Dataset.""" + """HTML representation of a Dataset.""" - stream = StringIO() + stream = StringIO() - page = markup.page() - page.table.open() + page = markup.page() + page.table.open() - if dataset.headers is not None: - new_header = [item if item is not None else '' for item in dataset.headers] + if dataset.headers is not None: + new_header = [item if item is not None else '' for item in dataset.headers] - page.thead.open() - headers = markup.oneliner.th(new_header) - page.tr(headers) - page.thead.close() + page.thead.open() + headers = markup.oneliner.th(new_header) + page.tr(headers) + page.thead.close() - for row in dataset: - new_row = [item if item is not None else '' for item in row] + for row in dataset: + new_row = [item if item is not None else '' for item in row] - html_row = markup.oneliner.td(new_row) - page.tr(html_row) + html_row = markup.oneliner.td(new_row) + page.tr(html_row) - page.table.close() + page.table.close() # Allow unicode characters in output - wrapper = codecs.getwriter("utf8")(stream) - wrapper.writelines(unicode(page)) + wrapper = codecs.getwriter("utf8")(stream) + wrapper.writelines(unicode(page)) - return stream.getvalue().decode('utf-8') + return stream.getvalue().decode('utf-8') def export_book(databook): - """HTML representation of a Databook.""" + """HTML representation of a Databook.""" - stream = StringIO() + stream = StringIO() - for i, dset in enumerate(databook._datasets): - title = (dset.title if dset.title else 'Set %s' % (i)) - stream.write('<%s>%s</%s>\n' % (BOOK_ENDINGS, title, BOOK_ENDINGS)) - stream.write(dset.html) - stream.write('\n') + for i, dset in enumerate(databook._datasets): + title = (dset.title if dset.title else 'Set %s' % (i)) + stream.write('<%s>%s</%s>\n' % (BOOK_ENDINGS, title, BOOK_ENDINGS)) + stream.write(dset.html) + stream.write('\n') - return stream.getvalue() + return stream.getvalue() diff --git a/tablib/formats/_latex.py b/tablib/formats/_latex.py new file mode 100644 index 0000000..44ee101 --- /dev/null +++ b/tablib/formats/_latex.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- + +"""Tablib - LaTeX table export support. + + Generates a LaTeX booktabs-style table from the dataset. +""" +import re + +from tablib.compat import unicode + +title = 'latex' +extensions = ('tex',) + +TABLE_TEMPLATE = """\ +%% Note: add \\usepackage{booktabs} to your preamble +%% +\\begin{table}[!htbp] + \\centering + %(CAPTION)s + \\begin{tabular}{%(COLSPEC)s} + \\toprule +%(HEADER)s + %(MIDRULE)s +%(BODY)s + \\bottomrule + \\end{tabular} +\\end{table} +""" + +TEX_RESERVED_SYMBOLS_MAP = dict([ + ('\\', '\\textbackslash{}'), + ('{', '\\{'), + ('}', '\\}'), + ('$', '\\$'), + ('&', '\\&'), + ('#', '\\#'), + ('^', '\\textasciicircum{}'), + ('_', '\\_'), + ('~', '\\textasciitilde{}'), + ('%', '\\%'), +]) + +TEX_RESERVED_SYMBOLS_RE = re.compile( + '(%s)' % '|'.join(map(re.escape, TEX_RESERVED_SYMBOLS_MAP.keys()))) + + +def export_set(dataset): + """Returns LaTeX representation of dataset + + :param dataset: dataset to serialize + :type dataset: tablib.core.Dataset + """ + + caption = '\\caption{%s}' % dataset.title if dataset.title else '%' + colspec = _colspec(dataset.width) + header = _serialize_row(dataset.headers) if dataset.headers else '' + midrule = _midrule(dataset.width) + body = '\n'.join([_serialize_row(row) for row in dataset]) + return TABLE_TEMPLATE % dict(CAPTION=caption, COLSPEC=colspec, + HEADER=header, MIDRULE=midrule, BODY=body) + + +def _colspec(dataset_width): + """Generates the column specification for the LaTeX `tabular` environment + based on the dataset width. + + The first column is justified to the left, all further columns are aligned + to the right. + + .. note:: This is only a heuristic and most probably has to be fine-tuned + post export. Column alignment should depend on the data type, e.g., textual + content should usually be aligned to the left while numeric content almost + always should be aligned to the right. + + :param dataset_width: width of the dataset + """ + + spec = 'l' + for _ in range(1, dataset_width): + spec += 'r' + return spec + + +def _midrule(dataset_width): + """Generates the table `midrule`, which may be composed of several + `cmidrules`. + + :param dataset_width: width of the dataset to serialize + """ + + if not dataset_width or dataset_width == 1: + return '\\midrule' + return ' '.join([_cmidrule(colindex, dataset_width) for colindex in + range(1, dataset_width + 1)]) + + +def _cmidrule(colindex, dataset_width): + """Generates the `cmidrule` for a single column with appropriate trimming + based on the column position. + + :param colindex: Column index + :param dataset_width: width of the dataset + """ + + rule = '\\cmidrule(%s){%d-%d}' + if colindex == 1: + # Rule of first column is trimmed on the right + return rule % ('r', colindex, colindex) + if colindex == dataset_width: + # Rule of last column is trimmed on the left + return rule % ('l', colindex, colindex) + # Inner columns are trimmed on the left and right + return rule % ('lr', colindex, colindex) + + +def _serialize_row(row): + """Returns string representation of a single row. + + :param row: single dataset row + """ + + new_row = [_escape_tex_reserved_symbols(unicode(item)) if item else '' for + item in row] + return 6 * ' ' + ' & '.join(new_row) + ' \\\\' + + +def _escape_tex_reserved_symbols(input): + """Escapes all TeX reserved symbols ('_', '~', etc.) in a string. + + :param input: String to escape + """ + def replace(match): + return TEX_RESERVED_SYMBOLS_MAP[match.group()] + return TEX_RESERVED_SYMBOLS_RE.sub(replace, input) diff --git a/tablib/formats/_tsv.py b/tablib/formats/_tsv.py index 8ef2b67..9380b3b 100644 --- a/tablib/formats/_tsv.py +++ b/tablib/formats/_tsv.py @@ -3,57 +3,28 @@ """ Tablib - TSV (Tab Separated Values) Support. """ -from tablib.compat import is_py3, csv, StringIO - - +from tablib.formats._csv import ( + export_set as export_set_wrapper, + import_set as import_set_wrapper, + detect as detect_wrapper, +) title = 'tsv' extensions = ('tsv',) DEFAULT_ENCODING = 'utf-8' +DELIMITER = '\t' def export_set(dataset): - """Returns a TSV representation of Dataset.""" - - stream = StringIO() - - if is_py3: - _tsv = csv.writer(stream, delimiter='\t') - else: - _tsv = csv.writer(stream, encoding=DEFAULT_ENCODING, delimiter='\t') - - for row in dataset._package(dicts=False): - _tsv.writerow(row) - - return stream.getvalue() + """Returns TSV representation of Dataset.""" + return export_set_wrapper(dataset, delimiter=DELIMITER) def import_set(dset, in_stream, headers=True): """Returns dataset from TSV stream.""" - - dset.wipe() - - if is_py3: - rows = csv.reader(in_stream.splitlines(), delimiter='\t') - else: - rows = csv.reader(in_stream.splitlines(), delimiter='\t', - encoding=DEFAULT_ENCODING) - - for i, row in enumerate(rows): - # Skip empty rows - if not row: - continue - - if (i == 0) and (headers): - dset.headers = row - else: - dset.append(row) + return import_set_wrapper(dset, in_stream, headers=headers, delimiter=DELIMITER) def detect(stream): """Returns True if given stream is valid TSV.""" - try: - csv.Sniffer().sniff(stream, delimiters='\t') - return True - except (csv.Error, TypeError): - return False + return detect_wrapper(stream, delimiter=DELIMITER) diff --git a/tablib/formats/_xls.py b/tablib/formats/_xls.py index 67b87ea..787907a 100644 --- a/tablib/formats/_xls.py +++ b/tablib/formats/_xls.py @@ -5,7 +5,7 @@ import sys -from tablib.compat import BytesIO, xlwt, xlrd, XLRDError +from tablib.compat import BytesIO, xlwt, xlrd, XLRDError, xrange import tablib title = 'xls' diff --git a/tablib/formats/_xlsx.py b/tablib/formats/_xlsx.py index 0cd8500..d9d3d57 100644 --- a/tablib/formats/_xlsx.py +++ b/tablib/formats/_xlsx.py @@ -69,7 +69,7 @@ def import_set(dset, in_stream, headers=True): dset.wipe() - xls_book = openpyxl.reader.excel.load_workbook(in_stream) + xls_book = openpyxl.reader.excel.load_workbook(BytesIO(in_stream)) sheet = xls_book.get_active_sheet() dset.title = sheet.title @@ -87,7 +87,7 @@ def import_book(dbook, in_stream, headers=True): dbook.wipe() - xls_book = openpyxl.reader.excel.load_workbook(in_stream) + xls_book = openpyxl.reader.excel.load_workbook(BytesIO(in_stream)) for sheet in xls_book.worksheets: data = tablib.Dataset() @@ -115,8 +115,6 @@ def dset_sheet(dataset, ws, freeze_panes=True): row_number = i + 1 for j, col in enumerate(row): col_idx = get_column_letter(j + 1) - # We want to freeze the column after the last column - frzn_col_idx = get_column_letter(j + 2) # bold headers if (row_number == 1) and dataset.headers: |