diff options
| -rw-r--r-- | AUTHORS | 4 | ||||
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | TODO.rst | 7 | ||||
| -rw-r--r-- | docs/development.rst | 32 | ||||
| -rw-r--r-- | docs/index.rst | 8 | ||||
| -rw-r--r-- | docs/install.rst | 19 | ||||
| -rw-r--r-- | docs/tutorial.rst | 10 | ||||
| -rw-r--r-- | setup.cfg | 2 | ||||
| -rw-r--r-- | tablib/compat.py | 2 | ||||
| -rw-r--r-- | tablib/core.py | 163 | ||||
| -rw-r--r-- | tablib/formats/__init__.py | 3 | ||||
| -rw-r--r-- | tablib/formats/_csv.py | 30 | ||||
| -rw-r--r-- | tablib/formats/_html.py | 52 | ||||
| -rw-r--r-- | tablib/formats/_json.py | 8 | ||||
| -rw-r--r-- | tablib/formats/_latex.py | 134 | ||||
| -rw-r--r-- | tablib/formats/_tsv.py | 49 | ||||
| -rw-r--r-- | tablib/formats/_xls.py | 2 | ||||
| -rw-r--r-- | tablib/formats/_xlsx.py | 19 | ||||
| -rwxr-xr-x | test_tablib.py | 164 | ||||
| -rw-r--r-- | toy.py | 15 |
20 files changed, 518 insertions, 207 deletions
@@ -28,3 +28,7 @@ Patches and Suggestions - Marc Abramowitz - Alex Gaynor - James Douglass +- Tommy Anthony +- Rabin Nankhwa +- Marco Dallagiacoma +- Mathias Loesch @@ -1,4 +1,4 @@ publish: python setup.py register python setup.py sdist upload - python setup.py bdist_wheel upload + python setup.py bdist_wheel upload --universal diff --git a/TODO.rst b/TODO.rst deleted file mode 100644 index ecfa974..0000000 --- a/TODO.rst +++ /dev/null @@ -1,7 +0,0 @@ -* Hooks System - - pre/post-append - - pre/post-import - - pre/post-export -* Add Tablib.ext namespace -* Width detection for XLS output -* Documentation Improvements
\ No newline at end of file diff --git a/docs/development.rst b/docs/development.rst index c091669..3ab05d5 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -8,11 +8,6 @@ Tablib is under active development, and contributors are welcome. If you have a feature request, suggestion, or bug report, please open a new issue on GitHub_. To submit patches, please send a pull request on GitHub_. -If you'd like to contribute, there's plenty to do. Here's a short todo list. - - .. include:: ../TODO.rst - - .. _GitHub: http://github.com/kennethreitz/tablib/ @@ -66,8 +61,6 @@ Feature / Hotfix / Release branches follow a `Successful Git Branching Model`_ . The "next release" branch. Likely unstable. ``master`` Current production release (|version|) on PyPi. -``gh-pages`` - Current release of http://docs.python-tablib.org. Each release is tagged. @@ -205,34 +198,9 @@ Your ``docs/_build/html`` directory will then contain an HTML representation of You can also generate the documentation in **epub**, **latex**, **json**, *&c* similarly. -.. admonition:: GitHub Pages - - To push the documentation up to `GitHub Pages`_, you will first need to run `sphinx-to-github`_ against your ``docs/_build/html`` directory. - - GitHub Pages are powered by an HTML generation system called Jekyll_, which is configured to ignore files and folders that begin with "``_``" (*ie.* **_static**). - - - - - - - and `sphinx-to-github`_. :: - - Installing sphinx-to-github is simple. :: - - $ pip install sphinx-to-github - - Running it against the docs is even simpler. :: - - $ sphinx-to-github _build/html - - Move the resulting files to the **gh-pages** branch of your repository, and push it up to GitHub. - .. _`reStructured Text`: http://docutils.sourceforge.net/rst.html .. _Sphinx: http://sphinx.pocoo.org .. _`GitHub Pages`: http://pages.github.com -.. _Jekyll: http://github.com/mojombo/jekyll -.. _`sphinx-to-github`: http://github.com/michaeljones/sphinx-to-github ---------- diff --git a/docs/index.rst b/docs/index.rst index 7bf1a41..20103b6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,13 +26,13 @@ Tablib is an :ref:`MIT Licensed <mit>` format-agnostic tabular dataset library, :: - >>> data = tablib.Dataset(*[('Kenneth', 'Reitz', 23), ('Bessie', 'Monke', 22)], - headers=['First Name', 'Last Name', 'Age']) + >>> data = tablib.Dataset(headers=['First Name', 'Last Name', 'Age']) + >>> map(data.append, [('Kenneth', 'Reitz', 22), ('Bessie', 'Monke', 21)]) - >>> data.json + >>> print data.json [{"Last Name": "Reitz", "First Name": "Kenneth", "Age": 22}, {"Last Name": "Monke", "First Name": "Bessie", "Age": 21}] - >>> data.yaml + >>> print data.yaml - {Age: 22, First Name: Kenneth, Last Name: Reitz} - {Age: 21, First Name: Bessie, Last Name: Monke} diff --git a/docs/install.rst b/docs/install.rst index 6bb9ad6..7b0b00b 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -14,27 +14,10 @@ Installing Tablib Distribute & Pip ---------------- -Installing Tablib is simple with `pip <http://www.pip-installer.org/>`_:: +Of course, the recommended way to install Tablib is with `pip <http://www.pip-installer.org/>`_:: $ pip install tablib -or, with `easy_install <http://pypi.python.org/pypi/setuptools>`_:: - - $ easy_install tablib - -But, you really `shouldn't do that <http://www.pip-installer.org/en/latest/other-tools.html#pip-compared-to-easy-install>`_. - - - -Cheeseshop Mirror ------------------ - -If the Cheeseshop is down, you can also install Tablib from Kenneth Reitz's personal `Cheeseshop mirror <pip.kreitz.co/>`_:: - - $ pip install -i http://pip.kreitz.co/simple tablib - - - ------------------- Download the Source diff --git a/docs/tutorial.rst b/docs/tutorial.rst index bd2dbc0..551b191 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -39,6 +39,7 @@ You can now start filling this :class:`Dataset <tablib.Dataset>` object with dat + ----------- Adding Rows ----------- @@ -98,6 +99,15 @@ It's that easy. -------------- +Importing Data +-------------- +Creating a :class:`tablib.Dataset` object by importing a pre-existing file is simple. :: + + imported_data = tablib.import_set(open('data.csv').read()) + +This detects what sort of data is being passed in, and uses an appropriate formatter to do the import. So you can import from a variety of different file types. + +-------------- Exporting Data -------------- diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 5e40900..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[wheel] -universal = 1 diff --git a/tablib/compat.py b/tablib/compat.py index d4582d5..e03526d 100644 --- a/tablib/compat.py +++ b/tablib/compat.py @@ -37,6 +37,7 @@ if is_py3: unicode = str bytes = bytes basestring = str + xrange = range else: from cStringIO import StringIO as BytesIO @@ -53,3 +54,4 @@ else: import tablib.packages.dbfpy as dbfpy unicode = unicode + xrange = xrange diff --git a/tablib/core.py b/tablib/core.py index 7afdd71..7d09789 100644 --- a/tablib/core.py +++ b/tablib/core.py @@ -153,6 +153,8 @@ class Dataset(object): """ + _formats = {} + def __init__(self, *args, **kwargs): self._data = list(Row(arg) for arg in args) self.__headers = None @@ -163,15 +165,9 @@ class Dataset(object): # (column, callback) tuples self._formatters = [] - try: - self.headers = kwargs['headers'] - except KeyError: - self.headers = None + self.headers = kwargs.get('headers') - try: - self.title = kwargs['title'] - except KeyError: - self.title = None + self.title = kwargs.get('title') self._register_formats() @@ -254,11 +250,15 @@ class Dataset(object): try: try: setattr(cls, fmt.title, property(fmt.export_set, fmt.import_set)) + cls._formats[fmt.title] = (fmt.export_set, fmt.import_set) except AttributeError: setattr(cls, fmt.title, property(fmt.export_set)) + cls._formats[fmt.title] = (fmt.export_set, None) + setattr(cls, 'get_%s' % fmt.title, fmt.export_set) + setattr(cls, 'set_%s' % fmt.title, fmt.import_set) except AttributeError: - pass + cls._formats[fmt.title] = (None, None) def _validate(self, row=None, col=None, safety=False): @@ -349,7 +349,7 @@ class Dataset(object): A dataset object can also be imported by setting the `Dataset.dict` attribute: :: data = tablib.Dataset() - data.json = '[{"last_name": "Adams","age": 90,"first_name": "John"}]' + data.dict = [{'age': 90, 'first_name': 'Kenneth', 'last_name': 'Reitz'}] """ return self._package() @@ -428,12 +428,34 @@ class Dataset(object): except TypeError: return 0 + def import_(self, format, in_stream, **kwargs): + """ + Import `in_stream` to the :class:`Dataset` object using the `format`. + + :param \*\*kwargs: (optional) custom configuration to the format `import_set`. + """ + export_set, import_set = self._formats.get(format, (None, None)) + if not import_set: + raise UnsupportedFormat + + import_set(self, in_stream, **kwargs) + + def export(self, format, **kwargs): + """ + Export :class:`Dataset` object to `format`. + + :param \*\*kwargs: (optional) custom configuration to the format `export_set`. + """ + export_set, import_set = self._formats.get(format, (None, None)) + if not export_set: + raise UnsupportedFormat + + return export_set(self, **kwargs) # ------- # Formats # ------- - @property def xls(): """A Legacy Excel Spreadsheet representation of the :class:`Dataset` object, with :ref:`separators`. Cannot be set. @@ -584,6 +606,16 @@ class Dataset(object): pass + @property + def latex(): + """A LaTeX booktabs representation of the :class:`Dataset` object. If a + title has been set, it will be exported as the table caption. + + .. note:: This method can be used for export only. + """ + pass + + # ---- # Rows # ---- @@ -934,17 +966,66 @@ class Dataset(object): return _dset + def remove_duplicates(self): + """Removes all duplicate rows from the :class:`Dataset` object + while maintaining the original order.""" + seen = set() + self._data[:] = [row for row in self._data if not (tuple(row) in seen or seen.add(tuple(row)))] + + def wipe(self): """Removes all content and headers from the :class:`Dataset` object.""" self._data = list() self.__headers = None + def subset(self, rows=None, cols=None): + """Returns a new instance of the :class:`Dataset`, + including only specified rows and columns. + """ + + # Don't return if no data + if not self: + return + + if rows is None: + rows = list(range(self.height)) + + if cols is None: + cols = list(self.headers) + + #filter out impossible rows and columns + rows = [row for row in rows if row in range(self.height)] + cols = [header for header in cols if header in self.headers] + + _dset = Dataset() + + #filtering rows and columns + _dset.headers = list(cols) + + _dset._data = [] + for row_no, row in enumerate(self._data): + data_row = [] + for key in _dset.headers: + if key in self.headers: + pos = self.headers.index(key) + data_row.append(row[pos]) + else: + raise KeyError + + if row_no in rows: + _dset.append(row=Row(data_row)) + + return _dset + + class Databook(object): """A book of :class:`Dataset` objects. """ + _formats = {} + def __init__(self, sets=None): if sets is None: @@ -960,7 +1041,6 @@ class Databook(object): except AttributeError: return '<databook object>' - def wipe(self): """Removes all :class:`Dataset` objects from the :class:`Databook`.""" self._datasets = [] @@ -973,11 +1053,13 @@ class Databook(object): try: try: setattr(cls, fmt.title, property(fmt.export_book, fmt.import_book)) + cls._formats[fmt.title] = (fmt.export_book, fmt.import_book) except AttributeError: setattr(cls, fmt.title, property(fmt.export_book)) + cls._formats[fmt.title] = (fmt.export_book, None) except AttributeError: - pass + cls._formats[fmt.title] = (None, None) def sheets(self): return self._datasets @@ -1012,6 +1094,30 @@ class Databook(object): """The number of the :class:`Dataset` objects within :class:`Databook`.""" return len(self._datasets) + def import_(self, format, in_stream, **kwargs): + """ + Import `in_stream` to the :class:`Databook` object using the `format`. + + :param \*\*kwargs: (optional) custom configuration to the format `import_book`. + """ + export_book, import_book = self._formats.get(format, (None, None)) + if not import_book: + raise UnsupportedFormat + + import_book(self, in_stream, **kwargs) + + def export(self, format, **kwargs): + """ + Export :class:`Databook` object to `format`. + + :param \*\*kwargs: (optional) custom configuration to the format `export_book`. + """ + export_book, import_book = self._formats.get(format, (None, None)) + if not export_book: + raise UnsupportedFormat + + return export_book(self, **kwargs) + def detect(stream): """Return (format, stream) of given stream.""" @@ -1024,32 +1130,43 @@ def detect(stream): return (None, stream) -def import_set(stream): +def import_set(stream, format=None, **kwargs): """Return dataset of given stream.""" - (format, stream) = detect(stream) + if format: + format = get_formatter(format) + else: + format, stream = detect(stream) + data = Dataset() try: - data = Dataset() - format.import_set(data, stream) + format.import_set(data, stream, **kwargs) return data - except AttributeError: return None -def import_book(stream): +def import_book(stream, format=None, **kwargs): """Return dataset of given stream.""" - (format, stream) = detect(stream) + if format: + format = get_formatter(format) + else: + format, stream = detect(stream) + databook = Databook() try: - databook = Databook() - format.import_book(databook, stream) + format.import_book(databook, stream, **kwargs) return databook - except AttributeError: return None +def get_formatter(format): + for item in formats.available: + if item.title == format: + return item + raise UnsupportedFormat(format) + + class InvalidDatasetType(Exception): "Only Datasets can be added to a DataBook" diff --git a/tablib/formats/__init__.py b/tablib/formats/__init__.py index 1eda107..5cca19f 100644 --- a/tablib/formats/__init__.py +++ b/tablib/formats/__init__.py @@ -12,5 +12,6 @@ from . import _html as html from . import _xlsx as xlsx from . import _ods as ods from . import _dbf as dbf +from . import _latex as latex -available = (json, xls, yaml, csv, dbf, tsv, html, xlsx, ods) +available = (json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods) diff --git a/tablib/formats/_csv.py b/tablib/formats/_csv.py index 7597395..994b23b 100644 --- a/tablib/formats/_csv.py +++ b/tablib/formats/_csv.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" Tablib - CSV Support. +""" Tablib - *SV Support. """ from tablib.compat import is_py3, csv, StringIO @@ -11,17 +11,18 @@ extensions = ('csv',) DEFAULT_ENCODING = 'utf-8' +DEFAULT_DELIMITER = ',' - -def export_set(dataset): +def export_set(dataset, **kwargs): """Returns CSV representation of Dataset.""" stream = StringIO() - if is_py3: - _csv = csv.writer(stream) - else: - _csv = csv.writer(stream, encoding=DEFAULT_ENCODING) + kwargs.setdefault('delimiter', DEFAULT_DELIMITER) + if not is_py3: + kwargs.setdefault('encoding', DEFAULT_ENCODING) + + _csv = csv.writer(stream, **kwargs) for row in dataset._package(dicts=False): _csv.writerow(row) @@ -29,15 +30,16 @@ def export_set(dataset): return stream.getvalue() -def import_set(dset, in_stream, headers=True): +def import_set(dset, in_stream, headers=True, **kwargs): """Returns dataset from CSV stream.""" dset.wipe() - if is_py3: - rows = csv.reader(StringIO(in_stream)) - else: - rows = csv.reader(StringIO(in_stream), encoding=DEFAULT_ENCODING) + kwargs.setdefault('delimiter', DEFAULT_DELIMITER) + if not is_py3: + kwargs.setdefault('encoding', DEFAULT_ENCODING) + + rows = csv.reader(StringIO(in_stream), **kwargs) for i, row in enumerate(rows): if (i == 0) and (headers): @@ -46,10 +48,10 @@ def import_set(dset, in_stream, headers=True): dset.append(row) -def detect(stream): +def detect(stream, delimiter=DEFAULT_DELIMITER): """Returns True if given stream is valid CSV.""" try: - csv.Sniffer().sniff(stream, delimiters=',') + csv.Sniffer().sniff(stream, delimiters=delimiter) return True except (csv.Error, TypeError): return False diff --git a/tablib/formats/_html.py b/tablib/formats/_html.py index 7bb77f0..0b45f14 100644 --- a/tablib/formats/_html.py +++ b/tablib/formats/_html.py @@ -23,45 +23,45 @@ extensions = ('html', ) def export_set(dataset): - """HTML representation of a Dataset.""" + """HTML representation of a Dataset.""" - stream = StringIO() + stream = StringIO() - page = markup.page() - page.table.open() + page = markup.page() + page.table.open() - if dataset.headers is not None: - new_header = [item if item is not None else '' for item in dataset.headers] + if dataset.headers is not None: + new_header = [item if item is not None else '' for item in dataset.headers] - page.thead.open() - headers = markup.oneliner.th(new_header) - page.tr(headers) - page.thead.close() + page.thead.open() + headers = markup.oneliner.th(new_header) + page.tr(headers) + page.thead.close() - for row in dataset: - new_row = [item if item is not None else '' for item in row] + for row in dataset: + new_row = [item if item is not None else '' for item in row] - html_row = markup.oneliner.td(new_row) - page.tr(html_row) + html_row = markup.oneliner.td(new_row) + page.tr(html_row) - page.table.close() + page.table.close() # Allow unicode characters in output - wrapper = codecs.getwriter("utf8")(stream) - wrapper.writelines(unicode(page)) + wrapper = codecs.getwriter("utf8")(stream) + wrapper.writelines(unicode(page)) - return stream.getvalue().decode('utf-8') + return stream.getvalue().decode('utf-8') def export_book(databook): - """HTML representation of a Databook.""" + """HTML representation of a Databook.""" - stream = StringIO() + stream = StringIO() - for i, dset in enumerate(databook._datasets): - title = (dset.title if dset.title else 'Set %s' % (i)) - stream.write('<%s>%s</%s>\n' % (BOOK_ENDINGS, title, BOOK_ENDINGS)) - stream.write(dset.html) - stream.write('\n') + for i, dset in enumerate(databook._datasets): + title = (dset.title if dset.title else 'Set %s' % (i)) + stream.write('<%s>%s</%s>\n' % (BOOK_ENDINGS, title, BOOK_ENDINGS)) + stream.write(dset.html) + stream.write('\n') - return stream.getvalue() + return stream.getvalue() diff --git a/tablib/formats/_json.py b/tablib/formats/_json.py index 944afe0..777040a 100644 --- a/tablib/formats/_json.py +++ b/tablib/formats/_json.py @@ -13,14 +13,18 @@ title = 'json' extensions = ('json', 'jsn') +def date_handler(obj): + return obj.isoformat() if hasattr(obj, 'isoformat') else obj + + def export_set(dataset): """Returns JSON representation of Dataset.""" - return json.dumps(dataset.dict) + return json.dumps(dataset.dict, default=date_handler) def export_book(databook): """Returns JSON representation of Databook.""" - return json.dumps(databook._package()) + return json.dumps(databook._package(), default=date_handler) def import_set(dset, in_stream): diff --git a/tablib/formats/_latex.py b/tablib/formats/_latex.py new file mode 100644 index 0000000..44ee101 --- /dev/null +++ b/tablib/formats/_latex.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- + +"""Tablib - LaTeX table export support. + + Generates a LaTeX booktabs-style table from the dataset. +""" +import re + +from tablib.compat import unicode + +title = 'latex' +extensions = ('tex',) + +TABLE_TEMPLATE = """\ +%% Note: add \\usepackage{booktabs} to your preamble +%% +\\begin{table}[!htbp] + \\centering + %(CAPTION)s + \\begin{tabular}{%(COLSPEC)s} + \\toprule +%(HEADER)s + %(MIDRULE)s +%(BODY)s + \\bottomrule + \\end{tabular} +\\end{table} +""" + +TEX_RESERVED_SYMBOLS_MAP = dict([ + ('\\', '\\textbackslash{}'), + ('{', '\\{'), + ('}', '\\}'), + ('$', '\\$'), + ('&', '\\&'), + ('#', '\\#'), + ('^', '\\textasciicircum{}'), + ('_', '\\_'), + ('~', '\\textasciitilde{}'), + ('%', '\\%'), +]) + +TEX_RESERVED_SYMBOLS_RE = re.compile( + '(%s)' % '|'.join(map(re.escape, TEX_RESERVED_SYMBOLS_MAP.keys()))) + + +def export_set(dataset): + """Returns LaTeX representation of dataset + + :param dataset: dataset to serialize + :type dataset: tablib.core.Dataset + """ + + caption = '\\caption{%s}' % dataset.title if dataset.title else '%' + colspec = _colspec(dataset.width) + header = _serialize_row(dataset.headers) if dataset.headers else '' + midrule = _midrule(dataset.width) + body = '\n'.join([_serialize_row(row) for row in dataset]) + return TABLE_TEMPLATE % dict(CAPTION=caption, COLSPEC=colspec, + HEADER=header, MIDRULE=midrule, BODY=body) + + +def _colspec(dataset_width): + """Generates the column specification for the LaTeX `tabular` environment + based on the dataset width. + + The first column is justified to the left, all further columns are aligned + to the right. + + .. note:: This is only a heuristic and most probably has to be fine-tuned + post export. Column alignment should depend on the data type, e.g., textual + content should usually be aligned to the left while numeric content almost + always should be aligned to the right. + + :param dataset_width: width of the dataset + """ + + spec = 'l' + for _ in range(1, dataset_width): + spec += 'r' + return spec + + +def _midrule(dataset_width): + """Generates the table `midrule`, which may be composed of several + `cmidrules`. + + :param dataset_width: width of the dataset to serialize + """ + + if not dataset_width or dataset_width == 1: + return '\\midrule' + return ' '.join([_cmidrule(colindex, dataset_width) for colindex in + range(1, dataset_width + 1)]) + + +def _cmidrule(colindex, dataset_width): + """Generates the `cmidrule` for a single column with appropriate trimming + based on the column position. + + :param colindex: Column index + :param dataset_width: width of the dataset + """ + + rule = '\\cmidrule(%s){%d-%d}' + if colindex == 1: + # Rule of first column is trimmed on the right + return rule % ('r', colindex, colindex) + if colindex == dataset_width: + # Rule of last column is trimmed on the left + return rule % ('l', colindex, colindex) + # Inner columns are trimmed on the left and right + return rule % ('lr', colindex, colindex) + + +def _serialize_row(row): + """Returns string representation of a single row. + + :param row: single dataset row + """ + + new_row = [_escape_tex_reserved_symbols(unicode(item)) if item else '' for + item in row] + return 6 * ' ' + ' & '.join(new_row) + ' \\\\' + + +def _escape_tex_reserved_symbols(input): + """Escapes all TeX reserved symbols ('_', '~', etc.) in a string. + + :param input: String to escape + """ + def replace(match): + return TEX_RESERVED_SYMBOLS_MAP[match.group()] + return TEX_RESERVED_SYMBOLS_RE.sub(replace, input) diff --git a/tablib/formats/_tsv.py b/tablib/formats/_tsv.py index 8ef2b67..9380b3b 100644 --- a/tablib/formats/_tsv.py +++ b/tablib/formats/_tsv.py @@ -3,57 +3,28 @@ """ Tablib - TSV (Tab Separated Values) Support. """ -from tablib.compat import is_py3, csv, StringIO - - +from tablib.formats._csv import ( + export_set as export_set_wrapper, + import_set as import_set_wrapper, + detect as detect_wrapper, +) title = 'tsv' extensions = ('tsv',) DEFAULT_ENCODING = 'utf-8' +DELIMITER = '\t' def export_set(dataset): - """Returns a TSV representation of Dataset.""" - - stream = StringIO() - - if is_py3: - _tsv = csv.writer(stream, delimiter='\t') - else: - _tsv = csv.writer(stream, encoding=DEFAULT_ENCODING, delimiter='\t') - - for row in dataset._package(dicts=False): - _tsv.writerow(row) - - return stream.getvalue() + """Returns TSV representation of Dataset.""" + return export_set_wrapper(dataset, delimiter=DELIMITER) def import_set(dset, in_stream, headers=True): """Returns dataset from TSV stream.""" - - dset.wipe() - - if is_py3: - rows = csv.reader(in_stream.splitlines(), delimiter='\t') - else: - rows = csv.reader(in_stream.splitlines(), delimiter='\t', - encoding=DEFAULT_ENCODING) - - for i, row in enumerate(rows): - # Skip empty rows - if not row: - continue - - if (i == 0) and (headers): - dset.headers = row - else: - dset.append(row) + return import_set_wrapper(dset, in_stream, headers=headers, delimiter=DELIMITER) def detect(stream): """Returns True if given stream is valid TSV.""" - try: - csv.Sniffer().sniff(stream, delimiters='\t') - return True - except (csv.Error, TypeError): - return False + return detect_wrapper(stream, delimiter=DELIMITER) diff --git a/tablib/formats/_xls.py b/tablib/formats/_xls.py index 67b87ea..787907a 100644 --- a/tablib/formats/_xls.py +++ b/tablib/formats/_xls.py @@ -5,7 +5,7 @@ import sys -from tablib.compat import BytesIO, xlwt, xlrd, XLRDError +from tablib.compat import BytesIO, xlwt, xlrd, XLRDError, xrange import tablib title = 'xls' diff --git a/tablib/formats/_xlsx.py b/tablib/formats/_xlsx.py index 57058c8..d9d3d57 100644 --- a/tablib/formats/_xlsx.py +++ b/tablib/formats/_xlsx.py @@ -33,21 +33,21 @@ def detect(stream): except openpyxl.shared.exc.InvalidFileException: pass -def export_set(dataset): +def export_set(dataset, freeze_panes=True): """Returns XLSX representation of Dataset.""" wb = Workbook() ws = wb.worksheets[0] ws.title = dataset.title if dataset.title else 'Tablib Dataset' - dset_sheet(dataset, ws) + dset_sheet(dataset, ws, freeze_panes=freeze_panes) stream = BytesIO() wb.save(stream) return stream.getvalue() -def export_book(databook): +def export_book(databook, freeze_panes=True): """Returns XLSX representation of DataBook.""" wb = Workbook() @@ -56,7 +56,7 @@ def export_book(databook): ws = wb.create_sheet() ws.title = dset.title if dset.title else 'Sheet%s' % (i) - dset_sheet(dset, ws) + dset_sheet(dset, ws, freeze_panes=freeze_panes) stream = BytesIO() @@ -69,7 +69,7 @@ def import_set(dset, in_stream, headers=True): dset.wipe() - xls_book = openpyxl.reader.excel.load_workbook(in_stream) + xls_book = openpyxl.reader.excel.load_workbook(BytesIO(in_stream)) sheet = xls_book.get_active_sheet() dset.title = sheet.title @@ -87,7 +87,7 @@ def import_book(dbook, in_stream, headers=True): dbook.wipe() - xls_book = openpyxl.reader.excel.load_workbook(in_stream) + xls_book = openpyxl.reader.excel.load_workbook(BytesIO(in_stream)) for sheet in xls_book.worksheets: data = tablib.Dataset() @@ -103,7 +103,7 @@ def import_book(dbook, in_stream, headers=True): dbook.add_sheet(data) -def dset_sheet(dataset, ws): +def dset_sheet(dataset, ws, freeze_panes=True): """Completes given worksheet from given Dataset.""" _package = dataset._package(dicts=False) @@ -123,8 +123,9 @@ def dset_sheet(dataset, ws): ws.cell('%s%s'%(col_idx, row_number)).value = unicode(col) style = ws.get_style('%s%s' % (col_idx, row_number)) style.font.bold = True - ws.freeze_panes = 'A2' - + if freeze_panes: + # We want to freeze the column after the last column + ws.freeze_panes = '%s%s' % (frzn_col_idx, row_number) # bold separators elif len(row) < dataset.width: diff --git a/test_tablib.py b/test_tablib.py index a1f581c..3830dda 100755 --- a/test_tablib.py +++ b/test_tablib.py @@ -319,6 +319,67 @@ class TablibTestCase(unittest.TestCase): self.assertEqual(html, d.html) + def test_latex_export(self): + """LaTeX export""" + + expected = """\ +% Note: add \\usepackage{booktabs} to your preamble +% +\\begin{table}[!htbp] + \\centering + \\caption{Founders} + \\begin{tabular}{lrr} + \\toprule + first\\_name & last\\_name & gpa \\\\ + \\cmidrule(r){1-1} \\cmidrule(lr){2-2} \\cmidrule(l){3-3} + John & Adams & 90 \\\\ + George & Washington & 67 \\\\ + Thomas & Jefferson & 50 \\\\ + \\bottomrule + \\end{tabular} +\\end{table} +""" + output = self.founders.latex + self.assertEqual(output, expected) + + + def test_latex_export_empty_dataset(self): + self.assertTrue(tablib.Dataset().latex is not None) + + + def test_latex_export_no_headers(self): + d = tablib.Dataset() + d.append(('one', 'two', 'three')) + self.assertTrue('one' in d.latex) + + + def test_latex_export_caption(self): + d = tablib.Dataset() + d.append(('one', 'two', 'three')) + self.assertFalse('caption' in d.latex) + + d.title = 'Title' + self.assertTrue('\\caption{Title}' in d.latex) + + + def test_latex_export_none_values(self): + headers = ['foo', None, 'bar'] + d = tablib.Dataset(['foo', None, 'bar'], headers=headers) + output = d.latex + self.assertTrue('foo' in output) + self.assertFalse('None' in output) + + + def test_latex_escaping(self): + d = tablib.Dataset(['~', '^']) + output = d.latex + + self.assertFalse('~' in output) + self.assertTrue('textasciitilde' in output) + self.assertFalse('^' in output) + self.assertTrue('textasciicircum' in output) + + def test_unicode_append(self): """Passes in a single unicode character and exports.""" @@ -338,6 +399,7 @@ class TablibTestCase(unittest.TestCase): data.xlsx data.ods data.html + data.latex def test_book_export_no_exceptions(self): @@ -419,6 +481,17 @@ class TablibTestCase(unittest.TestCase): self.assertEqual(_csv, data.csv) + def test_csv_import_set_semicolons(self): + """Test for proper output with semicolon separated CSV.""" + data.append(self.john) + data.append(self.george) + data.headers = self.headers + + _csv = data.get_csv(delimiter=';') + + data.set_csv(_csv, delimiter=';') + + self.assertEqual(_csv, data.get_csv(delimiter=';')) def test_csv_import_set_with_spaces(self): """Generate and import CSV set serialization when row values have @@ -433,6 +506,19 @@ class TablibTestCase(unittest.TestCase): self.assertEqual(_csv, data.csv) + def test_csv_import_set_semicolon_with_spaces(self): + """Generate and import semicolon separated CSV set serialization when row values have + spaces.""" + data.append(('Bill Gates', 'Microsoft')) + data.append(('Steve Jobs', 'Apple')) + data.headers = ('Name', 'Company') + + _csv = data.get_csv(delimiter=';') + + data.set_csv(_csv, delimiter=';') + + self.assertEqual(_csv, data.get_csv(delimiter=';')) + def test_csv_import_set_with_newlines(self): """Generate and import CSV set serialization when row values have @@ -444,7 +530,6 @@ class TablibTestCase(unittest.TestCase): data.headers = ('title', 'body') _csv = data.csv - data.csv = _csv self.assertEqual(_csv, data.csv) @@ -709,6 +794,25 @@ class TablibTestCase(unittest.TestCase): self.assertEqual(third_row, expected_third) + def test_remove_duplicates(self): + """Unique Rows.""" + + self.founders.append(self.john) + self.founders.append(self.george) + self.founders.append(self.tom) + self.assertEqual(self.founders[0], self.founders[3]) + self.assertEqual(self.founders[1], self.founders[4]) + self.assertEqual(self.founders[2], self.founders[5]) + self.assertEqual(self.founders.height, 6) + + self.founders.remove_duplicates() + + self.assertEqual(self.founders[0], self.john) + self.assertEqual(self.founders[1], self.george) + self.assertEqual(self.founders[2], self.tom) + self.assertEqual(self.founders.height, 3) + + def test_wipe(self): """Purge a dataset.""" @@ -726,6 +830,26 @@ class TablibTestCase(unittest.TestCase): self.assertTrue(data[0] == new_row) + def test_subset(self): + """Create a subset of a dataset""" + + rows = (0, 2) + columns = ('first_name','gpa') + + data.headers = self.headers + + data.append(self.john) + data.append(self.george) + data.append(self.tom) + + #Verify data is truncated + subset = data.subset(rows=rows, cols=columns) + self.assertEqual(type(subset), tablib.Dataset) + self.assertEqual(subset.headers, list(columns)) + self.assertEqual(subset._data[0].list, ['John', 90]) + self.assertEqual(subset._data[1].list, ['Thomas', 50]) + + def test_formatters(self): """Confirm formatters are being triggered.""" @@ -795,18 +919,7 @@ class TablibTestCase(unittest.TestCase): # add another entry to test right field width for # integer self.founders.append(('Old', 'Man', 100500)) - - self.assertEqual( - """ -first_name|last_name |gpa -----------|----------|------ -John |Adams |90 -George |Washington|67 -Thomas |Jefferson |50 -Old |Man |100500 -""".strip(), - unicode(self.founders) - ) + self.assertEqual('first_name|last_name |gpa ', unicode(self.founders).split('\n')[0]) def test_databook_add_sheet_accepts_only_dataset_instances(self): @@ -834,5 +947,30 @@ Old |Man |100500 except tablib.InvalidDatasetType: self.fail("Subclass of tablib.Dataset should be accepted by Databook.add_sheet") + + def test_csv_formatter_support_kwargs(self): + """Test CSV import and export with formatter configuration.""" + data.append(self.john) + data.append(self.george) + data.headers = self.headers + + expected = 'first_name;last_name;gpa\nJohn;Adams;90\nGeorge;Washington;67\n' + + kwargs = dict(delimiter=';', lineterminator='\n') + _csv = data.export('csv', **kwargs) + self.assertEqual(expected, _csv) + + # the import works but consider default delimiter=',' + d1 = tablib.import_set(_csv, format="csv") + self.assertEqual(1, len(d1.headers)) + + d2 = tablib.import_set(_csv, format="csv", **kwargs) + self.assertEqual(3, len(d2.headers)) + + def test_databook_formatter_support_kwargs(self): + """Test XLSX export with formatter configuration.""" + self.founders.export('xlsx', freeze_panes=False) + + if __name__ == '__main__': unittest.main() @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- - -import tablib - -d = tablib.Dataset() - -with open('/Users/kreitz/Desktop/test.json') as f: - d.json = f.read() - -# del d[900:] - -# print d.height - -print len(d.ods) - |