path: root/src/lxml/parser.pxi

# Parsers for XML and HTML

cimport xmlparser
cimport htmlparser

class ParseError(LxmlSyntaxError):
    """Syntax error while parsing an XML document.

    For compatibility with ElementTree 1.3 and later.
    """
    pass

class XMLSyntaxError(ParseError):
    """Syntax error while parsing an XML document.
    """
    pass

class ParserError(LxmlError):
    """Internal lxml parser error.
    """
    pass

ctypedef enum LxmlParserType:
    LXML_XML_PARSER
    LXML_HTML_PARSER
    LXML_ITERPARSE_PARSER

cdef class _ParserDictionaryContext:
    # Global parser context to share the string dictionary.
    #
    # This class is a delegate singleton!
    #
    # It creates _ParserDictionaryContext objects for each thread to keep thread state,
    # but those must never be used directly.  Always stick to using the static
    # __GLOBAL_PARSER_CONTEXT as defined below the class.
    #

    cdef tree.xmlDict* _c_dict
    cdef _BaseParser _default_parser
    def __dealloc__(self):
        if self._c_dict is not NULL:
            xmlparser.xmlDictFree(self._c_dict)

    cdef void initMainParserContext(self):
        """Put the global context into the thread dictionary of the main
        thread.  To be called once and only in the main thread."""
        cdef python.PyObject* thread_dict
        cdef python.PyObject* result
        thread_dict = python.PyThreadState_GetDict()
        if thread_dict is not NULL:
            python.PyDict_SetItem(<object>thread_dict, "_ParserDictionaryContext", self)

    cdef _ParserDictionaryContext _findThreadParserContext(self):
        "Find (or create) the _ParserDictionaryContext object for the current thread"
        cdef python.PyObject* thread_dict
        cdef python.PyObject* result
        cdef _ParserDictionaryContext context
        thread_dict = python.PyThreadState_GetDict()
        if thread_dict is NULL:
            return self
        d = <object>thread_dict
        result = python.PyDict_GetItem(d, "_ParserDictionaryContext")
        if result is not NULL:
            return <object>result
        context = _ParserDictionaryContext()
        python.PyDict_SetItem(d, "_ParserDictionaryContext", context)
        return context

    cdef void setDefaultParser(self, _BaseParser parser):
        "Set the default parser for the current thread"
        cdef _ParserDictionaryContext context
        context = self._findThreadParserContext()
        context._default_parser = parser

    cdef _BaseParser getDefaultParser(self):
        "Return (or create) the default parser of the current thread"
        cdef _ParserDictionaryContext context
        context = self._findThreadParserContext()
        if context._default_parser is None:
            if self._default_parser is None:
                self._default_parser = __DEFAULT_XML_PARSER._copy()
            if context is not self:
                context._default_parser = self._default_parser._copy()
        return context._default_parser

    cdef tree.xmlDict* _getThreadDict(self, tree.xmlDict* default):
        "Return the thread-local dict or create a new one if necessary."
        cdef _ParserDictionaryContext context
        context = self._findThreadParserContext()
        if context._c_dict is NULL:
            # thread dict not yet set up => use default or create a new one
            if default is not NULL:
                context._c_dict = default
                xmlparser.xmlDictReference(default)
                return default
            if self._c_dict is NULL:
                self._c_dict = xmlparser.xmlDictCreate()
            if context is not self:
                context._c_dict = xmlparser.xmlDictCreateSub(self._c_dict)
        return context._c_dict

    cdef void initThreadDictRef(self, tree.xmlDict** c_dict_ref):
        cdef tree.xmlDict* c_dict
        cdef tree.xmlDict* c_thread_dict
        c_dict = c_dict_ref[0]
        c_thread_dict = self._getThreadDict(c_dict)
        if c_dict is c_thread_dict:
            return
        if c_dict is not NULL:
            xmlparser.xmlDictFree(c_dict)
        c_dict_ref[0] = c_thread_dict
        xmlparser.xmlDictReference(c_thread_dict)

    cdef void initParserDict(self, xmlparser.xmlParserCtxt* pctxt):
        "Assure we always use the same string dictionary."
        self.initThreadDictRef(&pctxt.dict)

    cdef void initXPathParserDict(self, xpath.xmlXPathContext* pctxt):
        "Assure we always use the same string dictionary."
        self.initThreadDictRef(&pctxt.dict)

    cdef void initDocDict(self, xmlDoc* result):
        "Store dict of last object parsed if no shared dict yet"
        # XXX We also free the result dict here if there already was one.
        # This case should only occur for new documents with empty dicts,
        # otherwise we'd free data that's in use => segfault
        self.initThreadDictRef(&result.dict)

cdef _ParserDictionaryContext __GLOBAL_PARSER_CONTEXT
__GLOBAL_PARSER_CONTEXT = _ParserDictionaryContext()
__GLOBAL_PARSER_CONTEXT.initMainParserContext()

cdef int _checkThreadDict(tree.xmlDict* c_dict):
    """Check that c_dict is either the local thread dictionary or the global
    parent dictionary.
    """
    if __GLOBAL_PARSER_CONTEXT._c_dict is c_dict:
        return 1 # main thread
    if __GLOBAL_PARSER_CONTEXT._getThreadDict(NULL) is c_dict:
        return 1 # local thread dict
    return 0

############################################################
## support for Python unicode I/O
############################################################

# name of Python unicode encoding as known to libxml2
cdef char* _UNICODE_ENCODING
_UNICODE_ENCODING = NULL

cdef void _setupPythonUnicode():
    """Sets _UNICODE_ENCODING to the internal encoding name of Python unicode
    strings if libxml2 supports reading native Python unicode.  This depends
    on iconv and the local Python installation, so we simply check if we find
    a matching encoding handler.
    """
    cdef tree.xmlCharEncodingHandler* enchandler
    cdef Py_ssize_t l
    cdef char* buffer
    cdef char* enc
    utext = python.PyUnicode_DecodeUTF8("<test/>", 7, 'strict')
    l = python.PyUnicode_GET_DATA_SIZE(utext)
    buffer = python.PyUnicode_AS_DATA(utext)
    enc = _findEncodingName(buffer, l)
    if enc == NULL:
        # apparently, libxml2 can't detect UTF-16 on some systems
        if l >= 4 and \
               buffer[0] == c'<' and buffer[1] == c'\0' and \
               buffer[2] == c't' and buffer[3] == c'\0':
            enc = "UTF-16LE"
        elif l >= 4 and \
               buffer[0] == c'\0' and buffer[1] == c'<' and \
               buffer[2] == c'\0' and buffer[3] == c't':
            enc = "UTF-16BE"
        else:
            # not my fault, it's YOUR broken system :)
            return
    enchandler = tree.xmlFindCharEncodingHandler(enc)
    if enchandler is not NULL:
        global _UNICODE_ENCODING
        tree.xmlCharEncCloseFunc(enchandler)
        _UNICODE_ENCODING = enc

cdef char* _findEncodingName(char* buffer, int size):
    "Work around bug in libxml2: find iconv name of encoding on our own."
    cdef tree.xmlCharEncoding enc
    enc = tree.xmlDetectCharEncoding(buffer, size)
    if enc == tree.XML_CHAR_ENCODING_UTF16LE:
        return "UTF-16LE"
    elif enc == tree.XML_CHAR_ENCODING_UTF16BE:
        return "UTF-16BE"
    elif enc == tree.XML_CHAR_ENCODING_UCS4LE:
        return "UCS-4LE"
    elif enc == tree.XML_CHAR_ENCODING_UCS4BE:
        return "UCS-4BE"
    elif enc == tree.XML_CHAR_ENCODING_NONE:
        return NULL
    else:
        return tree.xmlGetCharEncodingName(enc)

_setupPythonUnicode()

############################################################
## support for file-like objects
############################################################

cdef class _FileReaderContext:
    cdef object _filelike
    cdef object _url
    cdef object _bytes
    cdef _ExceptionContext _exc_context
    cdef cstd.size_t _bytes_read
    cdef char* _c_url
    def __init__(self, filelike, exc_context, url=None):
        self._exc_context = exc_context
        self._filelike = filelike
        self._url = url
        if url is None:
            self._c_url = NULL
        else:
            self._c_url = _cstr(url)
        self._bytes  = ''
        self._bytes_read = 0

    cdef xmlparser.xmlParserInput* _createParserInput(
            self, xmlparser.xmlParserCtxt* ctxt):
        cdef xmlparser.xmlParserInputBuffer* c_buffer
        c_buffer = xmlparser.xmlAllocParserInputBuffer(0)
        c_buffer.context = <python.PyObject*>self
        c_buffer.readcallback = _readFilelikeParser
        return xmlparser.xmlNewIOInputStream(ctxt, c_buffer, 0)

    cdef xmlDoc* _readDoc(self, xmlparser.xmlParserCtxt* ctxt, int options,
                          LxmlParserType parser_type):
        cdef python.PyThreadState* state
        cdef xmlDoc* result
        state = python.PyEval_SaveThread()
        if parser_type == LXML_XML_PARSER:
            result = xmlparser.xmlCtxtReadIO(
                ctxt, _readFilelikeParser, NULL, <python.PyObject*>self,
                self._c_url, NULL, options)
        else:
            result = htmlparser.htmlCtxtReadIO(
                ctxt, _readFilelikeParser, NULL, <python.PyObject*>self,
                self._c_url, NULL, options)
        python.PyEval_RestoreThread(state)
        return result

    cdef tree.xmlDtd* _readDtd(self):
        cdef python.PyThreadState* state
        cdef tree.xmlDtd* result
        cdef xmlparser.xmlParserInputBuffer* c_buffer
        c_buffer = xmlparser.xmlAllocParserInputBuffer(0)
        c_buffer.context = <python.PyObject*>self
        c_buffer.readcallback = _readFilelikeParser
        state = python.PyEval_SaveThread()
        result = xmlparser.xmlIOParseDTD(NULL, c_buffer, 0)
        python.PyEval_RestoreThread(state)
        return result

    cdef int copyToBuffer(self, char* c_buffer, int c_size):
        cdef char* c_start
        cdef Py_ssize_t byte_count, remaining
        if self._bytes_read < 0:
            return 0
        try:
            byte_count = python.PyString_GET_SIZE(self._bytes)
            remaining = byte_count - self._bytes_read
            if remaining <= 0:
                self._bytes = self._filelike.read(c_size)
                if not python.PyString_Check(self._bytes):
                    raise TypeError, "reading file objects must return plain strings"
                remaining = python.PyString_GET_SIZE(self._bytes)
                self._bytes_read = 0
                if remaining == 0:
                    self._bytes_read = -1
                    return 0
            if c_size > remaining:
                c_size = remaining
            c_start = _cstr(self._bytes) + self._bytes_read
            self._bytes_read = self._bytes_read + c_size
            cstd.memcpy(c_buffer, c_start, c_size)
            return c_size
        except:
            self._exc_context._store_raised()
            return -1

cdef int _readFilelikeParser(void* ctxt, char* c_buffer, int c_size) with GIL:
    return (<_FileReaderContext>ctxt).copyToBuffer(c_buffer, c_size)

############################################################
## support for custom document loaders
############################################################

cdef  xmlparser.xmlParserInput* _parser_resolve_from_python(
    char* c_url, char* c_pubid, xmlparser.xmlParserCtxt* c_context,
    int* error) with GIL:
    # call the Python document loaders
    cdef xmlparser.xmlParserInput* c_input
    cdef _ResolverContext context
    cdef _InputDocument   doc_ref
    cdef _FileReaderContext file_context
    error[0] = 0
    context = <_ResolverContext>c_context._private
    try:
        if c_url is NULL:
            url = None
        elif c_context.myDoc is NULL or c_context.myDoc.URL is NULL:
            # parsing a main document, so URL was passed verbatimly by user
            url = c_url
        else:
            # parsing a related document (DTD etc.) => UTF-8 encoded URL
            url = funicode(c_url)
        if c_pubid is NULL:
            pubid = None
        else:
            pubid = funicode(c_pubid) # always UTF-8

        doc_ref = context._resolvers.resolve(url, pubid, context)
        if doc_ref is None:
            return NULL
    except:
        context._store_raised()
        error[0] = 1
        return NULL

    c_input = NULL
    data = None
    if doc_ref._type == PARSER_DATA_STRING:
        data = doc_ref._data_bytes
        c_input = xmlparser.xmlNewStringInputStream(
            c_context, _cstr(data))
    elif doc_ref._type == PARSER_DATA_FILENAME:
        c_input = xmlparser.xmlNewInputFromFile(
            c_context, _cstr(doc_ref._data_bytes))
    elif doc_ref._type == PARSER_DATA_FILE:
        file_context = _FileReaderContext(doc_ref._file, context, url)
        c_input = file_context._createParserInput(c_context)
        data = file_context

    if data is not None:
        context._storage.add(data)
    return c_input

cdef xmlparser.xmlParserInput* _local_resolver(char* c_url, char* c_pubid,
                                               xmlparser.xmlParserCtxt* c_context):
    # no Python objects here, may be called without thread context !
    # when we declare a Python object, Pyrex will INCREF(None) !
    cdef xmlparser.xmlParserInput* c_input
    cdef int error
    if c_context._private is NULL:
        if __DEFAULT_ENTITY_LOADER is NULL:
            return NULL
        return __DEFAULT_ENTITY_LOADER(c_url, c_pubid, c_context)

    c_input = _parser_resolve_from_python(c_url, c_pubid, c_context, &error)

    if c_input is not NULL:
        return c_input
    if error:
        return NULL
    if __DEFAULT_ENTITY_LOADER is NULL:
        return NULL
    return __DEFAULT_ENTITY_LOADER(c_url, c_pubid, c_context)

cdef xmlparser.xmlExternalEntityLoader __DEFAULT_ENTITY_LOADER
__DEFAULT_ENTITY_LOADER = xmlparser.xmlGetExternalEntityLoader()

xmlparser.xmlSetExternalEntityLoader(_local_resolver)

############################################################
## Parsers
############################################################

cdef class _ParserContext(_ResolverContext)
cdef class _TargetParserContext(_ParserContext)

cdef class _ParserContext(_ResolverContext):
    cdef _ErrorLog _error_log
    cdef xmlparser.xmlParserCtxt* _c_ctxt

    cdef _ParserContext _copy(self):
        cdef _ParserContext context
        context = self.__class__()
        _initParserContext(context, self._resolvers._copy(), NULL)
        return context

    cdef void _initParserContext(self, xmlparser.xmlParserCtxt* c_ctxt):
        self._c_ctxt = c_ctxt
        c_ctxt._private = <void*>self

    cdef object _handleParseResult(self, _BaseParser parser,
                                   xmlDoc* result, filename):
        cdef xmlDoc* c_doc
        cdef int recover
        recover = parser._parse_options & xmlparser.XML_PARSE_RECOVER
        c_doc = _handleParseResult(self, self._c_ctxt, result,
                                   filename, recover)
        return _documentFactory(c_doc, parser)

    cdef xmlDoc* _handleParseResultDoc(self, _BaseParser parser,
                                       xmlDoc* result, filename) except NULL:
        cdef int recover
        recover = parser._parse_options & xmlparser.XML_PARSE_RECOVER
        return _handleParseResult(self, self._c_ctxt, result,
                                   filename, recover)

cdef _initParserContext(_ParserContext context,
                        _ResolverRegistry resolvers,
                        xmlparser.xmlParserCtxt* c_ctxt):
    _initResolverContext(context, resolvers)
    if c_ctxt is not NULL:
        context._initParserContext(c_ctxt)
    context._error_log = _ErrorLog()


cdef int _raiseParseError(xmlparser.xmlParserCtxt* ctxt, filename,
                          _ErrorLog error_log) except 0:
    if filename is not None and \
           ctxt.lastError.domain == xmlerror.XML_FROM_IO:
        if ctxt.lastError.message is not NULL:
            message = "Error reading file '%s': %s" % (
                filename, (ctxt.lastError.message).strip())
        else:
            message = "Error reading file '%s'" % filename
        raise IOError, message
    elif error_log:
        raise XMLSyntaxError, error_log._buildExceptionMessage(
            "Document is not well formed")
    elif ctxt.lastError.message is not NULL:
        message = (ctxt.lastError.message).strip()
        if ctxt.lastError.line > 0:
            message = "line %d: %s" % (ctxt.lastError.line, message)
        raise XMLSyntaxError, message
    else:
        raise XMLSyntaxError

cdef xmlDoc* _handleParseResult(_ParserContext context,
                                xmlparser.xmlParserCtxt* c_ctxt,
                                xmlDoc* result, filename,
                                int recover) except NULL:
    cdef int well_formed
    if c_ctxt.myDoc is not NULL:
        if c_ctxt.myDoc != result:
            tree.xmlFreeDoc(c_ctxt.myDoc)
        c_ctxt.myDoc = NULL

    if result is not NULL:
        if recover or (c_ctxt.wellFormed and \
                       c_ctxt.lastError.level < xmlerror.XML_ERR_ERROR):
            well_formed = 1
        elif not c_ctxt.replaceEntities and not c_ctxt.validate \
                 and context is not None:
            # in this mode, we ignore errors about undefined entities
            for error in context._error_log.filter_from_errors():
                if error.type != ErrorTypes.WAR_UNDECLARED_ENTITY and \
                       error.type != ErrorTypes.ERR_UNDECLARED_ENTITY:
                    well_formed = 0
                    break
            else:
                well_formed = 1
        else:
            well_formed = 0

        if well_formed:
            __GLOBAL_PARSER_CONTEXT.initDocDict(result)
        else:
            # free broken document
            tree.xmlFreeDoc(result)
            result = NULL

    if context is not None and context._has_raised():
        if result is not NULL:
            tree.xmlFreeDoc(result)
            result = NULL
        context._raise_if_stored()

    if result is NULL:
        if context is not None:
            _raiseParseError(c_ctxt, filename, context._error_log)
        else:
            _raiseParseError(c_ctxt, filename, None)
    elif result.URL is NULL and filename is not None:
        result.URL = tree.xmlStrdup(_cstr(filename))
    return result


cdef class _BaseParser:
    cdef int _parse_options
    cdef _ParserContext _context
    cdef LxmlParserType _parser_type
    cdef xmlparser.xmlParserCtxt* _parser_ctxt
    cdef ElementClassLookup _class_lookup
    cdef python.PyThread_type_lock _parser_lock
    cdef int _feed_parser_running

    def __init__(self, int parse_options, remove_comments, remove_pis,
                 target):
        cdef xmlparser.xmlParserCtxt* pctxt
        if isinstance(self, HTMLParser):
            self._parser_type = LXML_HTML_PARSER
        elif isinstance(self, XMLParser):
            self._parser_type = LXML_XML_PARSER
        elif isinstance(self, iterparse):
            self._parser_type = LXML_ITERPARSE_PARSER
        else:
            raise TypeError, "This class cannot be instantiated"

        self._parse_options = parse_options

        pctxt = self._newParserCtxt()
        self._parser_ctxt = pctxt
        if pctxt is NULL:
            python.PyErr_NoMemory()

        self._context = self._createContext(target)
        _initParserContext(self._context, None, pctxt)

        if remove_comments:
            pctxt.sax.comment = NULL
        if remove_pis:
            pctxt.sax.processingInstruction = NULL
        # hard switch-off for CDATA nodes => makes them plain text
        pctxt.sax.cdataBlock = NULL

        if not config.ENABLE_THREADING or \
               self._parser_type == LXML_ITERPARSE_PARSER:
            # no threading
            self._parser_lock = NULL
        else:
            self._parser_lock = python.PyThread_allocate_lock()

    cdef _ParserContext _createContext(self, target):
        cdef _TargetParserContext context
        if target is None:
            return _ParserContext()
        context = _TargetParserContext()
        context._setTarget(target)
        return context

    cdef xmlparser.xmlParserCtxt* _newParserCtxt(self):
        if self._parser_type == LXML_HTML_PARSER:
            return htmlparser.htmlCreateMemoryParserCtxt('dummy', 5)
        else:
            return xmlparser.xmlNewParserCtxt()

    def __dealloc__(self):
        if self._parser_ctxt is not NULL:
            xmlparser.xmlFreeParserCtxt(self._parser_ctxt)
        if self._parser_lock is not NULL:
            python.PyThread_free_lock(self._parser_lock)

    cdef void _cleanup(self):
        cdef xmlparser.xmlParserCtxt* pctxt
        pctxt = self._parser_ctxt
        if pctxt is not NULL:
            if pctxt.spaceTab is not NULL: # work around bug in libxml2
                xmlparser.xmlClearParserCtxt(pctxt)

    cdef int _lockParser(self) except -1:
        cdef python.PyThreadState* state
        cdef int result
        if config.ENABLE_THREADING and self._parser_lock != NULL:
            state = python.PyEval_SaveThread()
            result = python.PyThread_acquire_lock(
                self._parser_lock, python.WAIT_LOCK)
            python.PyEval_RestoreThread(state)
            if result == 0:
                raise ParserError, "parser locking failed"
        return 0

    cdef void _unlockParser(self):
        if config.ENABLE_THREADING and self._parser_lock != NULL:
            python.PyThread_release_lock(self._parser_lock)

    property error_log:
        def __get__(self):
            return self._context._error_log.copy()

    property resolvers:
        def __get__(self):
            return self._context._resolvers

    def setElementClassLookup(self, ElementClassLookup lookup = None):
        "Deprecated, use ``parser.set_element_class_lookup(lookup)`` instead."
        self.set_element_class_lookup(lookup)

    def set_element_class_lookup(self, ElementClassLookup lookup = None):
        """Set a lookup scheme for element classes generated from this parser.

        Reset it by passing None or nothing.
        """
        self._class_lookup = lookup

    cdef _BaseParser _copy(self):
        "Create a new parser with the same configuration."
        cdef _BaseParser parser
        parser = self.__class__()
        parser._parse_options = self._parse_options
        parser._class_lookup  = self._class_lookup
        parser._context = self._context._copy()
        parser._context._initParserContext(parser._parser_ctxt)
        return parser

    def copy(self):
        "Create a new parser with the same configuration."
        return self._copy()

    def makeelement(self, _tag, attrib=None, nsmap=None, **_extra):
        """Creates a new element associated with this parser.
        """
        return _makeElement(_tag, NULL, None, self, None, None,
                            attrib, nsmap, _extra)

    property version:
        "The version of the underlying XML parser."
        def __get__(self):
            return "libxml2 %d.%d.%d" % LIBXML_VERSION

    # internal parser methods

    cdef xmlDoc* _parseUnicodeDoc(self, utext, char* c_filename) except NULL:
        """Parse unicode document, share dictionary if possible.
        """
        cdef python.PyThreadState* state
        cdef xmlDoc* result
        cdef xmlparser.xmlParserCtxt* pctxt
        cdef int recover
        cdef Py_ssize_t py_buffer_len
        cdef int buffer_len
        cdef char* c_text
        py_buffer_len = python.PyUnicode_GET_DATA_SIZE(utext)
        if py_buffer_len > python.INT_MAX or _UNICODE_ENCODING is NULL:
            text_utf = python.PyUnicode_AsUTF8String(utext)
            py_buffer_len = python.PyString_GET_SIZE(text_utf)
            return self._parseDoc(_cstr(text_utf), py_buffer_len, c_filename)
        buffer_len = py_buffer_len

        self._lockParser()
        self._context._error_log.connect()
        try:
            pctxt = self._parser_ctxt
            __GLOBAL_PARSER_CONTEXT.initParserDict(pctxt)

            c_text = python.PyUnicode_AS_DATA(utext)
            state = python.PyEval_SaveThread()
            if self._parser_type == LXML_HTML_PARSER:
                result = htmlparser.htmlCtxtReadMemory(
                    pctxt, c_text, buffer_len, c_filename, _UNICODE_ENCODING,
                    self._parse_options)
            else:
                result = xmlparser.xmlCtxtReadMemory(
                    pctxt, c_text, buffer_len, c_filename, _UNICODE_ENCODING,
                    self._parse_options)
            python.PyEval_RestoreThread(state)

            return self._context._handleParseResultDoc(self, result, None)
        finally:
            self._cleanup()
            self._context.clear()
            self._context._error_log.disconnect()
            self._unlockParser()

    cdef xmlDoc* _parseDoc(self, char* c_text, Py_ssize_t c_len,
                           char* c_filename) except NULL:
        """Parse document, share dictionary if possible.
        """
        cdef python.PyThreadState* state
        cdef xmlDoc* result
        cdef xmlparser.xmlParserCtxt* pctxt
        cdef int recover
        if c_len > python.INT_MAX:
            raise ParserError, "string is too long to parse it with libxml2"
        self._lockParser()
        self._context._error_log.connect()
        try:
            pctxt = self._parser_ctxt
            __GLOBAL_PARSER_CONTEXT.initParserDict(pctxt)

            state = python.PyEval_SaveThread()
            if self._parser_type == LXML_HTML_PARSER:
                result = htmlparser.htmlCtxtReadMemory(
                    pctxt, c_text, c_len, c_filename, NULL, self._parse_options)
            else:
                result = xmlparser.xmlCtxtReadMemory(
                    pctxt, c_text, c_len, c_filename, NULL, self._parse_options)
            python.PyEval_RestoreThread(state)

            return self._context._handleParseResultDoc(self, result, None)
        finally:
            self._cleanup()
            self._context.clear()
            self._context._error_log.disconnect()
            self._unlockParser()

    cdef xmlDoc* _parseDocFromFile(self, char* c_filename) except NULL:
        cdef python.PyThreadState* state
        cdef xmlDoc* result
        cdef xmlparser.xmlParserCtxt* pctxt
        cdef int recover
        cdef int orig_options
        result = NULL
        self._lockParser()
        self._context._error_log.connect()
        try:
            pctxt = self._parser_ctxt
            __GLOBAL_PARSER_CONTEXT.initParserDict(pctxt)

            orig_options = pctxt.options
            state = python.PyEval_SaveThread()
            if self._parser_type == LXML_HTML_PARSER:
                result = htmlparser.htmlCtxtReadFile(
                    pctxt, c_filename, NULL, self._parse_options)
            else:
                result = xmlparser.xmlCtxtReadFile(
                    pctxt, c_filename, NULL, self._parse_options)
            python.PyEval_RestoreThread(state)
            pctxt.options = orig_options # work around libxml2 problem

            return self._context._handleParseResultDoc(
                self, result, c_filename)
        finally:
            self._cleanup()
            self._context.clear()
            self._context._error_log.disconnect()
            self._unlockParser()

    cdef xmlDoc* _parseDocFromFilelike(self, filelike, filename) except NULL:
        cdef _FileReaderContext file_context
        cdef xmlDoc* result
        cdef xmlparser.xmlParserCtxt* pctxt
        cdef char* c_filename
        cdef int recover
        if not filename:
            filename = None
        self._lockParser()
        self._context._error_log.connect()
        try:
            pctxt = self._parser_ctxt
            __GLOBAL_PARSER_CONTEXT.initParserDict(pctxt)
            file_context = _FileReaderContext(filelike, self._context, filename)
            result = file_context._readDoc(
                pctxt, self._parse_options, self._parser_type)

            return self._context._handleParseResultDoc(
                self, result, filename)
        finally:
            self._cleanup()
            self._context.clear()
            self._context._error_log.disconnect()
            self._unlockParser()

############################################################
## ET feed parser
############################################################

cdef class _FeedParser(_BaseParser):
    def feed(self, data):
        """Feeds data to the parser.  The argument should be an 8-bit string
        buffer containing encoded data, although Unicode is supported as long
        as both string types are not mixed.

        This is the main entry point to the consumer interface of a parser.
        The parser will parse as much of the XML stream as it can on each
        call.  To finish parsing, call the ``close()`` method.

        It is not possible to use the parser in any other way after calling
        the ``feed()`` method.  The parser can only be reset by calling
        ``close()``.
        """
        cdef xmlparser.xmlParserCtxt* pctxt
        cdef Py_ssize_t py_buffer_len
        cdef char* c_data
        cdef char* c_encoding
        cdef int buffer_len
        cdef int error
        cdef int recover
        if python.PyString_Check(data):
            c_encoding = NULL
            c_data = _cstr(data)
            py_buffer_len = python.PyString_GET_SIZE(data)
        elif python.PyUnicode_Check(data):
            if _UNICODE_ENCODING is NULL:
                raise ParserError, \
                      "Unicode parsing is not supported on this platform"
            c_encoding = _UNICODE_ENCODING
            c_data = python.PyUnicode_AS_DATA(data)
            py_buffer_len = python.PyUnicode_GET_DATA_SIZE(data)
        else:
            raise TypeError, "Parsing requires string data"

        pctxt = self._parser_ctxt
        error = 0
        if not self._feed_parser_running:
            self._lockParser()
            self._feed_parser_running = 1
            self._context._error_log.connect()
            __GLOBAL_PARSER_CONTEXT.initParserDict(pctxt)

            if py_buffer_len > python.INT_MAX:
                buffer_len = python.INT_MAX
            else:
                buffer_len = <int>py_buffer_len
            if self._parser_type == LXML_HTML_PARSER:
                error = _htmlCtxtResetPush(pctxt, c_data, buffer_len,
                                           c_encoding, self._parse_options)
            else:
                error = xmlparser.xmlCtxtResetPush(
                    pctxt, c_data, buffer_len, NULL, c_encoding)
                xmlparser.xmlCtxtUseOptions(pctxt, self._parse_options)
            py_buffer_len = py_buffer_len - buffer_len
            c_data = c_data + buffer_len

        while error == 0 and py_buffer_len > 0:
            if py_buffer_len > python.INT_MAX:
                buffer_len = python.INT_MAX
            else:
                buffer_len = <int>py_buffer_len
            if self._parser_type == LXML_HTML_PARSER:
                error = htmlparser.htmlParseChunk(pctxt, c_data, buffer_len, 0)
            else:
                error = xmlparser.xmlParseChunk(pctxt, c_data, buffer_len, 0)
            py_buffer_len = py_buffer_len - buffer_len
            c_data = c_data + buffer_len

        if error:
            self._feed_parser_running = 0
            try:
                self._context._handleParseResult(
                    self, pctxt.myDoc, None)
            finally:
                self._cleanup()
                self._context.clear()
                self._context._error_log.disconnect()
                self._unlockParser()

    def close(self):
        """Terminates feeding data to this parser.  This tells the parser to
        process any remaining data in the feed buffer, and then returns the
        root Element of the tree that was parsed.

        This method must be called after passing the last chunk of data into
        the ``feed()`` method.  It should only be called when using the feed
        parser interface, all other usage is undefined.
        """
        cdef xmlparser.xmlParserCtxt* pctxt
        cdef xmlDoc* c_doc
        cdef _Document doc
        if not self._feed_parser_running:
            raise XMLSyntaxError, "no element found"
        pctxt = self._parser_ctxt
        self._feed_parser_running = 0
        if self._parser_type == LXML_HTML_PARSER:
            htmlparser.htmlParseChunk(pctxt, NULL, 0, 1)
        else:
            xmlparser.xmlParseChunk(pctxt, NULL, 0, 1)
        try:
            result = self._context._handleParseResult(
                self, pctxt.myDoc, None)
        finally:
            self._cleanup()
            self._context.clear()
            self._context._error_log.disconnect()
            self._unlockParser()

        if isinstance(result, _Document):
            return (<_Document>result).getroot()
        else:
            return result

cdef int _htmlCtxtResetPush(xmlparser.xmlParserCtxt* c_ctxt,
                             char* c_data, int buffer_len,
                             char* c_encoding, int parse_options) except -1:
    cdef xmlparser.xmlParserInput* c_input_stream
    # libxml2 crashes if spaceTab is not initialised
    if _LIBXML_VERSION_INT < 20629 and c_ctxt.spaceTab is NULL:
        c_ctxt.spaceTab = <int*>tree.xmlMalloc(10 * sizeof(int))
        c_ctxt.spaceMax = 10

    # libxml2 lacks an HTML push parser setup function
    error = xmlparser.xmlCtxtResetPush(c_ctxt, NULL, 0, NULL, c_encoding)
    if error:
        return error

    # fix libxml2 setup for HTML
    c_ctxt.progressive = 1
    c_ctxt.html = 1
    htmlparser.htmlCtxtUseOptions(c_ctxt, parse_options)

    if c_data is not NULL and buffer_len > 0:
        return htmlparser.htmlParseChunk(c_ctxt, c_data, buffer_len, 0)
    return 0
        

############################################################
## XML parser
############################################################

cdef int _XML_DEFAULT_PARSE_OPTIONS
_XML_DEFAULT_PARSE_OPTIONS = (
    xmlparser.XML_PARSE_NOENT   |
    xmlparser.XML_PARSE_NOCDATA |
    xmlparser.XML_PARSE_NONET   |
    xmlparser.XML_PARSE_COMPACT
    )

cdef class XMLParser(_FeedParser):
    """The XML parser.  Parsers can be supplied as additional argument to
    various parse functions of the lxml API.  A default parser is always
    available and can be replaced by a call to the global function
    'set_default_parser'.  New parsers can be created at any time without a
    major run-time overhead.

    The keyword arguments in the constructor are mainly based on the libxml2
    parser configuration.  A DTD will also be loaded if validation or
    attribute default values are requested.

    Available boolean keyword arguments:
    * attribute_defaults - read default attributes from DTD
    * dtd_validation     - validate (if DTD is available)
    * load_dtd           - use DTD for parsing
    * no_network         - prevent network access (default: True)
    * ns_clean           - clean up redundant namespace declarations
    * recover            - try hard to parse through broken XML
    * remove_blank_text  - discard blank text nodes
    * remove_comments    - discard comments
    * remove_pis         - discard processing instructions
    * compact            - safe memory for short text content (default: True)
    * resolve_entities   - replace entities by their text value (default: True)

    You can pass a parser target as ``target`` keyword argument.

    Note that you should avoid sharing parsers between threads.  While this is
    not harmful, it is more efficient to use separate parsers.  This does not
    apply to the default parser.
    """
    def __init__(self, attribute_defaults=False, dtd_validation=False,
                 load_dtd=False, no_network=True, ns_clean=False,
                 recover=False, remove_blank_text=False, compact=True,
                 resolve_entities=True, remove_comments=False,
                 remove_pis=False, target=None):
        cdef int parse_options
        parse_options = _XML_DEFAULT_PARSE_OPTIONS
        if load_dtd:
            parse_options = parse_options | xmlparser.XML_PARSE_DTDLOAD
        if dtd_validation:
            parse_options = parse_options | xmlparser.XML_PARSE_DTDVALID | \
                            xmlparser.XML_PARSE_DTDLOAD
        if attribute_defaults:
            parse_options = parse_options | xmlparser.XML_PARSE_DTDATTR | \
                            xmlparser.XML_PARSE_DTDLOAD
        if ns_clean:
            parse_options = parse_options | xmlparser.XML_PARSE_NSCLEAN
        if recover:
            parse_options = parse_options | xmlparser.XML_PARSE_RECOVER
        if remove_blank_text:
            parse_options = parse_options | xmlparser.XML_PARSE_NOBLANKS
        if not no_network:
            parse_options = parse_options ^ xmlparser.XML_PARSE_NONET
        if not compact:
            parse_options = parse_options ^ xmlparser.XML_PARSE_COMPACT
        if not resolve_entities:
            parse_options = parse_options ^ xmlparser.XML_PARSE_NOENT

        _BaseParser.__init__(self, parse_options, remove_comments, remove_pis,
                             target)

cdef class ETCompatXMLParser(XMLParser):
    """An XML parser with an ElementTree compatible default setup.  See the
    XMLParser class for details.

    This parser defaults to removing processing instructions and comments from
    the tree.
    """
    def __init__(self, attribute_defaults=False, dtd_validation=False,
                 load_dtd=False, no_network=True, ns_clean=False,
                 recover=False, remove_blank_text=False, compact=True,
                 resolve_entities=True, remove_comments=True,
                 remove_pis=True, target=None):
        XMLParser.__init__(self,
                 attribute_defaults, dtd_validation,
                 load_dtd, no_network, ns_clean,
                 recover, remove_blank_text, compact,
                 resolve_entities, remove_comments,
                 remove_pis, target)


cdef XMLParser __DEFAULT_XML_PARSER
__DEFAULT_XML_PARSER = XMLParser()

__GLOBAL_PARSER_CONTEXT.setDefaultParser(__DEFAULT_XML_PARSER)

def setDefaultParser(parser=None):
    "Deprecated, please use set_default_parser instead."
    set_default_parser(parser)

def getDefaultParser():
    "Deprecated, please use get_default_parser instead."
    return get_default_parser()

def set_default_parser(_BaseParser parser=None):
    """Set a default parser for the current thread.  This parser is used
    globally whenever no parser is supplied to the various parse functions of
    the lxml API.  If this function is called without a parser (or if it is
    None), the default parser is reset to the original configuration.

    Note that the pre-installed default parser is not thread-safe.  Avoid the
    default parser in multi-threaded environments.  You can create a separate
    parser for each thread explicitly or use a parser pool.
    """
    if parser is None:
        parser = __DEFAULT_XML_PARSER
    __GLOBAL_PARSER_CONTEXT.setDefaultParser(parser)

def get_default_parser():
    return __GLOBAL_PARSER_CONTEXT.getDefaultParser()

############################################################
## HTML parser
############################################################

cdef int _HTML_DEFAULT_PARSE_OPTIONS
_HTML_DEFAULT_PARSE_OPTIONS = (
    htmlparser.HTML_PARSE_RECOVER |
    htmlparser.HTML_PARSE_NONET   |
    htmlparser.HTML_PARSE_COMPACT
    )

cdef class HTMLParser(_FeedParser):
    """The HTML parser.  This parser allows reading HTML into a normal XML
    tree.  By default, it can read broken (non well-formed) HTML, depending on
    the capabilities of libxml2.  Use the 'recover' option to switch this off.

    Available boolean keyword arguments:
    * recover            - try hard to parse through broken HTML (default: True)
    * no_network         - prevent network access (default: True)
    * remove_blank_text  - discard empty text nodes
    * remove_comments    - discard comments
    * remove_pis         - discard processing instructions
    * compact            - safe memory for short text content (default: True)

    You can pass a parser target as ``target`` keyword argument.

    Note that you should avoid sharing parsers between threads for performance
    reasons.
    """
    def __init__(self, recover=True, no_network=True, remove_blank_text=False,
                 compact=True, remove_comments=False, remove_pis=False,
                 target=None):
        cdef int parse_options
        parse_options = _HTML_DEFAULT_PARSE_OPTIONS
        if remove_blank_text:
            parse_options = parse_options | htmlparser.HTML_PARSE_NOBLANKS
        if not recover:
            parse_options = parse_options ^ htmlparser.HTML_PARSE_RECOVER
        if not no_network:
            parse_options = parse_options ^ htmlparser.HTML_PARSE_NONET
        if not compact:
            parse_options = parse_options ^ htmlparser.HTML_PARSE_COMPACT

        _BaseParser.__init__(self, parse_options, remove_comments, remove_pis,
                             target)

cdef HTMLParser __DEFAULT_HTML_PARSER
__DEFAULT_HTML_PARSER = HTMLParser()

############################################################
## helper functions for document creation
############################################################

cdef xmlDoc* _parseDoc(text, filename, _BaseParser parser) except NULL:
    cdef char* c_filename
    cdef char* c_text
    cdef Py_ssize_t c_len
    if parser is None:
        parser = __GLOBAL_PARSER_CONTEXT.getDefaultParser()
    if not filename:
        c_filename = NULL
    else:
        filename_utf = _encodeFilenameUTF8(filename)
        c_filename = _cstr(filename_utf)
    if python.PyUnicode_Check(text):
        return (<_BaseParser>parser)._parseUnicodeDoc(text, c_filename)
    else:
        c_text = _cstr(text)
        c_len  = python.PyString_GET_SIZE(text)
        return (<_BaseParser>parser)._parseDoc(c_text, c_len, c_filename)

cdef xmlDoc* _parseDocFromFile(filename8, _BaseParser parser) except NULL:
    if parser is None:
        parser = __GLOBAL_PARSER_CONTEXT.getDefaultParser()
    return (<_BaseParser>parser)._parseDocFromFile(_cstr(filename8))

cdef xmlDoc* _parseDocFromFilelike(source, filename,
                                   _BaseParser parser) except NULL:
    if parser is None:
        parser = __GLOBAL_PARSER_CONTEXT.getDefaultParser()
    return (<_BaseParser>parser)._parseDocFromFilelike(source, filename)

cdef xmlDoc* _newDoc():
    cdef xmlDoc* result
    result = tree.xmlNewDoc("1.0")
    __GLOBAL_PARSER_CONTEXT.initDocDict(result)
    return result

cdef xmlDoc* _copyDoc(xmlDoc* c_doc, int recursive):
    cdef python.PyThreadState* state
    cdef xmlDoc* result
    if recursive:
        state = python.PyEval_SaveThread()
    result = tree.xmlCopyDoc(c_doc, recursive)
    if recursive:
        python.PyEval_RestoreThread(state)
    __GLOBAL_PARSER_CONTEXT.initDocDict(result)
    return result

cdef xmlDoc* _copyDocRoot(xmlDoc* c_doc, xmlNode* c_new_root):
    "Recursively copy the document and make c_new_root the new root node."
    cdef python.PyThreadState* state
    cdef xmlDoc* result
    cdef xmlNode* c_node
    result = tree.xmlCopyDoc(c_doc, 0) # non recursive
    __GLOBAL_PARSER_CONTEXT.initDocDict(result)
    state = python.PyEval_SaveThread()
    c_node = tree.xmlDocCopyNode(c_new_root, result, 1) # recursive
    tree.xmlDocSetRootElement(result, c_node)
    _copyTail(c_new_root.next, c_node)
    python.PyEval_RestoreThread(state)
    return result

cdef xmlNode* _copyNodeToDoc(xmlNode* c_node, xmlDoc* c_doc):
    "Recursively copy the element into the document. c_doc is not modified."
    cdef xmlNode* c_root
    c_root = tree.xmlDocCopyNode(c_node, c_doc, 1) # recursive
    _copyTail(c_node.next, c_root)
    return c_root


############################################################
## API level helper functions for _Document creation
## (here we convert to UTF-8)
############################################################

cdef _Document _parseDocument(source, _BaseParser parser):
    filename = _getFilenameForFile(source)
    if hasattr(source, 'getvalue') and hasattr(source, 'tell'):
        # StringIO - reading from start?
        if source.tell() == 0:
            return _parseMemoryDocument(
                source.getvalue(), _encodeFilenameUTF8(filename), parser)

    # Support for file-like objects (urlgrabber.urlopen, ...)
    if hasattr(source, 'read'):
        return _parseFilelikeDocument(
            source, _encodeFilenameUTF8(filename), parser)

    # Otherwise parse the file directly from the filesystem
    if filename is None:
        filename = _encodeFilename(source)
    # open filename
    return _parseDocumentFromURL(filename, parser)

cdef _Document _parseDocumentFromURL(url, _BaseParser parser):
    cdef xmlDoc* c_doc
    c_doc = _parseDocFromFile(url, parser)
    return _documentFactory(c_doc, parser)

cdef _Document _parseMemoryDocument(text, url, _BaseParser parser):
    cdef xmlDoc* c_doc
    if python.PyUnicode_Check(text):
        if _hasEncodingDeclaration(text):
            raise ValueError, \
                  "Unicode strings with encoding declaration are not supported."
        # pass native unicode only if libxml2 can handle it
        if _UNICODE_ENCODING is NULL:
            text = python.PyUnicode_AsUTF8String(text)
    elif not python.PyString_Check(text):
        raise ValueError, "can only parse strings"
    if python.PyUnicode_Check(url):
        url = python.PyUnicode_AsUTF8String(url)
    c_doc = _parseDoc(text, url, parser)
    return _documentFactory(c_doc, parser)

cdef _Document _parseFilelikeDocument(source, url, _BaseParser parser):
    cdef xmlDoc* c_doc
    if python.PyUnicode_Check(url):
        url = python.PyUnicode_AsUTF8String(url)
    c_doc = _parseDocFromFilelike(source, url, parser)
    return _documentFactory(c_doc, parser)