diff options
author | Pearu Peterson <pearu.peterson@gmail.com> | 2007-08-10 13:57:35 +0000 |
---|---|---|
committer | Pearu Peterson <pearu.peterson@gmail.com> | 2007-08-10 13:57:35 +0000 |
commit | d4375f2985a3e5f3e503960b925b4f0e3a307171 (patch) | |
tree | f943ee782ac2be0ccb073ef28b663a73d06cfea8 /numpy/f2py | |
parent | 907a90f4c28fe811d878abe66ae375a757c9ece3 (diff) | |
download | numpy-d4375f2985a3e5f3e503960b925b4f0e3a307171.tar.gz |
extgen: rewrite, clean up, update docs, simple example from Python reference manual.
Diffstat (limited to 'numpy/f2py')
-rw-r--r-- | numpy/f2py/lib/extgen/__init__.py | 31 | ||||
-rw-r--r-- | numpy/f2py/lib/extgen/base.py | 113 | ||||
-rw-r--r-- | numpy/f2py/lib/extgen/c_support.py | 293 | ||||
-rw-r--r-- | numpy/f2py/lib/extgen/doc.txt | 362 | ||||
-rw-r--r-- | numpy/f2py/lib/extgen/py_support.py | 833 | ||||
-rw-r--r-- | numpy/f2py/lib/extgen/setup_py.py | 103 | ||||
-rw-r--r-- | numpy/f2py/lib/extgen/utils.py | 126 |
7 files changed, 1724 insertions, 137 deletions
diff --git a/numpy/f2py/lib/extgen/__init__.py b/numpy/f2py/lib/extgen/__init__.py index 8c0d8e18b..58c965745 100644 --- a/numpy/f2py/lib/extgen/__init__.py +++ b/numpy/f2py/lib/extgen/__init__.py @@ -2,19 +2,26 @@ Python Extensions Generator """ -__all__ = ['Component', 'ExtensionModule', 'PyCFunction', 'PyCArgument', - 'CCode'] +__all__ = ['Component'] from base import Component -from extension_module import ExtensionModule -from pyc_function import PyCFunction -from pyc_argument import PyCArgument -from c_code import CCode -import c_type -from c_type import * -__all__ += c_type.__all__ +for _m in ['utils', 'c_support', 'py_support', 'setup_py']: + exec 'from %s import *' % (_m) + exec 'import %s as _m' % (_m) + __all__.extend(_m.__all__) -import predefined_components -import converters -c_type.register() +#from pyc_function import PyCFunction +#from pyc_argument import PyCArgument +#from c_code import CCode + +#import c_type +#from c_type import * +#__all__ += c_type.__all__ +#import c_struct +#from c_struct import * +#__all__ += c_struct.__all__# + +#import predefined_components +#import converters +#c_type.register() diff --git a/numpy/f2py/lib/extgen/base.py b/numpy/f2py/lib/extgen/base.py index 61769af9d..35c79a553 100644 --- a/numpy/f2py/lib/extgen/base.py +++ b/numpy/f2py/lib/extgen/base.py @@ -4,6 +4,7 @@ ExtGen --- Python Extension module Generator. Defines Component and Container classes. """ +import os import re import sys import time @@ -12,14 +13,19 @@ class ComponentMetaClass(type): classnamespace = {} - def __init__(cls,*args,**kws): + def __new__(mcls, *args, **kws): + cls = type.__new__(mcls, *args, **kws) n = cls.__name__ c = ComponentMetaClass.classnamespace.get(n) if c is None: ComponentMetaClass.classnamespace[n] = cls else: - print 'Ignoring redefinition of %s: %s defined earlier than %s' % (n, c, cls) - type.__init__(cls, *args, **kws) + if not c.__module__=='__main__': + sys.stderr.write('ComponentMetaClass: returning %s as %s\n'\ + % (cls, c)) + ComponentMetaClass.classnamespace[n] = c + cls = c + return cls def __getattr__(cls, name): try: return ComponentMetaClass.classnamespace[name] @@ -27,14 +33,14 @@ class ComponentMetaClass(type): raise AttributeError("'%s' object has no attribute '%s'"% (cls.__name__, name)) -class Component(object): # XXX: rename Component to Component +class Component(object): __metaclass__ = ComponentMetaClass container_options = dict() component_container_map = dict() default_container_label = None - default_component_class_name = 'CCode' + default_component_class_name = 'Code' template = '' def __new__(cls, *args, **kws): @@ -42,10 +48,24 @@ class Component(object): # XXX: rename Component to Component obj._provides = kws.get('provides', None) obj.parent = None obj.containers = {} # holds containers for named string lists - obj.components = [] # holds pairs (<Component subclass instance>, <container name or None>) + obj._components = [] # holds pairs (<Component subclass instance>, <container name or None>) + obj._generate_components = {} # temporary copy of components used for finalize and generate methods. obj = obj.initialize(*args, **kws) # initialize from constructor arguments return obj + @property + def components(self): + if Component._running_generate: + try: + return self._generate_components[Component._running_generate_id] + except KeyError: + pass + while self._generate_components: # clean up old cache + self._generate_components.popitem() + self._generate_components[Component._running_generate_id] = l = list(self._components) + return l + return self._components + def initialize(self, *components, **options): """ Set additional attributes, add components to instance, etc. @@ -54,6 +74,12 @@ class Component(object): # XXX: rename Component to Component # map(self.add, components) return self + def finalize(self): + """ + Set components after all components are added. + """ + return + def __repr__(self): return '%s(%s)' % (self.__class__.__name__, ', '.join([repr(c) for (c,l) in self.components])) @@ -70,6 +96,7 @@ class Component(object): # XXX: rename Component to Component @staticmethod def warning(message): + #raise RuntimeError('extgen:' + message) print >> sys.stderr, 'extgen:',message @staticmethod def info(message): @@ -78,6 +105,8 @@ class Component(object): # XXX: rename Component to Component def __getattr__(self, attr): if attr.startswith('container_'): # convenience feature return self.get_container(attr[10:]) + if attr.startswith('component_'): # convenience feature + return self.get_component(attr[10:]) raise AttributeError('%s instance has no attribute %r' % (self.__class__.__name__, attr)) def __add__(self, other): # convenience method @@ -116,14 +145,56 @@ class Component(object): # XXX: rename Component to Component break except KeyError: pass + if container_label is None: + container_label = component.__class__.__name__ self.components.append((component, container_label)) + component.update_parent(self) return - def generate(self): + def update_parent(self, parent): + pass + + def get_path(self, *paths): + if not hasattr(self, 'path'): + if paths: + return os.path.join(*paths) + return '' + if not self.parent: + return os.path.join(*((self.path,) + paths)) + return os.path.join(*((self.parent.get_path(), self.path)+paths)) + + def get_component(self, cls): + if isinstance(cls, str): + cls = getattr(Component, cls) + if isinstance(self, cls): + return self + if self.parent: + return self.parent.get_component(cls) + self.warning('could not find %r parent component %s, returning self'\ + % (self.__class__.__name__, cls.__name__)) + return self + + _running_generate = False + _running_generate_id = 0 + _generate_dry_run = True + + def generate(self, dry_run=True): + old_dry_run = Component._generate_dry_run + Component._generate_dry_run = dry_run + Component._running_generate_id += 1 + Component._running_generate = True + result = self._generate() + Component._running_generate = False + Component._generate_dry_run = old_dry_run + return result + + def _generate(self): """ Generate code idioms (saved in containers) and return evaluated template strings. """ + self.finalize() + # clean up containers self.containers = {} for n in dir(self): @@ -151,7 +222,7 @@ class Component(object): # XXX: rename Component to Component continue old_parent = component.parent component.parent = self - result = component.generate() + result = component._generate() if container_key == '<IGNORE>': pass elif container_key is not None: @@ -167,6 +238,7 @@ class Component(object): # XXX: rename Component to Component container = component.get_container(k) container.add(r, component.provides) else: + self.warning('%s: no container label specified for component providing %r'\ % (self.__class__.__name__,component.provides)) component.parent = old_parent @@ -181,7 +253,6 @@ class Component(object): # XXX: rename Component to Component else: assert isinstance(templates, (tuple, list)),`type(templates)` result = tuple(map(self.evaluate, templates)) - return result def init_containers(self): @@ -225,7 +296,7 @@ class Component(object): # XXX: rename Component to Component # create local container self.warning('Created container for %r with name %r, define it in'\ - ' .container_options mapping to get rid of this warning' \ + ' parent .container_options mapping to get rid of this warning' \ % (self.__class__.__name__, name)) c = self.containers[name] = Container() return c @@ -259,7 +330,10 @@ class Component(object): # XXX: rename Component to Component i = template.index(s) template = template[:i] + str(container) + template[i+len(s):] container.indent_offset = old_indent - template = template % d + try: + template = template % d + except KeyError, msg: + raise KeyError('%s.container_options needs %s item' % (self.__class__.__name__, msg)) return re.sub(r'.*[<]KILLLINE[>].*(\n|$)','', template) @@ -330,7 +404,9 @@ class Container(object): use_indent = False, indent_offset = 0, use_firstline_indent = False, # implies use_indent - replace_map = {} + replace_map = {}, + ignore_empty_content = False, + skip_prefix_suffix_when_single = False ): self.list = [] self.label_map = {} @@ -347,6 +423,8 @@ class Container(object): self.indent_offset = indent_offset self.use_firstline_indent = use_firstline_indent self.replace_map = replace_map + self.ignore_empty_content = ignore_empty_content + self.skip_prefix_suffix_when_single = skip_prefix_suffix_when_single def __nonzero__(self): return bool(self.list) @@ -374,6 +452,8 @@ class Container(object): """ if content is None: return + if content=='' and self.ignore_empty_content: + return assert isinstance(content, str),`type(content)` if label is None: label = time.time() @@ -406,8 +486,9 @@ class Container(object): new_l.extend([indent + l2 for l2 in lines[1:]]) l = new_l r = self.separator.join(l) - r = self.prefix + r - r = r + self.suffix + if not (len(self.list)==1 and self.skip_prefix_suffix_when_single): + r = self.prefix + r + r = r + self.suffix else: r = self.default if not self.skip_prefix: @@ -429,7 +510,9 @@ class Container(object): use_indent = self.use_indent, indent_offset = self.indent_offset, use_firstline_indent = self.use_firstline_indent, - replace_map = self.replace_map + replace_map = self.replace_map, + ignore_empty_content = self.ignore_empty_content, + skip_prefix_suffix_when_single = self.skip_prefix_suffix_when_single ) options.update(extra_options) cpy = Container(**options) diff --git a/numpy/f2py/lib/extgen/c_support.py b/numpy/f2py/lib/extgen/c_support.py new file mode 100644 index 000000000..95b7bb675 --- /dev/null +++ b/numpy/f2py/lib/extgen/c_support.py @@ -0,0 +1,293 @@ + +__all__ = ['CLine', 'Keyword', 'CTypeSpec', 'CDeclarator', 'CDeclaration', + 'CArgument', 'CCode', 'CFunction', 'CSource', 'CHeader', 'CStdHeader'] + +from base import Component +from utils import Line, Code, FileSource + +class CLine(Line): + pass + +class Keyword(CLine): + pass + +class CInitExpr(CLine): + pass + +class CTypeSpec(CLine): + + """ + >>> i = CTypeSpec('int') + >>> print i.generate() + int + >>> print i.as_ptr().generate() + int* + """ + def as_ptr(self): return self.__class__(self.generate()+'*') + + +class CDeclarator(Component): + + """ + + >>> CDeclarator('name').generate() + 'name' + >>> CDeclarator('name','0').generate() + 'name = 0' + """ + container_options = dict( + Initializer = dict(default='',prefix=' = ', skip_prefix_when_empty=True, + ignore_empty_content = True + ), + ScalarInitializer = dict(default='',prefix=' = ', skip_prefix_when_empty=True, + ignore_empty_content = True + ), + SequenceInitializer = dict(default='',prefix=' = {\n', skip_prefix_when_empty=True, + suffix='}', skip_suffix_when_empty=True, + ignore_empty_content = True, + separator = ',\n', use_indent=True, + ), + StringInitializer = dict(default='',prefix=' = "', skip_prefix_when_empty=True, + suffix='"', skip_suffix_when_empty=True, + ignore_empty_content = True, + separator='\\n"\n"', replace_map = {'\n':'\\n'}, + use_firstline_indent = True, + ), + ) + + default_component_class_name = 'CInitExpr' + + component_container_map = dict( + CInitExpr = 'Initializer' + ) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr,[self.name]+[c for (c,l) in self.components]))) + + def initialize(self, name, *initvalues, **options): + self.name = name + self.is_string = options.get('is_string', None) + if self.is_string: + assert not options.get('is_scalar', None) + self.is_scalar = False + else: + if name.endswith(']'): + self.is_scalar = False + else: + self.is_scalar = options.get('is_scalar', True) + + map(self.add, initvalues) + return self + + def update_containers(self): + if self.is_scalar: + self.container_ScalarInitializer += self.container_Initializer + self.template = '%(name)s%(ScalarInitializer)s' + elif self.is_string: + self.container_StringInitializer += self.container_Initializer + self.template = '%(name)s%(StringInitializer)s' + elif len(self.containers)>1 or not self.is_scalar: + self.container_SequenceInitializer += self.container_Initializer + self.template = '%(name)s%(SequenceInitializer)s' + else: + self.container_ScalarInitializer += self.container_Initializer + self.template = '%(name)s%(ScalarInitializer)s' + +class CDeclaration(Component): + + """ + >>> d = CDeclaration('int', 'a') + >>> print d.generate() + int a + >>> d += 'b' + >>> print d.generate() + int a, b + >>> d += CDeclarator('c',1) + >>> print d.generate() + int a, b, c = 1 + """ + + template = '%(CTypeSpec)s %(CDeclarator)s' + + container_options = dict( + CTypeSpec = dict(default='int', separator=' '), + CDeclarator = dict(default='<KILLLINE>', separator=', '), + ) + + component_container_map = dict( + CTypeSpec = 'CTypeSpec', + CDeclarator = 'CDeclarator', + ) + + default_component_class_name = 'CDeclarator' + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr,[c for (c,l) in self.components]))) + + def initialize(self, ctype, *declarators, **options): + ctype = CTypeSpec(ctype) + self.ctype = ctype + self.add(ctype) + map(self.add, declarators) + return self + +class CArgument(CDeclaration): + + def initialize(self, name, ctype, **options): + return CDeclaration.initialize(ctype, name, **options) + + +class CCode(Code): + parent_container_options = dict(default='<KILLLINE>', use_indent=True, ignore_empty_content=True) + +class CFunction(Component): + + """ + >>> f = CFunction('foo') + >>> print f.generate() + int + foo(void) { + } + >>> f += Keyword('static') + >>> f += CArgument('int', 'a') + >>> f += 'a = 2;' + >>> print f.generate() + static + int + foo(int a) { + a = 2; + } + >>> f += CArgument('float', 'b') + >>> f += CDeclaration('float', 'c') + >>> f += CDeclaration('float', CDeclarator('d','3.0')) + >>> print f.generate() + static + int + foo(int a, float b) { + float c; + float d = 3.0; + a = 2; + } + """ + + template = '''\ +%(CSpecifier)s +%(CTypeSpec)s +%(name)s(%(CArgument)s) { + %(CDeclaration)s + %(CBody)s +}''' + + container_options = dict( + CArgument = dict(separator=', ', default='void'), + CDeclaration = dict(default='<KILLLINE>', use_indent=True, ignore_empty_content=True, + separator = ';\n', suffix=';', skip_suffix_when_empty=True), + CBody = dict(default='<KILLLINE>', use_indent=True, ignore_empty_content=True), + CTypeSpec = dict(default='int', separator = ' ', ignore_empty_content=True), + CSpecifier = dict(default='<KILLLINE>', separator = ' ', ignore_empty_content = True) + ) + + component_container_map = dict( + CArgument = 'CArgument', + CDeclaration = 'CDeclaration', + CCode = 'CBody', + CTypeSpec = 'CTypeSpec', + Keyword = 'CSpecifier', + ) + + default_component_class_name = 'CCode' + + def initialize(self, name, rctype='int', *components, **options): + self.name = name + rctype = CTypeSpec(rctype) + self.rctype = rctype + self.add(rctype) + map(self.add, components) + if options: self.warning('%s unused options: %s\n' % (self.__class__.__name__, options)) + return self + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr,[self.name, self.rctype]+[c for (c,l) in self.components]))) + +class CHeader(CLine): + + """ + >>> h = CHeader('noddy.h') + >>> print h.generate() + #include "noddy.h" + + """ + template = '#include "%(line)s"' + +class CStdHeader(CHeader): + template = '#include <%(line)s>' + +class CSource(FileSource): + + """ + >>> s = CSource('foo.c') + >>> print s.generate() #doctest: +ELLIPSIS + /* -*- c -*- */ + /* This file 'foo.c' is generated using ExtGen tool + from NumPy version ... + ExtGen is developed by Pearu Peterson <pearu.peterson@gmail.com>. + For more information see http://www.scipy.org/ExtGen/ . + */ + #ifdef __cplusplus + extern "C" { + #endif + #ifdef __cplusplus + } + #endif + <BLANKLINE> + """ + + container_options = dict( + CHeader = dict(default='<KILLLINE>', prefix='\n/* CHeader */\n', skip_prefix_when_empty=True), + CTypeDef = dict(default='<KILLLINE>', prefix='\n/* CTypeDef */\n', skip_prefix_when_empty=True), + CProto = dict(default='<KILLLINE>', prefix='\n/* CProto */\n', skip_prefix_when_empty=True), + CDefinition = dict(default='<KILLLINE>', prefix='\n/* CDefinition */\n', skip_prefix_when_empty=True), + CDeclaration = dict(default='<KILLLINE>', separator=';\n', suffix=';', + prefix='\n/* CDeclaration */\n', skip_prefix_when_empty=True), + CMainProgram = dict(default='<KILLLINE>', prefix='\n/* CMainProgram */\n', skip_prefix_when_empty=True), + ) + + template_c_header = '''\ +/* -*- c -*- */ +/* This file %(path)r is generated using ExtGen tool + from NumPy version %(numpy_version)s. + ExtGen is developed by Pearu Peterson <pearu.peterson@gmail.com>. + For more information see http://www.scipy.org/ExtGen/ . +*/''' + + + template = template_c_header + ''' +#ifdef __cplusplus +extern \"C\" { +#endif +%(CHeader)s +%(CTypeDef)s +%(CProto)s +%(CDefinition)s +%(CDeclaration)s +%(CMainProgram)s +#ifdef __cplusplus +} +#endif +''' + + component_container_map = dict( + CHeader = 'CHeader', + CFunction = 'CDefinition', + CDeclaration = 'CDeclaration', + ) + + + + +def _test(): + import doctest + doctest.testmod() + +if __name__ == "__main__": + _test() diff --git a/numpy/f2py/lib/extgen/doc.txt b/numpy/f2py/lib/extgen/doc.txt index d1ec1e4db..c86af8d96 100644 --- a/numpy/f2py/lib/extgen/doc.txt +++ b/numpy/f2py/lib/extgen/doc.txt @@ -15,22 +15,251 @@ Introduction ExtGen is a pure Python package that provides a high-level tool for constructing and building Python extension modules. -Hello example follows:: +Even an inexperienced user with no background on writing extension +modules can build extension modules on fly when using ExtGen tool! + +Hello example follows >>> from numpy.f2py.lib.extgen import * - >>> m = ExtensionModule('foo') - >>> f = PyCFunction('hello') - >>> f += 'printf("Hello!\\n");' - >>> m += f - >>> print m.generate() # shows a string containing C source to extension module, useful for debugging - >>> foo = m.build() - >>> foo.hello() + >>> m = PyCModule('foo') # define extension module component + >>> f = PyCFunction('hello') # define function component + >>> f += 'printf("Hello!\\n");' # put a C statement into function body + >>> m += f # add function to module + >>> print m.generate() # shows a string containing C source to extension module + # useful for debugging + >>> foo = m.build() # compile, build, and return extension module object + >>> foo.hello() # call function Hello! - >>> -Description of the ExtGen model -=============================== +Users reference manual +====================== + +Writing a python extension module requires a knowledge of Pyhton C/API +details and may take lots of effort to get your first extension module +compile and working, even for a simple problem. See the `Simple Example`__ +in Python reference manual. ExtGen provides a high level tool for +constructing extension modules by automatically taking care of the +Pyhton C/API details while providing full control how an extension +module is created. + +__ http://docs.python.org/ext/ + +Getting started +--------------- + +Creating the `Simple Example`__ with the help of ExtGen tool is really simple + + >>> system = PyCFunction('system', + PyCArgument('command', 'c_const_char_ptr'), + PyCReturn('sts','c_int')) + >>> system += 'sts = system(command);' + >>> module = PyCModule('spam', system) + >>> spam = module.build() + >>> spam.system('pwd') + /home/pearu/svn/numpy/numpy/f2py/lib + 0 + +__ http://docs.python.org/ext/ + +ExtGen generated modules have automatically generated documentation +strings that accept also user input + + >>> a = PyCArgument('command', 'c_const_char_ptr', + input_description='a shell command string') + >>> r = PyCReturn('sts', 'c_int', + output_description='status value returned by shell command') + >>> system = PyCFunction('system', title='Execute a shell command.') + >>> system += a # add argument component to function + >>> system += r # add return value component to function + >>> system += 'sts = system(command);' # add C code to functon body + >>> module = PyCModule('spam', system) # create module instance with function component + >>> spam = module.build() + >>> print spam.__doc__ + This module 'spam' is generated with ExtGen from NumPy version 1.0.4.dev3744. + :Functions: + system(command) -> sts + >>> print spam.system.__doc__ + system(command) -> sts + Execute a shell command. + :Parameters: + command : a to C const char ptr convertable object + a shell command string + :Returns: + sts : a to C int convertable object + status value returned by shell command + >>> + +To see the source code that ExtGen generates, use `.generate()` method for any component instance + + >>> print system.generate() + static + char pyc_function_system_doc[] = + " system(command) -> sts" + "\n\nExecute a shell command." + "\n\n:Parameters:\n" + " command : a to C const char ptr convertable object\n" + " a shell command string" + "\n\n:Returns:\n" + " sts : a to C int convertable object\n" + " status value returned by shell command" + ; + static + PyObject* + pyc_function_system(PyObject *pyc_self, PyObject *pyc_args, PyObject *pyc_keywds) { + PyObject * volatile pyc_buildvalue = NULL; + volatile int capi_success = 1; + const char * command = NULL; + int sts = 0; + static char *capi_kwlist[] = {"command", NULL}; + if (PyArg_ParseTupleAndKeywords(pyc_args, pyc_keywds,"z", + capi_kwlist, &command)) { + sts = system(command); + capi_success = !PyErr_Occurred(); + if (capi_success) { + pyc_buildvalue = Py_BuildValue("i", sts); + } + } + return pyc_buildvalue; + } + >>> print module.generate() # prints full extension module source + ... + +Components +---------- + +All components are subclassed of `Component` base class that provides +the following methods: + +- `.generate(self)` --- return a generated component string +- `.add(self, component, container_label=None)` --- add subcomponent + to component instance. The `container_label` argument can be used to tell + `ExtGen` where this component should be used, see `Developers reference + manual` for more details, otherwise `EgtGen` figures that out by + looking at subcomponent class properties. +- `.__add__(self, other)`, `.__iadd__(self, other)` --- shortcuts + for `self.add(other)` call. + +ExtGen provides the following Python C/API related components: + +- `SetupPy(<build directory>, *components)` --- generates a `setup.py` file + that is used to build extension modules to the given build directory. + It provides the following methods: + + - `.execute(self, *args)` --- runs `python setup.py` command with given + arguments. + + One can add `PyCModule` and `PySource` components to `SetupPy`. + +- `PyCModule(<modulename>, *components, title=..., description=...)` --- + represents python extension module source. It provides the following + methods: + + - `.build(self, build_dir=None, clean_at_exit=None)` --- compilers, + builds, and returns extension module object. + + One can add `PyCFunction` components to `PyCModule`. + +- `PyCFunction(<funcname>, *components, title=..., description=...)` --- + represents python extension module function source. + + One can add `PyCArgument`, `PyCReturn`, `CCode` components to `PyCfunction`. + String components are converted `CCode` components by default. + +- `PyCArgument(<argname>, ctype=<expected C type>, **components, + input_intent='required', input_title=..., input_description=..., + output_intent='hide', output_title=..., output_description=..., + title=..., description=..., + depends=<list of argument dependencies>)` --- represents argument + to python extension module function. + + `ctype` is `PyCTypeSpec` component instance or string. In the latter case + it is converted to `PyCTypeSpec` component. + +- `PyCReturn(<retname>, ctype=<expected C type>, **components)` --- + same as `PyCArgument` but with `input_intent='hide'` and `output_intent='return'` + options set. + +- `PyCTypeSpec(<typeobj>)` --- represents variable type in a + python extension module. Over 70 types are supported: + + >>> typenames = PyCTypeSpec.typeinfo_map.keys() + >>> typenames.sort() + >>> print ', '.join(typenames) + buffer, c_Py_UNICODE, c_Py_complex, c_Py_ssize_t, c_char, c_char1, + c_const_char_ptr, c_double, c_float, c_int, c_long, c_long_long, + c_short, c_unsigned_char, c_unsigned_int, c_unsigned_long, + c_unsigned_long_long, c_unsigned_short, cell, cobject, complex, dict, + file, float, frozenset, function, generator, instance, int, iter, + list, long, method, module, numeric_array, numpy_complex128, + numpy_complex160, numpy_complex192, numpy_complex256, numpy_complex32, + numpy_complex64, numpy_descr, numpy_float128, numpy_float16, + numpy_float32, numpy_float64, numpy_float80, numpy_float96, + numpy_int128, numpy_int16, numpy_int32, numpy_int64, numpy_int8, + numpy_iter, numpy_multiiter, numpy_ndarray, numpy_ufunc, + numpy_uint128, numpy_uint16, numpy_uint32, numpy_uint64, numpy_uint8, + object, property, set, slice, str, tuple, type, unicode + + `typeobj` can be python type instance (e.g. `int`) or one of the string values + from the list of typenames above. + +- `PySource(<filename>, *components)` --- represents pure python module file. + + One can add `Code` components to `PySource`. + +ExtGen provides the following C related components: + +- `CSource` --- represents C file. Derived from `FileSource`. + + One can add `CCode` components to `CSource`. + String input is converted to `CCode` component. + +- `CHeader(<header filename>)`, `CStdHeader(<std header filename>)`. Derived from `Line`. + +- `CCode(*components)` --- represents any C code block. Derived from `Code`. + +- `CFunction(<funcname>, rctype=CTypeSpec('int'), *components)` --- represents + a C function. + + One can add `CArgument`, `CDeclaration`, `CCode` components to `CFunction`. + +- `CDeclaration(<ctype>, *declarators)` --- represenets `<ctype> <declarators list>` + code idiom. `<ctype>` is `CTypeSpec` instance. + + One can add `CDeclarator` components to `CDeclaration`. + +- `CArgument(<argument name>, <ctype>)` --- C function argument. Derived from `CDeclaration`. + +- `CTypeSpec(<typename>)` --- C type specifier. Derived from `Line`. + +- `CDeclarator(<name>, *initvalues, is_string=..., is_scalar=...)` --- represents + `<name> [= <initialzer>]` code idiom. + + String input is converted to `CInitExpr` component. + +ExtGen provides the following general purpose components: + +- `Word(<word>)` + + Nothing can be added to `Word`. + +- `Line(*strings)` + + One can add `Line` component to `Line`. + String input is converted to `Line` component. + +- `Code(*lines)` + + One can add `Line` and `Code` components to `Code`. + String input is converted to `Line` component. + +- `FileSource(<filename>, *components)`. + + One can add `Line` and `Code` components to `FileSource`. + String input is converted to `Code` component. + +Developers reference manual +=========================== To extend ExtGen, one needs to understand the infrastructure of generating extension modules. @@ -43,7 +272,7 @@ specified rules resulting actual code sources. ExtGen uses two steps for constructing code sources: - creating code components and adding them together to a parent - component. For example, the `ExtensionModule` instance in the + component. For example, the `PyCModule` instance in the hello example becomes a parent component to a `PyCFunction` instance after executing `m += f`. @@ -52,7 +281,7 @@ for constructing code sources: One can iterate the above process as one wishes. -The method `ExtensionModule.build()` is defined for convenience. +The method `PyCModule.build()` is defined for convenience. It compiles the generated sources, builds an extension module, imports the resulting module to Python, and returns the module object. @@ -79,6 +308,11 @@ following methods and attributes: - `.component_containe_map` is used to find `container_label` corresponding to `component` argument class. +- `.update_parent(self, parent)` is called after `parent.add(self,..)`. + +- `.finalize(self)` is called after finishing adding new components + and before `.generate()` method call. + - `.generate(self)` returns a source code string. It recursively processes all subcomponents, creates code containers, and evaluates code templates. @@ -129,7 +363,8 @@ tasks: - A component class must have a base class `Component`. - A component class may redefine `.initialize()`, - `.init_containers()`, `.update_containers()`, `.get_templates()` + `.init_containers()`, `.add()`, `update_parent()`, + `.update_containers()`, `.get_templates()` methods, `.provides()` property method and `.container_options`, `.component_container_map`, `.default_container_label`, `.default_component_class_name`, `.template` attributes. @@ -157,43 +392,12 @@ tasks: - All classes derived from `Component` are available as `Component.<subclass name>`. -Here follows a simplified version of `ExtensionModule.template`:: - - #include "Python.h" - - %(Header)s - %(TypeDef)s - %(Extern)s - %(CCode)s - %(CAPICode)s - %(ObjDecl)s - - static PyObject* extgen_module; - - static PyMethodDef extgen_module_methods[] = { - %(ModuleMethod)s - {NULL,NULL,0,NULL} - }; - - PyMODINIT_FUNC init%(modulename)s(void) { - extgen_module = Py_InitModule("%(modulename)s", extgen_module_methods); - %(ModuleInit)s - return; - capi_error: - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_RuntimeError, "failed to initialize %(modulename)s module."); - } - return; - } - -Here formatting mapping keys `Header`, `TypeDef`, etc are the labels -of containers which will be used in the templare evaluation. See `extgen/*.py` files for more examples how to redefine `Component` class methods and attributes. Using `Container` class -======================= +----------------------- `Container` class has the following optional arguments: @@ -209,6 +413,8 @@ Using `Container` class - `indent_offset=0` - `user_defined_str=None` - `replace_map={}` + - `ignore_empty_content=False` + - `skip_prefix_suffix_when_single=False` that are used to enhance the behaviour of `Container.__str__()` method. By default, `Container.__str__()` returns @@ -241,67 +447,3 @@ in `.line[0]` is used. The `replace_map` is used to apply Full control over the `__str__()` method is obtained via defining `user_defined_str` that should be a callable object taking list as input and return a string. - - -Component classes -================= - -ExtGen package defines the following extension module component classes: - - - `ExtensionModule(<modulename>, *components, numpy=False, - provides=.., title=.., description=..)` --- - represents an extension module component. If `numpy` is `True` then - `NumPy` support will be added to an extension module. - - - `PyCFunction(<name>, *components, provides=.., title=.., description=..)` --- - represents an extension function component. - - - `PyCArgument(<name>, *components, provides=.., input_intent=.., - output_intent=.., input_title=.., input_description=.., - output_title=, output_description=..)` --- represents an argument component for - `PyCFunction`. Keyword arguments `input_intent` and - `output_intent` may have values `'required'`, `'optional'`, - `'extra'`, `'hide'` and `'hide'`, `'return'`, respectively. - - - `CCode(*lines, provides=..)` --- represents any C code block or - statement component. - - - `CType(<name or python type obj>)` --- represents a predefined or intrinsic C type - with a given name. - - - `CTypeAlias(<name>, <ctype>)` --- represents `typedef ctype name;` - declaration. - - - `CTypeFuncAlias(<name>, <return ctype>, <argument ctypes>)` --- - represents `typedef rctype (*name)(actype1,..)` declaration. - Use `.add()` method to add more argument types. - - - `CTypePtr(<ctype>)` --- represents `typedef ctype* ctype_ptr;` - declaration. - - - `CTypeStruct(<name>, *declarations)` --- represents `typedef struct { - <declarations> } name;` declaration. Use `.add()` method to add - more declarations. - - - `CDecl(<ctype>, *names)` --- represents `ctype name1, name2, ..;` - declaration. Use `.add()` method to add more names. - - - `CTypePython(<python type object or type name>)` - --- represents python type object in C, see - `CTypePython.typeinfo_map.keys()` for supported type names - (there are over 70 supported types). - - -Predefined components -===================== - -ExtGen defines the following components that can be retrived -using `Component.get(<provides>)` method: - -- `'Python.h'` - include `Pyhton.h` header file. - -- `'arrayobject.h'` - include NumPy header files. - -- `'import_array'` - code for importing numpy package to extension - module. - diff --git a/numpy/f2py/lib/extgen/py_support.py b/numpy/f2py/lib/extgen/py_support.py new file mode 100644 index 000000000..8fc2bac31 --- /dev/null +++ b/numpy/f2py/lib/extgen/py_support.py @@ -0,0 +1,833 @@ + +__all__ = ['PySource', 'PyCFunction', 'PyCModule', 'PyCTypeSpec', 'PyCArgument', 'PyCReturn'] + +import os +import sys +from base import Component +from utils import * +from c_support import * + +class PySource(FileSource): + + template_py_header = '''\ +#!/usr/bin/env python +# This file %(path)r is generated using ExtGen tool +# from NumPy version %(numpy_version)s. +# ExtGen is developed by Pearu Peterson <pearu.peterson@gmail.com>. +# For more information see http://www.scipy.org/ExtGen/ .''' + + container_options = dict( + Content = dict(default='', + prefix = template_py_header + '\n', + suffix = '\n', + use_indent=True) + ) + + pass + +class PyCModule(CSource): + + """ + >>> m = PyCModule('PyCModule_test', title='This is first line.\\nSecond line.', description='This is a module.\\nYes, it is.') + >>> mod = m.build() + >>> print mod.__doc__ #doctest: +ELLIPSIS + This module 'PyCModule_test' is generated with ExtGen from NumPy version ... + <BLANKLINE> + This is first line. + Second line. + <BLANKLINE> + This is a module. + Yes, it is. + """ + + template = CSource.template_c_header + ''' +#ifdef __cplusplus +extern \"C\" { +#endif +#include "Python.h" +%(CHeader)s +%(CTypeDef)s +%(CProto)s +%(CDefinition)s +%(CAPIDefinition)s +%(CDeclaration)s +%(PyCModuleCDeclaration)s +%(CMainProgram)s +#ifdef __cplusplus +} +#endif +''' + + container_options = CSource.container_options.copy() + container_options.update(CAPIDefinition=container_options['CDefinition'], + PyCModuleCDeclaration=dict(default='<KILLLINE>', + ignore_empty_content=True), + ) + + component_container_map = dict( + PyCModuleInitFunction = 'CMainProgram', + PyCModuleCDeclaration = 'PyCModuleCDeclaration', + PyCFunction = 'CAPIDefinition', + ) + + def initialize(self, pyname, *components, **options): + self.pyname = pyname + self.title = options.pop('title', None) + self.description = options.pop('description', None) + + self = CSource.initialize(self, '%smodule.c' % (pyname), **options) + + self.cdecl = PyCModuleCDeclaration(pyname) + self += self.cdecl + + self.main = PyCModuleInitFunction(pyname) + self += self.main + map(self.add, components) + return self + + def update_parent(self, parent): + if isinstance(parent, Component.SetupPy): + self.update_SetupPy(parent) + + def update_SetupPy(self, parent): + parent.setup_py += self.evaluate(' config.add_extension(%(pyname)r, sources = ["%(extmodulesrc)s"])', + extmodulesrc = self.path) + parent.init_py += 'import %s' % (self.pyname) + + def build(self, build_dir=None, clean_at_exit=None): + """ build(build_dir=None, clean_at_exit=None) + + A convenience function to build, import, an return + an extension module object. + """ + if build_dir is None: + import tempfile + import time + packagename = 'extgen_' + str(hex(int(time.time()*10000000)))[2:] + build_dir = os.path.join(tempfile.gettempdir(), packagename) + clean_at_exit = True + if clean_at_exit: + import atexit + import shutil + atexit.register(lambda d=build_dir: shutil.rmtree(d)) + self.info('directory %r will be removed at exit from python.' % (build_dir)) + + setup = Component.SetupPy(build_dir) + setup += self + s,o = setup.execute('build_ext','--inplace') + if s: + self.info('return status=%s' % (s)) + self.info(o) + raise RuntimeError('failed to build extension module %r' % (self.pyname)) + sys.path.insert(0, os.path.dirname(build_dir)) + packagename = os.path.basename(build_dir) + try: + p = __import__(packagename) + #exec 'import %s as p' % (packagename) + m = getattr(p, self.pyname) + except: + self.info(sys.path) + del sys.path[0] + raise + else: + del sys.path[0] + return m + +class PyCModuleCDeclaration(Component): + + template = '''\ +static PyObject* extgen_module; +static +PyMethodDef extgen_module_methods[] = { + %(PyMethodDef)s + {NULL,NULL,0,NULL} +}; +static +char extgen_module_doc[] = +"This module %(pyname)r is generated with ExtGen from NumPy version %(numpy_version)s." +%(Title)s +%(Description)s +%(FunctionSignature)s +;''' + container_options = dict( + PyMethodDef = dict(suffix=',', skip_suffix_when_empty=True,separator=',\n', + default='<KILLLINE>', use_indent=True, ignore_empty_content=True), + FunctionSignature = dict(prefix='"\\n\\n:Functions:\\n"\n" ', skip_prefix_when_empty=True, use_indent=True, + ignore_empty_content=True, default='<KILLLINE>', + separator = '"\n" ', suffix='"', skip_suffix_when_empty=True, + ), + Title = dict(default='<KILLLINE>',prefix='"\\n\\n',suffix='"',separator='\\n"\n"', + skip_prefix_when_empty=True, skip_suffix_when_empty=True, + use_firstline_indent=True, replace_map={'\n':'\\n'}), + Description = dict(default='<KILLLINE>',prefix='"\\n\\n"\n"', + suffix='"',separator='\\n"\n"', + skip_prefix_when_empty=True, skip_suffix_when_empty=True, + use_firstline_indent=True, replace_map={'\n':'\\n'}), + ) + + default_component_class_name = 'Line' + + def initialize(self, pyname): + self.pyname = pyname + return self + + def update_parent(self, parent): + if isinstance(parent, PyCModule): + self.update_PyCModule(parent) + + def update_PyCModule(self, parent): + if parent.title: + self.add(parent.title, 'Title') + if parent.description: + self.add(parent.description, 'Description') + + +class PyCModuleInitFunction(CFunction): + + """ + >>> f = PyCModuleInitFunction('test_PyCModuleInitFunction') + >>> print f.generate() + PyMODINIT_FUNC + inittest_PyCModuleInitFunction(void) { + PyObject* extgen_module_dict = NULL; + PyObject* extgen_str_obj = NULL; + extgen_module = Py_InitModule(\"test_PyCModuleInitFunction\", extgen_module_methods); + if ((extgen_module_dict = PyModule_GetDict(extgen_module))==NULL) goto capi_error; + if ((extgen_str_obj = PyString_FromString(extgen_module_doc))==NULL) goto capi_error; + PyDict_SetItemString(extgen_module_dict, \"__doc__\", extgen_str_obj); + Py_DECREF(extgen_str_obj); + if ((extgen_str_obj = PyString_FromString(\"restructuredtext\"))==NULL) goto capi_error; + PyDict_SetItemString(extgen_module_dict, \"__docformat__\", extgen_str_obj); + Py_DECREF(extgen_str_obj); + return; + capi_error: + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, \"failed to initialize 'test_PyCModuleInitFunction' module.\"); + } + return; + } + """ + + template = '''\ +%(CSpecifier)s +%(CTypeSpec)s +%(name)s(void) { + PyObject* extgen_module_dict = NULL; + PyObject* extgen_str_obj = NULL; + %(CDeclaration)s + extgen_module = Py_InitModule("%(pyname)s", extgen_module_methods); + if ((extgen_module_dict = PyModule_GetDict(extgen_module))==NULL) goto capi_error; + if ((extgen_str_obj = PyString_FromString(extgen_module_doc))==NULL) goto capi_error; + PyDict_SetItemString(extgen_module_dict, "__doc__", extgen_str_obj); + Py_DECREF(extgen_str_obj); + if ((extgen_str_obj = PyString_FromString("restructuredtext"))==NULL) goto capi_error; + PyDict_SetItemString(extgen_module_dict, "__docformat__", extgen_str_obj); + Py_DECREF(extgen_str_obj); + %(CBody)s + return; +capi_error: + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "failed to initialize %(pyname)r module."); + } + return; +}''' + + def initialize(self, pyname, *components, **options): + self.pyname = pyname + self.title = options.pop('title', None) + self.description = options.pop('description', None) + self = CFunction.initialize(self, 'init'+pyname, 'PyMODINIT_FUNC', *components, **options) + return self + +#helper classes for PyCFunction +class KWListBase(Word): parent_container_options = dict(separator=', ', suffix=', ', skip_suffix_when_empty=True) +class ReqKWList(KWListBase): pass +class OptKWList(KWListBase): pass +class ExtKWList(KWListBase): pass +class ArgBase(Word): parent_container_options = dict(separator=', ') +class ReqArg(ArgBase): pass +class OptArg(ArgBase): pass +class ExtArg(ArgBase): pass +class RetArg(ArgBase): + parent_container_options = dict(separator=', ', prefix='(', suffix=')', default = 'None', + skip_prefix_when_empty=True, skip_suffix_when_empty=True, + skip_prefix_suffix_when_single=True) +class OptExtArg(ArgBase): + parent_container_options = dict(separator=', ', prefix=' [, ', skip_prefix_when_empty=True, + suffix=']', skip_suffix_when_empty=True) +class ArgDocBase(Word): + parent_container_options = dict(default='<KILLLINE>', prefix='"\\n\\nArguments:\\n"\n" ', + separator='\\n"\n" ', suffix='"', + skip_prefix_when_empty=True, skip_suffix_when_empty=True, + use_firstline_indent=True, replace_map={'\n':'\\n'}) +class ReqArgDoc(ArgDocBase): + parent_container_options = ArgDocBase.parent_container_options.copy() + parent_container_options.update(prefix='"\\n\\n:Parameters:\\n"\n" ') +class OptArgDoc(ArgDocBase): + parent_container_options = ArgDocBase.parent_container_options.copy() + parent_container_options.update(prefix='"\\n\\n:Optional parameters:\\n"\n" ') +class ExtArgDoc(ArgDocBase): + parent_container_options = ArgDocBase.parent_container_options.copy() + parent_container_options.update(prefix='"\\n\\n:Extra parameters:\\n"\n" ') +class RetArgDoc(ArgDocBase): + parent_container_options = ArgDocBase.parent_container_options.copy() + parent_container_options.update(prefix='"\\n\\n:Returns:\\n"\n" ', + default='"\\n\\n:Returns:\\n None"') +class ArgFmtBase(Word): parent_container_options = dict(separator='') +class ReqArgFmt(ArgFmtBase): pass +class OptArgFmt(ArgFmtBase): pass +class ExtArgFmt(ArgFmtBase): pass +class RetArgFmt(ArgFmtBase): pass +class OptExtArgFmt(ArgFmtBase): + parent_container_options = dict(separator='', prefix='|', skip_prefix_when_empty=True) +class ArgObjBase(Word): parent_container_options = dict(separator=', ', prefix=', ', skip_prefix_when_empty=True) +class ReqArgObj(ArgObjBase): pass +class OptArgObj(ArgObjBase): pass +class ExtArgObj(ArgObjBase): pass +class RetArgObj(ArgObjBase): pass + +class FunctionSignature(Component): + template = '%(name)s(%(ReqArg)s%(OptExtArg)s) -> %(RetArg)s' + parent_container_options = dict() + container_options = dict( + ReqArg = ReqArg.parent_container_options, + OptArg = OptArg.parent_container_options, + ExtArg = ExtArg.parent_container_options, + RetArg = RetArg.parent_container_options, + OptExtArg = OptExtArg.parent_container_options, + ) + def initialize(self, name, *components, **options): + self.name = name + map(self.add, components) + return self + def update_containers(self): + self.container_OptExtArg += self.container_OptArg + self.container_ExtArg + +class PyCFunction(CFunction): + + """ + >>> from __init__ import * + >>> f = PyCFunction('foo') + >>> print f.generate() + static + char pyc_function_foo_doc[] = + \" foo() -> None\" + \"\\n\\n:Returns:\\n None\" + ; + static + PyObject* + pyc_function_foo(PyObject *pyc_self, PyObject *pyc_args, PyObject *pyc_keywds) { + PyObject * volatile pyc_buildvalue = NULL; + volatile int capi_success = 1; + static char *capi_kwlist[] = {NULL}; + if (PyArg_ParseTupleAndKeywords(pyc_args, pyc_keywds,"", + capi_kwlist)) { + capi_success = !PyErr_Occurred(); + if (capi_success) { + pyc_buildvalue = Py_BuildValue(""); + } + } + return pyc_buildvalue; + } + >>> f = PyCFunction('foo', title=' Function title.\\nSecond line.', description=' This is a function.\\n2nd line.') + >>> e = PyCModule('PyCFunction_test', f) + >>> mod = e.build() + >>> print mod.foo.__doc__ + foo() -> None + <BLANKLINE> + Function title. + Second line. + <BLANKLINE> + This is a function. + 2nd line. + <BLANKLINE> + :Returns: + None + """ + + template = '''\ +static +char %(name)s_doc[] = +" %(FunctionSignature)s" +%(Title)s +%(Description)s +%(ReqArgDoc)s +%(RetArgDoc)s +%(OptArgDoc)s +%(ExtArgDoc)s +; +static +PyObject* +%(name)s(PyObject *pyc_self, PyObject *pyc_args, PyObject *pyc_keywds) { + PyObject * volatile pyc_buildvalue = NULL; + volatile int capi_success = 1; + %(CDeclaration)s + static char *capi_kwlist[] = {%(ReqKWList)s%(OptKWList)s%(ExtKWList)sNULL}; + if (PyArg_ParseTupleAndKeywords(pyc_args, pyc_keywds,"%(ReqArgFmt)s%(OptExtArgFmt)s", + capi_kwlist%(ReqArgObj)s%(OptArgObj)s%(ExtArgObj)s)) { + %(FromPyObj)s + %(CBody)s + capi_success = !PyErr_Occurred(); + if (capi_success) { + %(PyObjFrom)s + pyc_buildvalue = Py_BuildValue("%(RetArgFmt)s"%(RetArgObj)s); + %(CleanPyObjFrom)s + } + %(CleanCBody)s + %(CleanFromPyObj)s + } + return pyc_buildvalue; +}''' + + container_options = CFunction.container_options.copy() + + container_options.update(\ + + TMP = dict(), + + ReqArg = ReqArg.parent_container_options, + OptArg = OptArg.parent_container_options, + ExtArg = ExtArg.parent_container_options, + RetArg = RetArg.parent_container_options, + + FunctionSignature = FunctionSignature.parent_container_options, + + OptExtArg = OptExtArg.parent_container_options, + + Title = dict(default='<KILLLINE>',prefix='"\\n\\n',suffix='"',separator='\\n"\n"', + skip_prefix_when_empty=True, skip_suffix_when_empty=True, + use_firstline_indent=True, replace_map={'\n':'\\n'}), + Description = dict(default='<KILLLINE>',prefix='"\\n\\n"\n"', + suffix='"',separator='\\n"\n"', + skip_prefix_when_empty=True, skip_suffix_when_empty=True, + use_firstline_indent=True, replace_map={'\n':'\\n'}), + + ReqArgDoc = ReqArgDoc.parent_container_options, + OptArgDoc = OptArgDoc.parent_container_options, + ExtArgDoc = ExtArgDoc.parent_container_options, + RetArgDoc = RetArgDoc.parent_container_options, + + ReqKWList = ReqKWList.parent_container_options, + OptKWList = OptKWList.parent_container_options, + ExtKWList = ExtKWList.parent_container_options, + + ReqArgFmt = ReqArgFmt.parent_container_options, + OptArgFmt = OptArgFmt.parent_container_options, + ExtArgFmt = ExtArgFmt.parent_container_options, + OptExtArgFmt = OptExtArgFmt.ExtArgFmt.parent_container_options, + RetArgFmt = ExtArgFmt.parent_container_options, + + ReqArgObj = ReqArgObj.parent_container_options, + OptArgObj = OptArgObj.parent_container_options, + ExtArgObj = ExtArgObj.parent_container_options, + RetArgObj = RetArgObj.parent_container_options, + + FromPyObj = CCode.parent_container_options, + PyObjFrom = CCode.parent_container_options, + + CleanPyObjFrom = dict(default='<KILLLINE>', reverse=True, use_indent=True, ignore_empty_content=True), + CleanCBody = dict(default='<KILLLINE>', reverse=True, use_indent=True, ignore_empty_content=True), + CleanFromPyObj = dict(default='<KILLLINE>', reverse=True, use_indent=True, ignore_empty_content=True), + + ) + + default_component_class_name = 'CCode' + + component_container_map = CFunction.component_container_map.copy() + component_container_map.update( + PyCArgument = 'TMP', + CCode = 'CBody', + ) + + def initialize(self, pyname, *components, **options): + self.pyname = pyname + self.title = options.pop('title', None) + self.description = options.pop('description', None) + self = CFunction.initialize(self, 'pyc_function_'+pyname, 'PyObject*', **options) + self.signature = FunctionSignature(pyname) + self += self.signature + if self.title: + self.add(self.title, 'Title') + if self.description: + self.add(self.description, 'Description') + map(self.add, components) + return self + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr,[self.pyname]+[c for (c,l) in self.components]))) + + def update_parent(self, parent): + if isinstance(parent, PyCModule): + self.update_PyCModule(parent) + + def update_PyCModule(self, parent): + t = ' {"%(pyname)s", (PyCFunction)%(name)s, METH_VARARGS | METH_KEYWORDS, %(name)s_doc}' + parent.cdecl.add(self.evaluate(t),'PyMethodDef') + parent.cdecl.add(self.signature,'FunctionSignature') + + def update_containers(self): + self.container_OptExtArg += self.container_OptArg + self.container_ExtArg + self.container_OptExtArgFmt += self.container_OptArgFmt + self.container_ExtArgFmt + + +class PyCArgument(Component): + + """ + >>> from __init__ import * + >>> a = PyCArgument('a') + >>> print a + PyCArgument('a', PyCTypeSpec('object')) + >>> print a.generate() + a + >>> f = PyCFunction('foo') + >>> f += a + >>> f += PyCArgument('b') + >>> m = PyCModule('PyCArgument_test') + >>> m += f + >>> #print m.generate() + >>> mod = m.build() + >>> print mod.__doc__ #doctest: +ELLIPSIS + This module 'PyCArgument_test' is generated with ExtGen from NumPy version ... + <BLANKLINE> + :Functions: + foo(a, b) -> None + + """ + + container_options = dict( + TMP = dict() + ) + + component_container_map = dict( + PyCTypeSpec = 'TMP' + ) + + template = '%(name)s' + + def initialize(self, name, ctype = object, *components, **options): + self.input_intent = options.pop('input_intent','required') # 'optional', 'extra', 'hide' + self.output_intent = options.pop('output_intent','hide') # 'return' + self.input_title = options.pop('input_title', None) + self.output_title = options.pop('output_title', None) + self.input_description = options.pop('input_description', None) + self.output_description = options.pop('output_description', None) + self.depends = options.pop('depends', []) + title = options.pop('title', None) + description = options.pop('description', None) + if title is not None: + if self.input_intent!='hide': + if self.input_title is None: + self.input_title = title + elif self.output_intent!='hide': + if self.output_title is None: + self.output_title = title + if description is not None: + if self.input_intent!='hide': + if self.input_description is None: + self.input_description = description + elif self.output_intent!='hide': + if self.output_description is None: + self.output_description = description + if options: self.warning('%s unused options: %s\n' % (self.__class__.__name__, options)) + + self.name = name + self.ctype = ctype = PyCTypeSpec(ctype) + self += ctype + + self.cvar = name + self.pycvar = None + self.retpycvar = None + + retfmt = ctype.get_pyret_fmt(self) + if isinstance(ctype, PyCTypeSpec): + if retfmt and retfmt in 'SON': + if self.output_intent == 'return': + if self.input_intent=='hide': + self.retpycvar = name + else: + self.pycvar = name + self.retpycvar = name + '_return' + elif self.input_intent!='hide': + self.pycvar = name + else: + self.pycvar = name + self.retpycvar = name + else: + self.pycvar = name + '_pyc' + self.retpycvar = name + '_pyc_r' + + ctype.set_titles(self) + + map(self.add, components) + return self + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr,[self.name]+[c for (c,l) in self.components]))) + + def update_parent(self, parent): + if isinstance(parent, PyCFunction): + self.update_PyCFunction(parent) + + def update_PyCFunction(self, parent): + ctype = self.ctype + + input_doc_title = '%s : %s' % (self.name, self.input_title) + output_doc_title = '%s : %s' % (self.name, self.output_title) + if self.input_description is not None: + input_doc_descr = ' %s' % (self.input_description) + else: + input_doc_descr = None + if self.output_description is not None: + output_doc_descr = ' %s' % (self.output_description) + else: + output_doc_descr = None + + # add components to parent: + parent += ctype.get_decl(self) + if self.input_intent=='required': + parent += ReqArg(self.name) + parent.signature += ReqArg(self.name) + parent += ReqKWList('"' + self.name + '"') + parent += ReqArgFmt(ctype.get_pyarg_fmt(self)) + parent += ReqArgObj(ctype.get_pyarg_obj(self)) + parent += ReqArgDoc(input_doc_title) + parent += ReqArgDoc(input_doc_descr) + elif self.input_intent=='optional': + parent += OptArg(self.name) + parent.signature += OptArg(self.name) + parent += OptKWList('"' + self.name + '"') + parent += OptArgFmt(ctype.get_pyarg_fmt(self)) + parent += OptArgObj(ctype.get_pyarg_obj(self)) + parent += OptArgDoc(input_doc_title) + parent += OptArgDoc(input_doc_descr) + elif self.input_intent=='extra': + parent += ExtArg(self.name) + parent.signature += ExtArg(self.name) + parent += ExtKWList('"' + self.name + '"') + parent += ExtArgFmt(ctype.get_pyarg_fmt(self)) + parent += ExtArgObj(ctype.get_pyarg_obj(self)) + parent += ExtArgDoc(input_doc_title) + parent += ExtArgDoc(input_doc_descr) + elif self.input_intent=='hide': + pass + else: + raise NotImplementedError('input_intent=%r' % (self.input_intent)) + + if self.output_intent=='return': + parent += RetArg(self.name) + parent.signature += RetArg(self.name) + parent += RetArgFmt(ctype.get_pyret_fmt(self)) + parent += RetArgObj(ctype.get_pyret_obj(self)) + parent += RetArgDoc(output_doc_title) + parent += RetArgDoc(output_doc_descr) + elif self.output_intent=='hide': + pass + else: + raise NotImplementedError('output_intent=%r' % (self.output_intent)) + +class PyCReturn(PyCArgument): + + def initialize(self, name, ctype = object, *components, **options): + return PyCArgument(name, ctype, input_intent='hide', output_intent='return', *components, **options) + +class PyCTypeSpec(CTypeSpec): + + """ + >>> s = PyCTypeSpec(object) + >>> print s + PyCTypeSpec('object') + >>> print s.generate() + PyObject* + """ + + typeinfo_map = dict( + int = ('PyInt_Type', 'PyIntObject*', 'O!', 'N', 'NULL'), + long = ('PyLong_Type', 'PyLongObject*', 'O!', 'N', 'NULL'), + float = ('PyFloat_Type', 'PyFloatObject*', 'O!', 'N', 'NULL'), + complex = ('PyComplex_Type', 'PyComplexObject*', 'O!', 'N', 'NULL'), + str = ('PyString_Type', 'PyStringObject*', 'S', 'N', 'NULL'), + unicode = ('PyUnicode_Type', 'PyUnicodeObject*', 'U', 'N', 'NULL'), + buffer = ('PyBuffer_Type', 'PyBufferObject*', 'O!', 'N', 'NULL'), + tuple = ('PyTuple_Type', 'PyTupleObject*', 'O!', 'N', 'NULL'), + list = ('PyList_Type', 'PyListObject*', 'O!', 'N', 'NULL'), + dict = ('PyDict_Type', 'PyDictObject*', 'O!', 'N', 'NULL'), + file = ('PyFile_Type', 'PyFileObject*', 'O!', 'N', 'NULL'), + instance = ('PyInstance_Type', 'PyObject*', 'O!', 'N', 'NULL'), + function = ('PyFunction_Type', 'PyFunctionObject*', 'O!', 'N', 'NULL'), + method = ('PyMethod_Type', 'PyObject*', 'O!', 'N', 'NULL'), + module = ('PyModule_Type', 'PyObject*', 'O!', 'N', 'NULL'), + iter = ('PySeqIter_Type', 'PyObject*', 'O!', 'N', 'NULL'), + property = ('PyProperty_Type', 'PyObject*', 'O!', 'N', 'NULL'), + slice = ('PySlice_Type', 'PyObject*', 'O!', 'N', 'NULL'), + cell = ('PyCell_Type', 'PyCellObject*', 'O!', 'N', 'NULL'), + generator = ('PyGen_Type', 'PyGenObject*', 'O!', 'N', 'NULL'), + set = ('PySet_Type', 'PySetObject*', 'O!', 'N', 'NULL'), + frozenset = ('PyFrozenSet_Type', 'PySetObject*', 'O!', 'N', 'NULL'), + cobject = (None, 'PyCObject*', 'O', 'N', 'NULL'), + type = ('PyType_Type', 'PyTypeObject*', 'O!', 'N', 'NULL'), + object = (None, 'PyObject*', 'O', 'N', 'NULL'), + numpy_ndarray = ('PyArray_Type', 'PyArrayObject*', 'O!', 'N', 'NULL'), + numpy_descr = ('PyArrayDescr_Type','PyArray_Descr', 'O!', 'N', 'NULL'), + numpy_ufunc = ('PyUFunc_Type', 'PyUFuncObject*', 'O!', 'N', 'NULL'), + numpy_iter = ('PyArrayIter_Type', 'PyArrayIterObject*', 'O!', 'N', 'NULL'), + numpy_multiiter = ('PyArrayMultiIter_Type', 'PyArrayMultiIterObject*', 'O!', 'N', 'NULL'), + numpy_int8 = ('PyInt8ArrType_Type', 'PyInt8ScalarObject*', 'O!', 'N', 'NULL'), + numpy_int16 = ('PyInt16ArrType_Type', 'PyInt16ScalarObject*', 'O!', 'N', 'NULL'), + numpy_int32 = ('PyInt32ArrType_Type', 'PyInt32ScalarObject*', 'O!', 'N', 'NULL'), + numpy_int64 = ('PyInt64ArrType_Type', 'PyInt64ScalarObject*', 'O!', 'N', 'NULL'), + numpy_int128 = ('PyInt128ArrType_Type', 'PyInt128ScalarObject*', 'O!', 'N', 'NULL'), + numpy_uint8 = ('PyUInt8ArrType_Type', 'PyUInt8ScalarObject*', 'O!', 'N', 'NULL'), + numpy_uint16 = ('PyUInt16ArrType_Type', 'PyUInt16ScalarObject*', 'O!', 'N', 'NULL'), + numpy_uint32 = ('PyUInt32ArrType_Type', 'PyUInt32ScalarObject*', 'O!', 'N', 'NULL'), + numpy_uint64 = ('PyUInt64ArrType_Type', 'PyUInt64ScalarObject*', 'O!', 'N', 'NULL'), + numpy_uint128 = ('PyUInt128ArrType_Type', 'PyUInt128ScalarObject*', 'O!', 'N', 'NULL'), + numpy_float16 = ('PyFloat16ArrType_Type', 'PyFloat16ScalarObject*', 'O!', 'N', 'NULL'), + numpy_float32 = ('PyFloat32ArrType_Type', 'PyFloat32ScalarObject*', 'O!', 'N', 'NULL'), + numpy_float64 = ('PyFloat64ArrType_Type', 'PyFloat64ScalarObject*', 'O!', 'N', 'NULL'), + numpy_float80 = ('PyFloat80ArrType_Type', 'PyFloat80ScalarObject*', 'O!', 'N', 'NULL'), + numpy_float96 = ('PyFloat96ArrType_Type', 'PyFloat96ScalarObject*', 'O!', 'N', 'NULL'), + numpy_float128 = ('PyFloat128ArrType_Type', 'PyFloat128ScalarObject*', 'O!', 'N', 'NULL'), + numpy_complex32 = ('PyComplex32ArrType_Type', 'PyComplex32ScalarObject*', 'O!', 'N', 'NULL'), + numpy_complex64 = ('PyComplex64ArrType_Type', 'PyComplex64ScalarObject*', 'O!', 'N', 'NULL'), + numpy_complex128 = ('PyComplex128ArrType_Type', 'PyComplex128ScalarObject*', 'O!', 'N', 'NULL'), + numpy_complex160 = ('PyComplex160ArrType_Type', 'PyComplex160ScalarObject*', 'O!', 'N', 'NULL'), + numpy_complex192 = ('PyComplex192ArrType_Type', 'PyComplex192ScalarObject*', 'O!', 'N', 'NULL'), + numpy_complex256 = ('PyComplex256ArrType_Type', 'PyComplex256ScalarObject*', 'O!', 'N', 'NULL'), + numeric_array = ('PyArray_Type', 'PyArrayObject*', 'O!', 'N', 'NULL'), + c_char = (None, 'char', 'b', 'b', '0'), + c_unsigned_char = (None, 'unsigned char', 'B', 'B', '0'), + c_short = (None, 'short int', 'h', 'h', '0'), + c_unsigned_short = (None, 'unsigned short int', 'H', 'H', '0'), + c_int = (None,'int', 'i', 'i', '0'), + c_unsigned_int = (None,'unsigned int', 'I', 'I', '0'), + c_long = (None,'long', 'l', 'l', '0'), + c_unsigned_long = (None,'unsigned long', 'k', 'k', '0'), + c_long_long = (None,'PY_LONG_LONG', 'L', 'L', '0'), + c_unsigned_long_long = (None,'unsigned PY_LONG_LONG', 'K', 'K', '0'), + c_Py_ssize_t = (None,'Py_ssize_t', 'n', 'n', '0'), + c_char1 = (None,'char', 'c', 'c', '"\\0"'), + c_float = (None,'float', 'f', 'f', '0.0'), + c_double = (None,'double', 'd', 'd', '0.0'), + c_Py_complex = (None,'Py_complex', 'D', 'D', '{0.0, 0.0}'), + c_const_char_ptr = (None,'const char *', 'z', 'z', 'NULL'), + c_Py_UNICODE = (None,'Py_UNICODE*','u','u', 'NULL'), + ) + + def initialize(self, typeobj): + if isinstance(typeobj, self.__class__): + return typeobj + + m = self.typeinfo_map + + key = None + if isinstance(typeobj, type): + if typeobj.__module__=='__builtin__': + key = typeobj.__name__ + if key=='array': + key = 'numeric_array' + elif typeobj.__module__=='numpy': + key = 'numpy_' + typeobj.__name__ + elif isinstance(typeobj, str): + key = typeobj + if key.startswith('numpy_'): + k = key[6:] + named_scalars = ['byte','short','int','long','longlong', + 'ubyte','ushort','uint','ulong','ulonglong', + 'intp','uintp', + 'float_','double', + 'longfloat','longdouble', + 'complex_', + ] + if k in named_scalars: + import numpy + key = 'numpy_' + getattr(numpy, k).__name__ + + try: item = m[key] + except KeyError: + raise NotImplementedError('%s: need %s support' % (self.__class__.__name__, typeobj)) + + self.typeobj_name = key + self.ctypeobj = item[0] + self.line = item[1] + self.arg_fmt = item[2] + self.ret_fmt = item[3] + self.cinit_value = item[4] + + #if key.startswith('numpy_'): + # self.add(Component.get('arrayobject.h'), 'Header') + # self.add(Component.get('import_array'), 'ModuleInit') + if key.startswith('numeric_'): + raise NotImplementedError(self.__class__.__name__ + ': Numeric support') + + return self + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, ', '.join([repr(self.typeobj_name)]+[repr(c) for (c,l) in self.components])) + + def get_pyarg_fmt(self, arg): + if arg.input_intent=='hide': return None + return self.arg_fmt + + def get_pyarg_obj(self, arg): + if arg.input_intent=='hide': return None + if self.arg_fmt=='O!': + return '&%s, &%s' % (self.ctypeobj, arg.pycvar) + return '&' + arg.pycvar + + def get_pyret_fmt(self, arg): + if arg.output_intent=='hide': return None + return self.ret_fmt + + def get_pyret_obj(self, arg): + if arg.output_intent=='return': + if self.get_pyret_fmt(arg)=='D': + return '&' + arg.retpycvar + return arg.retpycvar + return + + def get_init_value(self, arg): + return self.cinit_value + + def set_titles(self, arg): + if self.typeobj_name == 'object': + tn = 'a python ' + self.typeobj_name + else: + if self.typeobj_name.startswith('numpy_'): + tn = 'a numpy.' + self.typeobj_name[6:] + ' object' + elif self.typeobj_name.startswith('c_'): + n = self.typeobj_name[2:] + if not n.startswith('Py_'): + n = ' '.join(n.split('_')) + tn = 'a to C ' + n + ' convertable object' + else: + tn = 'a python ' + self.typeobj_name + ' object' + if arg.input_intent!='hide': + r = '' + if arg.input_title: r = ', ' + arg.input_title + arg.input_title = tn + r + if arg.output_intent!='hide': + r = '' + if arg.output_title: r = ', ' + arg.output_title + arg.output_title = tn + r + + def get_decl(self, arg): + init_value = self.get_init_value(arg) + if init_value: + init = ' = %s' % (init_value) + else: + init = '' + if arg.pycvar and arg.pycvar==arg.retpycvar: + return CDeclaration(self, '%s%s' % (arg.pycvar, init)) + else: + if arg.input_intent!='hide': + return CDeclaration(self, '%s%s' % (arg.pycvar, init)) + if arg.output_intent!='hide': + return CDeclaration(self, '%s%s' % (arg.retpycvar, init)) + return + +def _test(): + import doctest + doctest.testmod() + +if __name__ == "__main__": + _test() diff --git a/numpy/f2py/lib/extgen/setup_py.py b/numpy/f2py/lib/extgen/setup_py.py new file mode 100644 index 000000000..9c34194d9 --- /dev/null +++ b/numpy/f2py/lib/extgen/setup_py.py @@ -0,0 +1,103 @@ + +__all__ = ['SetupPy'] + +import os +import sys +from numpy.distutils.exec_command import exec_command +from base import Component +from utils import FileSource + +def write_files(container): + s = ['creating files and directories:'] + for filename, i in container.label_map.items(): + content = container.list[i] + d,f = os.path.split(filename) + if d and not os.path.isdir(d): + s.append(' %s/' % (d)) + if not Component._generate_dry_run: + os.makedirs(d) + s.append(' %s' % (filename)) + if not Component._generate_dry_run: + f = file(filename,'w') + f.write(content) + f.close() + return '\n'.join(s) + + +class SetupPy(Component): + + """ + >>> from __init__ import * + >>> s = SetupPy('SetupPy_doctest') + >>> s += PyCModule('foo') + >>> s,o = s.execute('build_ext', '--inplace') + >>> assert s==0,`s` + >>> import SetupPy_doctest as mypackage + >>> print mypackage.foo.__doc__ #doctest: +ELLIPSIS + This module 'foo' is generated with ExtGen from NumPy version... + + """ + template_setup_py_start = '''\ +def configuration(parent_package='', top_path = ''): + from numpy.distutils.misc_util import Configuration + config = Configuration('',parent_package,top_path)''' + template_setup_py_end = '''\ + return config +if __name__ == "__main__": + from numpy.distutils.core import setup + setup(configuration=configuration) +''' + template = '%(SourceWriter)s' + + container_options = dict( + SourceWriter = dict(user_defined_str = write_files), + TMP = dict() + ) + + component_container_map = dict( + FileSource = 'SourceWriter', + ExtensionModule = 'TMP', + ) + + def initialize(self, build_dir, *components, **options): + self.name = self.path = build_dir + if not self.path: + self.setup_py = setup_py = Component.PySource('extgen_setup.py') + self.init_py = init_py = Component.PySource('extgen__init__.py') + else: + self.setup_py = setup_py = Component.PySource('setup.py') + self.init_py = init_py = Component.PySource('__init__.py') + + setup_py += self.template_setup_py_start + + self += init_py + self += setup_py + + map(self.add, components) + + return self + + def finalize(self): + self.setup_py += self.template_setup_py_end + + def execute(self, *args): + """ + Run generated setup.py file with given arguments. + """ + if not args: + raise ValueError('need setup.py arguments') + self.info(self.generate(dry_run=False)) + cmd = [sys.executable,'setup.py'] + list(args) + self.info('entering %r directory' % (self.path)) + self.info('executing command %r' % (' '.join(cmd))) + r = exec_command(cmd, execute_in=self.path, use_tee=False) + self.info('leaving %r directory' % (self.path)) + return r + + +def _test(): + import doctest + doctest.testmod() + +if __name__ == "__main__": + _test() diff --git a/numpy/f2py/lib/extgen/utils.py b/numpy/f2py/lib/extgen/utils.py new file mode 100644 index 000000000..b3666c8f6 --- /dev/null +++ b/numpy/f2py/lib/extgen/utils.py @@ -0,0 +1,126 @@ + +__all__ = ['Word', 'Line', 'Code', 'FileSource'] + +from base import Component + +class Word(Component): + template = '%(word)s' + + def initialize(self, word): + if not word: return None + self.word = word + return self + + def add(self, component, container_label=None): + raise ValueError('%s does not take components' % (self.__class__.__name__)) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr,[self.word]+[c for (c,l) in self.components]))) + + +class Line(Component): + + """ + >>> l = Line('hey') + >>> l += ' you ' + >>> l += 2 + >>> print l + Line('hey you 2') + >>> print l.generate() + hey you 2 + >>> l += l + >>> print l.generate() + hey you 2hey you 2 + """ + + template = '%(line)s' + + def initialize(self, *strings): + self.line = '' + map(self.add, strings) + return self + + def add(self, component, container_label=None): + if isinstance(component, Line): + self.line += component.line + elif isinstance(component, str): + self.line += component + elif component is None: + pass + else: + self.line += str(component) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr,[self.line]+[c for (c,l) in self.components]))) + + +class Code(Component): + + """ + >>> c = Code('start') + >>> c += 2 + >>> c += 'end' + >>> c + Code(Line('start'), Line('2'), Line('end')) + >>> print c.generate() + start + 2 + end + """ + + template = '%(Line)s' + + container_options = dict( + Line = dict(default = '<KILLLINE>', ignore_empty_content=True) + ) + component_container_map = dict( + Line = 'Line' + ) + default_component_class_name = 'Line' + + def initialize(self, *lines): + map(self.add, lines) + return self + + def add(self, component, label=None): + if isinstance(component, Code): + assert label is None,`label` + self.components += component.components + else: + Component.add(self, component, label) + + +class FileSource(Component): + + container_options = dict( + Content = dict(default='<KILLLINE>') + ) + + template = '%(Content)s' + + default_component_class_name = 'Code' + + component_container_map = dict( + Line = 'Content', + Code = 'Content', + ) + + def initialize(self, path, *components, **options): + self.path = path + map(self.add, components) + self._provides = options.pop('provides', path) + if options: self.warning('%s unused options: %s\n' % (self.__class__.__name__, options)) + return self + + def finalize(self): + self._provides = self.get_path() or self._provides + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr,[self.path]+[c for (c,l) in self.components]))) + +def _test(): + import doctest + doctest.testmod() + +if __name__ == "__main__": + _test() |