diff options
author | Jim Rollenhagen <jim@jimrollenhagen.com> | 2019-09-26 09:56:42 -0400 |
---|---|---|
committer | Jim Rollenhagen <jim@jimrollenhagen.com> | 2019-09-26 09:56:42 -0400 |
commit | 52672a64cc0cab4ea14a4a756fce850eb03315e3 (patch) | |
tree | a86024e4e6141aa8983c750f751c58d924f5b11a /migrate/versioning | |
parent | 8acab2cd75a5b23ac162e49c8e4fb1e3f958352a (diff) | |
download | sqlalchemy-migrate-master.tar.gz |
Diffstat (limited to 'migrate/versioning')
37 files changed, 0 insertions, 2939 deletions
diff --git a/migrate/versioning/__init__.py b/migrate/versioning/__init__.py deleted file mode 100644 index 8b5a736..0000000 --- a/migrate/versioning/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" - This package provides functionality to create and manage - repositories of database schema changesets and to apply these - changesets to databases. -""" diff --git a/migrate/versioning/api.py b/migrate/versioning/api.py deleted file mode 100644 index 570dc08..0000000 --- a/migrate/versioning/api.py +++ /dev/null @@ -1,384 +0,0 @@ -""" - This module provides an external API to the versioning system. - - .. versionchanged:: 0.6.0 - :func:`migrate.versioning.api.test` and schema diff functions - changed order of positional arguments so all accept `url` and `repository` - as first arguments. - - .. versionchanged:: 0.5.4 - ``--preview_sql`` displays source file when using SQL scripts. - If Python script is used, it runs the action with mocked engine and - returns captured SQL statements. - - .. versionchanged:: 0.5.4 - Deprecated ``--echo`` parameter in favour of new - :func:`migrate.versioning.util.construct_engine` behavior. -""" - -# Dear migrate developers, -# -# please do not comment this module using sphinx syntax because its -# docstrings are presented as user help and most users cannot -# interpret sphinx annotated ReStructuredText. -# -# Thanks, -# Jan Dittberner - -import sys -import inspect -import logging - -from migrate import exceptions -from migrate.versioning import (repository, schema, version, - script as script_) # command name conflict -from migrate.versioning.util import catch_known_errors, with_engine - - -log = logging.getLogger(__name__) -command_desc = { - 'help': 'displays help on a given command', - 'create': 'create an empty repository at the specified path', - 'script': 'create an empty change Python script', - 'script_sql': 'create empty change SQL scripts for given database', - 'version': 'display the latest version available in a repository', - 'db_version': 'show the current version of the repository under version control', - 'source': 'display the Python code for a particular version in this repository', - 'version_control': 'mark a database as under this repository\'s version control', - 'upgrade': 'upgrade a database to a later version', - 'downgrade': 'downgrade a database to an earlier version', - 'drop_version_control': 'removes version control from a database', - 'manage': 'creates a Python script that runs Migrate with a set of default values', - 'test': 'performs the upgrade and downgrade command on the given database', - 'compare_model_to_db': 'compare MetaData against the current database state', - 'create_model': 'dump the current database as a Python model to stdout', - 'make_update_script_for_model': 'create a script changing the old MetaData to the new (current) MetaData', - 'update_db_from_model': 'modify the database to match the structure of the current MetaData', -} -__all__ = command_desc.keys() - -Repository = repository.Repository -ControlledSchema = schema.ControlledSchema -VerNum = version.VerNum -PythonScript = script_.PythonScript -SqlScript = script_.SqlScript - - -# deprecated -def help(cmd=None, **opts): - """%prog help COMMAND - - Displays help on a given command. - """ - if cmd is None: - raise exceptions.UsageError(None) - try: - func = globals()[cmd] - except: - raise exceptions.UsageError( - "'%s' isn't a valid command. Try 'help COMMAND'" % cmd) - ret = func.__doc__ - if sys.argv[0]: - ret = ret.replace('%prog', sys.argv[0]) - return ret - -@catch_known_errors -def create(repository, name, **opts): - """%prog create REPOSITORY_PATH NAME [--table=TABLE] - - Create an empty repository at the specified path. - - You can specify the version_table to be used; by default, it is - 'migrate_version'. This table is created in all version-controlled - databases. - """ - repo_path = Repository.create(repository, name, **opts) - - -@catch_known_errors -def script(description, repository, **opts): - """%prog script DESCRIPTION REPOSITORY_PATH - - Create an empty change script using the next unused version number - appended with the given description. - - For instance, manage.py script "Add initial tables" creates: - repository/versions/001_Add_initial_tables.py - """ - repo = Repository(repository) - repo.create_script(description, **opts) - - -@catch_known_errors -def script_sql(database, description, repository, **opts): - """%prog script_sql DATABASE DESCRIPTION REPOSITORY_PATH - - Create empty change SQL scripts for given DATABASE, where DATABASE - is either specific ('postgresql', 'mysql', 'oracle', 'sqlite', etc.) - or generic ('default'). - - For instance, manage.py script_sql postgresql description creates: - repository/versions/001_description_postgresql_upgrade.sql and - repository/versions/001_description_postgresql_downgrade.sql - """ - repo = Repository(repository) - repo.create_script_sql(database, description, **opts) - - -def version(repository, **opts): - """%prog version REPOSITORY_PATH - - Display the latest version available in a repository. - """ - repo = Repository(repository) - return repo.latest - - -@with_engine -def db_version(url, repository, **opts): - """%prog db_version URL REPOSITORY_PATH - - Show the current version of the repository with the given - connection string, under version control of the specified - repository. - - The url should be any valid SQLAlchemy connection string. - """ - engine = opts.pop('engine') - schema = ControlledSchema(engine, repository) - return schema.version - - -def source(version, dest=None, repository=None, **opts): - """%prog source VERSION [DESTINATION] --repository=REPOSITORY_PATH - - Display the Python code for a particular version in this - repository. Save it to the file at DESTINATION or, if omitted, - send to stdout. - """ - if repository is None: - raise exceptions.UsageError("A repository must be specified") - repo = Repository(repository) - ret = repo.version(version).script().source() - if dest is not None: - dest = open(dest, 'w') - dest.write(ret) - dest.close() - ret = None - return ret - - -def upgrade(url, repository, version=None, **opts): - """%prog upgrade URL REPOSITORY_PATH [VERSION] [--preview_py|--preview_sql] - - Upgrade a database to a later version. - - This runs the upgrade() function defined in your change scripts. - - By default, the database is updated to the latest available - version. You may specify a version instead, if you wish. - - You may preview the Python or SQL code to be executed, rather than - actually executing it, using the appropriate 'preview' option. - """ - err = "Cannot upgrade a database of version %s to version %s. "\ - "Try 'downgrade' instead." - return _migrate(url, repository, version, upgrade=True, err=err, **opts) - - -def downgrade(url, repository, version, **opts): - """%prog downgrade URL REPOSITORY_PATH VERSION [--preview_py|--preview_sql] - - Downgrade a database to an earlier version. - - This is the reverse of upgrade; this runs the downgrade() function - defined in your change scripts. - - You may preview the Python or SQL code to be executed, rather than - actually executing it, using the appropriate 'preview' option. - """ - err = "Cannot downgrade a database of version %s to version %s. "\ - "Try 'upgrade' instead." - return _migrate(url, repository, version, upgrade=False, err=err, **opts) - -@with_engine -def test(url, repository, **opts): - """%prog test URL REPOSITORY_PATH [VERSION] - - Performs the upgrade and downgrade option on the given - database. This is not a real test and may leave the database in a - bad state. You should therefore better run the test on a copy of - your database. - """ - engine = opts.pop('engine') - repos = Repository(repository) - - # Upgrade - log.info("Upgrading...") - script = repos.version(None).script(engine.name, 'upgrade') - script.run(engine, 1) - log.info("done") - - log.info("Downgrading...") - script = repos.version(None).script(engine.name, 'downgrade') - script.run(engine, -1) - log.info("done") - log.info("Success") - - -@with_engine -def version_control(url, repository, version=None, **opts): - """%prog version_control URL REPOSITORY_PATH [VERSION] - - Mark a database as under this repository's version control. - - Once a database is under version control, schema changes should - only be done via change scripts in this repository. - - This creates the table version_table in the database. - - The url should be any valid SQLAlchemy connection string. - - By default, the database begins at version 0 and is assumed to be - empty. If the database is not empty, you may specify a version at - which to begin instead. No attempt is made to verify this - version's correctness - the database schema is expected to be - identical to what it would be if the database were created from - scratch. - """ - engine = opts.pop('engine') - ControlledSchema.create(engine, repository, version) - - -@with_engine -def drop_version_control(url, repository, **opts): - """%prog drop_version_control URL REPOSITORY_PATH - - Removes version control from a database. - """ - engine = opts.pop('engine') - schema = ControlledSchema(engine, repository) - schema.drop() - - -def manage(file, **opts): - """%prog manage FILENAME [VARIABLES...] - - Creates a script that runs Migrate with a set of default values. - - For example:: - - %prog manage manage.py --repository=/path/to/repository \ ---url=sqlite:///project.db - - would create the script manage.py. The following two commands - would then have exactly the same results:: - - python manage.py version - %prog version --repository=/path/to/repository - """ - Repository.create_manage_file(file, **opts) - - -@with_engine -def compare_model_to_db(url, repository, model, **opts): - """%prog compare_model_to_db URL REPOSITORY_PATH MODEL - - Compare the current model (assumed to be a module level variable - of type sqlalchemy.MetaData) against the current database. - - NOTE: This is EXPERIMENTAL. - """ # TODO: get rid of EXPERIMENTAL label - engine = opts.pop('engine') - return ControlledSchema.compare_model_to_db(engine, model, repository) - - -@with_engine -def create_model(url, repository, **opts): - """%prog create_model URL REPOSITORY_PATH [DECLERATIVE=True] - - Dump the current database as a Python model to stdout. - - NOTE: This is EXPERIMENTAL. - """ # TODO: get rid of EXPERIMENTAL label - engine = opts.pop('engine') - declarative = opts.get('declarative', False) - return ControlledSchema.create_model(engine, repository, declarative) - - -@catch_known_errors -@with_engine -def make_update_script_for_model(url, repository, oldmodel, model, **opts): - """%prog make_update_script_for_model URL OLDMODEL MODEL REPOSITORY_PATH - - Create a script changing the old Python model to the new (current) - Python model, sending to stdout. - - NOTE: This is EXPERIMENTAL. - """ # TODO: get rid of EXPERIMENTAL label - engine = opts.pop('engine') - return PythonScript.make_update_script_for_model( - engine, oldmodel, model, repository, **opts) - - -@with_engine -def update_db_from_model(url, repository, model, **opts): - """%prog update_db_from_model URL REPOSITORY_PATH MODEL - - Modify the database to match the structure of the current Python - model. This also sets the db_version number to the latest in the - repository. - - NOTE: This is EXPERIMENTAL. - """ # TODO: get rid of EXPERIMENTAL label - engine = opts.pop('engine') - schema = ControlledSchema(engine, repository) - schema.update_db_from_model(model) - -@with_engine -def _migrate(url, repository, version, upgrade, err, **opts): - engine = opts.pop('engine') - url = str(engine.url) - schema = ControlledSchema(engine, repository) - version = _migrate_version(schema, version, upgrade, err) - - changeset = schema.changeset(version) - for ver, change in changeset: - nextver = ver + changeset.step - log.info('%s -> %s... ', ver, nextver) - - if opts.get('preview_sql'): - if isinstance(change, PythonScript): - log.info(change.preview_sql(url, changeset.step, **opts)) - elif isinstance(change, SqlScript): - log.info(change.source()) - - elif opts.get('preview_py'): - if not isinstance(change, PythonScript): - raise exceptions.UsageError("Python source can be only displayed" - " for python migration files") - source_ver = max(ver, nextver) - module = schema.repository.version(source_ver).script().module - funcname = upgrade and "upgrade" or "downgrade" - func = getattr(module, funcname) - log.info(inspect.getsource(func)) - else: - schema.runchange(ver, change, changeset.step) - log.info('done') - - -def _migrate_version(schema, version, upgrade, err): - if version is None: - return version - # Version is specified: ensure we're upgrading in the right direction - # (current version < target version for upgrading; reverse for down) - version = VerNum(version) - cur = schema.version - if upgrade is not None: - if upgrade: - direction = cur <= version - else: - direction = cur >= version - if not direction: - raise exceptions.KnownError(err % (cur, version)) - return version diff --git a/migrate/versioning/cfgparse.py b/migrate/versioning/cfgparse.py deleted file mode 100644 index 8f1ccf9..0000000 --- a/migrate/versioning/cfgparse.py +++ /dev/null @@ -1,27 +0,0 @@ -""" - Configuration parser module. -""" - -from six.moves.configparser import ConfigParser - -from migrate.versioning.config import * -from migrate.versioning import pathed - - -class Parser(ConfigParser): - """A project configuration file.""" - - def to_dict(self, sections=None): - """It's easier to access config values like dictionaries""" - return self._sections - - -class Config(pathed.Pathed, Parser): - """Configuration class.""" - - def __init__(self, path, *p, **k): - """Confirm the config file exists; read it.""" - self.require_found(path) - pathed.Pathed.__init__(self, path) - Parser.__init__(self, *p, **k) - self.read(path) diff --git a/migrate/versioning/config.py b/migrate/versioning/config.py deleted file mode 100644 index 2429fd8..0000000 --- a/migrate/versioning/config.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -from sqlalchemy.util import OrderedDict - - -__all__ = ['databases', 'operations'] - -databases = ('sqlite', 'postgres', 'mysql', 'oracle', 'mssql', 'firebird') - -# Map operation names to function names -operations = OrderedDict() -operations['upgrade'] = 'upgrade' -operations['downgrade'] = 'downgrade' diff --git a/migrate/versioning/genmodel.py b/migrate/versioning/genmodel.py deleted file mode 100644 index 4d9cd12..0000000 --- a/migrate/versioning/genmodel.py +++ /dev/null @@ -1,306 +0,0 @@ -""" -Code to generate a Python model from a database or differences -between a model and database. - -Some of this is borrowed heavily from the AutoCode project at: -http://code.google.com/p/sqlautocode/ -""" - -import sys -import logging - -import six -import sqlalchemy - -import migrate -import migrate.changeset - - -log = logging.getLogger(__name__) -HEADER = """ -## File autogenerated by genmodel.py - -from sqlalchemy import * -""" - -META_DEFINITION = "meta = MetaData()" - -DECLARATIVE_DEFINITION = """ -from sqlalchemy.ext import declarative - -Base = declarative.declarative_base() -""" - - -class ModelGenerator(object): - """Various transformations from an A, B diff. - - In the implementation, A tends to be called the model and B - the database (although this is not true of all diffs). - The diff is directionless, but transformations apply the diff - in a particular direction, described in the method name. - """ - - def __init__(self, diff, engine, declarative=False): - self.diff = diff - self.engine = engine - self.declarative = declarative - - def column_repr(self, col): - kwarg = [] - if col.key != col.name: - kwarg.append('key') - if col.primary_key: - col.primary_key = True # otherwise it dumps it as 1 - kwarg.append('primary_key') - if not col.nullable: - kwarg.append('nullable') - if col.onupdate: - kwarg.append('onupdate') - if col.default: - if col.primary_key: - # I found that PostgreSQL automatically creates a - # default value for the sequence, but let's not show - # that. - pass - else: - kwarg.append('default') - args = ['%s=%r' % (k, getattr(col, k)) for k in kwarg] - - # crs: not sure if this is good idea, but it gets rid of extra - # u'' - if six.PY3: - name = col.name - else: - name = col.name.encode('utf8') - - type_ = col.type - for cls in col.type.__class__.__mro__: - if cls.__module__ == 'sqlalchemy.types' and \ - not cls.__name__.isupper(): - if cls is not type_.__class__: - type_ = cls() - break - - type_repr = repr(type_) - if type_repr.endswith('()'): - type_repr = type_repr[:-2] - - constraints = [repr(cn) for cn in col.constraints] - - data = { - 'name': name, - 'commonStuff': ', '.join([type_repr] + constraints + args), - } - - if self.declarative: - return """%(name)s = Column(%(commonStuff)s)""" % data - else: - return """Column(%(name)r, %(commonStuff)s)""" % data - - def _getTableDefn(self, table, metaName='meta'): - out = [] - tableName = table.name - if self.declarative: - out.append("class %(table)s(Base):" % {'table': tableName}) - out.append(" __tablename__ = '%(table)s'\n" % - {'table': tableName}) - for col in table.columns: - out.append(" %s" % self.column_repr(col)) - out.append('\n') - else: - out.append("%(table)s = Table('%(table)s', %(meta)s," % - {'table': tableName, 'meta': metaName}) - for col in table.columns: - out.append(" %s," % self.column_repr(col)) - out.append(")\n") - return out - - def _get_tables(self,missingA=False,missingB=False,modified=False): - to_process = [] - for bool_,names,metadata in ( - (missingA,self.diff.tables_missing_from_A,self.diff.metadataB), - (missingB,self.diff.tables_missing_from_B,self.diff.metadataA), - (modified,self.diff.tables_different,self.diff.metadataA), - ): - if bool_: - for name in names: - yield metadata.tables.get(name) - - def _genModelHeader(self, tables): - out = [] - import_index = [] - - out.append(HEADER) - - for table in tables: - for col in table.columns: - if "dialects" in col.type.__module__ and \ - col.type.__class__ not in import_index: - out.append("from " + col.type.__module__ + - " import " + col.type.__class__.__name__) - import_index.append(col.type.__class__) - - out.append("") - - if self.declarative: - out.append(DECLARATIVE_DEFINITION) - else: - out.append(META_DEFINITION) - out.append("") - - return out - - def genBDefinition(self): - """Generates the source code for a definition of B. - - Assumes a diff where A is empty. - - Was: toPython. Assume database (B) is current and model (A) is empty. - """ - - out = [] - out.extend(self._genModelHeader(self._get_tables(missingA=True))) - for table in self._get_tables(missingA=True): - out.extend(self._getTableDefn(table)) - return '\n'.join(out) - - def genB2AMigration(self, indent=' '): - '''Generate a migration from B to A. - - Was: toUpgradeDowngradePython - Assume model (A) is most current and database (B) is out-of-date. - ''' - - decls = ['from migrate.changeset import schema', - 'pre_meta = MetaData()', - 'post_meta = MetaData()', - ] - upgradeCommands = ['pre_meta.bind = migrate_engine', - 'post_meta.bind = migrate_engine'] - downgradeCommands = list(upgradeCommands) - - for tn in self.diff.tables_missing_from_A: - pre_table = self.diff.metadataB.tables[tn] - decls.extend(self._getTableDefn(pre_table, metaName='pre_meta')) - upgradeCommands.append( - "pre_meta.tables[%(table)r].drop()" % {'table': tn}) - downgradeCommands.append( - "pre_meta.tables[%(table)r].create()" % {'table': tn}) - - for tn in self.diff.tables_missing_from_B: - post_table = self.diff.metadataA.tables[tn] - decls.extend(self._getTableDefn(post_table, metaName='post_meta')) - upgradeCommands.append( - "post_meta.tables[%(table)r].create()" % {'table': tn}) - downgradeCommands.append( - "post_meta.tables[%(table)r].drop()" % {'table': tn}) - - for (tn, td) in six.iteritems(self.diff.tables_different): - if td.columns_missing_from_A or td.columns_different: - pre_table = self.diff.metadataB.tables[tn] - decls.extend(self._getTableDefn( - pre_table, metaName='pre_meta')) - if td.columns_missing_from_B or td.columns_different: - post_table = self.diff.metadataA.tables[tn] - decls.extend(self._getTableDefn( - post_table, metaName='post_meta')) - - for col in td.columns_missing_from_A: - upgradeCommands.append( - 'pre_meta.tables[%r].columns[%r].drop()' % (tn, col)) - downgradeCommands.append( - 'pre_meta.tables[%r].columns[%r].create()' % (tn, col)) - for col in td.columns_missing_from_B: - upgradeCommands.append( - 'post_meta.tables[%r].columns[%r].create()' % (tn, col)) - downgradeCommands.append( - 'post_meta.tables[%r].columns[%r].drop()' % (tn, col)) - for modelCol, databaseCol, modelDecl, databaseDecl in td.columns_different: - upgradeCommands.append( - 'assert False, "Can\'t alter columns: %s:%s=>%s"' % ( - tn, modelCol.name, databaseCol.name)) - downgradeCommands.append( - 'assert False, "Can\'t alter columns: %s:%s=>%s"' % ( - tn, modelCol.name, databaseCol.name)) - - return ( - '\n'.join(decls), - '\n'.join('%s%s' % (indent, line) for line in upgradeCommands), - '\n'.join('%s%s' % (indent, line) for line in downgradeCommands)) - - def _db_can_handle_this_change(self,td): - """Check if the database can handle going from B to A.""" - - if (td.columns_missing_from_B - and not td.columns_missing_from_A - and not td.columns_different): - # Even sqlite can handle column additions. - return True - else: - return not self.engine.url.drivername.startswith('sqlite') - - def runB2A(self): - """Goes from B to A. - - Was: applyModel. Apply model (A) to current database (B). - """ - - meta = sqlalchemy.MetaData(self.engine) - - for table in self._get_tables(missingA=True): - table = table.tometadata(meta) - table.drop() - for table in self._get_tables(missingB=True): - table = table.tometadata(meta) - table.create() - for modelTable in self._get_tables(modified=True): - tableName = modelTable.name - modelTable = modelTable.tometadata(meta) - dbTable = self.diff.metadataB.tables[tableName] - - td = self.diff.tables_different[tableName] - - if self._db_can_handle_this_change(td): - - for col in td.columns_missing_from_B: - modelTable.columns[col].create() - for col in td.columns_missing_from_A: - dbTable.columns[col].drop() - # XXX handle column changes here. - else: - # Sqlite doesn't support drop column, so you have to - # do more: create temp table, copy data to it, drop - # old table, create new table, copy data back. - # - # I wonder if this is guaranteed to be unique? - tempName = '_temp_%s' % modelTable.name - - def getCopyStatement(): - preparer = self.engine.dialect.preparer - commonCols = [] - for modelCol in modelTable.columns: - if modelCol.name in dbTable.columns: - commonCols.append(modelCol.name) - commonColsStr = ', '.join(commonCols) - return 'INSERT INTO %s (%s) SELECT %s FROM %s' % \ - (tableName, commonColsStr, commonColsStr, tempName) - - # Move the data in one transaction, so that we don't - # leave the database in a nasty state. - connection = self.engine.connect() - trans = connection.begin() - try: - connection.execute( - 'CREATE TEMPORARY TABLE %s as SELECT * from %s' % \ - (tempName, modelTable.name)) - # make sure the drop takes place inside our - # transaction with the bind parameter - modelTable.drop(bind=connection) - modelTable.create(bind=connection) - connection.execute(getCopyStatement()) - connection.execute('DROP TABLE %s' % tempName) - trans.commit() - except: - trans.rollback() - raise diff --git a/migrate/versioning/migrate_repository.py b/migrate/versioning/migrate_repository.py deleted file mode 100644 index 22bba47..0000000 --- a/migrate/versioning/migrate_repository.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - Script to migrate repository from sqlalchemy <= 0.4.4 to the new - repository schema. This shouldn't use any other migrate modules, so - that it can work in any version. -""" - -import os -import sys -import logging - -log = logging.getLogger(__name__) - - -def usage(): - """Gives usage information.""" - print("Usage: %s repository-to-migrate" % sys.argv[0]) - print("Upgrade your repository to the new flat format.") - print("NOTE: You should probably make a backup before running this.") - sys.exit(1) - - -def delete_file(filepath): - """Deletes a file and prints a message.""" - log.info('Deleting file: %s' % filepath) - os.remove(filepath) - - -def move_file(src, tgt): - """Moves a file and prints a message.""" - log.info('Moving file %s to %s' % (src, tgt)) - if os.path.exists(tgt): - raise Exception( - 'Cannot move file %s because target %s already exists' % \ - (src, tgt)) - os.rename(src, tgt) - - -def delete_directory(dirpath): - """Delete a directory and print a message.""" - log.info('Deleting directory: %s' % dirpath) - os.rmdir(dirpath) - - -def migrate_repository(repos): - """Does the actual migration to the new repository format.""" - log.info('Migrating repository at: %s to new format' % repos) - versions = '%s/versions' % repos - dirs = os.listdir(versions) - # Only use int's in list. - numdirs = [int(dirname) for dirname in dirs if dirname.isdigit()] - numdirs.sort() # Sort list. - for dirname in numdirs: - origdir = '%s/%s' % (versions, dirname) - log.info('Working on directory: %s' % origdir) - files = os.listdir(origdir) - files.sort() - for filename in files: - # Delete compiled Python files. - if filename.endswith('.pyc') or filename.endswith('.pyo'): - delete_file('%s/%s' % (origdir, filename)) - - # Delete empty __init__.py files. - origfile = '%s/__init__.py' % origdir - if os.path.exists(origfile) and len(open(origfile).read()) == 0: - delete_file(origfile) - - # Move sql upgrade scripts. - if filename.endswith('.sql'): - version, dbms, operation = filename.split('.', 3)[0:3] - origfile = '%s/%s' % (origdir, filename) - # For instance: 2.postgres.upgrade.sql -> - # 002_postgres_upgrade.sql - tgtfile = '%s/%03d_%s_%s.sql' % ( - versions, int(version), dbms, operation) - move_file(origfile, tgtfile) - - # Move Python upgrade script. - pyfile = '%s.py' % dirname - pyfilepath = '%s/%s' % (origdir, pyfile) - if os.path.exists(pyfilepath): - tgtfile = '%s/%03d.py' % (versions, int(dirname)) - move_file(pyfilepath, tgtfile) - - # Try to remove directory. Will fail if it's not empty. - delete_directory(origdir) - - -def main(): - """Main function to be called when using this script.""" - if len(sys.argv) != 2: - usage() - migrate_repository(sys.argv[1]) - - -if __name__ == '__main__': - main() diff --git a/migrate/versioning/pathed.py b/migrate/versioning/pathed.py deleted file mode 100644 index fbee0e4..0000000 --- a/migrate/versioning/pathed.py +++ /dev/null @@ -1,75 +0,0 @@ -""" - A path/directory class. -""" - -import os -import shutil -import logging - -from migrate import exceptions -from migrate.versioning.config import * -from migrate.versioning.util import KeyedInstance - - -log = logging.getLogger(__name__) - -class Pathed(KeyedInstance): - """ - A class associated with a path/directory tree. - - Only one instance of this class may exist for a particular file; - __new__ will return an existing instance if possible - """ - parent = None - - @classmethod - def _key(cls, path): - return str(path) - - def __init__(self, path): - self.path = path - if self.__class__.parent is not None: - self._init_parent(path) - - def _init_parent(self, path): - """Try to initialize this object's parent, if it has one""" - parent_path = self.__class__._parent_path(path) - self.parent = self.__class__.parent(parent_path) - log.debug("Getting parent %r:%r" % (self.__class__.parent, parent_path)) - self.parent._init_child(path, self) - - def _init_child(self, child, path): - """Run when a child of this object is initialized. - - Parameters: the child object; the path to this object (its - parent) - """ - - @classmethod - def _parent_path(cls, path): - """ - Fetch the path of this object's parent from this object's path. - """ - # os.path.dirname(), but strip directories like files (like - # unix basename) - # - # Treat directories like files... - if path[-1] == '/': - path = path[:-1] - ret = os.path.dirname(path) - return ret - - @classmethod - def require_notfound(cls, path): - """Ensures a given path does not already exist""" - if os.path.exists(path): - raise exceptions.PathFoundError(path) - - @classmethod - def require_found(cls, path): - """Ensures a given path already exists""" - if not os.path.exists(path): - raise exceptions.PathNotFoundError(path) - - def __str__(self): - return self.path diff --git a/migrate/versioning/repository.py b/migrate/versioning/repository.py deleted file mode 100644 index 8c8cd3b..0000000 --- a/migrate/versioning/repository.py +++ /dev/null @@ -1,242 +0,0 @@ -""" - SQLAlchemy migrate repository management. -""" -import os -import shutil -import string -import logging - -from pkg_resources import resource_filename -from tempita import Template as TempitaTemplate - -from migrate import exceptions -from migrate.versioning import version, pathed, cfgparse -from migrate.versioning.template import Template -from migrate.versioning.config import * - - -log = logging.getLogger(__name__) - -class Changeset(dict): - """A collection of changes to be applied to a database. - - Changesets are bound to a repository and manage a set of - scripts from that repository. - - Behaves like a dict, for the most part. Keys are ordered based on step value. - """ - - def __init__(self, start, *changes, **k): - """ - Give a start version; step must be explicitly stated. - """ - self.step = k.pop('step', 1) - self.start = version.VerNum(start) - self.end = self.start - for change in changes: - self.add(change) - - def __iter__(self): - return iter(self.items()) - - def keys(self): - """ - In a series of upgrades x -> y, keys are version x. Sorted. - """ - ret = list(super(Changeset, self).keys()) - # Reverse order if downgrading - ret.sort(reverse=(self.step < 1)) - return ret - - def values(self): - return [self[k] for k in self.keys()] - - def items(self): - return zip(self.keys(), self.values()) - - def add(self, change): - """Add new change to changeset""" - key = self.end - self.end += self.step - self[key] = change - - def run(self, *p, **k): - """Run the changeset scripts""" - for ver, script in self: - script.run(*p, **k) - - -class Repository(pathed.Pathed): - """A project's change script repository""" - - _config = 'migrate.cfg' - _versions = 'versions' - - def __init__(self, path): - log.debug('Loading repository %s...' % path) - self.verify(path) - super(Repository, self).__init__(path) - self.config = cfgparse.Config(os.path.join(self.path, self._config)) - self.versions = version.Collection(os.path.join(self.path, - self._versions)) - log.debug('Repository %s loaded successfully' % path) - log.debug('Config: %r' % self.config.to_dict()) - - @classmethod - def verify(cls, path): - """ - Ensure the target path is a valid repository. - - :raises: :exc:`InvalidRepositoryError <migrate.exceptions.InvalidRepositoryError>` - """ - # Ensure the existence of required files - try: - cls.require_found(path) - cls.require_found(os.path.join(path, cls._config)) - cls.require_found(os.path.join(path, cls._versions)) - except exceptions.PathNotFoundError: - raise exceptions.InvalidRepositoryError(path) - - @classmethod - def prepare_config(cls, tmpl_dir, name, options=None): - """ - Prepare a project configuration file for a new project. - - :param tmpl_dir: Path to Repository template - :param config_file: Name of the config file in Repository template - :param name: Repository name - :type tmpl_dir: string - :type config_file: string - :type name: string - :returns: Populated config file - """ - if options is None: - options = {} - options.setdefault('version_table', 'migrate_version') - options.setdefault('repository_id', name) - options.setdefault('required_dbs', []) - options.setdefault('use_timestamp_numbering', False) - - tmpl = open(os.path.join(tmpl_dir, cls._config)).read() - ret = TempitaTemplate(tmpl).substitute(options) - - # cleanup - del options['__template_name__'] - - return ret - - @classmethod - def create(cls, path, name, **opts): - """Create a repository at a specified path""" - cls.require_notfound(path) - theme = opts.pop('templates_theme', None) - t_path = opts.pop('templates_path', None) - - # Create repository - tmpl_dir = Template(t_path).get_repository(theme=theme) - shutil.copytree(tmpl_dir, path) - - # Edit config defaults - config_text = cls.prepare_config(tmpl_dir, name, options=opts) - fd = open(os.path.join(path, cls._config), 'w') - fd.write(config_text) - fd.close() - - opts['repository_name'] = name - - # Create a management script - manager = os.path.join(path, 'manage.py') - Repository.create_manage_file(manager, templates_theme=theme, - templates_path=t_path, **opts) - - return cls(path) - - def create_script(self, description, **k): - """API to :meth:`migrate.versioning.version.Collection.create_new_python_version`""" - - k['use_timestamp_numbering'] = self.use_timestamp_numbering - self.versions.create_new_python_version(description, **k) - - def create_script_sql(self, database, description, **k): - """API to :meth:`migrate.versioning.version.Collection.create_new_sql_version`""" - k['use_timestamp_numbering'] = self.use_timestamp_numbering - self.versions.create_new_sql_version(database, description, **k) - - @property - def latest(self): - """API to :attr:`migrate.versioning.version.Collection.latest`""" - return self.versions.latest - - @property - def version_table(self): - """Returns version_table name specified in config""" - return self.config.get('db_settings', 'version_table') - - @property - def id(self): - """Returns repository id specified in config""" - return self.config.get('db_settings', 'repository_id') - - @property - def use_timestamp_numbering(self): - """Returns use_timestamp_numbering specified in config""" - if self.config.has_option('db_settings', 'use_timestamp_numbering'): - return self.config.getboolean('db_settings', 'use_timestamp_numbering') - return False - - def version(self, *p, **k): - """API to :attr:`migrate.versioning.version.Collection.version`""" - return self.versions.version(*p, **k) - - @classmethod - def clear(cls): - # TODO: deletes repo - super(Repository, cls).clear() - version.Collection.clear() - - def changeset(self, database, start, end=None): - """Create a changeset to migrate this database from ver. start to end/latest. - - :param database: name of database to generate changeset - :param start: version to start at - :param end: version to end at (latest if None given) - :type database: string - :type start: int - :type end: int - :returns: :class:`Changeset instance <migration.versioning.repository.Changeset>` - """ - start = version.VerNum(start) - - if end is None: - end = self.latest - else: - end = version.VerNum(end) - - if start <= end: - step = 1 - range_mod = 1 - op = 'upgrade' - else: - step = -1 - range_mod = 0 - op = 'downgrade' - - versions = range(int(start) + range_mod, int(end) + range_mod, step) - changes = [self.version(v).script(database, op) for v in versions] - ret = Changeset(start, step=step, *changes) - return ret - - @classmethod - def create_manage_file(cls, file_, **opts): - """Create a project management script (manage.py) - - :param file_: Destination file to be written - :param opts: Options that are passed to :func:`migrate.versioning.shell.main` - """ - mng_file = Template(opts.pop('templates_path', None))\ - .get_manage(theme=opts.pop('templates_theme', None)) - - tmpl = open(mng_file).read() - fd = open(file_, 'w') - fd.write(TempitaTemplate(tmpl).substitute(opts)) - fd.close() diff --git a/migrate/versioning/schema.py b/migrate/versioning/schema.py deleted file mode 100644 index b525cef..0000000 --- a/migrate/versioning/schema.py +++ /dev/null @@ -1,222 +0,0 @@ -""" - Database schema version management. -""" -import sys -import logging - -import six -from sqlalchemy import (Table, Column, MetaData, String, Text, Integer, - create_engine) -from sqlalchemy.sql import and_ -from sqlalchemy import exc as sa_exceptions -from sqlalchemy.sql import bindparam - -from migrate import exceptions -from migrate.changeset import SQLA_07 -from migrate.versioning import genmodel, schemadiff -from migrate.versioning.repository import Repository -from migrate.versioning.util import load_model -from migrate.versioning.version import VerNum - - -log = logging.getLogger(__name__) - -class ControlledSchema(object): - """A database under version control""" - - def __init__(self, engine, repository): - if isinstance(repository, six.string_types): - repository = Repository(repository) - self.engine = engine - self.repository = repository - self.meta = MetaData(engine) - self.load() - - def __eq__(self, other): - """Compare two schemas by repositories and versions""" - return (self.repository is other.repository \ - and self.version == other.version) - - def load(self): - """Load controlled schema version info from DB""" - tname = self.repository.version_table - try: - if not hasattr(self, 'table') or self.table is None: - self.table = Table(tname, self.meta, autoload=True) - - result = self.engine.execute(self.table.select( - self.table.c.repository_id == str(self.repository.id))) - - data = list(result)[0] - except: - cls, exc, tb = sys.exc_info() - six.reraise(exceptions.DatabaseNotControlledError, - exceptions.DatabaseNotControlledError(str(exc)), tb) - - self.version = data['version'] - return data - - def drop(self): - """ - Remove version control from a database. - """ - if SQLA_07: - try: - self.table.drop() - except sa_exceptions.DatabaseError: - raise exceptions.DatabaseNotControlledError(str(self.table)) - else: - try: - self.table.drop() - except (sa_exceptions.SQLError): - raise exceptions.DatabaseNotControlledError(str(self.table)) - - def changeset(self, version=None): - """API to Changeset creation. - - Uses self.version for start version and engine.name - to get database name. - """ - database = self.engine.name - start_ver = self.version - changeset = self.repository.changeset(database, start_ver, version) - return changeset - - def runchange(self, ver, change, step): - startver = ver - endver = ver + step - # Current database version must be correct! Don't run if corrupt! - if self.version != startver: - raise exceptions.InvalidVersionError("%s is not %s" % \ - (self.version, startver)) - # Run the change - change.run(self.engine, step) - - # Update/refresh database version - self.update_repository_table(startver, endver) - self.load() - - def update_repository_table(self, startver, endver): - """Update version_table with new information""" - update = self.table.update(and_(self.table.c.version == int(startver), - self.table.c.repository_id == str(self.repository.id))) - self.engine.execute(update, version=int(endver)) - - def upgrade(self, version=None): - """ - Upgrade (or downgrade) to a specified version, or latest version. - """ - changeset = self.changeset(version) - for ver, change in changeset: - self.runchange(ver, change, changeset.step) - - def update_db_from_model(self, model): - """ - Modify the database to match the structure of the current Python model. - """ - model = load_model(model) - - diff = schemadiff.getDiffOfModelAgainstDatabase( - model, self.engine, excludeTables=[self.repository.version_table] - ) - genmodel.ModelGenerator(diff,self.engine).runB2A() - - self.update_repository_table(self.version, int(self.repository.latest)) - - self.load() - - @classmethod - def create(cls, engine, repository, version=None): - """ - Declare a database to be under a repository's version control. - - :raises: :exc:`DatabaseAlreadyControlledError` - :returns: :class:`ControlledSchema` - """ - # Confirm that the version # is valid: positive, integer, - # exists in repos - if isinstance(repository, six.string_types): - repository = Repository(repository) - version = cls._validate_version(repository, version) - table = cls._create_table_version(engine, repository, version) - # TODO: history table - # Load repository information and return - return cls(engine, repository) - - @classmethod - def _validate_version(cls, repository, version): - """ - Ensures this is a valid version number for this repository. - - :raises: :exc:`InvalidVersionError` if invalid - :return: valid version number - """ - if version is None: - version = 0 - try: - version = VerNum(version) # raises valueerror - if version < 0 or version > repository.latest: - raise ValueError() - except ValueError: - raise exceptions.InvalidVersionError(version) - return version - - @classmethod - def _create_table_version(cls, engine, repository, version): - """ - Creates the versioning table in a database. - - :raises: :exc:`DatabaseAlreadyControlledError` - """ - # Create tables - tname = repository.version_table - meta = MetaData(engine) - - table = Table( - tname, meta, - Column('repository_id', String(250), primary_key=True), - Column('repository_path', Text), - Column('version', Integer), ) - - # there can be multiple repositories/schemas in the same db - if not table.exists(): - table.create() - - # test for existing repository_id - s = table.select(table.c.repository_id == bindparam("repository_id")) - result = engine.execute(s, repository_id=repository.id) - if result.fetchone(): - raise exceptions.DatabaseAlreadyControlledError - - # Insert data - engine.execute(table.insert().values( - repository_id=repository.id, - repository_path=repository.path, - version=int(version))) - return table - - @classmethod - def compare_model_to_db(cls, engine, model, repository): - """ - Compare the current model against the current database. - """ - if isinstance(repository, six.string_types): - repository = Repository(repository) - model = load_model(model) - - diff = schemadiff.getDiffOfModelAgainstDatabase( - model, engine, excludeTables=[repository.version_table]) - return diff - - @classmethod - def create_model(cls, engine, repository, declarative=False): - """ - Dump the current database as a Python model. - """ - if isinstance(repository, six.string_types): - repository = Repository(repository) - - diff = schemadiff.getDiffOfModelAgainstDatabase( - MetaData(), engine, excludeTables=[repository.version_table] - ) - return genmodel.ModelGenerator(diff, engine, declarative).genBDefinition() diff --git a/migrate/versioning/schemadiff.py b/migrate/versioning/schemadiff.py deleted file mode 100644 index d9477bf..0000000 --- a/migrate/versioning/schemadiff.py +++ /dev/null @@ -1,298 +0,0 @@ -""" - Schema differencing support. -""" - -import logging -import sqlalchemy - -from sqlalchemy.types import Float - -log = logging.getLogger(__name__) - -def getDiffOfModelAgainstDatabase(metadata, engine, excludeTables=None): - """ - Return differences of model against database. - - :return: object which will evaluate to :keyword:`True` if there \ - are differences else :keyword:`False`. - """ - db_metadata = sqlalchemy.MetaData(engine) - db_metadata.reflect() - - # sqlite will include a dynamically generated 'sqlite_sequence' table if - # there are autoincrement sequences in the database; this should not be - # compared. - if engine.dialect.name == 'sqlite': - if 'sqlite_sequence' in db_metadata.tables: - db_metadata.remove(db_metadata.tables['sqlite_sequence']) - - return SchemaDiff(metadata, db_metadata, - labelA='model', - labelB='database', - excludeTables=excludeTables) - - -def getDiffOfModelAgainstModel(metadataA, metadataB, excludeTables=None): - """ - Return differences of model against another model. - - :return: object which will evaluate to :keyword:`True` if there \ - are differences else :keyword:`False`. - """ - return SchemaDiff(metadataA, metadataB, excludeTables=excludeTables) - - -class ColDiff(object): - """ - Container for differences in one :class:`~sqlalchemy.schema.Column` - between two :class:`~sqlalchemy.schema.Table` instances, ``A`` - and ``B``. - - .. attribute:: col_A - - The :class:`~sqlalchemy.schema.Column` object for A. - - .. attribute:: col_B - - The :class:`~sqlalchemy.schema.Column` object for B. - - .. attribute:: type_A - - The most generic type of the :class:`~sqlalchemy.schema.Column` - object in A. - - .. attribute:: type_B - - The most generic type of the :class:`~sqlalchemy.schema.Column` - object in A. - - """ - - diff = False - - def __init__(self,col_A,col_B): - self.col_A = col_A - self.col_B = col_B - - self.type_A = col_A.type - self.type_B = col_B.type - - self.affinity_A = self.type_A._type_affinity - self.affinity_B = self.type_B._type_affinity - - if self.affinity_A is not self.affinity_B: - self.diff = True - return - - if isinstance(self.type_A,Float) or isinstance(self.type_B,Float): - if not (isinstance(self.type_A,Float) and isinstance(self.type_B,Float)): - self.diff=True - return - - for attr in ('precision','scale','length'): - A = getattr(self.type_A,attr,None) - B = getattr(self.type_B,attr,None) - if not (A is None or B is None) and A!=B: - self.diff=True - return - - def __nonzero__(self): - return self.diff - - __bool__ = __nonzero__ - - -class TableDiff(object): - """ - Container for differences in one :class:`~sqlalchemy.schema.Table` - between two :class:`~sqlalchemy.schema.MetaData` instances, ``A`` - and ``B``. - - .. attribute:: columns_missing_from_A - - A sequence of column names that were found in B but weren't in - A. - - .. attribute:: columns_missing_from_B - - A sequence of column names that were found in A but weren't in - B. - - .. attribute:: columns_different - - A dictionary containing information about columns that were - found to be different. - It maps column names to a :class:`ColDiff` objects describing the - differences found. - """ - __slots__ = ( - 'columns_missing_from_A', - 'columns_missing_from_B', - 'columns_different', - ) - - def __nonzero__(self): - return bool( - self.columns_missing_from_A or - self.columns_missing_from_B or - self.columns_different - ) - - __bool__ = __nonzero__ - -class SchemaDiff(object): - """ - Compute the difference between two :class:`~sqlalchemy.schema.MetaData` - objects. - - The string representation of a :class:`SchemaDiff` will summarise - the changes found between the two - :class:`~sqlalchemy.schema.MetaData` objects. - - The length of a :class:`SchemaDiff` will give the number of - changes found, enabling it to be used much like a boolean in - expressions. - - :param metadataA: - First :class:`~sqlalchemy.schema.MetaData` to compare. - - :param metadataB: - Second :class:`~sqlalchemy.schema.MetaData` to compare. - - :param labelA: - The label to use in messages about the first - :class:`~sqlalchemy.schema.MetaData`. - - :param labelB: - The label to use in messages about the second - :class:`~sqlalchemy.schema.MetaData`. - - :param excludeTables: - A sequence of table names to exclude. - - .. attribute:: tables_missing_from_A - - A sequence of table names that were found in B but weren't in - A. - - .. attribute:: tables_missing_from_B - - A sequence of table names that were found in A but weren't in - B. - - .. attribute:: tables_different - - A dictionary containing information about tables that were found - to be different. - It maps table names to a :class:`TableDiff` objects describing the - differences found. - """ - - def __init__(self, - metadataA, metadataB, - labelA='metadataA', - labelB='metadataB', - excludeTables=None): - - self.metadataA, self.metadataB = metadataA, metadataB - self.labelA, self.labelB = labelA, labelB - self.label_width = max(len(labelA),len(labelB)) - excludeTables = set(excludeTables or []) - - A_table_names = set(metadataA.tables.keys()) - B_table_names = set(metadataB.tables.keys()) - - self.tables_missing_from_A = sorted( - B_table_names - A_table_names - excludeTables - ) - self.tables_missing_from_B = sorted( - A_table_names - B_table_names - excludeTables - ) - - self.tables_different = {} - for table_name in A_table_names.intersection(B_table_names): - - td = TableDiff() - - A_table = metadataA.tables[table_name] - B_table = metadataB.tables[table_name] - - A_column_names = set(A_table.columns.keys()) - B_column_names = set(B_table.columns.keys()) - - td.columns_missing_from_A = sorted( - B_column_names - A_column_names - ) - - td.columns_missing_from_B = sorted( - A_column_names - B_column_names - ) - - td.columns_different = {} - - for col_name in A_column_names.intersection(B_column_names): - - cd = ColDiff( - A_table.columns.get(col_name), - B_table.columns.get(col_name) - ) - - if cd: - td.columns_different[col_name]=cd - - # XXX - index and constraint differences should - # be checked for here - - if td: - self.tables_different[table_name]=td - - def __str__(self): - ''' Summarize differences. ''' - out = [] - column_template =' %%%is: %%r' % self.label_width - - for names,label in ( - (self.tables_missing_from_A,self.labelA), - (self.tables_missing_from_B,self.labelB), - ): - if names: - out.append( - ' tables missing from %s: %s' % ( - label,', '.join(sorted(names)) - ) - ) - - for name,td in sorted(self.tables_different.items()): - out.append( - ' table with differences: %s' % name - ) - for names,label in ( - (td.columns_missing_from_A,self.labelA), - (td.columns_missing_from_B,self.labelB), - ): - if names: - out.append( - ' %s missing these columns: %s' % ( - label,', '.join(sorted(names)) - ) - ) - for name,cd in td.columns_different.items(): - out.append(' column with differences: %s' % name) - out.append(column_template % (self.labelA,cd.col_A)) - out.append(column_template % (self.labelB,cd.col_B)) - - if out: - out.insert(0, 'Schema diffs:') - return '\n'.join(out) - else: - return 'No schema diffs' - - def __len__(self): - """ - Used in bool evaluation, return of 0 means no diffs. - """ - return ( - len(self.tables_missing_from_A) + - len(self.tables_missing_from_B) + - len(self.tables_different) - ) diff --git a/migrate/versioning/script/__init__.py b/migrate/versioning/script/__init__.py deleted file mode 100644 index c788eda..0000000 --- a/migrate/versioning/script/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from migrate.versioning.script.base import BaseScript -from migrate.versioning.script.py import PythonScript -from migrate.versioning.script.sql import SqlScript diff --git a/migrate/versioning/script/base.py b/migrate/versioning/script/base.py deleted file mode 100644 index 22ca7b4..0000000 --- a/migrate/versioning/script/base.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import logging - -from migrate import exceptions -from migrate.versioning.config import operations -from migrate.versioning import pathed - - -log = logging.getLogger(__name__) - -class BaseScript(pathed.Pathed): - """Base class for other types of scripts. - All scripts have the following properties: - - source (script.source()) - The source code of the script - version (script.version()) - The version number of the script - operations (script.operations()) - The operations defined by the script: upgrade(), downgrade() or both. - Returns a tuple of operations. - Can also check for an operation with ex. script.operation(Script.ops.up) - """ # TODO: sphinxfy this and implement it correctly - - def __init__(self, path): - log.debug('Loading script %s...' % path) - self.verify(path) - super(BaseScript, self).__init__(path) - log.debug('Script %s loaded successfully' % path) - - @classmethod - def verify(cls, path): - """Ensure this is a valid script - This version simply ensures the script file's existence - - :raises: :exc:`InvalidScriptError <migrate.exceptions.InvalidScriptError>` - """ - try: - cls.require_found(path) - except: - raise exceptions.InvalidScriptError(path) - - def source(self): - """:returns: source code of the script. - :rtype: string - """ - fd = open(self.path) - ret = fd.read() - fd.close() - return ret - - def run(self, engine): - """Core of each BaseScript subclass. - This method executes the script. - """ - raise NotImplementedError() diff --git a/migrate/versioning/script/py.py b/migrate/versioning/script/py.py deleted file mode 100644 index 92a8f6b..0000000 --- a/migrate/versioning/script/py.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import shutil -import warnings -import logging -import inspect - -import migrate -from migrate.versioning import genmodel, schemadiff -from migrate.versioning.config import operations -from migrate.versioning.template import Template -from migrate.versioning.script import base -from migrate.versioning.util import import_path, load_model, with_engine -from migrate.exceptions import MigrateDeprecationWarning, InvalidScriptError, ScriptError -import six -from six.moves import StringIO - -log = logging.getLogger(__name__) -__all__ = ['PythonScript'] - - -class PythonScript(base.BaseScript): - """Base for Python scripts""" - - @classmethod - def create(cls, path, **opts): - """Create an empty migration script at specified path - - :returns: :class:`PythonScript instance <migrate.versioning.script.py.PythonScript>`""" - cls.require_notfound(path) - - src = Template(opts.pop('templates_path', None)).get_script(theme=opts.pop('templates_theme', None)) - shutil.copy(src, path) - - return cls(path) - - @classmethod - def make_update_script_for_model(cls, engine, oldmodel, - model, repository, **opts): - """Create a migration script based on difference between two SA models. - - :param repository: path to migrate repository - :param oldmodel: dotted.module.name:SAClass or SAClass object - :param model: dotted.module.name:SAClass or SAClass object - :param engine: SQLAlchemy engine - :type repository: string or :class:`Repository instance <migrate.versioning.repository.Repository>` - :type oldmodel: string or Class - :type model: string or Class - :type engine: Engine instance - :returns: Upgrade / Downgrade script - :rtype: string - """ - - if isinstance(repository, six.string_types): - # oh dear, an import cycle! - from migrate.versioning.repository import Repository - repository = Repository(repository) - - oldmodel = load_model(oldmodel) - model = load_model(model) - - # Compute differences. - diff = schemadiff.getDiffOfModelAgainstModel( - model, - oldmodel, - excludeTables=[repository.version_table]) - # TODO: diff can be False (there is no difference?) - decls, upgradeCommands, downgradeCommands = \ - genmodel.ModelGenerator(diff,engine).genB2AMigration() - - # Store differences into file. - src = Template(opts.pop('templates_path', None)).get_script(opts.pop('templates_theme', None)) - f = open(src) - contents = f.read() - f.close() - - # generate source - search = 'def upgrade(migrate_engine):' - contents = contents.replace(search, '\n\n'.join((decls, search)), 1) - if upgradeCommands: - contents = contents.replace(' pass', upgradeCommands, 1) - if downgradeCommands: - contents = contents.replace(' pass', downgradeCommands, 1) - return contents - - @classmethod - def verify_module(cls, path): - """Ensure path is a valid script - - :param path: Script location - :type path: string - :raises: :exc:`InvalidScriptError <migrate.exceptions.InvalidScriptError>` - :returns: Python module - """ - # Try to import and get the upgrade() func - module = import_path(path) - try: - assert callable(module.upgrade) - except Exception as e: - raise InvalidScriptError(path + ': %s' % str(e)) - return module - - def preview_sql(self, url, step, **args): - """Mocks SQLAlchemy Engine to store all executed calls in a string - and runs :meth:`PythonScript.run <migrate.versioning.script.py.PythonScript.run>` - - :returns: SQL file - """ - buf = StringIO() - args['engine_arg_strategy'] = 'mock' - args['engine_arg_executor'] = lambda s, p = '': buf.write(str(s) + p) - - @with_engine - def go(url, step, **kw): - engine = kw.pop('engine') - self.run(engine, step) - return buf.getvalue() - - return go(url, step, **args) - - def run(self, engine, step): - """Core method of Script file. - Exectues :func:`update` or :func:`downgrade` functions - - :param engine: SQLAlchemy Engine - :param step: Operation to run - :type engine: string - :type step: int - """ - if step in ('downgrade', 'upgrade'): - op = step - elif step > 0: - op = 'upgrade' - elif step < 0: - op = 'downgrade' - else: - raise ScriptError("%d is not a valid step" % step) - - funcname = base.operations[op] - script_func = self._func(funcname) - - # check for old way of using engine - if not inspect.getargspec(script_func)[0]: - raise TypeError("upgrade/downgrade functions must accept engine" - " parameter (since version 0.5.4)") - - script_func(engine) - - @property - def module(self): - """Calls :meth:`migrate.versioning.script.py.verify_module` - and returns it. - """ - if not hasattr(self, '_module'): - self._module = self.verify_module(self.path) - return self._module - - def _func(self, funcname): - if not hasattr(self.module, funcname): - msg = "Function '%s' is not defined in this script" - raise ScriptError(msg % funcname) - return getattr(self.module, funcname) diff --git a/migrate/versioning/script/sql.py b/migrate/versioning/script/sql.py deleted file mode 100644 index 862bc9f..0000000 --- a/migrate/versioning/script/sql.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import logging -import re -import shutil - -import sqlparse - -from migrate.versioning.script import base -from migrate.versioning.template import Template - - -log = logging.getLogger(__name__) - -class SqlScript(base.BaseScript): - """A file containing plain SQL statements.""" - - @classmethod - def create(cls, path, **opts): - """Create an empty migration script at specified path - - :returns: :class:`SqlScript instance <migrate.versioning.script.sql.SqlScript>`""" - cls.require_notfound(path) - - src = Template(opts.pop('templates_path', None)).get_sql_script(theme=opts.pop('templates_theme', None)) - shutil.copy(src, path) - return cls(path) - - # TODO: why is step parameter even here? - def run(self, engine, step=None): - """Runs SQL script through raw dbapi execute call""" - text = self.source() - # Don't rely on SA's autocommit here - # (SA uses .startswith to check if a commit is needed. What if script - # starts with a comment?) - conn = engine.connect() - try: - trans = conn.begin() - try: - # ignore transaction management statements that are - # redundant in SQL script context and result in - # operational error being returned. - # - # Note: we don't ignore ROLLBACK in migration scripts - # since its usage would be insane anyway, and we're - # better to fail on its occurance instead of ignoring it - # (and committing transaction, which is contradictory to - # the whole idea of ROLLBACK) - ignored_statements = ('BEGIN', 'END', 'COMMIT') - ignored_regex = re.compile('^\s*(%s).*;?$' % '|'.join(ignored_statements), - re.IGNORECASE) - - # NOTE(ihrachys): script may contain multiple statements, and - # not all drivers reliably handle multistatement queries or - # commands passed to .execute(), so split them and execute one - # by one - text = sqlparse.format(text, strip_comments=True, strip_whitespace=True) - for statement in sqlparse.split(text): - if statement: - if re.match(ignored_regex, statement): - log.warning('"%s" found in SQL script; ignoring' % statement) - else: - conn.execute(statement) - trans.commit() - except Exception as e: - log.error("SQL script %s failed: %s", self.path, e) - trans.rollback() - raise - finally: - conn.close() diff --git a/migrate/versioning/shell.py b/migrate/versioning/shell.py deleted file mode 100644 index 5fb86b1..0000000 --- a/migrate/versioning/shell.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -"""The migrate command-line tool.""" - -import sys -import inspect -import logging -from optparse import OptionParser, BadOptionError - -from migrate import exceptions -from migrate.versioning import api -from migrate.versioning.config import * -from migrate.versioning.util import asbool -import six - - -alias = dict( - s=api.script, - vc=api.version_control, - dbv=api.db_version, - v=api.version, -) - -def alias_setup(): - global alias - for key, val in six.iteritems(alias): - setattr(api, key, val) -alias_setup() - - -class PassiveOptionParser(OptionParser): - - def _process_args(self, largs, rargs, values): - """little hack to support all --some_option=value parameters""" - - while rargs: - arg = rargs[0] - if arg == "--": - del rargs[0] - return - elif arg[0:2] == "--": - # if parser does not know about the option - # pass it along (make it anonymous) - try: - opt = arg.split('=', 1)[0] - self._match_long_opt(opt) - except BadOptionError: - largs.append(arg) - del rargs[0] - else: - self._process_long_opt(rargs, values) - elif arg[:1] == "-" and len(arg) > 1: - self._process_short_opts(rargs, values) - elif self.allow_interspersed_args: - largs.append(arg) - del rargs[0] - -def main(argv=None, **kwargs): - """Shell interface to :mod:`migrate.versioning.api`. - - kwargs are default options that can be overriden with passing - --some_option as command line option - - :param disable_logging: Let migrate configure logging - :type disable_logging: bool - """ - if argv is not None: - argv = argv - else: - argv = list(sys.argv[1:]) - commands = list(api.__all__) - commands.sort() - - usage = """%%prog COMMAND ... - - Available commands: - %s - - Enter "%%prog help COMMAND" for information on a particular command. - """ % '\n\t'.join(["%s - %s" % (command.ljust(28), api.command_desc.get(command)) for command in commands]) - - parser = PassiveOptionParser(usage=usage) - parser.add_option("-d", "--debug", - action="store_true", - dest="debug", - default=False, - help="Shortcut to turn on DEBUG mode for logging") - parser.add_option("-q", "--disable_logging", - action="store_true", - dest="disable_logging", - default=False, - help="Use this option to disable logging configuration") - help_commands = ['help', '-h', '--help'] - HELP = False - - try: - command = argv.pop(0) - if command in help_commands: - HELP = True - command = argv.pop(0) - except IndexError: - parser.print_help() - return - - command_func = getattr(api, command, None) - if command_func is None or command.startswith('_'): - parser.error("Invalid command %s" % command) - - parser.set_usage(inspect.getdoc(command_func)) - f_args, f_varargs, f_kwargs, f_defaults = inspect.getargspec(command_func) - for arg in f_args: - parser.add_option( - "--%s" % arg, - dest=arg, - action='store', - type="string") - - # display help of the current command - if HELP: - parser.print_help() - return - - options, args = parser.parse_args(argv) - - # override kwargs with anonymous parameters - override_kwargs = dict() - for arg in list(args): - if arg.startswith('--'): - args.remove(arg) - if '=' in arg: - opt, value = arg[2:].split('=', 1) - else: - opt = arg[2:] - value = True - override_kwargs[opt] = value - - # override kwargs with options if user is overwriting - for key, value in six.iteritems(options.__dict__): - if value is not None: - override_kwargs[key] = value - - # arguments that function accepts without passed kwargs - f_required = list(f_args) - candidates = dict(kwargs) - candidates.update(override_kwargs) - for key, value in six.iteritems(candidates): - if key in f_args: - f_required.remove(key) - - # map function arguments to parsed arguments - for arg in args: - try: - kw = f_required.pop(0) - except IndexError: - parser.error("Too many arguments for command %s: %s" % (command, - arg)) - kwargs[kw] = arg - - # apply overrides - kwargs.update(override_kwargs) - - # configure options - for key, value in six.iteritems(options.__dict__): - kwargs.setdefault(key, value) - - # configure logging - if not asbool(kwargs.pop('disable_logging', False)): - # filter to log =< INFO into stdout and rest to stderr - class SingleLevelFilter(logging.Filter): - def __init__(self, min=None, max=None): - self.min = min or 0 - self.max = max or 100 - - def filter(self, record): - return self.min <= record.levelno <= self.max - - logger = logging.getLogger() - h1 = logging.StreamHandler(sys.stdout) - f1 = SingleLevelFilter(max=logging.INFO) - h1.addFilter(f1) - h2 = logging.StreamHandler(sys.stderr) - f2 = SingleLevelFilter(min=logging.WARN) - h2.addFilter(f2) - logger.addHandler(h1) - logger.addHandler(h2) - - if options.debug: - logger.setLevel(logging.DEBUG) - else: - logger.setLevel(logging.INFO) - - log = logging.getLogger(__name__) - - # check if all args are given - try: - num_defaults = len(f_defaults) - except TypeError: - num_defaults = 0 - f_args_default = f_args[len(f_args) - num_defaults:] - required = list(set(f_required) - set(f_args_default)) - required.sort() - if required: - parser.error("Not enough arguments for command %s: %s not specified" \ - % (command, ', '.join(required))) - - # handle command - try: - ret = command_func(**kwargs) - if ret is not None: - log.info(ret) - except (exceptions.UsageError, exceptions.KnownError) as e: - parser.error(e.args[0]) - -if __name__ == "__main__": - main() diff --git a/migrate/versioning/template.py b/migrate/versioning/template.py deleted file mode 100644 index 8182e6b..0000000 --- a/migrate/versioning/template.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import shutil -import sys - -from pkg_resources import resource_filename - -from migrate.versioning.config import * -from migrate.versioning import pathed - - -class Collection(pathed.Pathed): - """A collection of templates of a specific type""" - _mask = None - - def get_path(self, file): - return os.path.join(self.path, str(file)) - - -class RepositoryCollection(Collection): - _mask = '%s' - -class ScriptCollection(Collection): - _mask = '%s.py_tmpl' - -class ManageCollection(Collection): - _mask = '%s.py_tmpl' - -class SQLScriptCollection(Collection): - _mask = '%s.py_tmpl' - -class Template(pathed.Pathed): - """Finds the paths/packages of various Migrate templates. - - :param path: Templates are loaded from migrate package - if `path` is not provided. - """ - pkg = 'migrate.versioning.templates' - - def __new__(cls, path=None): - if path is None: - path = cls._find_path(cls.pkg) - return super(Template, cls).__new__(cls, path) - - def __init__(self, path=None): - if path is None: - path = Template._find_path(self.pkg) - super(Template, self).__init__(path) - self.repository = RepositoryCollection(os.path.join(path, 'repository')) - self.script = ScriptCollection(os.path.join(path, 'script')) - self.manage = ManageCollection(os.path.join(path, 'manage')) - self.sql_script = SQLScriptCollection(os.path.join(path, 'sql_script')) - - @classmethod - def _find_path(cls, pkg): - """Returns absolute path to dotted python package.""" - tmp_pkg = pkg.rsplit('.', 1) - - if len(tmp_pkg) != 1: - return resource_filename(tmp_pkg[0], tmp_pkg[1]) - else: - return resource_filename(tmp_pkg[0], '') - - def _get_item(self, collection, theme=None): - """Locates and returns collection. - - :param collection: name of collection to locate - :param type_: type of subfolder in collection (defaults to "_default") - :returns: (package, source) - :rtype: str, str - """ - item = getattr(self, collection) - theme_mask = getattr(item, '_mask') - theme = theme_mask % (theme or 'default') - return item.get_path(theme) - - def get_repository(self, *a, **kw): - """Calls self._get_item('repository', *a, **kw)""" - return self._get_item('repository', *a, **kw) - - def get_script(self, *a, **kw): - """Calls self._get_item('script', *a, **kw)""" - return self._get_item('script', *a, **kw) - - def get_sql_script(self, *a, **kw): - """Calls self._get_item('sql_script', *a, **kw)""" - return self._get_item('sql_script', *a, **kw) - - def get_manage(self, *a, **kw): - """Calls self._get_item('manage', *a, **kw)""" - return self._get_item('manage', *a, **kw) diff --git a/migrate/versioning/templates/__init__.py b/migrate/versioning/templates/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/migrate/versioning/templates/__init__.py +++ /dev/null diff --git a/migrate/versioning/templates/manage/default.py_tmpl b/migrate/versioning/templates/manage/default.py_tmpl deleted file mode 100644 index 971c70f..0000000 --- a/migrate/versioning/templates/manage/default.py_tmpl +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -from migrate.versioning.shell import main - -{{py: -import six -_vars = locals().copy() -del _vars['__template_name__'] -del _vars['six'] -_vars.pop('repository_name', None) -defaults = ", ".join(["%s='%s'" % var for var in six.iteritems(_vars)]) -}} - -if __name__ == '__main__': - main({{ defaults }}) diff --git a/migrate/versioning/templates/manage/pylons.py_tmpl b/migrate/versioning/templates/manage/pylons.py_tmpl deleted file mode 100644 index 0d6c32c..0000000 --- a/migrate/versioning/templates/manage/pylons.py_tmpl +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -import sys - -from sqlalchemy import engine_from_config -from paste.deploy.loadwsgi import ConfigLoader - -from migrate.versioning.shell import main -from {{ locals().pop('repository_name') }}.model import migrations - - -if '-c' in sys.argv: - pos = sys.argv.index('-c') - conf_path = sys.argv[pos + 1] - del sys.argv[pos:pos + 2] -else: - conf_path = 'development.ini' - -{{py: -import six -_vars = locals().copy() -del _vars['__template_name__'] -del _vars['six'] -defaults = ", ".join(["%s='%s'" % var for var in six.iteritems(_vars)]) -}} - -conf_dict = ConfigLoader(conf_path).parser._sections['app:main'] - -# migrate supports passing url as an existing Engine instance (since 0.6.0) -# usage: migrate -c path/to/config.ini COMMANDS -if __name__ == '__main__': - main(url=engine_from_config(conf_dict), repository=migrations.__path__[0],{{ defaults }}) diff --git a/migrate/versioning/templates/repository/__init__.py b/migrate/versioning/templates/repository/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/migrate/versioning/templates/repository/__init__.py +++ /dev/null diff --git a/migrate/versioning/templates/repository/default/README b/migrate/versioning/templates/repository/default/README deleted file mode 100644 index 6218f8c..0000000 --- a/migrate/versioning/templates/repository/default/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -http://code.google.com/p/sqlalchemy-migrate/ diff --git a/migrate/versioning/templates/repository/default/__init__.py b/migrate/versioning/templates/repository/default/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/migrate/versioning/templates/repository/default/__init__.py +++ /dev/null diff --git a/migrate/versioning/templates/repository/default/migrate.cfg b/migrate/versioning/templates/repository/default/migrate.cfg deleted file mode 100644 index bcc33a7..0000000 --- a/migrate/versioning/templates/repository/default/migrate.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id={{ locals().pop('repository_id') }} - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table={{ locals().pop('version_table') }} - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs={{ locals().pop('required_dbs') }} - -# When creating new change scripts, Migrate will stamp the new script with -# a version number. By default this is latest_version + 1. You can set this -# to 'true' to tell Migrate to use the UTC timestamp instead. -use_timestamp_numbering={{ locals().pop('use_timestamp_numbering') }} diff --git a/migrate/versioning/templates/repository/default/versions/__init__.py b/migrate/versioning/templates/repository/default/versions/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/migrate/versioning/templates/repository/default/versions/__init__.py +++ /dev/null diff --git a/migrate/versioning/templates/repository/pylons/README b/migrate/versioning/templates/repository/pylons/README deleted file mode 100644 index 6218f8c..0000000 --- a/migrate/versioning/templates/repository/pylons/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -http://code.google.com/p/sqlalchemy-migrate/ diff --git a/migrate/versioning/templates/repository/pylons/__init__.py b/migrate/versioning/templates/repository/pylons/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/migrate/versioning/templates/repository/pylons/__init__.py +++ /dev/null diff --git a/migrate/versioning/templates/repository/pylons/migrate.cfg b/migrate/versioning/templates/repository/pylons/migrate.cfg deleted file mode 100644 index bcc33a7..0000000 --- a/migrate/versioning/templates/repository/pylons/migrate.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id={{ locals().pop('repository_id') }} - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table={{ locals().pop('version_table') }} - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs={{ locals().pop('required_dbs') }} - -# When creating new change scripts, Migrate will stamp the new script with -# a version number. By default this is latest_version + 1. You can set this -# to 'true' to tell Migrate to use the UTC timestamp instead. -use_timestamp_numbering={{ locals().pop('use_timestamp_numbering') }} diff --git a/migrate/versioning/templates/repository/pylons/versions/__init__.py b/migrate/versioning/templates/repository/pylons/versions/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/migrate/versioning/templates/repository/pylons/versions/__init__.py +++ /dev/null diff --git a/migrate/versioning/templates/script/__init__.py b/migrate/versioning/templates/script/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/migrate/versioning/templates/script/__init__.py +++ /dev/null diff --git a/migrate/versioning/templates/script/default.py_tmpl b/migrate/versioning/templates/script/default.py_tmpl deleted file mode 100644 index 58d874b..0000000 --- a/migrate/versioning/templates/script/default.py_tmpl +++ /dev/null @@ -1,13 +0,0 @@ -from sqlalchemy import * -from migrate import * - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - pass - - -def downgrade(migrate_engine): - # Operations to reverse the above upgrade go here. - pass diff --git a/migrate/versioning/templates/script/pylons.py_tmpl b/migrate/versioning/templates/script/pylons.py_tmpl deleted file mode 100644 index 58d874b..0000000 --- a/migrate/versioning/templates/script/pylons.py_tmpl +++ /dev/null @@ -1,13 +0,0 @@ -from sqlalchemy import * -from migrate import * - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - pass - - -def downgrade(migrate_engine): - # Operations to reverse the above upgrade go here. - pass diff --git a/migrate/versioning/templates/sql_script/default.py_tmpl b/migrate/versioning/templates/sql_script/default.py_tmpl deleted file mode 100644 index e69de29..0000000 --- a/migrate/versioning/templates/sql_script/default.py_tmpl +++ /dev/null diff --git a/migrate/versioning/templates/sql_script/pylons.py_tmpl b/migrate/versioning/templates/sql_script/pylons.py_tmpl deleted file mode 100644 index e69de29..0000000 --- a/migrate/versioning/templates/sql_script/pylons.py_tmpl +++ /dev/null diff --git a/migrate/versioning/util/__init__.py b/migrate/versioning/util/__init__.py deleted file mode 100644 index 55c72c9..0000000 --- a/migrate/versioning/util/__init__.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""".. currentmodule:: migrate.versioning.util""" - -import warnings -import logging -from decorator import decorator -from pkg_resources import EntryPoint - -import six -from sqlalchemy import create_engine -from sqlalchemy.engine import Engine -from sqlalchemy.pool import StaticPool - -from migrate import exceptions -from migrate.versioning.util.keyedinstance import KeyedInstance -from migrate.versioning.util.importpath import import_path - - -log = logging.getLogger(__name__) - -def load_model(dotted_name): - """Import module and use module-level variable". - - :param dotted_name: path to model in form of string: ``some.python.module:Class`` - - .. versionchanged:: 0.5.4 - - """ - if isinstance(dotted_name, six.string_types): - if ':' not in dotted_name: - # backwards compatibility - warnings.warn('model should be in form of module.model:User ' - 'and not module.model.User', exceptions.MigrateDeprecationWarning) - dotted_name = ':'.join(dotted_name.rsplit('.', 1)) - - ep = EntryPoint.parse('x=%s' % dotted_name) - if hasattr(ep, 'resolve'): - # this is available on setuptools >= 10.2 - return ep.resolve() - else: - # this causes a DeprecationWarning on setuptools >= 11.3 - return ep.load(False) - else: - # Assume it's already loaded. - return dotted_name - -def asbool(obj): - """Do everything to use object as bool""" - if isinstance(obj, six.string_types): - obj = obj.strip().lower() - if obj in ['true', 'yes', 'on', 'y', 't', '1']: - return True - elif obj in ['false', 'no', 'off', 'n', 'f', '0']: - return False - else: - raise ValueError("String is not true/false: %r" % obj) - if obj in (True, False): - return bool(obj) - else: - raise ValueError("String is not true/false: %r" % obj) - -def guess_obj_type(obj): - """Do everything to guess object type from string - - Tries to convert to `int`, `bool` and finally returns if not succeded. - - .. versionadded: 0.5.4 - """ - - result = None - - try: - result = int(obj) - except: - pass - - if result is None: - try: - result = asbool(obj) - except: - pass - - if result is not None: - return result - else: - return obj - -@decorator -def catch_known_errors(f, *a, **kw): - """Decorator that catches known api errors - - .. versionadded: 0.5.4 - """ - - try: - return f(*a, **kw) - except exceptions.PathFoundError as e: - raise exceptions.KnownError("The path %s already exists" % e.args[0]) - -def construct_engine(engine, **opts): - """.. versionadded:: 0.5.4 - - Constructs and returns SQLAlchemy engine. - - Currently, there are 2 ways to pass create_engine options to :mod:`migrate.versioning.api` functions: - - :param engine: connection string or a existing engine - :param engine_dict: python dictionary of options to pass to `create_engine` - :param engine_arg_*: keyword parameters to pass to `create_engine` (evaluated with :func:`migrate.versioning.util.guess_obj_type`) - :type engine_dict: dict - :type engine: string or Engine instance - :type engine_arg_*: string - :returns: SQLAlchemy Engine - - .. note:: - - keyword parameters override ``engine_dict`` values. - - """ - if isinstance(engine, Engine): - return engine - elif not isinstance(engine, six.string_types): - raise ValueError("you need to pass either an existing engine or a database uri") - - # get options for create_engine - if opts.get('engine_dict') and isinstance(opts['engine_dict'], dict): - kwargs = opts['engine_dict'] - else: - kwargs = dict() - - # DEPRECATED: handle echo the old way - echo = asbool(opts.get('echo', False)) - if echo: - warnings.warn('echo=True parameter is deprecated, pass ' - 'engine_arg_echo=True or engine_dict={"echo": True}', - exceptions.MigrateDeprecationWarning) - kwargs['echo'] = echo - - # parse keyword arguments - for key, value in six.iteritems(opts): - if key.startswith('engine_arg_'): - kwargs[key[11:]] = guess_obj_type(value) - - log.debug('Constructing engine') - # TODO: return create_engine(engine, poolclass=StaticPool, **kwargs) - # seems like 0.5.x branch does not work with engine.dispose and staticpool - return create_engine(engine, **kwargs) - -@decorator -def with_engine(f, *a, **kw): - """Decorator for :mod:`migrate.versioning.api` functions - to safely close resources after function usage. - - Passes engine parameters to :func:`construct_engine` and - resulting parameter is available as kw['engine']. - - Engine is disposed after wrapped function is executed. - - .. versionadded: 0.6.0 - """ - url = a[0] - engine = construct_engine(url, **kw) - - try: - kw['engine'] = engine - return f(*a, **kw) - finally: - if isinstance(engine, Engine) and engine is not url: - log.debug('Disposing SQLAlchemy engine %s', engine) - engine.dispose() - - -class Memoize(object): - """Memoize(fn) - an instance which acts like fn but memoizes its arguments - Will only work on functions with non-mutable arguments - - ActiveState Code 52201 - """ - def __init__(self, fn): - self.fn = fn - self.memo = {} - - def __call__(self, *args): - if args not in self.memo: - self.memo[args] = self.fn(*args) - return self.memo[args] diff --git a/migrate/versioning/util/importpath.py b/migrate/versioning/util/importpath.py deleted file mode 100644 index 529be89..0000000 --- a/migrate/versioning/util/importpath.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -import sys - -PY33 = sys.version_info >= (3, 3) - -if PY33: - from importlib import machinery -else: - from six.moves import reload_module as reload - - -def import_path(fullpath): - """ Import a file with full path specification. Allows one to - import from anywhere, something __import__ does not do. - """ - if PY33: - name = os.path.splitext(os.path.basename(fullpath))[0] - return machinery.SourceFileLoader( - name, fullpath).load_module(name) - else: - # http://zephyrfalcon.org/weblog/arch_d7_2002_08_31.html - path, filename = os.path.split(fullpath) - filename, ext = os.path.splitext(filename) - sys.path.append(path) - try: - module = __import__(filename) - reload(module) # Might be out of date during tests - return module - finally: - del sys.path[-1] diff --git a/migrate/versioning/util/keyedinstance.py b/migrate/versioning/util/keyedinstance.py deleted file mode 100644 index a692e08..0000000 --- a/migrate/versioning/util/keyedinstance.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -class KeyedInstance(object): - """A class whose instances have a unique identifier of some sort - No two instances with the same unique ID should exist - if we try to create - a second instance, the first should be returned. - """ - - _instances = dict() - - def __new__(cls, *p, **k): - instances = cls._instances - clskey = str(cls) - if clskey not in instances: - instances[clskey] = dict() - instances = instances[clskey] - - key = cls._key(*p, **k) - if key not in instances: - instances[key] = super(KeyedInstance, cls).__new__(cls) - return instances[key] - - @classmethod - def _key(cls, *p, **k): - """Given a unique identifier, return a dictionary key - This should be overridden by child classes, to specify which parameters - should determine an object's uniqueness - """ - raise NotImplementedError() - - @classmethod - def clear(cls): - # Allow cls.clear() as well as uniqueInstance.clear(cls) - if str(cls) in cls._instances: - del cls._instances[str(cls)] diff --git a/migrate/versioning/version.py b/migrate/versioning/version.py deleted file mode 100644 index 0633e1b..0000000 --- a/migrate/versioning/version.py +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import re -import shutil -import logging - -from migrate import exceptions -from migrate.versioning import pathed, script -from datetime import datetime -import six - - -log = logging.getLogger(__name__) - -class VerNum(object): - """A version number that behaves like a string and int at the same time""" - - _instances = dict() - - def __new__(cls, value): - val = str(value) - if val not in cls._instances: - cls._instances[val] = super(VerNum, cls).__new__(cls) - ret = cls._instances[val] - return ret - - def __init__(self,value): - self.value = str(int(value)) - if self < 0: - raise ValueError("Version number cannot be negative") - - def __add__(self, value): - ret = int(self) + int(value) - return VerNum(ret) - - def __sub__(self, value): - return self + (int(value) * -1) - - def __eq__(self, value): - return int(self) == int(value) - - def __ne__(self, value): - return int(self) != int(value) - - def __lt__(self, value): - return int(self) < int(value) - - def __gt__(self, value): - return int(self) > int(value) - - def __ge__(self, value): - return int(self) >= int(value) - - def __le__(self, value): - return int(self) <= int(value) - - def __repr__(self): - return "<VerNum(%s)>" % self.value - - def __str__(self): - return str(self.value) - - def __int__(self): - return int(self.value) - - def __index__(self): - return int(self.value) - - if six.PY3: - def __hash__(self): - return hash(self.value) - - -class Collection(pathed.Pathed): - """A collection of versioning scripts in a repository""" - - FILENAME_WITH_VERSION = re.compile(r'^(\d{3,}).*') - - def __init__(self, path): - """Collect current version scripts in repository - and store them in self.versions - """ - super(Collection, self).__init__(path) - - # Create temporary list of files, allowing skipped version numbers. - files = os.listdir(path) - if '1' in files: - # deprecation - raise Exception('It looks like you have a repository in the old ' - 'format (with directories for each version). ' - 'Please convert repository before proceeding.') - - tempVersions = dict() - for filename in files: - match = self.FILENAME_WITH_VERSION.match(filename) - if match: - num = int(match.group(1)) - tempVersions.setdefault(num, []).append(filename) - else: - pass # Must be a helper file or something, let's ignore it. - - # Create the versions member where the keys - # are VerNum's and the values are Version's. - self.versions = dict() - for num, files in tempVersions.items(): - self.versions[VerNum(num)] = Version(num, path, files) - - @property - def latest(self): - """:returns: Latest version in Collection""" - return max([VerNum(0)] + list(self.versions.keys())) - - def _next_ver_num(self, use_timestamp_numbering): - if use_timestamp_numbering == True: - return VerNum(int(datetime.utcnow().strftime('%Y%m%d%H%M%S'))) - else: - return self.latest + 1 - - def create_new_python_version(self, description, **k): - """Create Python files for new version""" - ver = self._next_ver_num(k.pop('use_timestamp_numbering', False)) - extra = str_to_filename(description) - - if extra: - if extra == '_': - extra = '' - elif not extra.startswith('_'): - extra = '_%s' % extra - - filename = '%03d%s.py' % (ver, extra) - filepath = self._version_path(filename) - - script.PythonScript.create(filepath, **k) - self.versions[ver] = Version(ver, self.path, [filename]) - - def create_new_sql_version(self, database, description, **k): - """Create SQL files for new version""" - ver = self._next_ver_num(k.pop('use_timestamp_numbering', False)) - self.versions[ver] = Version(ver, self.path, []) - - extra = str_to_filename(description) - - if extra: - if extra == '_': - extra = '' - elif not extra.startswith('_'): - extra = '_%s' % extra - - # Create new files. - for op in ('upgrade', 'downgrade'): - filename = '%03d%s_%s_%s.sql' % (ver, extra, database, op) - filepath = self._version_path(filename) - script.SqlScript.create(filepath, **k) - self.versions[ver].add_script(filepath) - - def version(self, vernum=None): - """Returns required version. - - If vernum is not given latest version will be returned otherwise - required version will be returned. - :raises: : exceptions.VersionNotFoundError if respective migration - script file of version is not present in the migration repository. - """ - if vernum is None: - vernum = self.latest - - try: - return self.versions[VerNum(vernum)] - except KeyError: - raise exceptions.VersionNotFoundError( - ("Database schema file with version %(args)s doesn't " - "exist.") % {'args': VerNum(vernum)}) - - @classmethod - def clear(cls): - super(Collection, cls).clear() - - def _version_path(self, ver): - """Returns path of file in versions repository""" - return os.path.join(self.path, str(ver)) - - -class Version(object): - """A single version in a collection - :param vernum: Version Number - :param path: Path to script files - :param filelist: List of scripts - :type vernum: int, VerNum - :type path: string - :type filelist: list - """ - - def __init__(self, vernum, path, filelist): - self.version = VerNum(vernum) - - # Collect scripts in this folder - self.sql = dict() - self.python = None - - for script in filelist: - self.add_script(os.path.join(path, script)) - - def script(self, database=None, operation=None): - """Returns SQL or Python Script""" - for db in (database, 'default'): - # Try to return a .sql script first - try: - return self.sql[db][operation] - except KeyError: - continue # No .sql script exists - - # TODO: maybe add force Python parameter? - ret = self.python - - assert ret is not None, \ - "There is no script for %d version" % self.version - return ret - - def add_script(self, path): - """Add script to Collection/Version""" - if path.endswith(Extensions.py): - self._add_script_py(path) - elif path.endswith(Extensions.sql): - self._add_script_sql(path) - - SQL_FILENAME = re.compile(r'^.*\.sql') - - def _add_script_sql(self, path): - basename = os.path.basename(path) - match = self.SQL_FILENAME.match(basename) - - if match: - basename = basename.replace('.sql', '') - parts = basename.split('_') - if len(parts) < 3: - raise exceptions.ScriptError( - "Invalid SQL script name %s " % basename + \ - "(needs to be ###_description_database_operation.sql)") - version = parts[0] - op = parts[-1] - # NOTE(mriedem): check for ibm_db_sa as the database in the name - if 'ibm_db_sa' in basename: - if len(parts) == 6: - dbms = '_'.join(parts[-4: -1]) - else: - raise exceptions.ScriptError( - "Invalid ibm_db_sa SQL script name '%s'; " - "(needs to be " - "###_description_ibm_db_sa_operation.sql)" % basename) - else: - dbms = parts[-2] - else: - raise exceptions.ScriptError( - "Invalid SQL script name %s " % basename + \ - "(needs to be ###_description_database_operation.sql)") - - # File the script into a dictionary - self.sql.setdefault(dbms, {})[op] = script.SqlScript(path) - - def _add_script_py(self, path): - if self.python is not None: - raise exceptions.ScriptError('You can only have one Python script ' - 'per version, but you have: %s and %s' % (self.python, path)) - self.python = script.PythonScript(path) - - -class Extensions(object): - """A namespace for file extensions""" - py = 'py' - sql = 'sql' - -def str_to_filename(s): - """Replaces spaces, (double and single) quotes - and double underscores to underscores - """ - - s = s.replace(' ', '_').replace('"', '_').replace("'", '_').replace(".", "_") - while '__' in s: - s = s.replace('__', '_') - return s |