From 2934ad567e8935873e67c0bae824104347c0f111 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Fri, 4 Feb 2022 17:48:21 +0000 Subject: [PATCH 01/91] Basic infrastructure for `versiondb` subcommand --- pyani/scripts/parsers/__init__.py | 4 ++ pyani/scripts/parsers/versiondb_parser.py | 50 +++++++++++++++++++ pyani/scripts/subcommands/__init__.py | 1 + pyani/scripts/subcommands/subcmd_versiondb.py | 30 +++++++++++ 4 files changed, 85 insertions(+) create mode 100644 pyani/scripts/parsers/versiondb_parser.py create mode 100644 pyani/scripts/subcommands/subcmd_versiondb.py diff --git a/pyani/scripts/parsers/__init__.py b/pyani/scripts/parsers/__init__.py index e1fc790c..1f7b1edf 100644 --- a/pyani/scripts/parsers/__init__.py +++ b/pyani/scripts/parsers/__init__.py @@ -58,6 +58,7 @@ common_parser, run_common_parser, listdeps_parser, + versiondb_parser, ) @@ -76,6 +77,8 @@ def parse_cmdline(argv: Optional[List] = None) -> Namespace: index genome sequence files in a subdirectory, for analysis - createdb generate SQLite database for data and analysis results + - versiondb + upgrade/downgrade pyani SQLite database between schemas - anim conduct ANIm analysis - anib @@ -119,6 +122,7 @@ def parse_cmdline(argv: Optional[List] = None) -> Namespace: download_parser.build(subparsers, parents=[parser_common]) index_parser.build(subparsers, parents=[parser_common]) createdb_parser.build(subparsers, parents=[parser_common]) + versiondb_parser.build(subparsers, parents=[parser_common]) anim_parser.build( subparsers, parents=[parser_common, parser_scheduler, parser_run_common] ) diff --git a/pyani/scripts/parsers/versiondb_parser.py b/pyani/scripts/parsers/versiondb_parser.py new file mode 100644 index 00000000..d0395624 --- /dev/null +++ b/pyani/scripts/parsers/versiondb_parser.py @@ -0,0 +1,50 @@ +"""Provides parser for versiondb subcommand.""" + +from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, _SubParsersAction +from pathlib import Path +from typing import List, Optional + +from pyani.scripts import subcommands + + +def build( + subps: _SubParsersAction, parents: Optional[List[ArgumentParser]] = None +) -> None: + """Return a command-line parser for the versiondb subcommand. + + :param subps: collection of subparsers in main parser + :param parents: parsers from which arguments are inherited + + """ + parser = subps.add_parser( + "versiondb", parents=parents, formatter_class=ArgumentDefaultsHelpFormatter + ) + # Path to database (default: .pyani/pyanidb) + parser.add_argument( + "--dbpath", + action="store", + dest="dbpath", + default=Path(".pyani/pyanidb"), + type=Path, + help="path to pyani database", + ) + direction = parser.add_mutually_exclusive_group(required=False) + direction.add_argument( + "-u", + "--upgrade", + action="store", + dest="upgrade", + default="head", + metavar="VERSION", + help="update an existing database to a newer schema; default is to upgrade to the newest version", + ) + direction.add_argument( + "-d", + "--downgrade", + action="store", + dest="downgrade", + default=None, + metavar="VERSION", + help="revert an existing database to a older schema", + ) + parser.set_defaults(func=subcommands.subcmd_versiondb) diff --git a/pyani/scripts/subcommands/__init__.py b/pyani/scripts/subcommands/__init__.py index 7e5e92ef..863a28ef 100644 --- a/pyani/scripts/subcommands/__init__.py +++ b/pyani/scripts/subcommands/__init__.py @@ -43,6 +43,7 @@ from .subcmd_anim import subcmd_anim from .subcmd_classify import subcmd_classify from .subcmd_createdb import subcmd_createdb +from .subcmd_versiondb import subcmd_versiondb from .subcmd_download import subcmd_download from .subcmd_index import subcmd_index from .subcmd_listdeps import subcmd_listdeps diff --git a/pyani/scripts/subcommands/subcmd_versiondb.py b/pyani/scripts/subcommands/subcmd_versiondb.py new file mode 100644 index 00000000..f79b311f --- /dev/null +++ b/pyani/scripts/subcommands/subcmd_versiondb.py @@ -0,0 +1,30 @@ +import logging + +from argparse import Namespace + +from pyani import pyani_orm + + +def subcmd_versiondb(args: Namespace) -> int: + """Create an empty pyani database. + + :param args: Namespace, command-line arguments + :param logger: logging object + """ + # Create logger + logger = logging.getLogger(__name__) + + # If the database exists, raise an error rather than overwrite + if not args.dbpath.is_file(): + logger.error("Database %s does not exist (exiting)", args.dbpath) + raise SystemError(1) + + # If the path to the database doesn't exist, create it + if args.upgrade: + logger.info("Upgrading database schema to: %s", args.upgrade) + elif args.downgrade: + logger.info("Downgrading database schema to: %s", args.downgrade) + + # Do some stuff in alembic + + return 0 From cbc6a71b5831fb6aece34e800b37167f4df3ed2e Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Fri, 4 Feb 2022 22:42:39 +0000 Subject: [PATCH 02/91] Add new requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 62841ab0..e58c7b23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ scipy seaborn sqlalchemy tqdm +alembic From 8fcd2245556b00867e7d2d60861b6afed2efa5e3 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 7 Feb 2022 22:56:24 +0000 Subject: [PATCH 03/91] Add `get_version()` for alembic --- pyani/dependencies.py | 1 + pyani/pyani_config.py | 1 + pyani/scripts/subcommands/subcmd_versiondb.py | 65 +++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/pyani/dependencies.py b/pyani/dependencies.py index 2f2ec865..32b9cdb0 100644 --- a/pyani/dependencies.py +++ b/pyani/dependencies.py @@ -58,6 +58,7 @@ "seaborn", "sqlalchemy", "tqdm", + "alembic", ] DEVELOPMENT = [ diff --git a/pyani/pyani_config.py b/pyani/pyani_config.py index caa268c0..7829a63f 100644 --- a/pyani/pyani_config.py +++ b/pyani/pyani_config.py @@ -55,6 +55,7 @@ FORMATDB_DEFAULT = Path("formatdb") QSUB_DEFAULT = Path("qsub") FASTANI_DEFAULT = Path("fastANI") +ALEMBIC_DEFAULT = Path("alembic") # Stems for output files ANIM_FILESTEMS = ( diff --git a/pyani/scripts/subcommands/subcmd_versiondb.py b/pyani/scripts/subcommands/subcmd_versiondb.py index f79b311f..2af1ced0 100644 --- a/pyani/scripts/subcommands/subcmd_versiondb.py +++ b/pyani/scripts/subcommands/subcmd_versiondb.py @@ -1,4 +1,16 @@ import logging +import os +import platform +import re +import shutil +import subprocess +import sys + +from logging import Logger +from pathlib import Path +from typing import List + +from pyani import pyani_config from argparse import Namespace @@ -28,3 +40,56 @@ def subcmd_versiondb(args: Namespace) -> int: # Do some stuff in alembic return 0 + + +def get_version(alembic_exe: Path = pyani_config.ALEMBIC_DEFAULT) -> str: + """Return ALembic package version as a string. + + :param alembic_exe: path to Alembic executable + + We expect Alembic to return a string on STDOUT as + + .. code-block:: bash + + $ alembic --version + alembic 1.7.5 + + we concatenate this with the OS name. + + The following circumstances are explicitly reported as strings: + + - no executable at passed path + - non-executable file at passed path (this includes cases where the user doesn't have execute permissions on the file) + - no version info returned + """ + + try: + alembic_path = Path(shutil.which(alembic_exe)) # type:ignore + except TypeError: + return f"{alembic_exe} is not found in $PATH" + + if not alembic_path.is_file(): # no executable + return f"No alembic at {alembic_path}" + + # This should catch cases when the file can't be executed by the user + if not os.access(alembic_path, os.X_OK): # file exists but not executable + return f"alembic exists at {alembic_path} but not executable" + + cmdline = [alembic_exe, "--version"] # type: List + result = subprocess.run( + cmdline, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + + if result.stdout: + match = re.search(r"(?<=alembic\s)[0-9\.]*", str(result.stdout, "utf-8")) + + version = match.group() # type: ignore + + if 0 == len(version.strip()): + return f"alembic exists at {alembic_path} but could not retrieve version" + + return f"{platform.system()}_{version} ({alembic_path})" From e7a5c30779bdfb57f05ced188437811dcd7d7a13 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 9 Feb 2022 05:51:48 +0000 Subject: [PATCH 04/91] Hook `pyani versiondb` infrastructure to functional code --- pyani/scripts/parsers/versiondb_parser.py | 28 ++++ pyani/scripts/subcommands/subcmd_versiondb.py | 94 ++++++-------- pyani/versiondb.py | 122 ++++++++++++++++++ 3 files changed, 188 insertions(+), 56 deletions(-) create mode 100644 pyani/versiondb.py diff --git a/pyani/scripts/parsers/versiondb_parser.py b/pyani/scripts/parsers/versiondb_parser.py index d0395624..12b38db1 100644 --- a/pyani/scripts/parsers/versiondb_parser.py +++ b/pyani/scripts/parsers/versiondb_parser.py @@ -4,6 +4,8 @@ from pathlib import Path from typing import List, Optional +from pyani import pyani_config + from pyani.scripts import subcommands @@ -47,4 +49,30 @@ def build( metavar="VERSION", help="revert an existing database to a older schema", ) + parser.add_argument( + "--alembic_exe", + action="store", + dest="alembic_exe", + default=pyani_config.ALEMBIC_DEFAULT, + type=Path, + help="path to alembic executable", + ) + parser.add_argument( + "-n", + "--name", + action="store", + dest="dbname", + default=None, + metavar="NAME", + help="used to specify an individual database in a multidb setup", + ) + parser.add_argument( + "-c", + "--config", + action="store", + dest="alembic_config", + default=None, + metavar="FILE", + help="used to specify a config file for alembic", + ) parser.set_defaults(func=subcommands.subcmd_versiondb) diff --git a/pyani/scripts/subcommands/subcmd_versiondb.py b/pyani/scripts/subcommands/subcmd_versiondb.py index 2af1ced0..3fb61a27 100644 --- a/pyani/scripts/subcommands/subcmd_versiondb.py +++ b/pyani/scripts/subcommands/subcmd_versiondb.py @@ -14,11 +14,16 @@ from argparse import Namespace -from pyani import pyani_orm +from pyani import ( + pyani_orm, + versiondb, +) + +from pyani.pyani_tools import termcolor def subcmd_versiondb(args: Namespace) -> int: - """Create an empty pyani database. + """Up/downgrade a pyani database. :param args: Namespace, command-line arguments :param logger: logging object @@ -26,70 +31,47 @@ def subcmd_versiondb(args: Namespace) -> int: # Create logger logger = logging.getLogger(__name__) - # If the database exists, raise an error rather than overwrite + # Announce what's happening + if args.upgrade: + logger.info(termcolor("Downgrading database to %s", bold=True), args.downgrade) + else: + logger.info(termcolor("Upgrading database to %s", bold=True), args.upgrade) + + # Get current alembic version + alembic_version = versiondb.get_version(args.alembic_exe) + logger.info(termcolor("Alembic version: %s", "cyan"), alembic_version) + + # If the database doesn't exist, raise an error if not args.dbpath.is_file(): logger.error("Database %s does not exist (exiting)", args.dbpath) raise SystemError(1) - # If the path to the database doesn't exist, create it - if args.upgrade: - logger.info("Upgrading database schema to: %s", args.upgrade) - elif args.downgrade: + # Up/downgrade database + if args.downgrade: logger.info("Downgrading database schema to: %s", args.downgrade) + versiondb.downgrade_database(args) + elif args.upgrade: + logger.info("Upgrading database schema to: %s", args.upgrade) + versiondb.upgrade_database(args) # Do some stuff in alembic return 0 -def get_version(alembic_exe: Path = pyani_config.ALEMBIC_DEFAULT) -> str: - """Return ALembic package version as a string. - - :param alembic_exe: path to Alembic executable - - We expect Alembic to return a string on STDOUT as - - .. code-block:: bash - - $ alembic --version - alembic 1.7.5 - - we concatenate this with the OS name. - - The following circumstances are explicitly reported as strings: - - - no executable at passed path - - non-executable file at passed path (this includes cases where the user doesn't have execute permissions on the file) - - no version info returned - """ - - try: - alembic_path = Path(shutil.which(alembic_exe)) # type:ignore - except TypeError: - return f"{alembic_exe} is not found in $PATH" - - if not alembic_path.is_file(): # no executable - return f"No alembic at {alembic_path}" - - # This should catch cases when the file can't be executed by the user - if not os.access(alembic_path, os.X_OK): # file exists but not executable - return f"alembic exists at {alembic_path} but not executable" - - cmdline = [alembic_exe, "--version"] # type: List - result = subprocess.run( - cmdline, - shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - ) - - if result.stdout: - match = re.search(r"(?<=alembic\s)[0-9\.]*", str(result.stdout, "utf-8")) - - version = match.group() # type: ignore +# Valid SQLite URL forms are: +# sqlite:///:memory: (or, sqlite://) +# sqlite:///relative/path/to/file.db +# sqlite:////absolute/path/to/file.db - if 0 == len(version.strip()): - return f"alembic exists at {alembic_path} but could not retrieve version" +# alembic init --package # alembic +# need to change location of sqlalchemy.url - return f"{platform.system()}_{version} ({alembic_path})" +# - alembic.ini +# - alembic +# - __init__.py +# - env.py +# - README +# - script.py.mako +# - versions +# - __init__.py diff --git a/pyani/versiondb.py b/pyani/versiondb.py new file mode 100644 index 00000000..a9770b97 --- /dev/null +++ b/pyani/versiondb.py @@ -0,0 +1,122 @@ +import logging +import os +import platform +import re +import shutil +import subprocess +import sys + +from typing import List + +from pathlib import Path + +from pyani import pyani_config + +from argparse import Namespace + + +def get_version(alembic_exe: Path = pyani_config.ALEMBIC_DEFAULT) -> str: + """Return ALembic package version as a string. + + :param alembic_exe: path to Alembic executable + + We expect Alembic to return a string on STDOUT as + + .. code-block:: bash + + $ alembic --version + alembic 1.7.5 + + we concatenate this with the OS name. + + The following circumstances are explicitly reported as strings: + + - no executable at passed path + - non-executable file at passed path (this includes cases where the user doesn't have execute permissions on the file) + - no version info returned + """ + + try: + alembic_path = Path(shutil.which(alembic_exe)) # type:ignore + except TypeError: + return f"{alembic_exe} is not found in $PATH" + + if not alembic_path.is_file(): # no executable + return f"No alembic at {alembic_path}" + + # This should catch cases when the file can't be executed by the user + if not os.access(alembic_path, os.X_OK): # file exists but not executable + return f"alembic exists at {alembic_path} but not executable" + + cmdline = [alembic_exe, "--version"] # type: List + result = subprocess.run( + cmdline, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + + if result.stdout: + match = re.search(r"(?<=alembic\s)[0-9\.]*", str(result.stdout, "utf-8")) + + version = match.group() # type: ignore + + if 0 == len(version.strip()): + return f"alembic exists at {alembic_path} but could not retrieve version" + + return f"{platform.system()}_{version} ({alembic_path})" + + +def get_optional_args(args: Namespace): + opts = [] + if args.dbname: + opts.extend(["-n", args.dbname]) + if args.alembic_config: + opts.extend(["-c", args.config]) + return opts + + +def construct_alembic_cmdline( + direction, + args: Namespace, + alembic_exe=pyani_config.ALEMBIC_DEFAULT, +): + if direction == "upgrade": + return [alembic_exe, direction, args.upgrade, *get_optional_args(args)] + elif direction == "downgrade": + return [alembic_exe, direction, args.downgrade, *get_optional_args(args)] + + +def upgrade_database(args: Namespace): + logger = logging.getLogger(__name__) + cmdline = construct_alembic_cmdline("upgrade", args) + result = subprocess.run( + cmdline, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + # with str(result.stdout, "utf-8") as pipe: + # for line in str(result.stdout, "utf-8"): + # logger.info('Alembic: %s', str(result.stderr, "utf-8")) + for line in str(result.stderr, "utf-8").split("\n"): + if line: + logger.info("Alembic: %s", line) + + +def downgrade_database(args: Namespace): + logger = logging.getLogger(__name__) + cmdline = construct_alembic_cmdline("downgrade", args) + result = subprocess.run( + cmdline, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + # logger.info('A: %s', str(result.stderr, "utf-8")) + for line in str(result.stderr, "utf-8").split("\n"): + if line: + logger.info("Alembic: %s", line) From 6b88b2bfcec0777b8c3b95f9df42e8ba1bf828d3 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sun, 13 Feb 2022 01:17:17 +0000 Subject: [PATCH 05/91] Add generic versions of files required for `alembic` to function --- alembic.ini | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++ alembic/env.py | 85 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 alembic.ini create mode 100644 alembic/env.py diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..15d1863b --- /dev/null +++ b/alembic.ini @@ -0,0 +1,99 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. Valid values are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # default: use os.pathsep + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = sqlite:///.pyani/pyanidb + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 00000000..acac8adf --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,85 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +import os + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +dbpath = os.environ.get("PYANI_DATABASE") +script_dir = os.environ.get("ALEMBIC_MIGRATIONS_DIR") +url = f"sqlite:///{dbpath}" +config.set_main_option("sqlalchemy.url", url) +config.set_main_option("script_location", "alembic") + + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() From bfd6a6060f456a74d68c723264eda6eb85df775e Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sun, 13 Feb 2022 01:22:33 +0000 Subject: [PATCH 06/91] Alter handling of `stderr` and `stdout`from `alembic` --- pyani/versiondb.py | 47 ++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/pyani/versiondb.py b/pyani/versiondb.py index a9770b97..84e559dc 100644 --- a/pyani/versiondb.py +++ b/pyani/versiondb.py @@ -83,40 +83,51 @@ def construct_alembic_cmdline( alembic_exe=pyani_config.ALEMBIC_DEFAULT, ): if direction == "upgrade": - return [alembic_exe, direction, args.upgrade, *get_optional_args(args)] + return [str(alembic_exe), direction, args.upgrade, *get_optional_args(args)] elif direction == "downgrade": - return [alembic_exe, direction, args.downgrade, *get_optional_args(args)] + return [str(alembic_exe), direction, args.downgrade, *get_optional_args(args)] -def upgrade_database(args: Namespace): +def log_output_and_errors(result): logger = logging.getLogger(__name__) + if result.stdout: + logger.info("Alembic stdout:") + for line in str(result.stdout, "utf-8").split("\n"): + if line: + logger.info(line) + if result.stderr: + logger.info("Alembic stderr:") + for line in str(result.stderr, "utf-8").split("\n"): + if line: + logger.info(line) + + +def upgrade_database(args: Namespace): cmdline = construct_alembic_cmdline("upgrade", args) result = subprocess.run( cmdline, shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE, + # check=True, + capture_output=True, ) # with str(result.stdout, "utf-8") as pipe: # for line in str(result.stdout, "utf-8"): - # logger.info('Alembic: %s', str(result.stderr, "utf-8")) - for line in str(result.stderr, "utf-8").split("\n"): - if line: - logger.info("Alembic: %s", line) + + log_output_and_errors(result) def downgrade_database(args: Namespace): - logger = logging.getLogger(__name__) cmdline = construct_alembic_cmdline("downgrade", args) + result = subprocess.run( cmdline, shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE, + # check=True, + capture_output=True, ) - # logger.info('A: %s', str(result.stderr, "utf-8")) - for line in str(result.stderr, "utf-8").split("\n"): - if line: - logger.info("Alembic: %s", line) + + log_output_and_errors(result) From ea6adf174376290bf35f5fe7a13dbbf2fb985279 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sun, 13 Feb 2022 01:25:18 +0000 Subject: [PATCH 07/91] Add logging and use environmental variables --- pyani/scripts/subcommands/subcmd_versiondb.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyani/scripts/subcommands/subcmd_versiondb.py b/pyani/scripts/subcommands/subcmd_versiondb.py index 3fb61a27..55e39b67 100644 --- a/pyani/scripts/subcommands/subcmd_versiondb.py +++ b/pyani/scripts/subcommands/subcmd_versiondb.py @@ -32,6 +32,7 @@ def subcmd_versiondb(args: Namespace) -> int: logger = logging.getLogger(__name__) # Announce what's happening + logger.info(termcolor("Database: %s", "cyan"), str(args.dbpath.resolve())) if args.upgrade: logger.info(termcolor("Downgrading database to %s", bold=True), args.downgrade) else: @@ -46,6 +47,10 @@ def subcmd_versiondb(args: Namespace) -> int: logger.error("Database %s does not exist (exiting)", args.dbpath) raise SystemError(1) + # Create environment variables for alembic to access + os.environ["PYANI_DATABASE"] = str(args.dbpath.resolve()) + os.environ["ALEMBIC_MIGRATIONS_DIR"] = "alembic" + # Up/downgrade database if args.downgrade: logger.info("Downgrading database schema to: %s", args.downgrade) From aed669d536fc97452f5ea47a1dbdf87424a3722f Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sun, 13 Feb 2022 01:26:36 +0000 Subject: [PATCH 08/91] Add migration script for `fastANI` changes to the ORM --- .../92f7f6b1626e_add_fastani_columns.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 alembic/versions/92f7f6b1626e_add_fastani_columns.py diff --git a/alembic/versions/92f7f6b1626e_add_fastani_columns.py b/alembic/versions/92f7f6b1626e_add_fastani_columns.py new file mode 100644 index 00000000..72c7d419 --- /dev/null +++ b/alembic/versions/92f7f6b1626e_add_fastani_columns.py @@ -0,0 +1,32 @@ +"""add fastani columns + +Revision ID: 92f7f6b1626e +Revises: +Create Date: 2022-02-07 22:57:35.779356 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "92f7f6b1626e" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("comparisons", sa.Column("kmersize", sa.Integer)) + op.add_column("comparisons", sa.Column("minmatch", sa.Float)) + # with op.batch_alter_table("comparisons") as batch_op: + # batch_op.add_column(sa.Column('kmersize', sa.Integer)) + # batch_op.add_column(sa.Column('minmatch', sa.Float)) + + +def downgrade(): + op.drop_column("comparisons", "kmersize") + op.drop_column("comparisons", "minmatch") + # with op.batch_alter_table("comparisons") as batch_op: + # batch_op.drop_column('kmersize') + # batch_op.drop_column('minmatch') From 3d669532e74641c4eec862cc5adf16e2878ce5bc Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sun, 13 Feb 2022 02:28:19 +0000 Subject: [PATCH 09/91] Backup existing database before migrating --- pyani/scripts/subcommands/subcmd_versiondb.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyani/scripts/subcommands/subcmd_versiondb.py b/pyani/scripts/subcommands/subcmd_versiondb.py index 55e39b67..d4256a1e 100644 --- a/pyani/scripts/subcommands/subcmd_versiondb.py +++ b/pyani/scripts/subcommands/subcmd_versiondb.py @@ -5,6 +5,7 @@ import shutil import subprocess import sys +import datetime from logging import Logger from pathlib import Path @@ -48,9 +49,14 @@ def subcmd_versiondb(args: Namespace) -> int: raise SystemError(1) # Create environment variables for alembic to access - os.environ["PYANI_DATABASE"] = str(args.dbpath.resolve()) + abs_path = args.dbpath.resolve() + os.environ["PYANI_DATABASE"] = str(abs_path) os.environ["ALEMBIC_MIGRATIONS_DIR"] = "alembic" + # Create a backup of the database + timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + shutil.copy(args.dbpath.resolve(), f"{abs_path}.{timestamp}.bak") + # Up/downgrade database if args.downgrade: logger.info("Downgrading database schema to: %s", args.downgrade) From 4a3563667c7504c6318e8f4731acef3d5cd20ec5 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Feb 2022 06:00:52 +0000 Subject: [PATCH 10/91] Use a local file to track versions in dry-run mode --- alembic/env.py | 19 +++++++++++++++++++ alembic/version.txt | 1 + 2 files changed, 20 insertions(+) create mode 100644 alembic/version.txt diff --git a/alembic/env.py b/alembic/env.py index acac8adf..69c7ffa1 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -80,6 +80,25 @@ def run_migrations_online(): if context.is_offline_mode(): + # Track the current version of the database in a local file + # this value is used in the --dry-run option + version_file = os.path.join( + os.path.dirname(config.config_file_name), "alembic/version.txt" + ) + if os.path.exists(version_file): + current_version = open(version_file).read().strip() + else: + current_version = None + context.configure(dialect_name="sqlite", starting_rev=current_version) + + # Perform the dry run run_migrations_offline() + + # Write 'new' version to file + end_version = context.get_revision_argument() + if end_version and end_version != current_version: + open(version_file, "w").write(end_version) + elif end_version is None: + open(version_file, "w").write("base") else: run_migrations_online() diff --git a/alembic/version.txt b/alembic/version.txt new file mode 100644 index 00000000..7fe54291 --- /dev/null +++ b/alembic/version.txt @@ -0,0 +1 @@ +65538af5a5e1 \ No newline at end of file From 18788ed5c442db99909025fdda7aa4f1a44fd312 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Feb 2022 06:03:55 +0000 Subject: [PATCH 11/91] Update parser to use `const` values instead of defaults --- pyani/scripts/parsers/versiondb_parser.py | 32 ++++++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/pyani/scripts/parsers/versiondb_parser.py b/pyani/scripts/parsers/versiondb_parser.py index 12b38db1..d0ab870f 100644 --- a/pyani/scripts/parsers/versiondb_parser.py +++ b/pyani/scripts/parsers/versiondb_parser.py @@ -9,6 +9,10 @@ from pyani.scripts import subcommands +def get_dry_run_args(): + pass + + def build( subps: _SubParsersAction, parents: Optional[List[ArgumentParser]] = None ) -> None: @@ -19,7 +23,10 @@ def build( """ parser = subps.add_parser( - "versiondb", parents=parents, formatter_class=ArgumentDefaultsHelpFormatter + "versiondb", + parents=parents, + formatter_class=ArgumentDefaultsHelpFormatter, + description="One of --upgrade, --downgrade, or --dry-run must be specified.", ) # Path to database (default: .pyani/pyanidb) parser.add_argument( @@ -30,24 +37,35 @@ def build( type=Path, help="path to pyani database", ) - direction = parser.add_mutually_exclusive_group(required=False) + direction = parser.add_mutually_exclusive_group(required=True) direction.add_argument( - "-u", "--upgrade", action="store", dest="upgrade", - default="head", + nargs="?", + default=None, + const="head", metavar="VERSION", - help="update an existing database to a newer schema; default is to upgrade to the newest version", + help="update an existing database to a newer schema; if no argument is given, 'head' will be used", ) direction.add_argument( - "-d", "--downgrade", action="store", dest="downgrade", + nargs="?", default=None, + const="base", metavar="VERSION", - help="revert an existing database to a older schema", + help="revert an existing database to a older schema; if no argument is given, 'base' will be used", + ) + direction.add_argument( + "--dry-run", + action="store", + dest="dry_run", + nargs="?", + metavar="START:END", + default=None, + help="produce the SQL that would be run in migrations, without altering the database; start and end versions must be specified", ) parser.add_argument( "--alembic_exe", From 0531a656a272848c2d62e1bfd179fd66ef2918ba Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Feb 2022 06:05:20 +0000 Subject: [PATCH 12/91] Update `versiondb` to accommodate `--dry-run` option --- pyani/versiondb.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pyani/versiondb.py b/pyani/versiondb.py index 84e559dc..15090abf 100644 --- a/pyani/versiondb.py +++ b/pyani/versiondb.py @@ -82,7 +82,15 @@ def construct_alembic_cmdline( args: Namespace, alembic_exe=pyani_config.ALEMBIC_DEFAULT, ): - if direction == "upgrade": + if args.dry_run: + return [ + str(alembic_exe), + direction, + args.dry_run, + "--sql", + *get_optional_args(args), + ] # FAILED: downgrade with --sql requires : + elif direction == "upgrade": return [str(alembic_exe), direction, args.upgrade, *get_optional_args(args)] elif direction == "downgrade": return [str(alembic_exe), direction, args.downgrade, *get_optional_args(args)] From b2976301030924722ed1dc09f108a24e4c1b76ed Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Feb 2022 06:05:45 +0000 Subject: [PATCH 13/91] Clean up `subcmd_versiondb` --- pyani/scripts/subcommands/subcmd_versiondb.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyani/scripts/subcommands/subcmd_versiondb.py b/pyani/scripts/subcommands/subcmd_versiondb.py index d4256a1e..8650f1a9 100644 --- a/pyani/scripts/subcommands/subcmd_versiondb.py +++ b/pyani/scripts/subcommands/subcmd_versiondb.py @@ -34,7 +34,7 @@ def subcmd_versiondb(args: Namespace) -> int: # Announce what's happening logger.info(termcolor("Database: %s", "cyan"), str(args.dbpath.resolve())) - if args.upgrade: + if args.downgrade: logger.info(termcolor("Downgrading database to %s", bold=True), args.downgrade) else: logger.info(termcolor("Upgrading database to %s", bold=True), args.upgrade) @@ -65,8 +65,6 @@ def subcmd_versiondb(args: Namespace) -> int: logger.info("Upgrading database schema to: %s", args.upgrade) versiondb.upgrade_database(args) - # Do some stuff in alembic - return 0 From 51bac6f6f77acb853d8801189f783415b9496c7c Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Feb 2022 07:46:06 +0000 Subject: [PATCH 14/91] Add initial tests for `versiondb` --- tests/test_subcmd_10_versiondb.py | 58 +++++++++++++++++++++++ tests/test_versiondb.py | 78 +++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 tests/test_subcmd_10_versiondb.py create mode 100644 tests/test_versiondb.py diff --git a/tests/test_subcmd_10_versiondb.py b/tests/test_subcmd_10_versiondb.py new file mode 100644 index 00000000..ec5d862e --- /dev/null +++ b/tests/test_subcmd_10_versiondb.py @@ -0,0 +1,58 @@ +import logging +import unittest + +from argparse import Namespace +from typing import NamedTuple +from pathlib import Path + +from pyani.scripts import subcommands + + +# Convenience struct with paths to third-party executables +class ThirdPartyExes(NamedTuple): + alembic_exe: Path + + +# Convenience struct with paths to working directories +class DirPaths(NamedTuple): + indir: Path + outdir: Path + + +class TestVersiondbSubcommand(unittest.TestCase): + + """Class defining tests of the pyani versiondb subcommand.""" + + def setUp(self): + """Configure parameters for tests.""" + testdir = Path("tests") + self.dirpaths = DirPaths( + testdir / "test_input" / "subcmd_versiondb", + testdir / "test_output" / "subcmd_versiondb", + ) + self.dirpaths.outdir.Mkdir(exist_ok=True) + self.dbpath = testdir / "test_input" / "subcmd_versiondb" / "pyanidb" + self.exes = ThirdPartyExes("alembic") + + # Null logger instance + self.logger = logging.getLogger("TestVersiondbSubcommand logger") + self.logger.addHandler(logging.NullHandler()) + + # Command line namespaces + self.argsdict = { + "versiondb": Namespace( + indir=self.dirpaths.indir, + outdir=self.dirpaths.outdir, + dbpath=self.dbpath, + name="test_versiondb", + cmdline="alembic test suite", + alembic_exe=self.exes.alembic_exe, + workers=None, + jobprefix="alembicTest", + ) + } + + def test_versiondb(self): + """Test versiondb run.""" + print(self.argsdict["versiondb"]) + subcommands.subcmd_versiondb(self.argsdict["versiondb"]) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py new file mode 100644 index 00000000..6f9a2de4 --- /dev/null +++ b/tests/test_versiondb.py @@ -0,0 +1,78 @@ +"""Test versiondb.py module. + +These tests are intended to be run from the repository root using: + +pytest -v +""" + +import os + +from pathlib import Path +from typing import List, NamedTuple, Tuple + +import pandas as pd +import pytest +import unittest + +from pandas.util.testing import assert_frame_equal + +from pyani import versiondb, pyani_files, pyani_tools + + +# Test get_version() +# Test case 0: no executable location is specified +def test_get_version_nonetype(): + """Test behaviour when no location for the executable is given.""" + test_file_0 = None + + assert versiondb.get_version(test_file_0) == f"{test_file_0} is not found in $PATH" + + +# Test case 1: there is no executable +def test_get_version_no_exe(executable_missing, monkeypatch): + """Test behaviour when there is no file at the specified executable location.""" + test_file_1 = Path("/non/existent/alembic") + assert ( + versiondb.get_version(test_file_1) == f"No alembic executable at {test_file_1}" + ) + + +# Test case 2: there is a file, but it is not executable +def test_get_version_exe_not_executable(executable_not_executable, monkeypatch): + """Test behaviour when the file at the executable location is not executable.""" + test_file_2 = Path("/non/executable/alembic") + assert ( + versiondb.get_version(test_file_2) + == f"alembic exists at {test_file_2} but not executable" + ) + + +# Test case 3: there is an executable file, but the version can't be retrieved +def test_get_version_exe_no_version(executable_without_version, monkeypatch): + """Test behaviour when the version for the executable can not be retrieved.""" + test_file_3 = Path("/missing/version/alembic") + assert ( + versiondb.get_version(test_file_3) + == f"alembic exists at {test_file_3} but could not retrieve version" + ) + + +# Test alembic command generation +def test_alembic_cmdline_generation(): + """Generate single alembic command line.""" + pass + # alembic_cmd = versiondb.construct_alembic_cmdline() + # dir_alembic = tmp_path / "versiondb_output" + # expected = "alembic upgrade" + + +# Test upgrade +def test_versiondb_upgrade(): + """ """ + pass + + +# Test downgrade + + +# Test dry-run result From b37a1ca9470a508ce77fc102d7e89fd2d0e386e3 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 23 Feb 2022 07:42:15 -1000 Subject: [PATCH 15/91] Fix test comparison value --- tests/test_versiondb.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 6f9a2de4..f9ae4178 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -32,9 +32,7 @@ def test_get_version_nonetype(): def test_get_version_no_exe(executable_missing, monkeypatch): """Test behaviour when there is no file at the specified executable location.""" test_file_1 = Path("/non/existent/alembic") - assert ( - versiondb.get_version(test_file_1) == f"No alembic executable at {test_file_1}" - ) + assert versiondb.get_version(test_file_1) == f"No alembic at {test_file_1}" # Test case 2: there is a file, but it is not executable From b2c74f408c4b2827e2d38d2e7afcdfc7b8cad0ab Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 23 Feb 2022 07:42:49 -1000 Subject: [PATCH 16/91] Specify which parser arguments are optional --- pyani/scripts/parsers/versiondb_parser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyani/scripts/parsers/versiondb_parser.py b/pyani/scripts/parsers/versiondb_parser.py index d0ab870f..2549a9f7 100644 --- a/pyani/scripts/parsers/versiondb_parser.py +++ b/pyani/scripts/parsers/versiondb_parser.py @@ -71,6 +71,7 @@ def build( "--alembic_exe", action="store", dest="alembic_exe", + required=False, default=pyani_config.ALEMBIC_DEFAULT, type=Path, help="path to alembic executable", @@ -81,6 +82,7 @@ def build( action="store", dest="dbname", default=None, + required=False, metavar="NAME", help="used to specify an individual database in a multidb setup", ) @@ -90,6 +92,7 @@ def build( action="store", dest="alembic_config", default=None, + required=False, metavar="FILE", help="used to specify a config file for alembic", ) From 0b9d47a3b19ffa386e1de38b0d4ae22db8a43ea1 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 23 Feb 2022 11:35:39 -1000 Subject: [PATCH 17/91] Abstract migration functions to one and implement dry-run --- pyani/scripts/parsers/versiondb_parser.py | 26 ++++++++--- pyani/scripts/subcommands/subcmd_versiondb.py | 14 ++++-- pyani/versiondb.py | 43 ++++++++----------- 3 files changed, 46 insertions(+), 37 deletions(-) diff --git a/pyani/scripts/parsers/versiondb_parser.py b/pyani/scripts/parsers/versiondb_parser.py index 2549a9f7..b03ef1f7 100644 --- a/pyani/scripts/parsers/versiondb_parser.py +++ b/pyani/scripts/parsers/versiondb_parser.py @@ -1,6 +1,11 @@ """Provides parser for versiondb subcommand.""" -from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, _SubParsersAction +from argparse import ( + ArgumentDefaultsHelpFormatter, + ArgumentParser, + _SubParsersAction, + Action, +) from pathlib import Path from typing import List, Optional @@ -9,8 +14,14 @@ from pyani.scripts import subcommands -def get_dry_run_args(): - pass +class DryRunAction(Action): + def __init__(self, option_strings, **kwargs): + super().__init__(option_strings, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + print("%r %r %r" % (namespace, values, option_string)) + setattr(namespace, "direction", values.pop(0)) + setattr(namespace, "dry_run", values.pop(0)) def build( @@ -60,12 +71,13 @@ def build( ) direction.add_argument( "--dry-run", - action="store", + action=DryRunAction, dest="dry_run", - nargs="?", - metavar="START:END", + required=False, + nargs=2, + metavar=("DIRECTION", "START:END"), default=None, - help="produce the SQL that would be run in migrations, without altering the database; start and end versions must be specified", + help="produce the SQL that would be run in migrations, without altering the database; a direction {upgrade or downgrade} and start and end versions e.g., {head:base, base:head, base:} must be specified", ) parser.add_argument( "--alembic_exe", diff --git a/pyani/scripts/subcommands/subcmd_versiondb.py b/pyani/scripts/subcommands/subcmd_versiondb.py index 8650f1a9..711f8ac8 100644 --- a/pyani/scripts/subcommands/subcmd_versiondb.py +++ b/pyani/scripts/subcommands/subcmd_versiondb.py @@ -55,15 +55,21 @@ def subcmd_versiondb(args: Namespace) -> int: # Create a backup of the database timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - shutil.copy(args.dbpath.resolve(), f"{abs_path}.{timestamp}.bak") # Up/downgrade database - if args.downgrade: + if args.dry_run: + logger.info( + "(Dry run): Migrating database from %s to %s", *args.dry_run.split(":") + ) + versiondb.migrate_database(args.direction, args, timestamp=timestamp) + elif args.downgrade: logger.info("Downgrading database schema to: %s", args.downgrade) - versiondb.downgrade_database(args) + shutil.copy(args.dbpath.resolve(), f"{abs_path}.{timestamp}.bak") + versiondb.migrate_database("downgrade", args) elif args.upgrade: logger.info("Upgrading database schema to: %s", args.upgrade) - versiondb.upgrade_database(args) + shutil.copy(args.dbpath.resolve(), f"{abs_path}.{timestamp}.bak") + versiondb.migrate_database("upgrade", args) return 0 diff --git a/pyani/versiondb.py b/pyani/versiondb.py index 15090abf..418fdc77 100644 --- a/pyani/versiondb.py +++ b/pyani/versiondb.py @@ -96,38 +96,29 @@ def construct_alembic_cmdline( return [str(alembic_exe), direction, args.downgrade, *get_optional_args(args)] -def log_output_and_errors(result): +def log_output_and_errors(result, args: Namespace, timestamp=None): logger = logging.getLogger(__name__) if result.stdout: logger.info("Alembic stdout:") - for line in str(result.stdout, "utf-8").split("\n"): - if line: - logger.info(line) + for line in str(result.stdout, "utf-8").split("\n"): + if line: + logger.info(line) + if args.dry_run: + abs_path = args.dbpath.resolve() + with open(f"{abs_path}.{timestamp}.sql", "w") as sqlfile: + for line in str(result.stdout, "utf-8").split("\n"): + if line: + sqlfile.write(f"{line}\n") + if result.stderr: logger.info("Alembic stderr:") - for line in str(result.stderr, "utf-8").split("\n"): - if line: - logger.info(line) - - -def upgrade_database(args: Namespace): - cmdline = construct_alembic_cmdline("upgrade", args) - result = subprocess.run( - cmdline, - shell=False, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE, - # check=True, - capture_output=True, - ) - # with str(result.stdout, "utf-8") as pipe: - # for line in str(result.stdout, "utf-8"): - - log_output_and_errors(result) + for line in str(result.stderr, "utf-8").split("\n"): + if line: + logger.info(line) -def downgrade_database(args: Namespace): - cmdline = construct_alembic_cmdline("downgrade", args) +def migrate_database(direction, args: Namespace): + cmdline = construct_alembic_cmdline(direction, args) result = subprocess.run( cmdline, @@ -138,4 +129,4 @@ def downgrade_database(args: Namespace): capture_output=True, ) - log_output_and_errors(result) + log_output_and_errors(result, args) From 7f933ba8ca6dc80c132cdcc813b9799c7818f0c3 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Feb 2022 15:08:07 -1000 Subject: [PATCH 18/91] Add `alembic_version` table to database model --- pyani/pyani_orm.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/pyani/pyani_orm.py b/pyani/pyani_orm.py index 40864afc..835df848 100644 --- a/pyani/pyani_orm.py +++ b/pyani/pyani_orm.py @@ -53,7 +53,17 @@ from sqlalchemy import and_ # type: ignore import sqlalchemy from sqlalchemy import UniqueConstraint, create_engine, Table -from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Float, Boolean +from sqlalchemy import ( + Column, + DateTime, + ForeignKey, + Integer, + String, + Float, + Boolean, + types, + PrimaryKeyConstraint, +) from sqlalchemy.ext.declarative import declarative_base # type: ignore from sqlalchemy.orm import relationship, sessionmaker # type: ignore @@ -104,6 +114,14 @@ class LabelTuple(NamedTuple): class_label: str +class Alembic(Base): + __tablename__ = "alembic_version" + __table_args__ = (PrimaryKeyConstraint("version_num", name="alembic_version_pkc"),) + # __table_args__ = tuple([PrimaryKeyConstraint('alembic_version_pkc', name='version_num')]) + + version_num = Column(types.VARCHAR(32), primary_key=True, nullable=False) + + class Label(Base): """Describes relationship between genome, run and genome label. @@ -347,6 +365,19 @@ def get_session(dbpath: Path) -> Any: return Session() +def add_alembic(session, version_num): + try: + db_version = Alembic(version_num=version_num) + except Exception: + raise PyaniORMException(f"Could not create Alembic() object {version_num}") + try: + session.add(db_version) + session.commit() + except Exception: + raise PyaniORMException(f"Could not add version {version_num}") + return db_version.version_num + + def get_comparison_dict(session: Any) -> Dict[Tuple, Any]: """Return a dictionary of comparisons in the session database. From 216778a571f8491f74eeed9a109c4a85d0ffeb27 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Feb 2022 15:09:27 -1000 Subject: [PATCH 19/91] Add current database version to `alembic_version` table --- pyani/scripts/subcommands/subcmd_createdb.py | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pyani/scripts/subcommands/subcmd_createdb.py b/pyani/scripts/subcommands/subcmd_createdb.py index ea445b1c..95ef39d3 100644 --- a/pyani/scripts/subcommands/subcmd_createdb.py +++ b/pyani/scripts/subcommands/subcmd_createdb.py @@ -45,6 +45,8 @@ from pyani import pyani_orm +from pyani.pyani_orm import PyaniORMException, get_session, add_alembic + def subcmd_createdb(args: Namespace) -> int: """Create an empty pyani database. @@ -72,4 +74,29 @@ def subcmd_createdb(args: Namespace) -> int: logger.info("Creating pyani database at %s", args.dbpath) pyani_orm.create_db(args.dbpath) + # Get connection to existing database. This may or may not have data + logger.debug("Connecting to database %s", args.dbpath) + try: + session = get_session(args.dbpath) + except Exception: + logger.error( + "Could not connect to database %s (exiting)", args.dbpath, exc_info=True + ) + raise SystemExit(1) + + # Add information about the database version to the database + logger.debug("Adding database version to database %s...", args.dbpath) + try: + version_num = add_alembic( + session, version_num="92f7f6b1626e" # most current version (fastani) + ) + except PyaniORMException: + logger.error( + "Could not add db_version %s to the database (exiting)", + version_num, + exc_info=True, + ) + raise SystemExit(1) + logger.debug("...added db_version: %s to the database", version_num) + return 0 From ac061d5918cef3bd45034b1a775f9f6df871cfd9 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Feb 2022 15:12:13 -1000 Subject: [PATCH 20/91] Update .gitignore to ignore some testing files --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1bb86dd5..c7f50db4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # Scratch directory for local testing scratch/ +alembic/version.txt +*/*add_third_column.py # Mac-related dreck .DS_Store @@ -72,4 +74,4 @@ venv-* # Extra documentation output classes_pyani.pdf -packages_pyani.pdf \ No newline at end of file +packages_pyani.pdf From f8755084774d6013295528518dcbeb0b0dd9f04f Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Feb 2022 21:50:01 -1000 Subject: [PATCH 21/91] Change unique constraints for `comparisons` table with Alembic --- .../92f7f6b1626e_add_fastani_columns.py | 40 ++++++++++++++----- pyani/pyani_orm.py | 1 + 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/alembic/versions/92f7f6b1626e_add_fastani_columns.py b/alembic/versions/92f7f6b1626e_add_fastani_columns.py index 72c7d419..f98c1430 100644 --- a/alembic/versions/92f7f6b1626e_add_fastani_columns.py +++ b/alembic/versions/92f7f6b1626e_add_fastani_columns.py @@ -17,16 +17,36 @@ def upgrade(): - op.add_column("comparisons", sa.Column("kmersize", sa.Integer)) - op.add_column("comparisons", sa.Column("minmatch", sa.Float)) - # with op.batch_alter_table("comparisons") as batch_op: - # batch_op.add_column(sa.Column('kmersize', sa.Integer)) - # batch_op.add_column(sa.Column('minmatch', sa.Float)) + # op.add_column("comparisons", sa.Column("kmersize", sa.Integer)) + # op.add_column("comparisons", sa.Column("minmatch", sa.Float)) + + with op.batch_alter_table("comparisons") as batch_op: + batch_op.add_column(sa.Column("kmersize", sa.Integer)) + batch_op.add_column(sa.Column("minmatch", sa.Float)) + batch_op.create_unique_constraint( + "fastani_reqs", + [ + "query_id", + "subject_id", + "program", + "version", + "fragsize", + "maxmatch", + "kmersize", + "minmatch", + ], + ) def downgrade(): - op.drop_column("comparisons", "kmersize") - op.drop_column("comparisons", "minmatch") - # with op.batch_alter_table("comparisons") as batch_op: - # batch_op.drop_column('kmersize') - # batch_op.drop_column('minmatch') + # op.drop_constraint("comparisons", 'kmersize') + # op.drop_column("comparisons", "kmersize") + + with op.batch_alter_table("comparisons") as batch_op: + batch_op.drop_column("kmersize") + batch_op.drop_column("minmatch") + batch_op.drop_constraint("fastani_reqs") + batch_op.create_unique_constraint( + "base_reqs", + ["query_id", "subject_id", "program", "version", "fragsize", "maxmatch"], + ) diff --git a/pyani/pyani_orm.py b/pyani/pyani_orm.py index 835df848..42fa468a 100644 --- a/pyani/pyani_orm.py +++ b/pyani/pyani_orm.py @@ -297,6 +297,7 @@ class Comparison(Base): "maxmatch", "kmersize", "minmatch", + name="fastani_reqs", ), ) From f8140b9a4604832a5868397d9e77cf9e94954795 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Feb 2022 22:00:40 -1000 Subject: [PATCH 22/91] Groundwork for `tests/test_versiondb.py` --- tests/conftest.py | 11 ++++++ tests/test_versiondb.py | 88 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 10e1ca21..37522c67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -66,6 +66,11 @@ FIXTUREPATH = TESTSPATH / "fixtures" +def pytest_configure(): + pytest.testspath = Path(__file__).parents[0] + pytest.fixturepath = pytest.testspath / "fixtures" + + # Convenience structs to emulate returned objects class MockGenome(NamedTuple): """Mock genome object.""" @@ -142,6 +147,12 @@ def dir_graphics_in(): return FIXTUREPATH / "graphics" +@pytest.fixture +def dir_versiondb_in(): + """Input files for versiondb ests.""" + return FIXTUREPATH / "versiondb" + + @pytest.fixture def dir_seq(): """Sequence files for tests.""" diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index f9ae4178..5649f15f 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -7,12 +7,15 @@ import os +from argparse import Namespace from pathlib import Path from typing import List, NamedTuple, Tuple import pandas as pd import pytest import unittest +import shutil +import filecmp from pandas.util.testing import assert_frame_equal @@ -55,6 +58,56 @@ def test_get_version_exe_no_version(executable_without_version, monkeypatch): ) +@pytest.fixture +def versiondb_namespaces(dir_versiondb_in): + { + "upgrade": Namespace( + dbpath=dir_versiondb_in / "pyanidb_upgrade", + upgrade="head", + downgrade=None, + dry_run=None, + direction=None, + ), + "downgrade": Namespace( + dbpath=dir_versiondb_in / "pyanidb_downgrade", + upgrade=None, + downgrade="base", + dry_run=None, + direction=None, + ), + "dry_down": Namespace( + dbpath=dir_versiondb_in / "pyanidb_dry_down", + upgrade=None, + downgrade=None, + dry_run="head:base", + direction="downgrade", + ), + "dry_up": Namespace( + dbpath=dir_versiondb_in / "pyanidb_dry_up", + upgrade=None, + downgrade=None, + dry_run="base:head", + direction="upgrade", + ), + "altdb": Namespace( + dbpath=dir_versiondb_in / "pyanidb_altdb", + upgrade="head", + downgrade=None, + dry_run=None, + direction=None, + name="altdb", + ), + "alt_config": Namespace( + dbpath=dir_versiondb_in / "pyanidb_alt_config", + upgrade="head", + downgrade=None, + dry_run=None, + direction=None, + config="alt_config", + ), + } + + # Test alembic command generation def test_alembic_cmdline_generation(): """Generate single alembic command line.""" @@ -65,12 +118,41 @@ def test_alembic_cmdline_generation(): # Test upgrade -def test_versiondb_upgrade(): +def test_versiondb_upgrade(dir_versiondb_in): """ """ - pass + shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_upgrade") # Test downgrade +def test_versiondb_downgrade(dir_versiondb_in): + """ """ + shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_downgrade") + + +# Test dry-run upgrade result +def test_versiondb_dry_upgrade(dir_versiondb_in): + shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_dry_up") + + assert filecmp.cmp( + dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_dry_up" + ) + + +# Test dry-run upgrade result +def test_versiondb_dry_downgrade(dir_versiondb_in): + shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_dry_down") + + assert filecmp.cmp( + dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_dry_down" + ) + + +# Test dry-run upgrade result +def test_versiondb_altname(dir_versiondb_in): + shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_altdb") + +# Test dry-run upgrade result +def test_versiondb_alt_config(dir_versiondb_in): -# Test dry-run result + shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_alt_config") From 2c3500ca10380089463e201076abd8bcbb0f3b86 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Tue, 1 Mar 2022 19:10:09 -1000 Subject: [PATCH 23/91] Add timestamp to dry-run output file name --- pyani/scripts/subcommands/subcmd_versiondb.py | 4 ++-- pyani/versiondb.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyani/scripts/subcommands/subcmd_versiondb.py b/pyani/scripts/subcommands/subcmd_versiondb.py index 711f8ac8..e34b87e7 100644 --- a/pyani/scripts/subcommands/subcmd_versiondb.py +++ b/pyani/scripts/subcommands/subcmd_versiondb.py @@ -65,11 +65,11 @@ def subcmd_versiondb(args: Namespace) -> int: elif args.downgrade: logger.info("Downgrading database schema to: %s", args.downgrade) shutil.copy(args.dbpath.resolve(), f"{abs_path}.{timestamp}.bak") - versiondb.migrate_database("downgrade", args) + versiondb.migrate_database("downgrade", args, timestamp=timestamp) elif args.upgrade: logger.info("Upgrading database schema to: %s", args.upgrade) shutil.copy(args.dbpath.resolve(), f"{abs_path}.{timestamp}.bak") - versiondb.migrate_database("upgrade", args) + versiondb.migrate_database("upgrade", args, timestamp=timestamp) return 0 diff --git a/pyani/versiondb.py b/pyani/versiondb.py index 418fdc77..befa0348 100644 --- a/pyani/versiondb.py +++ b/pyani/versiondb.py @@ -96,7 +96,7 @@ def construct_alembic_cmdline( return [str(alembic_exe), direction, args.downgrade, *get_optional_args(args)] -def log_output_and_errors(result, args: Namespace, timestamp=None): +def log_output_and_errors(result, direction, args: Namespace, timestamp=None): logger = logging.getLogger(__name__) if result.stdout: logger.info("Alembic stdout:") @@ -105,7 +105,7 @@ def log_output_and_errors(result, args: Namespace, timestamp=None): logger.info(line) if args.dry_run: abs_path = args.dbpath.resolve() - with open(f"{abs_path}.{timestamp}.sql", "w") as sqlfile: + with open(f"{abs_path}.{direction}.{timestamp}.sql", "w") as sqlfile: for line in str(result.stdout, "utf-8").split("\n"): if line: sqlfile.write(f"{line}\n") @@ -117,7 +117,7 @@ def log_output_and_errors(result, args: Namespace, timestamp=None): logger.info(line) -def migrate_database(direction, args: Namespace): +def migrate_database(direction, args: Namespace, timestamp=None): cmdline = construct_alembic_cmdline(direction, args) result = subprocess.run( @@ -129,4 +129,4 @@ def migrate_database(direction, args: Namespace): capture_output=True, ) - log_output_and_errors(result, args) + log_output_and_errors(result, direction, args, timestamp) From 9c642b39fb450a2a46157bebca2d61692a6cfb3b Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Tue, 1 Mar 2022 19:10:31 -1000 Subject: [PATCH 24/91] Update migration file to allow for reflection in offline mode --- .../92f7f6b1626e_add_fastani_columns.py | 73 ++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/alembic/versions/92f7f6b1626e_add_fastani_columns.py b/alembic/versions/92f7f6b1626e_add_fastani_columns.py index f98c1430..06075ff5 100644 --- a/alembic/versions/92f7f6b1626e_add_fastani_columns.py +++ b/alembic/versions/92f7f6b1626e_add_fastani_columns.py @@ -8,6 +8,17 @@ from alembic import op import sqlalchemy as sa +from sqlalchemy import ( + Column, + Table, + MetaData, + ForeignKey, + Integer, + String, + Float, + Boolean, + UniqueConstraint, +) # revision identifiers, used by Alembic. revision = "92f7f6b1626e" @@ -16,11 +27,69 @@ depends_on = None +meta = MetaData() +new_comparisons = Table( + "new_comparisons", + meta, + Column("comparisons_id", Integer, primary_key=True), + Column("query_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("subject_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("aln_length", Integer), + Column("sim_errs", Integer), + Column("identity", Float), + Column("cov_query", Float), + Column("cov_subject", Float), + Column("program", String), + Column("version", String), + Column("fragsize", Integer), + Column("maxmatch", Boolean), + Column("kmersize", Integer), + Column("minmatch", Float), + UniqueConstraint( + "query_id", + "subject_id", + "program", + "version", + "fragsize", + "maxmatch", + "kmersize", + "minmatch", + name="fastani_reqs", + ), +) + +old_comparisons = Table( + "old_comparisons", + meta, + Column("comparisons_id", Integer, primary_key=True), + Column("query_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("subject_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("aln_length", Integer), + Column("sim_errs", Integer), + Column("identity", Float), + Column("cov_query", Float), + Column("cov_subject", Float), + Column("program", String), + Column("version", String), + Column("fragsize", Integer), + Column("maxmatch", Boolean), + UniqueConstraint( + "query_id", + "subject_id", + "program", + "version", + "fragsize", + "maxmatch", + name="base_reqs", + ), +) + + def upgrade(): # op.add_column("comparisons", sa.Column("kmersize", sa.Integer)) # op.add_column("comparisons", sa.Column("minmatch", sa.Float)) - with op.batch_alter_table("comparisons") as batch_op: + with op.batch_alter_table("comparisons", copy_from=old_comparisons) as batch_op: batch_op.add_column(sa.Column("kmersize", sa.Integer)) batch_op.add_column(sa.Column("minmatch", sa.Float)) batch_op.create_unique_constraint( @@ -42,7 +111,7 @@ def downgrade(): # op.drop_constraint("comparisons", 'kmersize') # op.drop_column("comparisons", "kmersize") - with op.batch_alter_table("comparisons") as batch_op: + with op.batch_alter_table("comparisons", copy_from=new_comparisons) as batch_op: batch_op.drop_column("kmersize") batch_op.drop_column("minmatch") batch_op.drop_constraint("fastani_reqs") From 79e06f2d82a6ca3ec0e4cee87e28aa69e90ca341 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 3 Mar 2022 13:22:41 -1000 Subject: [PATCH 25/91] Add empty database fixtures for tests --- tests/fixtures/versiondb/base_pyanidb | Bin 0 -> 40960 bytes tests/fixtures/versiondb/head_pyanidb | Bin 0 -> 49152 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/fixtures/versiondb/base_pyanidb create mode 100644 tests/fixtures/versiondb/head_pyanidb diff --git a/tests/fixtures/versiondb/base_pyanidb b/tests/fixtures/versiondb/base_pyanidb new file mode 100644 index 0000000000000000000000000000000000000000..9f0ea475f35215440339d619b23b3726460a426b GIT binary patch literal 40960 zcmeI)O>5go7zc2vO`}+C+U_BU5&{DuV1rXAWiNYLH>N z({=4;@j`_J{UU+i4TC!8fk791k43`?OD<+i#l6ZY|>2$F^hlkL*`uxrUL- zX5z5&LFQd3dy65GX0gI-T^$&PwZE@_{2`yNbZlvHOPQp!V(t7$e=sEd(_T-E=e13B&uIz!R#IC`q_w`qJsX z=s0gg5I+~5(|-5l)F#amrA?LsZBh)SO^N}uN!e|il%2JzQ7#Xq#ChV!N#yfT&lkAw z%6`j^uiMjX{kXMhb@z2Gh&}&3dmHk*OP9{FB>yg-Anqe6KN^?MHw|lVPrnNC^csbf zW!{LDpN5iHWvj@lVtHL!EG8vKWA%g*c;}SojucvNRBb7VnY!6H^5h03uFK)As4j+f z9cnd<*PpDHd$$bh!2|tjC)W(=$Pd~4eN&N~w+Y95=_?iUW5^U^9!BP_G%G@7$N#6# z#mf!DIylfT$IIGXoCl)pxyAA=#Y$cy#1`@Ua%#zCcCpJh2zm3SVeRbbSC1D{5(C~? znSvPlJcG({D`htFvuUE_Gf(d@hB%yh&6Lw3rjfeK8BeaF z@X9CI8LtVvhS5oNYhKWcdaiEHOFFOU39spD`;>Y#qNy@;CDG=SZkeqMM>nk9UH$Tp zoT>J`fK**Fc(v&uz88F_K=qB55k2B}Lvslywa zR%drz-7V(fJ}MS&@|&$Kal^GO-tLPxydVGp2tWV=5P$##AOHafKmY;|sH;H!13#Ys z>l$9n3jz>;00bZa0SG_<0uX=z1R#(L;P-zd0t6rc0SG_<0uX=z1Rwwb2tc6z0(kzf ze~d9h2tWV=5P$##AOHafKmY;|fB>HVkpmEb00bZa0SG_<0uX=z1Rwx``U~Lszy2}C z3?Tpk2tWV=5P$##AOHafKmY=G{znc#00Izz00bZa0SG_<0uX=z1nMt<|NmeA7-NPI ufB*y_009U<00Izz00bZa0sQ`t9Do1>AOHafKmY;|fB*y_009WpU*JEg0=zT; literal 0 HcmV?d00001 diff --git a/tests/fixtures/versiondb/head_pyanidb b/tests/fixtures/versiondb/head_pyanidb new file mode 100644 index 0000000000000000000000000000000000000000..581f52226a3463ce74a13d63e0d4046fcd7dca29 GIT binary patch literal 49152 zcmeI)O>g2x7zc1WFE}PI^b%RCDAGt#Bv44DBxE09MWn}mN5q0wGC^V z)oyzV&7~irAEaMqzd$cN_Si9uVa6=Yp=woGe-wflzs>V|9?xJA<@ToG#t!KXL*I(Y zh4h&u%hET5NRp(|=MH_Qk1e{ek-nj?GPk^HQI$Ub{`qd{PiecbFO_ynzwN1eA9p_P zY!`o5e^oZr9&N@00uX=z1pY4q_ujUmou0}+Z^u^0bHZ^DnfA~hS)m&ZgDCrc=gQD) zZJo4hPa8VPt|jH7y4cxtyQJQ1>(6wfN{TAoIMZ_7GW7bhCV8R1E(^O=h@oHWhTgo= zZ^>d!L^7Kthvg44JB6@!(nV5cD~go|Q&qI1Bl(xjX}H|6xy7OoB)4K^e5BcGljdEc zk@V>8*a<&yp$aRbapwoej#sU=yukFFpdSx-y2$lSCk!LL&Fwlt?8YC+b)!{lC+D?? z@674B#U^&fMdK(O_Cw1j-_?w(=e22W-Z^394g09sVM+bXAp0-*I zz1B?5`o>T8&1~lev+d@6du7yL){NIlsy~&)yJr37PABD*u1Xe8Rl+)4C9IoOl6J02 z(k@j=+7a5uPG8B^M|yanLi$cH^c`9xvjv)&rTrG2d2G*O<%?oLs~^dd8+4ub(OZvx z(otE}zGi!DVvJd_{YX<>FeD~h|FD1B9hl1y8ZdSuJ13Thq{pU34( zF##P}wCcE`=ytL#DSIStX8T>fLCE!@drPYGu04jDP2=O2`Q%Vhv?ovGhyAIhXLTGe zn*H7qBxh}j<82=E-ZzRbbVGqX}^U5+IQl?P^f2G-kuvc zd|ta2y^;EsPK53HOFb#iD7NCUa9ISFFYeN##l?nf{vzwqF@cV2RzJHnoeJ1YAZ|`4 z13DKB>1-gj53H``TcOZ(q0nYQSBjN;V_ne>4&=#CQ!18h_Oc;>vBpoPTVP|E9axbt zhGa*O<+-ivM0V(o;>`3T^qd`>jm+s}!JU!@b-Gc|Y6q*vT^5IV#uhg3E3%@UoX8V( zYG8Se-*Ih|%`lm_Yl2B;Wn#m2GlTJbS}k9kR|KJ}R`a%P)aWN7k~xAodSeqIVD5M{ zw6E6++Re$T(6R?R%QURhkDAH-ytGT7a_J9={@?)t2tWV=5P$##AOHafKmY;|fIve{snOVpZ^>qLI^+r0uX=z1Rwwb2tWV= z5P$&g|4{=FfB*y_009U<00Izz00bZaf&2^L{y+aYMuZT600bZa0SG_<0uX=z1Rwwb z-2bBnAOHafKmY;|fB*y_009U<00Q|J!2kcxe~u9$1Rwwb2tWV=5P$##AOHafKmgzW bqXr-V0SG_<0uX=z1Rwwb2tWV=`4{*LU0EnF literal 0 HcmV?d00001 From 54e814f41fedc707d0e7daed4b6af034577cb51a Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 3 Mar 2022 13:39:17 -1000 Subject: [PATCH 26/91] Update `test_versiondb.py` input file names --- tests/test_versiondb.py | 55 +++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 5649f15f..994b4616 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -120,39 +120,78 @@ def test_alembic_cmdline_generation(): # Test upgrade def test_versiondb_upgrade(dir_versiondb_in): """ """ - shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_upgrade") + args = versiondb_namespaces["upgrade"] + timestamp = "testing" + shutil.copy(dir_versiondb_in / "base_pyanidb", dir_versiondb_in / "pyanidb_upgrade") + versiondb.migrate_database(args.direction, args, timestamp) + + assert filecmp.cmp( + dir_versiondb_in / "pyanidb_upgrade", dir_versiondb_in / "head_pyanidb" + ) # Test downgrade def test_versiondb_downgrade(dir_versiondb_in): """ """ - shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_downgrade") + args = versiondb_namespaces["downgrade"] + timestamp = "testing" + shutil.copy( + dir_versiondb_in / "head_pyanidb", dir_versiondb_in / "pyanidb_downgrade" + ) + versiondb.migrate_database(args.direction, args, timestamp) + + assert filecmp.cmp( + dir_versiondb_in / "pyanidb_downgrade", dir_versiondb_in / "base_pyanidb" + ) # Test dry-run upgrade result def test_versiondb_dry_upgrade(dir_versiondb_in): - shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_dry_up") + args = versiondb_namespaces["dry_up"] + timestamp = "testing" + shutil.copy(dir_versiondb_in / "base_pyanidb", dir_versiondb_in / "pyanidb_dry_up") + versiondb.migrate_database(args.direction, args, timestamp) assert filecmp.cmp( - dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_dry_up" + dir_versiondb_in / "pyanidb_dry_up", dir_versiondb_in / "head_pyanidb" ) # Test dry-run upgrade result def test_versiondb_dry_downgrade(dir_versiondb_in): - shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_dry_down") + args = versiondb_namespaces["dry_down"] + timestamp = "testing" + shutil.copy( + dir_versiondb_in / "head_pyanidb", dir_versiondb_in / "pyanidb_dry_down" + ) + versiondb.migrate_database(args.direction, args, timestamp) assert filecmp.cmp( - dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_dry_down" + dir_versiondb_in / "pyanidb_dry_down", dir_versiondb_in / "base_pyanidb" ) # Test dry-run upgrade result def test_versiondb_altname(dir_versiondb_in): - shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_altdb") + args = versiondb_namespaces["altname"] + timestamp = "testing" + shutil.copy(dir_versiondb_in / "head_pyanidb", dir_versiondb_in / "pyanidb_altdb") + versiondb.migrate_database(args.direction, args, timestamp) + + assert filecmp.cmp( + dir_versiondb_in / "pyanidb_altdb", dir_versiondb_in / "head_pyanidb" + ) # Test dry-run upgrade result def test_versiondb_alt_config(dir_versiondb_in): + args = versiondb_namespaces["alt_config"] + timestamp = "testing" + shutil.copy( + dir_versiondb_in / "head_pyanidb", dir_versiondb_in / "pyanidb_alt_config" + ) + versiondb.migrate_database(args.direction, args, timestamp) - shutil.copy(dir_versiondb_in / "pyanidb", dir_versiondb_in / "pyanidb_alt_config") + assert filecmp.cmp( + dir_versiondb_in / "pyanidb_alt_config", dir_versiondb_in / "head_pyanidb" + ) From a747d46b4d070f7fa474c435b28c25933e396a59 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 9 Mar 2022 05:17:01 -1000 Subject: [PATCH 27/91] Add docs for `versiondb` --- docs/run_versiondb.rst | 97 +++++++++++++++++++++++++++++++++++++++ docs/subcmd_versiondb.rst | 43 +++++++++++++++++ docs/subcommands.rst | 1 + 3 files changed, 141 insertions(+) create mode 100644 docs/run_versiondb.rst create mode 100644 docs/subcmd_versiondb.rst diff --git a/docs/run_versiondb.rst b/docs/run_versiondb.rst new file mode 100644 index 00000000..0f694bfb --- /dev/null +++ b/docs/run_versiondb.rst @@ -0,0 +1,97 @@ +.. _pyani-run_versiondb: + +================= +Running versiondb +================= + +``pyani`` allows for migration of databases between versions using `versiondb`_(*versiondb*), backed by `Alembic`_. To run versiondb on an existing database, use the ``pyani versiondb`` subcommand. + +In brief, ``versiondb`` can upgrade or downgrade a database; it also has an option to perform a dry run: + +1. ``--upgrade`` is used to upgrade a database to a more current version. If the flag is passed, but no value is specified, this defaults to `head` (the most up-to-date version). A backup of the original database is created prior to the upgrade. +2. ``--downgrade`` is used to downgrade a database to an older version. If the flag is passed, but no value is specified, this defaults to `base` (the oldest version). A backup of the original database is created prior to the downgrade. +3. ``--dry-run`` is used to show what changes would be made to the specified database, without actually performing any alterations. An SQL script containing the commands that would be run is output. + + +.. ATTENTION:: + ``pyani versiondb`` requires that a working copy of `Alembic`_ is available. This should happen automatically with new ``pyani`` installs. If it is missing, please see `pyani-installation`_ for information about installing this package. + +For more information about the ``pyani versiondb`` subcommand, please see the `pyani-subcmd-versiondb`_ page, or issue the command ``pyani versiondb -h`` to see the inline help. + +---------------------------- +Perform versiondb migrations +---------------------------- + +The basic form of the command is: + +.. code-block:: bash + + pyani versiondb [--dbpath DATABASE] {--upgrade [VERSION], --downgrade [vVERSION], --dry-run {up, down} START:END} + +One of ``--upgrade``, ``--downgrade``, or ``--dry-run`` must be provided; a migration version can be optionally specified for ``--upgrade`` and ``--downgrade``; if one is not, these will behave as ``--upgrade head`` and ``--downgrade base``, respectively. ``--dry-run`` takes two values; the first, a direction (one of 'up' or 'down'), the second the start and end database versions (given as ``START:END``), specified by their unique hashes, or 'head' and 'base', for the newest/oldest versions, respectively. + +All backups and SQL script outputs from these options will be written to the same directory as the database given to ``--dbpath``. + +This instructs ``pyani`` to modify the database at ``--dbpath``. For example, the following command for ``versiondb`` upgrades a database ``./test_database`` and writes a backup ``./test_database.upgrade.YYYY-MM-DD_HH-MM-SS.bak`` to the same containing directory: + +.. code-block:: bash + + pyani versiondb --dbpath ./test_database --upgrade head + +.. ATTENTION:: + To view the result of running ``versiondb``, you will need to use the ``sqlite3 ``. Please see `SQLite`_ for more details. + +The expectation is that the above command is what most ``pyani`` users will need in terms of migrations, as this will perform the necessary alterations to the database to accommodate changes merged into ``pyani`` with commit `fastani_merge`_, which broke compatibility with previous database schemas. + + +-------------- +Advanced Usage +-------------- + +However, there may be cases where it is necessary to downgrade a database to a specific, *earlier*, version, or the user wants to perform a dry run. Some users may also want to specify a non-standard location for an Alembic config file, or need to use this in a project with a ``multidb`` setup. Some examples of these advanced cases can be found below. + +~~~~~~~~~~~~~~~~~~~ +Performing a dry run +~~~~~~~~~~~~~~~~~~~ + +This following command creates an SQL file, ``./test_database.downgrade.YYYY-MM-DD_HH-MM-SS.sql``, (in the same directory as ``./test_database``) containing the raw SQL that would produce the necessary changes to the database to migrate it to the specified version (in this case, downgrading it to ``base``): + +.. code-block:: bash + + pyani versiondb --dbpath ./test_database --dry-run down head:base + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using a different config file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, ``Alembic``, and therefore ``pyani versiondb``, assume a config file, ``alembic.ini``, located in a standard location, which in the case of ``pyani``, is among the package files. To use a different file, add the ``-c`` or ``--config`` flag to your ``pyani`` command: + +.. code-block:: bash + + pyani versiondb --dbpath ./test_database --upgrade head --config ./config.ini + +If you need to specify additional settings for `Alembic`_, or have multiple databases in your ``pyani`` project (especially if not all should be upgraded/downgraded), this is the way you will need to use this option. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using pyani versiondb in a multidb setup +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. NOTE:: + + For information about how to set up a project with multiple databases managed by `Alembic`_, please see the `Alembic`_ documentation on `working with multiple databases `_. + +To specify a single database in a ``multidb`` setup, use the ``-n`` or ``--name`` option, along with the designation for the correct database from your ``multidb`` config file: + +.. code-block:: bash + + pyani versiondb --dbpath ./test_database --upgrade head --config ./multidb.ini --name database2 + +---------- +References +---------- + +.. _SQLite: https://www.sqlite.org/docs.html + +.. _fastani_merge: https://github.com/widdowquinn/pyani/pull/299/commits/254346cae24058b745bd9496b4205400da03fb4c + +.. _Alembic: https://alembic.sqlalchemy.org/en/latest/ diff --git a/docs/subcmd_versiondb.rst b/docs/subcmd_versiondb.rst new file mode 100644 index 00000000..19a74264 --- /dev/null +++ b/docs/subcmd_versiondb.rst @@ -0,0 +1,43 @@ +. _pyani-subcmd-versiondb: + +=============== +pyani versiondb +=============== + +The ``versiondb`` subcommand will migrate a ``pyani`` database between versions. `fastani_merge`_ + +usage: pyani versiondb [-h] [-l LOGFILE] [-v] [--debug] [--disable_tqdm] [--version] [--citation] [--dbpath DBPATH] + (--upgrade [VERSION] | --downgrade [VERSION] | --dry-run DIRECTION START:END) [--alembic_exe ALEMBIC_EXE] [-n NAME] [-c FILE] + +One of --upgrade, --downgrade, or --dry-run must be specified. + +optional arguments: + -h, --help show this help message and exit + -l LOGFILE, --logfile LOGFILE + logfile location (default: None) + -v, --verbose report verbose progress to log (default: False) + --debug report debug messages to log (default: False) + --disable_tqdm Turn off tqdm progress bar (default: False) + --version + --citation + --dbpath DBPATH path to pyani database (default: .pyani/pyanidb) + --upgrade [VERSION] update an existing database to a newer schema; if no argument is given, 'head' will be used (default: None) + --downgrade [VERSION] + revert an existing database to a older schema; if no argument is given, 'base' will be used (default: None) + --dry-run DIRECTION START:END + produce the SQL that would be run in migrations, without altering the database; a direction {upgrade or downgrade} and start and end versions e.g., {head:base, + base:head, base:} must be specified (default: None) + --alembic_exe ALEMBIC_EXE + path to alembic executable (default: alembic) + -n NAME, --name NAME used to specify an individual database in a multidb setup (default: None) + -c FILE, --config FILE + used to specify a config file for alembic (default: None) + + +---------- +References +---------- + +.. _SQLite: https://www.sqlite.org/docs.html + +.. _fastani_merge: https://github.com/widdowquinn/pyani/pull/299/commits/254346cae24058b745bd9496b4205400da03fb4c diff --git a/docs/subcommands.rst b/docs/subcommands.rst index dd12fb5e..42a31590 100644 --- a/docs/subcommands.rst +++ b/docs/subcommands.rst @@ -27,3 +27,4 @@ This document links out to detailed instructions for each of the ``pyani`` subco subcmd_classify subcmd_listdeps subcmd_fastani + subcmd_versiondb From 4ea0b672d1053df75c93174474418b950798d2b3 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Mar 2022 03:20:03 -0900 Subject: [PATCH 28/91] Add docs for `versiondbAdd a directory for `versionb` output files --- tests/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 37522c67..e763efdf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -149,10 +149,16 @@ def dir_graphics_in(): @pytest.fixture def dir_versiondb_in(): - """Input files for versiondb ests.""" + """Input files for versiondb tests.""" return FIXTUREPATH / "versiondb" +@pytest.fixture +def dir_versiondb_out(): + """Output files for versiondb tests.""" + return TESTSPATH / "test_output/versiondb" + + @pytest.fixture def dir_seq(): """Sequence files for tests.""" From 51fb69642e90eb2e7bebb7e3bcd55929fd222d9b Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Mar 2022 03:31:34 -0900 Subject: [PATCH 29/91] Rework tests to look for meaningful differences between output and targets --- tests/test_versiondb.py | 324 +++++++++++++++++++++++++++++++--------- 1 file changed, 253 insertions(+), 71 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 994b4616..5bda6370 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -6,6 +6,8 @@ """ import os +import sys +import subprocess from argparse import Namespace from pathlib import Path @@ -15,12 +17,19 @@ import pytest import unittest import shutil -import filecmp from pandas.util.testing import assert_frame_equal from pyani import versiondb, pyani_files, pyani_tools +from pyani.pyani_orm import PyaniORMException, get_session, add_alembic + + +# Create environment variables for alembic to access +def setenv(dir_versiondb_in, dbfile: Path): + abs_path = Path(dir_versiondb_in / dbfile).resolve() + os.environ["PYANI_DATABASE"] = str(abs_path) + # Test get_version() # Test case 0: no executable location is specified @@ -58,56 +67,132 @@ def test_get_version_exe_no_version(executable_without_version, monkeypatch): ) -@pytest.fixture -def versiondb_namespaces(dir_versiondb_in): - { +def versiondb_namespaces(namespace, dir_versiondb_in): + return { "upgrade": Namespace( - dbpath=dir_versiondb_in / "pyanidb_upgrade", + dbpath="pyanidb_upgrade", upgrade="head", downgrade=None, dry_run=None, - direction=None, + direction="upgrade", + dbname=None, + alembic_config=None, + start=dir_versiondb_in / "base_pyanidb", + target=dir_versiondb_in / "head_pyanidb", ), "downgrade": Namespace( - dbpath=dir_versiondb_in / "pyanidb_downgrade", + dbpath="pyanidb_downgrade", upgrade=None, downgrade="base", dry_run=None, - direction=None, + direction="downgrade", + dbname=None, + alembic_config=None, + start=dir_versiondb_in / "head_pyanidb", + target=dir_versiondb_in / "base_pyanidb", ), "dry_down": Namespace( - dbpath=dir_versiondb_in / "pyanidb_dry_down", + dbpath="pyanidb_dry_down", upgrade=None, downgrade=None, dry_run="head:base", direction="downgrade", + dbname=None, + alembic_config=None, ), "dry_up": Namespace( - dbpath=dir_versiondb_in / "pyanidb_dry_up", + dbpath="pyanidb_dry_up", upgrade=None, downgrade=None, dry_run="base:head", direction="upgrade", + dbname=None, + alembic_config=None, ), "altdb": Namespace( - dbpath=dir_versiondb_in / "pyanidb_altdb", + dbpath="pyanidb_altdb", upgrade="head", downgrade=None, dry_run=None, - direction=None, - name="altdb", + direction="upgrade", + dbname="altdb", + alembic_config=None, + start=dir_versiondb_in / "base_pyanidb", + target=dir_versiondb_in / "head_pyanidb", ), "alt_config": Namespace( - dbpath=dir_versiondb_in / "pyanidb_alt_config", + dbpath="pyanidb_alt_config", upgrade="head", downgrade=None, dry_run=None, - direction=None, - config="alt_config", + direction="upgrade", + dbname=None, + alembic_config="alt_config", + start=dir_versiondb_in / "base_pyanidb", + target=dir_versiondb_in / "head_pyanidb", ), + }.get(namespace, None) + + +def expected_diffs(): + return { + "upgrade": b"2a3,7\n> CREATE TABLE alembic_version (\n> \tversion_num VARCHAR(32) NOT NULL, \n> \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n> );\n> INSERT INTO alembic_version VALUES('92f7f6b1626e');\n54,65c59\n< CREATE TABLE runs_comparisons (\n< \tcomparison_id INTEGER, \n< \trun_id INTEGER, \n< \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n< \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n< );\n< CREATE TABLE alembic_version (\n< \tversion_num VARCHAR(32) NOT NULL, \n< \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n< );\n< INSERT INTO alembic_version VALUES('92f7f6b1626e');\n< CREATE TABLE IF NOT EXISTS \"comparisons\" (\n---\n> CREATE TABLE comparisons (\n81d74\n< \tCHECK (maxmatch IN (0, 1)), \n85a79,84\n> CREATE TABLE runs_comparisons (\n> \tcomparison_id INTEGER, \n> \trun_id INTEGER, \n> \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n> \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n> );\n", + "downgrade": b'3,6d2\n< CREATE TABLE alembic_version (\n< \tversion_num VARCHAR(32) NOT NULL, \n< \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n< );\n58,64c54\n< CREATE TABLE runs_comparisons (\n< \tcomparison_id INTEGER, \n< \trun_id INTEGER, \n< \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n< \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n< );\n< CREATE TABLE IF NOT EXISTS "comparisons" (\n---\n> CREATE TABLE comparisons (\n78,79c68\n< \tCHECK (maxmatch IN (0, 1)), \n< \tCONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n---\n> \tUNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n82a72,77\n> CREATE TABLE runs_comparisons (\n> \tcomparison_id INTEGER, \n> \trun_id INTEGER, \n> \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n> \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n> );\n', + "altdb": b"2a3,7\n> CREATE TABLE alembic_version (\n> \tversion_num VARCHAR(32) NOT NULL, \n> \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n> );\n> INSERT INTO alembic_version VALUES('92f7f6b1626e');\n66a72,73\n> \tkmersize INTEGER, \n> \tminmatch FLOAT, \n68c75\n< \tCONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n---\n> \tCONSTRAINT fastani_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch, kmersize, minmatch), \n", + "alt_config": b"2a3,7\n> CREATE TABLE alembic_version (\n> \tversion_num VARCHAR(32) NOT NULL, \n> \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n> );\n> INSERT INTO alembic_version VALUES('92f7f6b1626e');\n66a72,73\n> \tkmersize INTEGER, \n> \tminmatch FLOAT, \n68c75\n< \tCONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n---\n> \tCONSTRAINT fastani_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch, kmersize, minmatch), \n", } +# Create database dump +def dumpdb(abs_dbpath): + cmdline = ["sqlite3", f"{abs_dbpath}", ".dump"] + with open(f"{abs_dbpath}.sql", "w") as outfile: + subprocess.run( + cmdline, + shell=False, + stdout=outfile, + ) + return f"{abs_dbpath}.sql" + + +def name_base_reqs(startdb_dump): + """Name unique constraint in comparisons table of old database schema.""" + old_constraint = ( + "UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch)," + ) + new_constraint = "CONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch)," + + # Edit .dump file so that the unique constraint is named + # This is required in order to subsequently modify it + sed_cmd = [ + "sed", + "-i", + ".bak", + f"s/{old_constraint}/{new_constraint}/", + startdb_dump, + ] + subprocess.run( + sed_cmd, + shell=False, + capture_output=True, + ) + + +def cleanup(abs_dbpath, dir_versiondb_out, args): + """Remove files created for test.""" + + shutil.move(abs_dbpath, dir_versiondb_out) + shutil.move(f"{abs_dbpath}.sql", dir_versiondb_out) + shutil.move(f"{args.target}.sql", dir_versiondb_out) + shutil.move(f"{args.start}.sql", dir_versiondb_out) + + # This file is not generated in the downgrade test + try: + shutil.move(f"{args.start}.sql.bak", dir_versiondb_out) + except FileNotFoundError: + pass + + # Test alembic command generation def test_alembic_cmdline_generation(): """Generate single alembic command line.""" @@ -118,80 +203,177 @@ def test_alembic_cmdline_generation(): # Test upgrade -def test_versiondb_upgrade(dir_versiondb_in): - """ """ - args = versiondb_namespaces["upgrade"] - timestamp = "testing" - shutil.copy(dir_versiondb_in / "base_pyanidb", dir_versiondb_in / "pyanidb_upgrade") - versiondb.migrate_database(args.direction, args, timestamp) - - assert filecmp.cmp( - dir_versiondb_in / "pyanidb_upgrade", dir_versiondb_in / "head_pyanidb" +def test_versiondb_upgrade(dir_versiondb_in, dir_versiondb_out): + """Test upgrade of database.""" + # Test setup + # Retrieve test namespace and + # Set environment variables and resolve absolute path of database + args = versiondb_namespaces("upgrade", dir_versiondb_in) + setenv(dir_versiondb_in, args.dbpath) + abs_dbpath = os.environ.get("PYANI_DATABASE") + + # Create and edit dump file to fix constraint problem + startdb_dump = dumpdb(args.start) + name_base_reqs(startdb_dump) + + # Run `sqlite3 -init + init_cmd = ["sqlite3", abs_dbpath] + subprocess.run( + init_cmd, + stdin=open(startdb_dump), + shell=False, + capture_output=True, ) + # Run test migration + versiondb.migrate_database(args.direction, args, timestamp="testing") -# Test downgrade -def test_versiondb_downgrade(dir_versiondb_in): - """ """ - args = versiondb_namespaces["downgrade"] - timestamp = "testing" - shutil.copy( - dir_versiondb_in / "head_pyanidb", dir_versiondb_in / "pyanidb_downgrade" - ) - versiondb.migrate_database(args.direction, args, timestamp) + # Dump altered and target databases + enddb_dump = dumpdb(abs_dbpath) + targetdb_dump = dumpdb(args.target) - assert filecmp.cmp( - dir_versiondb_in / "pyanidb_downgrade", dir_versiondb_in / "base_pyanidb" + # Run diff + diff_cmd = ["diff", "--suppress-common-lines", enddb_dump, targetdb_dump] + result = subprocess.run( + diff_cmd, + shell=False, + capture_output=True, ) + # Move files + cleanup(abs_dbpath, dir_versiondb_out, args) + + assert result.stdout == expected_diffs()["upgrade"] + + +def test_versiondb_downgrade(dir_versiondb_in, dir_versiondb_out): + """Test downgrade of database.""" + # Test setup + # Retrieve test namespace and + # Set environment variables and resolve absolute path of database + args = versiondb_namespaces("downgrade", dir_versiondb_in) + setenv(dir_versiondb_in, args.dbpath) + abs_dbpath = os.environ.get("PYANI_DATABASE") -# Test dry-run upgrade result -def test_versiondb_dry_upgrade(dir_versiondb_in): - args = versiondb_namespaces["dry_up"] - timestamp = "testing" - shutil.copy(dir_versiondb_in / "base_pyanidb", dir_versiondb_in / "pyanidb_dry_up") - versiondb.migrate_database(args.direction, args, timestamp) + # Create dump file + startdb_dump = dumpdb(args.start) - assert filecmp.cmp( - dir_versiondb_in / "pyanidb_dry_up", dir_versiondb_in / "head_pyanidb" + # Run `sqlite3 -init + init_cmd = ["sqlite3", abs_dbpath] + subprocess.run( + init_cmd, + stdin=open(startdb_dump), + shell=False, + capture_output=True, ) + # Run test migration + versiondb.migrate_database(args.direction, args, timestamp="testing") -# Test dry-run upgrade result -def test_versiondb_dry_downgrade(dir_versiondb_in): - args = versiondb_namespaces["dry_down"] - timestamp = "testing" - shutil.copy( - dir_versiondb_in / "head_pyanidb", dir_versiondb_in / "pyanidb_dry_down" + # Dump altered and target databases + enddb_dump = dumpdb(abs_dbpath) + targetdb_dump = dumpdb(args.target) + + sys.stdout.write(f"{enddb_dump}\n") + sys.stdout.write(f"{targetdb_dump}\n") + + # Run diff + diff_cmd = ["diff", "--suppress-common-lines", enddb_dump, targetdb_dump] + result = subprocess.run( + diff_cmd, + shell=False, + capture_output=True, ) - versiondb.migrate_database(args.direction, args, timestamp) - assert filecmp.cmp( - dir_versiondb_in / "pyanidb_dry_down", dir_versiondb_in / "base_pyanidb" + # Move output files + cleanup(abs_dbpath, dir_versiondb_out, args) + + assert result.stdout == expected_diffs()["downgrade"] + + +# Test alternate dbname +def test_versiondb_altdb(dir_versiondb_in, dir_versiondb_out): + """Test upgrade of database using an alternate database name, such as in a multidb situation.""" + # Test setup + # Retrieve test namespace and + # Set environment variables and resolve absolute path of database + args = versiondb_namespaces("altdb", dir_versiondb_in) + setenv(dir_versiondb_in, args.dbpath) + abs_dbpath = os.environ.get("PYANI_DATABASE") + + # Create dump file + startdb_dump = dumpdb(args.start) + name_base_reqs(startdb_dump) + + # Run `sqlite3 -init + init_cmd = ["sqlite3", abs_dbpath] + subprocess.run( + init_cmd, + stdin=open(startdb_dump), + shell=False, + capture_output=True, ) + # Run test migration + versiondb.migrate_database(args.direction, args, timestamp="testing") -# Test dry-run upgrade result -def test_versiondb_altname(dir_versiondb_in): - args = versiondb_namespaces["altname"] - timestamp = "testing" - shutil.copy(dir_versiondb_in / "head_pyanidb", dir_versiondb_in / "pyanidb_altdb") - versiondb.migrate_database(args.direction, args, timestamp) + # Dump altered and target databases + enddb_dump = dumpdb(abs_dbpath) + targetdb_dump = dumpdb(args.target) - assert filecmp.cmp( - dir_versiondb_in / "pyanidb_altdb", dir_versiondb_in / "head_pyanidb" + # Run diff + diff_cmd = ["diff", "--suppress-common-lines", enddb_dump, targetdb_dump] + result = subprocess.run( + diff_cmd, + shell=False, + capture_output=True, ) - -# Test dry-run upgrade result -def test_versiondb_alt_config(dir_versiondb_in): - args = versiondb_namespaces["alt_config"] - timestamp = "testing" - shutil.copy( - dir_versiondb_in / "head_pyanidb", dir_versiondb_in / "pyanidb_alt_config" + # Move files + cleanup(abs_dbpath, dir_versiondb_out, args) + + assert result.stdout == expected_diffs()["altdb"] + + +# Test alt_config result +def test_versiondb_alt_config(dir_versiondb_in, dir_versiondb_out): + """Test upgrade of database using an alternate config file.""" + # Test setup + # Retrieve test namespace and + # Set environment variables and resolve absolute path of database + args = versiondb_namespaces("alt_config", dir_versiondb_in) + setenv(dir_versiondb_in, args.dbpath) + abs_dbpath = os.environ.get("PYANI_DATABASE") + + # Create dump file + startdb_dump = dumpdb(args.start) + name_base_reqs(startdb_dump) + + # Run `sqlite3 -init + init_cmd = ["sqlite3", abs_dbpath] + subprocess.run( + init_cmd, + stdin=open(startdb_dump), + shell=False, + capture_output=True, ) - versiondb.migrate_database(args.direction, args, timestamp) - assert filecmp.cmp( - dir_versiondb_in / "pyanidb_alt_config", dir_versiondb_in / "head_pyanidb" + # Run test migration + versiondb.migrate_database(args.direction, args, timestamp="testing") + + # Dump altered and target databases + enddb_dump = dumpdb(abs_dbpath) + targetdb_dump = dumpdb(args.target) + + # Run diff + diff_cmd = ["diff", "--suppress-common-lines", enddb_dump, targetdb_dump] + result = subprocess.run( + diff_cmd, + shell=False, + capture_output=True, ) + + # Move files + cleanup(abs_dbpath, dir_versiondb_out, args) + + assert result.stdout == expected_diffs()["alt_config"] From 871a14951d155d86a92aa0571e48b03f5774dc70 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Mar 2022 03:32:31 -0900 Subject: [PATCH 30/91] Update starting point for database downgrade tests --- tests/fixtures/versiondb/head_pyanidb | Bin 49152 -> 49152 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/fixtures/versiondb/head_pyanidb b/tests/fixtures/versiondb/head_pyanidb index 581f52226a3463ce74a13d63e0d4046fcd7dca29..a212532c9c7e8b707c5569fea8462e8f3cc50ced 100644 GIT binary patch delta 87 zcmV-d0I2_ffCGSl1CVJM1^oaI4G*~v2@c*2-V6&0>j|9(>j|9)1PNva6|)f#lLeFg1@H-9 V2?rM=BLRs8v-Sp!0kfM3Ww#zC5>NmD From d9da009b3d09bb1aed95b5e05ffc3528b6a6520b Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Mar 2022 03:35:38 -0900 Subject: [PATCH 31/91] Fix errors in `versiondb.py` arg names --- pyani/versiondb.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyani/versiondb.py b/pyani/versiondb.py index befa0348..a48255f4 100644 --- a/pyani/versiondb.py +++ b/pyani/versiondb.py @@ -73,7 +73,7 @@ def get_optional_args(args: Namespace): if args.dbname: opts.extend(["-n", args.dbname]) if args.alembic_config: - opts.extend(["-c", args.config]) + opts.extend(["-c", args.alembic_config]) return opts @@ -123,9 +123,6 @@ def migrate_database(direction, args: Namespace, timestamp=None): result = subprocess.run( cmdline, shell=False, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE, - # check=True, capture_output=True, ) From c1ff74f6934c3a9200a5f708670924c667dc5e54 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Mar 2022 08:30:24 -0900 Subject: [PATCH 32/91] Add `sqlite3` as a third-party dependency --- Makefile | 3 ++- requirements-thirdparty.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 629ff1f0..f58eff18 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ setup_env: @conda install --file requirements-dev.txt --yes @conda install --file requirements.txt --yes + @conda config --add channels blaze @conda install --file requirements-thirdparty.txt --yes @pip install -r requirements-pip.txt @pre-commit install @@ -54,7 +55,7 @@ walkthrough: clean_walkthrough # pyani report --runs C_blochmannia_ANIb/ --formats html,excel,stdout # pyani report --run_results 2 --formats html,excel,stdout C_blochmannia_ANIb/ # pyani report --run_matrices 2 --formats html,excel,stdout C_blochmannia_ANIb/ - # pyani plot --formats png,pdf --method seaborn C_blochmannia_ANIb 2 + # pyani plot --formats png,pdf --method seaborn C_blochmannia_ANIb 2 uml: pyreverse -o pdf -p pyani pyani diff --git a/requirements-thirdparty.txt b/requirements-thirdparty.txt index 71f899f0..d294fd9d 100644 --- a/requirements-thirdparty.txt +++ b/requirements-thirdparty.txt @@ -2,3 +2,4 @@ blast blast-legacy mummer fastani +sqlite3 From 5b74fc7cd74f788a79b2ed837cf8ea41da5b01ac Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Mar 2022 10:44:16 -0900 Subject: [PATCH 33/91] Fix failing `cli` test; (failure caused by code added for alembic) --- pyani/scripts/subcommands/subcmd_createdb.py | 7 +++---- tests/test_cli_parsing.py | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pyani/scripts/subcommands/subcmd_createdb.py b/pyani/scripts/subcommands/subcmd_createdb.py index 95ef39d3..bf69dc4f 100644 --- a/pyani/scripts/subcommands/subcmd_createdb.py +++ b/pyani/scripts/subcommands/subcmd_createdb.py @@ -45,7 +45,7 @@ from pyani import pyani_orm -from pyani.pyani_orm import PyaniORMException, get_session, add_alembic +from pyani.pyani_orm import PyaniORMException, get_session def subcmd_createdb(args: Namespace) -> int: @@ -87,9 +87,8 @@ def subcmd_createdb(args: Namespace) -> int: # Add information about the database version to the database logger.debug("Adding database version to database %s...", args.dbpath) try: - version_num = add_alembic( - session, version_num="92f7f6b1626e" # most current version (fastani) - ) + version_num = "92f7f6b1626e" # most current version (fastani) + pyani_orm.add_alembic(session, version_num=version_num) except PyaniORMException: logger.error( "Could not add db_version %s to the database (exiting)", diff --git a/tests/test_cli_parsing.py b/tests/test_cli_parsing.py index ecdeaa40..a064dd27 100644 --- a/tests/test_cli_parsing.py +++ b/tests/test_cli_parsing.py @@ -81,7 +81,11 @@ def test_createdb(args_createdb, monkeypatch): def mock_return_none(*args, **kwargs): return None + def mock_add_alembic(*args, **kwargs): + return "TEST VERSION" + monkeypatch.setattr(pyani_orm, "create_db", mock_return_none) + monkeypatch.setattr(pyani_orm, "add_alembic", mock_add_alembic) pyani_script.run_main(args_createdb) From 6e18256bc3cbb81a5b25028de1dad9e37a6d0581 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Mar 2022 11:13:43 -0900 Subject: [PATCH 34/91] Add `sqlite` installation code to `config.yml` --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 56cda4ac..6e51bdcf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,6 +68,14 @@ jobs: echo 'export PATH=$PWD:$PATH' >> $BASH_ENV source $BASH_ENV fastANI -h + - run: + name: install sqlite3 + command: | + wget https://www.sqlite.org/2022/sqlite-tools-linux-x86-3380100.zip + unzip sqlite-tools-linux-x86-3380100.zip + echo 'export PATH=$PWD:$PATH' >> $BASH_ENV + source $BASH_ENV + sqlite3 -help - run: From d2f9f28fab6d11ebaab9c142da4613cfe208afce Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 14 Mar 2022 16:17:20 -0900 Subject: [PATCH 35/91] Test alembic command-line generation and add docstrings --- tests/test_versiondb.py | 48 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 5bda6370..c63fd672 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -68,6 +68,8 @@ def test_get_version_exe_no_version(executable_without_version, monkeypatch): def versiondb_namespaces(namespace, dir_versiondb_in): + """Namespaces for different alembic scenarios.""" + return { "upgrade": Namespace( dbpath="pyanidb_upgrade", @@ -135,6 +137,7 @@ def versiondb_namespaces(namespace, dir_versiondb_in): def expected_diffs(): + """Expected (acceptable) differences between output and target databases.""" return { "upgrade": b"2a3,7\n> CREATE TABLE alembic_version (\n> \tversion_num VARCHAR(32) NOT NULL, \n> \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n> );\n> INSERT INTO alembic_version VALUES('92f7f6b1626e');\n54,65c59\n< CREATE TABLE runs_comparisons (\n< \tcomparison_id INTEGER, \n< \trun_id INTEGER, \n< \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n< \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n< );\n< CREATE TABLE alembic_version (\n< \tversion_num VARCHAR(32) NOT NULL, \n< \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n< );\n< INSERT INTO alembic_version VALUES('92f7f6b1626e');\n< CREATE TABLE IF NOT EXISTS \"comparisons\" (\n---\n> CREATE TABLE comparisons (\n81d74\n< \tCHECK (maxmatch IN (0, 1)), \n85a79,84\n> CREATE TABLE runs_comparisons (\n> \tcomparison_id INTEGER, \n> \trun_id INTEGER, \n> \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n> \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n> );\n", "downgrade": b'3,6d2\n< CREATE TABLE alembic_version (\n< \tversion_num VARCHAR(32) NOT NULL, \n< \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n< );\n58,64c54\n< CREATE TABLE runs_comparisons (\n< \tcomparison_id INTEGER, \n< \trun_id INTEGER, \n< \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n< \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n< );\n< CREATE TABLE IF NOT EXISTS "comparisons" (\n---\n> CREATE TABLE comparisons (\n78,79c68\n< \tCHECK (maxmatch IN (0, 1)), \n< \tCONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n---\n> \tUNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n82a72,77\n> CREATE TABLE runs_comparisons (\n> \tcomparison_id INTEGER, \n> \trun_id INTEGER, \n> \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n> \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n> );\n', @@ -145,6 +148,8 @@ def expected_diffs(): # Create database dump def dumpdb(abs_dbpath): + """Dump contents of database to a plain-text file.""" + cmdline = ["sqlite3", f"{abs_dbpath}", ".dump"] with open(f"{abs_dbpath}.sql", "w") as outfile: subprocess.run( @@ -194,8 +199,47 @@ def cleanup(abs_dbpath, dir_versiondb_out, args): # Test alembic command generation -def test_alembic_cmdline_generation(): - """Generate single alembic command line.""" +def test_alembic_cmdline_generation(dir_versiondb_in): + """Generate alembic command lines.""" + + alembic_cmds = [] + upgrade_args = versiondb_namespaces("upgrade", dir_versiondb_in) + alembic_cmds.append( + " ".join( + versiondb.construct_alembic_cmdline(upgrade_args.direction, upgrade_args) + ) + ) + + downgrade_args = versiondb_namespaces("downgrade", dir_versiondb_in) + alembic_cmds.append( + " ".join( + versiondb.construct_alembic_cmdline( + downgrade_args.direction, downgrade_args + ) + ) + ) + + altdb_args = versiondb_namespaces("altdb", dir_versiondb_in) + alembic_cmds.append( + " ".join(versiondb.construct_alembic_cmdline(altdb_args.direction, altdb_args)) + ) + + alt_config_args = versiondb_namespaces("alt_config", dir_versiondb_in) + alembic_cmds.append( + " ".join( + versiondb.construct_alembic_cmdline( + alt_config_args.direction, alt_config_args + ) + ) + ) + + assert alembic_cmds == [ + "alembic upgrade head", + "alembic downgrade base", + f"alembic upgrade head -n {altdb_args.dbname}", + f"alembic upgrade head -c {alt_config_args.alembic_config}", + ] + pass # alembic_cmd = versiondb.construct_alembic_cmdline() # dir_alembic = tmp_path / "versiondb_output" From 938440ba506184aa2272424dc2da2fe15287f13d Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 16 Mar 2022 12:41:05 +0000 Subject: [PATCH 36/91] Change how `sqlite3` is installed --- .circleci/config.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e51bdcf..84336684 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: - run: name: install third-party tools command: | - sudo apt-get install csh mummer ncbi-blast+ + sudo apt-get install csh mummer ncbi-blast+ sqlite3 - run: name: install legacy BLAST command: | @@ -68,14 +68,6 @@ jobs: echo 'export PATH=$PWD:$PATH' >> $BASH_ENV source $BASH_ENV fastANI -h - - run: - name: install sqlite3 - command: | - wget https://www.sqlite.org/2022/sqlite-tools-linux-x86-3380100.zip - unzip sqlite-tools-linux-x86-3380100.zip - echo 'export PATH=$PWD:$PATH' >> $BASH_ENV - source $BASH_ENV - sqlite3 -help - run: From 0efe216149b5da109933f16f76382a73049513e6 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 16 Mar 2022 12:52:41 +0000 Subject: [PATCH 37/91] Change how namespaces are retrieved to use fixtures --- tests/test_versiondb.py | 177 +++++++++++++++++++++------------------- 1 file changed, 95 insertions(+), 82 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index c63fd672..940298df 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -23,6 +23,7 @@ from pyani import versiondb, pyani_files, pyani_tools from pyani.pyani_orm import PyaniORMException, get_session, add_alembic +from tools import modify_namespace # Create environment variables for alembic to access @@ -67,76 +68,78 @@ def test_get_version_exe_no_version(executable_without_version, monkeypatch): ) -def versiondb_namespaces(namespace, dir_versiondb_in): - """Namespaces for different alembic scenarios.""" +@pytest.fixture +def generic_versiondb_namespace(dir_versiondb_in): + """Generic namespace for the pyani versiondb subcommand.""" + return Namespace( + dbpath="pyanidb_upgrade", + upgrade="head", + downgrade=None, + dry_run=None, + direction="upgrade", + dbname=None, + alembic_config=None, + start=dir_versiondb_in / "base_pyanidb", + target=dir_versiondb_in / "head_pyanidb", + ) - return { - "upgrade": Namespace( - dbpath="pyanidb_upgrade", - upgrade="head", - downgrade=None, - dry_run=None, - direction="upgrade", - dbname=None, - alembic_config=None, - start=dir_versiondb_in / "base_pyanidb", - target=dir_versiondb_in / "head_pyanidb", - ), - "downgrade": Namespace( - dbpath="pyanidb_downgrade", - upgrade=None, - downgrade="base", - dry_run=None, - direction="downgrade", - dbname=None, - alembic_config=None, - start=dir_versiondb_in / "head_pyanidb", - target=dir_versiondb_in / "base_pyanidb", - ), - "dry_down": Namespace( - dbpath="pyanidb_dry_down", - upgrade=None, - downgrade=None, - dry_run="head:base", - direction="downgrade", - dbname=None, - alembic_config=None, - ), - "dry_up": Namespace( - dbpath="pyanidb_dry_up", - upgrade=None, - downgrade=None, - dry_run="base:head", - direction="upgrade", - dbname=None, - alembic_config=None, - ), - "altdb": Namespace( - dbpath="pyanidb_altdb", - upgrade="head", - downgrade=None, - dry_run=None, - direction="upgrade", - dbname="altdb", - alembic_config=None, - start=dir_versiondb_in / "base_pyanidb", - target=dir_versiondb_in / "head_pyanidb", - ), - "alt_config": Namespace( - dbpath="pyanidb_alt_config", - upgrade="head", - downgrade=None, - dry_run=None, - direction="upgrade", - dbname=None, - alembic_config="alt_config", - start=dir_versiondb_in / "base_pyanidb", - target=dir_versiondb_in / "head_pyanidb", - ), - }.get(namespace, None) - - -def expected_diffs(): + +@pytest.fixture +def downgrade_namespace(generic_versiondb_namespace, dir_versiondb_in): + """Namespace for pyani versiondb downgrade.""" + return modify_namespace( + generic_versiondb_namespace, + dbpath="pyanidb_downgrade", + upgrade=None, + downgrade="base", + direction="downgrade", + start=dir_versiondb_in / "head_pyanidb", + target=dir_versiondb_in / "base_pyanidb", + ) + + +@pytest.fixture +def altdb_namespace(generic_versiondb_namespace): + """Namespace for pyani versiondb -n altdb.""" + return modify_namespace( + generic_versiondb_namespace, dbpath="pyanidb_altdb", dbname="altdb" + ) + + +@pytest.fixture +def alt_config_namespace(generic_versiondb_namespace): + """Namespace for pyani versiondb -c alt_config.""" + return modify_namespace( + generic_versiondb_namespace, + dbpath="pyanidb_alt_config", + alembic_config="alt_config", + ) + + +@pytest.fixture +def dry_up_namespace(generic_versiondb_namespace): + """Namespace for pyani versiondb dry-run upgrade.""" + return modify_namespace( + generic_versiondb_namespace, + dbpath="pyanidb_dry_up", + upgrade=None, + dry_run="base:head", + ) + + +@pytest.fixture +def dry_down_namespace(generic_versiondb_namespace): + """Namespace for pyani versiondb dry-run downgrade.""" + return modify_namespace( + generic_versiondb_namespace, + dbpath="pyanidb_dry_down", + direction="downgrade", + downgrade=None, + dry_run="head:base", + ) + + +def expected_diffs(namespace): """Expected (acceptable) differences between output and target databases.""" return { "upgrade": b"2a3,7\n> CREATE TABLE alembic_version (\n> \tversion_num VARCHAR(32) NOT NULL, \n> \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n> );\n> INSERT INTO alembic_version VALUES('92f7f6b1626e');\n54,65c59\n< CREATE TABLE runs_comparisons (\n< \tcomparison_id INTEGER, \n< \trun_id INTEGER, \n< \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n< \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n< );\n< CREATE TABLE alembic_version (\n< \tversion_num VARCHAR(32) NOT NULL, \n< \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n< );\n< INSERT INTO alembic_version VALUES('92f7f6b1626e');\n< CREATE TABLE IF NOT EXISTS \"comparisons\" (\n---\n> CREATE TABLE comparisons (\n81d74\n< \tCHECK (maxmatch IN (0, 1)), \n85a79,84\n> CREATE TABLE runs_comparisons (\n> \tcomparison_id INTEGER, \n> \trun_id INTEGER, \n> \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n> \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n> );\n", @@ -199,18 +202,24 @@ def cleanup(abs_dbpath, dir_versiondb_out, args): # Test alembic command generation -def test_alembic_cmdline_generation(dir_versiondb_in): +def test_alembic_cmdline_generation( + generic_versiondb_namespace, + downgrade_namespace, + altdb_namespace, + alt_config_namespace, + dir_versiondb_in, +): """Generate alembic command lines.""" alembic_cmds = [] - upgrade_args = versiondb_namespaces("upgrade", dir_versiondb_in) + upgrade_args = generic_versiondb_namespace alembic_cmds.append( " ".join( versiondb.construct_alembic_cmdline(upgrade_args.direction, upgrade_args) ) ) - downgrade_args = versiondb_namespaces("downgrade", dir_versiondb_in) + downgrade_args = downgrade_namespace alembic_cmds.append( " ".join( versiondb.construct_alembic_cmdline( @@ -219,12 +228,12 @@ def test_alembic_cmdline_generation(dir_versiondb_in): ) ) - altdb_args = versiondb_namespaces("altdb", dir_versiondb_in) + altdb_args = altdb_namespace alembic_cmds.append( " ".join(versiondb.construct_alembic_cmdline(altdb_args.direction, altdb_args)) ) - alt_config_args = versiondb_namespaces("alt_config", dir_versiondb_in) + alt_config_args = alt_config_namespace alembic_cmds.append( " ".join( versiondb.construct_alembic_cmdline( @@ -247,12 +256,14 @@ def test_alembic_cmdline_generation(dir_versiondb_in): # Test upgrade -def test_versiondb_upgrade(dir_versiondb_in, dir_versiondb_out): +def test_versiondb_upgrade( + generic_versiondb_namespace, dir_versiondb_in, dir_versiondb_out +): """Test upgrade of database.""" # Test setup # Retrieve test namespace and # Set environment variables and resolve absolute path of database - args = versiondb_namespaces("upgrade", dir_versiondb_in) + args = generic_versiondb_namespace setenv(dir_versiondb_in, args.dbpath) abs_dbpath = os.environ.get("PYANI_DATABASE") @@ -290,12 +301,12 @@ def test_versiondb_upgrade(dir_versiondb_in, dir_versiondb_out): assert result.stdout == expected_diffs()["upgrade"] -def test_versiondb_downgrade(dir_versiondb_in, dir_versiondb_out): +def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiondb_out): """Test downgrade of database.""" # Test setup # Retrieve test namespace and # Set environment variables and resolve absolute path of database - args = versiondb_namespaces("downgrade", dir_versiondb_in) + args = downgrade_namespace setenv(dir_versiondb_in, args.dbpath) abs_dbpath = os.environ.get("PYANI_DATABASE") @@ -336,12 +347,12 @@ def test_versiondb_downgrade(dir_versiondb_in, dir_versiondb_out): # Test alternate dbname -def test_versiondb_altdb(dir_versiondb_in, dir_versiondb_out): +def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): """Test upgrade of database using an alternate database name, such as in a multidb situation.""" # Test setup # Retrieve test namespace and # Set environment variables and resolve absolute path of database - args = versiondb_namespaces("altdb", dir_versiondb_in) + args = altdb_namespace setenv(dir_versiondb_in, args.dbpath) abs_dbpath = os.environ.get("PYANI_DATABASE") @@ -380,12 +391,14 @@ def test_versiondb_altdb(dir_versiondb_in, dir_versiondb_out): # Test alt_config result -def test_versiondb_alt_config(dir_versiondb_in, dir_versiondb_out): +def test_versiondb_alt_config( + alt_config_namespace, dir_versiondb_in, dir_versiondb_out +): """Test upgrade of database using an alternate config file.""" # Test setup # Retrieve test namespace and # Set environment variables and resolve absolute path of database - args = versiondb_namespaces("alt_config", dir_versiondb_in) + args = alt_config_namespace setenv(dir_versiondb_in, args.dbpath) abs_dbpath = os.environ.get("PYANI_DATABASE") From 8da38c7e881ab096e21db5b4f7030815daa04629 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 16 Mar 2022 12:58:10 +0000 Subject: [PATCH 38/91] Change how `expected_diffs()` returns values This might solve problems between comparisons seen on CircleCI --- tests/test_versiondb.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 940298df..114cec51 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -146,7 +146,7 @@ def expected_diffs(namespace): "downgrade": b'3,6d2\n< CREATE TABLE alembic_version (\n< \tversion_num VARCHAR(32) NOT NULL, \n< \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n< );\n58,64c54\n< CREATE TABLE runs_comparisons (\n< \tcomparison_id INTEGER, \n< \trun_id INTEGER, \n< \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n< \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n< );\n< CREATE TABLE IF NOT EXISTS "comparisons" (\n---\n> CREATE TABLE comparisons (\n78,79c68\n< \tCHECK (maxmatch IN (0, 1)), \n< \tCONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n---\n> \tUNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n82a72,77\n> CREATE TABLE runs_comparisons (\n> \tcomparison_id INTEGER, \n> \trun_id INTEGER, \n> \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n> \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n> );\n', "altdb": b"2a3,7\n> CREATE TABLE alembic_version (\n> \tversion_num VARCHAR(32) NOT NULL, \n> \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n> );\n> INSERT INTO alembic_version VALUES('92f7f6b1626e');\n66a72,73\n> \tkmersize INTEGER, \n> \tminmatch FLOAT, \n68c75\n< \tCONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n---\n> \tCONSTRAINT fastani_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch, kmersize, minmatch), \n", "alt_config": b"2a3,7\n> CREATE TABLE alembic_version (\n> \tversion_num VARCHAR(32) NOT NULL, \n> \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n> );\n> INSERT INTO alembic_version VALUES('92f7f6b1626e');\n66a72,73\n> \tkmersize INTEGER, \n> \tminmatch FLOAT, \n68c75\n< \tCONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n---\n> \tCONSTRAINT fastani_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch, kmersize, minmatch), \n", - } + }.get(namespace, None) # Create database dump @@ -298,7 +298,7 @@ def test_versiondb_upgrade( # Move files cleanup(abs_dbpath, dir_versiondb_out, args) - assert result.stdout == expected_diffs()["upgrade"] + assert result.stdout == expected_diffs("upgrade") def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiondb_out): @@ -343,7 +343,7 @@ def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiond # Move output files cleanup(abs_dbpath, dir_versiondb_out, args) - assert result.stdout == expected_diffs()["downgrade"] + assert result.stdout == expected_diffs("downgrade") # Test alternate dbname @@ -387,7 +387,7 @@ def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): # Move files cleanup(abs_dbpath, dir_versiondb_out, args) - assert result.stdout == expected_diffs()["altdb"] + assert result.stdout == expected_diffs("altdb") # Test alt_config result @@ -433,4 +433,4 @@ def test_versiondb_alt_config( # Move files cleanup(abs_dbpath, dir_versiondb_out, args) - assert result.stdout == expected_diffs()["alt_config"] + assert result.stdout == expected_diffs("alt_config") From 84f4a35db734e17fa1445a8c9751571a0943d88b Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 16 Mar 2022 13:01:06 +0000 Subject: [PATCH 39/91] Change handling of `stdout` and `stderr` from `subprocess.run` calls The keyword `capture_output` is not compatible with Python 3.6 --- tests/test_versiondb.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 114cec51..5d5a2b59 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -159,6 +159,7 @@ def dumpdb(abs_dbpath): cmdline, shell=False, stdout=outfile, + stderr=subprocess.PIPE, ) return f"{abs_dbpath}.sql" @@ -182,7 +183,8 @@ def name_base_reqs(startdb_dump): subprocess.run( sed_cmd, shell=False, - capture_output=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, ) @@ -277,7 +279,8 @@ def test_versiondb_upgrade( init_cmd, stdin=open(startdb_dump), shell=False, - capture_output=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, ) # Run test migration @@ -292,7 +295,8 @@ def test_versiondb_upgrade( result = subprocess.run( diff_cmd, shell=False, - capture_output=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, ) # Move files @@ -319,7 +323,8 @@ def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiond init_cmd, stdin=open(startdb_dump), shell=False, - capture_output=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, ) # Run test migration @@ -329,15 +334,13 @@ def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiond enddb_dump = dumpdb(abs_dbpath) targetdb_dump = dumpdb(args.target) - sys.stdout.write(f"{enddb_dump}\n") - sys.stdout.write(f"{targetdb_dump}\n") - # Run diff diff_cmd = ["diff", "--suppress-common-lines", enddb_dump, targetdb_dump] result = subprocess.run( diff_cmd, shell=False, - capture_output=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, ) # Move output files @@ -366,7 +369,8 @@ def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): init_cmd, stdin=open(startdb_dump), shell=False, - capture_output=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, ) # Run test migration @@ -381,7 +385,8 @@ def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): result = subprocess.run( diff_cmd, shell=False, - capture_output=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, ) # Move files @@ -412,7 +417,8 @@ def test_versiondb_alt_config( init_cmd, stdin=open(startdb_dump), shell=False, - capture_output=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, ) # Run test migration @@ -427,7 +433,8 @@ def test_versiondb_alt_config( result = subprocess.run( diff_cmd, shell=False, - capture_output=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, ) # Move files From d4d829311368cdd8184d43153e4824859767f446 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 16 Mar 2022 13:02:13 +0000 Subject: [PATCH 40/91] Change cleanup procedure to prevent artefacts being overwritten --- tests/conftest.py | 2 +- tests/test_versiondb.py | 38 +++++++++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e763efdf..4f9959b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -156,7 +156,7 @@ def dir_versiondb_in(): @pytest.fixture def dir_versiondb_out(): """Output files for versiondb tests.""" - return TESTSPATH / "test_output/versiondb" + return TESTSPATH / "test_output/subcmd_versiondb" @pytest.fixture diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 5d5a2b59..50ebd738 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -188,17 +188,33 @@ def name_base_reqs(startdb_dump): ) -def cleanup(abs_dbpath, dir_versiondb_out, args): +def cleanup(abs_dbpath, test, dir_versiondb_out, args): """Remove files created for test.""" - - shutil.move(abs_dbpath, dir_versiondb_out) - shutil.move(f"{abs_dbpath}.sql", dir_versiondb_out) - shutil.move(f"{args.target}.sql", dir_versiondb_out) - shutil.move(f"{args.start}.sql", dir_versiondb_out) + # db = abs_dbpath.name + # target = args.target.name + # start = args.start.name + + dir_versiondb_out.mkdir(exist_ok=True) + Path(f"{dir_versiondb_out}/{test}").mkdir(exist_ok=True) + + # The files must be moved, but shutil.move() does not work + # if the file already exists, so this is a two-step process + # Copy files to new location + shutil.copy(abs_dbpath, dir_versiondb_out / test) + shutil.copy(f"{abs_dbpath}.sql", dir_versiondb_out / test) + shutil.copy(f"{args.target}.sql", dir_versiondb_out / test) + shutil.copy(f"{args.start}.sql", dir_versiondb_out / test) + + # Remove old files + os.remove(abs_dbpath) + os.remove(f"{abs_dbpath}.sql") + os.remove(f"{args.target}.sql") + os.remove(f"{args.start}.sql") # This file is not generated in the downgrade test try: - shutil.move(f"{args.start}.sql.bak", dir_versiondb_out) + shutil.copy(f"{args.start}.sql.bak", dir_versiondb_out / test) + os.remove(f"{args.start}.sql.bak") except FileNotFoundError: pass @@ -300,7 +316,7 @@ def test_versiondb_upgrade( ) # Move files - cleanup(abs_dbpath, dir_versiondb_out, args) + cleanup(abs_dbpath, "upgrade", dir_versiondb_out, args) assert result.stdout == expected_diffs("upgrade") @@ -344,7 +360,7 @@ def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiond ) # Move output files - cleanup(abs_dbpath, dir_versiondb_out, args) + cleanup(abs_dbpath, "downgrade", dir_versiondb_out, args) assert result.stdout == expected_diffs("downgrade") @@ -390,7 +406,7 @@ def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): ) # Move files - cleanup(abs_dbpath, dir_versiondb_out, args) + cleanup(abs_dbpath, "altdb", dir_versiondb_out, args) assert result.stdout == expected_diffs("altdb") @@ -438,6 +454,6 @@ def test_versiondb_alt_config( ) # Move files - cleanup(abs_dbpath, dir_versiondb_out, args) + cleanup(abs_dbpath, "alt_config", dir_versiondb_out, args) assert result.stdout == expected_diffs("alt_config") From 02bb750ee00a26057fcaebaf6469c37e09ed79bc Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 16 Mar 2022 13:22:03 +0000 Subject: [PATCH 41/91] Replace remaining use of `capture_output` keyword; previously missed --- pyani/versiondb.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyani/versiondb.py b/pyani/versiondb.py index a48255f4..9d234508 100644 --- a/pyani/versiondb.py +++ b/pyani/versiondb.py @@ -121,9 +121,7 @@ def migrate_database(direction, args: Namespace, timestamp=None): cmdline = construct_alembic_cmdline(direction, args) result = subprocess.run( - cmdline, - shell=False, - capture_output=True, + cmdline, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) log_output_and_errors(result, direction, args, timestamp) From 55480eabd5d4c86534039210ec14c3ddd6c37c01 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Wed, 16 Mar 2022 15:06:03 +0000 Subject: [PATCH 42/91] Alter `sed` command to be OS-specific --- tests/test_versiondb.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 50ebd738..8ad8b3fe 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -8,6 +8,7 @@ import os import sys import subprocess +import platform from argparse import Namespace from pathlib import Path @@ -173,13 +174,21 @@ def name_base_reqs(startdb_dump): # Edit .dump file so that the unique constraint is named # This is required in order to subsequently modify it - sed_cmd = [ - "sed", - "-i", - ".bak", - f"s/{old_constraint}/{new_constraint}/", - startdb_dump, - ] + if platform.system() == "Darwin": + sed_cmd = [ + "sed", + "-i", + ".bak", + f"s/{old_constraint}/{new_constraint}/", + startdb_dump, + ] + else: + sed_cmd = [ + "sed", + "-i", + f"s/{old_constraint}/{new_constraint}/", + startdb_dump, + ] subprocess.run( sed_cmd, shell=False, From b0c094717de32bfc600fd14af800b184076b8c8a Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 17 Mar 2022 00:27:42 +0000 Subject: [PATCH 43/91] Add targets for `versiondb` tests --- .../subcmd_versiondb/altdb_minus_head.diff | 26 +++++++++++++++++++ .../downgrade_minus_base.diff | 19 ++++++++++++++ .../subcmd_versiondb/upgrade_minus_head.diff | 24 +++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 tests/test_targets/subcmd_versiondb/altdb_minus_head.diff create mode 100644 tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff create mode 100644 tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff diff --git a/tests/test_targets/subcmd_versiondb/altdb_minus_head.diff b/tests/test_targets/subcmd_versiondb/altdb_minus_head.diff new file mode 100644 index 00000000..7ff5186c --- /dev/null +++ b/tests/test_targets/subcmd_versiondb/altdb_minus_head.diff @@ -0,0 +1,26 @@ + > CREATE TABLE alembic_version ( + > version_num VARCHAR(32) NOT NULL, + > CONSTRAINT alembic_version_pkc PRIMARY KEY (version_n + > ); + > INSERT INTO alembic_version VALUES('92f7f6b1626e'); +CREATE TABLE runs_comparisons ( | CREATE TABLE comparisons ( + comparison_id INTEGER, < + run_id INTEGER, < + FOREIGN KEY(comparison_id) REFERENCES comparisons (co < + FOREIGN KEY(run_id) REFERENCES runs (run_id) < +); < +CREATE TABLE alembic_version ( < + version_num VARCHAR(32) NOT NULL, < + CONSTRAINT alembic_version_pkc PRIMARY KEY (version_n < +); < +INSERT INTO alembic_version VALUES('92f7f6b1626e'); < +CREATE TABLE IF NOT EXISTS "comparisons" ( < + CHECK (maxmatch IN (0, 1)), < + FOREIGN KEY(subject_id) REFERENCES genomes (genome_id | FOREIGN KEY(query_id) REFERENCES genomes (genome_id), + FOREIGN KEY(query_id) REFERENCES genomes (genome_id) | FOREIGN KEY(subject_id) REFERENCES genomes (genome_id + > ); + > CREATE TABLE runs_comparisons ( + > comparison_id INTEGER, + > run_id INTEGER, + > FOREIGN KEY(comparison_id) REFERENCES comparisons (co + > FOREIGN KEY(run_id) REFERENCES runs (run_id) diff --git a/tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff b/tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff new file mode 100644 index 00000000..be41bbe7 --- /dev/null +++ b/tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff @@ -0,0 +1,19 @@ +CREATE TABLE alembic_version ( < + version_num VARCHAR(32) NOT NULL, < + CONSTRAINT alembic_version_pkc PRIMARY KEY (version_n < +); < +CREATE TABLE runs_comparisons ( | CREATE TABLE comparisons ( + comparison_id INTEGER, < + run_id INTEGER, < + FOREIGN KEY(comparison_id) REFERENCES comparisons (co < + FOREIGN KEY(run_id) REFERENCES runs (run_id) < +); < +CREATE TABLE IF NOT EXISTS "comparisons" ( < + CHECK (maxmatch IN (0, 1)), | UNIQUE (query_id, subject_id, program, version, frags + CONSTRAINT base_reqs UNIQUE (query_id, subject_id, pr < + > CREATE TABLE runs_comparisons ( + > comparison_id INTEGER, + > run_id INTEGER, + > FOREIGN KEY(comparison_id) REFERENCES comparisons (co + > FOREIGN KEY(run_id) REFERENCES runs (run_id) + > ); diff --git a/tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff b/tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff new file mode 100644 index 00000000..95437981 --- /dev/null +++ b/tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff @@ -0,0 +1,24 @@ + > CREATE TABLE alembic_version ( + > version_num VARCHAR(32) NOT NULL, + > CONSTRAINT alembic_version_pkc PRIMARY KEY (version_n + > ); + > INSERT INTO alembic_version VALUES('92f7f6b1626e'); +CREATE TABLE runs_comparisons ( | CREATE TABLE comparisons ( + comparison_id INTEGER, < + run_id INTEGER, < + FOREIGN KEY(comparison_id) REFERENCES comparisons (co < + FOREIGN KEY(run_id) REFERENCES runs (run_id) < +); < +CREATE TABLE alembic_version ( < + version_num VARCHAR(32) NOT NULL, < + CONSTRAINT alembic_version_pkc PRIMARY KEY (version_n < +); < +INSERT INTO alembic_version VALUES('92f7f6b1626e'); < +CREATE TABLE IF NOT EXISTS "comparisons" ( < + CHECK (maxmatch IN (0, 1)), < + > CREATE TABLE runs_comparisons ( + > comparison_id INTEGER, + > run_id INTEGER, + > FOREIGN KEY(comparison_id) REFERENCES comparisons (co + > FOREIGN KEY(run_id) REFERENCES runs (run_id) + > ); From fdcf580ee9109b306035549a718390c553386db4 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 17 Mar 2022 00:28:03 +0000 Subject: [PATCH 44/91] Add target location for `versiondb` tests --- tests/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 4f9959b9..f5003fd5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -159,6 +159,12 @@ def dir_versiondb_out(): return TESTSPATH / "test_output/subcmd_versiondb" +@pytest.fixture +def dir_versiondb_targets(): + """Target diff files for versiondb tests.""" + return TESTSPATH / "test_targets/subcmd_versiondb" + + @pytest.fixture def dir_seq(): """Sequence files for tests.""" From d434f23c1cacb0b277e980ce3cff22be9d24eada Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 17 Mar 2022 00:28:40 +0000 Subject: [PATCH 45/91] Add alt_config file for `versiondb` tests --- alt_alembic_config.ini | 103 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 alt_alembic_config.ini diff --git a/alt_alembic_config.ini b/alt_alembic_config.ini new file mode 100644 index 00000000..1792ea18 --- /dev/null +++ b/alt_alembic_config.ini @@ -0,0 +1,103 @@ +# a multi-database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to multidb/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" +# version_locations = %(here)s/bar:%(here)s/bat:multidb/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. Valid values are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # default: use os.pathsep + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +databases = pyanidb_altdb + +[pyanidb_altdb] +sqlalchemy.url = sqlite:///.pyani/pyanidb +script_location = %(here)s/alembic + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S From 94590682c7572566ff351816dd279c3c0b1ab436 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 17 Mar 2022 00:49:01 +0000 Subject: [PATCH 46/91] Update tests for `versiondb` to make `diff` comparison more robust --- alembic.ini | 2 +- pyani/versiondb.py | 4 +-- tests/test_versiondb.py | 77 +++++++++++++++++++++++++---------------- 3 files changed, 50 insertions(+), 33 deletions(-) diff --git a/alembic.ini b/alembic.ini index 15d1863b..a340f706 100644 --- a/alembic.ini +++ b/alembic.ini @@ -2,7 +2,7 @@ [alembic] # path to migration scripts -script_location = alembic +script_location = %(here)s/alembic # template used to generate migration files # file_template = %%(rev)s_%%(slug)s diff --git a/pyani/versiondb.py b/pyani/versiondb.py index 9d234508..3deef4e0 100644 --- a/pyani/versiondb.py +++ b/pyani/versiondb.py @@ -91,9 +91,9 @@ def construct_alembic_cmdline( *get_optional_args(args), ] # FAILED: downgrade with --sql requires : elif direction == "upgrade": - return [str(alembic_exe), direction, args.upgrade, *get_optional_args(args)] + return [str(alembic_exe), *get_optional_args(args), direction, args.upgrade] elif direction == "downgrade": - return [str(alembic_exe), direction, args.downgrade, *get_optional_args(args)] + return [str(alembic_exe), *get_optional_args(args), direction, args.downgrade] def log_output_and_errors(result, direction, args: Namespace, timestamp=None): diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 8ad8b3fe..f6747347 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -103,7 +103,10 @@ def downgrade_namespace(generic_versiondb_namespace, dir_versiondb_in): def altdb_namespace(generic_versiondb_namespace): """Namespace for pyani versiondb -n altdb.""" return modify_namespace( - generic_versiondb_namespace, dbpath="pyanidb_altdb", dbname="altdb" + generic_versiondb_namespace, + dbpath="pyanidb_altdb", + dbname="pyanidb_altdb", + alembic_config="alt_alembic_config.ini", ) @@ -113,7 +116,8 @@ def alt_config_namespace(generic_versiondb_namespace): return modify_namespace( generic_versiondb_namespace, dbpath="pyanidb_alt_config", - alembic_config="alt_config", + dbname="pyanidb_altdb", + alembic_config="alt_alembic_config.ini", ) @@ -233,7 +237,6 @@ def test_alembic_cmdline_generation( generic_versiondb_namespace, downgrade_namespace, altdb_namespace, - alt_config_namespace, dir_versiondb_in, ): """Generate alembic command lines.""" @@ -260,31 +263,19 @@ def test_alembic_cmdline_generation( " ".join(versiondb.construct_alembic_cmdline(altdb_args.direction, altdb_args)) ) - alt_config_args = alt_config_namespace - alembic_cmds.append( - " ".join( - versiondb.construct_alembic_cmdline( - alt_config_args.direction, alt_config_args - ) - ) - ) - assert alembic_cmds == [ "alembic upgrade head", "alembic downgrade base", - f"alembic upgrade head -n {altdb_args.dbname}", - f"alembic upgrade head -c {alt_config_args.alembic_config}", + f"alembic -n {altdb_args.dbname} -c {altdb_args.alembic_config} upgrade head", ] - pass - # alembic_cmd = versiondb.construct_alembic_cmdline() - # dir_alembic = tmp_path / "versiondb_output" - # expected = "alembic upgrade" - # Test upgrade def test_versiondb_upgrade( - generic_versiondb_namespace, dir_versiondb_in, dir_versiondb_out + generic_versiondb_namespace, + dir_versiondb_in, + dir_versiondb_out, + dir_versiondb_targets, ): """Test upgrade of database.""" # Test setup @@ -313,10 +304,11 @@ def test_versiondb_upgrade( # Dump altered and target databases enddb_dump = dumpdb(abs_dbpath) + targetdb_dump = dumpdb(args.target) # Run diff - diff_cmd = ["diff", "--suppress-common-lines", enddb_dump, targetdb_dump] + diff_cmd = ["diff", "-y", "--suppress-common-lines", enddb_dump, targetdb_dump] result = subprocess.run( diff_cmd, shell=False, @@ -324,13 +316,19 @@ def test_versiondb_upgrade( stdout=subprocess.PIPE, ) + actual_diff = "".join( + open(dir_versiondb_targets / "upgrade_minus_head.diff", "r").readlines() + ) + # Move files cleanup(abs_dbpath, "upgrade", dir_versiondb_out, args) - assert result.stdout == expected_diffs("upgrade") + assert result.stdout.decode() == actual_diff -def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiondb_out): +def test_versiondb_downgrade( + downgrade_namespace, dir_versiondb_in, dir_versiondb_out, dir_versiondb_targets +): """Test downgrade of database.""" # Test setup # Retrieve test namespace and @@ -360,7 +358,7 @@ def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiond targetdb_dump = dumpdb(args.target) # Run diff - diff_cmd = ["diff", "--suppress-common-lines", enddb_dump, targetdb_dump] + diff_cmd = ["diff", "-y", "--suppress-common-lines", enddb_dump, targetdb_dump] result = subprocess.run( diff_cmd, shell=False, @@ -368,15 +366,21 @@ def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiond stdout=subprocess.PIPE, ) + actual_diff = "".join( + open(dir_versiondb_targets / "downgrade_minus_base.diff", "r").readlines() + ) + # Move output files cleanup(abs_dbpath, "downgrade", dir_versiondb_out, args) - assert result.stdout == expected_diffs("downgrade") + assert result.stdout.decode() == actual_diff # Test alternate dbname -def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): - """Test upgrade of database using an alternate database name, such as in a multidb situation.""" +def test_versiondb_altdb( + altdb_namespace, dir_versiondb_in, dir_versiondb_out, dir_versiondb_targets +): + """Test upgrade of database using an alternate database name and config file, such as in a multidb situation.""" # Test setup # Retrieve test namespace and # Set environment variables and resolve absolute path of database @@ -388,6 +392,7 @@ def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): startdb_dump = dumpdb(args.start) name_base_reqs(startdb_dump) + # assert False # Run `sqlite3 -init init_cmd = ["sqlite3", abs_dbpath] subprocess.run( @@ -406,7 +411,7 @@ def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): targetdb_dump = dumpdb(args.target) # Run diff - diff_cmd = ["diff", "--suppress-common-lines", enddb_dump, targetdb_dump] + diff_cmd = ["diff", "-y", "--suppress-common-lines", enddb_dump, targetdb_dump] result = subprocess.run( diff_cmd, shell=False, @@ -414,13 +419,18 @@ def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): stdout=subprocess.PIPE, ) + actual_diff = "".join( + open(dir_versiondb_targets / "altdb_minus_head.diff", "r").readlines() + ) + # Move files cleanup(abs_dbpath, "altdb", dir_versiondb_out, args) - assert result.stdout == expected_diffs("altdb") + assert result.stdout.decode() == actual_diff # Test alt_config result +@pytest.mark.skip(reason="May no be needed.") def test_versiondb_alt_config( alt_config_namespace, dir_versiondb_in, dir_versiondb_out ): @@ -462,7 +472,14 @@ def test_versiondb_alt_config( stdout=subprocess.PIPE, ) + actual_diff = "".join( + open( + "/Users/baileythegreen/Software/pyani/tests/fixtures/versiondb/upgrade_minus_head.diff", + "r", + ).readlines() + ) + # Move files cleanup(abs_dbpath, "alt_config", dir_versiondb_out, args) - assert result.stdout == expected_diffs("alt_config") + assert result.stdout.decode() == actual_diff From 34a694b4143adf0a96e3d9cb43ad7f2422fe535b Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 02:25:05 +0000 Subject: [PATCH 47/91] Updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84336684..3cd972bf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: - run: name: install third-party tools command: | - sudo apt-get install csh mummer ncbi-blast+ sqlite3 + sudo apt-get install csh mummer ncbi-blast+ sqlite3=3.36.0 - run: name: install legacy BLAST command: | From 12daf2c18ec74f01060b0603ac8f79afbe686775 Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 02:26:41 +0000 Subject: [PATCH 48/91] Updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3cd972bf..391c4b1d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: - run: name: install third-party tools command: | - sudo apt-get install csh mummer ncbi-blast+ sqlite3=3.36.0 + sudo apt-get install csh mummer ncbi-blast+ sqlite3=3.36 - run: name: install legacy BLAST command: | From ed18de562b633838305f86f2e3b4e755ffb07a5b Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 02:28:35 +0000 Subject: [PATCH 49/91] Updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 391c4b1d..fbbce613 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: - run: name: install third-party tools command: | - sudo apt-get install csh mummer ncbi-blast+ sqlite3=3.36 + sudo apt-get install csh mummer ncbi-blast+ sqlite3>3.27 - run: name: install legacy BLAST command: | From a3efcbcbae9047f4652df666c6865bd5ce66ea49 Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 02:39:38 +0000 Subject: [PATCH 50/91] Updated config.yml --- .circleci/config.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index fbbce613..67e52fbe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,6 +68,17 @@ jobs: echo 'export PATH=$PWD:$PATH' >> $BASH_ENV source $BASH_ENV fastANI -h + - run: + name: install sqlite3 + command: | + wget https://sqlite.org/2022/sqlite-autoconf-33600.tar.gz + tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite/configure ;# Run the configure script + make ;# Run the makefile. + make sqlite3.c ;# Build the "amalgamation" source file + make test ;# Run some tests (requires Tcl) - run: From 84b71e6922d628ec249162ca08b2b44c5d06c98a Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 02:44:29 +0000 Subject: [PATCH 51/91] Updated config.yml --- .circleci/config.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 67e52fbe..e9229cee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: - run: name: install third-party tools command: | - sudo apt-get install csh mummer ncbi-blast+ sqlite3>3.27 + sudo apt-get install csh mummer ncbi-blast+ - run: name: install legacy BLAST command: | @@ -69,16 +69,16 @@ jobs: source $BASH_ENV fastANI -h - run: - name: install sqlite3 - command: | - wget https://sqlite.org/2022/sqlite-autoconf-33600.tar.gz - tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build will occur in a sibling directory - cd bld ;# Change to the build directory - ../sqlite/configure ;# Run the configure script - make ;# Run the makefile. - make sqlite3.c ;# Build the "amalgamation" source file - make test ;# Run some tests (requires Tcl) + name: install sqlite3 + command: | + wget https://sqlite.org/2022/sqlite-autoconf-33600.tar.gz + tar -xzf sqlite-autoconf-33600.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite/configure ;# Run the configure script + make ;# Run the makefile. + make sqlite3.c ;# Build the "amalgamation" source file + make test ;# Run some tests (requires Tcl) - run: From 27ee6bb5d58ab66a3cf3fb3c32c08187ebc9b242 Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 02:53:41 +0000 Subject: [PATCH 52/91] Updated config.yml --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9229cee..8cc3a2e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,11 +71,11 @@ jobs: - run: name: install sqlite3 command: | - wget https://sqlite.org/2022/sqlite-autoconf-33600.tar.gz - tar -xzf sqlite-autoconf-33600.tar.gz ;# Unpack the source tree into "sqlite" + wget https://github.com/sqlite/sqlite/archive/refs/tags/version-3.36.0.tar.gz + tar xzf version-3.36.0.tar.gz ;# Unpack the source tree into "sqlite" mkdir bld ;# Build will occur in a sibling directory cd bld ;# Change to the build directory - ../sqlite/configure ;# Run the configure script + ../sqlite-version-3.36.0/configure ;# Run the configure script make ;# Run the makefile. make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) From 0bfef520d8116ab874e80d72c13db3e113aaa7c7 Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 03:26:50 +0000 Subject: [PATCH 53/91] Updated config.yml --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8cc3a2e5..60e60156 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,6 +79,8 @@ jobs: make ;# Run the makefile. make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) + ls + ls ../sqlite-version-3.36.0 - run: From 373725fb5605ae73c61d8b90935e07ed2922f217 Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 11:44:31 +0000 Subject: [PATCH 54/91] Updated config.yml --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 60e60156..963dcbc9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,7 +80,9 @@ jobs: make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) ls - ls ../sqlite-version-3.36.0 + echo 'export PATH=$PWD:$PATH' >> $BASH_ENV + source $BASH_ENV + sqlite3 -help - run: From baaec35025e8e68f142ad7aba1300237cf73cabe Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 11:58:07 +0000 Subject: [PATCH 55/91] Updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 963dcbc9..659883d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,7 +82,7 @@ jobs: ls echo 'export PATH=$PWD:$PATH' >> $BASH_ENV source $BASH_ENV - sqlite3 -help + #sqlite3 -help - run: From d3a967c66b904b92be257384a74167467fa556fe Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 14:20:01 +0000 Subject: [PATCH 56/91] Updated config.yml --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 659883d0..17e3a828 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,9 +80,10 @@ jobs: make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) ls - echo 'export PATH=$PWD:$PATH' >> $BASH_ENV + echo 'export PATH=$PWD/sqlite3:$PATH' >> $BASH_ENV source $BASH_ENV - #sqlite3 -help + cd .. + sqlite3 -help - run: From 359c00dbba865edd00d0b7b8fa1ae6990559a049 Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 14:32:40 +0000 Subject: [PATCH 57/91] Updated config.yml --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 17e3a828..c3e6ebe8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,8 +82,7 @@ jobs: ls echo 'export PATH=$PWD/sqlite3:$PATH' >> $BASH_ENV source $BASH_ENV - cd .. - sqlite3 -help + - run: From 1a549e477cd942666d61b1ef748af2be17861686 Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 14:57:04 +0000 Subject: [PATCH 58/91] Updated config.yml --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3e6ebe8..640c4549 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,7 +82,8 @@ jobs: ls echo 'export PATH=$PWD/sqlite3:$PATH' >> $BASH_ENV source $BASH_ENV - + cd + sqlite -h - run: From 3c44d357bc498a441478fa931ead419511b1c4b1 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 17 Mar 2022 20:54:41 +0000 Subject: [PATCH 59/91] Add default location and install instructions for `sqlite3` --- .circleci/config.yml | 17 ++++++++++++++++- pyani/pyani_config.py | 1 + tests/test_versiondb.py | 12 +++++++----- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84336684..af8ff4a2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,7 +68,22 @@ jobs: echo 'export PATH=$PWD:$PATH' >> $BASH_ENV source $BASH_ENV fastANI -h - + - run: + name: install sqlite3 + command: | + wget https://github.com/sqlite/sqlite/archive/refs/tags/version-3.36.0.tar.gz + tar xzf version-3.36.0.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite-version-3.36.0/configure ;# Run the configure script + make ;# Run the makefile. + make sqlite3.c ;# Build the "amalgamation" source file + make test ;# Run some tests (requires Tcl) + ls + echo 'export PATH=$PWD/sqlite3:$PATH' >> $BASH_ENV + source $BASH_ENV + # cd + # sqlite3 -help - run: name: install dependencies diff --git a/pyani/pyani_config.py b/pyani/pyani_config.py index 7829a63f..a8070cf6 100644 --- a/pyani/pyani_config.py +++ b/pyani/pyani_config.py @@ -56,6 +56,7 @@ QSUB_DEFAULT = Path("qsub") FASTANI_DEFAULT = Path("fastANI") ALEMBIC_DEFAULT = Path("alembic") +SQLITE_DEFAULT = Path("sqlite3") # Stems for output files ANIM_FILESTEMS = ( diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index f6747347..3423ec5d 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -26,6 +26,8 @@ from pyani.pyani_orm import PyaniORMException, get_session, add_alembic from tools import modify_namespace +import pyani_config + # Create environment variables for alembic to access def setenv(dir_versiondb_in, dbfile: Path): @@ -158,7 +160,7 @@ def expected_diffs(namespace): def dumpdb(abs_dbpath): """Dump contents of database to a plain-text file.""" - cmdline = ["sqlite3", f"{abs_dbpath}", ".dump"] + cmdline = [pyani_config.SQLITE_DEFAULT, f"{abs_dbpath}", ".dump"] with open(f"{abs_dbpath}.sql", "w") as outfile: subprocess.run( cmdline, @@ -290,7 +292,7 @@ def test_versiondb_upgrade( name_base_reqs(startdb_dump) # Run `sqlite3 -init - init_cmd = ["sqlite3", abs_dbpath] + init_cmd = [pyani_config.SQLITE_DEFAULT, abs_dbpath] subprocess.run( init_cmd, stdin=open(startdb_dump), @@ -341,7 +343,7 @@ def test_versiondb_downgrade( startdb_dump = dumpdb(args.start) # Run `sqlite3 -init - init_cmd = ["sqlite3", abs_dbpath] + init_cmd = [pyani_config.SQLITE_DEFAULT, abs_dbpath] subprocess.run( init_cmd, stdin=open(startdb_dump), @@ -394,7 +396,7 @@ def test_versiondb_altdb( # assert False # Run `sqlite3 -init - init_cmd = ["sqlite3", abs_dbpath] + init_cmd = [pyani_config.SQLITE_DEFAULT, abs_dbpath] subprocess.run( init_cmd, stdin=open(startdb_dump), @@ -447,7 +449,7 @@ def test_versiondb_alt_config( name_base_reqs(startdb_dump) # Run `sqlite3 -init - init_cmd = ["sqlite3", abs_dbpath] + init_cmd = [pyani_config.SQLITE_DEFAULT, abs_dbpath] subprocess.run( init_cmd, stdin=open(startdb_dump), From 983cd8729c7005304bd61bf4c424e0562206772c Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 17 Mar 2022 21:08:31 +0000 Subject: [PATCH 60/91] Fix import syntax --- tests/test_versiondb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 3423ec5d..9e08d764 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -26,7 +26,7 @@ from pyani.pyani_orm import PyaniORMException, get_session, add_alembic from tools import modify_namespace -import pyani_config +from pyani import pyani_config # Create environment variables for alembic to access From 8e713344191032886da5b16d6e9f8798e596b882 Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 21:49:56 +0000 Subject: [PATCH 61/91] Updated config.yml --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 858c90fd..11a8e32b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,8 +82,9 @@ jobs: ls echo 'export PATH=$PWD/sqlite3:$PATH' >> $BASH_ENV source $BASH_ENV + echo $PATH # cd - # sqlite3 -help + sqlite3 -help - run: name: install dependencies From 72dd9203653938a4075f9bab841c55b5452d247b Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 21:59:20 +0000 Subject: [PATCH 62/91] Updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11a8e32b..08902080 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ jobs: make ;# Run the makefile. make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) - ls + ls sqlite3 echo 'export PATH=$PWD/sqlite3:$PATH' >> $BASH_ENV source $BASH_ENV echo $PATH From 0892fc4956881f7473a5f9f90edeb25e796d801e Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 22:10:57 +0000 Subject: [PATCH 63/91] Updated config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08902080..444d737f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,8 +79,8 @@ jobs: make ;# Run the makefile. make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) - ls sqlite3 - echo 'export PATH=$PWD/sqlite3:$PATH' >> $BASH_ENV + ls sqlite3/* + echo 'export PATH=$PWD/sqlite3/:$PATH' >> $BASH_ENV source $BASH_ENV echo $PATH # cd From b3b31f99982da65916d09e65908fcc7b81a4981f Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 22:42:09 +0000 Subject: [PATCH 64/91] Updated config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 444d737f..1b31666f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,11 +80,11 @@ jobs: make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) ls sqlite3/* - echo 'export PATH=$PWD/sqlite3/:$PATH' >> $BASH_ENV + echo "export PATH=~/repo/bld/sqlite3:$PATH" >> $BASH_ENV source $BASH_ENV echo $PATH # cd - sqlite3 -help + # sqlite3 -help - run: name: install dependencies From 1f0e3d4e4bb0db8568b0ab39458c2ee5e88f9052 Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 22:57:28 +0000 Subject: [PATCH 65/91] Updated config.yml --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b31666f..1ebbdb3b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,8 +83,7 @@ jobs: echo "export PATH=~/repo/bld/sqlite3:$PATH" >> $BASH_ENV source $BASH_ENV echo $PATH - # cd - # sqlite3 -help + - run: name: install dependencies From 720e0af423d6cbc510ba7f180d2d4f58455eb343 Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 23:08:50 +0000 Subject: [PATCH 66/91] Updated config.yml --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ebbdb3b..cb4acd1e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,6 @@ jobs: make ;# Run the makefile. make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) - ls sqlite3/* echo "export PATH=~/repo/bld/sqlite3:$PATH" >> $BASH_ENV source $BASH_ENV echo $PATH From 8b34edc21254e1e02e5747cbd24798d9131b88fd Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 17 Mar 2022 23:51:00 +0000 Subject: [PATCH 67/91] Updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb4acd1e..1b23a46a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ jobs: make ;# Run the makefile. make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) - echo "export PATH=~/repo/bld/sqlite3:$PATH" >> $BASH_ENV + echo "export PATH=~/repo/bld:$PATH" >> $BASH_ENV source $BASH_ENV echo $PATH From f351084e8b8e96b7f847176f34ec5931065ee63d Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Fri, 18 Mar 2022 14:52:28 +0000 Subject: [PATCH 68/91] Remove some `dry-run` specific stuff for the time being --- .../92f7f6b1626e_add_fastani_columns.py | 140 +++++++++--------- 1 file changed, 71 insertions(+), 69 deletions(-) diff --git a/alembic/versions/92f7f6b1626e_add_fastani_columns.py b/alembic/versions/92f7f6b1626e_add_fastani_columns.py index 06075ff5..8c39a1d7 100644 --- a/alembic/versions/92f7f6b1626e_add_fastani_columns.py +++ b/alembic/versions/92f7f6b1626e_add_fastani_columns.py @@ -7,16 +7,17 @@ """ from alembic import op import sqlalchemy as sa +import sys from sqlalchemy import ( Column, - Table, - MetaData, - ForeignKey, + # Table, + # MetaData, + # ForeignKey, Integer, - String, + # String, Float, - Boolean, + # Boolean, UniqueConstraint, ) @@ -27,71 +28,42 @@ depends_on = None -meta = MetaData() -new_comparisons = Table( - "new_comparisons", - meta, - Column("comparisons_id", Integer, primary_key=True), - Column("query_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), - Column("subject_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), - Column("aln_length", Integer), - Column("sim_errs", Integer), - Column("identity", Float), - Column("cov_query", Float), - Column("cov_subject", Float), - Column("program", String), - Column("version", String), - Column("fragsize", Integer), - Column("maxmatch", Boolean), - Column("kmersize", Integer), - Column("minmatch", Float), - UniqueConstraint( - "query_id", - "subject_id", - "program", - "version", - "fragsize", - "maxmatch", - "kmersize", - "minmatch", - name="fastani_reqs", - ), -) - -old_comparisons = Table( - "old_comparisons", - meta, - Column("comparisons_id", Integer, primary_key=True), - Column("query_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), - Column("subject_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), - Column("aln_length", Integer), - Column("sim_errs", Integer), - Column("identity", Float), - Column("cov_query", Float), - Column("cov_subject", Float), - Column("program", String), - Column("version", String), - Column("fragsize", Integer), - Column("maxmatch", Boolean), - UniqueConstraint( - "query_id", - "subject_id", - "program", - "version", - "fragsize", - "maxmatch", - name="base_reqs", - ), -) - - def upgrade(): # op.add_column("comparisons", sa.Column("kmersize", sa.Integer)) # op.add_column("comparisons", sa.Column("minmatch", sa.Float)) - - with op.batch_alter_table("comparisons", copy_from=old_comparisons) as batch_op: - batch_op.add_column(sa.Column("kmersize", sa.Integer)) - batch_op.add_column(sa.Column("minmatch", sa.Float)) + """ + comparisons = Table( + "comparisons", + meta, + Column("comparisons_id", Integer, primary_key=True), + Column("query_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("subject_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("aln_length", Integer), + Column("sim_errs", Integer), + Column("identity", Float), + Column("cov_query", Float), + Column("cov_subject", Float), + Column("program", String), + Column("version", String), + Column("fragsize", Integer), + Column("maxmatch", Boolean), + UniqueConstraint( + "query_id", + "subject_id", + "program", + "version", + "fragsize", + "maxmatch", + name="base_reqs", + ), + ) + """ + with op.batch_alter_table("comparisons") as batch_op: + # batch_op.add_column(sa.Column("kmersize", sa.Integer, default=None)) + # batch_op.add_column(sa.Column("minmatch", sa.Float, default=None)) + batch_op.drop_constraint("base_reqs") + batch_op.add_column(sa.Column("kmersize", sa.Integer, default=None)) + batch_op.add_column(sa.Column("minmatch", sa.Float, default=None)) batch_op.create_unique_constraint( "fastani_reqs", [ @@ -110,8 +82,38 @@ def upgrade(): def downgrade(): # op.drop_constraint("comparisons", 'kmersize') # op.drop_column("comparisons", "kmersize") - - with op.batch_alter_table("comparisons", copy_from=new_comparisons) as batch_op: + """ + comparisons = Table( + "comparisons", + meta, + Column("comparisons_id", Integer, primary_key=True), + Column("query_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("subject_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("aln_length", Integer), + Column("sim_errs", Integer), + Column("identity", Float), + Column("cov_query", Float), + Column("cov_subject", Float), + Column("program", String), + Column("version", String), + Column("fragsize", Integer), + Column("maxmatch", Boolean), + Column("kmersize", Integer), + Column("minmatch", Float), + UniqueConstraint( + "query_id", + "subject_id", + "program", + "version", + "fragsize", + "maxmatch", + "kmersize", + "minmatch", + name="fastani_reqs", + ), + ) + """ + with op.batch_alter_table("comparisons") as batch_op: batch_op.drop_column("kmersize") batch_op.drop_column("minmatch") batch_op.drop_constraint("fastani_reqs") From 86b79a40296663c4ec01481a22c577382b32d366 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Fri, 18 Mar 2022 18:04:34 +0000 Subject: [PATCH 69/91] Set `create_constraint=True` for compatibility between sqlalchemy versions --- pyani/pyani_orm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyani/pyani_orm.py b/pyani/pyani_orm.py index 42fa468a..7974b9a5 100644 --- a/pyani/pyani_orm.py +++ b/pyani/pyani_orm.py @@ -312,7 +312,9 @@ class Comparison(Base): program = Column(String) version = Column(String) fragsize = Column(Integer) # in fastANI this is fragLength - maxmatch = Column(Boolean) # in fastANi this is Null + + # create_constraint keyword is needed for portability between sqlalchemy 1.3 and 1.4 + maxmatch = Column(Boolean, create_constraint=True) # in fastANi this is Null kmersize = Column(Integer) minmatch = Column(Float) From 8aa105e85e3c434d52ce25cfb8f5522d7c355a51 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sat, 19 Mar 2022 00:49:30 +0000 Subject: [PATCH 70/91] Set `create_constraint` flag only if `sqlalchemy` version is >= 1.4 --- pyani/pyani_orm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyani/pyani_orm.py b/pyani/pyani_orm.py index 7974b9a5..2feef423 100644 --- a/pyani/pyani_orm.py +++ b/pyani/pyani_orm.py @@ -314,7 +314,10 @@ class Comparison(Base): fragsize = Column(Integer) # in fastANI this is fragLength # create_constraint keyword is needed for portability between sqlalchemy 1.3 and 1.4 - maxmatch = Column(Boolean, create_constraint=True) # in fastANi this is Null + if float(sqlalchemy.__version__.rsplit(".", 1)[0]) < 1.4: + maxmatch = Column(Boolean) + else: + maxmatch = Column(Boolean, create_constraint=True) # in fastANi this is Null kmersize = Column(Integer) minmatch = Column(Float) From 9a10cb5974cbcc7e1a16e3b692416eff7d654523 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sat, 19 Mar 2022 01:42:35 +0000 Subject: [PATCH 71/91] Update `versiondb` test targets --- tests/test_targets/subcmd_versiondb/altdb_minus_head.diff | 4 ++-- tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff | 4 +++- tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_targets/subcmd_versiondb/altdb_minus_head.diff b/tests/test_targets/subcmd_versiondb/altdb_minus_head.diff index 7ff5186c..1e7eb865 100644 --- a/tests/test_targets/subcmd_versiondb/altdb_minus_head.diff +++ b/tests/test_targets/subcmd_versiondb/altdb_minus_head.diff @@ -16,8 +16,8 @@ CREATE TABLE alembic_version ( < INSERT INTO alembic_version VALUES('92f7f6b1626e'); < CREATE TABLE IF NOT EXISTS "comparisons" ( < CHECK (maxmatch IN (0, 1)), < - FOREIGN KEY(subject_id) REFERENCES genomes (genome_id | FOREIGN KEY(query_id) REFERENCES genomes (genome_id), - FOREIGN KEY(query_id) REFERENCES genomes (genome_id) | FOREIGN KEY(subject_id) REFERENCES genomes (genome_id + > FOREIGN KEY(query_id) REFERENCES genomes (genome_id), + FOREIGN KEY(query_id) REFERENCES genomes (genome_id) | CHECK (maxmatch IN (0, 1)) > ); > CREATE TABLE runs_comparisons ( > comparison_id INTEGER, diff --git a/tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff b/tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff index be41bbe7..f219143c 100644 --- a/tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff +++ b/tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff @@ -11,9 +11,11 @@ CREATE TABLE runs_comparisons ( | CREATE TABLE comparisons ( CREATE TABLE IF NOT EXISTS "comparisons" ( < CHECK (maxmatch IN (0, 1)), | UNIQUE (query_id, subject_id, program, version, frags CONSTRAINT base_reqs UNIQUE (query_id, subject_id, pr < + FOREIGN KEY(subject_id) REFERENCES genomes (genome_id | FOREIGN KEY(subject_id) REFERENCES genomes (genome_id + > CHECK (maxmatch IN (0, 1)) + > ); > CREATE TABLE runs_comparisons ( > comparison_id INTEGER, > run_id INTEGER, > FOREIGN KEY(comparison_id) REFERENCES comparisons (co > FOREIGN KEY(run_id) REFERENCES runs (run_id) - > ); diff --git a/tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff b/tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff index 95437981..6143b581 100644 --- a/tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff +++ b/tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff @@ -16,9 +16,11 @@ CREATE TABLE alembic_version ( < INSERT INTO alembic_version VALUES('92f7f6b1626e'); < CREATE TABLE IF NOT EXISTS "comparisons" ( < CHECK (maxmatch IN (0, 1)), < + FOREIGN KEY(subject_id) REFERENCES genomes (genome_id | FOREIGN KEY(subject_id) REFERENCES genomes (genome_id + > CHECK (maxmatch IN (0, 1)) + > ); > CREATE TABLE runs_comparisons ( > comparison_id INTEGER, > run_id INTEGER, > FOREIGN KEY(comparison_id) REFERENCES comparisons (co > FOREIGN KEY(run_id) REFERENCES runs (run_id) - > ); From 35296914c032d01cfe92bb66723c4c9e37990295 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sat, 19 Mar 2022 01:42:52 +0000 Subject: [PATCH 72/91] Update `versiondb` test fixtures --- tests/fixtures/versiondb/base_pyanidb | Bin 40960 -> 40960 bytes tests/fixtures/versiondb/head_pyanidb | Bin 49152 -> 49152 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/fixtures/versiondb/base_pyanidb b/tests/fixtures/versiondb/base_pyanidb index 9f0ea475f35215440339d619b23b3726460a426b..2b6634f140e6e3c41936c7e7bc51d17339f4a2d3 100644 GIT binary patch delta 95 zcmV-l0HFVXzyg540+3=HQ(z4b_y7q98~_gV56KP54E77w3Wx{`3Q-3fv4NfjlR5|R z2}1`57b7D9h)uII2pj?(EFcOALr6tKOCTt1VR&s}bYo~BNlqXrFf1T3DYH%rvXw!i B8w&sc delta 66 zcmV-I0KNZ!zyg540+3=HS4a&H_y7q9Isgvz56KP54E77w3Wx|P3Q-3-vk?%y29q2I Y@ChXc2NxqF0f-T^6$l&xvrh`Km8sejF#rGn diff --git a/tests/fixtures/versiondb/head_pyanidb b/tests/fixtures/versiondb/head_pyanidb index a212532c9c7e8b707c5569fea8462e8f3cc50ced..904de8a7b6e42947c83b077cbc071a9f6e0edeb6 100644 GIT binary patch delta 113 zcmV-%0FM8FfCGSl1CU}MQ(z4b_y7w9-vAE{54jEr4&Dsj3=0bD37rPc31$V~v5{Q` zll=wn3WXI32NxqF0f}+5_6BYOCM+Nd2}4LlLrWkiZDDwAVRU0?AW2RjC@?G_F)1ku TDK!cP7fB2?rM=BLRsuv)Bf10tgBzH3|k7Nh6cL4;ZtA34FFds~MR9 From cf810c59c8507bc65be9a8f748d85b9b368c74a8 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sun, 20 Mar 2022 20:40:19 +0000 Subject: [PATCH 73/91] Send diffs to stdout so we can debug why tests sporadically fail --- tests/test_versiondb.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 9e08d764..a9967735 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -474,14 +474,15 @@ def test_versiondb_alt_config( stdout=subprocess.PIPE, ) - actual_diff = "".join( + expected_diff = "".join( open( "/Users/baileythegreen/Software/pyani/tests/fixtures/versiondb/upgrade_minus_head.diff", "r", ).readlines() ) - + sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") + sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") # Move files cleanup(abs_dbpath, "alt_config", dir_versiondb_out, args) - assert result.stdout.decode() == actual_diff + assert result.stdout.decode() == expected_diff From 03c00573d2207843fac88a0b2e3886f96f0b3840 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sun, 20 Mar 2022 21:07:44 +0000 Subject: [PATCH 74/91] Send diffs to stdout for debugging --- tests/test_versiondb.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index a9967735..94c5c55a 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -318,14 +318,15 @@ def test_versiondb_upgrade( stdout=subprocess.PIPE, ) - actual_diff = "".join( + expected_diff = "".join( open(dir_versiondb_targets / "upgrade_minus_head.diff", "r").readlines() ) - + sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") + sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") # Move files cleanup(abs_dbpath, "upgrade", dir_versiondb_out, args) - assert result.stdout.decode() == actual_diff + assert result.stdout.decode() == expected_diff def test_versiondb_downgrade( @@ -368,14 +369,15 @@ def test_versiondb_downgrade( stdout=subprocess.PIPE, ) - actual_diff = "".join( + expected_diff = "".join( open(dir_versiondb_targets / "downgrade_minus_base.diff", "r").readlines() ) - + sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") + sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") # Move output files cleanup(abs_dbpath, "downgrade", dir_versiondb_out, args) - assert result.stdout.decode() == actual_diff + assert result.stdout.decode() == expected_diff # Test alternate dbname @@ -421,14 +423,15 @@ def test_versiondb_altdb( stdout=subprocess.PIPE, ) - actual_diff = "".join( + expected_diff = "".join( open(dir_versiondb_targets / "altdb_minus_head.diff", "r").readlines() ) - + sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") + sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") # Move files cleanup(abs_dbpath, "altdb", dir_versiondb_out, args) - assert result.stdout.decode() == actual_diff + assert result.stdout.decode() == expected_diff # Test alt_config result @@ -466,7 +469,7 @@ def test_versiondb_alt_config( targetdb_dump = dumpdb(args.target) # Run diff - diff_cmd = ["diff", "--suppress-common-lines", enddb_dump, targetdb_dump] + diff_cmd = ["diff", "-y", "--suppress-common-lines", enddb_dump, targetdb_dump] result = subprocess.run( diff_cmd, shell=False, From bc274602234cd273f10d773290ae51ec90ece485 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Sun, 20 Mar 2022 22:20:42 +0000 Subject: [PATCH 75/91] Use `sqldiff` instead of `diff` --- tests/test_versiondb.py | 44 +++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 94c5c55a..2fd5c09f 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -216,14 +216,14 @@ def cleanup(abs_dbpath, test, dir_versiondb_out, args): # if the file already exists, so this is a two-step process # Copy files to new location shutil.copy(abs_dbpath, dir_versiondb_out / test) - shutil.copy(f"{abs_dbpath}.sql", dir_versiondb_out / test) - shutil.copy(f"{args.target}.sql", dir_versiondb_out / test) + # shutil.copy(f"{abs_dbpath}.sql", dir_versiondb_out / test) + # shutil.copy(f"{args.target}.sql", dir_versiondb_out / test) shutil.copy(f"{args.start}.sql", dir_versiondb_out / test) # Remove old files os.remove(abs_dbpath) - os.remove(f"{abs_dbpath}.sql") - os.remove(f"{args.target}.sql") + # os.remove(f"{abs_dbpath}.sql") + # os.remove(f"{args.target}.sql") os.remove(f"{args.start}.sql") # This file is not generated in the downgrade test @@ -305,12 +305,12 @@ def test_versiondb_upgrade( versiondb.migrate_database(args.direction, args, timestamp="testing") # Dump altered and target databases - enddb_dump = dumpdb(abs_dbpath) + # enddb_dump = dumpdb(abs_dbpath) - targetdb_dump = dumpdb(args.target) + # targetdb_dump = dumpdb(args.target) # Run diff - diff_cmd = ["diff", "-y", "--suppress-common-lines", enddb_dump, targetdb_dump] + diff_cmd = ["sqldiff", "--schema", abs_dbpath, args.target] result = subprocess.run( diff_cmd, shell=False, @@ -318,11 +318,11 @@ def test_versiondb_upgrade( stdout=subprocess.PIPE, ) - expected_diff = "".join( - open(dir_versiondb_targets / "upgrade_minus_head.diff", "r").readlines() - ) + expected_diff = "" + sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") + # Move files cleanup(abs_dbpath, "upgrade", dir_versiondb_out, args) @@ -356,12 +356,8 @@ def test_versiondb_downgrade( # Run test migration versiondb.migrate_database(args.direction, args, timestamp="testing") - # Dump altered and target databases - enddb_dump = dumpdb(abs_dbpath) - targetdb_dump = dumpdb(args.target) - # Run diff - diff_cmd = ["diff", "-y", "--suppress-common-lines", enddb_dump, targetdb_dump] + diff_cmd = ["sqldiff", "--schema", abs_dbpath, args.target] result = subprocess.run( diff_cmd, shell=False, @@ -369,11 +365,11 @@ def test_versiondb_downgrade( stdout=subprocess.PIPE, ) - expected_diff = "".join( - open(dir_versiondb_targets / "downgrade_minus_base.diff", "r").readlines() - ) + expected_diff = "DROP TABLE alembic_version;\n" + sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") + # Move output files cleanup(abs_dbpath, "downgrade", dir_versiondb_out, args) @@ -410,12 +406,8 @@ def test_versiondb_altdb( # Run test migration versiondb.migrate_database(args.direction, args, timestamp="testing") - # Dump altered and target databases - enddb_dump = dumpdb(abs_dbpath) - targetdb_dump = dumpdb(args.target) - # Run diff - diff_cmd = ["diff", "-y", "--suppress-common-lines", enddb_dump, targetdb_dump] + diff_cmd = ["sqldiff", "--schema", abs_dbpath, args.target] result = subprocess.run( diff_cmd, shell=False, @@ -423,11 +415,11 @@ def test_versiondb_altdb( stdout=subprocess.PIPE, ) - expected_diff = "".join( - open(dir_versiondb_targets / "altdb_minus_head.diff", "r").readlines() - ) + expected_diff = "" + sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") + # Move files cleanup(abs_dbpath, "altdb", dir_versiondb_out, args) From cd5306c3ffb6cb65aaf507cfa7165a3cce29dd58 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Mar 2022 11:20:14 +0100 Subject: [PATCH 76/91] Move `sqldiff` specification to `pyani_config.py` --- pyani/pyani_config.py | 1 + tests/test_versiondb.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyani/pyani_config.py b/pyani/pyani_config.py index a8070cf6..22f70174 100644 --- a/pyani/pyani_config.py +++ b/pyani/pyani_config.py @@ -57,6 +57,7 @@ FASTANI_DEFAULT = Path("fastANI") ALEMBIC_DEFAULT = Path("alembic") SQLITE_DEFAULT = Path("sqlite3") +SQLDIFF_DEFAULT = Path("sqldiff") # Stems for output files ANIM_FILESTEMS = ( diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 2fd5c09f..d90b8c10 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -310,7 +310,7 @@ def test_versiondb_upgrade( # targetdb_dump = dumpdb(args.target) # Run diff - diff_cmd = ["sqldiff", "--schema", abs_dbpath, args.target] + diff_cmd = [pyani_config.SQLDIFF_DEFAULT, "--schema", abs_dbpath, args.target] result = subprocess.run( diff_cmd, shell=False, @@ -357,7 +357,7 @@ def test_versiondb_downgrade( versiondb.migrate_database(args.direction, args, timestamp="testing") # Run diff - diff_cmd = ["sqldiff", "--schema", abs_dbpath, args.target] + diff_cmd = [pyani_config.SQLDIFF_DEFAULT, "--schema", abs_dbpath, args.target] result = subprocess.run( diff_cmd, shell=False, @@ -407,7 +407,7 @@ def test_versiondb_altdb( versiondb.migrate_database(args.direction, args, timestamp="testing") # Run diff - diff_cmd = ["sqldiff", "--schema", abs_dbpath, args.target] + diff_cmd = [pyani_config.SQLDIFF_DEFAULT, "--schema", abs_dbpath, args.target] result = subprocess.run( diff_cmd, shell=False, From 2ee06367fe7e6098bd6153d44073292e3d80035e Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Mar 2022 11:22:14 +0100 Subject: [PATCH 77/91] Remove code from earlier versions of tests --- .../subcmd_versiondb/altdb_minus_head.diff | 26 ------ .../downgrade_minus_base.diff | 21 ----- .../subcmd_versiondb/upgrade_minus_head.diff | 26 ------ tests/test_versiondb.py | 87 +------------------ 4 files changed, 3 insertions(+), 157 deletions(-) delete mode 100644 tests/test_targets/subcmd_versiondb/altdb_minus_head.diff delete mode 100644 tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff delete mode 100644 tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff diff --git a/tests/test_targets/subcmd_versiondb/altdb_minus_head.diff b/tests/test_targets/subcmd_versiondb/altdb_minus_head.diff deleted file mode 100644 index 1e7eb865..00000000 --- a/tests/test_targets/subcmd_versiondb/altdb_minus_head.diff +++ /dev/null @@ -1,26 +0,0 @@ - > CREATE TABLE alembic_version ( - > version_num VARCHAR(32) NOT NULL, - > CONSTRAINT alembic_version_pkc PRIMARY KEY (version_n - > ); - > INSERT INTO alembic_version VALUES('92f7f6b1626e'); -CREATE TABLE runs_comparisons ( | CREATE TABLE comparisons ( - comparison_id INTEGER, < - run_id INTEGER, < - FOREIGN KEY(comparison_id) REFERENCES comparisons (co < - FOREIGN KEY(run_id) REFERENCES runs (run_id) < -); < -CREATE TABLE alembic_version ( < - version_num VARCHAR(32) NOT NULL, < - CONSTRAINT alembic_version_pkc PRIMARY KEY (version_n < -); < -INSERT INTO alembic_version VALUES('92f7f6b1626e'); < -CREATE TABLE IF NOT EXISTS "comparisons" ( < - CHECK (maxmatch IN (0, 1)), < - > FOREIGN KEY(query_id) REFERENCES genomes (genome_id), - FOREIGN KEY(query_id) REFERENCES genomes (genome_id) | CHECK (maxmatch IN (0, 1)) - > ); - > CREATE TABLE runs_comparisons ( - > comparison_id INTEGER, - > run_id INTEGER, - > FOREIGN KEY(comparison_id) REFERENCES comparisons (co - > FOREIGN KEY(run_id) REFERENCES runs (run_id) diff --git a/tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff b/tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff deleted file mode 100644 index f219143c..00000000 --- a/tests/test_targets/subcmd_versiondb/downgrade_minus_base.diff +++ /dev/null @@ -1,21 +0,0 @@ -CREATE TABLE alembic_version ( < - version_num VARCHAR(32) NOT NULL, < - CONSTRAINT alembic_version_pkc PRIMARY KEY (version_n < -); < -CREATE TABLE runs_comparisons ( | CREATE TABLE comparisons ( - comparison_id INTEGER, < - run_id INTEGER, < - FOREIGN KEY(comparison_id) REFERENCES comparisons (co < - FOREIGN KEY(run_id) REFERENCES runs (run_id) < -); < -CREATE TABLE IF NOT EXISTS "comparisons" ( < - CHECK (maxmatch IN (0, 1)), | UNIQUE (query_id, subject_id, program, version, frags - CONSTRAINT base_reqs UNIQUE (query_id, subject_id, pr < - FOREIGN KEY(subject_id) REFERENCES genomes (genome_id | FOREIGN KEY(subject_id) REFERENCES genomes (genome_id - > CHECK (maxmatch IN (0, 1)) - > ); - > CREATE TABLE runs_comparisons ( - > comparison_id INTEGER, - > run_id INTEGER, - > FOREIGN KEY(comparison_id) REFERENCES comparisons (co - > FOREIGN KEY(run_id) REFERENCES runs (run_id) diff --git a/tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff b/tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff deleted file mode 100644 index 6143b581..00000000 --- a/tests/test_targets/subcmd_versiondb/upgrade_minus_head.diff +++ /dev/null @@ -1,26 +0,0 @@ - > CREATE TABLE alembic_version ( - > version_num VARCHAR(32) NOT NULL, - > CONSTRAINT alembic_version_pkc PRIMARY KEY (version_n - > ); - > INSERT INTO alembic_version VALUES('92f7f6b1626e'); -CREATE TABLE runs_comparisons ( | CREATE TABLE comparisons ( - comparison_id INTEGER, < - run_id INTEGER, < - FOREIGN KEY(comparison_id) REFERENCES comparisons (co < - FOREIGN KEY(run_id) REFERENCES runs (run_id) < -); < -CREATE TABLE alembic_version ( < - version_num VARCHAR(32) NOT NULL, < - CONSTRAINT alembic_version_pkc PRIMARY KEY (version_n < -); < -INSERT INTO alembic_version VALUES('92f7f6b1626e'); < -CREATE TABLE IF NOT EXISTS "comparisons" ( < - CHECK (maxmatch IN (0, 1)), < - FOREIGN KEY(subject_id) REFERENCES genomes (genome_id | FOREIGN KEY(subject_id) REFERENCES genomes (genome_id - > CHECK (maxmatch IN (0, 1)) - > ); - > CREATE TABLE runs_comparisons ( - > comparison_id INTEGER, - > run_id INTEGER, - > FOREIGN KEY(comparison_id) REFERENCES comparisons (co - > FOREIGN KEY(run_id) REFERENCES runs (run_id) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index d90b8c10..6caeed57 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -112,17 +112,6 @@ def altdb_namespace(generic_versiondb_namespace): ) -@pytest.fixture -def alt_config_namespace(generic_versiondb_namespace): - """Namespace for pyani versiondb -c alt_config.""" - return modify_namespace( - generic_versiondb_namespace, - dbpath="pyanidb_alt_config", - dbname="pyanidb_altdb", - alembic_config="alt_alembic_config.ini", - ) - - @pytest.fixture def dry_up_namespace(generic_versiondb_namespace): """Namespace for pyani versiondb dry-run upgrade.""" @@ -149,10 +138,9 @@ def dry_down_namespace(generic_versiondb_namespace): def expected_diffs(namespace): """Expected (acceptable) differences between output and target databases.""" return { - "upgrade": b"2a3,7\n> CREATE TABLE alembic_version (\n> \tversion_num VARCHAR(32) NOT NULL, \n> \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n> );\n> INSERT INTO alembic_version VALUES('92f7f6b1626e');\n54,65c59\n< CREATE TABLE runs_comparisons (\n< \tcomparison_id INTEGER, \n< \trun_id INTEGER, \n< \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n< \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n< );\n< CREATE TABLE alembic_version (\n< \tversion_num VARCHAR(32) NOT NULL, \n< \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n< );\n< INSERT INTO alembic_version VALUES('92f7f6b1626e');\n< CREATE TABLE IF NOT EXISTS \"comparisons\" (\n---\n> CREATE TABLE comparisons (\n81d74\n< \tCHECK (maxmatch IN (0, 1)), \n85a79,84\n> CREATE TABLE runs_comparisons (\n> \tcomparison_id INTEGER, \n> \trun_id INTEGER, \n> \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n> \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n> );\n", - "downgrade": b'3,6d2\n< CREATE TABLE alembic_version (\n< \tversion_num VARCHAR(32) NOT NULL, \n< \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n< );\n58,64c54\n< CREATE TABLE runs_comparisons (\n< \tcomparison_id INTEGER, \n< \trun_id INTEGER, \n< \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n< \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n< );\n< CREATE TABLE IF NOT EXISTS "comparisons" (\n---\n> CREATE TABLE comparisons (\n78,79c68\n< \tCHECK (maxmatch IN (0, 1)), \n< \tCONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n---\n> \tUNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n82a72,77\n> CREATE TABLE runs_comparisons (\n> \tcomparison_id INTEGER, \n> \trun_id INTEGER, \n> \tFOREIGN KEY(comparison_id) REFERENCES comparisons (comparison_id), \n> \tFOREIGN KEY(run_id) REFERENCES runs (run_id)\n> );\n', - "altdb": b"2a3,7\n> CREATE TABLE alembic_version (\n> \tversion_num VARCHAR(32) NOT NULL, \n> \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n> );\n> INSERT INTO alembic_version VALUES('92f7f6b1626e');\n66a72,73\n> \tkmersize INTEGER, \n> \tminmatch FLOAT, \n68c75\n< \tCONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n---\n> \tCONSTRAINT fastani_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch, kmersize, minmatch), \n", - "alt_config": b"2a3,7\n> CREATE TABLE alembic_version (\n> \tversion_num VARCHAR(32) NOT NULL, \n> \tCONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)\n> );\n> INSERT INTO alembic_version VALUES('92f7f6b1626e');\n66a72,73\n> \tkmersize INTEGER, \n> \tminmatch FLOAT, \n68c75\n< \tCONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch), \n---\n> \tCONSTRAINT fastani_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch, kmersize, minmatch), \n", + "upgrade": "", + "downgrade": "DROP TABLE alembic_version;\n", + "altdb": "", }.get(namespace, None) @@ -205,9 +193,6 @@ def name_base_reqs(startdb_dump): def cleanup(abs_dbpath, test, dir_versiondb_out, args): """Remove files created for test.""" - # db = abs_dbpath.name - # target = args.target.name - # start = args.start.name dir_versiondb_out.mkdir(exist_ok=True) Path(f"{dir_versiondb_out}/{test}").mkdir(exist_ok=True) @@ -216,14 +201,10 @@ def cleanup(abs_dbpath, test, dir_versiondb_out, args): # if the file already exists, so this is a two-step process # Copy files to new location shutil.copy(abs_dbpath, dir_versiondb_out / test) - # shutil.copy(f"{abs_dbpath}.sql", dir_versiondb_out / test) - # shutil.copy(f"{args.target}.sql", dir_versiondb_out / test) shutil.copy(f"{args.start}.sql", dir_versiondb_out / test) # Remove old files os.remove(abs_dbpath) - # os.remove(f"{abs_dbpath}.sql") - # os.remove(f"{args.target}.sql") os.remove(f"{args.start}.sql") # This file is not generated in the downgrade test @@ -304,11 +285,6 @@ def test_versiondb_upgrade( # Run test migration versiondb.migrate_database(args.direction, args, timestamp="testing") - # Dump altered and target databases - # enddb_dump = dumpdb(abs_dbpath) - - # targetdb_dump = dumpdb(args.target) - # Run diff diff_cmd = [pyani_config.SQLDIFF_DEFAULT, "--schema", abs_dbpath, args.target] result = subprocess.run( @@ -424,60 +400,3 @@ def test_versiondb_altdb( cleanup(abs_dbpath, "altdb", dir_versiondb_out, args) assert result.stdout.decode() == expected_diff - - -# Test alt_config result -@pytest.mark.skip(reason="May no be needed.") -def test_versiondb_alt_config( - alt_config_namespace, dir_versiondb_in, dir_versiondb_out -): - """Test upgrade of database using an alternate config file.""" - # Test setup - # Retrieve test namespace and - # Set environment variables and resolve absolute path of database - args = alt_config_namespace - setenv(dir_versiondb_in, args.dbpath) - abs_dbpath = os.environ.get("PYANI_DATABASE") - - # Create dump file - startdb_dump = dumpdb(args.start) - name_base_reqs(startdb_dump) - - # Run `sqlite3 -init - init_cmd = [pyani_config.SQLITE_DEFAULT, abs_dbpath] - subprocess.run( - init_cmd, - stdin=open(startdb_dump), - shell=False, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - # Run test migration - versiondb.migrate_database(args.direction, args, timestamp="testing") - - # Dump altered and target databases - enddb_dump = dumpdb(abs_dbpath) - targetdb_dump = dumpdb(args.target) - - # Run diff - diff_cmd = ["diff", "-y", "--suppress-common-lines", enddb_dump, targetdb_dump] - result = subprocess.run( - diff_cmd, - shell=False, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - expected_diff = "".join( - open( - "/Users/baileythegreen/Software/pyani/tests/fixtures/versiondb/upgrade_minus_head.diff", - "r", - ).readlines() - ) - sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") - sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") - # Move files - cleanup(abs_dbpath, "alt_config", dir_versiondb_out, args) - - assert result.stdout.decode() == expected_diff From cc59b981b86dc323a91fe22322ee4612c719a873 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Mar 2022 11:27:38 +0100 Subject: [PATCH 78/91] Enhance comments --- tests/test_versiondb.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 6caeed57..9ad81070 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -144,7 +144,7 @@ def expected_diffs(namespace): }.get(namespace, None) -# Create database dump +# Create database dump—a version that can be edited using sed def dumpdb(abs_dbpath): """Dump contents of database to a plain-text file.""" @@ -168,6 +168,7 @@ def name_base_reqs(startdb_dump): # Edit .dump file so that the unique constraint is named # This is required in order to subsequently modify it + # In-place usage differs on macOs vs Linux if platform.system() == "Darwin": sed_cmd = [ "sed", @@ -400,3 +401,6 @@ def test_versiondb_altdb( cleanup(abs_dbpath, "altdb", dir_versiondb_out, args) assert result.stdout.decode() == expected_diff + + +# Dry-run tests still to be done From 67ff5f9f3d1336c26b07f2455806c43eebbedfc4 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Mar 2022 11:29:29 +0100 Subject: [PATCH 79/91] Remove structures from `conftest.py` that are not being used after all --- tests/conftest.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f5003fd5..63486f18 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -66,11 +66,6 @@ FIXTUREPATH = TESTSPATH / "fixtures" -def pytest_configure(): - pytest.testspath = Path(__file__).parents[0] - pytest.fixturepath = pytest.testspath / "fixtures" - - # Convenience structs to emulate returned objects class MockGenome(NamedTuple): """Mock genome object.""" @@ -159,12 +154,6 @@ def dir_versiondb_out(): return TESTSPATH / "test_output/subcmd_versiondb" -@pytest.fixture -def dir_versiondb_targets(): - """Target diff files for versiondb tests.""" - return TESTSPATH / "test_targets/subcmd_versiondb" - - @pytest.fixture def dir_seq(): """Sequence files for tests.""" From 6c76bfb81dd22ec88f8dab2b033ef306ec727a8b Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Mar 2022 11:40:38 +0100 Subject: [PATCH 80/91] Add note about missing `--dry-run` tests --- docs/run_versiondb.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/run_versiondb.rst b/docs/run_versiondb.rst index 0f694bfb..5cbcbb46 100644 --- a/docs/run_versiondb.rst +++ b/docs/run_versiondb.rst @@ -54,6 +54,10 @@ However, there may be cases where it is necessary to downgrade a database to a s Performing a dry run ~~~~~~~~~~~~~~~~~~~ +.. NOTE:: + + This option has been implemented, but tests for it are still in the works. + This following command creates an SQL file, ``./test_database.downgrade.YYYY-MM-DD_HH-MM-SS.sql``, (in the same directory as ``./test_database``) containing the raw SQL that would produce the necessary changes to the database to migrate it to the specified version (in this case, downgrading it to ``base``): .. code-block:: bash From 40470a4d6d2dccb93ad5a1f9d75e1e266a0140f8 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Mar 2022 11:43:08 +0100 Subject: [PATCH 81/91] Fix typos --- docs/run_fastani.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/run_fastani.rst b/docs/run_fastani.rst index a1c73537..42a74a89 100644 --- a/docs/run_fastani.rst +++ b/docs/run_fastani.rst @@ -37,7 +37,7 @@ The basic form of the command is: pyani fastani -i -o -This instructs ``pyani`` to perform fast ANI on the genome FASTA files in ````, and write any output files to ````. For example, the following command performs fastANI on genomes in the directory ``genomes`` and writes output to a new directory ``genoems_fastANI``: +This instructs ``pyani`` to perform fast ANI on the genome FASTA files in ````, and write any output files to ````. For example, the following command performs fastANI on genomes in the directory ``genomes`` and writes output to a new directory ``genomes_fastANI``: .. code-block:: bash @@ -49,7 +49,7 @@ This instructs ``pyani`` to perform fast ANI on the genome FASTA files in `` Date: Mon, 28 Mar 2022 11:49:39 +0100 Subject: [PATCH 82/91] `sqldiff` is needed for `pyani versiondb`, but can't install from `conda` --- requirements-pip.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-pip.txt b/requirements-pip.txt index c561085a..04fff056 100644 --- a/requirements-pip.txt +++ b/requirements-pip.txt @@ -1,3 +1,4 @@ pre-commit pytest-ordering sphinx-rtd-theme +sqldiff From 78d6f8a7b5e301fe3f56dcc38f541ffeb663440c Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Mon, 28 Mar 2022 12:54:28 +0100 Subject: [PATCH 83/91] Remove references to non-existent fixtures These should have been removed in 2ee06367f, but were missed --- tests/test_versiondb.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index 9ad81070..a3eefde2 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -259,7 +259,6 @@ def test_versiondb_upgrade( generic_versiondb_namespace, dir_versiondb_in, dir_versiondb_out, - dir_versiondb_targets, ): """Test upgrade of database.""" # Test setup @@ -306,9 +305,7 @@ def test_versiondb_upgrade( assert result.stdout.decode() == expected_diff -def test_versiondb_downgrade( - downgrade_namespace, dir_versiondb_in, dir_versiondb_out, dir_versiondb_targets -): +def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiondb_out): """Test downgrade of database.""" # Test setup # Retrieve test namespace and @@ -354,9 +351,7 @@ def test_versiondb_downgrade( # Test alternate dbname -def test_versiondb_altdb( - altdb_namespace, dir_versiondb_in, dir_versiondb_out, dir_versiondb_targets -): +def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): """Test upgrade of database using an alternate database name and config file, such as in a multidb situation.""" # Test setup # Retrieve test namespace and From 7ddd894330503b3aed8ca6cbbf88337fd7d2bca2 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Tue, 29 Mar 2022 12:45:31 +0100 Subject: [PATCH 84/91] Add `alembic` directories to package data for non-dev use cases --- setup.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ee8e3ef6..26177663 100644 --- a/setup.py +++ b/setup.py @@ -99,7 +99,14 @@ ] }, packages=setuptools.find_packages(), - package_data={"pyani": ["tests/test_JSpecies/*.tab"]}, + package_data={ + "pyani": [ + "tests/test_JSpecies/*.tab", + "alembic.ini", + "alembic/*", + "alembic/**/*", + ] + }, include_package_date=True, install_requires=[ "biopython", From 063f1e91c8d4665f0c302900f3f50f951c5e990f Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 31 Mar 2022 12:15:18 +0100 Subject: [PATCH 85/91] Update version of `sqlite` to 3.37.0 and abstract version into a variable --- .circleci/config.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b23a46a..cf7bb446 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,18 +71,19 @@ jobs: - run: name: install sqlite3 command: | - wget https://github.com/sqlite/sqlite/archive/refs/tags/version-3.36.0.tar.gz - tar xzf version-3.36.0.tar.gz ;# Unpack the source tree into "sqlite" + version=verion-3.37.0 + wget https://github.com/sqlite/sqlite/archive/refs/tags/${version}.tar.gz + tar xzf ${version}.tar.gz ;# Unpack the source tree into "sqlite" mkdir bld ;# Build will occur in a sibling directory cd bld ;# Change to the build directory - ../sqlite-version-3.36.0/configure ;# Run the configure script + ../sqlite-${version}/configure ;# Run the configure script make ;# Run the makefile. make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) echo "export PATH=~/repo/bld:$PATH" >> $BASH_ENV source $BASH_ENV echo $PATH - + - run: name: install dependencies From 9b7005e406ff13870976d5f2163077a50c1cf2a0 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 31 Mar 2022 12:22:18 +0100 Subject: [PATCH 86/91] Add explanations surrounding `alembic` tests and when they run/dependencies --- docs/testing.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/testing.rst b/docs/testing.rst index ee40949b..da69b22e 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -9,6 +9,11 @@ We are currently writing tests formatted for the `pytest`_ package, for testing .. WARNING:: Some tests are still targeted at `nosetests`_, which is in maintenance mode and, although we are still in transition, our plan is to change the test framework completely to use `pytest`_. +.. Note:: + Tests for database migration (upgrades and downgrades) using ``alembic`` require ``sqldiff``, one of the ``sqlite-tools`` that is available as part of the compiled binaries, or source code, from `sqlite downloads`_. This tool is not available through package managers, and is *only* required to run the related tests locally, not to have the database migration functionality ``alembic`` affords ``pyani`` users; as such, it has been installed as part of our maintenance testing on CircleCI, and by our devs, but it does not come as part of the ``pyani`` installation. The related tests will be skipped if ``sqldiff`` is not present on the system. + + Should you wish to run those tests locally, installation details can be found below under **Installing sqldiff**. + ------------------------ Test directory structure ------------------------ @@ -71,7 +76,36 @@ And to test only "multiple command generation" we can issue the following: nosetests -v tests/test_anim.py:TestNUCmerCmdline.test_multi_cmd_generation +------------------ +Installing sqldiff +------------------ + +The CircleCI testing setup for ``pyani`` uses ``sqlite 3.37.0``, which comes with ``sqldiff``. This version is available to download as a zip file from GitHub: `sqlite 3.37.0 download`_. + +Installation instructions are available in the `sqlite Readme`_, but will need to be tweaked for the older archive. For example, in CircleCI (a Unix environment), we do: + +.. code-block:: bash + + version=verion-3.37.0 ;# Pull version number into a variable + wget https://github.com/sqlite/sqlite/archive/refs/tags/${version}.tar.gz + tar xzf ${version}.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite-${version}/configure ;# Run the configure script + make ;# Run the makefile. + make sqlite3.c ;# Build the "amalgamation" source file + make test ;# Run some tests (requires Tcl) + echo "export PATH=~/repo/bld:$PATH" >> $BASH_ENV ;# Add to shell configuration + source $BASH_ENV ;# Effect changes + +.. Note:: + This will also place ``sqlite 3.37.0`` into your ``$PATH``. ``pyani`` installs ``sqlite3`` via a package manager, already. If having two copies is not desirable, you may wish to copy the ``sqldiff`` binary itself into somewhere already on your path, instead. .. _nosetests: https://nose.readthedocs.io/en/latest/ .. _pytest: https://docs.pytest.org/en/latest/ +.. _sqlite downloads:: https://www.sqlite.org/download.html + + +.. _sqlite 3.37.0 download:: https://github.com/sqlite/sqlite/tags +.. _sqlite Readme:: https://github.com/sqlite/sqlite From 38d6e586cec04de2dccb5544c1fd375bc144d93a Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Thu, 31 Mar 2022 12:31:10 +0100 Subject: [PATCH 87/91] Updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf7bb446..1e0c679c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,7 +71,7 @@ jobs: - run: name: install sqlite3 command: | - version=verion-3.37.0 + version=version-3.37.0 wget https://github.com/sqlite/sqlite/archive/refs/tags/${version}.tar.gz tar xzf ${version}.tar.gz ;# Unpack the source tree into "sqlite" mkdir bld ;# Build will occur in a sibling directory From cefb5775a80ee7053fe6488db4508db24f2ac967 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 31 Mar 2022 12:32:14 +0100 Subject: [PATCH 88/91] Fix typo in code explanation --- docs/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing.rst b/docs/testing.rst index da69b22e..995b4c2d 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -86,7 +86,7 @@ Installation instructions are available in the `sqlite Readme`_, but will need t .. code-block:: bash - version=verion-3.37.0 ;# Pull version number into a variable + version=version-3.37.0 ;# Pull version number into a variable wget https://github.com/sqlite/sqlite/archive/refs/tags/${version}.tar.gz tar xzf ${version}.tar.gz ;# Unpack the source tree into "sqlite" mkdir bld ;# Build will occur in a sibling directory From 6847534757e2823ec415a28a2e9b504b0475021e Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Thu, 31 Mar 2022 13:34:58 +0100 Subject: [PATCH 89/91] Skip tests requiring `sqldiff` when it is not installed --- tests/test_versiondb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index a3eefde2..ca10dd51 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -255,6 +255,7 @@ def test_alembic_cmdline_generation( # Test upgrade +@pytest.mark.skip_if_exe_missing("sqldiff") def test_versiondb_upgrade( generic_versiondb_namespace, dir_versiondb_in, @@ -305,6 +306,7 @@ def test_versiondb_upgrade( assert result.stdout.decode() == expected_diff +@pytest.mark.skip_if_exe_missing("sqldiff") def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiondb_out): """Test downgrade of database.""" # Test setup @@ -351,6 +353,7 @@ def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiond # Test alternate dbname +@pytest.mark.skip_if_exe_missing("sqldiff") def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): """Test upgrade of database using an alternate database name and config file, such as in a multidb situation.""" # Test setup From 93cc99f055c6bd2031f3a38139d9f2098581a89c Mon Sep 17 00:00:00 2001 From: Bailey Harrington Date: Tue, 21 Jun 2022 20:37:03 +0100 Subject: [PATCH 90/91] Updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c6653c78..1ca7baf2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,7 +81,7 @@ jobs: make ;# Run the makefile. make sqlite3.c ;# Build the "amalgamation" source file make test ;# Run some tests (requires Tcl) - echo "export PATH=~/repo/bld:$PATH" >> $BASH_ENV + echo 'export PATH=~/repo/bld:$PATH' >> $BASH_ENV source $BASH_ENV echo $PATH From 582bdc4001c82242a1ab84d21a54e1b61eaab710 Mon Sep 17 00:00:00 2001 From: baileythegreen Date: Fri, 29 Jul 2022 15:20:52 +0100 Subject: [PATCH 91/91] Make changes to tests as discussed in Issue #405 --- pyani/versiondb.py | 9 ++++++++- tests/test_versiondb.py | 35 +++++++++++++++++++++++------------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/pyani/versiondb.py b/pyani/versiondb.py index 3deef4e0..91e0ce23 100644 --- a/pyani/versiondb.py +++ b/pyani/versiondb.py @@ -31,13 +31,20 @@ def get_version(alembic_exe: Path = pyani_config.ALEMBIC_DEFAULT) -> str: The following circumstances are explicitly reported as strings: + - a value of None given for the executable - no executable at passed path - non-executable file at passed path (this includes cases where the user doesn't have execute permissions on the file) - no version info returned """ try: - alembic_path = Path(shutil.which(alembic_exe)) # type:ignore + # Returns a TypeError if `alembic_exe` is None + try: + alembic_path = shutil.which(alembic_exe) # type:ignore + except TypeError: + return f"expected path to alembic executable; received {alembic_exe}" + # Returns a TypeError if `alembic_path` is not on the PATH + alembic_path = Path(alembic_path) except TypeError: return f"{alembic_exe} is not found in $PATH" diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py index ca10dd51..d0e643e1 100644 --- a/tests/test_versiondb.py +++ b/tests/test_versiondb.py @@ -41,33 +41,44 @@ def test_get_version_nonetype(): """Test behaviour when no location for the executable is given.""" test_file_0 = None - assert versiondb.get_version(test_file_0) == f"{test_file_0} is not found in $PATH" + assert ( + versiondb.get_version(test_file_0) + == f"expected path to alembic executable; received {test_file_0}" + ) + + +# Test case 1: no such file exists +def test_get_version_random_string(): + """Test behaviour when the given 'file' is not one.""" + test_file_1 = "string" + assert versiondb.get_version(test_file_1) == f"{test_file_1} is not found in $PATH" -# Test case 1: there is no executable + +# Test case 2: there is no executable def test_get_version_no_exe(executable_missing, monkeypatch): """Test behaviour when there is no file at the specified executable location.""" - test_file_1 = Path("/non/existent/alembic") - assert versiondb.get_version(test_file_1) == f"No alembic at {test_file_1}" + test_file_2 = Path("/non/existent/alembic") + assert versiondb.get_version(test_file_2) == f"No alembic at {test_file_2}" -# Test case 2: there is a file, but it is not executable +# Test case 3: there is a file, but it is not executable def test_get_version_exe_not_executable(executable_not_executable, monkeypatch): """Test behaviour when the file at the executable location is not executable.""" - test_file_2 = Path("/non/executable/alembic") + test_file_3 = Path("/non/executable/alembic") assert ( - versiondb.get_version(test_file_2) - == f"alembic exists at {test_file_2} but not executable" + versiondb.get_version(test_file_3) + == f"alembic exists at {test_file_3} but not executable" ) -# Test case 3: there is an executable file, but the version can't be retrieved +# Test case 4: there is an executable file, but the version can't be retrieved def test_get_version_exe_no_version(executable_without_version, monkeypatch): """Test behaviour when the version for the executable can not be retrieved.""" - test_file_3 = Path("/missing/version/alembic") + test_file_4 = Path("/missing/version/alembic") assert ( - versiondb.get_version(test_file_3) - == f"alembic exists at {test_file_3} but could not retrieve version" + versiondb.get_version(test_file_4) + == f"alembic exists at {test_file_4} but could not retrieve version" )