Source code for bioconda_utils.build

"""
Package Builder
"""

import subprocess as sp
from collections import defaultdict, namedtuple
import os
import logging
import itertools

from typing import List

# TODO: UnsatisfiableError is not yet in exports for conda 4.5.4
# from conda.exports import UnsatisfiableError
from conda.exceptions import UnsatisfiableError
import networkx as nx
import pandas

from . import utils
from . import docker_utils
from . import pkg_test
from . import upload
from . import lint
from . import graph

logger = logging.getLogger(__name__)


#: Result tuple for builds comprising success status and list of docker images
BuildResult = namedtuple("BuildResult", ["success", "mulled_images"])


[docs]def conda_build_purge() -> None: """Calls conda build purge and optionally conda clean ``conda clean --all`` is called if we haveless than 300 MB free space on the current disk. """ utils.run(["conda", "build", "purge"], mask=False) free_mb = utils.get_free_space() if free_mb < 300: logger.info("CLEANING UP PACKAGE CACHE (free space: %iMB).", free_mb) utils.run(["conda", "clean", "--all"], mask=False) logger.info("CLEANED UP PACKAGE CACHE (free space: %iMB).", utils.get_free_space())
[docs]def build(recipe: str, pkg_paths: List[str] = None, testonly: bool = False, mulled_test: bool = True, channels: List[str] = None, docker_builder: docker_utils.RecipeBuilder = None, raise_error: bool = False, linter=None) -> BuildResult: """ Build a single recipe for a single env Arguments: recipe: Path to recipe pkg_paths: List of paths to expected packages testonly: Only run the tests described in the meta.yaml mulled_test: Run tests in minimal docker container channels: Channels to include via the ``--channel`` argument to conda-build. Higher priority channels should come first. docker_builder : docker_utils.RecipeBuilder object Use this docker builder to build the recipe, copying over the built recipe to the host's conda-bld directory. raise_error: Instead of returning a failed build result, raise the error instead. Used for testing. linter: Linter to use for checking recipes """ if linter: logger.info('Linting recipe %s', recipe) linter.clear_messages() if linter.lint([recipe]): logger.error('\n\nThe recipe %s failed linting. See ' 'https://bioconda.github.io/linting.html for details:\n\n%s\n', recipe, linter.get_report()) return BuildResult(False, None) logger.info("Lint checks passed") # Copy env allowing only whitelisted vars whitelisted_env = { k: str(v) for k, v in os.environ.items() if utils.allowed_env_var(k, docker_builder is not None) } logger.info("BUILD START %s", recipe) args = ['--override-channels'] if testonly: args += ["--test"] else: args += ["--no-anaconda-upload"] for channel in channels or ['local']: args += ['-c', channel] logger.debug('Build and Channel Args: %s', args) # Even though there may be variants of the recipe that will be built, we # will only be checking attributes that are independent of variants (pkg # name, version, noarch, whether or not an extended container was used) meta = utils.load_first_metadata(recipe, finalize=False) is_noarch = bool(meta.get_value('build/noarch', default=False)) use_base_image = meta.get_value('extra/container', {}).get('extended-base', False) base_image = 'bioconda/extended-base-image' if use_base_image else None try: if docker_builder is not None: docker_builder.build_recipe(recipe_dir=os.path.abspath(recipe), build_args=' '.join(args), env=whitelisted_env, noarch=is_noarch) # Use presence of expected packages to check for success for pkg_path in pkg_paths: if not os.path.exists(pkg_path): logger.error( "BUILD FAILED: the built package %s " "cannot be found", pkg_path) return BuildResult(False, None) else: conda_build_cmd = [utils.bin_for('conda'), 'build'] # - Temporarily reset os.environ to avoid leaking env vars # - Also pass filtered env to run() # - Point conda-build to meta.yaml, to avoid building subdirs with utils.sandboxed_env(whitelisted_env): cmd = conda_build_cmd + args for config_file in utils.get_conda_build_config_files(): cmd += [config_file.arg, config_file.path] cmd += [os.path.join(recipe, 'meta.yaml')] with utils.Progress(): utils.run(cmd, env=os.environ, mask=False) logger.info('BUILD SUCCESS %s', ' '.join(os.path.basename(p) for p in pkg_paths)) except (docker_utils.DockerCalledProcessError, sp.CalledProcessError) as exc: logger.error('BUILD FAILED %s', recipe) if raise_error: raise exc return BuildResult(False, None) if mulled_test: logger.info('TEST START via mulled-build %s', recipe) mulled_images = [] for pkg_path in pkg_paths: try: pkg_test.test_package(pkg_path, base_image=base_image) except sp.CalledProcessError: logger.error('TEST FAILED: %s', recipe) return BuildResult(False, None) logger.info("TEST SUCCESS %s", recipe) mulled_images.append(pkg_test.get_image_name(pkg_path)) return BuildResult(True, mulled_images) return BuildResult(True, None)
def remove_cycles(dag, name2recipes, failed, skip_dependent): nodes_in_cycles = set() for cycle in list(nx.simple_cycles(dag)): logger.error('BUILD ERROR: dependency cycle found: %s', cycle) nodes_in_cycles.update(cycle) for name in sorted(nodes_in_cycles): cycle_fail_recipes = sorted(name2recipes[name]) logger.error('BUILD ERROR: cannot build recipes for %s since ' 'it cyclically depends on other packages in the ' 'current build job. Failed recipes: %s', name, cycle_fail_recipes) failed.extend(cycle_fail_recipes) for node in nx.algorithms.descendants(dag, name): if node not in nodes_in_cycles: skip_dependent[node].extend(cycle_fail_recipes) return dag.subgraph(name for name in dag if name not in nodes_in_cycles) def get_subdags(dag, n_workers, worker_offset): if n_workers > 1 and worker_offset >= n_workers: raise ValueError( "n-workers is less than the worker-offset given! " "Either decrease --n-workers or decrease --worker-offset!") # Get connected subdags and sort by nodes if n_workers > 1: root_nodes = sorted([k for k, v in dag.in_degree().items() if v == 0]) nodes = set() found = set() for idx, root_node in enumerate(root_nodes): # Flatten the nested list children = itertools.chain(*nx.dfs_successors(dag, root_node).values()) # This is the only obvious way of ensuring that all nodes are included # in exactly 1 subgraph found.add(root_node) if idx % n_workers == worker_offset: nodes.add(root_node) for child in children: if child not in found: nodes.add(child) found.add(child) else: for child in children: found.add(child) subdags = dag.subgraph(list(nodes)) logger.info("Building and testing sub-DAGs %i in each group of %i, which is %i packages", worker_offset, n_workers, len(subdags.nodes())) else: subdags = dag return subdags
[docs]def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], mulled_test: bool = True, testonly: bool = False, force: bool = False, docker_builder: docker_utils.RecipeBuilder = None, label: str = None, anaconda_upload: bool = False, mulled_upload_target=None, check_channels: List[str] = None, do_lint: bool = None, lint_exclude: List[str] = None, n_workers: int = 1, worker_offset: int = 0, keep_old_work: bool = False): """ Build one or many bioconda packages. Arguments: recipe_folder: Directory containing possibly many, and possibly nested, recipes. config_path: Path to config file packages: Glob indicating which packages should be considered. Note that packages matching the glob will still be filtered out by any blacklists specified in the config. mulled_test: If true, test the package in a minimal container. testonly: If true, only run test. force: If true, build the recipe even though it would otherwise be filtered out. docker_builder: If specified, use to build all recipes label: If specified, use to label uploaded packages on anaconda. Default is "main" label. anaconda_upload: If true, upload the package(s) to anaconda.org. mulled_upload_target: If specified, upload the mulled docker image to the given target on quay.io. check_channels: Channels to check to see if packages already exist in them. Defaults to every channel in the config file except "defaults". do_lint: Whether to run linter lint_exclude: List of linting functions to exclude. n_workers: The number of parallel instances of bioconda-utils being run. The sub-DAGs are then split into groups of n_workers size. worker_offset: If n_workers is >1, then every worker_offset within a given group of sub-DAGs will be processed. keep_old_work: Do not remove anything from environment, even after successful build and test. """ if not recipes: logger.info("Nothing to be done.") return True config = utils.load_config(config_path) blacklist = utils.get_blacklist(config, recipe_folder) # get channels to check if check_channels is None: if config['channels']: check_channels = [c for c in config['channels'] if c != "defaults"] else: check_channels = [] # setup linting if do_lint: always_exclude = ('build_number_needs_bump',) if not lint_exclude: lint_exclude = always_exclude else: lint_exclude = tuple(set(lint_exclude) | set(always_exclude)) linter = lint.Linter(config, recipe_folder, lint_exclude) else: linter = None failed = [] dag, name2recipes = graph.build(recipes, config=config_path, blacklist=blacklist) if not dag: logger.info("Nothing to be done.") return True skip_dependent = defaultdict(list) dag = remove_cycles(dag, name2recipes, failed, skip_dependent) subdag = get_subdags(dag, n_workers, worker_offset) if not subdag: logger.info("Nothing to be done.") return True logger.info("%i recipes to build and test: \n%s", len(subdag), "\n".join(subdag.nodes())) recipe2name = {} for name, recipe_list in name2recipes.items(): for recipe in recipe_list: recipe2name[recipe] = name recipes = [(recipe, recipe2name[recipe]) for package in nx.topological_sort(subdag) for recipe in name2recipes[package]] built_recipes = [] skipped_recipes = [] failed_uploads = [] for recipe, name in recipes: if name in skip_dependent: logger.info('BUILD SKIP: skipping %s because it depends on %s ' 'which had a failed build.', recipe, skip_dependent[name]) skipped_recipes.append(recipe) continue logger.info('Determining expected packages for %s', recipe) try: pkg_paths = utils.get_package_paths(recipe, check_channels, force=force) except utils.DivergentBuildsError as exc: logger.error('BUILD ERROR: packages with divergent build strings in repository ' 'for recipe %s. A build number bump is likely needed: %s', recipe, exc) failed.append(recipe) for pkg in nx.algorithms.descendants(subdag, name): skip_dependent[pkg].append(recipe) continue except UnsatisfiableError as exc: logger.error('BUILD ERROR: could not determine dependencies for recipe %s: %s', recipe, exc) failed.append(recipe) for pkg in nx.algorithms.descendants(subdag, name): skip_dependent[pkg].append(recipe) continue if not pkg_paths: logger.info("Nothing to be done for recipe %s", recipe) continue res = build(recipe=recipe, pkg_paths=pkg_paths, testonly=testonly, mulled_test=mulled_test, channels=config['channels'], docker_builder=docker_builder, linter=linter) if not res.success: failed.append(recipe) for pkg in nx.algorithms.descendants(subdag, name): skip_dependent[pkg].append(recipe) else: built_recipes.append(recipe) if not testonly: if anaconda_upload: for pkg in pkg_paths: if not upload.anaconda_upload(pkg, label=label): failed_uploads.append(pkg) if mulled_upload_target: for img in res.mulled_images: upload.mulled_upload(img, mulled_upload_target) docker_utils.purgeImage(mulled_upload_target, img) # remove traces of the build if not keep_old_work: conda_build_purge() if failed or failed_uploads: logger.error('BUILD SUMMARY: of %s recipes, ' '%s failed and %s were skipped. ' 'Details of recipes and environments follow.', len(recipes), len(failed), len(skipped_recipes)) if built_recipes: logger.error('BUILD SUMMARY: while the entire build failed, ' 'the following recipes were built successfully:\n%s', '\n'.join(built_recipes)) for recipe in failed: logger.error('BUILD SUMMARY: FAILED recipe %s', recipe) for name, dep in skip_dependent.items(): logger.error('BUILD SUMMARY: SKIPPED recipe %s ' 'due to failed dependencies %s', name, dep) if failed_uploads: logger.error('UPLOAD SUMMARY: the following packages failed to upload:\n%s', '\n'.join(failed_uploads)) return False logger.info("BUILD SUMMARY: successfully built %s of %s recipes", len(built_recipes), len(recipes)) return True