Source code for dxpy.app_builder

# Copyright (C) 2013-2016 DNAnexus, Inc.
#
# This file is part of dx-toolkit (DNAnexus platform client libraries).
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
#   use this file except in compliance with the License. You may obtain a copy
#   of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.

'''
App Builder Library
+++++++++++++++++++

Contains utility methods useful for compiling and deploying applets and apps
onto the platform.

You can specify the destination project in the following ways (with the earlier
ones taking precedence):

* Supply the *project* argument to :func:`upload_resources()` or
  :func:`upload_applet()`.
* Supply the 'project' attribute in your ``dxapp.json``.
* Set the ``DX_WORKSPACE_ID`` environment variable (when running in a job context).

You can use the function :func:`get_destination_project` to determine
the effective destination project.

'''

from __future__ import print_function, unicode_literals, division, absolute_import

import os, sys, json, subprocess, tempfile, multiprocessing
import datetime
import gzip
import hashlib
import io
import tarfile
import stat

import dxpy
import dxpy.executable_builder
from . import logger
from .utils import merge
from .utils.printing import fill
from .compat import input
from .cli import INTERACTIVE_CLI

NUM_CORES = multiprocessing.cpu_count()

DX_TOOLKIT_PKGS = ('dx-toolkit',)
DX_TOOLKIT_GIT_URLS = ("git@github.com:dnanexus/dx-toolkit.git",)


[docs]class AppBuilderException(Exception): """ This exception is raised by the methods in this module when app or applet building fails. """ pass
def _validate_applet_spec(applet_spec): if 'runSpec' not in applet_spec: raise AppBuilderException("Required field 'runSpec' not found in dxapp.json") def _validate_app_spec(app_spec): pass def _get_applet_spec(src_dir): applet_spec_file = os.path.join(src_dir, "dxapp.json") with open(applet_spec_file) as fh: applet_spec = json.load(fh) _validate_applet_spec(applet_spec) if 'project' not in applet_spec: applet_spec['project'] = dxpy.WORKSPACE_ID return applet_spec def _get_app_spec(src_dir): app_spec_file = os.path.join(src_dir, "dxapp.json") with open(app_spec_file) as fh: app_spec = json.load(fh) _validate_app_spec(app_spec) return app_spec
[docs]def build(src_dir, parallel_build=True): """ Runs any build scripts that are found in the specified directory. In particular, runs ``./configure`` if it exists, followed by ``make -jN`` if it exists (building with as many parallel tasks as there are CPUs on the system). """ # TODO: use Gentoo or deb buildsystem config_script = os.path.join(src_dir, "configure") if os.path.isfile(config_script) and os.access(config_script, os.X_OK): logger.debug("Running ./configure in {cwd}".format(cwd=os.path.abspath(src_dir))) try: subprocess.check_call([config_script]) except subprocess.CalledProcessError as e: raise AppBuilderException("./configure in target directory failed with exit code %d" % (e.returncode,)) if os.path.isfile(os.path.join(src_dir, "Makefile")) \ or os.path.isfile(os.path.join(src_dir, "makefile")) \ or os.path.isfile(os.path.join(src_dir, "GNUmakefile")): if parallel_build: make_shortcmd = "make -j%d" % (NUM_CORES,) else: make_shortcmd = "make" logger.debug("Building with {make} in {cwd}".format(make=make_shortcmd, cwd=os.path.abspath(src_dir))) try: make_cmd = ["make", "-C", src_dir] if parallel_build: make_cmd.append("-j" + str(NUM_CORES)) subprocess.check_call(make_cmd) except subprocess.CalledProcessError as e: raise AppBuilderException("%s in target directory failed with exit code %d" % (make_shortcmd, e.returncode))
[docs]def get_destination_project(src_dir, project=None): """ :returns: Project ID where applet specified by src_dir would be written :rtype: str Returns the project ID where the applet specified in *src_dir* (or its associated resource bundles) would be written. This returns the same project that would be used by :func:`upload_resources()` and :func:`upload_applet()`, given the same *src_dir* and *project* parameters. """ if project is not None: return project return _get_applet_spec(src_dir)['project']
def _fix_perms(perm_obj): """ :param perm_obj: A permissions object, as given by os.stat() :type perm_obj: integer :returns: A permissions object that is the result of "chmod a+rX" on the given permission object. This is defined to be the permission object bitwise or-ed with all stat.S_IR*, and if the stat.S_IXUSR bit is set, then the permission object should also be returned bitwise or-ed with stat.S_IX* (stat.S_IXUSR not included because it would be redundant). :rtype: integer """ ret_perm = perm_obj | stat.S_IROTH | stat.S_IRGRP | stat.S_IRUSR if ret_perm & stat.S_IXUSR: ret_perm = ret_perm | stat.S_IXGRP | stat.S_IXOTH return ret_perm def _fix_perm_filter(tar_obj): """ :param tar_obj: A TarInfo object to be added to a tar file :tpye tar_obj: tarfile.TarInfo :returns: A TarInfo object with permissions changed (a+rX) :rtype: tarfile.TarInfo """ tar_obj.mode = _fix_perms(tar_obj.mode) return tar_obj
[docs]def upload_resources(src_dir, project=None, folder='/', ensure_upload=False, force_symlinks=False, brief=False, resources_dir=None, worker_resources_subpath=""): """ :param ensure_upload: If True, will bypass checksum of resources directory and upload resources bundle unconditionally; will NOT be able to reuse this bundle in future builds. Else if False, will compute checksum and upload bundle if checksum is different from a previously uploaded bundle's checksum. :type ensure_upload: boolean :param force_symlinks: If true, will bypass the attempt to dereference any non-local symlinks and will unconditionally include the link as-is. Note that this will almost certainly result in a broken link within the resource directory unless you really know what you're doing. :type force_symlinks: boolean :param resources_dir: Directory with resources to be archived and uploaded. If not given, uses `resources/`. :type resources_dir: str :param worker_resources_subpath: Path that will be prepended to the default directory where files are extracted on the worker. Default is empty string, therefore files would be extracted directly to the root folder. Example: If "home/dnanexus" is given, files will be extracted into /home/dnanexus. :type worker_resources_subpath: str :returns: A list (possibly empty) of references to the generated archive(s) :rtype: list If resources_dir exists, archives and uploads the contents of the resources_dir (usually ``resources/``) subdirectory of *src_dir* to a new remote file object, and returns a list describing a single bundled dependency in the form expected by the ``bundledDepends`` field of a run specification. Returns an empty list, if no archive was created. """ if not resources_dir: resources_dir = os.path.join(src_dir, "resources") applet_spec = _get_applet_spec(src_dir) if project is None: dest_project = applet_spec['project'] else: dest_project = project applet_spec['project'] = project if os.path.exists(resources_dir) and len(os.listdir(resources_dir)) > 0: target_folder = applet_spec['folder'] if 'folder' in applet_spec else folder # While creating the resource bundle, optimistically look for a # resource bundle with the same contents, and reuse it if possible. # The resource bundle carries a property 'resource_bundle_checksum' # that indicates the checksum; the way in which the checksum is # computed is given below. If the checksum matches (and # ensure_upload is False), then we will use the existing file, # otherwise, we will compress and upload the tarball. # The input to the SHA1 contains entries of the form (whitespace # only included here for readability): # # / \0 MODE \0 MTIME \0 # /foo \0 MODE \0 MTIME \0 # ... # # where there is one entry for each directory or file (order is # specified below), followed by a numeric representation of the # mode, and the mtime in milliseconds since the epoch. # # Note when looking at a link, if the link is to be dereferenced, # the mtime and mode used are that of the target (using os.stat()) # If the link is to be kept as a link, the mtime and mode are those # of the link itself (using os.lstat()) with tempfile.NamedTemporaryFile(suffix=".tar") as tar_tmp_fh: output_sha1 = hashlib.sha1() tar_fh = tarfile.open(fileobj=tar_tmp_fh, mode='w') for dirname, subdirs, files in os.walk(resources_dir): if not dirname.startswith(resources_dir): raise AssertionError('Expected %r to start with root directory %r' % (dirname, resources_dir)) # Add an entry for the directory itself relative_dirname = dirname[len(resources_dir):] dir_stat = os.lstat(dirname) if not relative_dirname.startswith('/'): relative_dirname = '/' + relative_dirname fields = [relative_dirname, str(_fix_perms(dir_stat.st_mode)), str(int(dir_stat.st_mtime * 1000))] output_sha1.update(b''.join(s.encode('utf-8') + b'\0' for s in fields)) # add an entry in the tar file for the current directory, but # do not recurse! tar_fh.add(dirname, arcname=worker_resources_subpath + relative_dirname, recursive=False, filter=_fix_perm_filter) # Canonicalize the order of subdirectories; this is the order in # which they will be visited by os.walk subdirs.sort() # check the subdirectories for symlinks. We should throw an error # if there are any links that point outside of the directory (unless # --force-symlinks is given). If a link is pointing internal to # the directory (or --force-symlinks is given), we should add it # as a file. for subdir_name in subdirs: dir_path = os.path.join(dirname, subdir_name) # If we do have a symlink, if os.path.islink(dir_path): # Let's get the pointed-to path to ensure that it is # still in the directory link_target = os.readlink(dir_path) # If this is a local link, add it to the list of files (case 1) # else raise an error if force_symlinks or is_link_local(link_target): files.append(subdir_name) else: raise AppBuilderException("Cannot include symlinks to directories outside of the resource directory. '%s' points to directory '%s'" % (dir_path, os.path.realpath(dir_path))) # Canonicalize the order of files so that we compute the # checksum in a consistent order for filename in sorted(files): deref_link = False relative_filename = os.path.join(relative_dirname, filename) true_filename = os.path.join(dirname, filename) file_stat = os.lstat(true_filename) # check for a link here, please! if os.path.islink(true_filename): # Get the pointed-to path link_target = os.readlink(true_filename) if not (force_symlinks or is_link_local(link_target)): # if we are pointing outside of the directory, then: # try to get the true stat of the file and make sure # to dereference the link! try: file_stat = os.stat(os.path.join(dirname, link_target)) deref_link = True except OSError: # uh-oh! looks like we have a broken link! # since this is guaranteed to cause problems (and # we know we're not forcing symlinks here), we # should throw an error raise AppBuilderException("Broken symlink: Link '%s' points to '%s', which does not exist" % (true_filename, os.path.realpath(true_filename)) ) fields = [relative_filename, str(_fix_perms(file_stat.st_mode)), str(int(file_stat.st_mtime * 1000))] output_sha1.update(b''.join(s.encode('utf-8') + b'\0' for s in fields)) # If we are to dereference, use the target fn if deref_link: true_filename = os.path.realpath(true_filename) tar_fh.add(true_filename, arcname=worker_resources_subpath + relative_filename, filter=_fix_perm_filter) # end for filename in sorted(files) # end for dirname, subdirs, files in os.walk(resources_dir): # at this point, the tar is complete, so close the tar_fh tar_fh.close() # Optimistically look for a resource bundle with the same # contents, and reuse it if possible. The resource bundle # carries a property 'resource_bundle_checksum' that indicates # the checksum; the way in which the checksum is computed is # given in the documentation of _directory_checksum. if ensure_upload: properties_dict = {} existing_resources = False else: directory_checksum = output_sha1.hexdigest() properties_dict = dict(resource_bundle_checksum=directory_checksum) existing_resources = dxpy.find_one_data_object( project=dest_project, folder=target_folder, properties=dict(resource_bundle_checksum=directory_checksum), visibility='either', zero_ok=True, state='closed', return_handler=True ) if existing_resources: if not brief: logger.info("Found existing resource bundle that matches local resources directory: " + existing_resources.get_id()) dx_resource_archive = existing_resources else: logger.debug("Uploading in " + src_dir) # We need to compress the tar that we've created targz_fh = tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) # compress the file by reading the tar file and passing # it though a GzipFile object, writing the given # block size (by default 8192 bytes) at a time targz_gzf = gzip.GzipFile(fileobj=targz_fh, mode='wb') tar_tmp_fh.seek(0) dat = tar_tmp_fh.read(io.DEFAULT_BUFFER_SIZE) while dat: targz_gzf.write(dat) dat = tar_tmp_fh.read(io.DEFAULT_BUFFER_SIZE) targz_gzf.flush() targz_gzf.close() targz_fh.close() if 'folder' in applet_spec: try: dxpy.get_handler(dest_project).new_folder(applet_spec['folder'], parents=True) except dxpy.exceptions.DXAPIError: pass # TODO: make this better dx_resource_archive = dxpy.upload_local_file( targz_fh.name, wait_on_close=True, project=dest_project, folder=target_folder, hidden=True, properties=properties_dict ) os.unlink(targz_fh.name) # end compressed file creation and upload archive_link = dxpy.dxlink(dx_resource_archive.get_id()) # end tempfile.NamedTemporaryFile(suffix=".tar") as tar_fh return [{'name': 'resources.tar.gz', 'id': archive_link}] else: return []
[docs]def upload_applet(src_dir, uploaded_resources, check_name_collisions=True, overwrite=False, archive=False, project=None, override_folder=None, override_name=None, dry_run=False, brief=False, **kwargs): """ Creates a new applet object. :param project: ID of container in which to create the applet. :type project: str, or None to use whatever is specified in dxapp.json :param override_folder: folder name for the resulting applet which, if specified, overrides that given in dxapp.json :type override_folder: str :param override_name: name for the resulting applet which, if specified, overrides that given in dxapp.json :type override_name: str """ applet_spec = _get_applet_spec(src_dir) if project is None: dest_project = applet_spec['project'] else: dest_project = project applet_spec['project'] = project if 'name' not in applet_spec: try: applet_spec['name'] = os.path.basename(os.path.abspath(src_dir)) except: raise AppBuilderException("Could not determine applet name from the specification (dxapp.json) or from the name of the working directory (%r)" % (src_dir,)) if override_folder: applet_spec['folder'] = override_folder if 'folder' not in applet_spec: applet_spec['folder'] = '/' if override_name: applet_spec['name'] = override_name if 'dxapi' not in applet_spec: applet_spec['dxapi'] = dxpy.API_VERSION applets_to_overwrite = [] archived_applet = None if check_name_collisions and not dry_run: destination_path = applet_spec['folder'] + ('/' if not applet_spec['folder'].endswith('/') else '') + applet_spec['name'] logger.debug("Checking for existing applet at " + destination_path) for result in dxpy.find_data_objects(classname="applet", name=applet_spec["name"], folder=applet_spec['folder'], project=dest_project, recurse=False): if overwrite: # Don't remove the old applet until after the new one # has been created. This avoids a race condition where # we remove the old applet, but that causes garbage # collection of the bundled resources that will be # shared with the new applet applets_to_overwrite.append(result['id']) elif archive: logger.debug("Archiving applet %s" % (result['id'])) proj = dxpy.DXProject(dest_project) archive_folder = '/.Applet_archive' try: proj.list_folder(archive_folder) except dxpy.DXAPIError: proj.new_folder(archive_folder) proj.move(objects=[result['id']], destination=archive_folder) archived_applet = dxpy.DXApplet(result['id'], project=dest_project) now = datetime.datetime.fromtimestamp(archived_applet.created/1000).ctime() new_name = archived_applet.name + " ({d})".format(d=now) archived_applet.rename(new_name) if not brief: logger.info("Archived applet %s to %s:\"%s/%s\"" % (result['id'], dest_project, archive_folder, new_name)) else: raise AppBuilderException("An applet already exists at %s (id %s) and the --overwrite (-f) or --archive (-a) options were not given" % (destination_path, result['id'])) # ----- # Override various fields from the pristine dxapp.json # Carry region-specific values from regionalOptions into the main # runSpec applet_spec["runSpec"].setdefault("bundledDepends", []) applet_spec["runSpec"].setdefault("assetDepends", []) if not dry_run: region = dxpy.api.project_describe(dest_project, input_params={"fields": {"region": True}})["region"] # if regionalOptions contain at least one region, they must include # the region of the target project if len(applet_spec.get('regionalOptions', {})) != 0 and region not in applet_spec.get('regionalOptions', {}): err_mesg = "destination project is in region {} but \"regionalOptions\" do not contain this region. ".format(region) err_mesg += "Please, update your \"regionalOptions\" specification" raise AppBuilderException(err_mesg) regional_options = applet_spec.get('regionalOptions', {}).get(region, {}) # We checked earlier that if region-specific values for the # fields below are given, the same fields are not also specified # in the top-level runSpec. So the operations below should not # result in any user-supplied settings being clobbered. if 'systemRequirements' in regional_options: applet_spec["runSpec"]["systemRequirements"] = regional_options['systemRequirements'] if 'bundledDepends' in regional_options: applet_spec["runSpec"]["bundledDepends"].extend(regional_options["bundledDepends"]) if 'assetDepends' in regional_options: applet_spec["runSpec"]["assetDepends"].extend(regional_options["assetDepends"]) # Inline Readme.md and Readme.developer.md dxpy.executable_builder.inline_documentation_files(applet_spec, src_dir) # Inline the code of the program if "file" in applet_spec["runSpec"]: # Put it into runSpec.code instead with open(os.path.join(src_dir, applet_spec["runSpec"]["file"])) as code_fh: applet_spec["runSpec"]["code"] = code_fh.read() del applet_spec["runSpec"]["file"] # If this is applet requires a cluster, inline any bootstrapScript code that may be provided. # bootstrapScript is an *optional* clusterSpec parameter. # NOTE: assumes bootstrapScript is always provided as a filename if "systemRequirements" in applet_spec["runSpec"]: sys_reqs = applet_spec["runSpec"]["systemRequirements"] for entry_point in sys_reqs: try: bootstrap_script = os.path.join(src_dir, sys_reqs[entry_point]["clusterSpec"]["bootstrapScript"]) with open(bootstrap_script) as code_fh: sys_reqs[entry_point]["clusterSpec"]["bootstrapScript"] = code_fh.read() except KeyError: # either no "clusterSpec" or no "bootstrapScript" within "clusterSpec" continue except IOError: raise AppBuilderException("The clusterSpec \"bootstrapScript\" could not be read.") # Attach bundled resources to the app if uploaded_resources is not None: applet_spec["runSpec"]["bundledDepends"].extend(uploaded_resources) # Validate and process assetDepends asset_depends = applet_spec["runSpec"]["assetDepends"] if type(asset_depends) is not list or any(type(dep) is not dict for dep in asset_depends): raise AppBuilderException("Expected runSpec.assetDepends to be an array of objects") for asset in asset_depends: asset_project = asset.get("project", None) asset_folder = asset.get("folder", '/') asset_stages = asset.get("stages", None) if "id" in asset: asset_record = dxpy.DXRecord(asset["id"]).describe(fields={'details'}, default_fields=True) elif "name" in asset and asset_project is not None and "version" in asset: try: asset_record = dxpy.find_one_data_object(zero_ok=True, classname="record", typename="AssetBundle", name=asset["name"], properties=dict(version=asset["version"]), project=asset_project, folder=asset_folder, recurse=False, describe={"defaultFields": True, "fields": {"details": True}}, state="closed", more_ok=False) except dxpy.exceptions.DXSearchError: msg = "Found more than one asset record that matches: name={0}, folder={1} in project={2}." raise AppBuilderException(msg.format(asset["name"], asset_folder, asset_project)) else: raise AppBuilderException("Each runSpec.assetDepends element must have either {'id'} or " "{'name', 'project' and 'version'} field(s).") if asset_record: if "id" in asset: asset_details = asset_record["details"] else: asset_details = asset_record["describe"]["details"] if "archiveFileId" in asset_details: archive_file_id = asset_details["archiveFileId"] else: raise AppBuilderException("The required field 'archiveFileId' was not found in " "the details of the asset bundle %s " % asset_record["id"]) archive_file_name = dxpy.DXFile(archive_file_id).describe()["name"] bundle_depends = { "name": archive_file_name, "id": archive_file_id } if asset_stages: bundle_depends["stages"] = asset_stages applet_spec["runSpec"]["bundledDepends"].append(bundle_depends) # If the file is not found in the applet destination project, clone it from the asset project if (not dry_run and dxpy.DXRecord(dxid=asset_record["id"], project=dest_project).describe()["project"] != dest_project): dxpy.DXRecord(asset_record["id"], project=asset_record["project"]).clone(dest_project) else: raise AppBuilderException("No asset bundle was found that matched the specification %s" % (json.dumps(asset))) merge(applet_spec, kwargs) # ----- # Now actually create the applet if dry_run: print("Would create the following applet:") print(json.dumps(applet_spec, indent=2)) print("*** DRY-RUN-- no applet was created ***") return None, None if applet_spec.get("categories", []): if "tags" not in applet_spec: applet_spec["tags"] = [] applet_spec["tags"] = list(set(applet_spec["tags"]) | set(applet_spec["categories"])) applet_id = dxpy.api.applet_new(applet_spec)["id"] if archived_applet: archived_applet.set_properties({'replacedWith': applet_id}) # Now it is permissible to delete the old applet(s), if any if applets_to_overwrite: if not brief: logger.info("Deleting applet(s) %s" % (','.join(applets_to_overwrite))) dxpy.DXProject(dest_project).remove_objects(applets_to_overwrite) return applet_id, applet_spec
def _create_or_update_version(app_name, version, app_spec, try_update=True): """ Creates a new version of the app. Returns an app_id, or None if the app has already been created and published. """ # This has a race condition since the app could have been created or # published since we last looked. try: app_id = dxpy.api.app_new(app_spec)["id"] return app_id except dxpy.exceptions.DXAPIError as e: # TODO: detect this error more reliably if e.name == 'InvalidInput' and e.msg == 'Specified name and version conflict with an existing alias': print('App %s/%s already exists' % (app_spec["name"], version), file=sys.stderr) # The version number was already taken, so app/new doesn't work. # However, maybe it hasn't been published yet, so we might be able # to app-xxxx/update it. app_describe = dxpy.api.app_describe("app-" + app_name, alias=version) if app_describe.get("published", 0) > 0: return None return _update_version(app_name, version, app_spec, try_update=try_update) raise e def _update_version(app_name, version, app_spec, try_update=True): """ Updates a version of the app in place. Returns an app_id, or None if the app has already been published. """ if not try_update: return None try: app_id = dxpy.api.app_update("app-" + app_name, version, app_spec)["id"] return app_id except dxpy.exceptions.DXAPIError as e: if e.name == 'InvalidState': print('App %s/%s has already been published' % (app_spec["name"], version), file=sys.stderr) return None raise e
[docs]def create_app_multi_region(regional_options, app_name, src_dir, publish=False, set_default=False, billTo=None, try_versions=None, try_update=True, confirm=True, inherited_metadata={}, brief=False): """ Creates a new app object from the specified applet(s). :param regional_options: Region-specific options for the app. See https://documentation.dnanexus.com/developer/api/running-analyses/apps#api-method-app-new for details; this should contain keys for each region the app is to be enabled in, and for the values, a dict containing (at minimum) a key "applet" whose value is an applet ID for that region. :type regional_options: dict """ return _create_app(dict(regionalOptions=regional_options), app_name, src_dir, publish=publish, set_default=set_default, billTo=billTo, try_versions=try_versions, try_update=try_update, confirm=confirm, inherited_metadata=inherited_metadata, brief=brief)
[docs]def create_app(applet_id, applet_name, src_dir, publish=False, set_default=False, billTo=None, try_versions=None, try_update=True, confirm=True, regional_options=None): """ Creates a new app object from the specified applet. .. deprecated:: 0.204.0 Use :func:`create_app_multi_region()` instead. """ # In this case we don't know the region of the applet, so we use the # legacy API {"applet": applet_id} without specifying a region # specifically. return _create_app(dict(applet=applet_id), applet_name, src_dir, publish=publish, set_default=set_default, billTo=billTo, try_versions=try_versions, try_update=try_update, confirm=confirm)
def _create_app(applet_or_regional_options, app_name, src_dir, publish=False, set_default=False, billTo=None, try_versions=None, try_update=True, confirm=True, inherited_metadata={}, brief=False): if src_dir: app_spec = _get_app_spec(src_dir) if not brief: logger.info("Will create app with spec: %s" % (json.dumps(app_spec),)) # Inline Readme.md and Readme.developer.md dxpy.executable_builder.inline_documentation_files(app_spec, src_dir) else: app_spec = inherited_metadata app_spec.update(applet_or_regional_options, name=app_name) if billTo: app_spec["billTo"] = billTo if not try_versions: try_versions = [app_spec["version"]] for version in try_versions: logger.debug("Attempting to create version %s..." % (version,)) app_spec['version'] = version app_describe = None try: # 404, which is rather likely in this app_describe request # (the purpose of the request is to find out whether the # version of interest exists), would ordinarily cause this # request to be retried multiple times, introducing a # substantial delay. So we disable retrying here for this # request. app_describe = dxpy.api.app_describe("app-" + app_spec["name"], alias=version, always_retry=False) except dxpy.exceptions.DXAPIError as e: if e.name == 'ResourceNotFound': pass else: raise e # Now app_describe is None if the app didn't exist, OR it contains the # app describe content. # The describe check does not eliminate race conditions since an app # may always have been created, or published, since we last looked at # it. So the describe that happens here is just to save time and avoid # unnecessary API calls, but we always have to be prepared to recover # from API errors. if app_describe is None: logger.debug('App %s/%s does not yet exist' % (app_spec["name"], version)) app_id = _create_or_update_version(app_spec['name'], app_spec['version'], app_spec, try_update=try_update) if app_id is None: continue if not brief: logger.info("Created app " + app_id) # Success! break elif app_describe.get("published", 0) == 0: logger.debug('App %s/%s already exists and has not been published' % (app_spec["name"], version)) app_id = _update_version(app_spec['name'], app_spec['version'], app_spec, try_update=try_update) if app_id is None: continue if not brief: logger.info("Updated existing app " + app_id) # Success! break else: logger.debug('App %s/%s already exists and has been published' % (app_spec["name"], version)) # App has already been published. Give up on this version. continue else: # All versions requested failed if len(try_versions) != 1: tried_versions = 'any of the requested versions: ' + ', '.join(try_versions) else: tried_versions = 'the requested version: ' + try_versions[0] raise AppBuilderException('Could not create %s' % (tried_versions,)) # Set categories appropriately. categories_to_set = app_spec.get("categories", []) existing_categories = dxpy.api.app_list_categories(app_id)['categories'] categories_to_add = set(categories_to_set).difference(set(existing_categories)) categories_to_remove = set(existing_categories).difference(set(categories_to_set)) if categories_to_add: dxpy.api.app_add_categories(app_id, input_params={'categories': list(categories_to_add)}) if categories_to_remove: dxpy.api.app_remove_categories(app_id, input_params={'categories': list(categories_to_remove)}) # Set developers list appropriately, but only if provided. developers_to_set = app_spec.get("developers") if developers_to_set is not None: existing_developers = dxpy.api.app_list_developers(app_id)['developers'] developers_to_add = set(developers_to_set) - set(existing_developers) developers_to_remove = set(existing_developers) - set(developers_to_set) skip_updating_developers = False if developers_to_add or developers_to_remove: parts = [] if developers_to_add: parts.append('the following developers will be added: ' + ', '.join(sorted(developers_to_add))) if developers_to_remove: parts.append('the following developers will be removed: ' + ', '.join(sorted(developers_to_remove))) developer_change_message = '; and '.join(parts) if confirm: if INTERACTIVE_CLI: try: print('***') print(fill('WARNING: ' + developer_change_message)) print('***') value = input('Confirm updating developers list [y/N]: ') except KeyboardInterrupt: value = 'n' if not value.lower().startswith('y'): skip_updating_developers = True else: # Default to NOT updating developers if operating # without a TTY. logger.warn('skipping requested change to the developer list. Rerun "dx build" interactively or pass --yes to confirm this change.') skip_updating_developers = True else: logger.warn(developer_change_message) if not skip_updating_developers: if developers_to_add: dxpy.api.app_add_developers(app_id, input_params={'developers': list(developers_to_add)}) if developers_to_remove: dxpy.api.app_remove_developers(app_id, input_params={'developers': list(developers_to_remove)}) # Set authorizedUsers list appropriately, but only if provided. authorized_users_to_set = app_spec.get("authorizedUsers") existing_authorized_users = dxpy.api.app_list_authorized_users(app_id)['authorizedUsers'] if authorized_users_to_set is not None: authorized_users_to_add = set(authorized_users_to_set) - set(existing_authorized_users) authorized_users_to_remove = set(existing_authorized_users) - set(authorized_users_to_set) skip_adding_public = False if 'PUBLIC' in authorized_users_to_add: acl_change_message = 'app-%s will be made public. Anyone will be able to view and run all published versions of this app.' % (app_spec['name'],) if confirm: if INTERACTIVE_CLI: try: print('***') print(fill('WARNING: ' + acl_change_message)) print('***') value = input('Confirm making this app public [y/N]: ') except KeyboardInterrupt: value = 'n' if not value.lower().startswith('y'): skip_adding_public = True else: # Default to NOT adding PUBLIC if operating # without a TTY. logger.warn('skipping requested change to add PUBLIC to the authorized users list. Rerun "dx build" interactively or pass --yes to confirm this change.') skip_adding_public = True else: logger.warn(acl_change_message) if skip_adding_public: authorized_users_to_add -= {'PUBLIC'} if authorized_users_to_add: dxpy.api.app_add_authorized_users(app_id, input_params={'authorizedUsers': list(authorized_users_to_add)}) if skip_adding_public: logger.warn('the app was NOT made public as requested in the app spec. To make it so, run "dx add users app-%s PUBLIC".' % (app_spec["name"],)) if authorized_users_to_remove: dxpy.api.app_remove_authorized_users(app_id, input_params={'authorizedUsers': list(authorized_users_to_remove)}) elif not len(existing_authorized_users) and not brief: # Apps that had authorized users added by any other means will # not have this message printed. logger.warn('authorizedUsers is missing from the app spec. No one will be able to view or run the app except the app\'s developers.') if publish: dxpy.api.app_publish(app_id, input_params={'makeDefault': set_default}) else: # If no versions of this app have ever been published, then # we'll set the "default" tag to point to the latest # (unpublished) version. no_published_versions = len(list(dxpy.find_apps(name=app_name, published=True, limit=1))) == 0 if no_published_versions: dxpy.api.app_add_tags(app_id, input_params={'tags': ['default']}) return app_id
[docs]def get_enabled_regions(app_spec, from_command_line): """Returns a list of the regions in which the app should be enabled. Also validates that app_spec['regionalOptions'], if supplied, is well-formed. :param app_spec: app specification :type app_spec: dict :param from_command_line: The regions specified on the command-line via --region :type from_command_line: list or None """ enabled_regions = dxpy.executable_builder.get_enabled_regions('app', app_spec, from_command_line, AppBuilderException) if enabled_regions is not None and len(enabled_regions) == 0: raise AssertionError("This app should be enabled in at least one region") return enabled_regions