From 0c932b9a9768a7af2041caa15ab0f39810a4e9b3 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 1 Oct 2018 15:03:11 -0400 Subject: [PATCH 1/6] Move SampleSource to its own package. --- README.md | 1 + metagenscope_cli/cli/upload_cli.py | 5 +- metagenscope_cli/cli/utils.py | 2 +- metagenscope_cli/sample_sources/__init__.py | 49 ------ metagenscope_cli/sample_sources/constants.py | 13 -- .../sample_sources/data_super_source.py | 35 ----- .../sample_sources/file_source.py | 42 ------ metagenscope_cli/tools/__init__.py | 1 - metagenscope_cli/tools/constants.py | 48 ------ metagenscope_cli/tools/parse_metadata.py | 18 --- metagenscope_cli/tools/parser_utils.py | 140 ------------------ metagenscope_cli/tools/parsers.py | 49 ------ requirements.txt | 4 + setup.py | 1 + 14 files changed, 9 insertions(+), 399 deletions(-) delete mode 100644 metagenscope_cli/sample_sources/__init__.py delete mode 100644 metagenscope_cli/sample_sources/constants.py delete mode 100644 metagenscope_cli/sample_sources/data_super_source.py delete mode 100644 metagenscope_cli/sample_sources/file_source.py delete mode 100644 metagenscope_cli/tools/__init__.py delete mode 100644 metagenscope_cli/tools/constants.py delete mode 100644 metagenscope_cli/tools/parse_metadata.py delete mode 100644 metagenscope_cli/tools/parser_utils.py delete mode 100644 metagenscope_cli/tools/parsers.py create mode 100644 requirements.txt diff --git a/README.md b/README.md index 9391351..c4446c6 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Simply run: $ pipsi install . +**Note:** this relies packages from the Longtail Bio PyPi index. See [`pangea-modules`](https://github.com/LongTailBio/pangea-modules/tree/develop#pangea-modules) for instructions on setting that up. ### Usage diff --git a/metagenscope_cli/cli/upload_cli.py b/metagenscope_cli/cli/upload_cli.py index 22d6bf0..055da4e 100644 --- a/metagenscope_cli/cli/upload_cli.py +++ b/metagenscope_cli/cli/upload_cli.py @@ -3,8 +3,7 @@ from sys import stderr import click -from metagenscope_cli.sample_sources.data_super_source import DataSuperSource -from metagenscope_cli.sample_sources.file_source import FileSource +from sample_source import DataSuperSource, FileSystemSource from .utils import batch_upload, add_authorization, parse_metadata @@ -52,7 +51,7 @@ def datasuper(uploader, group, group_name): @click.argument('result_files', nargs=-1) def files(uploader, group, result_files): """Upload all samples from llist of tool result files.""" - sample_source = FileSource(files=result_files) + sample_source = FileSystemSource(files=result_files) samples = sample_source.get_sample_payloads() batch_upload(uploader, samples, group_uuid=group) diff --git a/metagenscope_cli/cli/utils.py b/metagenscope_cli/cli/utils.py index 0189d45..fb2cec7 100644 --- a/metagenscope_cli/cli/utils.py +++ b/metagenscope_cli/cli/utils.py @@ -7,10 +7,10 @@ import click from requests.exceptions import HTTPError +from sample_source.parser import parse_metadata_from_csv from metagenscope_cli.network import Knex, Uploader from metagenscope_cli.network.token_auth import TokenAuth -from metagenscope_cli.tools.parse_metadata import parse_metadata_from_csv def parse_metadata(filename, sample_names): diff --git a/metagenscope_cli/sample_sources/__init__.py b/metagenscope_cli/sample_sources/__init__.py deleted file mode 100644 index e9780d3..0000000 --- a/metagenscope_cli/sample_sources/__init__.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Sources for sample data.""" - -from sys import stderr -from metagenscope_cli.tools.parsers import parse, UnparsableError - - -class SampleSource(object): - """Base SampleSource interface.""" - - def get_cataloged_files(self): - """ - Return dictionary of files cataloged by sample and type. - - Returns {: {: {: }}} - """ - raise NotImplementedError() - - def get_sample_payloads(self): - """ - Return list of sample payload components (name, endpoint, and body JSON). - - returns { - : [{ - 'result_type': string, - 'data': dict (payload), - }] - } - """ - cataloged_files = self.get_cataloged_files() - - results = {} - for sample_name, sample_schema in cataloged_files.items(): - results[sample_name] = [] - for result_type, files_dict in sample_schema.items(): - try: - data = parse(result_type, files_dict) - except UnparsableError: - print(f'[parse-error] could not parse {result_type}', file=stderr) - continue - except KeyError: - print(f'[key-error] {sample_name} :: {result_type}', file=stderr) - - result_payload = { - 'result_type': result_type, - 'data': data, - } - results[sample_name].append(result_payload) - - return results diff --git a/metagenscope_cli/sample_sources/constants.py b/metagenscope_cli/sample_sources/constants.py deleted file mode 100644 index d707f23..0000000 --- a/metagenscope_cli/sample_sources/constants.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Constants for sample source.""" - - -UNSUPPORTED_RESULT_TYPES = [ - 'filter_human_dna', - 'mash_sketch', - 'resistome_amrs', - 'shortbred_amr_profiling', - 'adapter_removal', - 'align_to_sa_n315', - 'bracken_abundance_estimation', - 'raw_short_read_dna', -] diff --git a/metagenscope_cli/sample_sources/data_super_source.py b/metagenscope_cli/sample_sources/data_super_source.py deleted file mode 100644 index c962f27..0000000 --- a/metagenscope_cli/sample_sources/data_super_source.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Samples from a DataSuper repository.""" - -import datasuper as ds - -from metagenscope_cli.sample_sources import SampleSource - -from .constants import UNSUPPORTED_RESULT_TYPES - - -class DataSuperSource(SampleSource): - """Samples from a DataSuper repository.""" - - def __init__(self, all_samples=True): - """Initialize a DataSuperSource instance.""" - self.all_samples = all_samples - - def get_cataloged_files(self): - """Return dictionary of files cataloged by sample and type.""" - repo = ds.Repo.loadRepo() - samples = repo.sampleTable.getAll() - - catalog = {} - for sample in samples: - catalog[sample.name] = {} - for result in sample.results(): - result_type = result.resultType() - if result_type in UNSUPPORTED_RESULT_TYPES: - continue - - catalog[sample.name][result_type] = { - file_type: file_record.filepath() - for file_type, file_record in result.files() - } - - return catalog diff --git a/metagenscope_cli/sample_sources/file_source.py b/metagenscope_cli/sample_sources/file_source.py deleted file mode 100644 index 64c5603..0000000 --- a/metagenscope_cli/sample_sources/file_source.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Samples from a list of files.""" - -from metagenscope_cli.sample_sources import SampleSource - -from .constants import UNSUPPORTED_RESULT_TYPES - - -def parse_file_path(file_path): - """Extract file metadata from its path.""" - file_name = file_path.split('/')[-1] - name_components = file_name.split('.') - - sample_name = name_components[0] - result_type = name_components[1] - file_type = name_components[2] - - return sample_name, result_type, file_type - - -class FileSource(SampleSource): - """Samples from a list of files.""" - - def __init__(self, files): - """Initialize FileSource from list of files.""" - self.files = files - - def get_cataloged_files(self): - """Return dictionary of files cataloged by sample and type.""" - catalog = {} - for file in self.files: - sample_name, result_type, file_type = parse_file_path(file) - if result_type in UNSUPPORTED_RESULT_TYPES: - continue - - try: - try: - catalog[sample_name][result_type][file_type] = file - except KeyError: - catalog[sample_name][result_type] = {file_type: file} - except KeyError: - catalog[sample_name] = {result_type: {file_type: file}} - return catalog diff --git a/metagenscope_cli/tools/__init__.py b/metagenscope_cli/tools/__init__.py deleted file mode 100644 index 71784dc..0000000 --- a/metagenscope_cli/tools/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Handlers for genomic analysis tools.""" diff --git a/metagenscope_cli/tools/constants.py b/metagenscope_cli/tools/constants.py deleted file mode 100644 index e9a6921..0000000 --- a/metagenscope_cli/tools/constants.py +++ /dev/null @@ -1,48 +0,0 @@ -"""MetaGenScope CLI constants. - -Note: tool types and file types are defined in the -metasub cap and stored in datasuper. -""" - -# Tool Types -KRAKEN = 'kraken_taxonomy_profiling' -KRAKENHLL = 'krakenhll_taxonomy_profiling' -METAPHLAN2 = 'metaphlan2_taxonomy_profiling' -HMP_SITES = 'hmp_site_dists' -MICROBE_CENSUS = 'microbe_census' -AMR_GENES = 'align_to_amr_genes' -RESISTOME_AMRS = 'resistome_amrs' -READ_CLASS_PROPS = 'read_classification_proportions' -READ_STATS = 'read_stats' -MICROBE_DIRECTORY = 'microbe_directory_annotate' -ALPHA_DIVERSITY = 'alpha_diversity_stats' -BETA_DIVERSITY = 'beta_diversity_stats' -HUMANN2 = 'humann2_functional_profiling' -HUMANN2_NORMALIZED = 'humann2_normalize_genes' -METHYLS = 'align_to_methyltransferases' -VFDB = 'vfdb_quantify' -MACROBES = 'quantify_macrobial' -ANCESTRY = 'human_ancestry' - - -# Other -TAXON_KEY = 'taxon' -ABUNDANCE_KEY = 'abundance' - -AGS_KEY = 'average_genome_size' -TOTAL_BASES_KEY = 'total_bases' -GENOME_EQUIVALENTS_KEY = 'genome_equivalents' - - -# Humann2 -PATHWAY_KEY = 'pathway' -ABUNDANCE_KEY = 'abundance' -COVERAGE_KEY = 'coverage' -GENE_KEY = 'gene_family' - -# Gene Quantification -GENE_ID = 'gene_name' -RPK_KEY = 'rpk' -RPKM_KEY = 'rpkm' -RPKMG_KEY = 'rpkmg' -TOP_N_FILTER = 1000 diff --git a/metagenscope_cli/tools/parse_metadata.py b/metagenscope_cli/tools/parse_metadata.py deleted file mode 100644 index 3875223..0000000 --- a/metagenscope_cli/tools/parse_metadata.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Parser for Sample metadata.""" - -import pandas as pd - -NA_TOKEN = 'n/a' - - -def parse_metadata_from_csv(csv_filename, sample_names): - """Parse sample metadata from a .csv file.""" - metadata_df = pd.read_csv(csv_filename, index_col=None, dtype=str).fillna(NA_TOKEN) - colnames = list(metadata_df.columns.values) - metadata_df = metadata_df.set_index(colnames[0]) - tbl = metadata_df.to_dict(orient='index') - for sample_name in sample_names: - if sample_name not in tbl: - tbl[sample_name] = {colname: NA_TOKEN for colname in colnames} - metadata_df = pd.DataFrame.from_dict(tbl, orient='index').fillna(NA_TOKEN) - return metadata_df.to_dict(orient='index') diff --git a/metagenscope_cli/tools/parser_utils.py b/metagenscope_cli/tools/parser_utils.py deleted file mode 100644 index 095f086..0000000 --- a/metagenscope_cli/tools/parser_utils.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Parsing utilities.""" - -from json import loads - -from .constants import (RPK_KEY, RPKM_KEY, RPKMG_KEY, TOP_N_FILTER, ABUNDANCE_KEY, - COVERAGE_KEY, GENOME_EQUIVALENTS_KEY, AGS_KEY, TOTAL_BASES_KEY) - - -def jloads(fname): - """Load JSON file into python dictionary.""" - return loads(open(fname).read()) - - -def scrub_keys(key): - """Replace periods (restricted by Mongo) with underscores.""" - return '_'.join(key.split('.')) - - -def tokenize(file_name, skip=0, sep='\t', skipchar='#'): - """Tokenize a tabular file.""" - with open(file_name) as file: - for _ in range(skip): - file.readline() - for line in file: - stripped = line.strip() - if stripped[0] == skipchar: - continue - tkns = stripped.split(sep) - if len(tkns) >= 2: - yield tkns - - -def parse_key_val_file(filename, # pylint:disable=too-many-arguments - skip=0, skipchar='#', sep='\t', - kind=float, key_column=0, val_column=1): - """Parse a key-value-type file.""" - tokens = tokenize(filename, skip=skip, sep=sep, skipchar=skipchar) - out = {scrub_keys(token[key_column]): kind(token[val_column]) - for token in tokens} - return out - - -def parse_resistome_tables(gene_table, group_table, - classus_table, mech_table): - """Parse a resistome table file.""" - result = { - 'genes': parse_key_val_file(gene_table, - key_column=1, val_column=2, - skip=1, kind=int), - 'groups': parse_key_val_file(group_table, - key_column=1, val_column=2, - skip=1, kind=int), - 'classus': parse_key_val_file(classus_table, - key_column=1, val_column=2, - skip=1, kind=int), - 'mechanism': parse_key_val_file(mech_table, - key_column=1, val_column=2, - skip=1, kind=int), - } - - return result - - -def parse_humann2_tables(rpkm_file, rpkmg_file): - """Ingest Humann2 table file.""" - rpkms = parse_key_val_file(rpkm_file) - rpkmgs = parse_key_val_file(rpkmg_file) - data = {} - rpkms = [(gene, rpkm) for gene, rpkm in rpkms.items()] - rpkms = sorted(rpkms, key=lambda x: -x[1])[:TOP_N_FILTER] - for gene, rpkm in rpkms: - row = { - RPK_KEY: rpkm, # hack since rpk does not matter - RPKM_KEY: rpkm, - RPKMG_KEY: rpkmgs[gene], - } - data[gene] = row - return data - - -def parse_humann2_pathways(path_abunds, path_covs): - """Ingest Humann2 pathways results file.""" - path_abunds = parse_key_val_file(path_abunds) - path_covs = parse_key_val_file(path_covs) - data = {} - for path, abund in path_abunds.items(): - cov = path_covs[path] - row = { - ABUNDANCE_KEY: abund, - COVERAGE_KEY: cov - } - data[path] = row - return data - - -def parse_gene_table(gene_table): - """Return a parsed gene quantification table.""" - data = {} - with open(gene_table) as gfile: - gfile.readline() - for line in gfile: - tkns = line.strip().split(',') - gene_name = tkns[0] - row = { - RPK_KEY: float(tkns[1]), - RPKM_KEY: float(tkns[2]), - RPKMG_KEY: float(tkns[3]), - } - data[scrub_keys(gene_name)] = row - data = [(key, val) for key, val in data.items()] - data = sorted(data, key=lambda x: -x[1][RPKM_KEY])[:TOP_N_FILTER] - data = {key: val for key, val in data} - return data - - -def parse_mpa(mpa_file): - """Ingest MPA results file.""" - data = [(taxa, val) for taxa, val in parse_key_val_file(mpa_file).items()] - data = sorted(data, key=lambda x: -x[1])[:TOP_N_FILTER] - return {key: val for key, val in data} - - -def parse_microbe_census(input_file): - """Ingest microbe census results file.""" - data = {} - with open(input_file) as file: - for line in file: - parts = line.strip().split() - for key in [AGS_KEY, TOTAL_BASES_KEY, GENOME_EQUIVALENTS_KEY]: - if parts and key in parts[0]: - if key == TOTAL_BASES_KEY: - data[key] = int(parts[1]) - else: - data[key] = float(parts[1]) - # Require valid values - for key in [AGS_KEY, TOTAL_BASES_KEY, GENOME_EQUIVALENTS_KEY]: - if key not in data: - assert False, 'Missing key in MicrobeCensus' - - return data diff --git a/metagenscope_cli/tools/parsers.py b/metagenscope_cli/tools/parsers.py deleted file mode 100644 index c3ca58f..0000000 --- a/metagenscope_cli/tools/parsers.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Parsers for different Tool Result types.""" - -from . import parser_utils as utils, constants as const - - -class UnparsableError(Exception): - """Custom exception signaling an unknown Tool Result type.""" - - pass - - -JSON_TOOLS = { - const.ALPHA_DIVERSITY: 'json', - const.MICROBE_DIRECTORY: 'json', - const.READ_STATS: 'json', - const.READ_CLASS_PROPS: 'json', - const.HMP_SITES: 'metaphlan2', - const.BETA_DIVERSITY: 'json', - const.MACROBES: 'tbl', -} - -SIMPLE_PARSE = { - const.MICROBE_CENSUS: (utils.parse_microbe_census, 'stats'), - const.KRAKEN: (utils.parse_mpa, 'mpa'), - const.METAPHLAN2: (utils.parse_mpa, 'mpa'), - const.KRAKENHLL: (utils.parse_mpa, 'report'), - const.METHYLS: (utils.parse_gene_table, 'table'), - const.VFDB: (utils.parse_gene_table, 'table'), - const.AMR_GENES: (utils.parse_gene_table, 'table'), - const.ANCESTRY: (utils.parse_key_val_file, 'table'), - const.RESISTOME_AMRS: (utils.parse_resistome_tables, - 'gene', 'group', 'classus', 'mech'), - const.HUMANN2: (utils.parse_humann2_pathways, - 'path_abunds', 'path_cov'), - const.HUMANN2_NORMALIZED: (utils.parse_humann2_tables, - 'read_depth_norm_genes', 'ags_norm_genes'), -} - - -def parse(tool_type, schema): - """Parse schema as tool_type.""" - if tool_type in JSON_TOOLS: - key = JSON_TOOLS[tool_type] - return utils.jloads(schema[key]) - elif tool_type in SIMPLE_PARSE: - func = SIMPLE_PARSE[tool_type][0] - fnames = [schema[key] for key in SIMPLE_PARSE[tool_type][1:]] - return func(*fnames) - raise UnparsableError(f'{tool_type}, {schema}') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5ea9ae4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +--index-url https://pypi.python.org/simple +--index-url https://pypi.longtailbio.com/simple + +-e . diff --git a/setup.py b/setup.py index 7b886f4..c718ac0 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ 'configparser', 'pandas', 'datasuper==0.9.0', + 'sample-source', ] dependency_links = [ From 30873b0fbb802d1e6b3a75fbf88216441e098ba3 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 1 Oct 2018 15:26:44 -0400 Subject: [PATCH 2/6] Switch to namespaced package. --- metagenscope_cli/cli/upload_cli.py | 2 +- metagenscope_cli/cli/utils.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metagenscope_cli/cli/upload_cli.py b/metagenscope_cli/cli/upload_cli.py index 055da4e..fc8ae2a 100644 --- a/metagenscope_cli/cli/upload_cli.py +++ b/metagenscope_cli/cli/upload_cli.py @@ -3,7 +3,7 @@ from sys import stderr import click -from sample_source import DataSuperSource, FileSystemSource +from pangea_modules.sample_source import DataSuperSource, FileSystemSource from .utils import batch_upload, add_authorization, parse_metadata diff --git a/metagenscope_cli/cli/utils.py b/metagenscope_cli/cli/utils.py index fb2cec7..edd2803 100644 --- a/metagenscope_cli/cli/utils.py +++ b/metagenscope_cli/cli/utils.py @@ -7,7 +7,7 @@ import click from requests.exceptions import HTTPError -from sample_source.parser import parse_metadata_from_csv +from pangea_modules.sample_source.parser import parse_metadata_from_csv from metagenscope_cli.network import Knex, Uploader from metagenscope_cli.network.token_auth import TokenAuth diff --git a/setup.py b/setup.py index c718ac0..354b3b1 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ 'configparser', 'pandas', 'datasuper==0.9.0', - 'sample-source', + 'pangea_modules.sample-source', ] dependency_links = [ From 0bb0130821aa9d6cb35222f04875091a2e48167b Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 1 Oct 2018 15:44:57 -0400 Subject: [PATCH 3/6] Add Longtail package index to CI. --- .circleci/config.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1cdec4f..8a0d329 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,28 @@ jobs: - checkout + - run: + name: Init .pypirc + command: | + echo -e "[distutils]" >> ~/.pypirc + echo -e "index-servers =" >> ~/.pypirc + echo -e " pypi" >> ~/.pypirc + echo -e " longtailbio" >> ~/.pypirc + echo -e "" >> ~/.pypirc + echo -e "[pypi]" >> ~/.pypirc + echo -e "" >> ~/.pypirc + echo -e "[longtailbio]" >> ~/.pypirc + echo -e "repository: https://pypi.longtailbio.com" >> ~/.pypirc + echo -e "username: $LT_PYPI_USERNAME" >> ~/.pypirc + echo -e "password: $LT_PYPI_PASSWORD" >> ~/.pypirc + + - run: + name: Init ~/.pip/pip.conf + command: | + mkdir ~/.pip + echo -e "[global]" >> ~/.pip/pip.conf + echo -e "extra-index-url = https://pypi.longtailbio.com/simple/" >> ~/.pip/pip.conf + - run: name: Run tests with py35 command: tox -e py35 From 70d075230545d814b099836702a61c6e5e79f0ca Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 1 Oct 2018 16:03:25 -0400 Subject: [PATCH 4/6] Fix lint errors. --- metagenscope_cli/config.py | 2 +- metagenscope_cli/network/knex.py | 2 +- metagenscope_cli/network/token_auth.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metagenscope_cli/config.py b/metagenscope_cli/config.py index 9e4521b..3cf3c4f 100644 --- a/metagenscope_cli/config.py +++ b/metagenscope_cli/config.py @@ -9,7 +9,7 @@ TOKEN_OPTION_KEY = 'token' -class MetagenscopeConfiguration(object): +class MetagenscopeConfiguration: """MetaGenScope configuration.""" def __init__(self, filename, dirname='~'): diff --git a/metagenscope_cli/network/knex.py b/metagenscope_cli/network/knex.py index 7ef7755..b0b758f 100644 --- a/metagenscope_cli/network/knex.py +++ b/metagenscope_cli/network/knex.py @@ -5,7 +5,7 @@ from metagenscope_cli.constants import DEFAULT_HOST -class Knex(object): +class Knex: """Knex wraps MetaGenScope requests requiring authentication.""" def __init__(self, token_auth, host=None, headers=None): diff --git a/metagenscope_cli/network/token_auth.py b/metagenscope_cli/network/token_auth.py index d345bdc..42ef9c0 100644 --- a/metagenscope_cli/network/token_auth.py +++ b/metagenscope_cli/network/token_auth.py @@ -15,7 +15,7 @@ def __init__(self, jwt_token=None): if self.jwt_token is None: try: self.jwt_token = config.get_token() - except KeyError: + except KeyError: # pylint: disable=try-except-raise # No saved token raise From c65c4ecd9acb01a667bb84b8e413aac6c5e88918 Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 1 Oct 2018 16:31:21 -0400 Subject: [PATCH 5/6] Update build steps to include deploy. --- .circleci/config.yml | 101 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 80 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8a0d329..4c839ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,27 @@ version: 2 +# Common steps for building server Docker images +pypi_step: &pypi_step + name: Init .pypirc + command: | + echo -e "[distutils]" >> ~/.pypirc + echo -e "index-servers =" >> ~/.pypirc + echo -e " pypi" >> ~/.pypirc + echo -e " longtailbio" >> ~/.pypirc + echo -e "" >> ~/.pypirc + echo -e "[pypi]" >> ~/.pypirc + echo -e "" >> ~/.pypirc + echo -e "[longtailbio]" >> ~/.pypirc + echo -e "repository: https://pypi.longtailbio.com" >> ~/.pypirc + echo -e "username: $LT_PYPI_USERNAME" >> ~/.pypirc + echo -e "password: $LT_PYPI_PASSWORD" >> ~/.pypirc + mkdir ~/.pip + echo -e "[global]" >> ~/.pip/pip.conf + echo -e "extra-index-url = https://pypi.longtailbio.com/simple/" >> ~/.pip/pip.conf + + jobs: - build: + test_package: docker: - image: themattrix/tox @@ -13,26 +33,7 @@ jobs: - checkout - run: - name: Init .pypirc - command: | - echo -e "[distutils]" >> ~/.pypirc - echo -e "index-servers =" >> ~/.pypirc - echo -e " pypi" >> ~/.pypirc - echo -e " longtailbio" >> ~/.pypirc - echo -e "" >> ~/.pypirc - echo -e "[pypi]" >> ~/.pypirc - echo -e "" >> ~/.pypirc - echo -e "[longtailbio]" >> ~/.pypirc - echo -e "repository: https://pypi.longtailbio.com" >> ~/.pypirc - echo -e "username: $LT_PYPI_USERNAME" >> ~/.pypirc - echo -e "password: $LT_PYPI_PASSWORD" >> ~/.pypirc - - - run: - name: Init ~/.pip/pip.conf - command: | - mkdir ~/.pip - echo -e "[global]" >> ~/.pip/pip.conf - echo -e "extra-index-url = https://pypi.longtailbio.com/simple/" >> ~/.pip/pip.conf + <<: *pypi_step - run: name: Run tests with py35 @@ -50,3 +51,61 @@ jobs: name: Run pylint command: tox -e pylint when: always + + deploy: + docker: + - image: circleci/python:3.6.3-jessie + + steps: + - checkout + + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum "requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: + name: Install Python Dependencies + command: | + python3 -m venv venv + . venv/bin/activate + pip install -r requirements.txt + + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + + - run: + <<: *pypi_step + + - run: + name: Build package + command: | + . venv/bin/activate + python setup.py bdist_wheel --universal + + - run: + name: Upload to Longtail PyPi + command: | + . venv/bin/activate + pip install twine + twine upload -r longtailbio dist/* + + + +workflows: + version: 2 + build_and_deploy: + jobs: + - test_package: + context: longtail-pypi + - deploy: + context: longtail-pypi + filters: + branches: + only: master + requires: + - test_package From 1b0155c5bbd0384fad56f097f7edffe29ae8256d Mon Sep 17 00:00:00 2001 From: Benjamin Chrobot Date: Mon, 1 Oct 2018 16:56:45 -0400 Subject: [PATCH 6/6] Fix yaml anchor usage. --- .circleci/config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c839ee..acb9933 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,8 +32,7 @@ jobs: - checkout - - run: - <<: *pypi_step + - run: *pypi_step - run: name: Run tests with py35 @@ -78,8 +77,7 @@ jobs: - ./venv key: v1-dependencies-{{ checksum "requirements.txt" }} - - run: - <<: *pypi_step + - run: *pypi_step - run: name: Build package