testing.yml

name: Tests

on:
  push:
    paths-ignore:
    - 'docs/arch/**'
  pull_request:
    paths-ignore:
    - 'docs/arch/**'
  schedule:
  - cron:  '0 3 * * *'

jobs:
  lint:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 40
      matrix:
        check: [changelog, whitespace, style, commit, import, docs, lines]
        python-version: [3.7]
        node-version: [12.x]

    steps:
    - uses: actions/checkout@v2
    - name: Checkout full upstream repo
      run: |
        git remote set-url origin https://github.com/intel/dffml
        git fetch --prune --unshallow
        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
        git config --global user.email "dffml@intel.com"
        git config --global user.name "DFFML CI/CD"
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Use Node.js ${{ matrix.node-version }}
      uses: actions/setup-node@v1
      with:
        node-version: ${{ matrix.node-version }}
    - name: Get pip cache
      id: pip-cache
      run: |
        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
    - name: pip cache
      uses: actions/cache@v1
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    - name: Install dependencies
      run: |
        set -x
        ./.ci/deps.sh ${{ matrix.check }}
    - name: Run check
      run: |
        export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
        SSH_DFFML_GH_PAGES=${{ secrets.SSH_DFFML_GH_PAGES }} ./.ci/run.sh ${{ matrix.check }}

  container:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2
    - name: Install dependencies
      run: |
        set -x
        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
        sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
        sudo apt-get update && sudo apt-get install -y docker-ce docker-ce-cli containerd.io
    - name: Run check
      run: |
        ./.ci/run.sh container

  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 100
      matrix:
        plugin:
        - .
        - examples/shouldi
        - model/daal4py
        - model/tensorflow
        - model/tensorflow_hub
        - model/scratch
        - model/scikit
        - model/vowpalWabbit
        - model/autosklearn
        - model/spacy
        - model/pytorch
        - model/xgboost
        - operations/binsec
        - operations/data
        - operations/deploy
        - operations/image
        - operations/nlp
        - source/mysql
        - feature/git
        - feature/auth
        - service/http
        - configloader/yaml
        - configloader/image
        python-version: [3.7, 3.8]

    steps:
    - uses: actions/checkout@v2
    - name: Checkout full upstream repo
      run: |
        git remote set-url origin https://github.com/intel/dffml
        git fetch --prune --unshallow
        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
        git config --global user.email "dffml@intel.com"
        git config --global user.name "DFFML CI/CD"
    - name: Remove unused software
      run: |
        sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Get pip cache
      id: pip-cache
      run: |
        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
    - name: pip cache
      uses: actions/cache@v1
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    - name: shouldi test binary cache
      uses: actions/cache@v1
      with:
        path: examples/shouldi/tests/downloads
        key: ${{ hashFiles('examples/shouldi/tests/binaries.py') }}
        restore-keys: |
          ${{ runner.os }}-shouldi-test-binaries-
    - name: Install dependencies
      run: |
        set -x
        ./.ci/deps.sh ${{ matrix.plugin }}
    - name: Test
      run: |
        set -x
        export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
        export PYTHON=python${{ matrix.python-version }}
        export PLUGIN=${{ matrix.plugin }}
        export LOGGING=debug
        export PATH="${HOME}/.local/bin:${PATH}"
        export PYPI_TOKENS=$(mktemp)
        cat > ${PYPI_TOKENS} <<EOF
        examples/shouldi=${{ secrets.PYPI_SHOULDI }}
        model/daal4py=${{ secrets.PYPI_MODEL_DAAL4PY }}
        model/tensorflow=${{ secrets.PYPI_MODEL_TENSORFLOW }}
        model/tensorflow_hub=${{ secrets.PYPI_MODEL_TENSORFLOW_HUB }}
        model/scratch=${{ secrets.PYPI_MODEL_SCRATCH }}
        model/scikit=${{ secrets.PYPI_MODEL_SCIKIT }}
        model/spacy=${{ secrets.PYPI_MODEL_SPACY }}
        model/vowpalWabbit=${{ secrets.PYPI_MODEL_VOWPALWABBIT }}
        model/autosklearn=${{ secrets.PYPI_MODEL_AUTOSKLEARN }}
        model/pytorch=${{ secrets.PYPI_MODEL_PYTORCH }}
        model/xgboost=${{ secrets.PYPI_MODEL_XGBOOST }}
        source/mysql=${{ secrets.PYPI_SOURCE_MYSQL }}
        feature/git=${{ secrets.PYPI_FEATURE_GIT }}
        feature/auth=${{ secrets.PYPI_FEATURE_AUTH }}
        operations/binsec=${{ secrets.PYPI_OPERATIONS_BINSEC }}
        operations/data=${{ secrets.PYPI_OPERATIONS_DATA }}
        operations/deploy=${{ secrets.PYPI_OPERATIONS_DEPLOY }}
        operations/image=${{ secrets.PYPI_OPERATIONS_IMAGE }}
        operations/nlp=${{ secrets.PYPI_OPERATIONS_NLP }}
        service/http=${{ secrets.PYPI_SERVICE_HTTP }}
        configloader/yaml=${{ secrets.PYPI_CONFIG_YAML }}
        configloader/image=${{ secrets.PYPI_CONFIG_IMAGE }}
        EOF
        export TWINE_USERNAME=__token__
        if [ "x${PLUGIN}" = "x." ]; then
          export TWINE_PASSWORD=${{ secrets.PYPI_DFFML }}
        else
          export TWINE_PASSWORD=$(grep "${PLUGIN}=" "${PYPI_TOKENS}" | sed 's/^[^=]*=//g')
        fi
        rm ${PYPI_TOKENS}
        ./.ci/run.sh "${PLUGIN}"
        if [ "x${PLUGIN}" = "x." ]; then
          CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} codecov
        fi

  tutorials:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 100
      matrix:
        python-version: [3.7]
        docs:
        - docs/cli.rst
        - docs/examples/dataflows.rst
        - docs/examples/integration.rst
        - docs/examples/or_covid_data_by_county.rst
        - docs/examples/shouldi.rst
        - docs/examples/innersource/swportal.rst
        - docs/examples/innersource/microservice.rst
        - docs/examples/icecream_sales.rst
        - docs/examples/data_cleanup/data_cleanup.rst
        - docs/examples/data_cleanup/data_cleanup_classfication.rst
        - docs/installation.rst
        - docs/troubleshooting.rst
        - docs/tutorials/accuracy/mse.rst
        - docs/tutorials/dataflows/io.rst
        - docs/tutorials/dataflows/nlp.rst
        - docs/tutorials/models/iris.rst
        - docs/tutorials/models/package.rst
        - docs/tutorials/models/docs.rst
        - docs/tutorials/models/slr.rst
        - docs/tutorials/sources/complex.rst
        - docs/tutorials/sources/file.rst

    steps:
    - uses: actions/checkout@v2
    - name: Checkout full upstream repo
      run: |
        git remote set-url origin https://github.com/intel/dffml
        git fetch --prune --unshallow
        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
        git config --global user.email "dffml@intel.com"
        git config --global user.name "DFFML CI/CD"
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Get pip cache
      id: pip-cache
      run: |
        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
    - name: pip cache
      uses: actions/cache@v1
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    - name: Install dependencies
      run: |
        set -x
        ./.ci/deps.sh .
    - name: Test
      run: |
        set -x
        export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
        export PYTHON=python${{ matrix.python-version }}
        export PLUGIN=${{ matrix.plugin }}
        export LOGGING=debug
        export PATH="${HOME}/.local/bin:${PATH}"
        ./.ci/run.sh consoletest ${{ matrix.docs }}

  macos:
    runs-on: macos-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.7, 3.8]
    steps:
    - uses: actions/checkout@v2
    - name: Checkout full upstream repo
      run: |
        git remote set-url origin https://github.com/intel/dffml
        git fetch --prune --unshallow
        git config --global user.email "dffml@intel.com"
        git config --global user.name "DFFML CI/CD"
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Get pip cache
      id: pip-cache
      run: |
        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
    - name: pip cache
      uses: actions/cache@v1
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    - name: Setup DFFML
      run: |
        pip install -U pip setuptools wheel
        pip install -e .[dev]
        dffml service dev install -skip model/daal4py
        # XGBoost requires libomp on OSX
        brew install libomp
    - name: Test
      run: |
        python -m unittest discover -v
        python -m pip freeze