Fixes #135 - Add a job to fetch Webcompat Knowldege Base related bugs and store them in BQ

.circleci/config.yml

@@ -279,6 +279,22 @@ jobs:
             command: docker run app:build pytest
 
 
+  build-job-webcompat-kb:
+    docker:
+      - image: << pipeline.parameters.git-image >>
+    steps:
+      - checkout
+      - compare-branch:
+          pattern: ^jobs/webcompat-kb/
+      - setup_remote_docker:
+          version: << pipeline.parameters.docker-version >>
+      - run:
+          name: Build Docker image
+          command: docker build -t app:build jobs/webcompat-kb/
+      - run:
+          name: Test Code
+          command: docker run app:build pytest --flake8 --black
+
 workflows:
   docker-etl:
     jobs:
@@ -475,3 +491,17 @@ workflows:
             branches:
               only: main
 
+
+  job-webcompat-kb:
+    jobs:
+      - build-job-webcompat-kb
+      - gcp-gcr/build-and-push-image:
+          context: data-eng-airflow-gcr
+          docker-context: jobs/webcompat-kb/
+          path: jobs/webcompat-kb/
+          image: webcompat-kb_docker_etl
+          requires:
+            - build-job-webcompat-kb
+          filters:
+            branches:
+              only: main

jobs/webcompat-kb/.dockerignore

@@ -0,0 +1,7 @@
+.ci_job.yaml
+.ci_workflow.yaml
+.DS_Store
+*.pyc
+.pytest_cache/
+__pycache__/
+venv/

jobs/webcompat-kb/.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 88

jobs/webcompat-kb/.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+*.pyc
+__pycache__/
+venv/

jobs/webcompat-kb/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.8
+MAINTAINER <kberezina@mozilla.com>
+
+# https://github.com/mozilla-services/Dockerflow/blob/master/docs/building-container.md
+ARG USER_ID="10001"
+ARG GROUP_ID="app"
+ARG HOME="/app"
+
+ENV HOME=${HOME}
+RUN groupadd --gid ${USER_ID} ${GROUP_ID} && \
+    useradd --create-home --uid ${USER_ID} --gid ${GROUP_ID} --home-dir ${HOME} ${GROUP_ID}
+
+WORKDIR ${HOME}
+
+RUN pip install --upgrade pip
+
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+COPY . .
+
+RUN pip install .
+
+# Drop root and change ownership of the application folder to the user
+RUN chown -R ${USER_ID}:${GROUP_ID} ${HOME}
+USER ${USER_ID}

jobs/webcompat-kb/README.md

@@ -0,0 +1,47 @@
+# Web compatibility Knowledge Base bugs import from bugzilla 
+
+This job fetches bugzilla bugs from Web Compatibility > Knowledge Base component, 
+as well as their core bugs dependencies and breakage reports and puts them into BQ.  
+
+## Usage
+
+This script is intended to be run in a docker container.
+Build the docker image with:
+
+```sh
+docker build -t webcompat-kb .
+```
+
+To run locally, first install dependencies in `jobs/webcompat-kb`:
+
+```sh
+pip install -r requirements.txt
+```
+
+And then run the script after authentication with gcloud: 
+
+```sh
+gcloud auth application-default login
+python3 webcompat_kb/main.py --bq_project_id=<your_project_id> --bq_dataset_id=<your_dataset_id>
+```
+
+## Development
+
+Run tests with:
+
+```sh
+pytest
+```
+
+`flake8` and `black` are included for code linting and formatting:
+
+```sh
+pytest --black --flake8
+```
+
+or
+
+```sh
+flake8 webcompat_kb/ tests/
+black --diff webcompat_kb/ tests/
+```

jobs/webcompat-kb/ci_job.yaml

@@ -0,0 +1,15 @@
+build-job-webcompat-kb:
+  docker:
+    - image: << pipeline.parameters.git-image >>
+  steps:
+    - checkout
+    - compare-branch:
+        pattern: ^jobs/webcompat-kb/
+    - setup_remote_docker:
+        version: << pipeline.parameters.docker-version >>
+    - run:
+        name: Build Docker image
+        command: docker build -t app:build jobs/webcompat-kb/
+    - run:
+        name: Test Code
+        command: docker run app:build pytest --flake8 --black

jobs/webcompat-kb/ci_workflow.yaml

@@ -0,0 +1,13 @@
+job-webcompat-kb:
+  jobs:
+    - build-job-webcompat-kb
+    - gcp-gcr/build-and-push-image:
+        context: data-eng-airflow-gcr
+        docker-context: jobs/webcompat-kb/
+        path: jobs/webcompat-kb/
+        image: webcompat-kb_docker_etl
+        requires:
+          - build-job-webcompat-kb
+        filters:
+          branches:
+            only: main

jobs/webcompat-kb/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+testpaths =
+    tests

jobs/webcompat-kb/requirements.txt

@@ -0,0 +1,6 @@
+click==8.0.4
+flake8==3.8.4
+google-cloud-bigquery==3.11.4
+pytest==6.0.2
+pytest-black==0.3.11
+pytest-flake8==1.0.6

jobs/webcompat-kb/setup.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+from setuptools import setup, find_packages
+
+readme = open("README.md").read()
+
+setup(
+    name="webcompat-kb",
+    version="0.1.0",
+    author="Mozilla Corporation",
+    packages=find_packages(include=["webcompat_kb"]),
+    long_description=readme,
+    include_package_data=True,
+    license="MPL 2.0",
+)