From 2100145bad7b6481feb50dbcf6e1689f058c49a9 Mon Sep 17 00:00:00 2001 From: Mark Story Date: Mon, 26 Jan 2026 17:45:11 -0500 Subject: [PATCH] feat: Add docker image and readme for example app When the example application or client libraries are changed we should make new docker images. These docker images could be useful in sandbox testing as the example application has less overhead and complexity than sentry does. Refs STREAM-717 --- .github/workflows/python-example-image.yml | 36 ++++++++++++ clients/python/Dockerfile.example | 34 +++++++++++ clients/python/README.md | 29 +--------- clients/python/pyproject.toml | 5 +- clients/python/src/examples/README.md | 66 ++++++++++++++++++++++ clients/python/src/examples/app.py | 6 +- 6 files changed, 144 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/python-example-image.yml create mode 100644 clients/python/Dockerfile.example create mode 100644 clients/python/src/examples/README.md diff --git a/.github/workflows/python-example-image.yml b/.github/workflows/python-example-image.yml new file mode 100644 index 00000000..3b6fd9d5 --- /dev/null +++ b/.github/workflows/python-example-image.yml @@ -0,0 +1,36 @@ +--- +name: Python Example Docker Image + +on: + pull_request: + paths: + - 'clients/python/**' + push: + branches: + - main + paths: + - 'clients/python/**' + +jobs: + build: + runs-on: ubuntu-24.04 + strategy: + name: build-python-example + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Build and push Python example image + uses: getsentry/action-build-and-push-images@a53f146fc1ea3cb404f2dcf7378f5b60dd98d3ca + with: + image_name: 'taskbroker-python-example' + platforms: linux/${{ matrix.platform }} + dockerfile_path: './clients/python/Dockerfile.example' + dockerfile_context: './clients/python' + ghcr: true + publish_on_pr: true + google_ar: false + tag_nightly: false + tag_latest: false diff --git a/clients/python/Dockerfile.example b/clients/python/Dockerfile.example new file mode 100644 index 00000000..c56b3428 --- /dev/null +++ b/clients/python/Dockerfile.example @@ -0,0 +1,34 @@ +# Python example application Docker image +FROM python:3.12-slim-bookworm + +# Install system dependencies +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ + gcc \ + g++ \ + librdkafka-dev \ + && rm -rf /var/lib/apt/lists/* + +# Create app directory +WORKDIR /app + +# Install uv for package management +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +# Copy Python package files +COPY pyproject.toml README.md /app/ + +# Copy source code +COPY src/ /app/src/ + +# Install the package with examples dependencies +RUN uv pip install --system -e ".[examples]" + +# Environment variables for Kafka and Redis hosts +ENV KAFKA_HOST=localhost:9092 +ENV REDIS_HOST=localhost +ENV REDIS_PORT=6379 + +# Set the entrypoint to the examples CLI +ENTRYPOINT ["python", "-m", "examples.cli"] +CMD ["--help"] diff --git a/clients/python/README.md b/clients/python/README.md index d54303b7..28c658ef 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -9,31 +9,4 @@ This package provides python client libraries for taskbroker. The client librari ## Example application -The `src/examples` directory contains a sample application that is used in tests, and can be run locally. First -install the required dependencies: - -```bash -# Install the required and development dependencies -uv sync --dev - -# Install the optional dependencies for the example application -uv sync --extra examples -``` -Before running the examples, make sure you have: - -- Kafka running -- Redis running (for the scheduler) -- Taskbroker running. Use `cargo run` in the repository root for this. - -With all of those pre-requisites complete, you can run the example application: - -```bash -# Generate 5 tasks -python src/examples/cli.py generate --count 5 - -# Run the scheduler which emits a task every 1m -python src/examples/cli.py scheduler - -# Run the worker -python src/examples/cli.py scheduler -``` +You can find an example application in the `src/examples` directory. diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index becfbca2..7f969c63 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -33,11 +33,12 @@ dev = [ ] [project.optional-dependencies] examples = [ - "click>=8.3" + "click>=8.3", + "setuptools>=80.0", ] [build-system] -requires = ["uv_build>=0.8.2,<0.9.0"] +requires = ["uv_build>=0.8.2"] build-backend = "uv_build" [tool.uv] diff --git a/clients/python/src/examples/README.md b/clients/python/src/examples/README.md new file mode 100644 index 00000000..259a7e60 --- /dev/null +++ b/clients/python/src/examples/README.md @@ -0,0 +1,66 @@ +# Taskbroker client example application + +This directory contains a sample application that is used by `taskbroker_client` +tests, and can be run locally as a minimal example client for taskbroker. + +## Running the example application + +First, install the required dependencies: + +```bash +# Install the required and development dependencies +uv sync --dev + +# Install the optional dependencies for the example application +uv sync --extra examples +``` +Before running the examples, make sure you have: + +- Kafka running +- Redis running (for the scheduler) +- Taskbroker running. Use `cargo run` in the repository root for this. + +With all of those pre-requisites complete, you can run the example application: + +```bash +# Generate 5 tasks +python src/examples/cli.py generate --count 5 + +# Run the scheduler which emits a task every 1m +python src/examples/cli.py scheduler + +# Run the worker +python src/examples/cli.py worker +``` + +## Docker image + +The example application also comes with a docker image that will contain the +example application and all required dependencies. This can be useful for end +to end testing changes to taskbroker. + +### Building the image + +```bash +cd clients/python +docker build -t taskbroker-example -f Dockerfile.example . +``` + +### Running the image + +You'll need to have both Kafka running to produce tasks, and Redis +running to run a scheduler. + +```bash +# Spawn one timed_task activation +docker run \ + -e KAFKA_HOST=kafka-host:9092 \ + -e REDIS_HOST=redis-host \ + taskbroker-example spawn --count=1 + +# Run a worker +docker run \ + -e KAFKA_HOST=kafka-host:9092 \ + -e REDIS_HOST=redis-host \ + taskbroker-example worker --rpc-host=127.0.0.1:50051 +``` diff --git a/clients/python/src/examples/app.py b/clients/python/src/examples/app.py index df059332..30a31639 100644 --- a/clients/python/src/examples/app.py +++ b/clients/python/src/examples/app.py @@ -1,3 +1,5 @@ +import os + from arroyo.backends.kafka import KafkaProducer from examples.store import StubAtMostOnce @@ -5,9 +7,9 @@ def producer_factory(topic: str) -> KafkaProducer: - # TODO use env vars for kafka host/port + kafka_host = os.getenv("KAFKA_HOST") or "127.0.0.1:9092" config = { - "bootstrap.servers": "127.0.0.1:9092", + "bootstrap.servers": kafka_host, "compression.type": "lz4", "message.max.bytes": 50000000, # 50MB }