diff --git a/.github/workflows/python-example-image.yml b/.github/workflows/python-example-image.yml new file mode 100644 index 00000000..3b6fd9d5 --- /dev/null +++ b/.github/workflows/python-example-image.yml @@ -0,0 +1,36 @@ +--- +name: Python Example Docker Image + +on: + pull_request: + paths: + - 'clients/python/**' + push: + branches: + - main + paths: + - 'clients/python/**' + +jobs: + build: + runs-on: ubuntu-24.04 + strategy: + name: build-python-example + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Build and push Python example image + uses: getsentry/action-build-and-push-images@a53f146fc1ea3cb404f2dcf7378f5b60dd98d3ca + with: + image_name: 'taskbroker-python-example' + platforms: linux/${{ matrix.platform }} + dockerfile_path: './clients/python/Dockerfile.example' + dockerfile_context: './clients/python' + ghcr: true + publish_on_pr: true + google_ar: false + tag_nightly: false + tag_latest: false diff --git a/clients/python/Dockerfile.example b/clients/python/Dockerfile.example new file mode 100644 index 00000000..c56b3428 --- /dev/null +++ b/clients/python/Dockerfile.example @@ -0,0 +1,34 @@ +# Python example application Docker image +FROM python:3.12-slim-bookworm + +# Install system dependencies +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ + gcc \ + g++ \ + librdkafka-dev \ + && rm -rf /var/lib/apt/lists/* + +# Create app directory +WORKDIR /app + +# Install uv for package management +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +# Copy Python package files +COPY pyproject.toml README.md /app/ + +# Copy source code +COPY src/ /app/src/ + +# Install the package with examples dependencies +RUN uv pip install --system -e ".[examples]" + +# Environment variables for Kafka and Redis hosts +ENV KAFKA_HOST=localhost:9092 +ENV REDIS_HOST=localhost +ENV REDIS_PORT=6379 + +# Set the entrypoint to the examples CLI +ENTRYPOINT ["python", "-m", "examples.cli"] +CMD ["--help"] diff --git a/clients/python/README.md b/clients/python/README.md index d54303b7..28c658ef 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -9,31 +9,4 @@ This package provides python client libraries for taskbroker. The client librari ## Example application -The `src/examples` directory contains a sample application that is used in tests, and can be run locally. First -install the required dependencies: - -```bash -# Install the required and development dependencies -uv sync --dev - -# Install the optional dependencies for the example application -uv sync --extra examples -``` -Before running the examples, make sure you have: - -- Kafka running -- Redis running (for the scheduler) -- Taskbroker running. Use `cargo run` in the repository root for this. - -With all of those pre-requisites complete, you can run the example application: - -```bash -# Generate 5 tasks -python src/examples/cli.py generate --count 5 - -# Run the scheduler which emits a task every 1m -python src/examples/cli.py scheduler - -# Run the worker -python src/examples/cli.py scheduler -``` +You can find an example application in the `src/examples` directory. diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index becfbca2..7f969c63 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -33,11 +33,12 @@ dev = [ ] [project.optional-dependencies] examples = [ - "click>=8.3" + "click>=8.3", + "setuptools>=80.0", ] [build-system] -requires = ["uv_build>=0.8.2,<0.9.0"] +requires = ["uv_build>=0.8.2"] build-backend = "uv_build" [tool.uv] diff --git a/clients/python/src/examples/README.md b/clients/python/src/examples/README.md new file mode 100644 index 00000000..259a7e60 --- /dev/null +++ b/clients/python/src/examples/README.md @@ -0,0 +1,66 @@ +# Taskbroker client example application + +This directory contains a sample application that is used by `taskbroker_client` +tests, and can be run locally as a minimal example client for taskbroker. + +## Running the example application + +First, install the required dependencies: + +```bash +# Install the required and development dependencies +uv sync --dev + +# Install the optional dependencies for the example application +uv sync --extra examples +``` +Before running the examples, make sure you have: + +- Kafka running +- Redis running (for the scheduler) +- Taskbroker running. Use `cargo run` in the repository root for this. + +With all of those pre-requisites complete, you can run the example application: + +```bash +# Generate 5 tasks +python src/examples/cli.py generate --count 5 + +# Run the scheduler which emits a task every 1m +python src/examples/cli.py scheduler + +# Run the worker +python src/examples/cli.py worker +``` + +## Docker image + +The example application also comes with a docker image that will contain the +example application and all required dependencies. This can be useful for end +to end testing changes to taskbroker. + +### Building the image + +```bash +cd clients/python +docker build -t taskbroker-example -f Dockerfile.example . +``` + +### Running the image + +You'll need to have both Kafka running to produce tasks, and Redis +running to run a scheduler. + +```bash +# Spawn one timed_task activation +docker run \ + -e KAFKA_HOST=kafka-host:9092 \ + -e REDIS_HOST=redis-host \ + taskbroker-example spawn --count=1 + +# Run a worker +docker run \ + -e KAFKA_HOST=kafka-host:9092 \ + -e REDIS_HOST=redis-host \ + taskbroker-example worker --rpc-host=127.0.0.1:50051 +``` diff --git a/clients/python/src/examples/app.py b/clients/python/src/examples/app.py index df059332..30a31639 100644 --- a/clients/python/src/examples/app.py +++ b/clients/python/src/examples/app.py @@ -1,3 +1,5 @@ +import os + from arroyo.backends.kafka import KafkaProducer from examples.store import StubAtMostOnce @@ -5,9 +7,9 @@ def producer_factory(topic: str) -> KafkaProducer: - # TODO use env vars for kafka host/port + kafka_host = os.getenv("KAFKA_HOST") or "127.0.0.1:9092" config = { - "bootstrap.servers": "127.0.0.1:9092", + "bootstrap.servers": kafka_host, "compression.type": "lz4", "message.max.bytes": 50000000, # 50MB }