From 9b1779983c5042784d2839ac30ee8b0b31b0e4b5 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Tue, 11 Nov 2025 18:19:33 -0500 Subject: [PATCH 01/11] refactor public api for endpoints, added tests --- .../api_public/src => api_public}/__init__.py | 0 .../src/app => api_public/core}/.gitkeep | 0 .../src => api_public}/core/__init__.py | 0 .../src => api_public}/core/_cache.py | 0 .../src => api_public}/core/config.py | 0 .../core/fastapi/dependencies/kafka.py | 0 .../core/fastapi/dependencies/session.py | 2 +- .../fastapi/dependencies/to_jagex_name.py | 0 .../core/fastapi/middleware/__init__.py | 0 .../core/fastapi/middleware/logging.py | 0 .../core/fastapi/middleware/metrics.py | 0 .../src => api_public}/core/logging.py | 0 .../src => api_public}/core/server.py | 158 ++++----- bases/api_public/feedback/__init__.py | 3 + .../feedback/repository.py} | 2 +- .../feedback/routes.py} | 10 +- .../feedback/schemas.py} | 8 + bases/api_public/labels/__init__.py | 3 + .../labels/repository.py} | 0 .../labels.py => api_public/labels/routes.py} | 6 +- .../label.py => api_public/labels/schemas.py} | 0 bases/api_public/player/__init__.py | 3 + .../player/repository.py} | 322 +++++++++--------- .../player.py => api_public/player/routes.py} | 212 ++++++------ .../player/schemas.py} | 53 +++ bases/api_public/reports/__init__.py | 3 + .../reports/repository.py} | 224 ++++++------ .../reports/routes.py} | 140 ++++---- .../reports/schemas.py} | 12 +- bases/api_public/routes.py | 18 + .../shared}/__init__.py | 0 .../ok.py => api_public/shared/responses.py} | 0 .../api_public/src/api/__init__.py | 6 - .../bot_detector/api_public/src/api/readme.md | 1 - .../api_public/src/api/v2/__init__.py | 9 - .../bot_detector/api_public/src/app/readme.md | 6 - .../api_public/src/app/repositories/.gitkeep | 0 .../src/app/views/input/_metadata.py | 5 - .../src/app/views/response/feedback.py | 11 - .../src/app/views/response/feedback_score.py | 8 - .../src/app/views/response/player.py | 19 -- .../src/app/views/response/prediction.py | 27 -- .../src/app/views/response/report_score.py | 9 - .../bot_detector/api_public/src/core/.gitkeep | 0 docker-compose-dev.yml | 4 +- docker-compose.yml | 6 +- projects/api_public/Dockerfile | 2 +- projects/api_public/pyproject.toml | 4 +- 48 files changed, 643 insertions(+), 653 deletions(-) rename bases/{bot_detector/api_public/src => api_public}/__init__.py (100%) rename bases/{bot_detector/api_public/src/app => api_public/core}/.gitkeep (100%) rename bases/{bot_detector/api_public/src => api_public}/core/__init__.py (100%) rename bases/{bot_detector/api_public/src => api_public}/core/_cache.py (100%) rename bases/{bot_detector/api_public/src => api_public}/core/config.py (100%) rename bases/{bot_detector/api_public/src => api_public}/core/fastapi/dependencies/kafka.py (100%) rename bases/{bot_detector/api_public/src => api_public}/core/fastapi/dependencies/session.py (89%) rename bases/{bot_detector/api_public/src => api_public}/core/fastapi/dependencies/to_jagex_name.py (100%) rename bases/{bot_detector/api_public/src => api_public}/core/fastapi/middleware/__init__.py (100%) rename bases/{bot_detector/api_public/src => api_public}/core/fastapi/middleware/logging.py (100%) rename bases/{bot_detector/api_public/src => api_public}/core/fastapi/middleware/metrics.py (100%) rename bases/{bot_detector/api_public/src => api_public}/core/logging.py (100%) rename bases/{bot_detector/api_public/src => api_public}/core/server.py (85%) create mode 100644 bases/api_public/feedback/__init__.py rename bases/{bot_detector/api_public/src/app/repositories/feedback.py => api_public/feedback/repository.py} (96%) rename bases/{bot_detector/api_public/src/api/v2/feedback.py => api_public/feedback/routes.py} (62%) rename bases/{bot_detector/api_public/src/app/views/input/feedback.py => api_public/feedback/schemas.py} (88%) create mode 100644 bases/api_public/labels/__init__.py rename bases/{bot_detector/api_public/src/app/repositories/labels.py => api_public/labels/repository.py} (100%) rename bases/{bot_detector/api_public/src/api/v2/labels.py => api_public/labels/routes.py} (81%) rename bases/{bot_detector/api_public/src/app/views/response/label.py => api_public/labels/schemas.py} (100%) create mode 100644 bases/api_public/player/__init__.py rename bases/{bot_detector/api_public/src/app/repositories/player.py => api_public/player/repository.py} (94%) rename bases/{bot_detector/api_public/src/api/v2/player.py => api_public/player/routes.py} (83%) rename bases/{bot_detector/api_public/src/app/views/player.py => api_public/player/schemas.py} (50%) create mode 100644 bases/api_public/reports/__init__.py rename bases/{bot_detector/api_public/src/app/repositories/report.py => api_public/reports/repository.py} (94%) rename bases/{bot_detector/api_public/src/api/v2/report.py => api_public/reports/routes.py} (82%) rename bases/{bot_detector/api_public/src/app/views/input/report.py => api_public/reports/schemas.py} (89%) create mode 100644 bases/api_public/routes.py rename bases/{bot_detector/api_public/src/app/repositories => api_public/shared}/__init__.py (100%) rename bases/{bot_detector/api_public/src/app/views/response/ok.py => api_public/shared/responses.py} (100%) delete mode 100644 bases/bot_detector/api_public/src/api/__init__.py delete mode 100644 bases/bot_detector/api_public/src/api/readme.md delete mode 100644 bases/bot_detector/api_public/src/api/v2/__init__.py delete mode 100644 bases/bot_detector/api_public/src/app/readme.md delete mode 100644 bases/bot_detector/api_public/src/app/repositories/.gitkeep delete mode 100644 bases/bot_detector/api_public/src/app/views/input/_metadata.py delete mode 100644 bases/bot_detector/api_public/src/app/views/response/feedback.py delete mode 100644 bases/bot_detector/api_public/src/app/views/response/feedback_score.py delete mode 100644 bases/bot_detector/api_public/src/app/views/response/player.py delete mode 100644 bases/bot_detector/api_public/src/app/views/response/prediction.py delete mode 100644 bases/bot_detector/api_public/src/app/views/response/report_score.py delete mode 100644 bases/bot_detector/api_public/src/core/.gitkeep diff --git a/bases/bot_detector/api_public/src/__init__.py b/bases/api_public/__init__.py similarity index 100% rename from bases/bot_detector/api_public/src/__init__.py rename to bases/api_public/__init__.py diff --git a/bases/bot_detector/api_public/src/app/.gitkeep b/bases/api_public/core/.gitkeep similarity index 100% rename from bases/bot_detector/api_public/src/app/.gitkeep rename to bases/api_public/core/.gitkeep diff --git a/bases/bot_detector/api_public/src/core/__init__.py b/bases/api_public/core/__init__.py similarity index 100% rename from bases/bot_detector/api_public/src/core/__init__.py rename to bases/api_public/core/__init__.py diff --git a/bases/bot_detector/api_public/src/core/_cache.py b/bases/api_public/core/_cache.py similarity index 100% rename from bases/bot_detector/api_public/src/core/_cache.py rename to bases/api_public/core/_cache.py diff --git a/bases/bot_detector/api_public/src/core/config.py b/bases/api_public/core/config.py similarity index 100% rename from bases/bot_detector/api_public/src/core/config.py rename to bases/api_public/core/config.py diff --git a/bases/bot_detector/api_public/src/core/fastapi/dependencies/kafka.py b/bases/api_public/core/fastapi/dependencies/kafka.py similarity index 100% rename from bases/bot_detector/api_public/src/core/fastapi/dependencies/kafka.py rename to bases/api_public/core/fastapi/dependencies/kafka.py diff --git a/bases/bot_detector/api_public/src/core/fastapi/dependencies/session.py b/bases/api_public/core/fastapi/dependencies/session.py similarity index 89% rename from bases/bot_detector/api_public/src/core/fastapi/dependencies/session.py rename to bases/api_public/core/fastapi/dependencies/session.py index 8bad32d..040fdd9 100644 --- a/bases/bot_detector/api_public/src/core/fastapi/dependencies/session.py +++ b/bases/api_public/core/fastapi/dependencies/session.py @@ -1,4 +1,4 @@ -from bot_detector.api_public.src.core.config import DB_SEMAPHORE, settings +from bases.api_public.core.config import DB_SEMAPHORE, settings from bot_detector.database import Settings as DBSettings from bot_detector.database import get_session_factory from sqlalchemy.ext.asyncio import AsyncSession diff --git a/bases/bot_detector/api_public/src/core/fastapi/dependencies/to_jagex_name.py b/bases/api_public/core/fastapi/dependencies/to_jagex_name.py similarity index 100% rename from bases/bot_detector/api_public/src/core/fastapi/dependencies/to_jagex_name.py rename to bases/api_public/core/fastapi/dependencies/to_jagex_name.py diff --git a/bases/bot_detector/api_public/src/core/fastapi/middleware/__init__.py b/bases/api_public/core/fastapi/middleware/__init__.py similarity index 100% rename from bases/bot_detector/api_public/src/core/fastapi/middleware/__init__.py rename to bases/api_public/core/fastapi/middleware/__init__.py diff --git a/bases/bot_detector/api_public/src/core/fastapi/middleware/logging.py b/bases/api_public/core/fastapi/middleware/logging.py similarity index 100% rename from bases/bot_detector/api_public/src/core/fastapi/middleware/logging.py rename to bases/api_public/core/fastapi/middleware/logging.py diff --git a/bases/bot_detector/api_public/src/core/fastapi/middleware/metrics.py b/bases/api_public/core/fastapi/middleware/metrics.py similarity index 100% rename from bases/bot_detector/api_public/src/core/fastapi/middleware/metrics.py rename to bases/api_public/core/fastapi/middleware/metrics.py diff --git a/bases/bot_detector/api_public/src/core/logging.py b/bases/api_public/core/logging.py similarity index 100% rename from bases/bot_detector/api_public/src/core/logging.py rename to bases/api_public/core/logging.py diff --git a/bases/bot_detector/api_public/src/core/server.py b/bases/api_public/core/server.py similarity index 85% rename from bases/bot_detector/api_public/src/core/server.py rename to bases/api_public/core/server.py index 073f065..e1ca6af 100644 --- a/bases/bot_detector/api_public/src/core/server.py +++ b/bases/api_public/core/server.py @@ -1,79 +1,79 @@ -import logging -from contextlib import asynccontextmanager - -from bot_detector.api_public.src import api -from bot_detector.api_public.src.core.fastapi.dependencies.kafka import kafka_manager -from bot_detector.api_public.src.core.fastapi.middleware import ( - LoggingMiddleware, - PrometheusMiddleware, -) -from bot_detector.kafka import Settings as KafkaSettings -from bot_detector.kafka.repositories import RepoReportsToInsertProducer -from fastapi import FastAPI -from fastapi.middleware import Middleware -from fastapi.middleware.cors import CORSMiddleware -from prometheus_client import start_http_server - -logger = logging.getLogger(__name__) - - -def init_routers(_app: FastAPI) -> None: - _app.include_router(api.router) - - -def make_middleware() -> list[Middleware]: - middleware = [ - Middleware( - CORSMiddleware, - allow_origins=[ - "http://osrsbotdetector.com/", - "https://osrsbotdetector.com/", - "http://localhost", - "http://localhost:8080", - ], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ), - Middleware(LoggingMiddleware), - Middleware(PrometheusMiddleware), - ] - return middleware - - -@asynccontextmanager -async def lifespan(app: FastAPI): - logger.info("startup initiated") - kafka_manager.set_producer( - key="reports_to_insert", - producer=RepoReportsToInsertProducer( - bootstrap_servers=KafkaSettings().KAFKA_BOOTSTRAP_SERVERS - ), - ) - producer = kafka_manager.get_producer(key="reports_to_insert") - await producer.start() - yield - await producer.stop() - logger.info("shutdown completed") - - -def create_app() -> FastAPI: - _app = FastAPI( - title="Bot-Detector-API", - description="Bot-Detector-API", - middleware=make_middleware(), - lifespan=lifespan, - ) - init_routers(_app=_app) - return _app - - -app = create_app() - - -start_http_server(8000) - - -@app.get("/") -async def root(): - return {"message": "Hello World"} +import logging +from contextlib import asynccontextmanager + +from bases.api_public import routes as routes_pkg +from bases.api_public.core.fastapi.dependencies.kafka import kafka_manager +from bases.api_public.core.fastapi.middleware import ( + LoggingMiddleware, + PrometheusMiddleware, +) +from bot_detector.kafka import Settings as KafkaSettings +from bot_detector.kafka.repositories import RepoReportsToInsertProducer +from fastapi import FastAPI +from fastapi.middleware import Middleware +from fastapi.middleware.cors import CORSMiddleware +from prometheus_client import start_http_server + +logger = logging.getLogger(__name__) + + +def init_routers(_app: FastAPI) -> None: + _app.include_router(routes_pkg.router) + + +def make_middleware() -> list[Middleware]: + middleware = [ + Middleware( + CORSMiddleware, + allow_origins=[ + "http://osrsbotdetector.com/", + "https://osrsbotdetector.com/", + "http://localhost", + "http://localhost:8080", + ], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ), + Middleware(LoggingMiddleware), + Middleware(PrometheusMiddleware), + ] + return middleware + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logger.info("startup initiated") + kafka_manager.set_producer( + key="reports_to_insert", + producer=RepoReportsToInsertProducer( + bootstrap_servers=KafkaSettings().KAFKA_BOOTSTRAP_SERVERS + ), + ) + producer = kafka_manager.get_producer(key="reports_to_insert") + await producer.start() + yield + await producer.stop() + logger.info("shutdown completed") + + +def create_app() -> FastAPI: + _app = FastAPI( + title="Bot-Detector-API", + description="Bot-Detector-API", + middleware=make_middleware(), + lifespan=lifespan, + ) + init_routers(_app=_app) + return _app + + +app = create_app() + + +start_http_server(8000) + + +@app.get("/") +async def root(): + return {"message": "Hello World"} diff --git a/bases/api_public/feedback/__init__.py b/bases/api_public/feedback/__init__.py new file mode 100644 index 0000000..ab02352 --- /dev/null +++ b/bases/api_public/feedback/__init__.py @@ -0,0 +1,3 @@ +from bases.api_public.feedback.routes import router + +__all__ = ["router"] diff --git a/bases/bot_detector/api_public/src/app/repositories/feedback.py b/bases/api_public/feedback/repository.py similarity index 96% rename from bases/bot_detector/api_public/src/app/repositories/feedback.py rename to bases/api_public/feedback/repository.py index 9177c23..283614f 100644 --- a/bases/bot_detector/api_public/src/app/repositories/feedback.py +++ b/bases/api_public/feedback/repository.py @@ -1,6 +1,6 @@ import logging -from bot_detector.api_public.src.app.views.input.feedback import FeedbackInput +from bases.api_public.feedback.schemas import FeedbackInput from bot_detector.database.api_public import ( Player as dbPlayer, PredictionFeedback as dbFeedback, diff --git a/bases/bot_detector/api_public/src/api/v2/feedback.py b/bases/api_public/feedback/routes.py similarity index 62% rename from bases/bot_detector/api_public/src/api/v2/feedback.py rename to bases/api_public/feedback/routes.py index d9ef4f4..aff0497 100644 --- a/bases/bot_detector/api_public/src/api/v2/feedback.py +++ b/bases/api_public/feedback/routes.py @@ -1,10 +1,10 @@ import logging -from bot_detector.api_public.src.app.repositories.feedback import Feedback -from bot_detector.api_public.src.app.views.input.feedback import FeedbackInput -from bot_detector.api_public.src.app.views.response.ok import Ok -from bot_detector.api_public.src.core.fastapi.dependencies.session import get_session -from bot_detector.api_public.src.core.fastapi.dependencies.to_jagex_name import ( +from bases.api_public.feedback.repository import Feedback +from bases.api_public.feedback.schemas import FeedbackInput +from bases.api_public.shared.responses import Ok +from bases.api_public.core.fastapi.dependencies.session import get_session +from bases.api_public.core.fastapi.dependencies.to_jagex_name import ( to_jagex_name, ) from fastapi import APIRouter, Depends, HTTPException, status diff --git a/bases/bot_detector/api_public/src/app/views/input/feedback.py b/bases/api_public/feedback/schemas.py similarity index 88% rename from bases/bot_detector/api_public/src/app/views/input/feedback.py rename to bases/api_public/feedback/schemas.py index 3741944..9aa46a8 100644 --- a/bases/bot_detector/api_public/src/app/views/input/feedback.py +++ b/bases/api_public/feedback/schemas.py @@ -43,3 +43,11 @@ def player_name_validator(cls, value: str): return value case _: raise ValueError("Invalid format for player_name") + + +class FeedbackScore(BaseModel): + count: int + possible_ban: bool + confirmed_ban: bool + confirmed_player: bool + vote: Optional[int] = Field(None, ge=-1, le=1) diff --git a/bases/api_public/labels/__init__.py b/bases/api_public/labels/__init__.py new file mode 100644 index 0000000..8636e97 --- /dev/null +++ b/bases/api_public/labels/__init__.py @@ -0,0 +1,3 @@ +from bases.api_public.labels.routes import router + +__all__ = ["router"] diff --git a/bases/bot_detector/api_public/src/app/repositories/labels.py b/bases/api_public/labels/repository.py similarity index 100% rename from bases/bot_detector/api_public/src/app/repositories/labels.py rename to bases/api_public/labels/repository.py diff --git a/bases/bot_detector/api_public/src/api/v2/labels.py b/bases/api_public/labels/routes.py similarity index 81% rename from bases/bot_detector/api_public/src/api/v2/labels.py rename to bases/api_public/labels/routes.py index 93c70a9..ad1aa42 100644 --- a/bases/bot_detector/api_public/src/api/v2/labels.py +++ b/bases/api_public/labels/routes.py @@ -1,8 +1,8 @@ import logging -from bot_detector.api_public.src.app.repositories.labels import LabelRepository -from bot_detector.api_public.src.app.views.response.label import LabelResponse -from bot_detector.api_public.src.core.fastapi.dependencies.session import get_session +from bases.api_public.labels.repository import LabelRepository +from bases.api_public.labels.schemas import LabelResponse +from bases.api_public.core.fastapi.dependencies.session import get_session from fastapi import APIRouter, Depends, status router = APIRouter(tags=["Labels"]) diff --git a/bases/bot_detector/api_public/src/app/views/response/label.py b/bases/api_public/labels/schemas.py similarity index 100% rename from bases/bot_detector/api_public/src/app/views/response/label.py rename to bases/api_public/labels/schemas.py diff --git a/bases/api_public/player/__init__.py b/bases/api_public/player/__init__.py new file mode 100644 index 0000000..9517486 --- /dev/null +++ b/bases/api_public/player/__init__.py @@ -0,0 +1,3 @@ +from bases.api_public.player.routes import router + +__all__ = ["router"] diff --git a/bases/bot_detector/api_public/src/app/repositories/player.py b/bases/api_public/player/repository.py similarity index 94% rename from bases/bot_detector/api_public/src/app/repositories/player.py rename to bases/api_public/player/repository.py index f272aa1..2a676b0 100644 --- a/bases/bot_detector/api_public/src/app/repositories/player.py +++ b/bases/api_public/player/repository.py @@ -1,161 +1,161 @@ -import logging - -import sqlalchemy as sqla -from bot_detector.api_public.src.app.views.player import PlayerCreate, PlayerInDB -from bot_detector.api_public.src.core._cache import SimpleALRUCache -from bot_detector.database.api_public import ( - Player as dbPlayer, - PredictionFeedback as dbFeedback, - Prediction_v2 as dbPrediction, -) -from fastapi.encoders import jsonable_encoder -from pydantic import ValidationError -from sqlalchemy import func, select -from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession -from sqlalchemy.orm import aliased -from sqlalchemy.sql.expression import Select - -# from bot_detector.database.api_public import Report as dbReport - -logger = logging.getLogger(__name__) - - -def model_to_dict(model): - """Converts an SQLAlchemy model instance to a dictionary.""" - return {c.name: getattr(model, c.name) for c in model.__table__.columns} - - -class Player: - def __init__( - self, - session: AsyncSession, - cache: SimpleALRUCache = SimpleALRUCache(), - ) -> None: - self.session = session - self.cache = cache - - def sanitize_name(self, player_name: str) -> str: - return player_name.lower().replace("_", " ").replace("-", " ").strip() - - async def update_session(self, session: AsyncSession): - self.session = session - - async def get_report_score(self, player_names: tuple[str, ...]): - if not isinstance(player_names, tuple): - raise Exception() - sql_select = """ - select - count(rs.reporting_id) as count, - subject.confirmed_ban, - subject.possible_ban, - subject.confirmed_player, - rs.manual_detect - from report_sighting rs - join Players voter ON rs.reporting_id = voter.id - join Players subject ON rs.reported_id = subject.id - WHERE voter.name in :name - GROUP BY - subject.confirmed_ban, - subject.possible_ban, - subject.confirmed_player, - rs.manual_detect - """ - params = {"name": player_names} - data = await self.session.execute(sqla.text(sql_select), params=params) - result = data.mappings().all() - return result - - async def get_feedback_score(self, player_names: list[str]): - # dbFeedback - fb_voter: dbPlayer = aliased(dbPlayer, name="feedback_voter") - fb_subject: dbPlayer = aliased(dbPlayer, name="feedback_subject") - - query: Select = select( - func.count(func.distinct(fb_subject.id)).label("count"), - fb_subject.possible_ban, - fb_subject.confirmed_ban, - fb_subject.confirmed_player, - ) - query = query.select_from(dbFeedback) - query = query.join(fb_voter, dbFeedback.voter_id == fb_voter.id) - query = query.join(fb_subject, dbFeedback.subject_id == fb_subject.id) - query = query.where(fb_voter.name.in_(player_names)) - query = query.group_by( - fb_subject.possible_ban, - fb_subject.confirmed_ban, - fb_subject.confirmed_player, - ) - - result: AsyncResult = await self.session.execute(query) - await self.session.commit() - return tuple(result.mappings()) - - async def get_prediction(self, player_names: list[str]): - query: Select = select( - dbPlayer.id.label("player_id"), - dbPlayer.name, - dbPrediction.created_at, - dbPrediction.model_name, - dbPrediction.prediction, - dbPrediction.confidence, - dbPrediction.predictions, - ) - query = query.select_from(dbPrediction) - query = query.join(dbPlayer, dbPrediction.player_id == dbPlayer.id) - query = query.where(dbPlayer.name.in_(player_names)) - - result: AsyncResult = await self.session.execute(query) - result = result.mappings().all() - return jsonable_encoder(result) - - async def get(self, player_name: str) -> PlayerInDB: - assert isinstance(player_name, str) - player_name = self.sanitize_name(player_name) - - sql = sqla.select(dbPlayer).where(dbPlayer.name == player_name) - - result = await self.session.execute(sql) - data = result.scalars().all() - try: - if len(data) == 0: - return None - player_in_db = PlayerInDB(**model_to_dict(data[0])) - except ValidationError as e: - logger.error(f"Validation error: {e.json()}") - return None - return player_in_db - - async def get_cache(self, player_name: str) -> PlayerInDB: - player_name = self.sanitize_name(player_name) - player = await self.cache.get(key=player_name) - - if isinstance(player, PlayerInDB): - if self.cache.hits % 100 == 0 and self.cache.hits > 0: - logger.info(f"hits: {self.cache.hits}, misses: {self.cache.misses}") - return player - - player = await self.get(player_name=player_name) - - if isinstance(player, PlayerInDB): - await self.cache.put(key=player_name, value=player) - return player - - async def insert(self, player: PlayerCreate) -> PlayerInDB: - player.name = self.sanitize_name(player.name) - sql = sqla.insert(dbPlayer).values(player.model_dump()).prefix_with("IGNORE") - await self.session.execute(sql) - await self.session.commit() - return await self.get(player_name=player.name) - - async def get_or_insert(self, player_name: str, cached=True) -> PlayerInDB: - player_name = self.sanitize_name(player_name) - - if cached: - player = await self.get_cache(player_name=player_name) - else: - player = await self.get(player_name=player_name) - - if player is None: - player = await self.insert(PlayerCreate(name=player_name)) - - return player +import logging + +import sqlalchemy as sqla +from bases.api_public.player.schemas import PlayerCreate, PlayerInDB +from bases.api_public.core._cache import SimpleALRUCache +from bot_detector.database.api_public import ( + Player as dbPlayer, + PredictionFeedback as dbFeedback, + Prediction_v2 as dbPrediction, +) +from fastapi.encoders import jsonable_encoder +from pydantic import ValidationError +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select + +# from bot_detector.database.api_public import Report as dbReport + +logger = logging.getLogger(__name__) + + +def model_to_dict(model): + """Converts an SQLAlchemy model instance to a dictionary.""" + return {c.name: getattr(model, c.name) for c in model.__table__.columns} + + +class Player: + def __init__( + self, + session: AsyncSession, + cache: SimpleALRUCache = SimpleALRUCache(), + ) -> None: + self.session = session + self.cache = cache + + def sanitize_name(self, player_name: str) -> str: + return player_name.lower().replace("_", " ").replace("-", " ").strip() + + async def update_session(self, session: AsyncSession): + self.session = session + + async def get_report_score(self, player_names: tuple[str, ...]): + if not isinstance(player_names, tuple): + raise Exception() + sql_select = """ + select + count(rs.reporting_id) as count, + subject.confirmed_ban, + subject.possible_ban, + subject.confirmed_player, + rs.manual_detect + from report_sighting rs + join Players voter ON rs.reporting_id = voter.id + join Players subject ON rs.reported_id = subject.id + WHERE voter.name in :name + GROUP BY + subject.confirmed_ban, + subject.possible_ban, + subject.confirmed_player, + rs.manual_detect + """ + params = {"name": player_names} + data = await self.session.execute(sqla.text(sql_select), params=params) + result = data.mappings().all() + return result + + async def get_feedback_score(self, player_names: list[str]): + # dbFeedback + fb_voter: dbPlayer = aliased(dbPlayer, name="feedback_voter") + fb_subject: dbPlayer = aliased(dbPlayer, name="feedback_subject") + + query: Select = select( + func.count(func.distinct(fb_subject.id)).label("count"), + fb_subject.possible_ban, + fb_subject.confirmed_ban, + fb_subject.confirmed_player, + ) + query = query.select_from(dbFeedback) + query = query.join(fb_voter, dbFeedback.voter_id == fb_voter.id) + query = query.join(fb_subject, dbFeedback.subject_id == fb_subject.id) + query = query.where(fb_voter.name.in_(player_names)) + query = query.group_by( + fb_subject.possible_ban, + fb_subject.confirmed_ban, + fb_subject.confirmed_player, + ) + + result: AsyncResult = await self.session.execute(query) + await self.session.commit() + return tuple(result.mappings()) + + async def get_prediction(self, player_names: list[str]): + query: Select = select( + dbPlayer.id.label("player_id"), + dbPlayer.name, + dbPrediction.created_at, + dbPrediction.model_name, + dbPrediction.prediction, + dbPrediction.confidence, + dbPrediction.predictions, + ) + query = query.select_from(dbPrediction) + query = query.join(dbPlayer, dbPrediction.player_id == dbPlayer.id) + query = query.where(dbPlayer.name.in_(player_names)) + + result: AsyncResult = await self.session.execute(query) + result = result.mappings().all() + return jsonable_encoder(result) + + async def get(self, player_name: str) -> PlayerInDB: + assert isinstance(player_name, str) + player_name = self.sanitize_name(player_name) + + sql = sqla.select(dbPlayer).where(dbPlayer.name == player_name) + + result = await self.session.execute(sql) + data = result.scalars().all() + try: + if len(data) == 0: + return None + player_in_db = PlayerInDB(**model_to_dict(data[0])) + except ValidationError as e: + logger.error(f"Validation error: {e.json()}") + return None + return player_in_db + + async def get_cache(self, player_name: str) -> PlayerInDB: + player_name = self.sanitize_name(player_name) + player = await self.cache.get(key=player_name) + + if isinstance(player, PlayerInDB): + if self.cache.hits % 100 == 0 and self.cache.hits > 0: + logger.info(f"hits: {self.cache.hits}, misses: {self.cache.misses}") + return player + + player = await self.get(player_name=player_name) + + if isinstance(player, PlayerInDB): + await self.cache.put(key=player_name, value=player) + return player + + async def insert(self, player: PlayerCreate) -> PlayerInDB: + player.name = self.sanitize_name(player.name) + sql = sqla.insert(dbPlayer).values(player.model_dump()).prefix_with("IGNORE") + await self.session.execute(sql) + await self.session.commit() + return await self.get(player_name=player.name) + + async def get_or_insert(self, player_name: str, cached=True) -> PlayerInDB: + player_name = self.sanitize_name(player_name) + + if cached: + player = await self.get_cache(player_name=player_name) + else: + player = await self.get(player_name=player_name) + + if player is None: + player = await self.insert(PlayerCreate(name=player_name)) + + return player diff --git a/bases/bot_detector/api_public/src/api/v2/player.py b/bases/api_public/player/routes.py similarity index 83% rename from bases/bot_detector/api_public/src/api/v2/player.py rename to bases/api_public/player/routes.py index 0f4bfd4..f7255bb 100644 --- a/bases/bot_detector/api_public/src/api/v2/player.py +++ b/bases/api_public/player/routes.py @@ -1,107 +1,105 @@ -import asyncio -import logging -from typing import Annotated - -from bot_detector.api_public.src.app.repositories.player import Player as repoPlayer -from bot_detector.api_public.src.app.views.response.feedback_score import ( - FeedbackScoreResponse, -) -from bot_detector.api_public.src.app.views.response.prediction import PredictionResponse -from bot_detector.api_public.src.app.views.response.report_score import ( - ReportScoreResponse, -) -from bot_detector.api_public.src.core.fastapi.dependencies.session import get_session -from bot_detector.api_public.src.core.fastapi.dependencies.to_jagex_name import ( - to_jagex_name, -) -from fastapi import APIRouter, Depends, HTTPException, Query, status -from pydantic.fields import Field - -router = APIRouter(tags=["Player"]) -logger = logging.getLogger(__name__) - - -@router.get("/player/report/score", response_model=list[ReportScoreResponse]) -async def get_players_kc( - name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( - ..., - min_length=1, - description="Name of the player", - examples=["Player1", "Player2"], - ), - session=Depends(get_session), -): - """ - Get the report score for one or multiple players. - - Args: - name (str): can be provided multiple times - - Returns: - list[ReportScoreResponse]: A list of dictionaries containing KC data for each player. - """ - repo = repoPlayer(session) - names = await asyncio.gather(*[to_jagex_name(n) for n in name]) - data = await repo.get_report_score(player_names=tuple(names)) - return data - - -@router.get("/player/feedback/score", response_model=list[FeedbackScoreResponse]) -async def get_feedback_score( - name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( - ..., - min_length=1, - description="Name of the player", - examples=["Player1", "Player2"], - ), - session=Depends(get_session), -): - """ - Get the feedback score for one or multiple players. - - Args: - name (str): can be provided multiple times - - Returns: - list[FeedbackScoreResponse]: A list of dictionaries containing KC data for each player. - """ - repo = repoPlayer(session) - names = await asyncio.gather(*[to_jagex_name(n) for n in name]) - data = await repo.get_feedback_score(player_names=names) - return data - - -@router.get("/player/prediction", response_model=list[PredictionResponse]) -async def get_prediction( - name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( - ..., - min_length=1, - max_length=5, - description="Name of the player", - examples=["Player1", "Player2"], - ), - breakdown: bool = Query(...), - session=Depends(get_session), -): - """ - Get prediction data for one or multiple users. - - Args: - name (str): The username of the user for whom predictions are requested. - breakdown (bool): A flag indicating whether to include a breakdown of predictions. - - Returns: - List[PredictionResponse]: A list of PredictionResponse objects containing prediction data. - - Raises: - HTTPException: Returns a 404 error with the message "Player not found" if no data is found for the user. - - """ - repo = repoPlayer(session) - names = await asyncio.gather(*[to_jagex_name(n) for n in name]) - data = await repo.get_prediction(player_names=names) - if not data: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Player not found" - ) - return [PredictionResponse.from_data(d, breakdown) for d in data] +import asyncio +import logging +from typing import Annotated + +from bases.api_public.player.repository import Player as repoPlayer +from bases.api_public.player.schemas import ( + FeedbackScoreResponse, + PredictionResponse, + ReportScoreResponse, +) +from bases.api_public.core.fastapi.dependencies.session import get_session +from bases.api_public.core.fastapi.dependencies.to_jagex_name import ( + to_jagex_name, +) +from fastapi import APIRouter, Depends, HTTPException, Query, status +from pydantic.fields import Field + +router = APIRouter(tags=["Player"]) +logger = logging.getLogger(__name__) + + +@router.get("/player/report/score", response_model=list[ReportScoreResponse]) +async def get_players_kc( + name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( + ..., + min_length=1, + description="Name of the player", + examples=["Player1", "Player2"], + ), + session=Depends(get_session), +): + """ + Get the report score for one or multiple players. + + Args: + name (str): can be provided multiple times + + Returns: + list[ReportScoreResponse]: A list of dictionaries containing KC data for each player. + """ + repo = repoPlayer(session) + names = await asyncio.gather(*[to_jagex_name(n) for n in name]) + data = await repo.get_report_score(player_names=tuple(names)) + return data + + +@router.get("/player/feedback/score", response_model=list[FeedbackScoreResponse]) +async def get_feedback_score( + name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( + ..., + min_length=1, + description="Name of the player", + examples=["Player1", "Player2"], + ), + session=Depends(get_session), +): + """ + Get the feedback score for one or multiple players. + + Args: + name (str): can be provided multiple times + + Returns: + list[FeedbackScoreResponse]: A list of dictionaries containing KC data for each player. + """ + repo = repoPlayer(session) + names = await asyncio.gather(*[to_jagex_name(n) for n in name]) + data = await repo.get_feedback_score(player_names=names) + return data + + +@router.get("/player/prediction", response_model=list[PredictionResponse]) +async def get_prediction( + name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( + ..., + min_length=1, + max_length=5, + description="Name of the player", + examples=["Player1", "Player2"], + ), + breakdown: bool = Query(...), + session=Depends(get_session), +): + """ + Get prediction data for one or multiple users. + + Args: + name (str): The username of the user for whom predictions are requested. + breakdown (bool): A flag indicating whether to include a breakdown of predictions. + + Returns: + List[PredictionResponse]: A list of PredictionResponse objects containing prediction data. + + Raises: + HTTPException: Returns a 404 error with the message "Player not found" if no data is found for the user. + + """ + repo = repoPlayer(session) + names = await asyncio.gather(*[to_jagex_name(n) for n in name]) + data = await repo.get_prediction(player_names=names) + if not data: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Player not found" + ) + return [PredictionResponse.from_data(d, breakdown) for d in data] diff --git a/bases/bot_detector/api_public/src/app/views/player.py b/bases/api_public/player/schemas.py similarity index 50% rename from bases/bot_detector/api_public/src/app/views/player.py rename to bases/api_public/player/schemas.py index 482b971..796093e 100644 --- a/bases/bot_detector/api_public/src/app/views/player.py +++ b/bases/api_public/player/schemas.py @@ -46,3 +46,56 @@ def parse_created_at(cls, value): class Player(PlayerInDB): pass + + +class PlayerResponse(BaseModel): + id: int + name: str + created_at: datetime + updated_at: datetime + possible_ban: bool + confirmed_ban: bool + confirmed_player: bool + label_id: int + label_jagex: int + ironman: bool + hardcore_ironman: bool + ultimate_ironman: bool + normalized_name: str + + +class ReportScoreResponse(BaseModel): + count: int + possible_ban: bool + confirmed_ban: bool + confirmed_player: bool + manual_detect: bool + + +class FeedbackScoreResponse(BaseModel): + count: int + possible_ban: bool + confirmed_ban: bool + confirmed_player: bool + + +class PredictionResponse(BaseModel): + player_id: int + player_name: str + prediction_label: str + prediction_confidence: float + created: datetime + predictions_breakdown: dict + + @classmethod + def from_data(cls, data: dict, breakdown: bool): + prediction_data: dict = data.pop("predictions", {}) + player_data = { + "player_id": data.pop("player_id"), + "player_name": data.pop("name"), + "created": data.pop("created_at"), + "prediction_label": data.pop("prediction").lower(), + "prediction_confidence": data.pop("confidence"), + "predictions_breakdown": prediction_data if breakdown else {}, + } + return cls(**player_data) diff --git a/bases/api_public/reports/__init__.py b/bases/api_public/reports/__init__.py new file mode 100644 index 0000000..e194a5d --- /dev/null +++ b/bases/api_public/reports/__init__.py @@ -0,0 +1,3 @@ +from bases.api_public.reports.routes import router + +__all__ = ["router"] diff --git a/bases/bot_detector/api_public/src/app/repositories/report.py b/bases/api_public/reports/repository.py similarity index 94% rename from bases/bot_detector/api_public/src/app/repositories/report.py rename to bases/api_public/reports/repository.py index 2a3f0e8..3826ac9 100644 --- a/bases/bot_detector/api_public/src/app/repositories/report.py +++ b/bases/api_public/reports/repository.py @@ -1,112 +1,112 @@ -import asyncio -import logging -import time - -from bot_detector.api_public.src.core.fastapi.dependencies.kafka import kafka_manager -from bot_detector.kafka.repositories.reports_to_insert import ( - RepoReportsToInsertProducer, -) -from bot_detector.structs import ( - Detection, - MetaData, - ParsedDetection, - ReportsToInsertStruct, -) -from pydantic import ValidationError - -logger = logging.getLogger(__name__) - - -class CustomError(Exception): ... - - -class Report: - def __init__(self) -> None: - pass - - def _check_data_size(self, data: list[Detection]) -> list[Detection] | None: - return None if len(data) > 5000 else data - - def _filter_valid_time(self, data: list[Detection]) -> list[Detection]: - current_time = int(time.time()) - min_ts = current_time - 25200 # 7 hours ago - max_ts = current_time + 3600 # 1 hour in the future - # [d for d in data if min_ts < d.ts < max_ts] - output = [] - - for d in data: - if d.ts <= min_ts: - logger.info( - f"invalid: {d.ts} <= {min_ts}, now={current_time}, {d.reporter}" - ) - continue - if d.ts >= max_ts: - logger.info( - f"invalid: {d.ts} >= {max_ts}, now={current_time}, {d.reporter}" - ) - continue - output.append(d) - - return output - - def _check_unique_reporter(self, data: list[Detection]) -> list[Detection] | None: - return None if len(set(d.reporter for d in data)) > 1 else data - - async def parse_data(self, data: list[Detection]) -> tuple[list[Detection], None]: - """ - Parse and validate a list of detection data. - """ - data = self._check_data_size(data) - if not data: - error = "invalid data size" - logger.warning(error) - return None, error - - data = self._filter_valid_time(data) - if not data: - error = "invalid time" - logger.warning(error) - return None, error - - data = self._check_unique_reporter(data) - if not data: - error = "invalid unique reporter" - logger.warning(error) - return None, error - return data, None - - def _transform_detection( - self, data: list[ParsedDetection] - ) -> tuple[list[ReportsToInsertStruct], list | None]: - reports = [] - errors = [] - - for d in data: - metadata = MetaData(version=1, source="api_public") - try: - report = ReportsToInsertStruct(metadata=metadata, report=d.model_dump()) - reports.append(report) - except ValidationError as e: - error = f"Validation error: {e.json()}" - errors.append(error) - return reports, errors - - async def send_to_kafka(self, data: list[ParsedDetection]) -> None: - producer = kafka_manager.get_producer(key="reports_to_insert") - producer: RepoReportsToInsertProducer | None - - if not producer: - raise CustomError("Producer not found") - - tasks = [] - - # Transform data to ReportsToInsertStruct - reports, error = self._transform_detection(data) - - tasks = [producer.produce_one(report=report) for report in reports] - await asyncio.gather(*tasks) - - if len(error) > 0: - error_msg = f"Received {len(error)} validation errors like this: {error[0]}" - logger.error(error_msg) - raise CustomError(error_msg) +import asyncio +import logging +import time + +from bases.api_public.core.fastapi.dependencies.kafka import kafka_manager +from bot_detector.kafka.repositories.reports_to_insert import ( + RepoReportsToInsertProducer, +) +from bot_detector.structs import ( + Detection, + MetaData, + ParsedDetection, + ReportsToInsertStruct, +) +from pydantic import ValidationError + +logger = logging.getLogger(__name__) + + +class CustomError(Exception): ... + + +class Report: + def __init__(self) -> None: + pass + + def _check_data_size(self, data: list[Detection]) -> list[Detection] | None: + return None if len(data) > 5000 else data + + def _filter_valid_time(self, data: list[Detection]) -> list[Detection]: + current_time = int(time.time()) + min_ts = current_time - 25200 # 7 hours ago + max_ts = current_time + 3600 # 1 hour in the future + # [d for d in data if min_ts < d.ts < max_ts] + output = [] + + for d in data: + if d.ts <= min_ts: + logger.info( + f"invalid: {d.ts} <= {min_ts}, now={current_time}, {d.reporter}" + ) + continue + if d.ts >= max_ts: + logger.info( + f"invalid: {d.ts} >= {max_ts}, now={current_time}, {d.reporter}" + ) + continue + output.append(d) + + return output + + def _check_unique_reporter(self, data: list[Detection]) -> list[Detection] | None: + return None if len(set(d.reporter for d in data)) > 1 else data + + async def parse_data(self, data: list[Detection]) -> tuple[list[Detection], None]: + """ + Parse and validate a list of detection data. + """ + data = self._check_data_size(data) + if not data: + error = "invalid data size" + logger.warning(error) + return None, error + + data = self._filter_valid_time(data) + if not data: + error = "invalid time" + logger.warning(error) + return None, error + + data = self._check_unique_reporter(data) + if not data: + error = "invalid unique reporter" + logger.warning(error) + return None, error + return data, None + + def _transform_detection( + self, data: list[ParsedDetection] + ) -> tuple[list[ReportsToInsertStruct], list | None]: + reports = [] + errors = [] + + for d in data: + metadata = MetaData(version=1, source="api_public") + try: + report = ReportsToInsertStruct(metadata=metadata, report=d.model_dump()) + reports.append(report) + except ValidationError as e: + error = f"Validation error: {e.json()}" + errors.append(error) + return reports, errors + + async def send_to_kafka(self, data: list[ParsedDetection]) -> None: + producer = kafka_manager.get_producer(key="reports_to_insert") + producer: RepoReportsToInsertProducer | None + + if not producer: + raise CustomError("Producer not found") + + tasks = [] + + # Transform data to ReportsToInsertStruct + reports, error = self._transform_detection(data) + + tasks = [producer.produce_one(report=report) for report in reports] + await asyncio.gather(*tasks) + + if len(error) > 0: + error_msg = f"Received {len(error)} validation errors like this: {error[0]}" + logger.error(error_msg) + raise CustomError(error_msg) diff --git a/bases/bot_detector/api_public/src/api/v2/report.py b/bases/api_public/reports/routes.py similarity index 82% rename from bases/bot_detector/api_public/src/api/v2/report.py rename to bases/api_public/reports/routes.py index dad6a5f..dad34d4 100644 --- a/bases/bot_detector/api_public/src/api/v2/report.py +++ b/bases/api_public/reports/routes.py @@ -1,70 +1,70 @@ -import logging - -from bot_detector.api_public.src.app.repositories.player import Player -from bot_detector.api_public.src.app.repositories.report import CustomError, Report -from bot_detector.api_public.src.app.views.response.ok import Ok -from bot_detector.api_public.src.core._cache import SimpleALRUCache -from bot_detector.api_public.src.core.fastapi.dependencies.session import get_session -from bot_detector.structs import Detection, ParsedDetection -from fastapi import APIRouter, Depends, status -from fastapi.exceptions import HTTPException -from sqlalchemy.ext.asyncio import AsyncSession - -logger = logging.getLogger(__name__) -router = APIRouter(tags=["Report"]) - -player_cache = SimpleALRUCache(max_size=100_000) - - -@router.post("/report", status_code=status.HTTP_201_CREATED, response_model=Ok) -async def post_reports( - detections: list[Detection], - session: AsyncSession = Depends(get_session), -): - global player_cache - report_repo = Report() - player_repo = Player(session=session, cache=player_cache) - - data, error = await report_repo.parse_data(detections) - if error: - raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=error) - - logger.debug(f"Received: {len(data)}, Reporter: {data[0].reporter}") - - # get unique list of names - player_names = list(set([d.reported for d in data] + [d.reporter for d in data])) - players = [await player_repo.get_or_insert(player_name=p) for p in player_names] - players = {p.name: p.id for p in players if p} - - _data = [] - for d in data: - _d = d.model_dump() - # get reported_id from name - reported = player_repo.sanitize_name(_d.pop("reported")) - reported_id = players.get(reported) - - # get reporter_id from name - reporter = player_repo.sanitize_name(_d.pop("reporter")) - reporter_id = players.get(reporter) - - # some validation - if reporter_id is None or reported_id is None: - logger.warning(msg=f"{reported_id=}, {reporter_id=}, {d}") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="something went wrong", - ) - _d["reported_id"] = reported_id - _d["reporter_id"] = reporter_id - - _data.append(ParsedDetection(**_d)) - - # print(_data) - try: - await report_repo.send_to_kafka(data=_data) - except CustomError: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal error", - ) - return Ok() +import logging + +from bases.api_public.player.repository import Player +from bases.api_public.reports.repository import CustomError, Report +from bases.api_public.shared.responses import Ok +from bases.api_public.core._cache import SimpleALRUCache +from bases.api_public.core.fastapi.dependencies.session import get_session +from bot_detector.structs import Detection, ParsedDetection +from fastapi import APIRouter, Depends, status +from fastapi.exceptions import HTTPException +from sqlalchemy.ext.asyncio import AsyncSession + +logger = logging.getLogger(__name__) +router = APIRouter(tags=["Report"]) + +player_cache = SimpleALRUCache(max_size=100_000) + + +@router.post("/report", status_code=status.HTTP_201_CREATED, response_model=Ok) +async def post_reports( + detections: list[Detection], + session: AsyncSession = Depends(get_session), +): + global player_cache + report_repo = Report() + player_repo = Player(session=session, cache=player_cache) + + data, error = await report_repo.parse_data(detections) + if error: + raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=error) + + logger.debug(f"Received: {len(data)}, Reporter: {data[0].reporter}") + + # get unique list of names + player_names = list(set([d.reported for d in data] + [d.reporter for d in data])) + players = [await player_repo.get_or_insert(player_name=p) for p in player_names] + players = {p.name: p.id for p in players if p} + + _data = [] + for d in data: + _d = d.model_dump() + # get reported_id from name + reported = player_repo.sanitize_name(_d.pop("reported")) + reported_id = players.get(reported) + + # get reporter_id from name + reporter = player_repo.sanitize_name(_d.pop("reporter")) + reporter_id = players.get(reporter) + + # some validation + if reporter_id is None or reported_id is None: + logger.warning(msg=f"{reported_id=}, {reporter_id=}, {d}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="something went wrong", + ) + _d["reported_id"] = reported_id + _d["reporter_id"] = reporter_id + + _data.append(ParsedDetection(**_d)) + + # print(_data) + try: + await report_repo.send_to_kafka(data=_data) + except CustomError: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal error", + ) + return Ok() diff --git a/bases/bot_detector/api_public/src/app/views/input/report.py b/bases/api_public/reports/schemas.py similarity index 89% rename from bases/bot_detector/api_public/src/app/views/input/report.py rename to bases/api_public/reports/schemas.py index 2d897c8..59a9a54 100644 --- a/bases/bot_detector/api_public/src/app/views/input/report.py +++ b/bases/api_public/reports/schemas.py @@ -1,9 +1,11 @@ import time -from typing import Optional - -from bot_detector.api_public.src.app.views.input._metadata import Metadata -from pydantic import BaseModel -from pydantic.fields import Field +from typing import Optional + +from pydantic import BaseModel, Field + + +class Metadata(BaseModel): + version: str class Equipment(BaseModel): diff --git a/bases/api_public/routes.py b/bases/api_public/routes.py new file mode 100644 index 0000000..f6912e7 --- /dev/null +++ b/bases/api_public/routes.py @@ -0,0 +1,18 @@ +from fastapi import APIRouter + +from bases.api_public import feedback, labels, player, reports + +router = APIRouter() +v2_router = APIRouter(prefix="/v2") + +for feature_router in ( + player.router, + reports.router, + feedback.router, + labels.router, +): + v2_router.include_router(feature_router) + +router.include_router(v2_router) + +__all__ = ["router"] diff --git a/bases/bot_detector/api_public/src/app/repositories/__init__.py b/bases/api_public/shared/__init__.py similarity index 100% rename from bases/bot_detector/api_public/src/app/repositories/__init__.py rename to bases/api_public/shared/__init__.py diff --git a/bases/bot_detector/api_public/src/app/views/response/ok.py b/bases/api_public/shared/responses.py similarity index 100% rename from bases/bot_detector/api_public/src/app/views/response/ok.py rename to bases/api_public/shared/responses.py diff --git a/bases/bot_detector/api_public/src/api/__init__.py b/bases/bot_detector/api_public/src/api/__init__.py deleted file mode 100644 index a121193..0000000 --- a/bases/bot_detector/api_public/src/api/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from fastapi import APIRouter - -from . import v2 - -router = APIRouter() -router.include_router(v2.router, prefix="/v2") diff --git a/bases/bot_detector/api_public/src/api/readme.md b/bases/bot_detector/api_public/src/api/readme.md deleted file mode 100644 index 0ff400d..0000000 --- a/bases/bot_detector/api_public/src/api/readme.md +++ /dev/null @@ -1 +0,0 @@ -the api folder can be considered the controller in the MVC approach \ No newline at end of file diff --git a/bases/bot_detector/api_public/src/api/v2/__init__.py b/bases/bot_detector/api_public/src/api/v2/__init__.py deleted file mode 100644 index cc967e7..0000000 --- a/bases/bot_detector/api_public/src/api/v2/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from fastapi import APIRouter - -from . import feedback, labels, player, report - -router = APIRouter() -router.include_router(player.router) -router.include_router(report.router) -router.include_router(feedback.router) -router.include_router(labels.router) diff --git a/bases/bot_detector/api_public/src/app/readme.md b/bases/bot_detector/api_public/src/app/readme.md deleted file mode 100644 index 64b97ea..0000000 --- a/bases/bot_detector/api_public/src/app/readme.md +++ /dev/null @@ -1,6 +0,0 @@ -the model is responsible for all the data handeling -- getting data from the database -- handles data logic - -the view is responsible for the data representation -- return format etc \ No newline at end of file diff --git a/bases/bot_detector/api_public/src/app/repositories/.gitkeep b/bases/bot_detector/api_public/src/app/repositories/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/bases/bot_detector/api_public/src/app/views/input/_metadata.py b/bases/bot_detector/api_public/src/app/views/input/_metadata.py deleted file mode 100644 index bd87da8..0000000 --- a/bases/bot_detector/api_public/src/app/views/input/_metadata.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - - -class Metadata(BaseModel): - version: str diff --git a/bases/bot_detector/api_public/src/app/views/response/feedback.py b/bases/bot_detector/api_public/src/app/views/response/feedback.py deleted file mode 100644 index f0bf81f..0000000 --- a/bases/bot_detector/api_public/src/app/views/response/feedback.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Optional - -from pydantic import BaseModel, Field - - -class FeedbackScore(BaseModel): - count: int - possible_ban: bool - confirmed_ban: bool - confirmed_player: bool - vote: Optional[int] = Field(None, ge=-1, le=1) diff --git a/bases/bot_detector/api_public/src/app/views/response/feedback_score.py b/bases/bot_detector/api_public/src/app/views/response/feedback_score.py deleted file mode 100644 index 0107c64..0000000 --- a/bases/bot_detector/api_public/src/app/views/response/feedback_score.py +++ /dev/null @@ -1,8 +0,0 @@ -from pydantic import BaseModel - - -class FeedbackScoreResponse(BaseModel): - count: int - possible_ban: bool - confirmed_ban: bool - confirmed_player: bool diff --git a/bases/bot_detector/api_public/src/app/views/response/player.py b/bases/bot_detector/api_public/src/app/views/response/player.py deleted file mode 100644 index 89d7bfd..0000000 --- a/bases/bot_detector/api_public/src/app/views/response/player.py +++ /dev/null @@ -1,19 +0,0 @@ -from datetime import datetime - -from pydantic import BaseModel - - -class PlayerResponse(BaseModel): - id: int - name: str - created_at: datetime - updated_at: datetime - possible_ban: bool - confirmed_ban: bool - confirmed_player: bool - label_id: int - label_jagex: int - ironman: bool - hardcore_ironman: bool - ultimate_ironman: bool - normalized_name: str diff --git a/bases/bot_detector/api_public/src/app/views/response/prediction.py b/bases/bot_detector/api_public/src/app/views/response/prediction.py deleted file mode 100644 index 2ef02d5..0000000 --- a/bases/bot_detector/api_public/src/app/views/response/prediction.py +++ /dev/null @@ -1,27 +0,0 @@ -from datetime import datetime - -from pydantic import BaseModel - - -class PredictionResponse(BaseModel): - player_id: int - player_name: str - prediction_label: str - prediction_confidence: float - created: datetime - predictions_breakdown: dict - - @classmethod - def from_data(self, data: dict, breakdown: bool): - # Create the player data dictionary with only the relevant fields - prediction_data: dict = data.pop("predictions", {}) - player_data = { - "player_id": data.pop("player_id"), - "player_name": data.pop("name"), - "created": data.pop("created_at"), - "prediction_label": data.pop("prediction").lower(), - "prediction_confidence": data.pop("confidence"), - "predictions_breakdown": prediction_data if breakdown else {}, - } - - return self(**player_data) diff --git a/bases/bot_detector/api_public/src/app/views/response/report_score.py b/bases/bot_detector/api_public/src/app/views/response/report_score.py deleted file mode 100644 index bcbcfcc..0000000 --- a/bases/bot_detector/api_public/src/app/views/response/report_score.py +++ /dev/null @@ -1,9 +0,0 @@ -from pydantic import BaseModel - - -class ReportScoreResponse(BaseModel): - count: int - possible_ban: bool - confirmed_ban: bool - confirmed_player: bool - manual_detect: bool diff --git a/bases/bot_detector/api_public/src/core/.gitkeep b/bases/bot_detector/api_public/src/core/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 82bdf87..84aa8cf 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -121,7 +121,7 @@ services: context: ./ dockerfile: Dockerfile # command: ["uv", "run", "bases/bot_detector/hiscore_scraper/core.py"] - # command: uv run uvicorn bases.bot_detector.api_public.src.core.server:app --host 0.0.0.0 --reload --port 5000 + # command: uv run uvicorn bases.api_public.core.server:app --host 0.0.0.0 --reload --port 5000 # command: uv run uvicorn bases.bot_detector.website.core.server:app --host 0.0.0.0 --reload --port 5000 # command: uv run uvicorn bases.bot_detector.api_ml.core.server:app --host 0.0.0.0 --reload --port 5000 ports: @@ -146,4 +146,4 @@ networks: botdetector-network: name: bd-network volumes: - uv_cache: \ No newline at end of file + uv_cache: diff --git a/docker-compose.yml b/docker-compose.yml index ea124cd..95d707e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -137,7 +137,7 @@ services: context: ./ dockerfile: ./projects/hiscore_scraper/Dockerfile target: production - # command: ["uv", "run", "uvicorn", "bot_detector.api_public.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning", "--reload", "--reload-dir", "/app/bot_detector/api_public/"] + # command: ["uv", "run", "uvicorn", "bases.api_public.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning", "--reload", "--reload-dir", "/app/bot_detector/api_public/"] networks: - botdetector-network env_file: @@ -283,7 +283,7 @@ services: target: builder # command: ["sleep", "infinity"] command: > - sh -c "cd ../.. && projects/api_public/.venv/bin/uvicorn bases.bot_detector.api_public.src.core.server:app --host 0.0.0.0 --port 5000 --log-level warning --reload --reload-dir /app" + sh -c "cd ../.. && projects/api_public/.venv/bin/uvicorn bases.api_public.core.server:app --host 0.0.0.0 --port 5000 --log-level warning --reload --reload-dir /app" env_file: - .env environment: @@ -395,4 +395,4 @@ networks: # alpine sh -c "cp -r /source/* /data/" volumes: uv_cache: - external: true \ No newline at end of file + external: true diff --git a/projects/api_public/Dockerfile b/projects/api_public/Dockerfile index c7b3656..45cb95b 100644 --- a/projects/api_public/Dockerfile +++ b/projects/api_public/Dockerfile @@ -27,4 +27,4 @@ COPY --from=builder --chown=appuser /app/projects/api_public/.venv /app/projects USER appuser -CMD [".venv/bin/uvicorn", "bot_detector.api_public.src.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file +CMD [".venv/bin/uvicorn", "bases.api_public.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] diff --git a/projects/api_public/pyproject.toml b/projects/api_public/pyproject.toml index ed7ff76..d7b9ff6 100644 --- a/projects/api_public/pyproject.toml +++ b/projects/api_public/pyproject.toml @@ -20,13 +20,13 @@ dependencies = [ ] [project.scripts] -scrape_task_producer = "bot_detector.api_public.core.server:run" +scrape_task_producer = "bases.api_public.core.server:run" [tool.hatch.build.hooks.polylith-bricks] packages = ["bot_detector"] [tool.polylith.bricks] -"../../bases/bot_detector/api_public" = "bot_detector/api_public" +"../../bases/api_public" = "bot_detector/api_public" "../../components/bot_detector/database" = "bot_detector/database" "../../components/bot_detector/kafka" = "bot_detector/kafka" "../../components/bot_detector/structs" = "bot_detector/structs" From e4e1364017281b1119633a3799d288dba119adf7 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Tue, 11 Nov 2025 18:19:59 -0500 Subject: [PATCH 02/11] adding tests --- .../bot_detector/api_public/test_core.py | 64 ++++++++ .../bot_detector/api_public/test_endpoints.py | 146 ++++++++++++++++++ .../bot_detector/api_public/test_routes.py | 31 ++++ .../bot_detector/api_public/test_schemas.py | 25 +++ .../bot_detector/api_public/test_utils.py | 23 +++ 5 files changed, 289 insertions(+) create mode 100644 test/bases/bot_detector/api_public/test_endpoints.py create mode 100644 test/bases/bot_detector/api_public/test_routes.py create mode 100644 test/bases/bot_detector/api_public/test_schemas.py create mode 100644 test/bases/bot_detector/api_public/test_utils.py diff --git a/test/bases/bot_detector/api_public/test_core.py b/test/bases/bot_detector/api_public/test_core.py index e69de29..c53de4a 100644 --- a/test/bases/bot_detector/api_public/test_core.py +++ b/test/bases/bot_detector/api_public/test_core.py @@ -0,0 +1,64 @@ +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import FastAPI + +from bases.api_public.core import server +from bases.api_public.core.fastapi.dependencies import session as session_dep +from bases.api_public.core.fastapi.dependencies import kafka as kafka_dep + + +def test_create_app_wires_routes_and_middleware(): + app = server.create_app() + assert isinstance(app, FastAPI) + paths = {route.path for route in app.routes} + assert "/v2/player/prediction" in paths + middleware_names = {mw.cls.__name__ for mw in app.user_middleware} + assert {"LoggingMiddleware", "PrometheusMiddleware"}.issubset(middleware_names) + + +@pytest.mark.asyncio() +async def test_get_session_yields_session(monkeypatch): + fake_session = AsyncMock() + + class _Factory: + async def __aenter__(self): + return fake_session + + async def __aexit__(self, exc_type, exc, tb): + pass + + async def _session_factory(): + return _Factory() + + monkeypatch.setattr( + session_dep, + "SessionFactory", + MagicMock(return_value=_Factory()), + ) + + gen = session_dep.get_session() + session = await gen.__anext__() + assert session is fake_session + with pytest.raises(StopAsyncIteration): + await gen.__anext__() + + +@pytest.mark.asyncio() +async def test_lifespan_starts_and_stops_producer(monkeypatch): + fake_producer = AsyncMock() + monkeypatch.setattr( + kafka_dep.kafka_manager, + "set_producer", + lambda key, producer: None, + ) + monkeypatch.setattr( + kafka_dep.kafka_manager, + "get_producer", + lambda key: fake_producer, + ) + + async with server.lifespan(server.create_app()): + fake_producer.start.assert_awaited_once() + fake_producer.stop.assert_awaited_once() diff --git a/test/bases/bot_detector/api_public/test_endpoints.py b/test/bases/bot_detector/api_public/test_endpoints.py new file mode 100644 index 0000000..b234fa5 --- /dev/null +++ b/test/bases/bot_detector/api_public/test_endpoints.py @@ -0,0 +1,146 @@ +from fastapi.testclient import TestClient + +from bases.api_public.core import server +from bases.api_public.core.fastapi.dependencies.session import get_session +from bases.api_public.player.repository import Player as PlayerRepo +from bases.api_public.feedback.repository import Feedback as FeedbackRepo +from bases.api_public.reports.repository import Report as ReportRepo, CustomError + + +async def _dummy_session(): + yield object() + + +def _client(monkeypatch): + app = server.create_app() + app.dependency_overrides[get_session] = _dummy_session + return TestClient(app) + + +def test_player_prediction_not_found(monkeypatch): + async def fake_prediction(*args, **kwargs): + return [] + + monkeypatch.setattr(PlayerRepo, "get_prediction", fake_prediction) + client = _client(monkeypatch) + resp = client.get( + "/v2/player/prediction", + params={"name": ["abc"], "breakdown": "false"}, + ) + assert resp.status_code == 404 + assert resp.json()["detail"] == "Player not found" + + +def test_feedback_duplicate_returns_422(monkeypatch): + async def fake_insert(*args, **kwargs): + return False, "duplicate_record" + + monkeypatch.setattr(FeedbackRepo, "insert_feedback", fake_insert) + client = _client(monkeypatch) + payload = { + "player_name": "abc", + "vote": 1, + "prediction": "bot", + "confidence": 0.5, + "subject_id": 1, + } + resp = client.post("/v2/feedback", json=payload) + assert resp.status_code == 422 + assert resp.json()["detail"] == "duplicate_record" + + +def test_report_validation_error(monkeypatch): + async def fake_parse(self, data): + return None, "invalid data size" + + monkeypatch.setattr(ReportRepo, "parse_data", fake_parse) + client = _client(monkeypatch) + resp = client.post("/v2/report", json=[]) + assert resp.status_code == 400 + assert resp.json()["detail"] == "invalid data size" + + +def test_report_producer_failure(monkeypatch): + async def fake_parse(self, data): + class _Detection: + reporter = "a" + reported = "b" + + def model_dump(self): + return { + "reporter": "a", + "reported": "b", + "region_id": 1, + "x_coord": 1, + "y_coord": 1, + "z_coord": 0, + "ts": 1, + "manual_detect": 0, + "on_members_world": 0, + "on_pvp_world": 0, + "world_number": 301, + "equipment": { + "equip_head_id": 1, + "equip_amulet_id": 1, + "equip_torso_id": 1, + "equip_legs_id": 1, + "equip_boots_id": 1, + "equip_cape_id": 1, + "equip_hands_id": 1, + "equip_weapon_id": 1, + "equip_shield_id": 1, + }, + "equip_ge_value": 1, + } + + return [_Detection()], None + + class _Player: + def __init__(self, name, id_): + self.name = name + self.id = id_ + + async def fake_get_or_insert(self, player_name, **kwargs): + return _Player(player_name, 1 if player_name == "a" else 2) + + async def fake_send(self, *args, **kwargs): + raise CustomError("boom") + + monkeypatch.setattr(ReportRepo, "parse_data", fake_parse) + monkeypatch.setattr(PlayerRepo, "get_or_insert", fake_get_or_insert) + monkeypatch.setattr(PlayerRepo, "sanitize_name", lambda self, n: n) + monkeypatch.setattr(ReportRepo, "send_to_kafka", fake_send) + + client = _client(monkeypatch) + resp = client.post( + "/v2/report", + json=[ + { + "reporter": "a", + "reported": "b", + "region_id": 1, + "x_coord": 1, + "y_coord": 1, + "z_coord": 0, + "ts": 1, + "manual_detect": 0, + "on_members_world": 0, + "on_pvp_world": 0, + "world_number": 301, + "equipment": { + "equip_head_id": 1, + "equip_amulet_id": 1, + "equip_torso_id": 1, + "equip_legs_id": 1, + "equip_boots_id": 1, + "equip_cape_id": 1, + "equip_hands_id": 1, + "equip_weapon_id": 1, + "equip_shield_id": 1, + }, + "equip_ge_value": 1, + } + ], + ) + assert resp.status_code == 500 + assert resp.json()["detail"] == "Internal error" diff --git a/test/bases/bot_detector/api_public/test_routes.py b/test/bases/bot_detector/api_public/test_routes.py new file mode 100644 index 0000000..d70c962 --- /dev/null +++ b/test/bases/bot_detector/api_public/test_routes.py @@ -0,0 +1,31 @@ +from fastapi.routing import APIRoute + +from bases.api_public import routes + + +def _api_paths() -> set[str]: + return { + route.path + for route in routes.router.routes + if isinstance(route, APIRoute) + } + + +def test_router_registers_all_feature_paths(): + expected = { + "/v2/player/report/score", + "/v2/player/feedback/score", + "/v2/player/prediction", + "/v2/report", + "/v2/feedback", + "/v2/labels", + "/v2/labels/{label_id}", + } + paths = _api_paths() + assert expected.issubset(paths) + + +def test_router_applies_v2_prefix_once(): + paths = _api_paths() + assert paths, "router should expose routes" + assert all(path.startswith("/v2/") for path in paths) diff --git a/test/bases/bot_detector/api_public/test_schemas.py b/test/bases/bot_detector/api_public/test_schemas.py new file mode 100644 index 0000000..5914aaa --- /dev/null +++ b/test/bases/bot_detector/api_public/test_schemas.py @@ -0,0 +1,25 @@ +import pytest + +from bases.api_public.feedback.schemas import FeedbackInput + + +def _base_feedback_kwargs() -> dict: + return { + "player_name": "Some_Player", + "vote": 1, + "prediction": "bot", + "confidence": 0.5, + "subject_id": 123, + } + + +def test_feedback_input_accepts_valid_osrs_name(): + data = FeedbackInput(**_base_feedback_kwargs()) + assert data.player_name == "Some_Player" + + +def test_feedback_input_rejects_invalid_name(): + bad_kwargs = _base_feedback_kwargs() + bad_kwargs["player_name"] = "this-name-is-way-too-long" + with pytest.raises(ValueError): + FeedbackInput(**bad_kwargs) diff --git a/test/bases/bot_detector/api_public/test_utils.py b/test/bases/bot_detector/api_public/test_utils.py new file mode 100644 index 0000000..0daadb6 --- /dev/null +++ b/test/bases/bot_detector/api_public/test_utils.py @@ -0,0 +1,23 @@ +import asyncio + +import pytest + +from bases.api_public.core.fastapi.dependencies.to_jagex_name import to_jagex_name +from bases.api_public.player.repository import Player +from bases.api_public.core._cache import SimpleALRUCache + + +class _DummySession: + async def execute(self, *args, **kwargs): # pragma: no cover - not used here + raise AssertionError("execute should not be called") + + +@pytest.mark.asyncio() +async def test_to_jagex_name_normalizes_variants(): + assert await to_jagex_name("Some_Name") == "some name" + assert await to_jagex_name("AlreadyClean") == "alreadyclean" + + +def test_player_sanitize_name_is_consistent(): + repo = Player(session=_DummySession(), cache=SimpleALRUCache()) + assert repo.sanitize_name("My_Name-Here ") == "my name here" From 55f6fc02ccc5c8fed42a9f2ec24f37506e252d4a Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Tue, 11 Nov 2025 18:46:22 -0500 Subject: [PATCH 03/11] moved api back under bot_detector in bases --- bases/api_public/feedback/__init__.py | 3 --- bases/api_public/labels/__init__.py | 3 --- bases/api_public/player/__init__.py | 3 --- bases/api_public/reports/__init__.py | 3 --- bases/{ => bot_detector}/api_public/__init__.py | 0 bases/{ => bot_detector}/api_public/core/.gitkeep | 0 bases/{ => bot_detector}/api_public/core/__init__.py | 0 bases/{ => bot_detector}/api_public/core/_cache.py | 0 bases/{ => bot_detector}/api_public/core/config.py | 0 .../api_public/core/fastapi/dependencies/kafka.py | 0 .../api_public/core/fastapi/dependencies/session.py | 2 +- .../core/fastapi/dependencies/to_jagex_name.py | 0 .../api_public/core/fastapi/middleware/__init__.py | 0 .../api_public/core/fastapi/middleware/logging.py | 0 .../api_public/core/fastapi/middleware/metrics.py | 0 bases/{ => bot_detector}/api_public/core/logging.py | 0 bases/{ => bot_detector}/api_public/core/server.py | 6 +++--- bases/bot_detector/api_public/feedback/__init__.py | 3 +++ .../api_public/feedback/repository.py | 2 +- bases/{ => bot_detector}/api_public/feedback/routes.py | 10 +++++----- .../{ => bot_detector}/api_public/feedback/schemas.py | 0 bases/bot_detector/api_public/labels/__init__.py | 3 +++ .../{ => bot_detector}/api_public/labels/repository.py | 0 bases/{ => bot_detector}/api_public/labels/routes.py | 6 +++--- bases/{ => bot_detector}/api_public/labels/schemas.py | 0 bases/bot_detector/api_public/player/__init__.py | 3 +++ .../{ => bot_detector}/api_public/player/repository.py | 4 ++-- bases/{ => bot_detector}/api_public/player/routes.py | 8 ++++---- bases/{ => bot_detector}/api_public/player/schemas.py | 0 bases/bot_detector/api_public/reports/__init__.py | 3 +++ .../api_public/reports/repository.py | 2 +- bases/{ => bot_detector}/api_public/reports/routes.py | 10 +++++----- bases/{ => bot_detector}/api_public/reports/schemas.py | 0 bases/{ => bot_detector}/api_public/routes.py | 2 +- bases/{ => bot_detector}/api_public/shared/__init__.py | 0 .../{ => bot_detector}/api_public/shared/responses.py | 0 docker-compose-dev.yml | 2 +- docker-compose.yml | 4 ++-- projects/api_public/Dockerfile | 2 +- projects/api_public/pyproject.toml | 4 ++-- test/bases/bot_detector/api_public/test_core.py | 6 +++--- test/bases/bot_detector/api_public/test_endpoints.py | 10 +++++----- test/bases/bot_detector/api_public/test_routes.py | 2 +- test/bases/bot_detector/api_public/test_schemas.py | 2 +- test/bases/bot_detector/api_public/test_utils.py | 6 +++--- 45 files changed, 57 insertions(+), 57 deletions(-) delete mode 100644 bases/api_public/feedback/__init__.py delete mode 100644 bases/api_public/labels/__init__.py delete mode 100644 bases/api_public/player/__init__.py delete mode 100644 bases/api_public/reports/__init__.py rename bases/{ => bot_detector}/api_public/__init__.py (100%) rename bases/{ => bot_detector}/api_public/core/.gitkeep (100%) rename bases/{ => bot_detector}/api_public/core/__init__.py (100%) rename bases/{ => bot_detector}/api_public/core/_cache.py (100%) rename bases/{ => bot_detector}/api_public/core/config.py (100%) rename bases/{ => bot_detector}/api_public/core/fastapi/dependencies/kafka.py (100%) rename bases/{ => bot_detector}/api_public/core/fastapi/dependencies/session.py (89%) rename bases/{ => bot_detector}/api_public/core/fastapi/dependencies/to_jagex_name.py (100%) rename bases/{ => bot_detector}/api_public/core/fastapi/middleware/__init__.py (100%) rename bases/{ => bot_detector}/api_public/core/fastapi/middleware/logging.py (100%) rename bases/{ => bot_detector}/api_public/core/fastapi/middleware/metrics.py (100%) rename bases/{ => bot_detector}/api_public/core/logging.py (100%) rename bases/{ => bot_detector}/api_public/core/server.py (89%) create mode 100644 bases/bot_detector/api_public/feedback/__init__.py rename bases/{ => bot_detector}/api_public/feedback/repository.py (97%) rename bases/{ => bot_detector}/api_public/feedback/routes.py (63%) rename bases/{ => bot_detector}/api_public/feedback/schemas.py (100%) create mode 100644 bases/bot_detector/api_public/labels/__init__.py rename bases/{ => bot_detector}/api_public/labels/repository.py (100%) rename bases/{ => bot_detector}/api_public/labels/routes.py (81%) rename bases/{ => bot_detector}/api_public/labels/schemas.py (100%) create mode 100644 bases/bot_detector/api_public/player/__init__.py rename bases/{ => bot_detector}/api_public/player/repository.py (97%) rename bases/{ => bot_detector}/api_public/player/routes.py (90%) rename bases/{ => bot_detector}/api_public/player/schemas.py (100%) create mode 100644 bases/bot_detector/api_public/reports/__init__.py rename bases/{ => bot_detector}/api_public/reports/repository.py (97%) rename bases/{ => bot_detector}/api_public/reports/routes.py (85%) rename bases/{ => bot_detector}/api_public/reports/schemas.py (100%) rename bases/{ => bot_detector}/api_public/routes.py (79%) rename bases/{ => bot_detector}/api_public/shared/__init__.py (100%) rename bases/{ => bot_detector}/api_public/shared/responses.py (100%) diff --git a/bases/api_public/feedback/__init__.py b/bases/api_public/feedback/__init__.py deleted file mode 100644 index ab02352..0000000 --- a/bases/api_public/feedback/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bases.api_public.feedback.routes import router - -__all__ = ["router"] diff --git a/bases/api_public/labels/__init__.py b/bases/api_public/labels/__init__.py deleted file mode 100644 index 8636e97..0000000 --- a/bases/api_public/labels/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bases.api_public.labels.routes import router - -__all__ = ["router"] diff --git a/bases/api_public/player/__init__.py b/bases/api_public/player/__init__.py deleted file mode 100644 index 9517486..0000000 --- a/bases/api_public/player/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bases.api_public.player.routes import router - -__all__ = ["router"] diff --git a/bases/api_public/reports/__init__.py b/bases/api_public/reports/__init__.py deleted file mode 100644 index e194a5d..0000000 --- a/bases/api_public/reports/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bases.api_public.reports.routes import router - -__all__ = ["router"] diff --git a/bases/api_public/__init__.py b/bases/bot_detector/api_public/__init__.py similarity index 100% rename from bases/api_public/__init__.py rename to bases/bot_detector/api_public/__init__.py diff --git a/bases/api_public/core/.gitkeep b/bases/bot_detector/api_public/core/.gitkeep similarity index 100% rename from bases/api_public/core/.gitkeep rename to bases/bot_detector/api_public/core/.gitkeep diff --git a/bases/api_public/core/__init__.py b/bases/bot_detector/api_public/core/__init__.py similarity index 100% rename from bases/api_public/core/__init__.py rename to bases/bot_detector/api_public/core/__init__.py diff --git a/bases/api_public/core/_cache.py b/bases/bot_detector/api_public/core/_cache.py similarity index 100% rename from bases/api_public/core/_cache.py rename to bases/bot_detector/api_public/core/_cache.py diff --git a/bases/api_public/core/config.py b/bases/bot_detector/api_public/core/config.py similarity index 100% rename from bases/api_public/core/config.py rename to bases/bot_detector/api_public/core/config.py diff --git a/bases/api_public/core/fastapi/dependencies/kafka.py b/bases/bot_detector/api_public/core/fastapi/dependencies/kafka.py similarity index 100% rename from bases/api_public/core/fastapi/dependencies/kafka.py rename to bases/bot_detector/api_public/core/fastapi/dependencies/kafka.py diff --git a/bases/api_public/core/fastapi/dependencies/session.py b/bases/bot_detector/api_public/core/fastapi/dependencies/session.py similarity index 89% rename from bases/api_public/core/fastapi/dependencies/session.py rename to bases/bot_detector/api_public/core/fastapi/dependencies/session.py index 040fdd9..f4ce83d 100644 --- a/bases/api_public/core/fastapi/dependencies/session.py +++ b/bases/bot_detector/api_public/core/fastapi/dependencies/session.py @@ -1,4 +1,4 @@ -from bases.api_public.core.config import DB_SEMAPHORE, settings +from bases.bot_detector.api_public.core.config import DB_SEMAPHORE, settings from bot_detector.database import Settings as DBSettings from bot_detector.database import get_session_factory from sqlalchemy.ext.asyncio import AsyncSession diff --git a/bases/api_public/core/fastapi/dependencies/to_jagex_name.py b/bases/bot_detector/api_public/core/fastapi/dependencies/to_jagex_name.py similarity index 100% rename from bases/api_public/core/fastapi/dependencies/to_jagex_name.py rename to bases/bot_detector/api_public/core/fastapi/dependencies/to_jagex_name.py diff --git a/bases/api_public/core/fastapi/middleware/__init__.py b/bases/bot_detector/api_public/core/fastapi/middleware/__init__.py similarity index 100% rename from bases/api_public/core/fastapi/middleware/__init__.py rename to bases/bot_detector/api_public/core/fastapi/middleware/__init__.py diff --git a/bases/api_public/core/fastapi/middleware/logging.py b/bases/bot_detector/api_public/core/fastapi/middleware/logging.py similarity index 100% rename from bases/api_public/core/fastapi/middleware/logging.py rename to bases/bot_detector/api_public/core/fastapi/middleware/logging.py diff --git a/bases/api_public/core/fastapi/middleware/metrics.py b/bases/bot_detector/api_public/core/fastapi/middleware/metrics.py similarity index 100% rename from bases/api_public/core/fastapi/middleware/metrics.py rename to bases/bot_detector/api_public/core/fastapi/middleware/metrics.py diff --git a/bases/api_public/core/logging.py b/bases/bot_detector/api_public/core/logging.py similarity index 100% rename from bases/api_public/core/logging.py rename to bases/bot_detector/api_public/core/logging.py diff --git a/bases/api_public/core/server.py b/bases/bot_detector/api_public/core/server.py similarity index 89% rename from bases/api_public/core/server.py rename to bases/bot_detector/api_public/core/server.py index e1ca6af..e1d6a73 100644 --- a/bases/api_public/core/server.py +++ b/bases/bot_detector/api_public/core/server.py @@ -1,9 +1,9 @@ import logging from contextlib import asynccontextmanager -from bases.api_public import routes as routes_pkg -from bases.api_public.core.fastapi.dependencies.kafka import kafka_manager -from bases.api_public.core.fastapi.middleware import ( +from bases.bot_detector.api_public import routes as routes_pkg +from bases.bot_detector.api_public.core.fastapi.dependencies.kafka import kafka_manager +from bases.bot_detector.api_public.core.fastapi.middleware import ( LoggingMiddleware, PrometheusMiddleware, ) diff --git a/bases/bot_detector/api_public/feedback/__init__.py b/bases/bot_detector/api_public/feedback/__init__.py new file mode 100644 index 0000000..ab97598 --- /dev/null +++ b/bases/bot_detector/api_public/feedback/__init__.py @@ -0,0 +1,3 @@ +from bases.bot_detector.api_public.feedback.routes import router + +__all__ = ["router"] diff --git a/bases/api_public/feedback/repository.py b/bases/bot_detector/api_public/feedback/repository.py similarity index 97% rename from bases/api_public/feedback/repository.py rename to bases/bot_detector/api_public/feedback/repository.py index 283614f..9f9597d 100644 --- a/bases/api_public/feedback/repository.py +++ b/bases/bot_detector/api_public/feedback/repository.py @@ -1,6 +1,6 @@ import logging -from bases.api_public.feedback.schemas import FeedbackInput +from bases.bot_detector.api_public.feedback.schemas import FeedbackInput from bot_detector.database.api_public import ( Player as dbPlayer, PredictionFeedback as dbFeedback, diff --git a/bases/api_public/feedback/routes.py b/bases/bot_detector/api_public/feedback/routes.py similarity index 63% rename from bases/api_public/feedback/routes.py rename to bases/bot_detector/api_public/feedback/routes.py index aff0497..cac8e61 100644 --- a/bases/api_public/feedback/routes.py +++ b/bases/bot_detector/api_public/feedback/routes.py @@ -1,10 +1,10 @@ import logging -from bases.api_public.feedback.repository import Feedback -from bases.api_public.feedback.schemas import FeedbackInput -from bases.api_public.shared.responses import Ok -from bases.api_public.core.fastapi.dependencies.session import get_session -from bases.api_public.core.fastapi.dependencies.to_jagex_name import ( +from bases.bot_detector.api_public.feedback.repository import Feedback +from bases.bot_detector.api_public.feedback.schemas import FeedbackInput +from bases.bot_detector.api_public.shared.responses import Ok +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session +from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import ( to_jagex_name, ) from fastapi import APIRouter, Depends, HTTPException, status diff --git a/bases/api_public/feedback/schemas.py b/bases/bot_detector/api_public/feedback/schemas.py similarity index 100% rename from bases/api_public/feedback/schemas.py rename to bases/bot_detector/api_public/feedback/schemas.py diff --git a/bases/bot_detector/api_public/labels/__init__.py b/bases/bot_detector/api_public/labels/__init__.py new file mode 100644 index 0000000..34dd84e --- /dev/null +++ b/bases/bot_detector/api_public/labels/__init__.py @@ -0,0 +1,3 @@ +from bases.bot_detector.api_public.labels.routes import router + +__all__ = ["router"] diff --git a/bases/api_public/labels/repository.py b/bases/bot_detector/api_public/labels/repository.py similarity index 100% rename from bases/api_public/labels/repository.py rename to bases/bot_detector/api_public/labels/repository.py diff --git a/bases/api_public/labels/routes.py b/bases/bot_detector/api_public/labels/routes.py similarity index 81% rename from bases/api_public/labels/routes.py rename to bases/bot_detector/api_public/labels/routes.py index ad1aa42..dfa06c0 100644 --- a/bases/api_public/labels/routes.py +++ b/bases/bot_detector/api_public/labels/routes.py @@ -1,8 +1,8 @@ import logging -from bases.api_public.labels.repository import LabelRepository -from bases.api_public.labels.schemas import LabelResponse -from bases.api_public.core.fastapi.dependencies.session import get_session +from bases.bot_detector.api_public.labels.repository import LabelRepository +from bases.bot_detector.api_public.labels.schemas import LabelResponse +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session from fastapi import APIRouter, Depends, status router = APIRouter(tags=["Labels"]) diff --git a/bases/api_public/labels/schemas.py b/bases/bot_detector/api_public/labels/schemas.py similarity index 100% rename from bases/api_public/labels/schemas.py rename to bases/bot_detector/api_public/labels/schemas.py diff --git a/bases/bot_detector/api_public/player/__init__.py b/bases/bot_detector/api_public/player/__init__.py new file mode 100644 index 0000000..351c7b3 --- /dev/null +++ b/bases/bot_detector/api_public/player/__init__.py @@ -0,0 +1,3 @@ +from bases.bot_detector.api_public.player.routes import router + +__all__ = ["router"] diff --git a/bases/api_public/player/repository.py b/bases/bot_detector/api_public/player/repository.py similarity index 97% rename from bases/api_public/player/repository.py rename to bases/bot_detector/api_public/player/repository.py index 2a676b0..5f9bba7 100644 --- a/bases/api_public/player/repository.py +++ b/bases/bot_detector/api_public/player/repository.py @@ -1,8 +1,8 @@ import logging import sqlalchemy as sqla -from bases.api_public.player.schemas import PlayerCreate, PlayerInDB -from bases.api_public.core._cache import SimpleALRUCache +from bases.bot_detector.api_public.player.schemas import PlayerCreate, PlayerInDB +from bases.bot_detector.api_public.core._cache import SimpleALRUCache from bot_detector.database.api_public import ( Player as dbPlayer, PredictionFeedback as dbFeedback, diff --git a/bases/api_public/player/routes.py b/bases/bot_detector/api_public/player/routes.py similarity index 90% rename from bases/api_public/player/routes.py rename to bases/bot_detector/api_public/player/routes.py index f7255bb..6d24dcb 100644 --- a/bases/api_public/player/routes.py +++ b/bases/bot_detector/api_public/player/routes.py @@ -2,14 +2,14 @@ import logging from typing import Annotated -from bases.api_public.player.repository import Player as repoPlayer -from bases.api_public.player.schemas import ( +from bases.bot_detector.api_public.player.repository import Player as repoPlayer +from bases.bot_detector.api_public.player.schemas import ( FeedbackScoreResponse, PredictionResponse, ReportScoreResponse, ) -from bases.api_public.core.fastapi.dependencies.session import get_session -from bases.api_public.core.fastapi.dependencies.to_jagex_name import ( +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session +from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import ( to_jagex_name, ) from fastapi import APIRouter, Depends, HTTPException, Query, status diff --git a/bases/api_public/player/schemas.py b/bases/bot_detector/api_public/player/schemas.py similarity index 100% rename from bases/api_public/player/schemas.py rename to bases/bot_detector/api_public/player/schemas.py diff --git a/bases/bot_detector/api_public/reports/__init__.py b/bases/bot_detector/api_public/reports/__init__.py new file mode 100644 index 0000000..f30b497 --- /dev/null +++ b/bases/bot_detector/api_public/reports/__init__.py @@ -0,0 +1,3 @@ +from bases.bot_detector.api_public.reports.routes import router + +__all__ = ["router"] diff --git a/bases/api_public/reports/repository.py b/bases/bot_detector/api_public/reports/repository.py similarity index 97% rename from bases/api_public/reports/repository.py rename to bases/bot_detector/api_public/reports/repository.py index 3826ac9..2662dc8 100644 --- a/bases/api_public/reports/repository.py +++ b/bases/bot_detector/api_public/reports/repository.py @@ -2,7 +2,7 @@ import logging import time -from bases.api_public.core.fastapi.dependencies.kafka import kafka_manager +from bases.bot_detector.api_public.core.fastapi.dependencies.kafka import kafka_manager from bot_detector.kafka.repositories.reports_to_insert import ( RepoReportsToInsertProducer, ) diff --git a/bases/api_public/reports/routes.py b/bases/bot_detector/api_public/reports/routes.py similarity index 85% rename from bases/api_public/reports/routes.py rename to bases/bot_detector/api_public/reports/routes.py index dad34d4..2aec5b1 100644 --- a/bases/api_public/reports/routes.py +++ b/bases/bot_detector/api_public/reports/routes.py @@ -1,10 +1,10 @@ import logging -from bases.api_public.player.repository import Player -from bases.api_public.reports.repository import CustomError, Report -from bases.api_public.shared.responses import Ok -from bases.api_public.core._cache import SimpleALRUCache -from bases.api_public.core.fastapi.dependencies.session import get_session +from bases.bot_detector.api_public.player.repository import Player +from bases.bot_detector.api_public.reports.repository import CustomError, Report +from bases.bot_detector.api_public.shared.responses import Ok +from bases.bot_detector.api_public.core._cache import SimpleALRUCache +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session from bot_detector.structs import Detection, ParsedDetection from fastapi import APIRouter, Depends, status from fastapi.exceptions import HTTPException diff --git a/bases/api_public/reports/schemas.py b/bases/bot_detector/api_public/reports/schemas.py similarity index 100% rename from bases/api_public/reports/schemas.py rename to bases/bot_detector/api_public/reports/schemas.py diff --git a/bases/api_public/routes.py b/bases/bot_detector/api_public/routes.py similarity index 79% rename from bases/api_public/routes.py rename to bases/bot_detector/api_public/routes.py index f6912e7..d928e69 100644 --- a/bases/api_public/routes.py +++ b/bases/bot_detector/api_public/routes.py @@ -1,6 +1,6 @@ from fastapi import APIRouter -from bases.api_public import feedback, labels, player, reports +from bases.bot_detector.api_public import feedback, labels, player, reports router = APIRouter() v2_router = APIRouter(prefix="/v2") diff --git a/bases/api_public/shared/__init__.py b/bases/bot_detector/api_public/shared/__init__.py similarity index 100% rename from bases/api_public/shared/__init__.py rename to bases/bot_detector/api_public/shared/__init__.py diff --git a/bases/api_public/shared/responses.py b/bases/bot_detector/api_public/shared/responses.py similarity index 100% rename from bases/api_public/shared/responses.py rename to bases/bot_detector/api_public/shared/responses.py diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 84aa8cf..1325dce 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -121,7 +121,7 @@ services: context: ./ dockerfile: Dockerfile # command: ["uv", "run", "bases/bot_detector/hiscore_scraper/core.py"] - # command: uv run uvicorn bases.api_public.core.server:app --host 0.0.0.0 --reload --port 5000 + # command: uv run uvicorn bases.bot_detector.api_public.core.server:app --host 0.0.0.0 --reload --port 5000 # command: uv run uvicorn bases.bot_detector.website.core.server:app --host 0.0.0.0 --reload --port 5000 # command: uv run uvicorn bases.bot_detector.api_ml.core.server:app --host 0.0.0.0 --reload --port 5000 ports: diff --git a/docker-compose.yml b/docker-compose.yml index 95d707e..c1825f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -137,7 +137,7 @@ services: context: ./ dockerfile: ./projects/hiscore_scraper/Dockerfile target: production - # command: ["uv", "run", "uvicorn", "bases.api_public.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning", "--reload", "--reload-dir", "/app/bot_detector/api_public/"] + # command: ["uv", "run", "uvicorn", "bases.bot_detector.api_public.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning", "--reload", "--reload-dir", "/app/bot_detector/api_public/"] networks: - botdetector-network env_file: @@ -283,7 +283,7 @@ services: target: builder # command: ["sleep", "infinity"] command: > - sh -c "cd ../.. && projects/api_public/.venv/bin/uvicorn bases.api_public.core.server:app --host 0.0.0.0 --port 5000 --log-level warning --reload --reload-dir /app" + sh -c "cd ../.. && projects/api_public/.venv/bin/uvicorn bases.bot_detector.api_public.core.server:app --host 0.0.0.0 --port 5000 --log-level warning --reload --reload-dir /app" env_file: - .env environment: diff --git a/projects/api_public/Dockerfile b/projects/api_public/Dockerfile index 45cb95b..c71fc01 100644 --- a/projects/api_public/Dockerfile +++ b/projects/api_public/Dockerfile @@ -27,4 +27,4 @@ COPY --from=builder --chown=appuser /app/projects/api_public/.venv /app/projects USER appuser -CMD [".venv/bin/uvicorn", "bases.api_public.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] +CMD [".venv/bin/uvicorn", "bases.bot_detector.api_public.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] diff --git a/projects/api_public/pyproject.toml b/projects/api_public/pyproject.toml index d7b9ff6..90bc986 100644 --- a/projects/api_public/pyproject.toml +++ b/projects/api_public/pyproject.toml @@ -20,13 +20,13 @@ dependencies = [ ] [project.scripts] -scrape_task_producer = "bases.api_public.core.server:run" +scrape_task_producer = "bases.bot_detector.api_public.core.server:run" [tool.hatch.build.hooks.polylith-bricks] packages = ["bot_detector"] [tool.polylith.bricks] -"../../bases/api_public" = "bot_detector/api_public" +"../../bases/bot_detector/api_public" = "bot_detector/api_public" "../../components/bot_detector/database" = "bot_detector/database" "../../components/bot_detector/kafka" = "bot_detector/kafka" "../../components/bot_detector/structs" = "bot_detector/structs" diff --git a/test/bases/bot_detector/api_public/test_core.py b/test/bases/bot_detector/api_public/test_core.py index c53de4a..85ca40f 100644 --- a/test/bases/bot_detector/api_public/test_core.py +++ b/test/bases/bot_detector/api_public/test_core.py @@ -4,9 +4,9 @@ import pytest from fastapi import FastAPI -from bases.api_public.core import server -from bases.api_public.core.fastapi.dependencies import session as session_dep -from bases.api_public.core.fastapi.dependencies import kafka as kafka_dep +from bases.bot_detector.api_public.core import server +from bases.bot_detector.api_public.core.fastapi.dependencies import session as session_dep +from bases.bot_detector.api_public.core.fastapi.dependencies import kafka as kafka_dep def test_create_app_wires_routes_and_middleware(): diff --git a/test/bases/bot_detector/api_public/test_endpoints.py b/test/bases/bot_detector/api_public/test_endpoints.py index b234fa5..7fa20b2 100644 --- a/test/bases/bot_detector/api_public/test_endpoints.py +++ b/test/bases/bot_detector/api_public/test_endpoints.py @@ -1,10 +1,10 @@ from fastapi.testclient import TestClient -from bases.api_public.core import server -from bases.api_public.core.fastapi.dependencies.session import get_session -from bases.api_public.player.repository import Player as PlayerRepo -from bases.api_public.feedback.repository import Feedback as FeedbackRepo -from bases.api_public.reports.repository import Report as ReportRepo, CustomError +from bases.bot_detector.api_public.core import server +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session +from bases.bot_detector.api_public.player.repository import Player as PlayerRepo +from bases.bot_detector.api_public.feedback.repository import Feedback as FeedbackRepo +from bases.bot_detector.api_public.reports.repository import Report as ReportRepo, CustomError async def _dummy_session(): diff --git a/test/bases/bot_detector/api_public/test_routes.py b/test/bases/bot_detector/api_public/test_routes.py index d70c962..8ca5a89 100644 --- a/test/bases/bot_detector/api_public/test_routes.py +++ b/test/bases/bot_detector/api_public/test_routes.py @@ -1,6 +1,6 @@ from fastapi.routing import APIRoute -from bases.api_public import routes +from bases.bot_detector.api_public import routes def _api_paths() -> set[str]: diff --git a/test/bases/bot_detector/api_public/test_schemas.py b/test/bases/bot_detector/api_public/test_schemas.py index 5914aaa..7e63f6c 100644 --- a/test/bases/bot_detector/api_public/test_schemas.py +++ b/test/bases/bot_detector/api_public/test_schemas.py @@ -1,6 +1,6 @@ import pytest -from bases.api_public.feedback.schemas import FeedbackInput +from bases.bot_detector.api_public.feedback.schemas import FeedbackInput def _base_feedback_kwargs() -> dict: diff --git a/test/bases/bot_detector/api_public/test_utils.py b/test/bases/bot_detector/api_public/test_utils.py index 0daadb6..b309497 100644 --- a/test/bases/bot_detector/api_public/test_utils.py +++ b/test/bases/bot_detector/api_public/test_utils.py @@ -2,9 +2,9 @@ import pytest -from bases.api_public.core.fastapi.dependencies.to_jagex_name import to_jagex_name -from bases.api_public.player.repository import Player -from bases.api_public.core._cache import SimpleALRUCache +from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import to_jagex_name +from bases.bot_detector.api_public.player.repository import Player +from bases.bot_detector.api_public.core._cache import SimpleALRUCache class _DummySession: From c21ffcf1124059596384c03836c241d865acce27 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Tue, 11 Nov 2025 18:48:45 -0500 Subject: [PATCH 04/11] user player struct --- .../database/api_public/player.py | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/components/bot_detector/database/api_public/player.py b/components/bot_detector/database/api_public/player.py index de772dd..d424f05 100644 --- a/components/bot_detector/database/api_public/player.py +++ b/components/bot_detector/database/api_public/player.py @@ -1,20 +1,5 @@ -from bot_detector.database import Base -from sqlalchemy import Boolean, Column, DateTime, Integer, Text +"""Re-export the shared Players table for API Public consumers.""" +from bot_detector.database.player.structs import PlayersTableStruct as Player -class Player(Base): - __tablename__ = "Players" - - id = Column(Integer, primary_key=True, autoincrement=True) - name = Column(Text) - created_at = Column(DateTime) - updated_at = Column(DateTime) - possible_ban = Column(Boolean) - confirmed_ban = Column(Boolean) - confirmed_player = Column(Boolean) - label_id = Column(Integer) - label_jagex = Column(Integer) - ironman = Column(Boolean) - hardcore_ironman = Column(Boolean) - ultimate_ironman = Column(Boolean) - normalized_name = Column(Text) +__all__ = ["Player"] From c3572d3ef5e093af2ef963f60cdbb1e0cf14e1a4 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Tue, 11 Nov 2025 18:49:29 -0500 Subject: [PATCH 05/11] config parsing issue --- components/bot_detector/logfmt/core.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/components/bot_detector/logfmt/core.py b/components/bot_detector/logfmt/core.py index 5b28b98..75b4dee 100644 --- a/components/bot_detector/logfmt/core.py +++ b/components/bot_detector/logfmt/core.py @@ -2,12 +2,25 @@ import json import logging +from pydantic import field_validator from pydantic_settings import BaseSettings class Settings(BaseSettings): DEBUG: bool = False + @field_validator("DEBUG", mode="before") + @classmethod + def _coerce_debug(cls, value): + if isinstance(value, str): + lowered = value.lower() + if lowered in {"1", "true", "yes", "on", "debug"}: + return True + if lowered in {"0", "false", "no", "off"}: + return False + return False + return value + def can_convert_to_json(s: str) -> dict | None: try: From 37cd75a4e6120b8460f63b295d4a964b233aea09 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Thu, 13 Nov 2025 18:07:04 -0500 Subject: [PATCH 06/11] remove bases --- .../api_public/core/fastapi/dependencies/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bases/bot_detector/api_public/core/fastapi/dependencies/session.py b/bases/bot_detector/api_public/core/fastapi/dependencies/session.py index f4ce83d..e99502c 100644 --- a/bases/bot_detector/api_public/core/fastapi/dependencies/session.py +++ b/bases/bot_detector/api_public/core/fastapi/dependencies/session.py @@ -1,4 +1,4 @@ -from bases.bot_detector.api_public.core.config import DB_SEMAPHORE, settings +from bot_detector.api_public.core.config import DB_SEMAPHORE, settings from bot_detector.database import Settings as DBSettings from bot_detector.database import get_session_factory from sqlalchemy.ext.asyncio import AsyncSession From 9e8234d0f531e5bf321f0a1f6135cba0fab8532a Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Thu, 13 Nov 2025 22:03:24 -0500 Subject: [PATCH 07/11] repo to services rename, moved to components and set structure --- bases/bot_detector/api_public/core/server.py | 15 ++--- .../api_public/feedback/__init__.py | 3 - .../api_public/labels/__init__.py | 3 - .../bot_detector/api_public/labels/routes.py | 44 ------------- .../api_public/player/__init__.py | 3 - .../api_public/reports/__init__.py | 3 - bases/bot_detector/api_public/routes.py | 18 ------ .../api_public/routes/__init__.py | 24 +++++++ .../routes.py => routes/v2/feedback.py} | 17 ++--- .../api_public/routes/v2/labels.py | 32 ++++++++++ .../{player/routes.py => routes/v2/player.py} | 48 +++----------- .../routes.py => routes/v2/reports.py} | 40 +++++------- .../api_public/shared/__init__.py | 0 .../bot_detector/api_public/__init__.py | 10 +++ .../api_public/database/__init__.py | 18 ++++++ .../api_public/database/feedback.py | 28 +++++++++ .../bot_detector/api_public/database/label.py | 9 +++ .../api_public/database/player.py | 5 ++ .../api_public/database/prediction.py | 51 +++++++++++++++ .../api_public/database/report.py | 30 +++++++++ .../api_public/services/__init__.py | 6 ++ .../api_public/services/feedback.py | 4 +- .../api_public/services/labels.py | 2 +- .../api_public/services/player.py | 6 +- .../api_public/services/reports.py | 12 ++-- .../api_public/structs/__init__.py | 42 +++++++++++++ .../api_public/structs/feedback.py | 0 .../bot_detector/api_public/structs/labels.py | 0 .../bot_detector/api_public/structs/player.py | 0 .../api_public/structs/reports.py | 0 .../api_public/structs}/responses.py | 10 +-- components/bot_detector/cache/__init__.py | 3 + components/bot_detector/cache/simple.py | 63 +++++++++++++++++++ components/bot_detector/kafka/__init__.py | 3 + components/bot_detector/kafka/manager.py | 57 +++++++++++++++++ projects/api_public/pyproject.toml | 1 + .../bot_detector/api_public/test_endpoints.py | 23 ++++--- .../bot_detector/api_public/test_schemas.py | 2 +- .../bot_detector/api_public/test_utils.py | 6 +- .../api_public/test_reports_service.py | 60 ++++++++++++++++++ 40 files changed, 512 insertions(+), 189 deletions(-) delete mode 100644 bases/bot_detector/api_public/feedback/__init__.py delete mode 100644 bases/bot_detector/api_public/labels/__init__.py delete mode 100644 bases/bot_detector/api_public/labels/routes.py delete mode 100644 bases/bot_detector/api_public/player/__init__.py delete mode 100644 bases/bot_detector/api_public/reports/__init__.py delete mode 100644 bases/bot_detector/api_public/routes.py create mode 100644 bases/bot_detector/api_public/routes/__init__.py rename bases/bot_detector/api_public/{feedback/routes.py => routes/v2/feedback.py} (64%) create mode 100644 bases/bot_detector/api_public/routes/v2/labels.py rename bases/bot_detector/api_public/{player/routes.py => routes/v2/player.py} (60%) rename bases/bot_detector/api_public/{reports/routes.py => routes/v2/reports.py} (65%) delete mode 100644 bases/bot_detector/api_public/shared/__init__.py create mode 100644 components/bot_detector/api_public/__init__.py create mode 100644 components/bot_detector/api_public/database/__init__.py create mode 100644 components/bot_detector/api_public/database/feedback.py create mode 100644 components/bot_detector/api_public/database/label.py create mode 100644 components/bot_detector/api_public/database/player.py create mode 100644 components/bot_detector/api_public/database/prediction.py create mode 100644 components/bot_detector/api_public/database/report.py create mode 100644 components/bot_detector/api_public/services/__init__.py rename bases/bot_detector/api_public/feedback/repository.py => components/bot_detector/api_public/services/feedback.py (95%) rename bases/bot_detector/api_public/labels/repository.py => components/bot_detector/api_public/services/labels.py (97%) rename bases/bot_detector/api_public/player/repository.py => components/bot_detector/api_public/services/player.py (97%) rename bases/bot_detector/api_public/reports/repository.py => components/bot_detector/api_public/services/reports.py (93%) create mode 100644 components/bot_detector/api_public/structs/__init__.py rename bases/bot_detector/api_public/feedback/schemas.py => components/bot_detector/api_public/structs/feedback.py (100%) rename bases/bot_detector/api_public/labels/schemas.py => components/bot_detector/api_public/structs/labels.py (100%) rename bases/bot_detector/api_public/player/schemas.py => components/bot_detector/api_public/structs/player.py (100%) rename bases/bot_detector/api_public/reports/schemas.py => components/bot_detector/api_public/structs/reports.py (100%) rename {bases/bot_detector/api_public/shared => components/bot_detector/api_public/structs}/responses.py (93%) create mode 100644 components/bot_detector/cache/__init__.py create mode 100644 components/bot_detector/cache/simple.py create mode 100644 components/bot_detector/kafka/manager.py create mode 100644 test/components/bot_detector/api_public/test_reports_service.py diff --git a/bases/bot_detector/api_public/core/server.py b/bases/bot_detector/api_public/core/server.py index e1d6a73..056be97 100644 --- a/bases/bot_detector/api_public/core/server.py +++ b/bases/bot_detector/api_public/core/server.py @@ -1,12 +1,6 @@ import logging from contextlib import asynccontextmanager -from bases.bot_detector.api_public import routes as routes_pkg -from bases.bot_detector.api_public.core.fastapi.dependencies.kafka import kafka_manager -from bases.bot_detector.api_public.core.fastapi.middleware import ( - LoggingMiddleware, - PrometheusMiddleware, -) from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import RepoReportsToInsertProducer from fastapi import FastAPI @@ -14,11 +8,18 @@ from fastapi.middleware.cors import CORSMiddleware from prometheus_client import start_http_server +from bases.bot_detector.api_public import routes +from bases.bot_detector.api_public.core.fastapi.dependencies.kafka import kafka_manager +from bases.bot_detector.api_public.core.fastapi.middleware import ( + LoggingMiddleware, + PrometheusMiddleware, +) + logger = logging.getLogger(__name__) def init_routers(_app: FastAPI) -> None: - _app.include_router(routes_pkg.router) + _app.include_router(routes.router) def make_middleware() -> list[Middleware]: diff --git a/bases/bot_detector/api_public/feedback/__init__.py b/bases/bot_detector/api_public/feedback/__init__.py deleted file mode 100644 index ab97598..0000000 --- a/bases/bot_detector/api_public/feedback/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bases.bot_detector.api_public.feedback.routes import router - -__all__ = ["router"] diff --git a/bases/bot_detector/api_public/labels/__init__.py b/bases/bot_detector/api_public/labels/__init__.py deleted file mode 100644 index 34dd84e..0000000 --- a/bases/bot_detector/api_public/labels/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bases.bot_detector.api_public.labels.routes import router - -__all__ = ["router"] diff --git a/bases/bot_detector/api_public/labels/routes.py b/bases/bot_detector/api_public/labels/routes.py deleted file mode 100644 index dfa06c0..0000000 --- a/bases/bot_detector/api_public/labels/routes.py +++ /dev/null @@ -1,44 +0,0 @@ -import logging - -from bases.bot_detector.api_public.labels.repository import LabelRepository -from bases.bot_detector.api_public.labels.schemas import LabelResponse -from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session -from fastapi import APIRouter, Depends, status - -router = APIRouter(tags=["Labels"]) -logger = logging.getLogger(__name__) - - -@router.get( - "/labels", - response_model=list[LabelResponse], - status_code=status.HTTP_200_OK, -) -async def get_labels(session=Depends(get_session)): - _label_repo = LabelRepository(session) - labels = await _label_repo.get_labels() - - _labels = [] - for label in labels: - _label = LabelResponse(**label.__dict__) - _label.label = _label.label.lower() - _labels.append(_label) - return _labels - - -@router.get( - "/labels/{label_id}", - response_model=LabelResponse | None, - status_code=status.HTTP_200_OK, -) -async def get_label_by_id( - label_id: int, session=Depends(get_session) -) -> LabelResponse | None: - _label_repo = LabelRepository(session) - label = await _label_repo.get_label_by_id(label_id=label_id) - - if label is None: - return None - _label = LabelResponse(**label.__dict__) - _label.label = _label.label.lower() - return _label diff --git a/bases/bot_detector/api_public/player/__init__.py b/bases/bot_detector/api_public/player/__init__.py deleted file mode 100644 index 351c7b3..0000000 --- a/bases/bot_detector/api_public/player/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bases.bot_detector.api_public.player.routes import router - -__all__ = ["router"] diff --git a/bases/bot_detector/api_public/reports/__init__.py b/bases/bot_detector/api_public/reports/__init__.py deleted file mode 100644 index f30b497..0000000 --- a/bases/bot_detector/api_public/reports/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bases.bot_detector.api_public.reports.routes import router - -__all__ = ["router"] diff --git a/bases/bot_detector/api_public/routes.py b/bases/bot_detector/api_public/routes.py deleted file mode 100644 index d928e69..0000000 --- a/bases/bot_detector/api_public/routes.py +++ /dev/null @@ -1,18 +0,0 @@ -from fastapi import APIRouter - -from bases.bot_detector.api_public import feedback, labels, player, reports - -router = APIRouter() -v2_router = APIRouter(prefix="/v2") - -for feature_router in ( - player.router, - reports.router, - feedback.router, - labels.router, -): - v2_router.include_router(feature_router) - -router.include_router(v2_router) - -__all__ = ["router"] diff --git a/bases/bot_detector/api_public/routes/__init__.py b/bases/bot_detector/api_public/routes/__init__.py new file mode 100644 index 0000000..dc15a3a --- /dev/null +++ b/bases/bot_detector/api_public/routes/__init__.py @@ -0,0 +1,24 @@ +from fastapi import APIRouter + +from bases.bot_detector.api_public.routes.v2.player import router as player_router +from bases.bot_detector.api_public.routes.v2.reports import router as reports_router +from bases.bot_detector.api_public.routes.v2.feedback import router as feedback_router +from bases.bot_detector.api_public.routes.v2.labels import router as labels_router + + +def _build_v2() -> APIRouter: + api_router = APIRouter(prefix="/v2") + for feature_router in ( + player_router, + reports_router, + feedback_router, + labels_router, + ): + api_router.include_router(feature_router) + return api_router + + +router = APIRouter() +router.include_router(_build_v2()) + +__all__ = ["router"] diff --git a/bases/bot_detector/api_public/feedback/routes.py b/bases/bot_detector/api_public/routes/v2/feedback.py similarity index 64% rename from bases/bot_detector/api_public/feedback/routes.py rename to bases/bot_detector/api_public/routes/v2/feedback.py index cac8e61..b9076dc 100644 --- a/bases/bot_detector/api_public/feedback/routes.py +++ b/bases/bot_detector/api_public/routes/v2/feedback.py @@ -1,12 +1,10 @@ import logging -from bases.bot_detector.api_public.feedback.repository import Feedback -from bases.bot_detector.api_public.feedback.schemas import FeedbackInput -from bases.bot_detector.api_public.shared.responses import Ok +from components.bot_detector.api_public.services import FeedbackService +from components.bot_detector.api_public.structs.feedback import FeedbackInput +from components.bot_detector.api_public.structs.responses import Ok from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session -from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import ( - to_jagex_name, -) +from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import to_jagex_name from fastapi import APIRouter, Depends, HTTPException, status router = APIRouter(tags=["Feedback"]) @@ -18,12 +16,9 @@ async def post_feedback( feedback: FeedbackInput, session=Depends(get_session), ): - """ """ - _feedback = Feedback(session) - + repo = FeedbackService(session) feedback.player_name = await to_jagex_name(feedback.player_name) - - success, detail = await _feedback.insert_feedback(feedback=feedback) + success, detail = await repo.insert_feedback(feedback=feedback) if not success: raise HTTPException(status_code=422, detail=detail) return Ok(detail=detail) diff --git a/bases/bot_detector/api_public/routes/v2/labels.py b/bases/bot_detector/api_public/routes/v2/labels.py new file mode 100644 index 0000000..930ec27 --- /dev/null +++ b/bases/bot_detector/api_public/routes/v2/labels.py @@ -0,0 +1,32 @@ +import logging + +from components.bot_detector.api_public.services import LabelService +from components.bot_detector.api_public.structs.labels import LabelResponse +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session +from fastapi import APIRouter, Depends, status + +router = APIRouter(tags=["Labels"]) +logger = logging.getLogger(__name__) + + +@router.get("/labels", response_model=list[LabelResponse], status_code=status.HTTP_200_OK) +async def get_labels(session=Depends(get_session)): + repo = LabelService(session) + labels = await repo.get_labels() + _labels = [] + for label in labels: + res = LabelResponse(**label.__dict__) + res.label = res.label.lower() + _labels.append(res) + return _labels + + +@router.get("/labels/{label_id}", response_model=LabelResponse | None) +async def get_label_by_id(label_id: int, session=Depends(get_session)) -> LabelResponse | None: + repo = LabelService(session) + label = await repo.get_label_by_id(label_id=label_id) + if label is None: + return None + res = LabelResponse(**label.__dict__) + res.label = res.label.lower() + return res diff --git a/bases/bot_detector/api_public/player/routes.py b/bases/bot_detector/api_public/routes/v2/player.py similarity index 60% rename from bases/bot_detector/api_public/player/routes.py rename to bases/bot_detector/api_public/routes/v2/player.py index 6d24dcb..14bd32c 100644 --- a/bases/bot_detector/api_public/player/routes.py +++ b/bases/bot_detector/api_public/routes/v2/player.py @@ -2,8 +2,8 @@ import logging from typing import Annotated -from bases.bot_detector.api_public.player.repository import Player as repoPlayer -from bases.bot_detector.api_public.player.schemas import ( +from components.bot_detector.api_public.services import PlayerService +from components.bot_detector.api_public.structs.player import ( FeedbackScoreResponse, PredictionResponse, ReportScoreResponse, @@ -29,19 +29,9 @@ async def get_players_kc( ), session=Depends(get_session), ): - """ - Get the report score for one or multiple players. - - Args: - name (str): can be provided multiple times - - Returns: - list[ReportScoreResponse]: A list of dictionaries containing KC data for each player. - """ - repo = repoPlayer(session) + repo = PlayerService(session) names = await asyncio.gather(*[to_jagex_name(n) for n in name]) - data = await repo.get_report_score(player_names=tuple(names)) - return data + return await repo.get_report_score(player_names=tuple(names)) @router.get("/player/feedback/score", response_model=list[FeedbackScoreResponse]) @@ -54,19 +44,9 @@ async def get_feedback_score( ), session=Depends(get_session), ): - """ - Get the feedback score for one or multiple players. - - Args: - name (str): can be provided multiple times - - Returns: - list[FeedbackScoreResponse]: A list of dictionaries containing KC data for each player. - """ - repo = repoPlayer(session) + repo = PlayerService(session) names = await asyncio.gather(*[to_jagex_name(n) for n in name]) - data = await repo.get_feedback_score(player_names=names) - return data + return await repo.get_feedback_score(player_names=names) @router.get("/player/prediction", response_model=list[PredictionResponse]) @@ -81,21 +61,7 @@ async def get_prediction( breakdown: bool = Query(...), session=Depends(get_session), ): - """ - Get prediction data for one or multiple users. - - Args: - name (str): The username of the user for whom predictions are requested. - breakdown (bool): A flag indicating whether to include a breakdown of predictions. - - Returns: - List[PredictionResponse]: A list of PredictionResponse objects containing prediction data. - - Raises: - HTTPException: Returns a 404 error with the message "Player not found" if no data is found for the user. - - """ - repo = repoPlayer(session) + repo = PlayerService(session) names = await asyncio.gather(*[to_jagex_name(n) for n in name]) data = await repo.get_prediction(player_names=names) if not data: diff --git a/bases/bot_detector/api_public/reports/routes.py b/bases/bot_detector/api_public/routes/v2/reports.py similarity index 65% rename from bases/bot_detector/api_public/reports/routes.py rename to bases/bot_detector/api_public/routes/v2/reports.py index 2aec5b1..c8c9ab1 100644 --- a/bases/bot_detector/api_public/reports/routes.py +++ b/bases/bot_detector/api_public/routes/v2/reports.py @@ -1,18 +1,20 @@ import logging -from bases.bot_detector.api_public.player.repository import Player -from bases.bot_detector.api_public.reports.repository import CustomError, Report -from bases.bot_detector.api_public.shared.responses import Ok -from bases.bot_detector.api_public.core._cache import SimpleALRUCache +from components.bot_detector.api_public.services import PlayerService, ReportsService +from components.bot_detector.api_public.services.reports import CustomError +from components.bot_detector.api_public.structs.reports import ( + Detection, + ParsedDetection, +) +from components.bot_detector.api_public.structs.responses import Ok +from bot_detector.cache.simple import SimpleALRUCache from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session -from bot_detector.structs import Detection, ParsedDetection from fastapi import APIRouter, Depends, status from fastapi.exceptions import HTTPException from sqlalchemy.ext.asyncio import AsyncSession -logger = logging.getLogger(__name__) router = APIRouter(tags=["Report"]) - +logger = logging.getLogger(__name__) player_cache = SimpleALRUCache(max_size=100_000) @@ -22,8 +24,8 @@ async def post_reports( session: AsyncSession = Depends(get_session), ): global player_cache - report_repo = Report() - player_repo = Player(session=session, cache=player_cache) + report_repo = ReportsService() + player_repo = PlayerService(session=session, cache=player_cache) data, error = await report_repo.parse_data(detections) if error: @@ -31,35 +33,27 @@ async def post_reports( logger.debug(f"Received: {len(data)}, Reporter: {data[0].reporter}") - # get unique list of names player_names = list(set([d.reported for d in data] + [d.reporter for d in data])) players = [await player_repo.get_or_insert(player_name=p) for p in player_names] players = {p.name: p.id for p in players if p} _data = [] for d in data: - _d = d.model_dump() - # get reported_id from name - reported = player_repo.sanitize_name(_d.pop("reported")) + payload = d.model_dump() + reported = player_repo.sanitize_name(payload.pop("reported")) reported_id = players.get(reported) - - # get reporter_id from name - reporter = player_repo.sanitize_name(_d.pop("reporter")) + reporter = player_repo.sanitize_name(payload.pop("reporter")) reporter_id = players.get(reporter) - - # some validation if reporter_id is None or reported_id is None: logger.warning(msg=f"{reported_id=}, {reporter_id=}, {d}") raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="something went wrong", ) - _d["reported_id"] = reported_id - _d["reporter_id"] = reporter_id - - _data.append(ParsedDetection(**_d)) + payload["reported_id"] = reported_id + payload["reporter_id"] = reporter_id + _data.append(ParsedDetection(**payload)) - # print(_data) try: await report_repo.send_to_kafka(data=_data) except CustomError: diff --git a/bases/bot_detector/api_public/shared/__init__.py b/bases/bot_detector/api_public/shared/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/components/bot_detector/api_public/__init__.py b/components/bot_detector/api_public/__init__.py new file mode 100644 index 0000000..4cd40a9 --- /dev/null +++ b/components/bot_detector/api_public/__init__.py @@ -0,0 +1,10 @@ +"""Reusable API Public components.""" + +from . import services, structs +from .structs import Ok + +__all__ = [ + "services", + "structs", + "Ok", +] diff --git a/components/bot_detector/api_public/database/__init__.py b/components/bot_detector/api_public/database/__init__.py new file mode 100644 index 0000000..dcb16b7 --- /dev/null +++ b/components/bot_detector/api_public/database/__init__.py @@ -0,0 +1,18 @@ +""" +API Public specific database models and helpers. +""" + +from .feedback import PredictionFeedback +from .label import Label +from .player import Player +from .prediction import Prediction_v1, Prediction_v2 +from .report import Report + +__all__ = [ + "Player", + "Prediction_v1", + "Prediction_v2", + "PredictionFeedback", + "Report", + "Label", +] diff --git a/components/bot_detector/api_public/database/feedback.py b/components/bot_detector/api_public/database/feedback.py new file mode 100644 index 0000000..8c06d63 --- /dev/null +++ b/components/bot_detector/api_public/database/feedback.py @@ -0,0 +1,28 @@ +from bot_detector.database import Base +from sqlalchemy import ( + TIMESTAMP, + Column, + Float, + ForeignKey, + Integer, + SmallInteger, + String, + Text, +) + + +class PredictionFeedback(Base): + __tablename__ = "PredictionsFeedback" + + id = Column(Integer, primary_key=True, autoincrement=True) + ts = Column(TIMESTAMP, nullable=False, server_default="CURRENT_TIMESTAMP") + voter_id = Column(Integer, ForeignKey("Players.id"), nullable=False) + subject_id = Column(Integer, ForeignKey("Players.id"), nullable=False) + prediction = Column(String(50), nullable=False) + confidence = Column(Float, nullable=False) + vote = Column(Integer, nullable=False, server_default="0") + feedback_text = Column(Text(collation="utf8mb4_0900_ai_ci")) + reviewed = Column(SmallInteger, nullable=False, server_default="0") + reviewer_id = Column(Integer) + user_notified = Column(SmallInteger, nullable=False, server_default="0") + proposed_label = Column(String(50)) diff --git a/components/bot_detector/api_public/database/label.py b/components/bot_detector/api_public/database/label.py new file mode 100644 index 0000000..fdb9229 --- /dev/null +++ b/components/bot_detector/api_public/database/label.py @@ -0,0 +1,9 @@ +from bot_detector.database import Base +from sqlalchemy import Column, Integer, Text + + +class Label(Base): + __tablename__ = "Labels" + + id = Column(Integer, primary_key=True, autoincrement=True) + label = Column(Text) diff --git a/components/bot_detector/api_public/database/player.py b/components/bot_detector/api_public/database/player.py new file mode 100644 index 0000000..d424f05 --- /dev/null +++ b/components/bot_detector/api_public/database/player.py @@ -0,0 +1,5 @@ +"""Re-export the shared Players table for API Public consumers.""" + +from bot_detector.database.player.structs import PlayersTableStruct as Player + +__all__ = ["Player"] diff --git a/components/bot_detector/api_public/database/prediction.py b/components/bot_detector/api_public/database/prediction.py new file mode 100644 index 0000000..09e583f --- /dev/null +++ b/components/bot_detector/api_public/database/prediction.py @@ -0,0 +1,51 @@ +from bot_detector.database import Base +from sqlalchemy import DECIMAL, JSON, TIMESTAMP, Column, Integer, String + + +class Prediction_v1(Base): + __tablename__ = "Predictions" + + id = Column(Integer, primary_key=True, autoincrement=True) + name = Column(String(12)) + prediction = Column(String(50)) + created = Column(TIMESTAMP) + predicted_confidence = Column(DECIMAL(5, 2)) + real_player = Column(DECIMAL(5, 2), default=0) + pvm_melee_bot = Column(DECIMAL(5, 2), default=0) + smithing_bot = Column(DECIMAL(5, 2), default=0) + magic_bot = Column(DECIMAL(5, 2), default=0) + fishing_bot = Column(DECIMAL(5, 2), default=0) + mining_bot = Column(DECIMAL(5, 2), default=0) + crafting_bot = Column(DECIMAL(5, 2), default=0) + pvm_ranged_magic_bot = Column(DECIMAL(5, 2), default=0) + pvm_ranged_bot = Column(DECIMAL(5, 2), default=0) + hunter_bot = Column(DECIMAL(5, 2), default=0) + fletching_bot = Column(DECIMAL(5, 2), default=0) + clue_scroll_bot = Column(DECIMAL(5, 2), default=0) + lms_bot = Column(DECIMAL(5, 2), default=0) + agility_bot = Column(DECIMAL(5, 2), default=0) + wintertodt_bot = Column(DECIMAL(5, 2), default=0) + runecrafting_bot = Column(DECIMAL(5, 2), default=0) + zalcano_bot = Column(DECIMAL(5, 2), default=0) + woodcutting_bot = Column(DECIMAL(5, 2), default=0) + thieving_bot = Column(DECIMAL(5, 2), default=0) + soul_wars_bot = Column(DECIMAL(5, 2), default=0) + cooking_bot = Column(DECIMAL(5, 2), default=0) + vorkath_bot = Column(DECIMAL(5, 2), default=0) + barrows_bot = Column(DECIMAL(5, 2), default=0) + herblore_bot = Column(DECIMAL(5, 2), default=0) + zulrah_bot = Column(DECIMAL(5, 2), default=0) + gauntlet_bot = Column(DECIMAL(5, 2), default=0) + nex_bot = Column(DECIMAL(5, 2), default=0) + unknown_bot = Column(DECIMAL(5, 2), default=0) + + +class Prediction_v2(Base): + __tablename__ = "prediction_latest" + + created_at = Column(TIMESTAMP) + player_id = Column(Integer, primary_key=True) + model_name = Column(String(50)) + prediction = Column(String(50)) + confidence = Column(DECIMAL(5, 2)) + predictions = Column(JSON) diff --git a/components/bot_detector/api_public/database/report.py b/components/bot_detector/api_public/database/report.py new file mode 100644 index 0000000..ffbd0bf --- /dev/null +++ b/components/bot_detector/api_public/database/report.py @@ -0,0 +1,30 @@ +from bot_detector.database import Base +from sqlalchemy import TIMESTAMP, BigInteger, Column, Integer, SmallInteger + + +class Report(Base): + __tablename__ = "Reports" + + ID = Column(BigInteger, primary_key=True, autoincrement=True) + created_at = Column(TIMESTAMP) + reportedID = Column(Integer) + reportingID = Column(Integer) + region_id = Column(Integer) + x_coord = Column(Integer) + y_coord = Column(Integer) + z_coord = Column(Integer) + timestamp = Column(TIMESTAMP) + manual_detect = Column(SmallInteger) + on_members_world = Column(Integer) + on_pvp_world = Column(SmallInteger) + world_number = Column(Integer) + equip_head_id = Column(Integer) + equip_amulet_id = Column(Integer) + equip_torso_id = Column(Integer) + equip_legs_id = Column(Integer) + equip_boots_id = Column(Integer) + equip_cape_id = Column(Integer) + equip_hands_id = Column(Integer) + equip_weapon_id = Column(Integer) + equip_shield_id = Column(Integer) + equip_ge_value = Column(BigInteger) diff --git a/components/bot_detector/api_public/services/__init__.py b/components/bot_detector/api_public/services/__init__.py new file mode 100644 index 0000000..25dd069 --- /dev/null +++ b/components/bot_detector/api_public/services/__init__.py @@ -0,0 +1,6 @@ +from .feedback import FeedbackService +from .labels import LabelService +from .player import PlayerService +from .reports import ReportsService + +__all__ = ["PlayerService", "FeedbackService", "LabelService", "ReportsService"] diff --git a/bases/bot_detector/api_public/feedback/repository.py b/components/bot_detector/api_public/services/feedback.py similarity index 95% rename from bases/bot_detector/api_public/feedback/repository.py rename to components/bot_detector/api_public/services/feedback.py index 9f9597d..d7bf319 100644 --- a/bases/bot_detector/api_public/feedback/repository.py +++ b/components/bot_detector/api_public/services/feedback.py @@ -1,6 +1,6 @@ import logging -from bases.bot_detector.api_public.feedback.schemas import FeedbackInput +from components.bot_detector.api_public.structs.feedback import FeedbackInput from bot_detector.database.api_public import ( Player as dbPlayer, PredictionFeedback as dbFeedback, @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) -class Feedback: +class FeedbackService: def __init__(self, session: AsyncSession) -> None: self.session = session diff --git a/bases/bot_detector/api_public/labels/repository.py b/components/bot_detector/api_public/services/labels.py similarity index 97% rename from bases/bot_detector/api_public/labels/repository.py rename to components/bot_detector/api_public/services/labels.py index 90b347d..5085fb0 100644 --- a/bases/bot_detector/api_public/labels/repository.py +++ b/components/bot_detector/api_public/services/labels.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) -class LabelRepository: +class LabelService: def __init__(self, session: AsyncSession) -> None: self.session = session diff --git a/bases/bot_detector/api_public/player/repository.py b/components/bot_detector/api_public/services/player.py similarity index 97% rename from bases/bot_detector/api_public/player/repository.py rename to components/bot_detector/api_public/services/player.py index 5f9bba7..511ea91 100644 --- a/bases/bot_detector/api_public/player/repository.py +++ b/components/bot_detector/api_public/services/player.py @@ -1,8 +1,8 @@ import logging import sqlalchemy as sqla -from bases.bot_detector.api_public.player.schemas import PlayerCreate, PlayerInDB -from bases.bot_detector.api_public.core._cache import SimpleALRUCache +from components.bot_detector.api_public.structs.player import PlayerCreate, PlayerInDB +from bot_detector.cache.simple import SimpleALRUCache from bot_detector.database.api_public import ( Player as dbPlayer, PredictionFeedback as dbFeedback, @@ -25,7 +25,7 @@ def model_to_dict(model): return {c.name: getattr(model, c.name) for c in model.__table__.columns} -class Player: +class PlayerService: def __init__( self, session: AsyncSession, diff --git a/bases/bot_detector/api_public/reports/repository.py b/components/bot_detector/api_public/services/reports.py similarity index 93% rename from bases/bot_detector/api_public/reports/repository.py rename to components/bot_detector/api_public/services/reports.py index 2662dc8..f5cefd7 100644 --- a/bases/bot_detector/api_public/reports/repository.py +++ b/components/bot_detector/api_public/services/reports.py @@ -2,16 +2,12 @@ import logging import time -from bases.bot_detector.api_public.core.fastapi.dependencies.kafka import kafka_manager +from bot_detector.kafka import kafka_manager from bot_detector.kafka.repositories.reports_to_insert import ( RepoReportsToInsertProducer, ) -from bot_detector.structs import ( - Detection, - MetaData, - ParsedDetection, - ReportsToInsertStruct, -) +from components.bot_detector.api_public.structs.reports import Detection, ParsedDetection +from bot_detector.structs import MetaData, ReportsToInsertStruct from pydantic import ValidationError logger = logging.getLogger(__name__) @@ -20,7 +16,7 @@ class CustomError(Exception): ... -class Report: +class ReportsService: def __init__(self) -> None: pass diff --git a/components/bot_detector/api_public/structs/__init__.py b/components/bot_detector/api_public/structs/__init__.py new file mode 100644 index 0000000..99a9ea7 --- /dev/null +++ b/components/bot_detector/api_public/structs/__init__.py @@ -0,0 +1,42 @@ +from .player import ( + PlayerCreate, + PlayerUpdate, + PlayerInDB, + Player, + PlayerResponse, + ReportScoreResponse, + FeedbackScoreResponse, + PredictionResponse, +) +from .feedback import FeedbackInput, FeedbackScore +from .labels import LabelResponse +from .reports import ( + Equipment, + BaseDetection, + Detection, + ParsedDetection, + KafkaDetectionV1, + KafkaDetectionV2, +) +from .responses import Ok + +__all__ = [ + "PlayerCreate", + "PlayerUpdate", + "PlayerInDB", + "Player", + "PlayerResponse", + "ReportScoreResponse", + "FeedbackScoreResponse", + "PredictionResponse", + "FeedbackInput", + "FeedbackScore", + "LabelResponse", + "Equipment", + "BaseDetection", + "Detection", + "ParsedDetection", + "KafkaDetectionV1", + "KafkaDetectionV2", + "Ok", +] diff --git a/bases/bot_detector/api_public/feedback/schemas.py b/components/bot_detector/api_public/structs/feedback.py similarity index 100% rename from bases/bot_detector/api_public/feedback/schemas.py rename to components/bot_detector/api_public/structs/feedback.py diff --git a/bases/bot_detector/api_public/labels/schemas.py b/components/bot_detector/api_public/structs/labels.py similarity index 100% rename from bases/bot_detector/api_public/labels/schemas.py rename to components/bot_detector/api_public/structs/labels.py diff --git a/bases/bot_detector/api_public/player/schemas.py b/components/bot_detector/api_public/structs/player.py similarity index 100% rename from bases/bot_detector/api_public/player/schemas.py rename to components/bot_detector/api_public/structs/player.py diff --git a/bases/bot_detector/api_public/reports/schemas.py b/components/bot_detector/api_public/structs/reports.py similarity index 100% rename from bases/bot_detector/api_public/reports/schemas.py rename to components/bot_detector/api_public/structs/reports.py diff --git a/bases/bot_detector/api_public/shared/responses.py b/components/bot_detector/api_public/structs/responses.py similarity index 93% rename from bases/bot_detector/api_public/shared/responses.py rename to components/bot_detector/api_public/structs/responses.py index 85be632..59896f5 100644 --- a/bases/bot_detector/api_public/shared/responses.py +++ b/components/bot_detector/api_public/structs/responses.py @@ -1,5 +1,5 @@ -from pydantic import BaseModel - - -class Ok(BaseModel): - detail: str = "ok" +from pydantic import BaseModel + + +class Ok(BaseModel): + detail: str = "ok" diff --git a/components/bot_detector/cache/__init__.py b/components/bot_detector/cache/__init__.py new file mode 100644 index 0000000..353db39 --- /dev/null +++ b/components/bot_detector/cache/__init__.py @@ -0,0 +1,3 @@ +from .simple import SimpleALRUCache + +__all__ = ["SimpleALRUCache"] diff --git a/components/bot_detector/cache/simple.py b/components/bot_detector/cache/simple.py new file mode 100644 index 0000000..4d13c79 --- /dev/null +++ b/components/bot_detector/cache/simple.py @@ -0,0 +1,63 @@ +import asyncio +import logging +from collections import OrderedDict + +logger = logging.getLogger(__name__) + + +class SimpleALRUCache: + def __init__(self, max_size=10_000): + self.cache = OrderedDict() + self.max_size = max_size + self.lock = asyncio.Lock() + self.hits: int = 0 + self.misses: int = 0 + + async def get(self, key): + async with self.lock: + if key in self.cache: + # Move the accessed key to the end to mark it as recently used + self.cache.move_to_end(key) + self.hits += 1 + return self.cache[key] + self.misses += 1 + return None + + async def put(self, key, value): + async with self.lock: + if key in self.cache: + # Update the value and mark it as recently used + self.cache.move_to_end(key) + self.cache[key] = value + else: + # If the cache is full, remove the first (least recently used) item + if len(self.cache) >= self.max_size: + self.cache.popitem(last=False) + self.cache[key] = value + + async def clear(self): + async with self.lock: + self.cache.clear() + + +# Example usage +async def main(): + cache = SimpleALRUCache(max_size=3) + + await cache.put("a", 1) + await cache.put("b", 2) + await cache.put("c", 3) + + print(await cache.get("a")) # Output: 1 + print(await cache.get("b")) # Output: 2 + print(await cache.get("c")) # Output: 3 + + await cache.put("d", 4) # This will evict 'a' because it's the LRU item + + print(await cache.get("a")) # Output: None, 'a' has been evicted + print(await cache.get("d")) # Output: 4 + + +if __name__ == "__main__": + # Run the example + asyncio.run(main()) diff --git a/components/bot_detector/kafka/__init__.py b/components/bot_detector/kafka/__init__.py index 825e652..f430919 100644 --- a/components/bot_detector/kafka/__init__.py +++ b/components/bot_detector/kafka/__init__.py @@ -4,11 +4,14 @@ DataToPredictProducer, DataToPredictStruct, ) +from .manager import KafkaManager, kafka_manager __all__ = [ "Settings", "ConsumerInterface", "ProducerInterface", + "KafkaManager", + "kafka_manager", "DataToPredictConsumer", "DataToPredictProducer", "DataToPredictStruct", diff --git a/components/bot_detector/kafka/manager.py b/components/bot_detector/kafka/manager.py new file mode 100644 index 0000000..1a48f27 --- /dev/null +++ b/components/bot_detector/kafka/manager.py @@ -0,0 +1,57 @@ +import logging + +from bot_detector.kafka.interface import ConsumerInterface, ProducerInterface + +logger = logging.getLogger(__name__) + + +class KafkaManager: + def __init__(self): + self.producers = {} + self.consumers = {} + logger.debug("KafkaManager initialized.") + + def set_producer(self, key: str, producer: ProducerInterface) -> None: + self.producers[key] = producer + logger.debug(f"Producer set for key: {key}") + + def get_producer(self, key: str | None) -> ProducerInterface | None: + logger.debug(f"Retrieving producer for key: {key}") + + if key is None: + return self.producers + + producer = self.producers.get(key) + + if not producer: + logger.debug(self.producers) + logger.warning(f"Producer not found for key: {key}") + return None + + logger.debug(f"Producer retrieved for key: {key}") + return producer + + def set_consumer(self, key: str, consumer: ConsumerInterface) -> None: + self.consumers[key] = consumer + logger.debug(f"Consumer set for key: {key}") + + def get_consumer(self, key: str | None) -> ConsumerInterface | None: + if key is None: + return self.consumers + + consumer = self.consumers.get(key) + + if not consumer: + logger.debug(self.consumers) + logger.warning(f"Consumer not found for key: {key}") + return None + + logger.debug(f"Consumer retrieved for key: {key}") + return consumer + + +# Create a global KafkaManager instance +kafka_manager = KafkaManager() + +# memory location +# print(id(kafka_manager)) diff --git a/projects/api_public/pyproject.toml b/projects/api_public/pyproject.toml index 90bc986..2abf132 100644 --- a/projects/api_public/pyproject.toml +++ b/projects/api_public/pyproject.toml @@ -31,3 +31,4 @@ packages = ["bot_detector"] "../../components/bot_detector/kafka" = "bot_detector/kafka" "../../components/bot_detector/structs" = "bot_detector/structs" "../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/cache" = "bot_detector/cache" diff --git a/test/bases/bot_detector/api_public/test_endpoints.py b/test/bases/bot_detector/api_public/test_endpoints.py index 7fa20b2..a0957d3 100644 --- a/test/bases/bot_detector/api_public/test_endpoints.py +++ b/test/bases/bot_detector/api_public/test_endpoints.py @@ -2,9 +2,12 @@ from bases.bot_detector.api_public.core import server from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session -from bases.bot_detector.api_public.player.repository import Player as PlayerRepo -from bases.bot_detector.api_public.feedback.repository import Feedback as FeedbackRepo -from bases.bot_detector.api_public.reports.repository import Report as ReportRepo, CustomError +from components.bot_detector.api_public.services import ( + FeedbackService, + PlayerService, + ReportsService, +) +from components.bot_detector.api_public.services.reports import CustomError async def _dummy_session(): @@ -21,7 +24,7 @@ def test_player_prediction_not_found(monkeypatch): async def fake_prediction(*args, **kwargs): return [] - monkeypatch.setattr(PlayerRepo, "get_prediction", fake_prediction) + monkeypatch.setattr(PlayerService, "get_prediction", fake_prediction) client = _client(monkeypatch) resp = client.get( "/v2/player/prediction", @@ -35,7 +38,7 @@ def test_feedback_duplicate_returns_422(monkeypatch): async def fake_insert(*args, **kwargs): return False, "duplicate_record" - monkeypatch.setattr(FeedbackRepo, "insert_feedback", fake_insert) + monkeypatch.setattr(FeedbackService, "insert_feedback", fake_insert) client = _client(monkeypatch) payload = { "player_name": "abc", @@ -53,7 +56,7 @@ def test_report_validation_error(monkeypatch): async def fake_parse(self, data): return None, "invalid data size" - monkeypatch.setattr(ReportRepo, "parse_data", fake_parse) + monkeypatch.setattr(ReportsService, "parse_data", fake_parse) client = _client(monkeypatch) resp = client.post("/v2/report", json=[]) assert resp.status_code == 400 @@ -106,10 +109,10 @@ async def fake_get_or_insert(self, player_name, **kwargs): async def fake_send(self, *args, **kwargs): raise CustomError("boom") - monkeypatch.setattr(ReportRepo, "parse_data", fake_parse) - monkeypatch.setattr(PlayerRepo, "get_or_insert", fake_get_or_insert) - monkeypatch.setattr(PlayerRepo, "sanitize_name", lambda self, n: n) - monkeypatch.setattr(ReportRepo, "send_to_kafka", fake_send) + monkeypatch.setattr(ReportsService, "parse_data", fake_parse) + monkeypatch.setattr(PlayerService, "get_or_insert", fake_get_or_insert) + monkeypatch.setattr(PlayerService, "sanitize_name", lambda self, n: n) + monkeypatch.setattr(ReportsService, "send_to_kafka", fake_send) client = _client(monkeypatch) resp = client.post( diff --git a/test/bases/bot_detector/api_public/test_schemas.py b/test/bases/bot_detector/api_public/test_schemas.py index 7e63f6c..9fc5a05 100644 --- a/test/bases/bot_detector/api_public/test_schemas.py +++ b/test/bases/bot_detector/api_public/test_schemas.py @@ -1,6 +1,6 @@ import pytest -from bases.bot_detector.api_public.feedback.schemas import FeedbackInput +from components.bot_detector.api_public.structs.feedback import FeedbackInput def _base_feedback_kwargs() -> dict: diff --git a/test/bases/bot_detector/api_public/test_utils.py b/test/bases/bot_detector/api_public/test_utils.py index b309497..98cdce0 100644 --- a/test/bases/bot_detector/api_public/test_utils.py +++ b/test/bases/bot_detector/api_public/test_utils.py @@ -3,8 +3,8 @@ import pytest from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import to_jagex_name -from bases.bot_detector.api_public.player.repository import Player -from bases.bot_detector.api_public.core._cache import SimpleALRUCache +from components.bot_detector.api_public.services import PlayerService +from bot_detector.cache.simple import SimpleALRUCache class _DummySession: @@ -19,5 +19,5 @@ async def test_to_jagex_name_normalizes_variants(): def test_player_sanitize_name_is_consistent(): - repo = Player(session=_DummySession(), cache=SimpleALRUCache()) + repo = PlayerService(session=_DummySession(), cache=SimpleALRUCache()) assert repo.sanitize_name("My_Name-Here ") == "my name here" diff --git a/test/components/bot_detector/api_public/test_reports_service.py b/test/components/bot_detector/api_public/test_reports_service.py new file mode 100644 index 0000000..d5a1d91 --- /dev/null +++ b/test/components/bot_detector/api_public/test_reports_service.py @@ -0,0 +1,60 @@ +import time + +import pytest + +from components.bot_detector.api_public.services import ReportsService +from components.bot_detector.api_public.structs.reports import Detection, Equipment + + +def _make_detection(ts: int | None = None, reporter: str = "tester", reported: str = "target") -> Detection: + equipment = Equipment( + equip_head_id=1, + equip_amulet_id=1, + equip_torso_id=1, + equip_legs_id=1, + equip_boots_id=1, + equip_cape_id=1, + equip_hands_id=1, + equip_weapon_id=1, + equip_shield_id=1, + ) + return Detection( + reporter=reporter, + reported=reported, + region_id=1, + x_coord=1, + y_coord=1, + z_coord=0, + ts=ts if ts is not None else int(time.time()), + manual_detect=0, + on_members_world=0, + on_pvp_world=0, + world_number=301, + equipment=equipment, + equip_ge_value=1, + ) + + +@pytest.mark.asyncio() +async def test_parse_data_rejects_large_payload(): + service = ReportsService() + detections = [_make_detection(reported=f"target-{i}") for i in range(5001)] + + data, error = await service.parse_data(detections) + + assert data is None + assert error == "invalid data size" + + +@pytest.mark.asyncio() +async def test_parse_data_detects_invalid_unique_reporter(): + service = ReportsService() + detections = [ + _make_detection(reporter="alpha", reported="x"), + _make_detection(reporter="beta", reported="y"), + ] + + data, error = await service.parse_data(detections) + + assert data is None + assert error == "invalid unique reporter" From 6ff9443a0e1e3d65f82e44489ca16f71886b60b5 Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Thu, 13 Nov 2025 22:22:29 -0500 Subject: [PATCH 08/11] update readme --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 88da561..f7ce860 100644 --- a/README.md +++ b/README.md @@ -35,12 +35,16 @@ - **Feature**: a cohesive capability (feedback reporting, player scraping, proxy rotation, etc.) that owns its domain rules, ~~DTOs~~ structs, ports, and integrations. Each feature lives inside a component so it can be reused by multiple bases/projects without duplication. - **Models** = SQLAlchemy ORM classes mapped to concrete tables (live under `components/bot_detector/database/**/models`). Only the persistence layer (repositories/adapters that talk to storage) should touch them. - **Structs** = Pydantic data shapes (requests/responses/contracts) shared across components/bases; they live under `components/bot_detector/structs` and replace the old “DTO” term. +- **Service**: the domain façade exposed by a component (e.g., `PlayerService`). A service orchestrates one cohesive use-case—validation, repositories, messaging, caching—and is what bases call once they finish plumbing work. +- **Repository**: a persistence/adapter layer focused solely on talking to infrastructure (SQLAlchemy, Kafka, S3, HTTP APIs, etc.). Repositories are consumed by services and do not contain orchestration or HTTP-specific logic. +- **Manager**: an infrastructure helper that owns shared resources (Kafka producers, sessions, caches). Managers live alongside plumbing or support libraries and keep long-lived connections healthy. +- **Cache**: lightweight, in-process memoization layers (like `SimpleALRUCache`) that services can use for hot-path data. Caches never contain business logic; they simply store/retrieve entities to reduce load on repositories. ## Working Standards -- **Components** encapsulate each feature’s business logic plus adapters, and may depend on other components/libraries only. -- **Bases** expose public APIs and handle plumbing only (routing, request parsing, dependency wiring) before delegating to components. +- **Components** encapsulate each feature’s business logic plus adapters, and may depend on other components/libraries only. Services may call repositories/adapters but never import FastAPI or base code. +- **Bases** expose public APIs and must stay thin: they handle routing, validation, dependency wiring, and immediately delegate to services. Bases never import ORM models or implement business rules. - **Projects** only compose bricks + libraries into deployable artifacts; they hold wiring/config, never feature code. -- **Shared structs** (DTOs, interfaces) belong in reusable components like `components/bot_detector/structs` so every base/project can import them without circular dependencies. +- **Shared structs** (DTOs, interfaces) and persistence code live in reusable components such as `components/bot_detector/structs` and `components/bot_detector/database`, so every base/project imports the same contracts and models without circular dependencies. Use the `structs` naming everywhere (no `schemas` leftovers) and suffix types consistently (`FooInput`, `FooResponse`, etc.). - **Tests** live under the workspace-level `test/` directory via `[tool.polylith.test]`, so base/component fixtures and contract tests should be added there rather than inside each brick folder. Add per-base `resources/` directories only when a base needs static assets or config that isn’t shared elsewhere. # The Polylith Architecture @@ -125,4 +129,4 @@ find . -type f -name "pyproject.toml" -not -path "*/.venv/*" -execdir sh -c 'ech # syncing in all directories, so uv cache is setup ```sh find . -type f -name "pyproject.toml" -not -path "*/.venv/*" -execdir sh -c 'echo "🔄 syncing in $(pwd)"; uv sync' \; -``` \ No newline at end of file +``` From 30973b9f10687d0660dd2ac7f1166c565baea26c Mon Sep 17 00:00:00 2001 From: RusticPotato Date: Fri, 14 Nov 2025 16:01:27 -0500 Subject: [PATCH 09/11] working refactor --- README.md | 6 +- .../core/fastapi/dependencies/session.py | 3 +- .../api_public/routes/v2/feedback.py | 6 +- .../api_public/routes/v2/labels.py | 4 +- .../api_public/routes/v2/player.py | 4 +- .../api_public/routes/v2/reports.py | 13 +- bases/bot_detector/hiscore_scraper/core.py | 9 +- .../bot_detector/job_hs_migration_v3/core.py | 12 +- bases/bot_detector/job_prune_hs_data/core.py | 3 +- .../bot_detector/runemetrics_scraper/core.py | 3 +- .../bot_detector/scrape_task_producer/core.py | 8 +- bases/bot_detector/worker_hiscore/core.py | 11 +- bases/bot_detector/worker_ml/core.py | 11 +- bases/bot_detector/worker_report/main.py | 10 +- .../bot_detector/api_public/__init__.py | 10 -- .../api_public/database/__init__.py | 18 --- .../bot_detector/api_public/database/label.py | 9 -- .../api_public/database/player.py | 5 - .../api_public/services/__init__.py | 6 - .../api_public/structs/__init__.py | 42 ------ components/bot_detector/cache/__init__.py | 3 - .../{cache/simple.py => core/cache.py} | 28 +--- .../{database/core.py => core/database.py} | 2 +- .../bot_detector/core/structs/__init__.py | 10 ++ .../{ => core}/structs/_metadata.py | 0 .../{ => core}/structs/kafka/__init__.py | 0 .../{ => core}/structs/kafka/not_found.py | 4 +- .../structs/kafka/reports_to_insert.py | 4 +- .../core/structs/kafka/scraped.py | 10 ++ .../{ => core}/structs/kafka/to_scrape.py | 4 +- .../{api_public => core}/structs/responses.py | 0 components/bot_detector/database/__init__.py | 3 - .../database/api_public/__init__.py | 18 --- .../database/api_public/feedback.py | 28 ---- .../database/api_public/player.py | 5 - .../database/api_public/prediction.py | 51 ------- .../database/api_public/report.py | 30 ---- .../bot_detector/database/hiscore/__init__.py | 35 ----- .../bot_detector/database/player/__init__.py | 5 - .../database/prediction/__init__.py | 12 -- .../bot_detector/database/report/__init__.py | 4 - .../feedback.py => feedback/database.py} | 2 +- .../feedback.py => feedback/services.py} | 8 +- .../feedback.py => feedback/structs.py} | 0 .../highscore_worker/database/__init__.py | 17 +++ .../database}/interface.py | 2 +- .../database}/repository.py | 2 +- .../database}/structs.py | 2 +- .../structs.py} | 10 ++ .../kafka/repositories/players_not_found.py | 2 +- .../kafka/repositories/players_scraped.py | 2 +- .../kafka/repositories/players_to_scrape.py | 2 +- .../kafka/repositories/reports_to_insert.py | 2 +- .../label.py => labels/database.py} | 2 +- .../services/labels.py => labels/services.py} | 2 +- .../structs/labels.py => labels/structs.py} | 0 components/bot_detector/player/__init__.py | 3 + .../bot_detector/player/database/__init__.py | 6 + .../player => player/database}/interface.py | 2 +- .../player => player/database}/repository.py | 2 +- .../player => player/database}/structs.py | 2 +- .../services/player.py => player/services.py} | 15 +- .../structs/player.py => player/structs.py} | 31 +++- .../database}/interface.py | 2 +- .../database/prediction.py | 2 +- .../database}/repository.py | 4 +- .../database}/structs.py | 2 +- .../prediction.py => prediction/structs.py} | 8 + .../report => report/database}/interface.py | 2 +- .../{api_public => report}/database/report.py | 2 +- .../report => report/database}/repository.py | 2 +- .../services/reports.py | 4 +- .../structs/reports.py => report/structs.py} | 109 ++++++++------ components/bot_detector/structs/__init__.py | 40 ----- .../bot_detector/structs/kafka/scraped.py | 10 -- components/bot_detector/structs/player.py | 20 --- components/bot_detector/structs/reports.py | 41 ------ projects/api_public/pyproject.toml | 10 +- projects/hiscore_scraper/pyproject.toml | 5 +- projects/job_hs_migration_v3/pyproject.toml | 6 +- projects/job_prune_hs_data/pyproject.toml | 3 +- projects/runemetrics_scraper/pyproject.toml | 4 +- projects/scrape_task_producer/pyproject.toml | 5 +- projects/worker_hiscore/pyproject.toml | 6 +- projects/worker_ml/pyproject.toml | 5 +- projects/worker_report/pyproject.toml | 5 +- .../bot_detector/api_public/test_endpoints.py | 139 +++++++++++++++++- .../bot_detector/api_public/test_schemas.py | 2 +- .../bot_detector/api_public/test_utils.py | 8 +- .../runemetrics_scraper/test_core.py | 2 +- .../scrape_task_producer/test_core.py | 2 +- .../api_public/test_reports_service.py | 95 +++++++++++- .../bot_detector/database/test_core.py | 5 - .../database/test_report_repository.py | 4 +- .../bot_detector/report/test_report_repo.py | 51 +++++++ 95 files changed, 557 insertions(+), 613 deletions(-) delete mode 100644 components/bot_detector/api_public/__init__.py delete mode 100644 components/bot_detector/api_public/database/__init__.py delete mode 100644 components/bot_detector/api_public/database/label.py delete mode 100644 components/bot_detector/api_public/database/player.py delete mode 100644 components/bot_detector/api_public/services/__init__.py delete mode 100644 components/bot_detector/api_public/structs/__init__.py delete mode 100644 components/bot_detector/cache/__init__.py rename components/bot_detector/{cache/simple.py => core/cache.py} (54%) rename components/bot_detector/{database/core.py => core/database.py} (94%) create mode 100644 components/bot_detector/core/structs/__init__.py rename components/bot_detector/{ => core}/structs/_metadata.py (100%) rename components/bot_detector/{ => core}/structs/kafka/__init__.py (100%) rename components/bot_detector/{ => core}/structs/kafka/not_found.py (53%) rename components/bot_detector/{ => core}/structs/kafka/reports_to_insert.py (53%) create mode 100644 components/bot_detector/core/structs/kafka/scraped.py rename components/bot_detector/{ => core}/structs/kafka/to_scrape.py (53%) rename components/bot_detector/{api_public => core}/structs/responses.py (100%) delete mode 100644 components/bot_detector/database/__init__.py delete mode 100644 components/bot_detector/database/api_public/__init__.py delete mode 100644 components/bot_detector/database/api_public/feedback.py delete mode 100644 components/bot_detector/database/api_public/player.py delete mode 100644 components/bot_detector/database/api_public/prediction.py delete mode 100644 components/bot_detector/database/api_public/report.py delete mode 100644 components/bot_detector/database/hiscore/__init__.py delete mode 100644 components/bot_detector/database/player/__init__.py delete mode 100644 components/bot_detector/database/prediction/__init__.py delete mode 100644 components/bot_detector/database/report/__init__.py rename components/bot_detector/{api_public/database/feedback.py => feedback/database.py} (95%) rename components/bot_detector/{api_public/services/feedback.py => feedback/services.py} (91%) rename components/bot_detector/{api_public/structs/feedback.py => feedback/structs.py} (100%) create mode 100644 components/bot_detector/highscore_worker/database/__init__.py rename components/bot_detector/{database/hiscore => highscore_worker/database}/interface.py (97%) rename components/bot_detector/{database/hiscore => highscore_worker/database}/repository.py (99%) rename components/bot_detector/{database/hiscore => highscore_worker/database}/structs.py (98%) rename components/bot_detector/{structs/hiscore.py => highscore_worker/structs.py} (76%) rename components/bot_detector/{database/api_public/label.py => labels/database.py} (80%) rename components/bot_detector/{api_public/services/labels.py => labels/services.py} (94%) rename components/bot_detector/{api_public/structs/labels.py => labels/structs.py} (100%) create mode 100644 components/bot_detector/player/__init__.py create mode 100644 components/bot_detector/player/database/__init__.py rename components/bot_detector/{database/player => player/database}/interface.py (94%) rename components/bot_detector/{database/player => player/database}/repository.py (99%) rename components/bot_detector/{database/player => player/database}/structs.py (95%) rename components/bot_detector/{api_public/services/player.py => player/services.py} (93%) rename components/bot_detector/{api_public/structs/player.py => player/structs.py} (78%) rename components/bot_detector/{database/prediction => prediction/database}/interface.py (95%) rename components/bot_detector/{api_public => prediction}/database/prediction.py (97%) rename components/bot_detector/{database/prediction => prediction/database}/repository.py (96%) rename components/bot_detector/{database/prediction => prediction/database}/structs.py (96%) rename components/bot_detector/{structs/prediction.py => prediction/structs.py} (85%) rename components/bot_detector/{database/report => report/database}/interface.py (92%) rename components/bot_detector/{api_public => report}/database/report.py (95%) rename components/bot_detector/{database/report => report/database}/repository.py (99%) rename components/bot_detector/{api_public => report}/services/reports.py (95%) rename components/bot_detector/{api_public/structs/reports.py => report/structs.py} (91%) delete mode 100644 components/bot_detector/structs/__init__.py delete mode 100644 components/bot_detector/structs/kafka/scraped.py delete mode 100644 components/bot_detector/structs/player.py delete mode 100644 components/bot_detector/structs/reports.py delete mode 100644 test/components/bot_detector/database/test_core.py create mode 100644 test/components/bot_detector/report/test_report_repo.py diff --git a/README.md b/README.md index f7ce860..e1503ef 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ - **Business logic**: the rules that determine how the domain behaves; validations, decisions, orchestration of use-cases, state transitions, etc. - **Plumbing**: transport/infrastructure glue (HTTP routing, request parsing, wiring dependencies) that carries inputs to the correct business logic and returns the result. - **Feature**: a cohesive capability (feedback reporting, player scraping, proxy rotation, etc.) that owns its domain rules, ~~DTOs~~ structs, ports, and integrations. Each feature lives inside a component so it can be reused by multiple bases/projects without duplication. -- **Models** = SQLAlchemy ORM classes mapped to concrete tables (live under `components/bot_detector/database/**/models`). Only the persistence layer (repositories/adapters that talk to storage) should touch them. -- **Structs** = Pydantic data shapes (requests/responses/contracts) shared across components/bases; they live under `components/bot_detector/structs` and replace the old “DTO” term. +- **Models** = SQLAlchemy ORM classes mapped to concrete tables (each feature owns its models under `components/bot_detector//database*.py`). Only the persistence layer (repositories/adapters that talk to storage) should touch them. +- **Structs** = Pydantic data shapes (requests/responses/contracts) shared across components/bases; each feature owns its structs (e.g., `components/bot_detector/player/structs.py`) and shared ones live under `bot_detector.core.structs`. They replace the old “DTO” term. - **Service**: the domain façade exposed by a component (e.g., `PlayerService`). A service orchestrates one cohesive use-case—validation, repositories, messaging, caching—and is what bases call once they finish plumbing work. - **Repository**: a persistence/adapter layer focused solely on talking to infrastructure (SQLAlchemy, Kafka, S3, HTTP APIs, etc.). Repositories are consumed by services and do not contain orchestration or HTTP-specific logic. - **Manager**: an infrastructure helper that owns shared resources (Kafka producers, sessions, caches). Managers live alongside plumbing or support libraries and keep long-lived connections healthy. @@ -44,7 +44,7 @@ - **Components** encapsulate each feature’s business logic plus adapters, and may depend on other components/libraries only. Services may call repositories/adapters but never import FastAPI or base code. - **Bases** expose public APIs and must stay thin: they handle routing, validation, dependency wiring, and immediately delegate to services. Bases never import ORM models or implement business rules. - **Projects** only compose bricks + libraries into deployable artifacts; they hold wiring/config, never feature code. -- **Shared structs** (DTOs, interfaces) and persistence code live in reusable components such as `components/bot_detector/structs` and `components/bot_detector/database`, so every base/project imports the same contracts and models without circular dependencies. Use the `structs` naming everywhere (no `schemas` leftovers) and suffix types consistently (`FooInput`, `FooResponse`, etc.). +- **Shared structs** (DTOs, interfaces) live in reusable modules such as `bot_detector.core.structs`, while feature-specific structs stay within their feature packages. Use the `structs` naming everywhere (no `schemas` leftovers) and suffix types consistently (`FooInput`, `FooResponse`, etc.). - **Tests** live under the workspace-level `test/` directory via `[tool.polylith.test]`, so base/component fixtures and contract tests should be added there rather than inside each brick folder. Add per-base `resources/` directories only when a base needs static assets or config that isn’t shared elsewhere. # The Polylith Architecture diff --git a/bases/bot_detector/api_public/core/fastapi/dependencies/session.py b/bases/bot_detector/api_public/core/fastapi/dependencies/session.py index e99502c..0772815 100644 --- a/bases/bot_detector/api_public/core/fastapi/dependencies/session.py +++ b/bases/bot_detector/api_public/core/fastapi/dependencies/session.py @@ -1,6 +1,5 @@ from bot_detector.api_public.core.config import DB_SEMAPHORE, settings -from bot_detector.database import Settings as DBSettings -from bot_detector.database import get_session_factory +from bot_detector.core.database import Settings as DBSettings, get_session_factory from sqlalchemy.ext.asyncio import AsyncSession # Reuse the shared database component instead of maintaining copy diff --git a/bases/bot_detector/api_public/routes/v2/feedback.py b/bases/bot_detector/api_public/routes/v2/feedback.py index b9076dc..4c625db 100644 --- a/bases/bot_detector/api_public/routes/v2/feedback.py +++ b/bases/bot_detector/api_public/routes/v2/feedback.py @@ -1,8 +1,8 @@ import logging -from components.bot_detector.api_public.services import FeedbackService -from components.bot_detector.api_public.structs.feedback import FeedbackInput -from components.bot_detector.api_public.structs.responses import Ok +from bot_detector.feedback.services import FeedbackService +from bot_detector.feedback.structs import FeedbackInput +from bot_detector.core.structs.responses import Ok from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import to_jagex_name from fastapi import APIRouter, Depends, HTTPException, status diff --git a/bases/bot_detector/api_public/routes/v2/labels.py b/bases/bot_detector/api_public/routes/v2/labels.py index 930ec27..7696d06 100644 --- a/bases/bot_detector/api_public/routes/v2/labels.py +++ b/bases/bot_detector/api_public/routes/v2/labels.py @@ -1,7 +1,7 @@ import logging -from components.bot_detector.api_public.services import LabelService -from components.bot_detector.api_public.structs.labels import LabelResponse +from bot_detector.labels.services import LabelService +from bot_detector.labels.structs import LabelResponse from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session from fastapi import APIRouter, Depends, status diff --git a/bases/bot_detector/api_public/routes/v2/player.py b/bases/bot_detector/api_public/routes/v2/player.py index 14bd32c..5d08fab 100644 --- a/bases/bot_detector/api_public/routes/v2/player.py +++ b/bases/bot_detector/api_public/routes/v2/player.py @@ -2,8 +2,8 @@ import logging from typing import Annotated -from components.bot_detector.api_public.services import PlayerService -from components.bot_detector.api_public.structs.player import ( +from bot_detector.player.services import PlayerService +from bot_detector.player.structs import ( FeedbackScoreResponse, PredictionResponse, ReportScoreResponse, diff --git a/bases/bot_detector/api_public/routes/v2/reports.py b/bases/bot_detector/api_public/routes/v2/reports.py index c8c9ab1..0683a23 100644 --- a/bases/bot_detector/api_public/routes/v2/reports.py +++ b/bases/bot_detector/api_public/routes/v2/reports.py @@ -1,18 +1,19 @@ import logging -from components.bot_detector.api_public.services import PlayerService, ReportsService -from components.bot_detector.api_public.services.reports import CustomError -from components.bot_detector.api_public.structs.reports import ( +from bot_detector.core.cache import SimpleALRUCache +from bot_detector.core.structs.responses import Ok +from bot_detector.player.services import PlayerService +from bot_detector.report.services.reports import CustomError, ReportsService +from bot_detector.report.structs import ( Detection, ParsedDetection, ) -from components.bot_detector.api_public.structs.responses import Ok -from bot_detector.cache.simple import SimpleALRUCache -from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session from fastapi import APIRouter, Depends, status from fastapi.exceptions import HTTPException from sqlalchemy.ext.asyncio import AsyncSession +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session + router = APIRouter(tags=["Report"]) logger = logging.getLogger(__name__) player_cache = SimpleALRUCache(max_size=100_000) diff --git a/bases/bot_detector/hiscore_scraper/core.py b/bases/bot_detector/hiscore_scraper/core.py index 97d335a..4d06673 100644 --- a/bases/bot_detector/hiscore_scraper/core.py +++ b/bases/bot_detector/hiscore_scraper/core.py @@ -16,12 +16,9 @@ ) from bot_detector.proxy_manager import ProxyManager from bot_detector.proxy_manager import Settings as ProxySettings -from bot_detector.structs import ( - HighscoreBaseStruct, - MetaData, - PlayerStruct, -) -from bot_detector.structs.kafka import NotFoundStruct, ScrapedStruct +from bot_detector.highscore_worker.structs import HighscoreBaseStruct +from bot_detector.core.structs import MetaData, NotFoundStruct, ScrapedStruct +from bot_detector.player.structs import PlayerStruct from osrs.asyncio import Hiscore, HSMode from osrs.asyncio.osrs.hiscores import PlayerStats from osrs.exceptions import PlayerDoesNotExist, UnexpectedRedirection diff --git a/bases/bot_detector/job_hs_migration_v3/core.py b/bases/bot_detector/job_hs_migration_v3/core.py index 5d71131..96c1fe1 100644 --- a/bases/bot_detector/job_hs_migration_v3/core.py +++ b/bases/bot_detector/job_hs_migration_v3/core.py @@ -5,18 +5,14 @@ from datetime import timedelta import sqlalchemy as sqla -from bot_detector.database import Settings as DBSettings -from bot_detector.database import get_session_factory +from bot_detector.core.database import Settings as DBSettings, get_session_factory from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayerScrapedProducer, ) -from bot_detector.structs import ( - HighscoreBaseStruct, - MetaData, - PlayerStruct, - ScrapedStruct, -) +from bot_detector.highscore_worker.structs import HighscoreBaseStruct +from bot_detector.core.structs import MetaData, ScrapedStruct +from bot_detector.player.structs import PlayerStruct from pydantic_settings import BaseSettings from sqlalchemy import TextClause from sqlalchemy.exc import OperationalError diff --git a/bases/bot_detector/job_prune_hs_data/core.py b/bases/bot_detector/job_prune_hs_data/core.py index 4d692aa..41befff 100644 --- a/bases/bot_detector/job_prune_hs_data/core.py +++ b/bases/bot_detector/job_prune_hs_data/core.py @@ -2,8 +2,7 @@ import logging import sqlalchemy as sqla -from bot_detector.database import Settings as DBSettings -from bot_detector.database import get_session_factory +from bot_detector.core.database import Settings as DBSettings, get_session_factory from pydantic_settings import BaseSettings from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker diff --git a/bases/bot_detector/runemetrics_scraper/core.py b/bases/bot_detector/runemetrics_scraper/core.py index dba5e1b..e69476d 100644 --- a/bases/bot_detector/runemetrics_scraper/core.py +++ b/bases/bot_detector/runemetrics_scraper/core.py @@ -15,7 +15,8 @@ from bot_detector.proxy_manager import Settings as ProxySettings from bot_detector.runemetrics_api import RuneMetrics, RuneMetricsResponse from bot_detector.runemetrics_api.exceptions import UnexpectedRedirection -from bot_detector.structs import MetaData, PlayerStruct, ScrapedStruct +from bot_detector.core.structs import MetaData, ScrapedStruct +from bot_detector.player.structs import PlayerStruct from osrs.utils import RateLimiter from prometheus_client import Counter, Histogram, start_http_server from pydantic import ValidationError diff --git a/bases/bot_detector/scrape_task_producer/core.py b/bases/bot_detector/scrape_task_producer/core.py index 74c01bb..35a6dd9 100644 --- a/bases/bot_detector/scrape_task_producer/core.py +++ b/bases/bot_detector/scrape_task_producer/core.py @@ -3,15 +3,15 @@ from dataclasses import asdict, dataclass from datetime import date, datetime, time, timedelta -from bot_detector.database import Settings as DBSettings -from bot_detector.database import get_session_factory -from bot_detector.database.player import PlayerRepo +from bot_detector.core.database import Settings as DBSettings, get_session_factory +from bot_detector.player.database.repository import PlayerRepo from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayersToScrapeConsumer, RepoPlayersToScrapeProducer, ) -from bot_detector.structs import MetaData, PlayerStruct, ToScrapeStruct +from bot_detector.core.structs import MetaData, ToScrapeStruct +from bot_detector.player.structs import PlayerStruct from pydantic_settings import BaseSettings from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from typing_extensions import Literal diff --git a/bases/bot_detector/worker_hiscore/core.py b/bases/bot_detector/worker_hiscore/core.py index 7a27f31..178764d 100644 --- a/bases/bot_detector/worker_hiscore/core.py +++ b/bases/bot_detector/worker_hiscore/core.py @@ -2,16 +2,15 @@ import logging import traceback -from bot_detector import database as db -from bot_detector.database import Settings as DBSettings -from bot_detector.database.hiscore import HighscoreDataRepo -from bot_detector.database.player import PlayerRepo +from bot_detector.core.database import Settings as DBSettings, get_session_factory +from bot_detector.highscore_worker.database.repository import HighscoreDataRepo +from bot_detector.player.database.repository import PlayerRepo from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayerScrapedConsumer, RepoPlayerScrapedProducer, ) -from bot_detector.structs import ScrapedStruct +from bot_detector.core.structs import ScrapedStruct from pydantic_settings import BaseSettings from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -108,7 +107,7 @@ async def consume_many_task( async def main(): - session_factory, async_engine = db.get_session_factory(SETTINGS=DBSettings()) + session_factory, async_engine = get_session_factory(SETTINGS=DBSettings()) player_repo = PlayerRepo() highscore_repo = HighscoreDataRepo() diff --git a/bases/bot_detector/worker_ml/core.py b/bases/bot_detector/worker_ml/core.py index 4b3cba7..9be4822 100644 --- a/bases/bot_detector/worker_ml/core.py +++ b/bases/bot_detector/worker_ml/core.py @@ -3,9 +3,11 @@ import traceback import aiohttp -from bot_detector.database import Settings as DBSettings -from bot_detector.database import get_session_factory -from bot_detector.database.prediction import PredictionLatestRepo, PredictionRepo +from bot_detector.core.database import Settings as DBSettings, get_session_factory +from bot_detector.prediction.database.repository import ( + PredictionLatestRepo, + PredictionRepo, +) from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayerScrapedConsumer, @@ -13,7 +15,8 @@ ) from bot_detector.ml_api.core import MLApiClient from bot_detector.ml_api.structs import InputData, Prediction -from bot_detector.structs import PredictionCreate, ScrapedStruct +from bot_detector.prediction.structs import PredictionCreate +from bot_detector.core.structs import ScrapedStruct from bot_detector.worker_ml.settings import Settings from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker diff --git a/bases/bot_detector/worker_report/main.py b/bases/bot_detector/worker_report/main.py index 5d66e1b..1d526da 100644 --- a/bases/bot_detector/worker_report/main.py +++ b/bases/bot_detector/worker_report/main.py @@ -3,15 +3,15 @@ import traceback from asyncio import Queue -from bot_detector import database as db -from bot_detector.database import Settings as DBSettings -from bot_detector.database.report import ReportRepo +from bot_detector.core.database import Settings as DBSettings, get_session_factory +from bot_detector.report.database.repository import ReportRepo from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoReportsToInsertConsumer, RepoReportsToInsertProducer, ) -from bot_detector.structs import ParsedDetection, ReportsToInsertStruct +from bot_detector.report.structs import ParsedDetection +from bot_detector.core.structs import ReportsToInsertStruct from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -120,7 +120,7 @@ async def error_task(error_queue: Queue, report_producer: RepoReportsToInsertPro async def main(): - session_factory, async_engine = db.get_session_factory(SETTINGS=DBSettings()) + session_factory, async_engine = get_session_factory(SETTINGS=DBSettings()) report_repo = ReportRepo() MAX_BATCH_SIZE = 10_000 # TODO: env variable? MAX_INTERVAL_MS = 1_000 # TODO: env variable? diff --git a/components/bot_detector/api_public/__init__.py b/components/bot_detector/api_public/__init__.py deleted file mode 100644 index 4cd40a9..0000000 --- a/components/bot_detector/api_public/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Reusable API Public components.""" - -from . import services, structs -from .structs import Ok - -__all__ = [ - "services", - "structs", - "Ok", -] diff --git a/components/bot_detector/api_public/database/__init__.py b/components/bot_detector/api_public/database/__init__.py deleted file mode 100644 index dcb16b7..0000000 --- a/components/bot_detector/api_public/database/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -API Public specific database models and helpers. -""" - -from .feedback import PredictionFeedback -from .label import Label -from .player import Player -from .prediction import Prediction_v1, Prediction_v2 -from .report import Report - -__all__ = [ - "Player", - "Prediction_v1", - "Prediction_v2", - "PredictionFeedback", - "Report", - "Label", -] diff --git a/components/bot_detector/api_public/database/label.py b/components/bot_detector/api_public/database/label.py deleted file mode 100644 index fdb9229..0000000 --- a/components/bot_detector/api_public/database/label.py +++ /dev/null @@ -1,9 +0,0 @@ -from bot_detector.database import Base -from sqlalchemy import Column, Integer, Text - - -class Label(Base): - __tablename__ = "Labels" - - id = Column(Integer, primary_key=True, autoincrement=True) - label = Column(Text) diff --git a/components/bot_detector/api_public/database/player.py b/components/bot_detector/api_public/database/player.py deleted file mode 100644 index d424f05..0000000 --- a/components/bot_detector/api_public/database/player.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Re-export the shared Players table for API Public consumers.""" - -from bot_detector.database.player.structs import PlayersTableStruct as Player - -__all__ = ["Player"] diff --git a/components/bot_detector/api_public/services/__init__.py b/components/bot_detector/api_public/services/__init__.py deleted file mode 100644 index 25dd069..0000000 --- a/components/bot_detector/api_public/services/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .feedback import FeedbackService -from .labels import LabelService -from .player import PlayerService -from .reports import ReportsService - -__all__ = ["PlayerService", "FeedbackService", "LabelService", "ReportsService"] diff --git a/components/bot_detector/api_public/structs/__init__.py b/components/bot_detector/api_public/structs/__init__.py deleted file mode 100644 index 99a9ea7..0000000 --- a/components/bot_detector/api_public/structs/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -from .player import ( - PlayerCreate, - PlayerUpdate, - PlayerInDB, - Player, - PlayerResponse, - ReportScoreResponse, - FeedbackScoreResponse, - PredictionResponse, -) -from .feedback import FeedbackInput, FeedbackScore -from .labels import LabelResponse -from .reports import ( - Equipment, - BaseDetection, - Detection, - ParsedDetection, - KafkaDetectionV1, - KafkaDetectionV2, -) -from .responses import Ok - -__all__ = [ - "PlayerCreate", - "PlayerUpdate", - "PlayerInDB", - "Player", - "PlayerResponse", - "ReportScoreResponse", - "FeedbackScoreResponse", - "PredictionResponse", - "FeedbackInput", - "FeedbackScore", - "LabelResponse", - "Equipment", - "BaseDetection", - "Detection", - "ParsedDetection", - "KafkaDetectionV1", - "KafkaDetectionV2", - "Ok", -] diff --git a/components/bot_detector/cache/__init__.py b/components/bot_detector/cache/__init__.py deleted file mode 100644 index 353db39..0000000 --- a/components/bot_detector/cache/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .simple import SimpleALRUCache - -__all__ = ["SimpleALRUCache"] diff --git a/components/bot_detector/cache/simple.py b/components/bot_detector/core/cache.py similarity index 54% rename from components/bot_detector/cache/simple.py rename to components/bot_detector/core/cache.py index 4d13c79..1ff5b68 100644 --- a/components/bot_detector/cache/simple.py +++ b/components/bot_detector/core/cache.py @@ -6,7 +6,7 @@ class SimpleALRUCache: - def __init__(self, max_size=10_000): + def __init__(self, max_size: int = 10_000): self.cache = OrderedDict() self.max_size = max_size self.lock = asyncio.Lock() @@ -16,7 +16,6 @@ def __init__(self, max_size=10_000): async def get(self, key): async with self.lock: if key in self.cache: - # Move the accessed key to the end to mark it as recently used self.cache.move_to_end(key) self.hits += 1 return self.cache[key] @@ -26,11 +25,9 @@ async def get(self, key): async def put(self, key, value): async with self.lock: if key in self.cache: - # Update the value and mark it as recently used self.cache.move_to_end(key) self.cache[key] = value else: - # If the cache is full, remove the first (least recently used) item if len(self.cache) >= self.max_size: self.cache.popitem(last=False) self.cache[key] = value @@ -38,26 +35,3 @@ async def put(self, key, value): async def clear(self): async with self.lock: self.cache.clear() - - -# Example usage -async def main(): - cache = SimpleALRUCache(max_size=3) - - await cache.put("a", 1) - await cache.put("b", 2) - await cache.put("c", 3) - - print(await cache.get("a")) # Output: 1 - print(await cache.get("b")) # Output: 2 - print(await cache.get("c")) # Output: 3 - - await cache.put("d", 4) # This will evict 'a' because it's the LRU item - - print(await cache.get("a")) # Output: None, 'a' has been evicted - print(await cache.get("d")) # Output: 4 - - -if __name__ == "__main__": - # Run the example - asyncio.run(main()) diff --git a/components/bot_detector/database/core.py b/components/bot_detector/core/database.py similarity index 94% rename from components/bot_detector/database/core.py rename to components/bot_detector/core/database.py index bb1ddb2..7c6712f 100644 --- a/components/bot_detector/database/core.py +++ b/components/bot_detector/core/database.py @@ -17,7 +17,7 @@ class Settings(BaseSettings): class Base(MappedAsDataclass, DeclarativeBase): - """subclasses will be converted to dataclasses""" + """Subclasses will be converted to dataclasses.""" def get_session_factory( diff --git a/components/bot_detector/core/structs/__init__.py b/components/bot_detector/core/structs/__init__.py new file mode 100644 index 0000000..b216e06 --- /dev/null +++ b/components/bot_detector/core/structs/__init__.py @@ -0,0 +1,10 @@ +from ._metadata import MetaData +from .kafka import NotFoundStruct, ReportsToInsertStruct, ScrapedStruct, ToScrapeStruct + +__all__ = [ + "MetaData", + "NotFoundStruct", + "ToScrapeStruct", + "ScrapedStruct", + "ReportsToInsertStruct", +] diff --git a/components/bot_detector/structs/_metadata.py b/components/bot_detector/core/structs/_metadata.py similarity index 100% rename from components/bot_detector/structs/_metadata.py rename to components/bot_detector/core/structs/_metadata.py diff --git a/components/bot_detector/structs/kafka/__init__.py b/components/bot_detector/core/structs/kafka/__init__.py similarity index 100% rename from components/bot_detector/structs/kafka/__init__.py rename to components/bot_detector/core/structs/kafka/__init__.py diff --git a/components/bot_detector/structs/kafka/not_found.py b/components/bot_detector/core/structs/kafka/not_found.py similarity index 53% rename from components/bot_detector/structs/kafka/not_found.py rename to components/bot_detector/core/structs/kafka/not_found.py index e8996ee..e551f91 100644 --- a/components/bot_detector/structs/kafka/not_found.py +++ b/components/bot_detector/core/structs/kafka/not_found.py @@ -1,5 +1,5 @@ -from bot_detector.structs._metadata import MetaData -from bot_detector.structs.player import PlayerStruct +from .._metadata import MetaData +from bot_detector.player.structs import PlayerStruct from pydantic import BaseModel diff --git a/components/bot_detector/structs/kafka/reports_to_insert.py b/components/bot_detector/core/structs/kafka/reports_to_insert.py similarity index 53% rename from components/bot_detector/structs/kafka/reports_to_insert.py rename to components/bot_detector/core/structs/kafka/reports_to_insert.py index 82898aa..85acb9a 100644 --- a/components/bot_detector/structs/kafka/reports_to_insert.py +++ b/components/bot_detector/core/structs/kafka/reports_to_insert.py @@ -1,5 +1,5 @@ -from bot_detector.structs._metadata import MetaData -from bot_detector.structs.reports import ParsedDetection +from .._metadata import MetaData +from bot_detector.report.structs import ParsedDetection from pydantic import BaseModel diff --git a/components/bot_detector/core/structs/kafka/scraped.py b/components/bot_detector/core/structs/kafka/scraped.py new file mode 100644 index 0000000..f129b1c --- /dev/null +++ b/components/bot_detector/core/structs/kafka/scraped.py @@ -0,0 +1,10 @@ +from .._metadata import MetaData +from bot_detector.highscore_worker.structs import HighscoreBaseStruct +from bot_detector.player.structs import PlayerStruct +from pydantic import BaseModel + + +class ScrapedStruct(BaseModel): + metadata: MetaData + player_data: PlayerStruct + highscore_data: HighscoreBaseStruct | None diff --git a/components/bot_detector/structs/kafka/to_scrape.py b/components/bot_detector/core/structs/kafka/to_scrape.py similarity index 53% rename from components/bot_detector/structs/kafka/to_scrape.py rename to components/bot_detector/core/structs/kafka/to_scrape.py index 4015fa1..2feac79 100644 --- a/components/bot_detector/structs/kafka/to_scrape.py +++ b/components/bot_detector/core/structs/kafka/to_scrape.py @@ -1,5 +1,5 @@ -from bot_detector.structs._metadata import MetaData -from bot_detector.structs.player import PlayerStruct +from .._metadata import MetaData +from bot_detector.player.structs import PlayerStruct from pydantic import BaseModel diff --git a/components/bot_detector/api_public/structs/responses.py b/components/bot_detector/core/structs/responses.py similarity index 100% rename from components/bot_detector/api_public/structs/responses.py rename to components/bot_detector/core/structs/responses.py diff --git a/components/bot_detector/database/__init__.py b/components/bot_detector/database/__init__.py deleted file mode 100644 index 17db9e8..0000000 --- a/components/bot_detector/database/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bot_detector.database.core import Base, Settings, get_session_factory - -__all__ = ["Base", "Settings", "get_session_factory"] diff --git a/components/bot_detector/database/api_public/__init__.py b/components/bot_detector/database/api_public/__init__.py deleted file mode 100644 index dcb16b7..0000000 --- a/components/bot_detector/database/api_public/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -API Public specific database models and helpers. -""" - -from .feedback import PredictionFeedback -from .label import Label -from .player import Player -from .prediction import Prediction_v1, Prediction_v2 -from .report import Report - -__all__ = [ - "Player", - "Prediction_v1", - "Prediction_v2", - "PredictionFeedback", - "Report", - "Label", -] diff --git a/components/bot_detector/database/api_public/feedback.py b/components/bot_detector/database/api_public/feedback.py deleted file mode 100644 index 8c06d63..0000000 --- a/components/bot_detector/database/api_public/feedback.py +++ /dev/null @@ -1,28 +0,0 @@ -from bot_detector.database import Base -from sqlalchemy import ( - TIMESTAMP, - Column, - Float, - ForeignKey, - Integer, - SmallInteger, - String, - Text, -) - - -class PredictionFeedback(Base): - __tablename__ = "PredictionsFeedback" - - id = Column(Integer, primary_key=True, autoincrement=True) - ts = Column(TIMESTAMP, nullable=False, server_default="CURRENT_TIMESTAMP") - voter_id = Column(Integer, ForeignKey("Players.id"), nullable=False) - subject_id = Column(Integer, ForeignKey("Players.id"), nullable=False) - prediction = Column(String(50), nullable=False) - confidence = Column(Float, nullable=False) - vote = Column(Integer, nullable=False, server_default="0") - feedback_text = Column(Text(collation="utf8mb4_0900_ai_ci")) - reviewed = Column(SmallInteger, nullable=False, server_default="0") - reviewer_id = Column(Integer) - user_notified = Column(SmallInteger, nullable=False, server_default="0") - proposed_label = Column(String(50)) diff --git a/components/bot_detector/database/api_public/player.py b/components/bot_detector/database/api_public/player.py deleted file mode 100644 index d424f05..0000000 --- a/components/bot_detector/database/api_public/player.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Re-export the shared Players table for API Public consumers.""" - -from bot_detector.database.player.structs import PlayersTableStruct as Player - -__all__ = ["Player"] diff --git a/components/bot_detector/database/api_public/prediction.py b/components/bot_detector/database/api_public/prediction.py deleted file mode 100644 index 09e583f..0000000 --- a/components/bot_detector/database/api_public/prediction.py +++ /dev/null @@ -1,51 +0,0 @@ -from bot_detector.database import Base -from sqlalchemy import DECIMAL, JSON, TIMESTAMP, Column, Integer, String - - -class Prediction_v1(Base): - __tablename__ = "Predictions" - - id = Column(Integer, primary_key=True, autoincrement=True) - name = Column(String(12)) - prediction = Column(String(50)) - created = Column(TIMESTAMP) - predicted_confidence = Column(DECIMAL(5, 2)) - real_player = Column(DECIMAL(5, 2), default=0) - pvm_melee_bot = Column(DECIMAL(5, 2), default=0) - smithing_bot = Column(DECIMAL(5, 2), default=0) - magic_bot = Column(DECIMAL(5, 2), default=0) - fishing_bot = Column(DECIMAL(5, 2), default=0) - mining_bot = Column(DECIMAL(5, 2), default=0) - crafting_bot = Column(DECIMAL(5, 2), default=0) - pvm_ranged_magic_bot = Column(DECIMAL(5, 2), default=0) - pvm_ranged_bot = Column(DECIMAL(5, 2), default=0) - hunter_bot = Column(DECIMAL(5, 2), default=0) - fletching_bot = Column(DECIMAL(5, 2), default=0) - clue_scroll_bot = Column(DECIMAL(5, 2), default=0) - lms_bot = Column(DECIMAL(5, 2), default=0) - agility_bot = Column(DECIMAL(5, 2), default=0) - wintertodt_bot = Column(DECIMAL(5, 2), default=0) - runecrafting_bot = Column(DECIMAL(5, 2), default=0) - zalcano_bot = Column(DECIMAL(5, 2), default=0) - woodcutting_bot = Column(DECIMAL(5, 2), default=0) - thieving_bot = Column(DECIMAL(5, 2), default=0) - soul_wars_bot = Column(DECIMAL(5, 2), default=0) - cooking_bot = Column(DECIMAL(5, 2), default=0) - vorkath_bot = Column(DECIMAL(5, 2), default=0) - barrows_bot = Column(DECIMAL(5, 2), default=0) - herblore_bot = Column(DECIMAL(5, 2), default=0) - zulrah_bot = Column(DECIMAL(5, 2), default=0) - gauntlet_bot = Column(DECIMAL(5, 2), default=0) - nex_bot = Column(DECIMAL(5, 2), default=0) - unknown_bot = Column(DECIMAL(5, 2), default=0) - - -class Prediction_v2(Base): - __tablename__ = "prediction_latest" - - created_at = Column(TIMESTAMP) - player_id = Column(Integer, primary_key=True) - model_name = Column(String(50)) - prediction = Column(String(50)) - confidence = Column(DECIMAL(5, 2)) - predictions = Column(JSON) diff --git a/components/bot_detector/database/api_public/report.py b/components/bot_detector/database/api_public/report.py deleted file mode 100644 index ffbd0bf..0000000 --- a/components/bot_detector/database/api_public/report.py +++ /dev/null @@ -1,30 +0,0 @@ -from bot_detector.database import Base -from sqlalchemy import TIMESTAMP, BigInteger, Column, Integer, SmallInteger - - -class Report(Base): - __tablename__ = "Reports" - - ID = Column(BigInteger, primary_key=True, autoincrement=True) - created_at = Column(TIMESTAMP) - reportedID = Column(Integer) - reportingID = Column(Integer) - region_id = Column(Integer) - x_coord = Column(Integer) - y_coord = Column(Integer) - z_coord = Column(Integer) - timestamp = Column(TIMESTAMP) - manual_detect = Column(SmallInteger) - on_members_world = Column(Integer) - on_pvp_world = Column(SmallInteger) - world_number = Column(Integer) - equip_head_id = Column(Integer) - equip_amulet_id = Column(Integer) - equip_torso_id = Column(Integer) - equip_legs_id = Column(Integer) - equip_boots_id = Column(Integer) - equip_cape_id = Column(Integer) - equip_hands_id = Column(Integer) - equip_weapon_id = Column(Integer) - equip_shield_id = Column(Integer) - equip_ge_value = Column(BigInteger) diff --git a/components/bot_detector/database/hiscore/__init__.py b/components/bot_detector/database/hiscore/__init__.py deleted file mode 100644 index 2f7510f..0000000 --- a/components/bot_detector/database/hiscore/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -from .interface import ( - HighscoreDataDailyInterface, - HighscoreDataLatestInterface, - HighscoreDataMonthlyInterface, - HighscoreDataWeeklyInterface, -) -from .repository import ( - HighscoreDataDailyRepo, - HighscoreDataLatestRepo, - HighscoreDataMonthlyRepo, - HighscoreDataRepo, - HighscoreDataWeeklyRepo, -) -from .structs import ( - HighscoreDataDailyTableStruct, - HighscoreDataLatestTableStruct, - HighscoreDataMonthlyTableStruct, - HighscoreDataWeeklyTableStruct, -) - -__all__ = [ - "HighscoreDataDailyInterface", - "HighscoreDataWeeklyInterface", - "HighscoreDataMonthlyInterface", - "HighscoreDataLatestInterface", - "HighscoreDataDailyRepo", - "HighscoreDataWeeklyRepo", - "HighscoreDataMonthlyRepo", - "HighscoreDataRepo", - "HighscoreDataLatestRepo", - "HighscoreDataDailyTableStruct", - "HighscoreDataWeeklyTableStruct", - "HighscoreDataMonthlyTableStruct", - "HighscoreDataLatestTableStruct", -] diff --git a/components/bot_detector/database/player/__init__.py b/components/bot_detector/database/player/__init__.py deleted file mode 100644 index 3bbcf41..0000000 --- a/components/bot_detector/database/player/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .interface import playerInterface -from .repository import PlayerRepo -from .structs import PlayersTableStruct - -__all__ = ["playerInterface", "PlayerRepo", "PlayersTableStruct"] diff --git a/components/bot_detector/database/prediction/__init__.py b/components/bot_detector/database/prediction/__init__.py deleted file mode 100644 index e68f32e..0000000 --- a/components/bot_detector/database/prediction/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from .interface import PredictionInterface, PredictionLatestInterface -from .repository import PredictionLatestRepo, PredictionRepo -from .structs import PredictionLatestStruct, PredictionStruct - -__all__ = [ - "PredictionInterface", - "PredictionLatestInterface", - "PredictionRepo", - "PredictionLatestRepo", - "PredictionStruct", - "PredictionLatestStruct", -] diff --git a/components/bot_detector/database/report/__init__.py b/components/bot_detector/database/report/__init__.py deleted file mode 100644 index 15cda4d..0000000 --- a/components/bot_detector/database/report/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .interface import ReportInterface -from .repository import ReportRepo - -__all__ = ["ReportInterface", "ReportRepo"] diff --git a/components/bot_detector/api_public/database/feedback.py b/components/bot_detector/feedback/database.py similarity index 95% rename from components/bot_detector/api_public/database/feedback.py rename to components/bot_detector/feedback/database.py index 8c06d63..3003576 100644 --- a/components/bot_detector/api_public/database/feedback.py +++ b/components/bot_detector/feedback/database.py @@ -1,4 +1,4 @@ -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import ( TIMESTAMP, Column, diff --git a/components/bot_detector/api_public/services/feedback.py b/components/bot_detector/feedback/services.py similarity index 91% rename from components/bot_detector/api_public/services/feedback.py rename to components/bot_detector/feedback/services.py index d7bf319..54bebd5 100644 --- a/components/bot_detector/api_public/services/feedback.py +++ b/components/bot_detector/feedback/services.py @@ -1,10 +1,8 @@ import logging -from components.bot_detector.api_public.structs.feedback import FeedbackInput -from bot_detector.database.api_public import ( - Player as dbPlayer, - PredictionFeedback as dbFeedback, -) +from bot_detector.feedback.structs import FeedbackInput +from bot_detector.player.database.structs import PlayersTableStruct as dbPlayer +from bot_detector.feedback.database import PredictionFeedback as dbFeedback from sqlalchemy import and_, insert, select from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession from sqlalchemy.sql.expression import Insert, Select diff --git a/components/bot_detector/api_public/structs/feedback.py b/components/bot_detector/feedback/structs.py similarity index 100% rename from components/bot_detector/api_public/structs/feedback.py rename to components/bot_detector/feedback/structs.py diff --git a/components/bot_detector/highscore_worker/database/__init__.py b/components/bot_detector/highscore_worker/database/__init__.py new file mode 100644 index 0000000..57afd1f --- /dev/null +++ b/components/bot_detector/highscore_worker/database/__init__.py @@ -0,0 +1,17 @@ +"""Highscore worker persistence helpers.""" + +from .repository import HighscoreDataRepo +from .structs import ( + HighscoreDataDailyTableStruct, + HighscoreDataLatestTableStruct, + HighscoreDataMonthlyTableStruct, + HighscoreDataWeeklyTableStruct, +) + +__all__ = [ + "HighscoreDataRepo", + "HighscoreDataDailyTableStruct", + "HighscoreDataWeeklyTableStruct", + "HighscoreDataMonthlyTableStruct", + "HighscoreDataLatestTableStruct", +] diff --git a/components/bot_detector/database/hiscore/interface.py b/components/bot_detector/highscore_worker/database/interface.py similarity index 97% rename from components/bot_detector/database/hiscore/interface.py rename to components/bot_detector/highscore_worker/database/interface.py index 619e1d2..70e908d 100644 --- a/components/bot_detector/database/hiscore/interface.py +++ b/components/bot_detector/highscore_worker/database/interface.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from bot_detector.structs import ( +from bot_detector.highscore_worker.structs import ( HighscoreDataDailyStruct, HighscoreDataLatestStruct, HighscoreDataMonthlyStruct, diff --git a/components/bot_detector/database/hiscore/repository.py b/components/bot_detector/highscore_worker/database/repository.py similarity index 99% rename from components/bot_detector/database/hiscore/repository.py rename to components/bot_detector/highscore_worker/database/repository.py index 628e256..74f2f5a 100644 --- a/components/bot_detector/database/hiscore/repository.py +++ b/components/bot_detector/highscore_worker/database/repository.py @@ -15,7 +15,7 @@ HighscoreDataMonthlyTableStruct, HighscoreDataWeeklyTableStruct, ) -from bot_detector.structs import HighscoreBaseStruct +from bot_detector.highscore_worker.structs import HighscoreBaseStruct from sqlalchemy import TextClause, func from sqlalchemy.ext.asyncio import AsyncSession diff --git a/components/bot_detector/database/hiscore/structs.py b/components/bot_detector/highscore_worker/database/structs.py similarity index 98% rename from components/bot_detector/database/hiscore/structs.py rename to components/bot_detector/highscore_worker/database/structs.py index 85432b7..7bb764f 100644 --- a/components/bot_detector/database/hiscore/structs.py +++ b/components/bot_detector/highscore_worker/database/structs.py @@ -1,7 +1,7 @@ from datetime import date from typing import Optional -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import ( JSON, Computed, diff --git a/components/bot_detector/structs/hiscore.py b/components/bot_detector/highscore_worker/structs.py similarity index 76% rename from components/bot_detector/structs/hiscore.py rename to components/bot_detector/highscore_worker/structs.py index b6ffd3c..9d5d453 100644 --- a/components/bot_detector/structs/hiscore.py +++ b/components/bot_detector/highscore_worker/structs.py @@ -32,3 +32,13 @@ class HighscoreDataWeeklyStruct(HighscoreDataBaseStruct): class HighscoreDataMonthlyStruct(HighscoreDataBaseStruct): pass + + +__all__ = [ + "HighscoreBaseStruct", + "HighscoreDataBaseStruct", + "HighscoreDataDailyStruct", + "HighscoreDataWeeklyStruct", + "HighscoreDataMonthlyStruct", + "HighscoreDataLatestStruct", +] diff --git a/components/bot_detector/kafka/repositories/players_not_found.py b/components/bot_detector/kafka/repositories/players_not_found.py index 4e77fc2..3c94908 100644 --- a/components/bot_detector/kafka/repositories/players_not_found.py +++ b/components/bot_detector/kafka/repositories/players_not_found.py @@ -6,7 +6,7 @@ ConsumerInterface, ProducerInterface, ) -from bot_detector.structs import NotFoundStruct +from bot_detector.core.structs import NotFoundStruct logger = logging.getLogger(__name__) diff --git a/components/bot_detector/kafka/repositories/players_scraped.py b/components/bot_detector/kafka/repositories/players_scraped.py index eb68ce2..6844f9a 100644 --- a/components/bot_detector/kafka/repositories/players_scraped.py +++ b/components/bot_detector/kafka/repositories/players_scraped.py @@ -7,7 +7,7 @@ ConsumerInterface, ProducerInterface, ) -from bot_detector.structs import ScrapedStruct +from bot_detector.core.structs import ScrapedStruct logger = logging.getLogger(__name__) diff --git a/components/bot_detector/kafka/repositories/players_to_scrape.py b/components/bot_detector/kafka/repositories/players_to_scrape.py index bf4df72..8f331cc 100644 --- a/components/bot_detector/kafka/repositories/players_to_scrape.py +++ b/components/bot_detector/kafka/repositories/players_to_scrape.py @@ -6,7 +6,7 @@ ConsumerInterface, ProducerInterface, ) -from bot_detector.structs import ToScrapeStruct +from bot_detector.core.structs import ToScrapeStruct logger = logging.getLogger(__name__) diff --git a/components/bot_detector/kafka/repositories/reports_to_insert.py b/components/bot_detector/kafka/repositories/reports_to_insert.py index f138225..f45621e 100644 --- a/components/bot_detector/kafka/repositories/reports_to_insert.py +++ b/components/bot_detector/kafka/repositories/reports_to_insert.py @@ -7,7 +7,7 @@ ConsumerInterface, ProducerInterface, ) -from bot_detector.structs import ReportsToInsertStruct +from bot_detector.core.structs import ReportsToInsertStruct logger = logging.getLogger(__name__) diff --git a/components/bot_detector/database/api_public/label.py b/components/bot_detector/labels/database.py similarity index 80% rename from components/bot_detector/database/api_public/label.py rename to components/bot_detector/labels/database.py index fdb9229..f008d2d 100644 --- a/components/bot_detector/database/api_public/label.py +++ b/components/bot_detector/labels/database.py @@ -1,4 +1,4 @@ -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import Column, Integer, Text diff --git a/components/bot_detector/api_public/services/labels.py b/components/bot_detector/labels/services.py similarity index 94% rename from components/bot_detector/api_public/services/labels.py rename to components/bot_detector/labels/services.py index 5085fb0..d8abe5d 100644 --- a/components/bot_detector/api_public/services/labels.py +++ b/components/bot_detector/labels/services.py @@ -1,6 +1,6 @@ import logging -from bot_detector.database.api_public import Label as dbLabel +from bot_detector.labels.database import Label as dbLabel from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession from sqlalchemy.sql.expression import Select diff --git a/components/bot_detector/api_public/structs/labels.py b/components/bot_detector/labels/structs.py similarity index 100% rename from components/bot_detector/api_public/structs/labels.py rename to components/bot_detector/labels/structs.py diff --git a/components/bot_detector/player/__init__.py b/components/bot_detector/player/__init__.py new file mode 100644 index 0000000..1bc6f6e --- /dev/null +++ b/components/bot_detector/player/__init__.py @@ -0,0 +1,3 @@ +"""Player feature package.""" + +__all__ = ["database", "services", "structs"] diff --git a/components/bot_detector/player/database/__init__.py b/components/bot_detector/player/database/__init__.py new file mode 100644 index 0000000..c81d000 --- /dev/null +++ b/components/bot_detector/player/database/__init__.py @@ -0,0 +1,6 @@ +"""Player database helpers.""" + +from .repository import PlayerRepo +from .structs import PlayersTableStruct + +__all__ = ["PlayerRepo", "PlayersTableStruct"] diff --git a/components/bot_detector/database/player/interface.py b/components/bot_detector/player/database/interface.py similarity index 94% rename from components/bot_detector/database/player/interface.py rename to components/bot_detector/player/database/interface.py index 4bc99c1..44edac4 100644 --- a/components/bot_detector/database/player/interface.py +++ b/components/bot_detector/player/database/interface.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from bot_detector.structs import PlayerStruct +from bot_detector.player.structs import PlayerStruct from sqlalchemy.ext.asyncio import AsyncSession diff --git a/components/bot_detector/database/player/repository.py b/components/bot_detector/player/database/repository.py similarity index 99% rename from components/bot_detector/database/player/repository.py rename to components/bot_detector/player/database/repository.py index f301dca..4f5d974 100644 --- a/components/bot_detector/database/player/repository.py +++ b/components/bot_detector/player/database/repository.py @@ -5,7 +5,7 @@ import sqlalchemy as sqla from .interface import playerInterface from .structs import PlayersTableStruct -from bot_detector.structs import PlayerStruct +from bot_detector.player.structs import PlayerStruct from sqlalchemy import TextClause from sqlalchemy.ext.asyncio import AsyncSession diff --git a/components/bot_detector/database/player/structs.py b/components/bot_detector/player/database/structs.py similarity index 95% rename from components/bot_detector/database/player/structs.py rename to components/bot_detector/player/database/structs.py index 98c50c6..3bd4c76 100644 --- a/components/bot_detector/database/player/structs.py +++ b/components/bot_detector/player/database/structs.py @@ -1,6 +1,6 @@ from datetime import datetime -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import Boolean, DateTime, Integer, Text from sqlalchemy.orm import Mapped, mapped_column diff --git a/components/bot_detector/api_public/services/player.py b/components/bot_detector/player/services.py similarity index 93% rename from components/bot_detector/api_public/services/player.py rename to components/bot_detector/player/services.py index 511ea91..d7bd0c0 100644 --- a/components/bot_detector/api_public/services/player.py +++ b/components/bot_detector/player/services.py @@ -1,13 +1,13 @@ import logging import sqlalchemy as sqla -from components.bot_detector.api_public.structs.player import PlayerCreate, PlayerInDB -from bot_detector.cache.simple import SimpleALRUCache -from bot_detector.database.api_public import ( - Player as dbPlayer, - PredictionFeedback as dbFeedback, +from bot_detector.core.cache import SimpleALRUCache +from bot_detector.player.database.structs import PlayersTableStruct as dbPlayer +from bot_detector.prediction.database.prediction import ( Prediction_v2 as dbPrediction, ) +from bot_detector.feedback.database import PredictionFeedback as dbFeedback +from bot_detector.player.structs import PlayerCreate, PlayerInDB from fastapi.encoders import jsonable_encoder from pydantic import ValidationError from sqlalchemy import func, select @@ -15,8 +15,6 @@ from sqlalchemy.orm import aliased from sqlalchemy.sql.expression import Select -# from bot_detector.database.api_public import Report as dbReport - logger = logging.getLogger(__name__) @@ -159,3 +157,6 @@ async def get_or_insert(self, player_name: str, cached=True) -> PlayerInDB: player = await self.insert(PlayerCreate(name=player_name)) return player + + +__all__ = ["PlayerService"] diff --git a/components/bot_detector/api_public/structs/player.py b/components/bot_detector/player/structs.py similarity index 78% rename from components/bot_detector/api_public/structs/player.py rename to components/bot_detector/player/structs.py index 796093e..8beef8c 100644 --- a/components/bot_detector/api_public/structs/player.py +++ b/components/bot_detector/player/structs.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Optional -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, Field, field_validator class PlayerCreate(BaseModel): @@ -99,3 +99,32 @@ def from_data(cls, data: dict, breakdown: bool): "predictions_breakdown": prediction_data if breakdown else {}, } return cls(**player_data) + + +class PlayerStruct(BaseModel): + id: int + name: str + created_at: datetime + updated_at: Optional[datetime] = None + possible_ban: bool = False + confirmed_ban: bool = False + confirmed_player: bool = False + label_id: int = 0 + label_jagex: int = 0 + ironman: Optional[bool] = None + hardcore_ironman: Optional[bool] = None + ultimate_ironman: Optional[bool] = None + normalized_name: Optional[str] = None + + +__all__ = [ + "PlayerCreate", + "PlayerUpdate", + "PlayerInDB", + "Player", + "PlayerResponse", + "ReportScoreResponse", + "FeedbackScoreResponse", + "PredictionResponse", + "PlayerStruct", +] diff --git a/components/bot_detector/database/prediction/interface.py b/components/bot_detector/prediction/database/interface.py similarity index 95% rename from components/bot_detector/database/prediction/interface.py rename to components/bot_detector/prediction/database/interface.py index a80eda7..a1bddfb 100644 --- a/components/bot_detector/database/prediction/interface.py +++ b/components/bot_detector/prediction/database/interface.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from bot_detector.structs import ( +from bot_detector.prediction.structs import ( PredictionCreate, PredictionLatestRead, PredictionRead, diff --git a/components/bot_detector/api_public/database/prediction.py b/components/bot_detector/prediction/database/prediction.py similarity index 97% rename from components/bot_detector/api_public/database/prediction.py rename to components/bot_detector/prediction/database/prediction.py index 09e583f..bef96ca 100644 --- a/components/bot_detector/api_public/database/prediction.py +++ b/components/bot_detector/prediction/database/prediction.py @@ -1,4 +1,4 @@ -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import DECIMAL, JSON, TIMESTAMP, Column, Integer, String diff --git a/components/bot_detector/database/prediction/repository.py b/components/bot_detector/prediction/database/repository.py similarity index 96% rename from components/bot_detector/database/prediction/repository.py rename to components/bot_detector/prediction/database/repository.py index 4b5a1ad..2624720 100644 --- a/components/bot_detector/database/prediction/repository.py +++ b/components/bot_detector/prediction/database/repository.py @@ -2,10 +2,10 @@ from dataclasses import asdict import sqlalchemy as sqla -from bot_detector.database.player.structs import PlayersTableStruct +from bot_detector.player.database.structs import PlayersTableStruct from .interface import PredictionInterface, PredictionLatestInterface from .structs import PredictionLatestStruct, PredictionStruct -from bot_detector.structs import ( +from bot_detector.prediction.structs import ( PredictionCreate, PredictionLatestRead, PredictionRead, diff --git a/components/bot_detector/database/prediction/structs.py b/components/bot_detector/prediction/database/structs.py similarity index 96% rename from components/bot_detector/database/prediction/structs.py rename to components/bot_detector/prediction/database/structs.py index a2e62d9..b69ebeb 100644 --- a/components/bot_detector/database/prediction/structs.py +++ b/components/bot_detector/prediction/database/structs.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Any, Optional -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import DECIMAL, JSON, DateTime, ForeignKey, Integer, String from sqlalchemy.orm import Mapped, mapped_column diff --git a/components/bot_detector/structs/prediction.py b/components/bot_detector/prediction/structs.py similarity index 85% rename from components/bot_detector/structs/prediction.py rename to components/bot_detector/prediction/structs.py index a598ecf..502bfe5 100644 --- a/components/bot_detector/structs/prediction.py +++ b/components/bot_detector/prediction/structs.py @@ -30,3 +30,11 @@ class PredictionLatestRead(PredictionBase): class Config: from_attributes = True + + +__all__ = [ + "PredictionBase", + "PredictionCreate", + "PredictionRead", + "PredictionLatestRead", +] diff --git a/components/bot_detector/database/report/interface.py b/components/bot_detector/report/database/interface.py similarity index 92% rename from components/bot_detector/database/report/interface.py rename to components/bot_detector/report/database/interface.py index 644eac4..5f102ac 100644 --- a/components/bot_detector/database/report/interface.py +++ b/components/bot_detector/report/database/interface.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from bot_detector.structs import ParsedDetection +from bot_detector.report.structs import ParsedDetection from sqlalchemy.ext.asyncio import AsyncSession diff --git a/components/bot_detector/api_public/database/report.py b/components/bot_detector/report/database/report.py similarity index 95% rename from components/bot_detector/api_public/database/report.py rename to components/bot_detector/report/database/report.py index ffbd0bf..b1644b7 100644 --- a/components/bot_detector/api_public/database/report.py +++ b/components/bot_detector/report/database/report.py @@ -1,4 +1,4 @@ -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import TIMESTAMP, BigInteger, Column, Integer, SmallInteger diff --git a/components/bot_detector/database/report/repository.py b/components/bot_detector/report/database/repository.py similarity index 99% rename from components/bot_detector/database/report/repository.py rename to components/bot_detector/report/database/repository.py index 3d9a3c7..d1360e1 100644 --- a/components/bot_detector/database/report/repository.py +++ b/components/bot_detector/report/database/repository.py @@ -3,7 +3,7 @@ import sqlalchemy as sqla from .interface import ReportInterface -from bot_detector.structs import ParsedDetection +from bot_detector.report.structs import ParsedDetection from sqlalchemy import TextClause from sqlalchemy.ext.asyncio import AsyncSession diff --git a/components/bot_detector/api_public/services/reports.py b/components/bot_detector/report/services/reports.py similarity index 95% rename from components/bot_detector/api_public/services/reports.py rename to components/bot_detector/report/services/reports.py index f5cefd7..1af8905 100644 --- a/components/bot_detector/api_public/services/reports.py +++ b/components/bot_detector/report/services/reports.py @@ -6,8 +6,8 @@ from bot_detector.kafka.repositories.reports_to_insert import ( RepoReportsToInsertProducer, ) -from components.bot_detector.api_public.structs.reports import Detection, ParsedDetection -from bot_detector.structs import MetaData, ReportsToInsertStruct +from bot_detector.report.structs import Detection, ParsedDetection +from bot_detector.core.structs import MetaData, ReportsToInsertStruct from pydantic import ValidationError logger = logging.getLogger(__name__) diff --git a/components/bot_detector/api_public/structs/reports.py b/components/bot_detector/report/structs.py similarity index 91% rename from components/bot_detector/api_public/structs/reports.py rename to components/bot_detector/report/structs.py index 59a9a54..6d78af9 100644 --- a/components/bot_detector/api_public/structs/reports.py +++ b/components/bot_detector/report/structs.py @@ -1,4 +1,4 @@ -import time +import time from typing import Optional from pydantic import BaseModel, Field @@ -6,51 +6,62 @@ class Metadata(BaseModel): version: str - - -class Equipment(BaseModel): - equip_head_id: Optional[int] = Field(None, ge=0) - equip_amulet_id: Optional[int] = Field(None, ge=0) - equip_torso_id: Optional[int] = Field(None, ge=0) - equip_legs_id: Optional[int] = Field(None, ge=0) - equip_boots_id: Optional[int] = Field(None, ge=0) - equip_cape_id: Optional[int] = Field(None, ge=0) - equip_hands_id: Optional[int] = Field(None, ge=0) - equip_weapon_id: Optional[int] = Field(None, ge=0) - equip_shield_id: Optional[int] = Field(None, ge=0) - - -class BaseDetection(BaseModel): - region_id: int = Field(0, ge=0, le=100_000) - x_coord: int = Field(0, ge=0) - y_coord: int = Field(0, ge=0) - z_coord: int = Field(0, ge=0) - ts: int = Field(int(time.time()), ge=0) - manual_detect: int = Field(0, ge=0, le=1) - on_members_world: int = Field(0, ge=0, le=1) - on_pvp_world: int = Field(0, ge=0, le=1) - world_number: int = Field(0, ge=300, le=1_000) - equipment: Equipment - equip_ge_value: int = Field(0, ge=0) - - -class Detection(BaseDetection): - reporter: str = Field(..., min_length=1, max_length=13) - reported: str = Field(..., min_length=1, max_length=12) - - -class ParsedDetection(BaseDetection): - reporter_id: int = Field(..., ge=0) - reported_id: int = Field(..., ge=0) - - -class KafkaDetectionV1(BaseDetection): - metadata: Metadata = Metadata(version="v1.0.0") - reporter: str = Field(..., min_length=1, max_length=13) - reported: str = Field(..., min_length=1, max_length=12) - - -class KafkaDetectionV2(BaseDetection): - metadata: Metadata = Metadata(version="v2.0.0") - reporter_id: int = Field(..., ge=0) - reported_id: int = Field(..., ge=0) + + +class Equipment(BaseModel): + equip_head_id: Optional[int] = Field(None, ge=0) + equip_amulet_id: Optional[int] = Field(None, ge=0) + equip_torso_id: Optional[int] = Field(None, ge=0) + equip_legs_id: Optional[int] = Field(None, ge=0) + equip_boots_id: Optional[int] = Field(None, ge=0) + equip_cape_id: Optional[int] = Field(None, ge=0) + equip_hands_id: Optional[int] = Field(None, ge=0) + equip_weapon_id: Optional[int] = Field(None, ge=0) + equip_shield_id: Optional[int] = Field(None, ge=0) + + +class BaseDetection(BaseModel): + region_id: int = Field(0, ge=0, le=100_000) + x_coord: int = Field(0, ge=0) + y_coord: int = Field(0, ge=0) + z_coord: int = Field(0, ge=0) + ts: int = Field(int(time.time()), ge=0) + manual_detect: int = Field(0, ge=0, le=1) + on_members_world: int = Field(0, ge=0, le=1) + on_pvp_world: int = Field(0, ge=0, le=1) + world_number: int = Field(0, ge=300, le=1_000) + equipment: Equipment + equip_ge_value: int = Field(0, ge=0) + + +class Detection(BaseDetection): + reporter: str = Field(..., min_length=1, max_length=13) + reported: str = Field(..., min_length=1, max_length=12) + + +class ParsedDetection(BaseDetection): + reporter_id: int = Field(..., ge=0) + reported_id: int = Field(..., ge=0) + + +class KafkaDetectionV1(BaseDetection): + metadata: Metadata = Metadata(version="v1.0.0") + reporter: str = Field(..., min_length=1, max_length=13) + reported: str = Field(..., min_length=1, max_length=12) + + +class KafkaDetectionV2(BaseDetection): + metadata: Metadata = Metadata(version="v2.0.0") + reporter_id: int = Field(..., ge=0) + reported_id: int = Field(..., ge=0) + + +__all__ = [ + "Metadata", + "Equipment", + "BaseDetection", + "Detection", + "ParsedDetection", + "KafkaDetectionV1", + "KafkaDetectionV2", +] diff --git a/components/bot_detector/structs/__init__.py b/components/bot_detector/structs/__init__.py deleted file mode 100644 index ae410de..0000000 --- a/components/bot_detector/structs/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -from ._metadata import MetaData -from .hiscore import ( - HighscoreBaseStruct, - HighscoreDataBaseStruct, - HighscoreDataDailyStruct, - HighscoreDataLatestStruct, - HighscoreDataMonthlyStruct, - HighscoreDataWeeklyStruct, -) -from .kafka import NotFoundStruct, ReportsToInsertStruct, ScrapedStruct, ToScrapeStruct -from .player import PlayerStruct -from .prediction import ( - PredictionBase, - PredictionCreate, - PredictionLatestRead, - PredictionRead, -) -from .reports import Detection, Equipment, ParsedDetection - -__all__ = [ - "MetaData", - "HighscoreBaseStruct", - "HighscoreDataLatestStruct", - "HighscoreDataBaseStruct", - "HighscoreDataDailyStruct", - "HighscoreDataWeeklyStruct", - "HighscoreDataMonthlyStruct", - "NotFoundStruct", - "PlayerStruct", - "ToScrapeStruct", - "ScrapedStruct", - "ReportsToInsertStruct", - "Detection", - "ParsedDetection", - "Equipment", - "PredictionLatestRead", - "PredictionBase", - "PredictionCreate", - "PredictionRead", -] diff --git a/components/bot_detector/structs/kafka/scraped.py b/components/bot_detector/structs/kafka/scraped.py deleted file mode 100644 index ac113e8..0000000 --- a/components/bot_detector/structs/kafka/scraped.py +++ /dev/null @@ -1,10 +0,0 @@ -from bot_detector.structs._metadata import MetaData -from bot_detector.structs.hiscore import HighscoreBaseStruct -from bot_detector.structs.player import PlayerStruct -from pydantic import BaseModel - - -class ScrapedStruct(BaseModel): - metadata: MetaData - player_data: PlayerStruct - highscore_data: HighscoreBaseStruct | None diff --git a/components/bot_detector/structs/player.py b/components/bot_detector/structs/player.py deleted file mode 100644 index b1d0ac2..0000000 --- a/components/bot_detector/structs/player.py +++ /dev/null @@ -1,20 +0,0 @@ -from datetime import datetime -from typing import Optional - -from pydantic import BaseModel - - -class PlayerStruct(BaseModel): - id: int - name: str - created_at: datetime - updated_at: Optional[datetime] = None - possible_ban: bool = False - confirmed_ban: bool = False - confirmed_player: bool = False - label_id: int = 0 - label_jagex: int = 0 - ironman: Optional[bool] = None - hardcore_ironman: Optional[bool] = None - ultimate_ironman: Optional[bool] = None - normalized_name: Optional[str] = None diff --git a/components/bot_detector/structs/reports.py b/components/bot_detector/structs/reports.py deleted file mode 100644 index 90a1d53..0000000 --- a/components/bot_detector/structs/reports.py +++ /dev/null @@ -1,41 +0,0 @@ -import time -from typing import Optional - -from pydantic import BaseModel -from pydantic.fields import Field - - -class Equipment(BaseModel): - equip_head_id: Optional[int] = Field(None, ge=0) - equip_amulet_id: Optional[int] = Field(None, ge=0) - equip_torso_id: Optional[int] = Field(None, ge=0) - equip_legs_id: Optional[int] = Field(None, ge=0) - equip_boots_id: Optional[int] = Field(None, ge=0) - equip_cape_id: Optional[int] = Field(None, ge=0) - equip_hands_id: Optional[int] = Field(None, ge=0) - equip_weapon_id: Optional[int] = Field(None, ge=0) - equip_shield_id: Optional[int] = Field(None, ge=0) - - -class BaseDetection(BaseModel): - region_id: int = Field(0, ge=0, le=100_000) - x_coord: int = Field(0, ge=0) - y_coord: int = Field(0, ge=0) - z_coord: int = Field(0, ge=0) - ts: int = Field(int(time.time()), ge=0) - manual_detect: int = Field(0, ge=0, le=1) - on_members_world: int = Field(0, ge=0, le=1) - on_pvp_world: int = Field(0, ge=0, le=1) - world_number: int = Field(0, ge=300, le=1_000) - equipment: Equipment - equip_ge_value: int = Field(0, ge=0) - - -class Detection(BaseDetection): - reporter: str = Field(..., min_length=1, max_length=13) - reported: str = Field(..., min_length=1, max_length=12) - - -class ParsedDetection(BaseDetection): - reporter_id: int = Field(..., ge=0) - reported_id: int = Field(..., ge=0) diff --git a/projects/api_public/pyproject.toml b/projects/api_public/pyproject.toml index 2abf132..633dffc 100644 --- a/projects/api_public/pyproject.toml +++ b/projects/api_public/pyproject.toml @@ -27,8 +27,12 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/api_public" = "bot_detector/api_public" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" "../../components/bot_detector/logfmt" = "bot_detector/logfmt" -"../../components/bot_detector/cache" = "bot_detector/cache" +"../../components/bot_detector/player" = "bot_detector/player" +"../../components/bot_detector/feedback" = "bot_detector/feedback" +"../../components/bot_detector/labels" = "bot_detector/labels" +"../../components/bot_detector/report" = "bot_detector/report" +"../../components/bot_detector/prediction" = "bot_detector/prediction" +"../../components/bot_detector/highscore_worker" = "bot_detector/highscore_worker" diff --git a/projects/hiscore_scraper/pyproject.toml b/projects/hiscore_scraper/pyproject.toml index 7582ce7..4d9771c 100644 --- a/projects/hiscore_scraper/pyproject.toml +++ b/projects/hiscore_scraper/pyproject.toml @@ -29,5 +29,6 @@ packages = ["bot_detector"] "../../bases/bot_detector/hiscore_scraper" = "bot_detector/hiscore_scraper" "../../components/bot_detector/proxy_manager" = "bot_detector/proxy_manager" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/core" = "bot_detector/core" +"../../components/bot_detector/player" = "bot_detector/player" +"../../components/bot_detector/highscore_worker" = "bot_detector/highscore_worker" diff --git a/projects/job_hs_migration_v3/pyproject.toml b/projects/job_hs_migration_v3/pyproject.toml index 38989a4..d283f0a 100644 --- a/projects/job_hs_migration_v3/pyproject.toml +++ b/projects/job_hs_migration_v3/pyproject.toml @@ -28,7 +28,7 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/job_hs_migration_v3" = "bot_detector/job_hs_migration_v3" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" \ No newline at end of file +"../../components/bot_detector/player" = "bot_detector/player" +"../../components/bot_detector/highscore_worker" = "bot_detector/highscore_worker" diff --git a/projects/job_prune_hs_data/pyproject.toml b/projects/job_prune_hs_data/pyproject.toml index 8dfdaca..42353e0 100644 --- a/projects/job_prune_hs_data/pyproject.toml +++ b/projects/job_prune_hs_data/pyproject.toml @@ -26,5 +26,4 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/job_prune_hs_data" = "bot_detector/job_prune_hs_data" -"../../components/bot_detector/database" = "bot_detector/database" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" \ No newline at end of file +"../../components/bot_detector/core" = "bot_detector/core" diff --git a/projects/runemetrics_scraper/pyproject.toml b/projects/runemetrics_scraper/pyproject.toml index 13bb0ce..1a86a75 100644 --- a/projects/runemetrics_scraper/pyproject.toml +++ b/projects/runemetrics_scraper/pyproject.toml @@ -30,5 +30,5 @@ packages = ["bot_detector"] "../../components/bot_detector/runemetrics_api" = "bot_detector/runemetrics_api" "../../components/bot_detector/proxy_manager" = "bot_detector/proxy_manager" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/core" = "bot_detector/core" +"../../components/bot_detector/player" = "bot_detector/player" diff --git a/projects/scrape_task_producer/pyproject.toml b/projects/scrape_task_producer/pyproject.toml index 1385e6e..36013a3 100644 --- a/projects/scrape_task_producer/pyproject.toml +++ b/projects/scrape_task_producer/pyproject.toml @@ -28,7 +28,6 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/scrape_task_producer" = "bot_detector/scrape_task_producer" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/player" = "bot_detector/player" diff --git a/projects/worker_hiscore/pyproject.toml b/projects/worker_hiscore/pyproject.toml index eb2f991..3b6fb62 100644 --- a/projects/worker_hiscore/pyproject.toml +++ b/projects/worker_hiscore/pyproject.toml @@ -28,7 +28,7 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/worker_hiscore" = "bot_detector/worker_hiscore" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/player" = "bot_detector/player" +"../../components/bot_detector/highscore_worker" = "bot_detector/highscore_worker" diff --git a/projects/worker_ml/pyproject.toml b/projects/worker_ml/pyproject.toml index de05fd9..ef351df 100644 --- a/projects/worker_ml/pyproject.toml +++ b/projects/worker_ml/pyproject.toml @@ -30,7 +30,6 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/worker_ml" = "bot_detector/worker_ml" "../../components/bot_detector/ml_api" = "bot_detector/ml_api" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/prediction" = "bot_detector/prediction" diff --git a/projects/worker_report/pyproject.toml b/projects/worker_report/pyproject.toml index 2d842fe..5d94116 100644 --- a/projects/worker_report/pyproject.toml +++ b/projects/worker_report/pyproject.toml @@ -28,7 +28,6 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/worker_report" = "bot_detector/worker_report" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/report" = "bot_detector/report" diff --git a/test/bases/bot_detector/api_public/test_endpoints.py b/test/bases/bot_detector/api_public/test_endpoints.py index a0957d3..2faea97 100644 --- a/test/bases/bot_detector/api_public/test_endpoints.py +++ b/test/bases/bot_detector/api_public/test_endpoints.py @@ -2,12 +2,9 @@ from bases.bot_detector.api_public.core import server from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session -from components.bot_detector.api_public.services import ( - FeedbackService, - PlayerService, - ReportsService, -) -from components.bot_detector.api_public.services.reports import CustomError +from bot_detector.player.services import PlayerService +from bot_detector.feedback.services import FeedbackService +from bot_detector.report.services.reports import CustomError, ReportsService async def _dummy_session(): @@ -147,3 +144,133 @@ async def fake_send(self, *args, **kwargs): ) assert resp.status_code == 500 assert resp.json()["detail"] == "Internal error" + + +def test_player_prediction_success(monkeypatch): + async def fake_prediction(*args, **kwargs): + return [ + { + "player_id": 1, + "name": "abc", + "created_at": "2024-01-01T00:00:00", + "model_name": "m", + "prediction": "bot", + "confidence": 0.9, + "predictions": {"bot": 0.9}, + } + ] + + monkeypatch.setattr(PlayerService, "get_prediction", fake_prediction) + client = _client(monkeypatch) + resp = client.get( + "/v2/player/prediction", + params={"name": ["abc"], "breakdown": "true"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body[0]["player_name"] == "abc" + assert body[0]["predictions_breakdown"] == {"bot": 0.9} + + +def test_feedback_success(monkeypatch): + async def fake_insert(*args, **kwargs): + return True, "success" + + monkeypatch.setattr(FeedbackService, "insert_feedback", fake_insert) + client = _client(monkeypatch) + payload = { + "player_name": "abc", + "vote": 1, + "prediction": "bot", + "confidence": 0.5, + "subject_id": 1, + } + resp = client.post("/v2/feedback", json=payload) + assert resp.status_code == 201 + assert resp.json()["detail"] == "success" + + +def test_report_success(monkeypatch): + async def fake_parse(self, data): + class _Detection: + reporter = "a" + reported = "b" + + def model_dump(self): + return { + "reporter": "a", + "reported": "b", + "region_id": 1, + "x_coord": 1, + "y_coord": 1, + "z_coord": 0, + "ts": 1, + "manual_detect": 0, + "on_members_world": 0, + "on_pvp_world": 0, + "world_number": 301, + "equipment": { + "equip_head_id": 1, + "equip_amulet_id": 1, + "equip_torso_id": 1, + "equip_legs_id": 1, + "equip_boots_id": 1, + "equip_cape_id": 1, + "equip_hands_id": 1, + "equip_weapon_id": 1, + "equip_shield_id": 1, + }, + "equip_ge_value": 1, + } + + return [_Detection()], None + + class _Player: + def __init__(self, name, id_): + self.name = name + self.id = id_ + + async def fake_get_or_insert(self, player_name, **kwargs): + return _Player(player_name, 1 if player_name == "a" else 2) + + async def fake_send(self, *args, **kwargs): + return None + + monkeypatch.setattr(ReportsService, "parse_data", fake_parse) + monkeypatch.setattr(PlayerService, "get_or_insert", fake_get_or_insert) + monkeypatch.setattr(PlayerService, "sanitize_name", lambda self, n: n) + monkeypatch.setattr(ReportsService, "send_to_kafka", fake_send) + + client = _client(monkeypatch) + resp = client.post( + "/v2/report", + json=[ + { + "reporter": "a", + "reported": "b", + "region_id": 1, + "x_coord": 1, + "y_coord": 1, + "z_coord": 0, + "ts": 1, + "manual_detect": 0, + "on_members_world": 0, + "on_pvp_world": 0, + "world_number": 301, + "equipment": { + "equip_head_id": 1, + "equip_amulet_id": 1, + "equip_torso_id": 1, + "equip_legs_id": 1, + "equip_boots_id": 1, + "equip_cape_id": 1, + "equip_hands_id": 1, + "equip_weapon_id": 1, + "equip_shield_id": 1, + }, + "equip_ge_value": 1, + } + ], + ) + assert resp.status_code == 201 + assert resp.json()["detail"] == "ok" diff --git a/test/bases/bot_detector/api_public/test_schemas.py b/test/bases/bot_detector/api_public/test_schemas.py index 9fc5a05..1ec248a 100644 --- a/test/bases/bot_detector/api_public/test_schemas.py +++ b/test/bases/bot_detector/api_public/test_schemas.py @@ -1,6 +1,6 @@ import pytest -from components.bot_detector.api_public.structs.feedback import FeedbackInput +from bot_detector.feedback.structs import FeedbackInput def _base_feedback_kwargs() -> dict: diff --git a/test/bases/bot_detector/api_public/test_utils.py b/test/bases/bot_detector/api_public/test_utils.py index 98cdce0..d99e030 100644 --- a/test/bases/bot_detector/api_public/test_utils.py +++ b/test/bases/bot_detector/api_public/test_utils.py @@ -1,10 +1,12 @@ import asyncio import pytest +from bot_detector.core.cache import SimpleALRUCache +from bot_detector.player.services import PlayerService -from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import to_jagex_name -from components.bot_detector.api_public.services import PlayerService -from bot_detector.cache.simple import SimpleALRUCache +from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import ( + to_jagex_name, +) class _DummySession: diff --git a/test/bases/bot_detector/runemetrics_scraper/test_core.py b/test/bases/bot_detector/runemetrics_scraper/test_core.py index 28f0da0..efd8281 100644 --- a/test/bases/bot_detector/runemetrics_scraper/test_core.py +++ b/test/bases/bot_detector/runemetrics_scraper/test_core.py @@ -7,7 +7,7 @@ RuneMetricsResponse, ) from bot_detector.runemetrics_scraper import core -from bot_detector.structs import PlayerStruct +from bot_detector.player.structs import PlayerStruct from pydantic import BaseModel os.environ["ENVIRONMENT"] = "test" diff --git a/test/bases/bot_detector/scrape_task_producer/test_core.py b/test/bases/bot_detector/scrape_task_producer/test_core.py index 879ef88..dcb829b 100644 --- a/test/bases/bot_detector/scrape_task_producer/test_core.py +++ b/test/bases/bot_detector/scrape_task_producer/test_core.py @@ -2,7 +2,7 @@ import pytest from bot_detector.scrape_task_producer.core import FetchParams, determine_fetch_params -from bot_detector.structs import PlayerStruct +from bot_detector.player.structs import PlayerStruct def make_fetch_params(**overrides) -> FetchParams: diff --git a/test/components/bot_detector/api_public/test_reports_service.py b/test/components/bot_detector/api_public/test_reports_service.py index d5a1d91..35c19f3 100644 --- a/test/components/bot_detector/api_public/test_reports_service.py +++ b/test/components/bot_detector/api_public/test_reports_service.py @@ -1,9 +1,11 @@ +import asyncio import time import pytest -from components.bot_detector.api_public.services import ReportsService -from components.bot_detector.api_public.structs.reports import Detection, Equipment +from bot_detector.report.services.reports import CustomError, ReportsService +from bot_detector.report.structs import Detection, Equipment, ParsedDetection +from bot_detector.core.structs import ReportsToInsertStruct, MetaData def _make_detection(ts: int | None = None, reporter: str = "tester", reported: str = "target") -> Detection: @@ -58,3 +60,92 @@ async def test_parse_data_detects_invalid_unique_reporter(): assert data is None assert error == "invalid unique reporter" + + +@pytest.mark.asyncio() +async def test_transform_detection_builds_structs(): + service = ReportsService() + parsed = ParsedDetection( + reporter_id=1, + reported_id=2, + region_id=1, + x_coord=1, + y_coord=1, + z_coord=0, + ts=1, + manual_detect=0, + on_members_world=0, + on_pvp_world=0, + world_number=301, + equip_ge_value=1, + equipment=Equipment( + equip_head_id=1, + equip_amulet_id=1, + equip_torso_id=1, + equip_legs_id=1, + equip_boots_id=1, + equip_cape_id=1, + equip_hands_id=1, + equip_weapon_id=1, + equip_shield_id=1, + ), + ) + + reports, errors = service._transform_detection([parsed]) + + assert not errors + assert isinstance(reports[0], ReportsToInsertStruct) + assert reports[0].report.reporter_id == 1 + + +@pytest.mark.asyncio() +async def test_send_to_kafka_raises_if_producer_missing(monkeypatch): + service = ReportsService() + monkeypatch.setattr(service, "_transform_detection", lambda data: ([], [])) + monkeypatch.setattr("bot_detector.kafka.kafka_manager.get_producer", lambda key: None) + + with pytest.raises(CustomError): + await service.send_to_kafka([]) + + +@pytest.mark.asyncio() +async def test_send_to_kafka_happy_path(monkeypatch): + service = ReportsService() + class _Producer: + def __init__(self): + self.calls = [] + + async def produce_one(self, report): + self.calls.append(report) + + producer = _Producer() + monkeypatch.setattr("bot_detector.kafka.kafka_manager.get_producer", lambda key: producer) + parsed = ParsedDetection( + reporter_id=1, + reported_id=2, + region_id=1, + x_coord=1, + y_coord=1, + z_coord=0, + ts=1, + manual_detect=0, + on_members_world=0, + on_pvp_world=0, + world_number=301, + equip_ge_value=1, + equipment=Equipment( + equip_head_id=1, + equip_amulet_id=1, + equip_torso_id=1, + equip_legs_id=1, + equip_boots_id=1, + equip_cape_id=1, + equip_hands_id=1, + equip_weapon_id=1, + equip_shield_id=1, + ), + ) + + await service.send_to_kafka([parsed]) + + assert producer.calls, "producer should be invoked" diff --git a/test/components/bot_detector/database/test_core.py b/test/components/bot_detector/database/test_core.py deleted file mode 100644 index b4e83c8..0000000 --- a/test/components/bot_detector/database/test_core.py +++ /dev/null @@ -1,5 +0,0 @@ -from bot_detector.database import core - - -def test_sample(): - assert core is not None diff --git a/test/components/bot_detector/database/test_report_repository.py b/test/components/bot_detector/database/test_report_repository.py index c179905..3353b87 100644 --- a/test/components/bot_detector/database/test_report_repository.py +++ b/test/components/bot_detector/database/test_report_repository.py @@ -1,7 +1,7 @@ from datetime import datetime -from bot_detector.database.report import ReportRepo -from bot_detector.structs import Equipment, ParsedDetection +from bot_detector.report.database.repository import ReportRepo +from bot_detector.report.structs import Equipment, ParsedDetection def _sample_detection( diff --git a/test/components/bot_detector/report/test_report_repo.py b/test/components/bot_detector/report/test_report_repo.py new file mode 100644 index 0000000..89862f3 --- /dev/null +++ b/test/components/bot_detector/report/test_report_repo.py @@ -0,0 +1,51 @@ +import datetime + +from bot_detector.report.database.repository import ReportRepo +from bot_detector.report.structs import Equipment, ParsedDetection + + +def _parsed_detection() -> ParsedDetection: + return ParsedDetection( + reporter_id=1, + reported_id=2, + region_id=1, + x_coord=10, + y_coord=20, + z_coord=0, + ts=1, + manual_detect=0, + on_members_world=0, + on_pvp_world=0, + world_number=301, + equip_ge_value=123, + equipment=Equipment( + equip_head_id=1, + equip_amulet_id=2, + equip_torso_id=3, + equip_legs_id=4, + equip_boots_id=5, + equip_cape_id=6, + equip_hands_id=7, + equip_weapon_id=8, + equip_shield_id=9, + ), + ) + + +def test_parse_reports_flattens_and_converts(): + repo = ReportRepo() + result = repo._parse_reports([_parsed_detection()]) + + assert len(result) == 1 + flattened = result[0] + assert flattened["reporter_id"] == 1 + assert flattened["equip_head_id"] == 1 + assert isinstance(flattened["timestamp"], datetime.datetime) + + +def test_parse_reports_skips_invalid_entries(): + repo = ReportRepo() + + result = repo._parse_reports(["not-a-detection"]) + + assert result == [] From c21ff88f0c0722498fe7aefe3b495ce61af212e7 Mon Sep 17 00:00:00 2001 From: extreme4all <> Date: Fri, 14 Nov 2025 22:51:13 +0100 Subject: [PATCH 10/11] rename --- bases/bot_detector/hiscore_scraper/core.py | 6 +- .../bot_detector/job_hs_migration_v3/core.py | 7 +- bases/bot_detector/worker_hiscore/core.py | 9 +- .../core/structs/kafka/scraped.py | 5 +- .../highscore_worker/database/__init__.py | 17 - .../highscore_worker/database/interface.py | 62 ---- .../highscore_worker/database/repository.py | 325 ------------------ .../highscore_worker/database/structs.py | 96 ------ .../bot_detector/highscore_worker/structs.py | 44 --- 9 files changed, 15 insertions(+), 556 deletions(-) delete mode 100644 components/bot_detector/highscore_worker/database/__init__.py delete mode 100644 components/bot_detector/highscore_worker/database/interface.py delete mode 100644 components/bot_detector/highscore_worker/database/repository.py delete mode 100644 components/bot_detector/highscore_worker/database/structs.py delete mode 100644 components/bot_detector/highscore_worker/structs.py diff --git a/bases/bot_detector/hiscore_scraper/core.py b/bases/bot_detector/hiscore_scraper/core.py index 4d06673..2be7b63 100644 --- a/bases/bot_detector/hiscore_scraper/core.py +++ b/bases/bot_detector/hiscore_scraper/core.py @@ -7,6 +7,8 @@ import aiohttp from aiohttp import ClientSession +from bot_detector.core.structs import MetaData, NotFoundStruct, ScrapedStruct +from bot_detector.highscore.structs import HighscoreBaseStruct from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayerScrapedProducer, @@ -14,11 +16,9 @@ RepoPlayersToScrapeConsumer, RepoPlayersToScrapeProducer, ) +from bot_detector.player.structs import PlayerStruct from bot_detector.proxy_manager import ProxyManager from bot_detector.proxy_manager import Settings as ProxySettings -from bot_detector.highscore_worker.structs import HighscoreBaseStruct -from bot_detector.core.structs import MetaData, NotFoundStruct, ScrapedStruct -from bot_detector.player.structs import PlayerStruct from osrs.asyncio import Hiscore, HSMode from osrs.asyncio.osrs.hiscores import PlayerStats from osrs.exceptions import PlayerDoesNotExist, UnexpectedRedirection diff --git a/bases/bot_detector/job_hs_migration_v3/core.py b/bases/bot_detector/job_hs_migration_v3/core.py index 96c1fe1..ff3d6f6 100644 --- a/bases/bot_detector/job_hs_migration_v3/core.py +++ b/bases/bot_detector/job_hs_migration_v3/core.py @@ -5,13 +5,14 @@ from datetime import timedelta import sqlalchemy as sqla -from bot_detector.core.database import Settings as DBSettings, get_session_factory +from bot_detector.core.database import Settings as DBSettings +from bot_detector.core.database import get_session_factory +from bot_detector.core.structs import MetaData, ScrapedStruct +from bot_detector.highscore.structs import HighscoreBaseStruct from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayerScrapedProducer, ) -from bot_detector.highscore_worker.structs import HighscoreBaseStruct -from bot_detector.core.structs import MetaData, ScrapedStruct from bot_detector.player.structs import PlayerStruct from pydantic_settings import BaseSettings from sqlalchemy import TextClause diff --git a/bases/bot_detector/worker_hiscore/core.py b/bases/bot_detector/worker_hiscore/core.py index 178764d..50eef8b 100644 --- a/bases/bot_detector/worker_hiscore/core.py +++ b/bases/bot_detector/worker_hiscore/core.py @@ -2,15 +2,16 @@ import logging import traceback -from bot_detector.core.database import Settings as DBSettings, get_session_factory -from bot_detector.highscore_worker.database.repository import HighscoreDataRepo -from bot_detector.player.database.repository import PlayerRepo +from bot_detector.core.database import Settings as DBSettings +from bot_detector.core.database import get_session_factory +from bot_detector.core.structs import ScrapedStruct +from bot_detector.highscore.database.repository import HighscoreDataRepo from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayerScrapedConsumer, RepoPlayerScrapedProducer, ) -from bot_detector.core.structs import ScrapedStruct +from bot_detector.player.database.repository import PlayerRepo from pydantic_settings import BaseSettings from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker diff --git a/components/bot_detector/core/structs/kafka/scraped.py b/components/bot_detector/core/structs/kafka/scraped.py index f129b1c..518c98a 100644 --- a/components/bot_detector/core/structs/kafka/scraped.py +++ b/components/bot_detector/core/structs/kafka/scraped.py @@ -1,8 +1,9 @@ -from .._metadata import MetaData -from bot_detector.highscore_worker.structs import HighscoreBaseStruct +from bot_detector.highscore.structs import HighscoreBaseStruct from bot_detector.player.structs import PlayerStruct from pydantic import BaseModel +from .._metadata import MetaData + class ScrapedStruct(BaseModel): metadata: MetaData diff --git a/components/bot_detector/highscore_worker/database/__init__.py b/components/bot_detector/highscore_worker/database/__init__.py deleted file mode 100644 index 57afd1f..0000000 --- a/components/bot_detector/highscore_worker/database/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Highscore worker persistence helpers.""" - -from .repository import HighscoreDataRepo -from .structs import ( - HighscoreDataDailyTableStruct, - HighscoreDataLatestTableStruct, - HighscoreDataMonthlyTableStruct, - HighscoreDataWeeklyTableStruct, -) - -__all__ = [ - "HighscoreDataRepo", - "HighscoreDataDailyTableStruct", - "HighscoreDataWeeklyTableStruct", - "HighscoreDataMonthlyTableStruct", - "HighscoreDataLatestTableStruct", -] diff --git a/components/bot_detector/highscore_worker/database/interface.py b/components/bot_detector/highscore_worker/database/interface.py deleted file mode 100644 index 70e908d..0000000 --- a/components/bot_detector/highscore_worker/database/interface.py +++ /dev/null @@ -1,62 +0,0 @@ -from abc import ABC, abstractmethod - -from bot_detector.highscore_worker.structs import ( - HighscoreDataDailyStruct, - HighscoreDataLatestStruct, - HighscoreDataMonthlyStruct, - HighscoreDataWeeklyStruct, -) -from sqlalchemy.ext.asyncio import AsyncSession - - -class HighscoreDataLatestInterface(ABC): - @abstractmethod - async def insert_highscore( - self, - async_session: AsyncSession, - highscore_data: HighscoreDataLatestStruct, - ) -> None: - """Insert a new highscore record into the database.""" - raise NotImplementedError() - - @abstractmethod - async def insert_highscore_many( - self, - async_session: AsyncSession, - highscore_data: list[HighscoreDataLatestStruct], - ) -> None: - """Insert multiple highscore records into the database.""" - raise NotImplementedError() - - -class HighscoreDataDailyInterface(ABC): - @abstractmethod - async def insert_highscore( - self, - async_session: AsyncSession, - highscore_data: HighscoreDataDailyStruct, - ) -> None: - """Insert a new highscore record into the database.""" - raise NotImplementedError() - - -class HighscoreDataWeeklyInterface(ABC): - @abstractmethod - async def insert_highscore( - self, - async_session: AsyncSession, - highscore_data: HighscoreDataWeeklyStruct, - ): - """Insert a new highscore record into the database.""" - raise NotImplementedError() - - -class HighscoreDataMonthlyInterface(ABC): - @abstractmethod - async def insert_highscore( - self, - async_session: AsyncSession, - highscore_data: HighscoreDataMonthlyStruct, - ): - """Insert a new highscore record into the database.""" - raise NotImplementedError() diff --git a/components/bot_detector/highscore_worker/database/repository.py b/components/bot_detector/highscore_worker/database/repository.py deleted file mode 100644 index 74f2f5a..0000000 --- a/components/bot_detector/highscore_worker/database/repository.py +++ /dev/null @@ -1,325 +0,0 @@ -import json -import logging - -import sqlalchemy as sqla -import sqlalchemy.dialects.mysql as sqla_mysql -from .interface import ( - HighscoreDataDailyInterface, - HighscoreDataLatestInterface, - HighscoreDataMonthlyInterface, - HighscoreDataWeeklyInterface, -) -from .structs import ( - HighscoreDataDailyTableStruct, - HighscoreDataLatestTableStruct, - HighscoreDataMonthlyTableStruct, - HighscoreDataWeeklyTableStruct, -) -from bot_detector.highscore_worker.structs import HighscoreBaseStruct -from sqlalchemy import TextClause, func -from sqlalchemy.ext.asyncio import AsyncSession - -logger = logging.getLogger(__name__) - - -class HighscoreDataRepo(HighscoreDataLatestInterface): - def _create_temp_highscore(self) -> TextClause: - """Create a temporary highscore table for the latest data.""" - return sqla.text( - """ - CREATE TEMPORARY TABLE temp_highscore_data ( - player_id BIGINT NOT NULL, - scrape_date DATE NOT NULL, - skills JSON NOT NULL, - activities JSON NOT NULL - ) ENGINE=InnoDB; - """ - ) - - def _insert_temp_highscore(self) -> TextClause: - """Insert data into the temporary highscore table.""" - return sqla.text( - """ - INSERT INTO temp_highscore_data (player_id, scrape_date, skills, activities) - VALUES (:player_id, :scrape_date, :skills, :activities); - """ - ) - - def _insert_highscore_data_latest(self) -> TextClause: - return sqla.text( - """ - INSERT INTO highscore_data_latest (player_id, scrape_date, skills, activities) - SELECT * FROM ( - SELECT - t.player_id, t.scrape_date, t.skills, t.activities - FROM temp_highscore_data t - ) AS new - ON DUPLICATE KEY UPDATE - scrape_date = new.scrape_date, - skills = new.skills, - activities = new.activities; - """ - ) - - def _insert_highscore_data_daily(self) -> TextClause: - # TODO: add if statement o nthe on duplicate key update - # TODO: figure out if we can avoid the update all together - - return sqla.text( - """ - INSERT INTO highscore_data_daily (player_id, scrape_date, time_to_live, skills, activities) - SELECT * FROM ( - SELECT - t.player_id, - t.scrape_date, - DATE_ADD(t.scrape_date, INTERVAL 30 DAY) AS time_to_live, - t.skills, - t.activities - FROM temp_highscore_data t - ) AS new - ON DUPLICATE KEY UPDATE - scrape_date = IF(new.scrape_date > highscore_data_daily.scrape_date, new.scrape_date, highscore_data_daily.scrape_date), - time_to_live = IF(new.scrape_date > highscore_data_daily.scrape_date, new.time_to_live, highscore_data_daily.time_to_live), - skills = IF(new.scrape_date > highscore_data_daily.scrape_date, new.skills, highscore_data_daily.skills), - activities = IF(new.scrape_date > highscore_data_daily.scrape_date, new.activities, highscore_data_daily.activities); - """ - ) - - def _insert_highscore_data_weekly(self) -> TextClause: - return sqla.text( - """ - INSERT INTO highscore_data_weekly (player_id, scrape_date, time_to_live, skills, activities) - SELECT * FROM ( - SELECT - t.player_id, - t.scrape_date, - DATE_ADD(t.scrape_date, INTERVAL 210 DAY) AS time_to_live, - t.skills, - t.activities - FROM temp_highscore_data t - ) AS new - ON DUPLICATE KEY UPDATE - scrape_date = IF(new.scrape_date > highscore_data_weekly.scrape_date, new.scrape_date, highscore_data_weekly.scrape_date), - time_to_live = IF(new.scrape_date > highscore_data_weekly.scrape_date, new.time_to_live, highscore_data_weekly.time_to_live), - skills = IF(new.scrape_date > highscore_data_weekly.scrape_date, new.skills, highscore_data_weekly.skills), - activities = IF(new.scrape_date > highscore_data_weekly.scrape_date, new.activities, highscore_data_weekly.activities); - """ - ) - - def _insert_highscore_data_monthly(self) -> TextClause: - return sqla.text( - """ - INSERT INTO highscore_data_monthly (player_id, scrape_date, time_to_live, skills, activities) - SELECT * FROM ( - SELECT - t.player_id, - t.scrape_date, - DATE_ADD(t.scrape_date, INTERVAL 900 DAY) AS time_to_live, - t.skills, - t.activities - FROM temp_highscore_data t - ) AS new - ON DUPLICATE KEY UPDATE - scrape_date = IF(new.scrape_date > highscore_data_monthly.scrape_date, new.scrape_date, highscore_data_monthly.scrape_date), - time_to_live = IF(new.scrape_date > highscore_data_monthly.scrape_date, new.time_to_live, highscore_data_monthly.time_to_live), - skills = IF(new.scrape_date > highscore_data_monthly.scrape_date, new.skills, highscore_data_monthly.skills), - activities = IF(new.scrape_date > highscore_data_monthly.scrape_date, new.activities, highscore_data_monthly.activities); - """ - ) - - async def insert_highscore( - self, - async_session: AsyncSession, - highscore_data: HighscoreBaseStruct, - ): - raise NotImplementedError("Use insert_highscore_many for batch inserts.") - - async def insert_highscore_many( - self, - async_session: AsyncSession, - highscore_data: list[HighscoreBaseStruct], - ): - """Insert multiple highscore records into the database.""" - if not highscore_data: - return - - # Drop the temporary table - await async_session.execute( - sqla.text("DROP TEMPORARY TABLE IF EXISTS temp_highscore_data;") - ) - - # Create temporary table - await async_session.execute(self._create_temp_highscore()) - - data_list = [] - for data in highscore_data: - _data = { - "player_id": data.player_id, - "scrape_date": data.scrape_date.isoformat(), - "skills": json.dumps(data.skills), - "activities": json.dumps(data.activities), - } - data_list.append(_data) - - # Insert data into temporary table - await async_session.execute(self._insert_temp_highscore(), data_list) - - # Insert into latest highscore table - await async_session.execute(self._insert_highscore_data_latest()) - - # Insert into daily highscore table - await async_session.execute(self._insert_highscore_data_daily()) - - # Insert into weekly highscore table - await async_session.execute(self._insert_highscore_data_weekly()) - - # Insert into monthly highscore table - await async_session.execute(self._insert_highscore_data_monthly()) - - # Drop the temporary table - await async_session.execute( - sqla.text("DROP TEMPORARY TABLE IF EXISTS temp_highscore_data;") - ) - - -class HighscoreDataLatestRepo(HighscoreDataLatestInterface): - async def insert_highscore( - self, - async_session: AsyncSession, - highscore_data: HighscoreBaseStruct, - ): - """Insert a new highscore record into the highscore_data_weekly table.""" - # The latest highscore data table has no time_to_live field, - _data = highscore_data.model_dump() - _data.pop("time_to_live", None) - - _table = HighscoreDataLatestTableStruct - sql = sqla_mysql.insert(_table) - sql = sql.values([_data]) - sql = sql.on_duplicate_key_update( - scrape_date=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.scrape_date, - _table.scrape_date, - ), - skills=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.skills, - _table.skills, - ), - activities=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.activities, - _table.activities, - ), - ) - sql = sql.prefix_with("IGNORE") - await async_session.execute(sql) - - -class HighscoreDataDailyRepo(HighscoreDataDailyInterface): - async def insert_highscore( - self, - async_session: AsyncSession, - highscore_data: HighscoreBaseStruct, - ): - """Insert a new highscore record into the database.""" - _table = HighscoreDataDailyTableStruct - sql = sqla_mysql.insert(_table) - sql = sql.values([highscore_data.model_dump()]) - sql = sql.on_duplicate_key_update( - scrape_date=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.scrape_date, - _table.scrape_date, - ), - time_to_live=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.time_to_live, - _table.time_to_live, - ), - skills=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.skills, - _table.skills, - ), - activities=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.activities, - _table.activities, - ), - ) - sql = sql.prefix_with("IGNORE") - await async_session.execute(sql) - - -class HighscoreDataWeeklyRepo(HighscoreDataWeeklyInterface): - async def insert_highscore( - self, - async_session: AsyncSession, - highscore_data: HighscoreBaseStruct, - ): - """Insert a new highscore record into the highscore_data_weekly table.""" - _table = HighscoreDataWeeklyTableStruct - sql = sqla_mysql.insert(_table) - sql = sql.values([highscore_data.model_dump()]) - sql = sql.on_duplicate_key_update( - scrape_date=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.scrape_date, - _table.scrape_date, - ), - time_to_live=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.time_to_live, - _table.time_to_live, - ), - skills=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.skills, - _table.skills, - ), - activities=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.activities, - _table.activities, - ), - ) - sql = sql.prefix_with("IGNORE") - await async_session.execute(sql) - - -class HighscoreDataMonthlyRepo(HighscoreDataMonthlyInterface): - async def insert_highscore( - self, - async_session: AsyncSession, - highscore_data: HighscoreBaseStruct, - ): - """Insert a new highscore record into the highscore_data_weekly table.""" - _table = HighscoreDataMonthlyTableStruct - sql = sqla_mysql.insert(_table) - sql = sql.values([highscore_data.model_dump()]) - sql = sql.on_duplicate_key_update( - scrape_date=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.scrape_date, - _table.scrape_date, - ), - time_to_live=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.time_to_live, - _table.time_to_live, - ), - skills=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.skills, - _table.skills, - ), - activities=func.if_( - sql.inserted.scrape_date > _table.scrape_date, - sql.inserted.activities, - _table.activities, - ), - ) - sql = sql.prefix_with("IGNORE") - await async_session.execute(sql) diff --git a/components/bot_detector/highscore_worker/database/structs.py b/components/bot_detector/highscore_worker/database/structs.py deleted file mode 100644 index 7bb764f..0000000 --- a/components/bot_detector/highscore_worker/database/structs.py +++ /dev/null @@ -1,96 +0,0 @@ -from datetime import date -from typing import Optional - -from bot_detector.core.database import Base -from sqlalchemy import ( - JSON, - Computed, - Date, - Integer, - PrimaryKeyConstraint, - SmallInteger, -) -from sqlalchemy.orm import Mapped, mapped_column - - -class HighscoreDataDailyTableStruct(Base): - __tablename__ = "highscore_data_daily" - - player_id: Mapped[int] = mapped_column(Integer, nullable=False) - scrape_date: Mapped[date] = mapped_column(Date, nullable=False) - time_to_live: Mapped[date] = mapped_column(Date, nullable=False) - scrape_year: Mapped[int] = mapped_column( - SmallInteger, Computed("YEAR(scrape_date)"), nullable=False - ) - scrape_month: Mapped[int] = mapped_column( - Integer, Computed("MONTH(scrape_date)"), nullable=False - ) - scrape_week: Mapped[int] = mapped_column( - Integer, Computed("WEEK(scrape_date, 3)"), nullable=False - ) - skills: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) - activities: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) - - __table_args__ = (PrimaryKeyConstraint("player_id", "scrape_date"),) - - -class HighscoreDataWeeklyTableStruct(Base): - __tablename__ = "highscore_data_weekly" - - player_id: Mapped[int] = mapped_column(Integer, nullable=False) - scrape_date: Mapped[date] = mapped_column(Date, nullable=False) - time_to_live: Mapped[date] = mapped_column(Date, nullable=False) - scrape_year: Mapped[int] = mapped_column( - SmallInteger, Computed("YEAR(scrape_date)"), nullable=False - ) - scrape_month: Mapped[int] = mapped_column( - Integer, Computed("MONTH(scrape_date)"), nullable=False - ) - scrape_week: Mapped[int] = mapped_column( - Integer, Computed("WEEK(scrape_date, 3)"), nullable=False - ) - skills: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) - activities: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) - - __table_args__ = (PrimaryKeyConstraint("player_id", "scrape_year", "scrape_week"),) - - -class HighscoreDataMonthlyTableStruct(Base): - __tablename__ = "highscore_data_monthly" - - player_id: Mapped[int] = mapped_column(Integer, nullable=False) - scrape_date: Mapped[date] = mapped_column(Date, nullable=False) - time_to_live: Mapped[date] = mapped_column(Date, nullable=False) - scrape_year: Mapped[int] = mapped_column( - SmallInteger, Computed("YEAR(scrape_date)"), nullable=False - ) - scrape_month: Mapped[int] = mapped_column( - Integer, Computed("MONTH(scrape_date)"), nullable=False - ) - scrape_week: Mapped[int] = mapped_column( - Integer, Computed("WEEK(scrape_date, 3)"), nullable=False - ) - skills: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) - activities: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) - - __table_args__ = (PrimaryKeyConstraint("player_id", "scrape_year", "scrape_month"),) - - -class HighscoreDataLatestTableStruct(Base): - __tablename__ = "highscore_data_latest" - - player_id: Mapped[int] = mapped_column(Integer, nullable=False) - scrape_date: Mapped[date] = mapped_column(Date, nullable=False) - scrape_year: Mapped[int] = mapped_column( - SmallInteger, Computed("YEAR(scrape_date)"), nullable=False - ) - scrape_month: Mapped[int] = mapped_column( - Integer, Computed("MONTH(scrape_date)"), nullable=False - ) - scrape_week: Mapped[int] = mapped_column( - Integer, Computed("WEEK(scrape_date, 3)"), nullable=False - ) - skills: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) - activities: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) - - __table_args__ = (PrimaryKeyConstraint("player_id"),) diff --git a/components/bot_detector/highscore_worker/structs.py b/components/bot_detector/highscore_worker/structs.py deleted file mode 100644 index 9d5d453..0000000 --- a/components/bot_detector/highscore_worker/structs.py +++ /dev/null @@ -1,44 +0,0 @@ -from datetime import date -from typing import Optional - -from pydantic import BaseModel - - -class HighscoreBaseStruct(BaseModel): - player_id: int - scrape_date: date - time_to_live: date - skills: Optional[dict[str, int]] = None - activities: Optional[dict[str, int]] = None - - -class HighscoreDataBaseStruct(HighscoreBaseStruct): - scrape_year: int - scrape_month: int - scrape_week: int - - -class HighscoreDataLatestStruct(HighscoreDataBaseStruct): - pass - - -class HighscoreDataDailyStruct(HighscoreDataBaseStruct): - pass - - -class HighscoreDataWeeklyStruct(HighscoreDataBaseStruct): - pass - - -class HighscoreDataMonthlyStruct(HighscoreDataBaseStruct): - pass - - -__all__ = [ - "HighscoreBaseStruct", - "HighscoreDataBaseStruct", - "HighscoreDataDailyStruct", - "HighscoreDataWeeklyStruct", - "HighscoreDataMonthlyStruct", - "HighscoreDataLatestStruct", -] From b5916a39c40a7d5c82fd66b581f0dc43547c3cd3 Mon Sep 17 00:00:00 2001 From: extreme4all <> Date: Fri, 14 Nov 2025 22:51:37 +0100 Subject: [PATCH 11/11] rename --- .../highscore/database/__init__.py | 17 + .../highscore/database/interface.py | 62 ++++ .../highscore/database/repository.py | 326 ++++++++++++++++++ .../highscore/database/structs.py | 96 ++++++ components/bot_detector/highscore/structs.py | 44 +++ 5 files changed, 545 insertions(+) create mode 100644 components/bot_detector/highscore/database/__init__.py create mode 100644 components/bot_detector/highscore/database/interface.py create mode 100644 components/bot_detector/highscore/database/repository.py create mode 100644 components/bot_detector/highscore/database/structs.py create mode 100644 components/bot_detector/highscore/structs.py diff --git a/components/bot_detector/highscore/database/__init__.py b/components/bot_detector/highscore/database/__init__.py new file mode 100644 index 0000000..57afd1f --- /dev/null +++ b/components/bot_detector/highscore/database/__init__.py @@ -0,0 +1,17 @@ +"""Highscore worker persistence helpers.""" + +from .repository import HighscoreDataRepo +from .structs import ( + HighscoreDataDailyTableStruct, + HighscoreDataLatestTableStruct, + HighscoreDataMonthlyTableStruct, + HighscoreDataWeeklyTableStruct, +) + +__all__ = [ + "HighscoreDataRepo", + "HighscoreDataDailyTableStruct", + "HighscoreDataWeeklyTableStruct", + "HighscoreDataMonthlyTableStruct", + "HighscoreDataLatestTableStruct", +] diff --git a/components/bot_detector/highscore/database/interface.py b/components/bot_detector/highscore/database/interface.py new file mode 100644 index 0000000..0178273 --- /dev/null +++ b/components/bot_detector/highscore/database/interface.py @@ -0,0 +1,62 @@ +from abc import ABC, abstractmethod + +from bot_detector.highscore.structs import ( + HighscoreDataDailyStruct, + HighscoreDataLatestStruct, + HighscoreDataMonthlyStruct, + HighscoreDataWeeklyStruct, +) +from sqlalchemy.ext.asyncio import AsyncSession + + +class HighscoreDataLatestInterface(ABC): + @abstractmethod + async def insert_highscore( + self, + async_session: AsyncSession, + highscore_data: HighscoreDataLatestStruct, + ) -> None: + """Insert a new highscore record into the database.""" + raise NotImplementedError() + + @abstractmethod + async def insert_highscore_many( + self, + async_session: AsyncSession, + highscore_data: list[HighscoreDataLatestStruct], + ) -> None: + """Insert multiple highscore records into the database.""" + raise NotImplementedError() + + +class HighscoreDataDailyInterface(ABC): + @abstractmethod + async def insert_highscore( + self, + async_session: AsyncSession, + highscore_data: HighscoreDataDailyStruct, + ) -> None: + """Insert a new highscore record into the database.""" + raise NotImplementedError() + + +class HighscoreDataWeeklyInterface(ABC): + @abstractmethod + async def insert_highscore( + self, + async_session: AsyncSession, + highscore_data: HighscoreDataWeeklyStruct, + ): + """Insert a new highscore record into the database.""" + raise NotImplementedError() + + +class HighscoreDataMonthlyInterface(ABC): + @abstractmethod + async def insert_highscore( + self, + async_session: AsyncSession, + highscore_data: HighscoreDataMonthlyStruct, + ): + """Insert a new highscore record into the database.""" + raise NotImplementedError() diff --git a/components/bot_detector/highscore/database/repository.py b/components/bot_detector/highscore/database/repository.py new file mode 100644 index 0000000..23ed97f --- /dev/null +++ b/components/bot_detector/highscore/database/repository.py @@ -0,0 +1,326 @@ +import json +import logging + +import sqlalchemy as sqla +import sqlalchemy.dialects.mysql as sqla_mysql +from bot_detector.highscore.structs import HighscoreBaseStruct +from sqlalchemy import TextClause, func +from sqlalchemy.ext.asyncio import AsyncSession + +from .interface import ( + HighscoreDataDailyInterface, + HighscoreDataLatestInterface, + HighscoreDataMonthlyInterface, + HighscoreDataWeeklyInterface, +) +from .structs import ( + HighscoreDataDailyTableStruct, + HighscoreDataLatestTableStruct, + HighscoreDataMonthlyTableStruct, + HighscoreDataWeeklyTableStruct, +) + +logger = logging.getLogger(__name__) + + +class HighscoreDataRepo(HighscoreDataLatestInterface): + def _create_temp_highscore(self) -> TextClause: + """Create a temporary highscore table for the latest data.""" + return sqla.text( + """ + CREATE TEMPORARY TABLE temp_highscore_data ( + player_id BIGINT NOT NULL, + scrape_date DATE NOT NULL, + skills JSON NOT NULL, + activities JSON NOT NULL + ) ENGINE=InnoDB; + """ + ) + + def _insert_temp_highscore(self) -> TextClause: + """Insert data into the temporary highscore table.""" + return sqla.text( + """ + INSERT INTO temp_highscore_data (player_id, scrape_date, skills, activities) + VALUES (:player_id, :scrape_date, :skills, :activities); + """ + ) + + def _insert_highscore_data_latest(self) -> TextClause: + return sqla.text( + """ + INSERT INTO highscore_data_latest (player_id, scrape_date, skills, activities) + SELECT * FROM ( + SELECT + t.player_id, t.scrape_date, t.skills, t.activities + FROM temp_highscore_data t + ) AS new + ON DUPLICATE KEY UPDATE + scrape_date = new.scrape_date, + skills = new.skills, + activities = new.activities; + """ + ) + + def _insert_highscore_data_daily(self) -> TextClause: + # TODO: add if statement o nthe on duplicate key update + # TODO: figure out if we can avoid the update all together + + return sqla.text( + """ + INSERT INTO highscore_data_daily (player_id, scrape_date, time_to_live, skills, activities) + SELECT * FROM ( + SELECT + t.player_id, + t.scrape_date, + DATE_ADD(t.scrape_date, INTERVAL 30 DAY) AS time_to_live, + t.skills, + t.activities + FROM temp_highscore_data t + ) AS new + ON DUPLICATE KEY UPDATE + scrape_date = IF(new.scrape_date > highscore_data_daily.scrape_date, new.scrape_date, highscore_data_daily.scrape_date), + time_to_live = IF(new.scrape_date > highscore_data_daily.scrape_date, new.time_to_live, highscore_data_daily.time_to_live), + skills = IF(new.scrape_date > highscore_data_daily.scrape_date, new.skills, highscore_data_daily.skills), + activities = IF(new.scrape_date > highscore_data_daily.scrape_date, new.activities, highscore_data_daily.activities); + """ + ) + + def _insert_highscore_data_weekly(self) -> TextClause: + return sqla.text( + """ + INSERT INTO highscore_data_weekly (player_id, scrape_date, time_to_live, skills, activities) + SELECT * FROM ( + SELECT + t.player_id, + t.scrape_date, + DATE_ADD(t.scrape_date, INTERVAL 210 DAY) AS time_to_live, + t.skills, + t.activities + FROM temp_highscore_data t + ) AS new + ON DUPLICATE KEY UPDATE + scrape_date = IF(new.scrape_date > highscore_data_weekly.scrape_date, new.scrape_date, highscore_data_weekly.scrape_date), + time_to_live = IF(new.scrape_date > highscore_data_weekly.scrape_date, new.time_to_live, highscore_data_weekly.time_to_live), + skills = IF(new.scrape_date > highscore_data_weekly.scrape_date, new.skills, highscore_data_weekly.skills), + activities = IF(new.scrape_date > highscore_data_weekly.scrape_date, new.activities, highscore_data_weekly.activities); + """ + ) + + def _insert_highscore_data_monthly(self) -> TextClause: + return sqla.text( + """ + INSERT INTO highscore_data_monthly (player_id, scrape_date, time_to_live, skills, activities) + SELECT * FROM ( + SELECT + t.player_id, + t.scrape_date, + DATE_ADD(t.scrape_date, INTERVAL 900 DAY) AS time_to_live, + t.skills, + t.activities + FROM temp_highscore_data t + ) AS new + ON DUPLICATE KEY UPDATE + scrape_date = IF(new.scrape_date > highscore_data_monthly.scrape_date, new.scrape_date, highscore_data_monthly.scrape_date), + time_to_live = IF(new.scrape_date > highscore_data_monthly.scrape_date, new.time_to_live, highscore_data_monthly.time_to_live), + skills = IF(new.scrape_date > highscore_data_monthly.scrape_date, new.skills, highscore_data_monthly.skills), + activities = IF(new.scrape_date > highscore_data_monthly.scrape_date, new.activities, highscore_data_monthly.activities); + """ + ) + + async def insert_highscore( + self, + async_session: AsyncSession, + highscore_data: HighscoreBaseStruct, + ): + raise NotImplementedError("Use insert_highscore_many for batch inserts.") + + async def insert_highscore_many( + self, + async_session: AsyncSession, + highscore_data: list[HighscoreBaseStruct], + ): + """Insert multiple highscore records into the database.""" + if not highscore_data: + return + + # Drop the temporary table + await async_session.execute( + sqla.text("DROP TEMPORARY TABLE IF EXISTS temp_highscore_data;") + ) + + # Create temporary table + await async_session.execute(self._create_temp_highscore()) + + data_list = [] + for data in highscore_data: + _data = { + "player_id": data.player_id, + "scrape_date": data.scrape_date.isoformat(), + "skills": json.dumps(data.skills), + "activities": json.dumps(data.activities), + } + data_list.append(_data) + + # Insert data into temporary table + await async_session.execute(self._insert_temp_highscore(), data_list) + + # Insert into latest highscore table + await async_session.execute(self._insert_highscore_data_latest()) + + # Insert into daily highscore table + await async_session.execute(self._insert_highscore_data_daily()) + + # Insert into weekly highscore table + await async_session.execute(self._insert_highscore_data_weekly()) + + # Insert into monthly highscore table + await async_session.execute(self._insert_highscore_data_monthly()) + + # Drop the temporary table + await async_session.execute( + sqla.text("DROP TEMPORARY TABLE IF EXISTS temp_highscore_data;") + ) + + +class HighscoreDataLatestRepo(HighscoreDataLatestInterface): + async def insert_highscore( + self, + async_session: AsyncSession, + highscore_data: HighscoreBaseStruct, + ): + """Insert a new highscore record into the highscore_data_weekly table.""" + # The latest highscore data table has no time_to_live field, + _data = highscore_data.model_dump() + _data.pop("time_to_live", None) + + _table = HighscoreDataLatestTableStruct + sql = sqla_mysql.insert(_table) + sql = sql.values([_data]) + sql = sql.on_duplicate_key_update( + scrape_date=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.scrape_date, + _table.scrape_date, + ), + skills=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.skills, + _table.skills, + ), + activities=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.activities, + _table.activities, + ), + ) + sql = sql.prefix_with("IGNORE") + await async_session.execute(sql) + + +class HighscoreDataDailyRepo(HighscoreDataDailyInterface): + async def insert_highscore( + self, + async_session: AsyncSession, + highscore_data: HighscoreBaseStruct, + ): + """Insert a new highscore record into the database.""" + _table = HighscoreDataDailyTableStruct + sql = sqla_mysql.insert(_table) + sql = sql.values([highscore_data.model_dump()]) + sql = sql.on_duplicate_key_update( + scrape_date=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.scrape_date, + _table.scrape_date, + ), + time_to_live=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.time_to_live, + _table.time_to_live, + ), + skills=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.skills, + _table.skills, + ), + activities=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.activities, + _table.activities, + ), + ) + sql = sql.prefix_with("IGNORE") + await async_session.execute(sql) + + +class HighscoreDataWeeklyRepo(HighscoreDataWeeklyInterface): + async def insert_highscore( + self, + async_session: AsyncSession, + highscore_data: HighscoreBaseStruct, + ): + """Insert a new highscore record into the highscore_data_weekly table.""" + _table = HighscoreDataWeeklyTableStruct + sql = sqla_mysql.insert(_table) + sql = sql.values([highscore_data.model_dump()]) + sql = sql.on_duplicate_key_update( + scrape_date=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.scrape_date, + _table.scrape_date, + ), + time_to_live=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.time_to_live, + _table.time_to_live, + ), + skills=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.skills, + _table.skills, + ), + activities=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.activities, + _table.activities, + ), + ) + sql = sql.prefix_with("IGNORE") + await async_session.execute(sql) + + +class HighscoreDataMonthlyRepo(HighscoreDataMonthlyInterface): + async def insert_highscore( + self, + async_session: AsyncSession, + highscore_data: HighscoreBaseStruct, + ): + """Insert a new highscore record into the highscore_data_weekly table.""" + _table = HighscoreDataMonthlyTableStruct + sql = sqla_mysql.insert(_table) + sql = sql.values([highscore_data.model_dump()]) + sql = sql.on_duplicate_key_update( + scrape_date=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.scrape_date, + _table.scrape_date, + ), + time_to_live=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.time_to_live, + _table.time_to_live, + ), + skills=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.skills, + _table.skills, + ), + activities=func.if_( + sql.inserted.scrape_date > _table.scrape_date, + sql.inserted.activities, + _table.activities, + ), + ) + sql = sql.prefix_with("IGNORE") + await async_session.execute(sql) diff --git a/components/bot_detector/highscore/database/structs.py b/components/bot_detector/highscore/database/structs.py new file mode 100644 index 0000000..7bb764f --- /dev/null +++ b/components/bot_detector/highscore/database/structs.py @@ -0,0 +1,96 @@ +from datetime import date +from typing import Optional + +from bot_detector.core.database import Base +from sqlalchemy import ( + JSON, + Computed, + Date, + Integer, + PrimaryKeyConstraint, + SmallInteger, +) +from sqlalchemy.orm import Mapped, mapped_column + + +class HighscoreDataDailyTableStruct(Base): + __tablename__ = "highscore_data_daily" + + player_id: Mapped[int] = mapped_column(Integer, nullable=False) + scrape_date: Mapped[date] = mapped_column(Date, nullable=False) + time_to_live: Mapped[date] = mapped_column(Date, nullable=False) + scrape_year: Mapped[int] = mapped_column( + SmallInteger, Computed("YEAR(scrape_date)"), nullable=False + ) + scrape_month: Mapped[int] = mapped_column( + Integer, Computed("MONTH(scrape_date)"), nullable=False + ) + scrape_week: Mapped[int] = mapped_column( + Integer, Computed("WEEK(scrape_date, 3)"), nullable=False + ) + skills: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) + activities: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) + + __table_args__ = (PrimaryKeyConstraint("player_id", "scrape_date"),) + + +class HighscoreDataWeeklyTableStruct(Base): + __tablename__ = "highscore_data_weekly" + + player_id: Mapped[int] = mapped_column(Integer, nullable=False) + scrape_date: Mapped[date] = mapped_column(Date, nullable=False) + time_to_live: Mapped[date] = mapped_column(Date, nullable=False) + scrape_year: Mapped[int] = mapped_column( + SmallInteger, Computed("YEAR(scrape_date)"), nullable=False + ) + scrape_month: Mapped[int] = mapped_column( + Integer, Computed("MONTH(scrape_date)"), nullable=False + ) + scrape_week: Mapped[int] = mapped_column( + Integer, Computed("WEEK(scrape_date, 3)"), nullable=False + ) + skills: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) + activities: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) + + __table_args__ = (PrimaryKeyConstraint("player_id", "scrape_year", "scrape_week"),) + + +class HighscoreDataMonthlyTableStruct(Base): + __tablename__ = "highscore_data_monthly" + + player_id: Mapped[int] = mapped_column(Integer, nullable=False) + scrape_date: Mapped[date] = mapped_column(Date, nullable=False) + time_to_live: Mapped[date] = mapped_column(Date, nullable=False) + scrape_year: Mapped[int] = mapped_column( + SmallInteger, Computed("YEAR(scrape_date)"), nullable=False + ) + scrape_month: Mapped[int] = mapped_column( + Integer, Computed("MONTH(scrape_date)"), nullable=False + ) + scrape_week: Mapped[int] = mapped_column( + Integer, Computed("WEEK(scrape_date, 3)"), nullable=False + ) + skills: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) + activities: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) + + __table_args__ = (PrimaryKeyConstraint("player_id", "scrape_year", "scrape_month"),) + + +class HighscoreDataLatestTableStruct(Base): + __tablename__ = "highscore_data_latest" + + player_id: Mapped[int] = mapped_column(Integer, nullable=False) + scrape_date: Mapped[date] = mapped_column(Date, nullable=False) + scrape_year: Mapped[int] = mapped_column( + SmallInteger, Computed("YEAR(scrape_date)"), nullable=False + ) + scrape_month: Mapped[int] = mapped_column( + Integer, Computed("MONTH(scrape_date)"), nullable=False + ) + scrape_week: Mapped[int] = mapped_column( + Integer, Computed("WEEK(scrape_date, 3)"), nullable=False + ) + skills: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) + activities: Mapped[Optional[dict[str, int]]] = mapped_column(JSON, nullable=True) + + __table_args__ = (PrimaryKeyConstraint("player_id"),) diff --git a/components/bot_detector/highscore/structs.py b/components/bot_detector/highscore/structs.py new file mode 100644 index 0000000..9d5d453 --- /dev/null +++ b/components/bot_detector/highscore/structs.py @@ -0,0 +1,44 @@ +from datetime import date +from typing import Optional + +from pydantic import BaseModel + + +class HighscoreBaseStruct(BaseModel): + player_id: int + scrape_date: date + time_to_live: date + skills: Optional[dict[str, int]] = None + activities: Optional[dict[str, int]] = None + + +class HighscoreDataBaseStruct(HighscoreBaseStruct): + scrape_year: int + scrape_month: int + scrape_week: int + + +class HighscoreDataLatestStruct(HighscoreDataBaseStruct): + pass + + +class HighscoreDataDailyStruct(HighscoreDataBaseStruct): + pass + + +class HighscoreDataWeeklyStruct(HighscoreDataBaseStruct): + pass + + +class HighscoreDataMonthlyStruct(HighscoreDataBaseStruct): + pass + + +__all__ = [ + "HighscoreBaseStruct", + "HighscoreDataBaseStruct", + "HighscoreDataDailyStruct", + "HighscoreDataWeeklyStruct", + "HighscoreDataMonthlyStruct", + "HighscoreDataLatestStruct", +]