diff --git a/oobe/src/App.tsx b/oobe/src/App.tsx index f3ecd16..0c98ccf 100644 --- a/oobe/src/App.tsx +++ b/oobe/src/App.tsx @@ -21,6 +21,9 @@ import SmartClinical from "./pages/SmartClinical"; import QualityInspection from "./pages/QualityInspection"; import HighResolutionVisuals from "./pages/HighResolutionVisuals"; import CrowdAndFallDetection from "./pages/CrowdAndFallDetection"; +import QualityInspectionWebcam from "./pages/QualityInspectionWebcam"; +import SampleIntegrityCheckWebcam from "./pages/SampleIntegrityCheckWebcam"; +import CrowdAndFallDetectionWebcam from "./pages/CrowdAndFallDetectionWebcam"; const HIDE_SIDEBAR_ROUTES = [ "/medical-alert-management", @@ -92,14 +95,26 @@ function App() { path="/quality-inspection" element={} /> + } + /> } /> + } + /> } /> + } + /> } diff --git a/oobe/src/api/APIClient.ts b/oobe/src/api/APIClient.ts index be261f1..26ada17 100644 --- a/oobe/src/api/APIClient.ts +++ b/oobe/src/api/APIClient.ts @@ -23,12 +23,6 @@ export type PersonResult = { score: number; }; -interface BackendPersonResult { - category_id: number; - bbox: number[]; - score: number; -} - export type InverterStatus = "ready" | "fault"; export type SmartUpdate = | { field: "plantStatus"; value: string } @@ -110,7 +104,12 @@ type FaceRecognitionMessage = { data: FaceRecognitionUpdate[]; }; -export type AnalysisMode = "cpu" | "npu"; +export type AnalysisMode = "cpu" | "gpu" | "npu"; + +export interface DetectionResponse { + results: T[]; + inferenceTime: number; +} export class APIClient { private config: Config; @@ -160,9 +159,12 @@ export class APIClient { }; } - async getBlisterPackResult(imageFile: File): Promise { - const response = await this.axiosInstance.post( - "/blister-pack-detect", + async getBlisterPackResult( + imageFile: File, + mode: AnalysisMode, + ): Promise> { + const response = await this.axiosInstance.post( + `/blister-pack-detect-${mode}`, imageFile, { headers: { @@ -170,13 +172,14 @@ export class APIClient { }, }, ); - return response.data.map( - (item): BlisterPackResult => ({ + return { + results: response.data.items.map((item) => ({ categoryId: item.category_id, bbox: item.bbox, score: item.score, - }), - ); + })), + inferenceTime: response.data.inferenceTime, + }; } async exitApp(): Promise { @@ -469,9 +472,12 @@ export class APIClient { }; } - async getPersonResult(imageFile: File): Promise { - const response = await this.axiosInstance.post( - "/people-detect", + async getPersonResult( + imageFile: File, + mode: AnalysisMode, + ): Promise> { + const response = await this.axiosInstance.post( + `/people-detect-${mode}`, imageFile, { headers: { @@ -480,13 +486,14 @@ export class APIClient { }, ); - return response.data.map( - (item): PersonResult => ({ + return { + results: response.data.items.map((item) => ({ categoryId: item.category_id, bbox: item.bbox, score: item.score, - }), - ); + })), + inferenceTime: response.data.inferenceTime, + }; } disconnectWebSocket(wsType?: string) { diff --git a/oobe/src/i18n/langs/en.json b/oobe/src/i18n/langs/en.json index b7c7a83..d0bf9b5 100644 --- a/oobe/src/i18n/langs/en.json +++ b/oobe/src/i18n/langs/en.json @@ -80,6 +80,15 @@ "components.BloodCountMedical.unit": { "defaultMessage": "Unit" }, + "components.CrowdAndFallDetection.cpuAnalysisButton": { + "defaultMessage": "CPU Analysis" + }, + "components.CrowdAndFallDetection.gpuAnalysisButton": { + "defaultMessage": "GPU Analysis" + }, + "components.CrowdAndFallDetection.npuAnalysisButton": { + "defaultMessage": "NPU Analysis" + }, "components.DeviceDetailsCard.cpuArchitecture": { "defaultMessage": "CPU architecture" }, @@ -122,32 +131,23 @@ "components.GeolocalizationCard.title": { "defaultMessage": "Geolocalization" }, - "components.QualityInspection.analyzeNextMessage": { - "defaultMessage": "This is a demo environment, the camera feed is simulated. Click ‘CPU Analysis’ or 'NPU Analysis' to run inference" - }, "components.QualityInspection.cpuAnalysisButton": { "defaultMessage": "CPU Analysis" }, + "components.QualityInspection.gpuAnalysisButton": { + "defaultMessage": "GPU Analysis" + }, "components.QualityInspection.npuAnalysisButton": { "defaultMessage": "NPU Analysis" }, - "components.QualityInspection.startAnalysisButton": { - "defaultMessage": "Start analysis" - }, - "components.QualityInspection.startAnalysisMessage": { - "defaultMessage": "This is a demo environment, the camera feed is simulated. Click ‘Start Analysis’ to run inference." - }, - "components.SampleIntegrityCheck.analyzeNextButton": { - "defaultMessage": "Analyze next object" - }, - "components.SampleIntegrityCheck.analyzeNextMessage": { - "defaultMessage": "This is a demo environment, the camera feed is simulated. Click ‘Analyze next object’ to run inference." + "components.SampleIntegrityCheck.cpuAnalysisButton": { + "defaultMessage": "CPU Analysis" }, - "components.SampleIntegrityCheck.startAnalysisMessage": { - "defaultMessage": "This is a demo environment, the camera feed is simulated. Click ‘Start Analysis’ to run inference." + "components.SampleIntegrityCheck.gpuAnalysisButton": { + "defaultMessage": "GPU Analysis" }, - "components.SampleIntegrityCheck.startButton": { - "defaultMessage": "Start analysis" + "components.SampleIntegrityCheck.npuAnalysisButton": { + "defaultMessage": "NPU Analysis" }, "components.Sidebar.dashboard": { "defaultMessage": "Dashboard" diff --git a/oobe/src/pages/CrowdAndFallDetection.scss b/oobe/src/pages/CrowdAndFallDetection.scss index 260417b..1acca0e 100644 --- a/oobe/src/pages/CrowdAndFallDetection.scss +++ b/oobe/src/pages/CrowdAndFallDetection.scss @@ -37,6 +37,41 @@ } } + .greeting-button { + background-color: #e2e5f3; + border: none; + border-radius: 6px; + color: #1a1a1a; + font-size: 1rem; + min-width: 200px; + height: 52px; + + &:hover { + background-color: #d1d5e8; + } + } + + .analyze-cpu-button, + .analyze-gpu-button, + .analyze-npu-button { + @extend .greeting-button; + height: 52px; + transition: all 0.3s ease; + + &.active-analysis { + color: #919191 !important; + background-color: #e2e5f3 !important; + opacity: 0.6; + cursor: wait; + } + + &:disabled:not(.active-analysis) { + opacity: 0.8; + background-color: #ffffff; + color: #ccc; + } + } + @media (max-width: 768px) { overflow-y: auto !important; .vh-100 { diff --git a/oobe/src/pages/CrowdAndFallDetection.tsx b/oobe/src/pages/CrowdAndFallDetection.tsx index 6dcde33..764881b 100644 --- a/oobe/src/pages/CrowdAndFallDetection.tsx +++ b/oobe/src/pages/CrowdAndFallDetection.tsx @@ -5,7 +5,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faX, faExclamationTriangle } from "@fortawesome/free-solid-svg-icons"; import { FormattedMessage, defineMessages } from "react-intl"; import ImageCarousel from "./ImageCarousel"; -import { APIClient } from "../api/APIClient"; +import { APIClient, type AnalysisMode } from "../api/APIClient"; import { logo, elevator_second_floor_1, @@ -126,9 +126,11 @@ const urlToFile = async (url: string): Promise => { const CrowdAndFallDetection = ({ apiClient }: CrowdAndFallDetectionProps) => { const [results, setResults] = useState([]); + const [analysisMode, setAnalysisMode] = useState("cpu"); + const [inferenceTime, setInferenceTime] = useState(null); const [error, setError] = useState(null); const [status, setStatus] = useState<"greeting" | "analysis" | "result">( - "greeting", + "analysis", ); const [currentTime, setCurrentTime] = useState(new Date()); const [analysisTime, setAnalysisTime] = useState(null); @@ -192,25 +194,20 @@ const CrowdAndFallDetection = ({ apiClient }: CrowdAndFallDetectionProps) => { const processImage = async () => { try { setResults([]); + setInferenceTime(null); const file = await urlToFile(currentImage); - const data = await apiClient.getPersonResult(file); - setResults(data); + const data = await apiClient.getPersonResult(file, analysisMode); + setResults(data.results); + setInferenceTime(data.inferenceTime); setAnalysisTime(new Date()); setStatus("result"); } catch { setError("Error"); - setStatus("greeting"); + setStatus("analysis"); } }; processImage(); - }, [apiClient, currentImage, status]); - - const changeImage = (dir: number) => { - const idx = imageOptions.indexOf(currentImage); - const next = (idx + dir + imageOptions.length) % imageOptions.length; - setCurrentImage(imageOptions[next]); - setStatus("analysis"); - }; + }, [apiClient, currentImage, status, analysisMode]); return ( { SECO -
+
{formatFullDate(currentTime)}
- {status === "greeting" ? ( -
- -
- ) : ( -
-
-
- {status === "result" && - results.map((r, i) => { - const scaleX = - imgRect.width / (imageRef.current?.naturalWidth || 1); - const scaleY = - imgRect.height / (imageRef.current?.naturalHeight || 1); - const isFall = r.categoryId === 1; - const boxColor = isFall - ? "#FF0000" - : DETECTION_COLORS[i % DETECTION_COLORS.length]; + {status === "result" && + results.map((r, i) => { + const scaleX = + imgRect.width / (imageRef.current?.naturalWidth || 1); + const scaleY = + imgRect.height / (imageRef.current?.naturalHeight || 1); + const isFall = r.categoryId === 1; + const boxColor = isFall + ? "#FF0000" + : DETECTION_COLORS[i % DETECTION_COLORS.length]; - return ( -
- ); - })} -
- + return ( +
+ ); + })}
+ +
-
- { - setCurrentImage(img); - setStatus("analysis"); - }} +
+ { + setCurrentImage(img); + setStatus("analysis"); + }} + /> +
+
+ +
+
+
+

+ -

+
-
-
-
-

- -

+ {status === "result" && inferenceTime !== null && ( +
+ Inference time: {inferenceTime.toFixed(2)} ms
+ )} -
- {status === "analysis" ? ( -
-
- -
- ) : ( -
- {results.length === 0 ? ( -
-

- -

-
- ) : ( - results.map((r, i) => ( -
-
- - #{i + 1} - - - - -
-
- - {analysisTime ? formatFullDate(analysisTime) : ""} +
+ {status === "analysis" ? ( +
+
+ +
+ ) : ( +
+ {results.length === 0 ? ( +
+

+ +

+
+ ) : ( + results.map((r, i) => ( +
+
+ #{i + 1} + + + +
+
+ + {analysisTime ? formatFullDate(analysisTime) : ""} + + {r.categoryId === 1 && ( + + + - {r.categoryId === 1 && ( - - - - - )} -
+ )}
- )) - )} -
- )} -
- -
-
- - - - 0 ? "text-primary" : "text-white"}`} - style={{ lineHeight: 1 }} - > - {results.length.toString().padStart(2, "0")} - -
-
- - +
+ )) + )}
+ )} +
+ +
+
+ + + + 0 ? "text-primary" : "text-white"}`} + style={{ lineHeight: 1 }} + > + {results.length.toString().padStart(2, "0")} + +
+
+ + + +
- )} +
{error && ( { + const streamRef = useRef(null); + const videoRef = useRef(null); + const canvasRef = useRef(null); + const imageRef = useRef(null); + const containerRef = useRef(null); + const isAnalyzingRef = useRef(false); + + const [status, setStatus] = useState("idle"); + const [results, setResults] = useState([]); + const [inferenceTime, setInferenceTime] = useState(null); + const [analysisMode, setAnalysisMode] = useState("cpu"); + const [error, setError] = useState(null); + const [captured, setCaptured] = useState(null); + const [currentTime] = useState(new Date()); + + const navigate = useNavigate(); + const apiClient = useMemo(() => new APIClient(), []); + + React.useEffect(() => { + return () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + } + }; + }, []); + + const startWebcam = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ video: true }); + streamRef.current = stream; + setStatus("greeting"); + } catch { + setError( + "Failed to access webcam. Please ensure you have given permission.", + ); + } + }; + + React.useEffect(() => { + if (status === "greeting" && videoRef.current && streamRef.current) { + videoRef.current.srcObject = streamRef.current; + } + }, [status]); + + React.useEffect(() => { + let active = true; + const analyzeLoop = async () => { + if ( + status !== "greeting" || + !streamRef.current || + !videoRef.current || + !canvasRef.current + ) + return; + if (isAnalyzingRef.current) return; + + isAnalyzingRef.current = true; + try { + const ctx = canvasRef.current.getContext("2d", { alpha: false }); + if (ctx) { + ctx.drawImage(videoRef.current, 0, 0, 980, 720); + + const blob = await new Promise((resolve) => + canvasRef.current?.toBlob((b) => resolve(b), "image/jpeg", 0.6), + ); + + if (blob && active && status === "greeting") { + const file = new File([blob], "webcam.jpg", { type: "image/jpeg" }); + const data = await apiClient.getPersonResult(file, analysisMode); + if (active && status === "greeting") { + const url = URL.createObjectURL(blob); + setCaptured((prev) => { + if (prev && prev.startsWith("blob:")) URL.revokeObjectURL(prev); + return url; + }); + setResults(data.results); + setInferenceTime(data.inferenceTime); + } + } + } + } catch (err) { + console.error("Loop analysis error:", err); + } finally { + isAnalyzingRef.current = false; + if (active && status === "greeting") { + setTimeout(analyzeLoop, 50); + } + } + }; + + if (status === "greeting") { + analyzeLoop(); + } + return () => { + active = false; + }; + }, [status, analysisMode, apiClient]); + + const formatFullDate = (date: Date): string => { + const time = date.toLocaleTimeString("en-GB", { hour12: false }); + const dayMonthYear = date.toLocaleDateString("en-GB", { + day: "2-digit", + month: "2-digit", + year: "numeric", + }); + return `${time} - ${dayMonthYear}`; + }; + + const handleAnalysisClick = (mode: AnalysisMode) => { + setAnalysisMode(mode); + if (status === "idle") { + startWebcam(); + } + }; + + const handleReturnPage = () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + } + navigate("/crowd-and-fall-detection", { state: { autoStart: true } }); + }; + + const renderBBoxes = () => { + const media = imageRef.current || videoRef.current; + const container = containerRef.current; + if (!media || !container || results.length === 0) return null; + + const rect = media.getBoundingClientRect(); + const containerRect = container.getBoundingClientRect(); + + const scaleX = rect.width / 980; + const scaleY = rect.height / 720; + + const offsetX = rect.left - containerRect.left; + const offsetY = rect.top - containerRect.top; + + return results.map((r, i) => { + const isFall = r.categoryId === 1; + const boxColor = isFall + ? "#FF0000" + : DETECTION_COLORS[i % DETECTION_COLORS.length]; + return ( +
+ ); + }); + }; + + return ( + +
+
+ +
+ SECO +
+ +
+ {formatFullDate(currentTime)} +
+
+ + {error && ( + setError(null)} + dismissible + variant="danger" + className="mb-3" + > + {error} + + )} + +
+
+
+ {status === "idle" && ( +
+

Webcam inactive.

+

Click an analysis mode to start.

+
+ )} + + {status === "greeting" && ( +
+ +
+
+
r.categoryId === 1) ? "bg-danger animate-pulse" : "bg-success"}`} + >
+

+ Live Webcam +

+
+ + {inferenceTime !== null && ( +
+ Inference time: {inferenceTime.toFixed(2)} ms +
+ )} + +
+
+ {results.length === 0 ? ( +
+

+ No people detected. +

+
+ ) : ( + results.map((r, i) => ( +
+
+ #{i + 1} + Person detected +
+
+ )) + )} +
+
+ +
+
+ + PEOPLE COUNTER + + r.categoryId === 1) + ? "text-danger" + : results.length > 0 + ? "text-primary" + : "text-white" + }`} + style={{ lineHeight: 1 }} + > + {results.length.toString().padStart(2, "0")} + +
+
+
+
+ +
+ + + + +
+ + ); +}; + +export default CrowdAndFallDetectionWebcam; diff --git a/oobe/src/pages/QualityInspection.scss b/oobe/src/pages/QualityInspection.scss index c989d52..a8aded2 100644 --- a/oobe/src/pages/QualityInspection.scss +++ b/oobe/src/pages/QualityInspection.scss @@ -65,6 +65,7 @@ } .analyze-cpu-button, + .analyze-gpu-button, .analyze-npu-button { @extend .greeting-button; height: 52px; diff --git a/oobe/src/pages/QualityInspection.tsx b/oobe/src/pages/QualityInspection.tsx index 8fe7f24..e2c6908 100644 --- a/oobe/src/pages/QualityInspection.tsx +++ b/oobe/src/pages/QualityInspection.tsx @@ -66,7 +66,7 @@ const QualityInspection = ({ apiClient }: QualityInspectionProps) => { const [analysisMode, setAnalysisMode] = useState("cpu"); const [inferenceTime, setInferenceTime] = useState(null); const [error, setError] = useState(null); - const [status, setStatus] = useState("greeting"); + const [status, setStatus] = useState("analysis"); const [currentImage, setCurrentImage] = useState(pcbMissingHole00); const [scale, setScale] = useState({ x: 1, y: 1 }); @@ -89,16 +89,6 @@ const QualityInspection = ({ apiClient }: QualityInspectionProps) => { return () => window.removeEventListener("resize", handleImageLoad); }, []); - useEffect(() => { - if (status !== "analysis") return; - - const timer = setTimeout(() => { - if (status === "analysis") setStatus("result"); - }, 2000); - - return () => clearTimeout(timer); - }, [status, currentImage]); - useEffect(() => { if (status !== "analysis") return; const processImage = async () => { @@ -109,6 +99,7 @@ const QualityInspection = ({ apiClient }: QualityInspectionProps) => { const data = await apiClient.getDefectResult(file, analysisMode); setDefectResults(data.results); setInferenceTime(data.inferenceTime); + setStatus("result"); } catch { setError( intl.formatMessage({ @@ -139,21 +130,15 @@ const QualityInspection = ({ apiClient }: QualityInspectionProps) => { className="quality-inspection-container vh-100 d-flex flex-column p-4 bg-black text-white" >
- {status !== "greeting" && ( - - )} +
- SECO Logo + SECO Logo
@@ -169,185 +154,161 @@ const QualityInspection = ({ apiClient }: QualityInspectionProps) => { )} -
-

- {status === "greeting" ? ( - - ) : ( - +
+
+ {status === "result" && + defectResults.map((defect, index) => ( +
+ ))} + Sample - )} -

-
- - {status !== "greeting" && ( -
-
-
- {status === "result" && - defectResults.map((defect, index) => ( -
- ))} - Sample -
+
-
- { - setCurrentImage(img); - setDefectResults([]); - }} - /> -
+
+ { + setCurrentImage(img); + setStatus("analysis"); + }} + />
+
-
-
- {status === "analysis" && ( -
- )} -

- {status === "analysis" ? ( +
+
+ {status === "analysis" && ( +
+ )} +

+ {status === "analysis" ? ( + + ) : ( +
- ) : ( -
- - - {status === "result" && inferenceTime !== null && ( -
- Inference time:{" "} - {inferenceTime.toFixed(2)} ms -
- )} -
- - + {status === "result" && inferenceTime !== null && ( +
+ Inference time:{" "} + {inferenceTime.toFixed(2)} ms
+ )} -
- - -
+
+ +
- )} -

-
- -
- - -
+
+ + +
+
+ )} +

-
- )} - {status === "greeting" && ( -
- +
+ + + + + + +
- )} +
); }; diff --git a/oobe/src/pages/QualityInspectionWebcam.tsx b/oobe/src/pages/QualityInspectionWebcam.tsx new file mode 100644 index 0000000..5538005 --- /dev/null +++ b/oobe/src/pages/QualityInspectionWebcam.tsx @@ -0,0 +1,363 @@ +import React, { useRef, useState, useMemo } from "react"; +import { Container, Button, Alert, Image } from "react-bootstrap"; +import { useNavigate } from "react-router-dom"; +import { APIClient, type AnalysisMode } from "../api/APIClient"; +import { type DefectResult } from "./QualityInspection.tsx"; +import { logo } from "../assets/images"; +import { FormattedMessage } from "react-intl"; + +const MISSING_HOLE_COLOR = "#FF0000"; +const SHORT_CIRCUIT_COLOR = "#FFC107"; +const DEFAULT_COLOR = "#222322"; + +const QualityInspectionWebcam = () => { + const streamRef = useRef(null); + const videoRef = useRef(null); + const canvasRef = useRef(null); + const imageRef = useRef(null); + const containerRef = useRef(null); + const isAnalyzingRef = useRef(false); + + const [status, setStatus] = useState("idle"); + const [defectResults, setDefectResults] = useState([]); + const [inferenceTime, setInferenceTime] = useState(null); + const [analysisMode, setAnalysisMode] = useState("cpu"); + const [error, setError] = useState(null); + const [captured, setCaptured] = useState(null); + const [currentTime] = useState(new Date()); + const navigate = useNavigate(); + const apiClient = useMemo(() => new APIClient(), []); + + React.useEffect(() => { + return () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + } + }; + }, []); + + const startWebcam = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ video: true }); + streamRef.current = stream; + setStatus("greeting"); + } catch { + setError( + "Failed to access webcam. Please ensure you have given permission.", + ); + } + }; + + React.useEffect(() => { + if (status === "greeting" && videoRef.current && streamRef.current) { + videoRef.current.srcObject = streamRef.current; + } + }, [status]); + + React.useEffect(() => { + let active = true; + const analyzeLoop = async () => { + if ( + status !== "greeting" || + !streamRef.current || + !videoRef.current || + !canvasRef.current + ) + return; + if (isAnalyzingRef.current) return; + + isAnalyzingRef.current = true; + try { + const ctx = canvasRef.current.getContext("2d", { alpha: false }); + if (ctx) { + ctx.drawImage(videoRef.current, 0, 0, 960, 720); + + const blob = await new Promise((resolve) => + canvasRef.current?.toBlob((b) => resolve(b), "image/jpeg", 0.7), + ); + + if (blob && active && status === "greeting") { + const file = new File([blob], "webcam.jpg", { type: "image/jpeg" }); + const data = await apiClient.getDefectResult(file, analysisMode); + if (active && status === "greeting") { + const url = URL.createObjectURL(blob); + setCaptured((prev) => { + if (prev && prev.startsWith("blob:")) URL.revokeObjectURL(prev); + return url; + }); + setDefectResults(data.results); + setInferenceTime(data.inferenceTime); + } + } + } + } catch (err) { + console.error("Loop analysis error:", err); + } finally { + isAnalyzingRef.current = false; + if (active && status === "greeting") { + setTimeout(analyzeLoop, 100); + } + } + }; + + if (status === "greeting") { + analyzeLoop(); + } + return () => { + active = false; + }; + }, [status, analysisMode, apiClient]); + + const formatFullDate = (date: Date): string => { + const time = date.toLocaleTimeString("en-GB", { hour12: false }); + const dayMonthYear = date.toLocaleDateString("en-GB", { + day: "2-digit", + month: "2-digit", + year: "numeric", + }); + return `${time} - ${dayMonthYear}`; + }; + + const handleAnalysisClick = (mode: AnalysisMode) => { + setAnalysisMode(mode); + if (status === "idle") { + startWebcam(); + } + }; + + const handleReturnPage = () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + } + navigate("/quality-inspection", { state: { autoStart: true } }); + }; + + const handleBBoxColor = (categoryId: number) => { + switch (categoryId) { + case 0: + return MISSING_HOLE_COLOR; + case 3: + return SHORT_CIRCUIT_COLOR; + default: + return DEFAULT_COLOR; + } + }; + + const renderBBoxes = () => { + const media = imageRef.current || videoRef.current; + const container = containerRef.current; + if (!media || !container || defectResults.length === 0) return null; + + const rect = media.getBoundingClientRect(); + const containerRect = container.getBoundingClientRect(); + + const scaleX = rect.width / 960; + const scaleY = rect.height / 720; + + const offsetX = rect.left - containerRect.left; + const offsetY = rect.top - containerRect.top; + + return defectResults.map((defect, index) => ( +
+ )); + }; + + return ( + +
+
+ +
+ SECO +
+ +
+ {formatFullDate(currentTime)} +
+
+ + {error && ( + setError(null)} + dismissible + variant="danger" + className="mb-3" + > + {error} + + )} + +
+
+
+ {status === "idle" && ( +
+

Webcam inactive.

+

Click an analysis mode to start.

+
+ )} + + {status === "greeting" && ( +
+
+
+

+
+ + + {inferenceTime !== null && ( +
+ Inference time:{" "} + {inferenceTime.toFixed(2)} ms +
+ )} + +
+ + +
+ +
+ + +
+
+

+
+
+
+
+ + + + +
+ + ); +}; + +export default QualityInspectionWebcam; diff --git a/oobe/src/pages/SampleIntegrityCheck.scss b/oobe/src/pages/SampleIntegrityCheck.scss index 8ee0e2b..4da6b43 100644 --- a/oobe/src/pages/SampleIntegrityCheck.scss +++ b/oobe/src/pages/SampleIntegrityCheck.scss @@ -56,7 +56,7 @@ border-radius: 6px; color: #1a1a1a; font-size: 1rem; - min-width: 300px; + min-width: 200px; height: 52px; &:hover { @@ -64,9 +64,25 @@ } } - .analysis-result-button { + .analysis-result-button, + .analyze-gpu-button, + .analyze-npu-button { @extend .greeting-button; height: 52px; + transition: all 0.3s ease; + + &.active-analysis { + color: #919191 !important; + background-color: #e2e5f3 !important; + opacity: 0.6; + cursor: wait; + } + + &:disabled:not(.active-analysis) { + opacity: 0.8; + background-color: #ffffff; + color: #ccc; + } } .image { diff --git a/oobe/src/pages/SampleIntegrityCheck.tsx b/oobe/src/pages/SampleIntegrityCheck.tsx index 0c836df..a8c1851 100644 --- a/oobe/src/pages/SampleIntegrityCheck.tsx +++ b/oobe/src/pages/SampleIntegrityCheck.tsx @@ -23,7 +23,7 @@ import { blisterPackPartial09, blisterPackPartial10, } from "../assets/images"; -import { APIClient } from "../api/APIClient"; +import { APIClient, type AnalysisMode } from "../api/APIClient"; import ImageCarousel from "./ImageCarousel"; const EMPTY_BLISTER_COLOR = "#FF0000"; @@ -52,8 +52,10 @@ const SampleIntegrityCheck = ({ apiClient }: SampleIntegrityCheckProps) => { const [blisterPackResults, setBlisterPackResults] = useState< BlisterPackResult[] >([]); + const [analysisMode, setAnalysisMode] = useState("cpu"); + const [inferenceTime, setInferenceTime] = useState(null); const [error, setError] = useState(null); - const [status, setStatus] = useState("greeting"); + const [status, setStatus] = useState("analysis"); const [currentImage, setCurrentImage] = useState(blisterPackEmpty00); const [scale, setScale] = useState({ x: 1, y: 1 }); @@ -94,24 +96,17 @@ const SampleIntegrityCheck = ({ apiClient }: SampleIntegrityCheckProps) => { return () => window.removeEventListener("resize", handleImageLoad); }, []); - useEffect(() => { - if (status !== "analysis") return; - - const timer = setTimeout(() => { - if (status === "analysis") setStatus("result"); - }, 5000); - - return () => clearTimeout(timer); - }, [status, currentImage]); - useEffect(() => { if (status !== "analysis") return; const processImage = async () => { try { setBlisterPackResults([]); + setInferenceTime(null); const file = await urlToFile(currentImage); - const data = await apiClient.getBlisterPackResult(file); - setBlisterPackResults(data); + const data = await apiClient.getBlisterPackResult(file, analysisMode); + setBlisterPackResults(data.results); + setInferenceTime(data.inferenceTime); + setStatus("result"); } catch { setError( intl.formatMessage({ @@ -123,7 +118,7 @@ const SampleIntegrityCheck = ({ apiClient }: SampleIntegrityCheckProps) => { }; if (currentImage) processImage(); - }, [apiClient, currentImage, status, intl]); + }, [apiClient, currentImage, status, intl, analysisMode]); const handleBBoxColor = (categoryId: number) => { switch (categoryId) { @@ -142,21 +137,15 @@ const SampleIntegrityCheck = ({ apiClient }: SampleIntegrityCheckProps) => { className="integrity-container vh-100 d-flex flex-column p-4 bg-black text-white" >
- {status !== "greeting" && ( - - )} +
- SECO Logo + SECO Logo
@@ -172,171 +161,161 @@ const SampleIntegrityCheck = ({ apiClient }: SampleIntegrityCheckProps) => { )} -
-

- {status === "greeting" ? ( - - ) : ( - +
+
+ {status === "result" && + blisterPackResults.map((defect, index) => ( +
+ ))} + Sample - )} -

-
- - {status !== "greeting" && ( -
-
-
- {status === "result" && - blisterPackResults.map((defect, index) => ( -
- ))} - Sample -
+
-
- { - setCurrentImage(img); - setStatus("analysis"); - }} - /> -
+
+ { + setCurrentImage(img); + setStatus("analysis"); + }} + />
+
-
-
- {status === "analysis" && ( -
- )} -

- {status === "analysis" ? ( +
+
+ {status === "analysis" && ( +
+ )} +

+ {status === "analysis" ? ( + + ) : ( +
- ) : ( -
- -
- - + {status === "result" && inferenceTime !== null && ( +
+ Inference time:{" "} + {inferenceTime.toFixed(2)} ms
-
- - -
-
- )}{" "} -

-
+ )} -
- -
+
+ + +
+
+ + +
+
+ )}{" "} +

-
- )} - {status === "greeting" && ( -
- +
+ + + + + + +
- )} +
); }; diff --git a/oobe/src/pages/SampleIntegrityCheckWebcam.tsx b/oobe/src/pages/SampleIntegrityCheckWebcam.tsx new file mode 100644 index 0000000..0be48d1 --- /dev/null +++ b/oobe/src/pages/SampleIntegrityCheckWebcam.tsx @@ -0,0 +1,372 @@ +import React, { useRef, useState, useMemo } from "react"; +import { Container, Button, Alert, Image } from "react-bootstrap"; +import { useNavigate } from "react-router-dom"; +import { APIClient, type AnalysisMode } from "../api/APIClient"; +import { logo } from "../assets/images"; +import { FormattedMessage } from "react-intl"; + +const EMPTY_BLISTER_COLOR = "#FF0000"; +const FULL_BLISTER_COLOR = "#FFC107"; +const DEFAULT_COLOR = "#222322"; + +export type BlisterPackResult = { + categoryId: number; + bbox: number[]; + score: number; +}; + +const SampleIntegrityCheckWebcam = () => { + const streamRef = useRef(null); + const videoRef = useRef(null); + const canvasRef = useRef(null); + const imageRef = useRef(null); + const containerRef = useRef(null); + const isAnalyzingRef = useRef(false); + + const [status, setStatus] = useState("idle"); + const [blisterPackResults, setBlisterPackResults] = useState< + BlisterPackResult[] + >([]); + const [inferenceTime, setInferenceTime] = useState(null); + const [analysisMode, setAnalysisMode] = useState("cpu"); + const [error, setError] = useState(null); + const [captured, setCaptured] = useState(null); + const [currentTime] = useState(new Date()); + const navigate = useNavigate(); + const apiClient = useMemo(() => new APIClient(), []); + + React.useEffect(() => { + return () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + } + }; + }, []); + + const startWebcam = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ video: true }); + streamRef.current = stream; + setStatus("greeting"); + } catch { + setError( + "Failed to access webcam. Please ensure you have given permission.", + ); + } + }; + + React.useEffect(() => { + if (status === "greeting" && videoRef.current && streamRef.current) { + videoRef.current.srcObject = streamRef.current; + } + }, [status]); + + React.useEffect(() => { + let active = true; + const analyzeLoop = async () => { + if ( + status !== "greeting" || + !streamRef.current || + !videoRef.current || + !canvasRef.current + ) + return; + if (isAnalyzingRef.current) return; + + isAnalyzingRef.current = true; + try { + const ctx = canvasRef.current.getContext("2d", { alpha: false }); + if (ctx) { + ctx.drawImage(videoRef.current, 0, 0, 960, 720); + + const blob = await new Promise((resolve) => + canvasRef.current?.toBlob((b) => resolve(b), "image/jpeg", 0.7), + ); + + if (blob && active && status === "greeting") { + const file = new File([blob], "webcam.jpg", { type: "image/jpeg" }); + const data = await apiClient.getBlisterPackResult( + file, + analysisMode, + ); + if (active && status === "greeting") { + const url = URL.createObjectURL(blob); + setCaptured((prev) => { + if (prev && prev.startsWith("blob:")) URL.revokeObjectURL(prev); + return url; + }); + setBlisterPackResults(data.results); + setInferenceTime(data.inferenceTime); + } + } + } + } catch (err) { + console.error("Loop analysis error:", err); + } finally { + isAnalyzingRef.current = false; + if (active && status === "greeting") { + setTimeout(analyzeLoop, 100); + } + } + }; + + if (status === "greeting") { + analyzeLoop(); + } + return () => { + active = false; + }; + }, [status, analysisMode, apiClient]); + + const formatFullDate = (date: Date): string => { + const time = date.toLocaleTimeString("en-GB", { hour12: false }); + const dayMonthYear = date.toLocaleDateString("en-GB", { + day: "2-digit", + month: "2-digit", + year: "numeric", + }); + return `${time} - ${dayMonthYear}`; + }; + + const handleAnalysisClick = (mode: AnalysisMode) => { + setAnalysisMode(mode); + if (status === "idle") { + startWebcam(); + } + }; + + const handleReturnPage = () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + } + navigate("/sample-integrity-check", { state: { autoStart: true } }); + }; + + const handleBBoxColor = (categoryId: number) => { + switch (categoryId) { + case 1: + return EMPTY_BLISTER_COLOR; + case 2: + return FULL_BLISTER_COLOR; + default: + return DEFAULT_COLOR; + } + }; + + const renderBBoxes = () => { + const media = imageRef.current || videoRef.current; + const container = containerRef.current; + if (!media || !container || blisterPackResults.length === 0) return null; + + const rect = media.getBoundingClientRect(); + const containerRect = container.getBoundingClientRect(); + + const scaleX = rect.width / 960; + const scaleY = rect.height / 720; + + const offsetX = rect.left - containerRect.left; + const offsetY = rect.top - containerRect.top; + + return blisterPackResults.map((defect, index) => ( +
+ )); + }; + + return ( + +
+
+ +
+ SECO +
+ +
+ {formatFullDate(currentTime)} +
+
+ + {error && ( + setError(null)} + dismissible + variant="danger" + className="mb-3" + > + {error} + + )} + +
+
+
+ {status === "idle" && ( +
+

Webcam inactive.

+

Click an analysis mode to start.

+
+ )} + + {status === "greeting" && ( +
+
+
+

+
+ + + {inferenceTime !== null && ( +
+ Inference time:{" "} + {inferenceTime.toFixed(2)} ms +
+ )} + +
+ + +
+
+ + +
+
+

+
+
+
+
+ + + + +
+ + ); +}; + +export default SampleIntegrityCheckWebcam;