From 5c2b01fc8b96b6d90287a1d59ab1ed3ebc1e9bf8 Mon Sep 17 00:00:00 2001 From: Gertjan Date: Wed, 4 Feb 2026 13:18:56 +0000 Subject: [PATCH 01/55] WIP: improve inclusion of most recent commits when latest major version is selected This works for V8, but still needs tweaking for browsers. Now, the latest major version is not adequately updated. Likely because of a bug in the bughog-service API. --- bughog/subject/state_oracle.py | 3 +++ bughog/version_control/conversion/bughog_service.py | 4 ++++ bughog/version_control/state_factory.py | 8 +++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/bughog/subject/state_oracle.py b/bughog/subject/state_oracle.py index 7418c866..eeada6be 100644 --- a/bughog/subject/state_oracle.py +++ b/bughog/subject/state_oracle.py @@ -34,6 +34,9 @@ def get_commit_url(self, commit_nb: int, commit_id: str | None) -> str | None: def get_most_recent_major_release_version(self) -> int: pass + def get_most_recent_commit_nb(self) -> int: + return bughog_service.find_latest_commit_info(self.subject_name).get('nb') + @staticmethod def is_valid_commit_id(commit_id: str) -> bool: """ diff --git a/bughog/version_control/conversion/bughog_service.py b/bughog/version_control/conversion/bughog_service.py index a07100f6..5f9c81b5 100644 --- a/bughog/version_control/conversion/bughog_service.py +++ b/bughog/version_control/conversion/bughog_service.py @@ -22,6 +22,10 @@ def find_commit_info(subject_name: str, commit_nb: str) -> dict[str, Any]: return __fetch_dict(url) +def find_latest_commit_info(subject_name: str) -> dict[str, Any]: + return find_commit_info(subject_name, 'latest') + + @lru_cache(maxsize=LRU_CACHE_SIZE) def find_commit_nb(subject_name: str, commit_id: str) -> int: url = urljoin(BASE_URL, f'{subject_name}/commits/{commit_id}') diff --git a/bughog/version_control/state_factory.py b/bughog/version_control/state_factory.py index e3d32edb..fb1dd2bc 100644 --- a/bughog/version_control/state_factory.py +++ b/bughog/version_control/state_factory.py @@ -50,8 +50,14 @@ def __create_boundary_states(self) -> tuple[State, State]: first_state = self.__create_release_state(eval_range.major_version_range[0]) last_state = self.__create_release_state(eval_range.major_version_range[1]) if not eval_range.only_release_commits: + # Only commits will be considered. first_state = first_state.convert_to_commit_state() - last_state = last_state.convert_to_commit_state() + if self.__oracle.get_most_recent_major_release_version() == last_state.index: + # If the upper boundary is the most recent major release, we simply set the latest commit as upper boundary. + last_commit = self.__oracle.get_most_recent_commit_nb() + last_state = self.__create_commit_state(last_commit) + else: + last_state = last_state.convert_to_commit_state() return first_state, last_state elif eval_range.commit_nb_range: if eval_range.only_release_commits: From 3655068354afc1401781706a7301ac3e14b44ac3 Mon Sep 17 00:00:00 2001 From: Gertjan Date: Wed, 4 Feb 2026 13:36:12 +0000 Subject: [PATCH 02/55] Fix bug where API is called for v8_sandbox instead of v8 --- bughog/subject/js_engine/v8_sandbox/state_oracle.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bughog/subject/js_engine/v8_sandbox/state_oracle.py b/bughog/subject/js_engine/v8_sandbox/state_oracle.py index 962f8c73..6e65f73b 100644 --- a/bughog/subject/js_engine/v8_sandbox/state_oracle.py +++ b/bughog/subject/js_engine/v8_sandbox/state_oracle.py @@ -2,6 +2,7 @@ from typing import Literal from bughog.subject.js_engine.v8.state_oracle import V8StateOracle +from bughog.version_control.conversion import bughog_service logger = logging.getLogger(__name__) @@ -19,6 +20,12 @@ def __init__(self, subject_type: str, subject_name: str) -> None: # There are no public executables, only artisanal. super().__init__(subject_type, subject_name, only_artisanal=True) + def get_most_recent_commit_nb(self) -> int: + """ + We override this method because we want to call the API for v8, not v8_sandbox. + """ + return bughog_service.find_latest_commit_info('v8').get('nb') + def has_public_executable(self, state_index: int, state_type: Literal['release', 'commit']) -> bool: return False From c8a61e9355ee67967c51f51022c929ddda3c679b Mon Sep 17 00:00:00 2001 From: Gertjan Date: Wed, 11 Feb 2026 17:00:13 +0000 Subject: [PATCH 03/55] Unify subject availability logic Also return 'min_commit' and 'max_commit' to web UI. --- bughog/subject/js_engine/v8/state_oracle.py | 3 +++ bughog/subject/js_engine/v8/subject.py | 10 --------- .../subject/js_engine/v8_sandbox/subject.py | 10 --------- bughog/subject/state_oracle.py | 4 ++++ bughog/subject/subject.py | 21 ++++++++++++------- .../wasm_runtime/wasmtime/state_oracle.py | 3 +++ .../subject/wasm_runtime/wasmtime/subject.py | 7 ------- .../web_browser/chromium/state_oracle.py | 3 +++ .../subject/web_browser/chromium/subject.py | 5 ----- .../web_browser/firefox/state_oracle.py | 3 +++ bughog/subject/web_browser/firefox/subject.py | 9 -------- 11 files changed, 30 insertions(+), 48 deletions(-) diff --git a/bughog/subject/js_engine/v8/state_oracle.py b/bughog/subject/js_engine/v8/state_oracle.py index 4a037cc6..bb957c0f 100644 --- a/bughog/subject/js_engine/v8/state_oracle.py +++ b/bughog/subject/js_engine/v8/state_oracle.py @@ -47,6 +47,9 @@ def get_commit_url(self, commit_nb: int, commit_id: str | None) -> str | None: # Public executables + def get_oldest_supported_release_version(self) -> int: + return 6 + def get_most_recent_major_release_version(self) -> int: all_release_tags = self.__get_all_release_tags() major_versions = set(int(tag.split('.')[0]) for tag in all_release_tags) diff --git a/bughog/subject/js_engine/v8/subject.py b/bughog/subject/js_engine/v8/subject.py index edb47ff4..b3aec75e 100644 --- a/bughog/subject/js_engine/v8/subject.py +++ b/bughog/subject/js_engine/v8/subject.py @@ -15,15 +15,5 @@ def name(self) -> str: def _state_oracle_class(self) -> type[V8StateOracle]: return V8StateOracle - def get_availability(self) -> dict: - """ - Returns availability data (minimum and maximu, release versions, and configuration options) of the subject. - """ - return { - 'name': 'v8', - 'min_version': 6, - 'max_version': self.state_oracle.get_most_recent_major_release_version() - } - def create_executable(self, subject_configuration: SubjectConfiguration, state: State) -> V8Executable: return V8Executable(subject_configuration, state) diff --git a/bughog/subject/js_engine/v8_sandbox/subject.py b/bughog/subject/js_engine/v8_sandbox/subject.py index 1712817c..b43fbfc8 100644 --- a/bughog/subject/js_engine/v8_sandbox/subject.py +++ b/bughog/subject/js_engine/v8_sandbox/subject.py @@ -15,15 +15,5 @@ def name(self) -> str: def _state_oracle_class(self) -> type[V8SandboxStateOracle]: return V8SandboxStateOracle - def get_availability(self) -> dict: - """ - Returns availability data (minimum and maximu, release versions, and configuration options) of the subject. - """ - return { - 'name': self.name, - 'min_version': 6, - 'max_version': self.state_oracle.get_most_recent_major_release_version() - } - def create_executable(self, subject_configuration: SubjectConfiguration, state: State) -> V8Executable: return V8Executable(subject_configuration, state) diff --git a/bughog/subject/state_oracle.py b/bughog/subject/state_oracle.py index eeada6be..eb159aaf 100644 --- a/bughog/subject/state_oracle.py +++ b/bughog/subject/state_oracle.py @@ -30,6 +30,10 @@ def find_commit_of_release(self, release_version: int) -> tuple[int, str]: def get_commit_url(self, commit_nb: int, commit_id: str | None) -> str | None: pass + @abstractmethod + def get_oldest_supported_release_version(self) -> int: + pass + @abstractmethod def get_most_recent_major_release_version(self) -> int: pass diff --git a/bughog/subject/subject.py b/bughog/subject/subject.py index 78ecd2ae..0c24dc9f 100644 --- a/bughog/subject/subject.py +++ b/bughog/subject/subject.py @@ -59,13 +59,6 @@ def create_executable(self, subject_configuration: SubjectConfiguration, state: """ pass - @abstractmethod - def get_availability(self) -> dict: - """ - Returns availability data (supported minimum and maximum release version) of this subject. - """ - pass - @staticmethod @abstractmethod def create_simulation(executable: Executable, context: Folder, params: EvaluationParameters) -> Simulation: @@ -95,3 +88,17 @@ def assets_folder_path(self) -> str: Returns the paths of the assets folder associated with this subject. """ return os.path.join('/app/subject', self.type, self.name) + + def get_availability(self) -> dict: + oldest_major_version = self.state_oracle.get_oldest_supported_release_version() + newest_major_version = self.state_oracle.get_most_recent_major_release_version() + + oldest_commit_number = self.state_oracle.find_commit_of_release(oldest_major_version)[0] + newest_commit_number = self.state_oracle.find_commit_of_release(newest_major_version)[0] + return { + 'name': self.name, + 'min_version': oldest_major_version, + 'max_version': newest_major_version, + 'min_commit': oldest_commit_number, + 'max_commit': newest_commit_number, + } diff --git a/bughog/subject/wasm_runtime/wasmtime/state_oracle.py b/bughog/subject/wasm_runtime/wasmtime/state_oracle.py index 82afc34d..003a524d 100644 --- a/bughog/subject/wasm_runtime/wasmtime/state_oracle.py +++ b/bughog/subject/wasm_runtime/wasmtime/state_oracle.py @@ -30,6 +30,9 @@ def find_commit_of_release(self, release_version: int) -> tuple[int, str]: commit_nb = self.find_commit_nb(commit_id) return commit_nb, commit_id + def get_oldest_supported_release_version(self) -> int: + return 1 + def get_most_recent_major_release_version(self) -> int: all_release_tags = self.__get_all_release_tags() truncated_tags = [self.get_full_version_from_release_tag(tag) for tag in all_release_tags] diff --git a/bughog/subject/wasm_runtime/wasmtime/subject.py b/bughog/subject/wasm_runtime/wasmtime/subject.py index 252b6808..b08dff6c 100644 --- a/bughog/subject/wasm_runtime/wasmtime/subject.py +++ b/bughog/subject/wasm_runtime/wasmtime/subject.py @@ -14,12 +14,5 @@ def name(self) -> str: def _state_oracle_class(self) -> type[WasmtimeStateOracle]: return WasmtimeStateOracle - def get_availability(self) -> dict: - return { - 'name': 'wasmtime', - 'min_version': 1, - 'max_version': self.state_oracle.get_most_recent_major_release_version() - } - def create_executable(self, subject_configuration: SubjectConfiguration, state: State) -> WasmtimeExecutable: return WasmtimeExecutable(subject_configuration, state) diff --git a/bughog/subject/web_browser/chromium/state_oracle.py b/bughog/subject/web_browser/chromium/state_oracle.py index 311d42c7..4543aabe 100644 --- a/bughog/subject/web_browser/chromium/state_oracle.py +++ b/bughog/subject/web_browser/chromium/state_oracle.py @@ -53,6 +53,9 @@ def find_commit_id(self, commit_nb: int) -> str | None: def find_commit_of_release(self, release_version: int) -> tuple[int, str]: return bughog_service.find_version_commit('chromium', release_version, has_public_executable=True) + def get_oldest_supported_release_version(self) -> int: + return 20 + def get_most_recent_major_release_version(self) -> int: return bughog_service.find_latest_major_version('chromium') diff --git a/bughog/subject/web_browser/chromium/subject.py b/bughog/subject/web_browser/chromium/subject.py index cc4d5002..64c71f36 100644 --- a/bughog/subject/web_browser/chromium/subject.py +++ b/bughog/subject/web_browser/chromium/subject.py @@ -5,7 +5,6 @@ from bughog.subject.web_browser.chromium.executable import ChromiumExecutable from bughog.subject.web_browser.chromium.state_oracle import ChromiumStateOracle from bughog.subject.web_browser.subject import WebBrowser -from bughog.version_control.conversion import bughog_service from bughog.version_control.state.base import State logger = logging.getLogger(__name__) @@ -22,7 +21,3 @@ def _state_oracle_class(self) -> type[StateOracle]: def create_executable(self, subject_configuration: SubjectConfiguration, state: State) -> ChromiumExecutable: return ChromiumExecutable(subject_configuration, state) - - def get_availability(self) -> dict: - most_recent_major_version = bughog_service.find_latest_major_version('chromium') - return {'name': 'chromium', 'min_version': 20, 'max_version': most_recent_major_version} diff --git a/bughog/subject/web_browser/firefox/state_oracle.py b/bughog/subject/web_browser/firefox/state_oracle.py index b049f88d..3056421e 100644 --- a/bughog/subject/web_browser/firefox/state_oracle.py +++ b/bughog/subject/web_browser/firefox/state_oracle.py @@ -18,6 +18,9 @@ def find_commit_id(self, commit_nb: int) -> str | None: def find_commit_of_release(self, release_version: int) -> tuple[int, str]: return bughog_service.find_version_commit('firefox', release_version) + def get_oldest_supported_release_version(self) -> int: + return 20 + def get_most_recent_major_release_version(self) -> int: return bughog_service.find_latest_major_version('firefox') diff --git a/bughog/subject/web_browser/firefox/subject.py b/bughog/subject/web_browser/firefox/subject.py index 33d8fca7..e6c9dcb3 100644 --- a/bughog/subject/web_browser/firefox/subject.py +++ b/bughog/subject/web_browser/firefox/subject.py @@ -2,7 +2,6 @@ from bughog.subject.state_oracle import StateOracle from bughog.subject.web_browser.firefox.executable import FirefoxExecutable from bughog.subject.web_browser.firefox.state_oracle import FirefoxStateOracle -from bughog.version_control.conversion import bughog_service from bughog.subject.web_browser.subject import WebBrowser from bughog.version_control.state.base import State @@ -18,11 +17,3 @@ def _state_oracle_class(self) -> type[StateOracle]: def create_executable(self, subject_configuration: SubjectConfiguration, state: State) -> FirefoxExecutable: return FirefoxExecutable(subject_configuration, state) - - def get_availability(self) -> dict: - max_version = bughog_service.find_latest_major_version('firefox') - return { - 'name': 'firefox', - 'min_version': 20, - 'max_version': max_version, - } From 68ed18819eae92895740d9c523b1aa44b297aa0d Mon Sep 17 00:00:00 2001 From: Gertjan Date: Wed, 18 Feb 2026 16:47:21 +0000 Subject: [PATCH 04/55] Add playground and refactor --- bughog/analysis/plot_factory.py | 12 +- bughog/database/mongo/mongodb.py | 68 +- bughog/distribution/worker_manager.py | 14 +- bughog/evaluation/evaluation.py | 12 +- bughog/evaluation/experiment_result.py | 10 + bughog/evaluation/experiments.py | 9 +- .../evaluation_configurations.py | 4 +- bughog/integration_tests/verify_results.py | 24 +- bughog/main.py | 49 +- bughog/parameters.py | 163 ++-- bughog/subject/factory.py | 25 +- bughog/subject/js_engine/subject.py | 4 +- bughog/subject/simulation.py | 4 +- bughog/subject/subject.py | 4 +- bughog/subject/wasm_runtime/subject.py | 6 +- .../web_browser/interaction/simulation.py | 23 +- bughog/subject/web_browser/subject.py | 4 +- bughog/version_control/state/base.py | 3 + bughog/web/blueprints/api.py | 33 +- bughog/web/clients.py | 22 +- bughog/web/evaluation_thread.py | 18 +- bughog/web/vue/package-lock.json | 22 + bughog/web/vue/package.json | 1 + bughog/web/vue/src/App.vue | 775 +----------------- bughog/web/vue/src/components/banner.vue | 83 ++ .../src/components/evaluation-controls.vue | 48 ++ ...ation_status.vue => evaluation-status.vue} | 0 .../src/components/experiment-controls.vue | 50 ++ bughog/web/vue/src/components/json-tree.vue | 124 +++ bughog/web/vue/src/components/poc-editor.vue | 29 +- .../src/components/subject-state-selector.vue | 103 +++ .../web/vue/src/composables/useServerInfo.js | 44 + bughog/web/vue/src/composables/useWebSocket | 169 ++++ bughog/web/vue/src/main.js | 11 +- bughog/web/vue/src/router/index.js | 22 + bughog/web/vue/src/views/Home.vue | 752 +++++++++++++++++ bughog/web/vue/src/views/Playground.vue | 337 ++++++++ bughog/worker.py | 8 +- 38 files changed, 2101 insertions(+), 988 deletions(-) create mode 100644 bughog/web/vue/src/components/banner.vue create mode 100644 bughog/web/vue/src/components/evaluation-controls.vue rename bughog/web/vue/src/components/{evaluation_status.vue => evaluation-status.vue} (100%) create mode 100644 bughog/web/vue/src/components/experiment-controls.vue create mode 100644 bughog/web/vue/src/components/json-tree.vue create mode 100644 bughog/web/vue/src/components/subject-state-selector.vue create mode 100644 bughog/web/vue/src/composables/useServerInfo.js create mode 100644 bughog/web/vue/src/composables/useWebSocket create mode 100644 bughog/web/vue/src/router/index.js create mode 100644 bughog/web/vue/src/views/Home.vue create mode 100644 bughog/web/vue/src/views/Playground.vue diff --git a/bughog/analysis/plot_factory.py b/bughog/analysis/plot_factory.py index cb23a87a..a137219d 100644 --- a/bughog/analysis/plot_factory.py +++ b/bughog/analysis/plot_factory.py @@ -13,7 +13,7 @@ class PlotFactory: @staticmethod def get_plot_commit_data(params: EvaluationParameters) -> dict: commit_docs = MongoDB().get_documents_for_plotting(params) - state_oracle = factory.get_subject_from_params(params).state_oracle + state_oracle = factory.get_subject_from_params(params.subject_configuration).state_oracle return PlotFactory.__add_outcome_info(commit_docs, state_oracle) @staticmethod @@ -24,7 +24,7 @@ def get_plot_release_data(params: EvaluationParameters) -> dict: @staticmethod def validate_params(params: EvaluationParameters) -> list[str]: missing_parameters = [] - if not params.evaluation_range.experiment_name: + if not params.experiment_name: missing_parameters.append('selected experiment') if not params.subject_configuration.subject_type: missing_parameters.append('subject_type') @@ -43,7 +43,7 @@ def __transform_to_bokeh_compatible(docs: list) -> dict: return new_docs @staticmethod - def __add_outcome_info(docs: list, state_oracle: StateOracle|None): + def __add_outcome_info(docs: list, state_oracle: StateOracle | None): if not docs: return {'commit_nb': [], 'major_version': [], 'version_printed_by_executable': [], 'outcome': []} @@ -63,7 +63,9 @@ def __add_outcome_info(docs: list, state_oracle: StateOracle|None): logger.error(f'Skipping state doc with unknown commit number (commit id: {commit_id}).') continue elif commit_id is None: - logger.error(f'Including state doc with unknown commit id (commit number: {commit_nb}), without supplying commit url.') + logger.error( + f'Including state doc with unknown commit id (commit number: {commit_nb}), without supplying commit url.' + ) commit_url = None else: if state_oracle: @@ -74,7 +76,7 @@ def __add_outcome_info(docs: list, state_oracle: StateOracle|None): new_doc = { 'commit_nb': commit_nb, 'commit_url': commit_url, - 'major_version': doc['state'].get('major_version', None), # commit states don't have this field + 'major_version': doc['state'].get('major_version', None), # commit states don't have this field 'version_printed_by_executable': doc['subject_version'], } if doc['dirty']: diff --git a/bughog/database/mongo/mongodb.py b/bughog/database/mongo/mongodb.py index ad0994be..e7bbe447 100644 --- a/bughog/database/mongo/mongodb.py +++ b/bughog/database/mongo/mongodb.py @@ -14,6 +14,7 @@ from bughog.parameters import ( DatabaseParameters, EvaluationParameters, + ExperimentParameters, SubjectConfiguration, ) from bughog.version_control.state.base import ShallowState, State @@ -139,13 +140,12 @@ def gridfs(self) -> GridFS: raise ServerException('Database server does not have a database') return GridFS(self._db) - def store_result(self, eval_params: EvaluationParameters, result: ExperimentResult): + def store_result(self, params: ExperimentParameters, result: ExperimentResult): """ Upserts the result. """ - subject_config = eval_params.subject_configuration - eval_params = eval_params - collection = self.__get_data_collection(eval_params) + subject_config = params.subject_configuration + collection = self.__get_data_collection(subject_config) query = { 'subject_version': result.executable_version, 'executable_origin': result.executable_origin, @@ -154,8 +154,8 @@ def store_result(self, eval_params: EvaluationParameters, result: ExperimentResu 'cli_options': subject_config.cli_options, 'extensions': subject_config.extensions, 'state': result.state, - 'project': eval_params.evaluation_range.project_name, - 'experiment': eval_params.evaluation_range.experiment_name, + 'project': params.project_name, + 'experiment': params.experiment_name, } # if browser_config.subject_name == 'firefox': # build_id = self.get_build_id_firefox(result.params.state) @@ -177,13 +177,13 @@ def store_result(self, eval_params: EvaluationParameters, result: ExperimentResu } collection.update_one(query, update, upsert=True) - def get_result(self, params: EvaluationParameters, state: ShallowState) -> Optional[ExperimentResult]: - collection = self.__get_data_collection(params) - query = self.__to_experiment_query(params, state) + def get_result(self, params: ExperimentParameters) -> Optional[ExperimentResult]: + collection = self.__get_data_collection(params.subject_configuration) + query = self.__to_experiment_query(params, params.state) doc = collection.find_one(query) if doc: return ExperimentResult( - doc['executable_version'], + doc['subject_version'], doc['executable_origin'], doc['state'], doc['result']['raw'], @@ -191,12 +191,12 @@ def get_result(self, params: EvaluationParameters, state: ShallowState) -> Optio doc['dirty'], ) else: - logger.error(f'Could not find document for query {query}.') + logger.info(f'Could not find document for query {query}.') return None - def has_result(self, params: EvaluationParameters, state: ShallowState) -> bool: - collection = self.__get_data_collection(params) - query = self.__to_experiment_query(params, state) + def has_result(self, params: ExperimentParameters) -> bool: + collection = self.__get_data_collection(params.subject_configuration) + query = self.__to_experiment_query(params, params.state) nb_of_documents = collection.count_documents(query) return nb_of_documents > 0 @@ -206,11 +206,11 @@ def get_evaluated_states( boundary_states: Optional[tuple[State, State]], dirty: Optional[bool] = None, ) -> list[State]: - collection = self.__get_data_collection(params) + collection = self.__get_data_collection(params.subject_configuration) query = { - 'project': params.evaluation_range.project_name, + 'project': params.project_name, 'subject_config': params.subject_configuration.subject_setting, - 'experiment': params.evaluation_range.experiment_name, + 'experiment': params.experiment_name, 'result': {'$exists': True}, 'state.type': 'release' if params.evaluation_range.only_release_commits else 'commit', } @@ -246,12 +246,12 @@ def get_evaluated_states( states.append(state) return states - def __to_experiment_query(self, params: EvaluationParameters, state: ShallowState) -> dict: + def __to_experiment_query(self, params: ExperimentParameters, state: ShallowState) -> dict: state_query = {'state.' + k: v for k, v in state.dict.items()} query = { - 'project': params.evaluation_range.project_name, + 'project': params.project_name, 'subject_config': params.subject_configuration.subject_setting, - 'experiment': params.evaluation_range.experiment_name, + 'experiment': params.experiment_name, } query.update(state_query) if len(params.subject_configuration.extensions) > 0: @@ -270,13 +270,11 @@ def __to_experiment_query(self, params: EvaluationParameters, state: ShallowStat query['cli_options'] = [] return query - def __get_data_collection(self, eval_params: EvaluationParameters) -> Collection: + def __get_data_collection(self, subject_config: SubjectConfiguration) -> Collection: """ Returns the data collection, of which the name is formatted as '{subject_type}_{subject_name}'. """ - collection_name = ( - f'{eval_params.subject_configuration.subject_type}_{eval_params.subject_configuration.subject_name}' - ) + collection_name = f'{subject_config.subject_type}_{subject_config.subject_name}' return self.get_collection(collection_name, create_if_not_found=True) def get_binary_availability_collection(self, subject_config: SubjectConfiguration) -> Collection: @@ -299,14 +297,14 @@ def get_stored_binary_availability(self, subject_config: SubjectConfiguration): return result def get_documents_for_plotting(self, params: EvaluationParameters, releases: bool = False) -> list: - collection = self.__get_data_collection(params) + collection = self.__get_data_collection(params.subject_configuration) evaluation_range = params.evaluation_range subject_config = params.subject_configuration query = { - 'project': evaluation_range.project_name, - 'experiment': evaluation_range.experiment_name, + 'project': params.project_name, + 'experiment': params.experiment_name, 'subject_config': subject_config.subject_setting, 'state.type': 'release' if releases else 'commit', 'extensions': {'$size': len(subject_config.extensions) if subject_config.extensions else 0}, @@ -343,22 +341,22 @@ def get_documents_for_plotting(self, params: EvaluationParameters, releases: boo ) return list(docs) - def remove_datapoint(self, params: EvaluationParameters, state: ShallowState) -> None: - collection = self.__get_data_collection(params) - query = self.__to_experiment_query(params, state) + def remove_datapoint(self, params: ExperimentParameters) -> None: + collection = self.__get_data_collection(params.subject_configuration) + query = self.__to_experiment_query(params, params.state) count = collection.delete_one(query) if count.deleted_count == 0: - logger.error(f'Could not remove datapoint for {state}.') + logger.error(f'Could not remove datapoint for {params.state}.') else: - logger.debug(f'Removed datapoint for {state}.') + logger.debug(f'Removed datapoint for {params.state}.') def remove_all_data_for(self, params_list: list[EvaluationParameters]) -> None: for params in params_list: - collection = self.__get_data_collection(params) + collection = self.__get_data_collection(params.subject_configuration) collection.delete_many( { - 'project': params.evaluation_range.project_name, - 'experiment': params.evaluation_range.experiment_name, + 'project': params.project_name, + 'experiment': params.experiment_name, } ) diff --git a/bughog/distribution/worker_manager.py b/bughog/distribution/worker_manager.py index 4a8869aa..8eaa217e 100644 --- a/bughog/distribution/worker_manager.py +++ b/bughog/distribution/worker_manager.py @@ -8,7 +8,7 @@ import docker.errors from bughog import configuration, worker -from bughog.parameters import EvaluationParameters +from bughog.parameters import ExperimentParameters from bughog.version_control.state.base import State from bughog.web.clients import Clients @@ -16,8 +16,8 @@ class WorkerManager: - def __init__(self, eval_params: EvaluationParameters) -> None: - self.max_nb_of_containers = eval_params.sequence_configuration.nb_of_containers + def __init__(self, subject_type: str, subject_name: str, max_nb_of_containers: int) -> None: + self.max_nb_of_containers = max_nb_of_containers if self.max_nb_of_containers == 1: logger.info('Running in single container mode') @@ -26,11 +26,9 @@ def __init__(self, eval_params: EvaluationParameters) -> None: for i in range(self.max_nb_of_containers): self.container_id_pool.put(i) self.client = docker.from_env() - subject_type = eval_params.subject_configuration.subject_type - subject_name = eval_params.subject_configuration.subject_name self.worker_image_ref = self.__get_worker_image_ref(subject_type, subject_name) - def start_experiment(self, params: EvaluationParameters, state: State, blocking_wait=True) -> None: + def start_experiment(self, params: ExperimentParameters, state: State, blocking_wait=True) -> None: if self.max_nb_of_containers != 1: return self.__run_container(params, state, blocking_wait) @@ -38,7 +36,7 @@ def start_experiment(self, params: EvaluationParameters, state: State, blocking_ worker.run(params, state) Clients.push_results_to_all() - def __run_container(self, params: EvaluationParameters, state: State, blocking_wait=True) -> None: + def __run_container(self, params: ExperimentParameters, state: State, blocking_wait=True) -> None: while blocking_wait and self.get_nb_of_running_worker_containers() >= self.max_nb_of_containers: time.sleep(1) container_id = self.container_id_pool.get() @@ -122,7 +120,7 @@ def start_container_thread(): thread.start() logger.info(f"Container '{container_name}' started experiments for '{state}'") # Sleep to avoid all workers downloading executables at once, clogging up all IO. - time.sleep(.1) + time.sleep(0.1) def get_nb_of_running_worker_containers(self): return len(self.get_runnning_containers()) diff --git a/bughog/evaluation/evaluation.py b/bughog/evaluation/evaluation.py index 7a335df1..e455ca88 100644 --- a/bughog/evaluation/evaluation.py +++ b/bughog/evaluation/evaluation.py @@ -5,7 +5,7 @@ from bughog.evaluation.collectors.collector import Collector from bughog.evaluation.experiment_result import ExperimentResult from bughog.evaluation.interaction import Interaction -from bughog.parameters import EvaluationParameters +from bughog.parameters import ExperimentParameters from bughog.subject import factory from bughog.subject.executable import Executable, ExecutableStatus from bughog.subject.simulation import Simulation @@ -20,16 +20,16 @@ def __init__(self, subject_type: str): self.experiments = factory.create_experiments(subject_type) self.should_stop = False - def evaluate(self, params: EvaluationParameters, state: State, is_worker=False): - if MongoDB().has_result(params, state.to_shallow_state()): + def evaluate(self, params: ExperimentParameters, state: State, is_worker=False): + if MongoDB().has_result(params): logger.warning( - f"Experiment '{params.evaluation_range.experiment_name}' for '{state}' was already performed, skipping." + f"Experiment '{params.experiment_name}' for '{params.state}' was already performed, skipping." ) return - subject = factory.get_subject_from_params(params) + subject = factory.get_subject_from_params(params.subject_configuration) - experiment_folder = self.experiments.get_experiment_folder(params) + experiment_folder = self.experiments.get_experiment_folder(params.project_name, params.experiment_name) executable = subject.create_executable(params.subject_configuration, state) runtime_flags = self.experiments.framework.get_runtime_flags(experiment_folder) runtime_env_vars = self.experiments.framework.get_runtime_env_vars(experiment_folder) diff --git a/bughog/evaluation/experiment_result.py b/bughog/evaluation/experiment_result.py index 706b6b6b..a64dab06 100644 --- a/bughog/evaluation/experiment_result.py +++ b/bughog/evaluation/experiment_result.py @@ -57,3 +57,13 @@ def padded_subject_version(self) -> Optional[str]: return None padded_version.append('0' * (padding_target - len(sub)) + sub) return '.'.join(padded_version) + + def to_dict(self) -> dict: + return { + 'executable_version': self.executable_version, + 'executable_origin': self.executable_origin, + 'state': self.state, + 'raw_results': self.raw_results, + 'result_variables': list(self.result_variables), + 'is_dirty': self.is_dirty, + } diff --git a/bughog/evaluation/experiments.py b/bughog/evaluation/experiments.py index 1016c509..72d13ea3 100644 --- a/bughog/evaluation/experiments.py +++ b/bughog/evaluation/experiments.py @@ -3,7 +3,6 @@ import shutil from bughog.evaluation.file_structure import Folder -from bughog.parameters import EvaluationParameters from bughog.subject.evaluation_framework import EvaluationFramework logger = logging.getLogger(__name__) @@ -104,7 +103,9 @@ def add_folder_or_file(self, project: str, poc: str, folder_name: str | None, fi poc_folder = project_folder.get_folder(poc) if folder_name is not None: if poc_folder.file_exists(folder_name): - raise Exception(f'Could not create {folder_name} in {poc_folder.path}, because a file with the same name exists.') + raise Exception( + f'Could not create {folder_name} in {poc_folder.path}, because a file with the same name exists.' + ) elif poc_folder.folder_exists(folder_name): folder = poc_folder.get_folder(folder_name) else: @@ -157,9 +158,7 @@ def __get_experiment_folder(self, project_name: str, experiment_name: str) -> Fo return experiment raise Exception(f"Could not find experiment '{experiment_name}'") - def get_experiment_folder(self, params: EvaluationParameters) -> Folder: - project_name = params.evaluation_range.project_name - experiment_name = params.evaluation_range.experiment_name + def get_experiment_folder(self, project_name: str, experiment_name: str) -> Folder: return self.__get_experiment_folder(project_name, experiment_name) def get_experiment_dir_tree(self, project_name: str, experiment_name: str) -> dict: diff --git a/bughog/integration_tests/evaluation_configurations.py b/bughog/integration_tests/evaluation_configurations.py index 1fe125d0..807506ec 100644 --- a/bughog/integration_tests/evaluation_configurations.py +++ b/bughog/integration_tests/evaluation_configurations.py @@ -26,8 +26,6 @@ def get_default_evaluation_range( ) -> EvaluationRange: min_version, max_version = factory.get_subject_availability(subject_type, subject_name) return EvaluationRange( - verify_results.TEST_PROJECT_NAME, - experiment, (min_version, max_version), None, only_releases, @@ -48,6 +46,8 @@ def get_default_evaluation_parameters( ) -> EvaluationParameters: database_params = configuration.get_database_params() return EvaluationParameters( + verify_results.TEST_PROJECT_NAME, + experiment, get_default_configuration(subject_type, subject_name), get_default_evaluation_range(subject_type, subject_name, experiment, only_releases), get_default_sequence_config(sequence_limit), diff --git a/bughog/integration_tests/verify_results.py b/bughog/integration_tests/verify_results.py index ea067914..05cc28fb 100644 --- a/bughog/integration_tests/verify_results.py +++ b/bughog/integration_tests/verify_results.py @@ -23,7 +23,7 @@ def get_all_testable_subject_types() -> Generator[str]: if TEST_PROJECT_NAME in all_experiments.get_projects(): yield subject_type else: - logger.warning(f'Skipping {subject_type} testing, because no "{TEST_PROJECT_NAME} was found.') + logger.warning(f'Skipping {subject_type} testing, because no "{TEST_PROJECT_NAME}" was found.') def verify_all() -> dict: @@ -43,25 +43,29 @@ def verify_all() -> dict: return grouped_results -def __verify_experiment(eval_parameters: EvaluationParameters, all_experiments: Experiments) -> dict | None: - experiment_name = eval_parameters.evaluation_range.experiment_name - experiment_folder = all_experiments.get_experiment_folder(eval_parameters) +def __verify_experiment(params: EvaluationParameters, all_experiments: Experiments) -> dict | None: + experiment_name = params.experiment_name + experiment_folder = all_experiments.get_experiment_folder(params.project_name, experiment_name) verification_func = __get_verification_function(all_experiments.framework, experiment_folder) if verification_func is None: return None - states = MongoDB().get_evaluated_states(eval_parameters, None) + states = MongoDB().get_evaluated_states(params, None) nb_of_success_results = len(list(filter(lambda x: verification_func(x), states))) - nb_of_fail_results = len(list(filter(lambda x: not verification_func(x) and not ExperimentResult.poc_is_dirty(x.result_variables), states))) + nb_of_fail_results = len( + list( + filter(lambda x: not verification_func(x) and not ExperimentResult.poc_is_dirty(x.result_variables), states) + ) + ) nb_of_error_results = len(list(filter(lambda x: ExperimentResult.poc_is_dirty(x.result_variables), states))) nb_of_results = nb_of_success_results + nb_of_fail_results + nb_of_error_results success_ratio = 0 if nb_of_results == 0 else round((nb_of_success_results / nb_of_results) * 100) return { 'experiment_name': experiment_name, - 'subject_type': eval_parameters.subject_configuration.subject_type, - 'subject_name': eval_parameters.subject_configuration.subject_name, + 'subject_type': params.subject_configuration.subject_type, + 'subject_name': params.subject_configuration.subject_name, 'nb_of_success_results': nb_of_success_results, 'nb_of_fail_results': nb_of_fail_results, 'nb_of_error_results': nb_of_error_results, @@ -85,14 +89,14 @@ def __get_verification_function(eval_framework: EvaluationFramework, experiment_ case _: if type(param_value) is str: reproducing_ranges = ast.literal_eval(param_value) - if type(reproducing_ranges) is list[tuple[int,int]]: + if type(reproducing_ranges) is list[tuple[int, int]]: return __create_complex_verification_function(reproducing_ranges) logger.warning(f'Skipping {experiment_folder.name}, because could not parse given "{param_name}".') return None -def __create_complex_verification_function(reproducing_ranges: list[tuple[int,int]]) -> Callable: +def __create_complex_verification_function(reproducing_ranges: list[tuple[int, int]]) -> Callable: def verification_function(state: State): for start, end in reproducing_ranges: if start <= state.index <= end: diff --git a/bughog/main.py b/bughog/main.py index 749df886..ff3151a6 100644 --- a/bughog/main.py +++ b/bughog/main.py @@ -11,13 +11,13 @@ from bughog.parameters import ( DatabaseParameters, EvaluationParameters, + ExperimentParameters, ) from bughog.search_strategy.bgb_search import BiggestGapBisectionSearch from bughog.search_strategy.bgb_sequence import BiggestGapBisectionSequence from bughog.search_strategy.composite_search import CompositeSearch from bughog.search_strategy.sequence_strategy import SequenceFinished, SequenceStrategy from bughog.subject import factory -from bughog.version_control.state.base import ShallowState from bughog.version_control.state_factory import StateFactory from bughog.web.clients import Clients @@ -52,7 +52,11 @@ def connect_to_database(self, db_connection_params: DatabaseParameters) -> None: def run(self, eval_params_list: list[EvaluationParameters]) -> None: # Sequence_configuration settings are the same over evaluation parameters (quick fix) self.__update_state(is_running=True, reason='user', status='running') - worker_manager = WorkerManager(eval_params_list[0]) + + subject_type = eval_params_list[0].subject_configuration.subject_type + subject_name = eval_params_list[0].subject_configuration.subject_name + nb_of_containers = eval_params_list[0].sequence_configuration.nb_of_containers + worker_manager = WorkerManager(subject_type, subject_name, nb_of_containers) self.stop_gracefully = False self.stop_forcefully = False try: @@ -60,7 +64,7 @@ def run(self, eval_params_list: list[EvaluationParameters]) -> None: for eval_params in eval_params_list: if self.stop_gracefully or self.stop_forcefully: break - self.__update_eval_queue(eval_params.evaluation_range.experiment_name, 'active') + self.__update_eval_queue(eval_params.experiment_name, 'active') self.__update_state( is_running=True, reason='user', @@ -111,8 +115,8 @@ def run_single_evaluation(self, eval_params: EvaluationParameters, worker_manage nb_of_iterations = 3 for i in range(1, nb_of_iterations + 1): start_time = time.time() - subject = factory.get_subject_from_params(eval_params) - experiment_name = eval_params.evaluation_range.experiment_name + subject = factory.get_subject_from_params(eval_params.subject_configuration) + experiment_name = eval_params.experiment_name search_strategy = self.create_sequence_strategy(eval_params) logger.info( @@ -125,7 +129,8 @@ def run_single_evaluation(self, eval_params: EvaluationParameters, worker_manage current_state = search_strategy.next(wait=False) # Start worker to perform evaluation - worker_manager.start_experiment(eval_params, current_state) + experiment_params = eval_params.to_experiment_parameters(current_state.to_shallow_state()) + worker_manager.start_experiment(experiment_params, current_state) except SequenceFinished: worker_manager.wait_until_all_evaluations_are_done() @@ -141,7 +146,7 @@ def run_single_evaluation(self, eval_params: EvaluationParameters, worker_manage worker_manager.wait_until_all_evaluations_are_done() self.state['reason'] = 'finished' - self.__update_eval_queue(eval_params.evaluation_range.experiment_name, 'done') + self.__update_eval_queue(eval_params.experiment_name, 'done') Clients.push_notification_to_all(f'Evaluation of {experiment_name} has finished.') def retry_dirty_tests(self, eval_params: EvaluationParameters, worker_manager: WorkerManager) -> None: @@ -150,7 +155,7 @@ def retry_dirty_tests(self, eval_params: EvaluationParameters, worker_manager: W logger.info('No tests are associated with a dirty result.') return - experiment = eval_params.evaluation_range.experiment_name + experiment = eval_params.experiment_name message = f'Retrying {nb_of_dirty_states} tests with a dirty result for {experiment}.' logger.info(message) @@ -158,19 +163,35 @@ def retry_dirty_tests(self, eval_params: EvaluationParameters, worker_manager: W for dirty_state in dirty_states: if self.stop_gracefully or self.stop_forcefully: return - MongoDB().remove_datapoint(eval_params, dirty_state.to_shallow_state()) - worker_manager.start_experiment(eval_params, dirty_state) + experiment_params = eval_params.to_experiment_parameters(dirty_state.to_shallow_state()) + MongoDB().remove_datapoint(experiment_params) + worker_manager.start_experiment(experiment_params, dirty_state) worker_manager.wait_until_all_evaluations_are_done() dirty_states_after_retry = MongoDB().get_evaluated_states(eval_params, None, dirty=True) logger.info(f'Dirty test results reduced from {nb_of_dirty_states} to {len(dirty_states_after_retry)}.') + def run_single_experiment(self, params: ExperimentParameters) -> None: + try: + self.__update_state(is_running=True, reason='user', status='running') + subject_config = params.subject_configuration + state = params.state.to_deep_state(subject_config.subject_type, subject_config.subject_name) + if not state.has_available_executable(): + Clients.push_notification_to_all(f'No available executable for state {state.commit_nb}.', type='error') + else: + worker_manager = WorkerManager(subject_config.subject_type, subject_config.subject_name, 1) + worker_manager.start_experiment(params, state) + Clients.push_complete_experiment_result(params) + finally: + # TODO: error handling + self.__update_state(is_running=False, reason='idle', status='running') + @staticmethod def create_sequence_strategy(eval_params: EvaluationParameters) -> SequenceStrategy: sequence_config = eval_params.sequence_configuration search_strategy = sequence_config.search_strategy sequence_limit = sequence_config.sequence_limit - subject = factory.get_subject_from_params(eval_params) + subject = factory.get_subject_from_params(eval_params.subject_configuration) state_factory = StateFactory(subject.state_oracle, eval_params) if search_strategy == 'bgb_sequence': @@ -229,8 +250,8 @@ def push_info(self, ws, *args) -> None: update['state'] = self.state Clients.push_info(ws, update) - def remove_datapoint(self, params: EvaluationParameters, state: ShallowState) -> None: - MongoDB().remove_datapoint(params, state) + def remove_datapoint(self, params: ExperimentParameters) -> None: + MongoDB().remove_datapoint(params) Clients.push_results_to_all() def remove_cached_executable(self, subject_type: str, subject_name: str, state_name: str) -> None: @@ -246,7 +267,7 @@ def __init_eval_queue(self, eval_params_list: list[EvaluationParameters]) -> Non for eval_params in eval_params_list: self.eval_queue.append( { - 'experiment': eval_params.evaluation_range.experiment_name, + 'experiment': eval_params.experiment_name, 'state': 'pending', } ) diff --git a/bughog/parameters.py b/bughog/parameters.py index 54a3a7b4..4832f527 100644 --- a/bughog/parameters.py +++ b/bughog/parameters.py @@ -7,19 +7,21 @@ from typing import Optional from bughog.exceptions import MissingParametersError +from bughog.version_control.state.base import ShallowState logger = logging.getLogger(__name__) @dataclass(frozen=True) -class EvaluationParameters: +class ExperimentParameters: """ - All parameters required to define an evaluation. + All parameters required to define an experiment. """ + project_name: str + experiment_name: str subject_configuration: SubjectConfiguration - evaluation_range: EvaluationRange - sequence_configuration: SequenceConfiguration + state: ShallowState database_params: DatabaseParameters def serialize(self) -> str: @@ -27,12 +29,33 @@ def serialize(self) -> str: return base64.b64encode(pickled_bytes).decode('ascii') @staticmethod - def deserialize(pickled_str: str) -> EvaluationParameters: + def deserialize(pickled_str: str) -> ExperimentParameters: pickled_bytes = base64.b64decode(pickled_str) return pickle.loads(pickled_bytes) + +@dataclass(frozen=True) +class EvaluationParameters: + """ + All parameters required to define an evaluation. + """ + + project_name: str + experiment_name: str + subject_configuration: SubjectConfiguration + evaluation_range: EvaluationRange + sequence_configuration: SequenceConfiguration + database_params: DatabaseParameters + + def to_experiment_parameters(self, state: ShallowState) -> ExperimentParameters: + return ExperimentParameters( + self.project_name, self.experiment_name, self.subject_configuration, state, self.database_params + ) + def to_plot_parameters(self, experiment_name: str, dirty_results_allowed: bool = True) -> PlotParameters: return PlotParameters( + self.project_name, + experiment_name, self.subject_configuration, self.evaluation_range, self.sequence_configuration, @@ -56,14 +79,16 @@ def to_dict(self) -> dict: @staticmethod def from_dict(data: dict) -> SubjectConfiguration: return SubjectConfiguration( - data['subject_type'], data['subject_name'], data['subject_setting'], data['cli_options'], data['extensions'] + data['subject_type'], + data['subject_name'], + data.get('subject_setting', 'default'), + data.get('cli_options', []), + data.get('extensions', []), ) @dataclass(frozen=True) class EvaluationRange: - project_name: str - experiment_name: str major_version_range: tuple[int, int] | None = None commit_nb_range: tuple[int, int] | None = None only_release_commits: bool = False @@ -76,6 +101,31 @@ def __post_init__(self): else: raise AttributeError('Evaluation ranges require either major versions or commit numbers') + @staticmethod + def from_dict(data: dict) -> EvaluationRange: + return EvaluationRange( + EvaluationRange.__get_version_range(data), + EvaluationRange.__get_commit_nb_range(data), + data.get('only_release_commits', False), + ) + + @staticmethod + def __get_version_range(form_data: dict[str, str]) -> tuple[int, int] | None: + if range := form_data.get('version_range', None): + if len(range) == 2: + return (int(range[0]), int(range[1])) + return None + + @staticmethod + def __get_commit_nb_range(form_data: dict[str, str]) -> tuple[int, int] | None: + lower_rev_number = form_data.get('lower_commit_nb', None) + upper_rev_number = form_data.get('upper_commit_nb', None) + lower_rev_number = int(lower_rev_number) if lower_rev_number else None + upper_rev_number = int(upper_rev_number) if upper_rev_number else None + if lower_rev_number is None or upper_rev_number is None: + return None + return (lower_rev_number, upper_rev_number) if lower_rev_number is not None else None + @dataclass(frozen=True) class SequenceConfiguration: @@ -83,6 +133,14 @@ class SequenceConfiguration: sequence_limit: int = 10000 search_strategy: str | None = None + @staticmethod + def from_dict(data: dict) -> SequenceConfiguration: + return SequenceConfiguration( + int(data.get('nb_of_containers', 1)), + int(data.get('sequence_limit', 50)), + data.get('search_strategy', None), + ) + @dataclass(frozen=True) class DatabaseParameters: @@ -116,42 +174,10 @@ def __repr__(self) -> str: class PlotParameters(EvaluationParameters): experiment: Optional[str] dirty_results_allowed: bool - # subject_name: Optional[str] - # database_collection: Optional[str] - # major_version_range: Optional[tuple[int, int]] = None - # revision_number_range: Optional[tuple[int, int]] = None - # subject_config: str = 'default' - # cli_options: Optional[list[str]] = None - # dirty_allowed: bool = True - - # @staticmethod - # def from_dict(data: dict) -> PlotParameters: - # if data.get('lower_version', None) and data.get('upper_version', None): - # major_version_range = (data['lower_version'], data['upper_version']) - # else: - # major_version_range = None - # if data.get('lower_revision_nb', None) and data.get('upper_revision_nb', None): - # revision_number_range = ( - # data['lower_revision_nb'], - # data['upper_revision_nb'], - # ) - # else: - # revision_number_range = None - # return PlotParameters( - # data.get('plot_experiment', None), - # data.get('target_mech_id', None), - # data.get('subject_name', None), - # data.get('db_collection', None), - # major_version_range=major_version_range, - # revision_number_range=revision_number_range, - # subject_config=data.get('subject_setting', 'default'), - # cli_options=data.get('cli_options', []), - # dirty_allowed=data.get('dirty_allowed', True), - # ) @staticmethod -def evaluation_factory( +def create_evaluation_params( kwargs: dict, database_params: DatabaseParameters, only_to_plot=False ) -> list[EvaluationParameters]: experiments = set(x for x in kwargs.get('experiments', []) + [kwargs.get('experiment_to_plot')] if x is not None) @@ -159,23 +185,15 @@ def evaluation_factory( raise MissingParametersError() subject_configuration = SubjectConfiguration.from_dict(kwargs) - sequence_configuration = SequenceConfiguration( - int(kwargs.get('nb_of_containers', 1)), - int(kwargs.get('sequence_limit', 50)), - kwargs.get('search_strategy'), - ) + sequence_configuration = SequenceConfiguration.from_dict(kwargs) evaluation_params_list = [] for experiment in sorted(experiments): if only_to_plot and experiment != kwargs.get('experiment_to_plot'): continue - evaluation_range = EvaluationRange( + evaluation_range = EvaluationRange.from_dict(kwargs) + evaluation_params = EvaluationParameters( kwargs['project_name'], experiment, - __get_version_range(kwargs), - __get_commit_nb_range(kwargs), - kwargs.get('only_release_commits', False), - ) - evaluation_params = EvaluationParameters( subject_configuration, evaluation_range, sequence_configuration, @@ -185,6 +203,28 @@ def evaluation_factory( return evaluation_params_list +@staticmethod +def create_experiment_params(kwargs: dict, database_params: DatabaseParameters) -> ExperimentParameters: + subject_configuration = SubjectConfiguration.from_dict(kwargs) + if 'major_version' in kwargs: + state_type = 'version' + elif 'commit_nb' in kwargs or 'commit_id' in kwargs: + state_type = 'commit' + else: + raise MissingParametersError( + 'Experiment parameters require either a major version, commit number, or commit id.' + ) + state = ShallowState( + state_type, + kwargs.get('major_version'), + kwargs.get('commit_nb'), + kwargs.get('commit_id'), + ) + return ExperimentParameters( + kwargs['project_name'], kwargs['poc_name'], subject_configuration, state, database_params + ) + + @staticmethod def __get_cookie_name(form_data: dict[str, str]) -> str | None: if form_data['check_for'] == 'request': @@ -192,22 +232,3 @@ def __get_cookie_name(form_data: dict[str, str]) -> str | None: if 'cookie_name' in form_data: return form_data['cookie_name'] return 'generic' - - -@staticmethod -def __get_version_range(form_data: dict[str, str]) -> tuple[int, int] | None: - if range := form_data.get('version_range', None): - if len(range) == 2: - return (int(range[0]), int(range[1])) - return None - - -@staticmethod -def __get_commit_nb_range(form_data: dict[str, str]) -> tuple[int, int] | None: - lower_rev_number = form_data.get('lower_commit_nb', None) - upper_rev_number = form_data.get('upper_commit_nb', None) - lower_rev_number = int(lower_rev_number) if lower_rev_number else None - upper_rev_number = int(upper_rev_number) if upper_rev_number else None - if lower_rev_number is None or upper_rev_number is None: - return None - return (lower_rev_number, upper_rev_number) if lower_rev_number is not None else None diff --git a/bughog/subject/factory.py b/bughog/subject/factory.py index 99f40412..90270a00 100644 --- a/bughog/subject/factory.py +++ b/bughog/subject/factory.py @@ -2,7 +2,7 @@ from functools import lru_cache from bughog.evaluation.experiments import Experiments -from bughog.parameters import EvaluationParameters +from bughog.parameters import SubjectConfiguration from bughog.subject.evaluation_framework import EvaluationFramework from bughog.subject.js_engine.evaluation_framework import JSEngineEvaluationFramework from bughog.subject.js_engine.v8.subject import V8Subject @@ -17,19 +17,8 @@ from bughog.subject.web_browser.firefox.subject import Firefox subjects = { - 'js_engine': { - 'evaluation_framework': JSEngineEvaluationFramework, - 'subjects': [ - V8Subject(), - V8SandboxSubject() - ] - }, - 'wasm_runtime': { - 'evaluation_framework': WasmRuntimeEvaluationFramework, - 'subjects': [ - WasmtimeSubject() - ] - }, + 'js_engine': {'evaluation_framework': JSEngineEvaluationFramework, 'subjects': [V8Subject(), V8SandboxSubject()]}, + 'wasm_runtime': {'evaluation_framework': WasmRuntimeEvaluationFramework, 'subjects': [WasmtimeSubject()]}, 'web_browser': { 'evaluation_framework': BrowserEvaluationFramework, 'subjects': [ @@ -86,15 +75,15 @@ def get_all_subject_availability() -> list[dict]: @staticmethod -def get_subject_availability(subject_type: str, subject_name: str) -> tuple[int,int]: +def get_subject_availability(subject_type: str, subject_name: str) -> tuple[int, int]: subject_availability = get_subject(subject_type, subject_name).get_availability() return subject_availability['min_version'], subject_availability['max_version'] @staticmethod -def get_subject_from_params(params: EvaluationParameters) -> Subject: - subject_type = params.subject_configuration.subject_type - subject_name = params.subject_configuration.subject_name +def get_subject_from_params(config: SubjectConfiguration) -> Subject: + subject_type = config.subject_type + subject_name = config.subject_name return get_subject(subject_type, subject_name) diff --git a/bughog/subject/js_engine/subject.py b/bughog/subject/js_engine/subject.py index b2ee67ab..1326fd5e 100644 --- a/bughog/subject/js_engine/subject.py +++ b/bughog/subject/js_engine/subject.py @@ -1,7 +1,7 @@ from bughog.evaluation.collectors.collector import Collector from bughog.evaluation.collectors.logs import LogCollector from bughog.evaluation.file_structure import Folder -from bughog.parameters import EvaluationParameters +from bughog.parameters import ExperimentParameters from bughog.subject.executable import Executable from bughog.subject.js_engine.simulation import JSEngineSimulation from bughog.subject.subject import Subject @@ -13,7 +13,7 @@ def type(self) -> str: return 'js_engine' @staticmethod - def create_simulation(executable: Executable, context: Folder, params: EvaluationParameters) -> JSEngineSimulation: + def create_simulation(executable: Executable, context: Folder, params: ExperimentParameters) -> JSEngineSimulation: return JSEngineSimulation(executable, context, params) @staticmethod diff --git a/bughog/subject/simulation.py b/bughog/subject/simulation.py index c9822393..e6a9cdfa 100644 --- a/bughog/subject/simulation.py +++ b/bughog/subject/simulation.py @@ -2,12 +2,12 @@ from abc import ABC, abstractmethod from bughog.evaluation.file_structure import Folder -from bughog.parameters import EvaluationParameters +from bughog.parameters import ExperimentParameters from bughog.subject.executable import Executable class Simulation(ABC): - def __init__(self, executable: Executable, context: Folder, params: EvaluationParameters) -> None: + def __init__(self, executable: Executable, context: Folder, params: ExperimentParameters) -> None: self.executable = executable self.context = context self.params = params diff --git a/bughog/subject/subject.py b/bughog/subject/subject.py index 0c24dc9f..b7709a9e 100644 --- a/bughog/subject/subject.py +++ b/bughog/subject/subject.py @@ -12,7 +12,7 @@ from bughog.evaluation.collectors.collector import Collector from bughog.evaluation.file_structure import Folder -from bughog.parameters import EvaluationParameters, SubjectConfiguration +from bughog.parameters import ExperimentParameters, SubjectConfiguration from bughog.subject.executable import Executable from bughog.subject.simulation import Simulation from bughog.subject.state_oracle import StateOracle @@ -61,7 +61,7 @@ def create_executable(self, subject_configuration: SubjectConfiguration, state: @staticmethod @abstractmethod - def create_simulation(executable: Executable, context: Folder, params: EvaluationParameters) -> Simulation: + def create_simulation(executable: Executable, context: Folder, params: ExperimentParameters) -> Simulation: """ Creates and returns the simulation object based on the given executable, experiment context and eval params. """ diff --git a/bughog/subject/wasm_runtime/subject.py b/bughog/subject/wasm_runtime/subject.py index 6229fb8a..1147a8b7 100644 --- a/bughog/subject/wasm_runtime/subject.py +++ b/bughog/subject/wasm_runtime/subject.py @@ -1,7 +1,7 @@ from bughog.evaluation.collectors.collector import Collector from bughog.evaluation.collectors.logs import LogCollector from bughog.evaluation.file_structure import Folder -from bughog.parameters import EvaluationParameters +from bughog.parameters import ExperimentParameters from bughog.subject.executable import Executable from bughog.subject.subject import Subject from bughog.subject.wasm_runtime.simulation import WasmRuntimeSimulation @@ -13,7 +13,9 @@ def type(self) -> str: return 'wasm_runtime' @staticmethod - def create_simulation(executable: Executable, context: Folder, params: EvaluationParameters) -> WasmRuntimeSimulation: + def create_simulation( + executable: Executable, context: Folder, params: ExperimentParameters + ) -> WasmRuntimeSimulation: return WasmRuntimeSimulation(executable, context, params) @staticmethod diff --git a/bughog/subject/web_browser/interaction/simulation.py b/bughog/subject/web_browser/interaction/simulation.py index 4f611e93..129c87a7 100644 --- a/bughog/subject/web_browser/interaction/simulation.py +++ b/bughog/subject/web_browser/interaction/simulation.py @@ -5,15 +5,17 @@ from pyvirtualdisplay.display import Display from bughog.evaluation.file_structure import Folder -from bughog.parameters import EvaluationParameters +from bughog.parameters import ExperimentParameters from bughog.subject.simulation import Simulation from bughog.subject.web_browser.executable import BrowserExecutable # TODO: all pyautogui are imported inside functions because the import needs DISPLAY var, while not all containers need and have that. + class BrowserSimulation(Simulation): - def __init__(self, executable: BrowserExecutable, folder: Folder, params: EvaluationParameters): + def __init__(self, executable: BrowserExecutable, folder: Folder, params: ExperimentParameters): import pyautogui as gui + super().__init__(executable, folder, params) disp = Display(visible=True, size=(1920, 1080), backend='xvfb', use_xauth=True) disp.start() @@ -72,6 +74,7 @@ def new_tab(self, url: str): def click_position(self, x: str, y: str): import pyautogui as gui + max_x, max_y = gui.size() gui.moveTo(self.parse_position(x, max_x), self.parse_position(y, max_y)) @@ -79,36 +82,46 @@ def click_position(self, x: str, y: str): def click(self, el_id: str): import pyautogui as gui + el_image_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), f'elements/{el_id}.png') x, y = gui.locateCenterOnScreen(el_image_path) self.click_position(str(x), str(y)) def write(self, text: str): import pyautogui as gui + gui.write(text, interval=0.1) def press(self, key: str): import pyautogui as gui + gui.press(key) def hold(self, key: str): import pyautogui as gui + gui.keyDown(key) def release(self, key: str): import pyautogui as gui + gui.keyUp(key) def hotkey(self, *keys: str): import pyautogui as gui + gui.hotkey(*keys) def screenshot(self, filename: str): import pyautogui as gui - project_name = self.params.evaluation_range.project_name - experiment_name = self.params.evaluation_range.experiment_name + + project_name = self.params.project_name + experiment_name = self.params.experiment_name executable_name = f'{self.executable.executable_name}-{self.executable.version}' - file_path = os.path.join('/app/logs/screenshots/', f'{project_name}-{experiment_name}-{self.executable.state.name}-{executable_name}.jpg') + file_path = os.path.join( + '/app/logs/screenshots/', + f'{project_name}-{experiment_name}-{self.executable.state.name}-{executable_name}.jpg', + ) gui.screenshot(file_path) def reproduced(self): diff --git a/bughog/subject/web_browser/subject.py b/bughog/subject/web_browser/subject.py index 42ece35d..5390112d 100644 --- a/bughog/subject/web_browser/subject.py +++ b/bughog/subject/web_browser/subject.py @@ -4,7 +4,7 @@ from bughog.evaluation.collectors.logs import LogCollector from bughog.evaluation.collectors.requests import RequestCollector from bughog.evaluation.file_structure import Folder -from bughog.parameters import EvaluationParameters +from bughog.parameters import ExperimentParameters from bughog.subject.executable import Executable from bughog.subject.subject import Subject from bughog.subject.web_browser.interaction.simulation import BrowserSimulation @@ -19,7 +19,7 @@ def type(self): return 'web_browser' @staticmethod - def create_simulation(executable: Executable, context: Folder, params: EvaluationParameters) -> BrowserSimulation: + def create_simulation(executable: Executable, context: Folder, params: ExperimentParameters) -> BrowserSimulation: return BrowserSimulation(executable, context, params) @staticmethod diff --git a/bughog/version_control/state/base.py b/bughog/version_control/state/base.py index e3711d81..3cef12fa 100644 --- a/bughog/version_control/state/base.py +++ b/bughog/version_control/state/base.py @@ -27,6 +27,9 @@ def dict(self) -> dict: } return {k: v for k, v in fields.items() if v is not None} + def to_deep_state(self, subject_type: str, subject_name: str) -> State: + return State.from_dict(subject_type, subject_name, self.dict) + class State(ABC): def __init__(self, oracle: StateOracle): diff --git a/bughog/web/blueprints/api.py b/bughog/web/blueprints/api.py index 8c431ff8..170449ab 100644 --- a/bughog/web/blueprints/api.py +++ b/bughog/web/blueprints/api.py @@ -13,9 +13,8 @@ from bughog.parameters import MissingParametersError from bughog.subject import factory from bughog.subject.factory import get_all_subject_availability -from bughog.version_control.state.base import ShallowState from bughog.web.clients import Clients -from bughog.web.evaluation_thread import run_eval_thread +from bughog.web.evaluation_thread import run_eval_thread, run_experiment_thread logger = logging.getLogger(__name__) api = Blueprint('api', __name__, url_prefix='/api') @@ -59,7 +58,7 @@ def start_evaluation(): data = request.json.copy() try: database_params = configuration.get_database_params() - params = application_logic.evaluation_factory(data, database_params) + params = application_logic.create_evaluation_params(data, database_params) run_eval_thread(__get_main(), params) return {'status': 'OK'} except MissingParametersError: @@ -80,6 +79,22 @@ def stop_evaluation(): return {'status': 'OK'} +@api.route('/experiment/start/', methods=['POST']) +def start_experiment(): + if request.json is None: + return {'status': 'NOK', 'msg': 'No experiment parameters found'} + + data = request.json.copy() + try: + database_params = configuration.get_database_params() + params = application_logic.create_experiment_params(data, database_params) + __get_main().remove_datapoint(params) + run_experiment_thread(__get_main(), params) + return {'status': 'OK'} + except MissingParametersError: + return {'status': 'NOK', 'msg': 'Could not start experiment due to missing parameters.'} + + """ Requesting information """ @@ -100,6 +115,9 @@ def init_websocket(ws): Clients.associate_params(ws, params) if requested_variables := message.get('get', []): __get_main().push_info(ws, *requested_variables) + if params_dict := message.get('request_experiment_result', None): + params = application_logic.create_experiment_params(params_dict, configuration.get_database_params()) + Clients.push_complete_experiment_result(params) except ValueError: logger.warning('Ignoring invalid message from client.') @@ -229,15 +247,12 @@ def remove_datapoint(): data = request.json.copy() if not isinstance(data, dict): return {'status': 'NOK', 'msg': 'Received dataformat is not a dictionary.'} - if (type := data.get('type')) not in ['release', 'commit']: + if (data.get('type')) not in ['release', 'commit']: return {'status': 'NOK', 'msg': 'Type argument should be release or commit.'} database_params = configuration.get_database_params() try: - params_list = application_logic.evaluation_factory(data, database_params, only_to_plot=True) - if len(params_list) < 1: - return {'status': 'NOK', 'msg': 'Could not construct removal parameters.'} - state = ShallowState(type, data.get('major_version'), data.get('commit_nb'), data.get('commit_id')) - __get_main().remove_datapoint(params_list[0], state) + params = application_logic.create_experiment_params(data, database_params) + __get_main().remove_datapoint(params) except MissingParametersError: return {'status': 'NOK', 'msg': 'Could not remove datapoint due to missing parameters'} return {'status': 'OK'} diff --git a/bughog/web/clients.py b/bughog/web/clients.py index eddbc566..4e37b5e8 100644 --- a/bughog/web/clients.py +++ b/bughog/web/clients.py @@ -7,7 +7,8 @@ from bughog import configuration from bughog.analysis.plot_factory import PlotFactory -from bughog.parameters import MissingParametersError, evaluation_factory +from bughog.database.mongo.mongodb import MongoDB +from bughog.parameters import ExperimentParameters, MissingParametersError, create_evaluation_params from bughog.subject import factory logger = logging.getLogger(__name__) @@ -62,7 +63,7 @@ def push_results(ws_client: Server): return params['experiments'] = [params['experiment_to_plot']] try: - eval_params = evaluation_factory(params, configuration.get_database_params()) + eval_params = create_evaluation_params(params, configuration.get_database_params()) if len(eval_params) < 1: return plot_params = eval_params[0].to_plot_parameters(params['experiment_to_plot']) @@ -131,3 +132,20 @@ def push_notification_to_all(message: str, type: Literal['info', 'error'] = 'inf Clients.__remove_disconnected_clients() for ws_client in Clients.__clients.keys(): ws_client.send(json.dumps({'notification': {'message': message, 'type': type}})) + + @staticmethod + def push_complete_experiment_result(params: ExperimentParameters) -> None: + Clients.__remove_disconnected_clients() + + if params is None: + logger.error('Could not find any associated parameters for this client.') + return + result = MongoDB().get_result(params) + + if result is None: + data = json.dumps({'update': {'experiment_result': None}}) + else: + data = json.dumps({'update': {'experiment_result': result.to_dict()}}) + + for ws_client in Clients.__clients.keys(): + ws_client.send(data) diff --git a/bughog/web/evaluation_thread.py b/bughog/web/evaluation_thread.py index 3c017efe..487424d9 100644 --- a/bughog/web/evaluation_thread.py +++ b/bughog/web/evaluation_thread.py @@ -2,7 +2,7 @@ import threading from bughog.main import Main -from bughog.parameters import EvaluationParameters +from bughog.parameters import EvaluationParameters, ExperimentParameters from bughog.web.clients import Clients logger = logging.getLogger(__name__) @@ -24,3 +24,19 @@ def thread_wrapper(): else: THREAD = threading.Thread(target=thread_wrapper) THREAD.start() + + +def run_experiment_thread(main: Main, experiment_params: ExperimentParameters): + global THREAD + + def thread_wrapper(): + try: + main.run_single_experiment(experiment_params) + except Exception as e: + Clients.push_notification_to_all(str(e), type='error') + + if THREAD and THREAD.is_alive(): + Clients.push_notification_to_all('Experiment thread is already running.', type='error') + else: + THREAD = threading.Thread(target=thread_wrapper) + THREAD.start() diff --git a/bughog/web/vue/package-lock.json b/bughog/web/vue/package-lock.json index e370e2ff..9a05c351 100644 --- a/bughog/web/vue/package-lock.json +++ b/bughog/web/vue/package-lock.json @@ -16,6 +16,7 @@ "oh-vue-icons": "^1.0.0-rc3", "vue": "^3.2.47", "vue-multiselect": "^2.1.9", + "vue-router": "^4.6.4", "vue3-toastify": "^0.2.8" }, "devDependencies": { @@ -615,6 +616,12 @@ "@vue/shared": "3.5.27" } }, + "node_modules/@vue/devtools-api": { + "version": "6.6.4", + "resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-6.6.4.tgz", + "integrity": "sha512-sGhTPMuXqZ1rVOk32RylztWkfXTRhuS7vgAKv0zjqk8gbsHkJ7xfFf+jbySxt7tWObEJwyKaHMikV/WGDiQm8g==", + "license": "MIT" + }, "node_modules/@vue/reactivity": { "version": "3.5.27", "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.27.tgz", @@ -2288,6 +2295,21 @@ "npm": ">= 3.0.0" } }, + "node_modules/vue-router": { + "version": "4.6.4", + "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-4.6.4.tgz", + "integrity": "sha512-Hz9q5sa33Yhduglwz6g9skT8OBPii+4bFn88w6J+J4MfEo4KRRpmiNG/hHHkdbRFlLBOqxN8y8gf2Fb0MTUgVg==", + "license": "MIT", + "dependencies": { + "@vue/devtools-api": "^6.6.4" + }, + "funding": { + "url": "https://github.com/sponsors/posva" + }, + "peerDependencies": { + "vue": "^3.5.0" + } + }, "node_modules/vue3-toastify": { "version": "0.2.8", "resolved": "https://registry.npmjs.org/vue3-toastify/-/vue3-toastify-0.2.8.tgz", diff --git a/bughog/web/vue/package.json b/bughog/web/vue/package.json index 6ae3ab9e..d0813380 100644 --- a/bughog/web/vue/package.json +++ b/bughog/web/vue/package.json @@ -17,6 +17,7 @@ "oh-vue-icons": "^1.0.0-rc3", "vue": "^3.2.47", "vue-multiselect": "^2.1.9", + "vue-router": "^4.6.4", "vue3-toastify": "^0.2.8" }, "devDependencies": { diff --git a/bughog/web/vue/src/App.vue b/bughog/web/vue/src/App.vue index 1d9e812e..b8e06903 100644 --- a/bughog/web/vue/src/App.vue +++ b/bughog/web/vue/src/App.vue @@ -1,773 +1,12 @@ - - - + diff --git a/bughog/web/vue/src/components/banner.vue b/bughog/web/vue/src/components/banner.vue new file mode 100644 index 00000000..e64404b6 --- /dev/null +++ b/bughog/web/vue/src/components/banner.vue @@ -0,0 +1,83 @@ + + + diff --git a/bughog/web/vue/src/components/evaluation-controls.vue b/bughog/web/vue/src/components/evaluation-controls.vue new file mode 100644 index 00000000..db1f112b --- /dev/null +++ b/bughog/web/vue/src/components/evaluation-controls.vue @@ -0,0 +1,48 @@ + + + diff --git a/bughog/web/vue/src/components/evaluation_status.vue b/bughog/web/vue/src/components/evaluation-status.vue similarity index 100% rename from bughog/web/vue/src/components/evaluation_status.vue rename to bughog/web/vue/src/components/evaluation-status.vue diff --git a/bughog/web/vue/src/components/experiment-controls.vue b/bughog/web/vue/src/components/experiment-controls.vue new file mode 100644 index 00000000..78139062 --- /dev/null +++ b/bughog/web/vue/src/components/experiment-controls.vue @@ -0,0 +1,50 @@ + + + diff --git a/bughog/web/vue/src/components/json-tree.vue b/bughog/web/vue/src/components/json-tree.vue new file mode 100644 index 00000000..4b85b7ab --- /dev/null +++ b/bughog/web/vue/src/components/json-tree.vue @@ -0,0 +1,124 @@ + + + diff --git a/bughog/web/vue/src/components/poc-editor.vue b/bughog/web/vue/src/components/poc-editor.vue index d8a06bb7..f2ccb737 100644 --- a/bughog/web/vue/src/components/poc-editor.vue +++ b/bughog/web/vue/src/components/poc-editor.vue @@ -93,14 +93,16 @@ import { getMode as getInteractionScriptMode } from '../interaction_script_mode' this.active_folder = folder_name; if (file_name === null) { console.log("Clearing PoC editor."); - this.editor.setValue(""); - this.editor.clearSelection(); - this.active_file.name = null; - this.active_file.content = null; - this.active_poc.name = null; - this.active_poc.active_domain = null; - this.active_poc.active_path = null; - this.active_poc.tree = null; + if (this.editor !== null) { + this.editor.setValue(""); + this.editor.clearSelection(); + this.active_file.name = null; + this.active_file.content = null; + this.active_poc.name = null; + this.active_poc.active_domain = null; + this.active_poc.active_path = null; + this.active_poc.tree = null; + } } else { axios.get(this.file_api_path) .then((res) => { @@ -274,10 +276,13 @@ import { getMode as getInteractionScriptMode } from '../interaction_script_mode' "readOnly": val === null }); }, - "poc": function(val) { - this.set_active_file(null, null); - this.active_poc.name = val; - this.update_poc_tree(); + "poc": { + immediate: true, + handler: function (val) { + this.set_active_file(null, null); + this.active_poc.name = val; + this.update_poc_tree(); + } }, "project": function() { this.set_active_file(null, null); diff --git a/bughog/web/vue/src/components/subject-state-selector.vue b/bughog/web/vue/src/components/subject-state-selector.vue new file mode 100644 index 00000000..8b4cd194 --- /dev/null +++ b/bughog/web/vue/src/components/subject-state-selector.vue @@ -0,0 +1,103 @@ + + + + + diff --git a/bughog/web/vue/src/composables/useServerInfo.js b/bughog/web/vue/src/composables/useServerInfo.js new file mode 100644 index 00000000..3ecc00d7 --- /dev/null +++ b/bughog/web/vue/src/composables/useServerInfo.js @@ -0,0 +1,44 @@ +import { reactive, computed } from 'vue'; + +export function useServerInfo() { + const serverInfo = reactive({ + db_info: { + host: null, + connected: false + }, + logs: [], + state: { + is_running: false + }, + }); + + const bannerMessage = computed(() => { + if (serverInfo.db_info.connected) { + return `Connected to MongoDB at ${serverInfo.db_info.host}`; + } else { + return `Connecting to database...`; + } + }); + + const updateServerInfo = (data) => { + if (data.logs) { + serverInfo.logs = data.logs; + } + + for (const key in data) { + if (key === 'logs') continue; + serverInfo[key] = data[key]; + } + }; + + const addLogEntry = (entry) => { + serverInfo.logs.push(entry); + }; + + return { + serverInfo, + bannerMessage, + updateServerInfo, + addLogEntry + }; +} diff --git a/bughog/web/vue/src/composables/useWebSocket b/bughog/web/vue/src/composables/useWebSocket new file mode 100644 index 00000000..b429a96a --- /dev/null +++ b/bughog/web/vue/src/composables/useWebSocket @@ -0,0 +1,169 @@ +import { ref, onMounted, onUnmounted } from 'vue'; +import { toast } from 'vue3-toastify'; + +export function useWebSocket(options = {}) { + const { + onMessage = () => {}, + onOpen = () => {}, + onError = () => {}, + onClose = () => {}, + autoConnect = true, + reconnectInterval = 3000, + maxReconnectAttempts = Infinity, + } = options; + + const websocket = ref(null); + const isConnected = ref(false); + const reconnectAttempts = ref(0); + const reconnectTimer = ref(null); + + const createSocket = () => { + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const url = `${protocol}//${window.location.host}/api/socket/`; + + const ws = new WebSocket(url); + + ws.addEventListener('open', () => { + console.log('WebSocket opened!'); + isConnected.value = true; + reconnectAttempts.value = 0; + + onOpen(ws); + }); + + ws.addEventListener('message', (event) => { + try { + const data = JSON.parse(event.data); + + // Handle notifications + if (data.hasOwnProperty('notification')) { + const { message, type } = data.notification; + + if (type === 'info') { + toast.info(message); + } else if (type === 'error') { + toast.error(message); + } + } + + // Call custom message handler with parsed data + onMessage(data); + } catch (error) { + console.error('Error parsing WebSocket message:', error); + } + }); + + ws.addEventListener('error', (event) => { + console.log('WebSocket error:', event.code, event.reason); + isConnected.value = false; + + // Call custom error handler + onError(event); + }); + + ws.addEventListener('close', (event) => { + console.log('WebSocket closed:', event.code, event.reason); + isConnected.value = false; + + // Call custom close handler + onClose(event); + + // Attempt to reconnect + attemptReconnect(); + }); + + return ws; + }; + + /** + * Attempts to reconnect to the WebSocket server + */ + const attemptReconnect = () => { + if (reconnectAttempts.value >= maxReconnectAttempts) { + console.log('Max reconnect attempts reached'); + return; + } + + reconnectAttempts.value++; + console.log(`Attempting to reconnect... (${reconnectAttempts.value}/${maxReconnectAttempts})`); + + reconnectTimer.value = setTimeout(() => { + websocket.value = createSocket(); + }, reconnectInterval); + }; + + /** + * Sends data through the WebSocket connection + * @param {Object} data - Data to send (will be stringified) + */ + const send = (data) => { + if (websocket.value === null || websocket.value === undefined || websocket.value.readyState > 1) { + console.log('WebSocket connection died, reviving...'); + websocket.value = createSocket(); + + // Queue the message to be sent after reconnection + setTimeout(() => send(data), 1000); + return; + } + + if (websocket.value.readyState === 0) { + console.log(`WebSocket is still trying to connect... (readyState: ${websocket.value.readyState})`); + // Queue the message to be sent after connection is established + setTimeout(() => send(data), 500); + return; + } + + if (websocket.value.readyState === 1) { + websocket.value.send(JSON.stringify(data)); + } + }; + + /** + * Manually connects the WebSocket + */ + const connect = () => { + if (websocket.value && websocket.value.readyState === 1) { + console.log('WebSocket already connected'); + return; + } + + websocket.value = createSocket(); + }; + + /** + * Manually disconnects the WebSocket + */ + const disconnect = () => { + if (reconnectTimer.value) { + clearTimeout(reconnectTimer.value); + reconnectTimer.value = null; + } + + if (websocket.value) { + websocket.value.close(); + websocket.value = null; + } + + isConnected.value = false; + }; + + // Lifecycle hooks + onMounted(() => { + if (autoConnect) { + connect(); + } + }); + + onUnmounted(() => { + disconnect(); + }); + + return { + websocket, + isConnected, + send, + connect, + disconnect, + reconnectAttempts, + }; +} diff --git a/bughog/web/vue/src/main.js b/bughog/web/vue/src/main.js index 32e46650..91c8452d 100644 --- a/bughog/web/vue/src/main.js +++ b/bughog/web/vue/src/main.js @@ -1,8 +1,10 @@ import { createApp } from 'vue' -import './style.css' import App from './App.vue' +import router from './router' + +import './style.css' import 'flowbite' -import 'axios' + import Vue3Toastify from 'vue3-toastify'; import 'vue3-toastify/dist/index.css'; @@ -10,6 +12,9 @@ import { OhVueIcon, addIcons } from "oh-vue-icons"; import { MdInfooutline, FaRegularEdit, FaLink, FaPlus } from "oh-vue-icons/icons"; addIcons(MdInfooutline, FaRegularEdit, FaLink, FaPlus); + const app = createApp(App); +app.use(router); app.use(Vue3Toastify, {autoclose: 5000, position: 'top-right'}); -app.component("v-icon", OhVueIcon).mount('#app') +app.component("v-icon", OhVueIcon); +app.mount('#app') diff --git a/bughog/web/vue/src/router/index.js b/bughog/web/vue/src/router/index.js new file mode 100644 index 00000000..8c685cb4 --- /dev/null +++ b/bughog/web/vue/src/router/index.js @@ -0,0 +1,22 @@ +import { createRouter, createWebHistory } from 'vue-router' +import Home from '../views/Home.vue' +import Playground from '../views/Playground.vue' + +const router = createRouter({ + history: createWebHistory(), + routes: [ + { + path: '/', + name: 'home', + component: Home + }, + { + path: '/play/:subject_type/:subject_name/:project_name/:poc_name', + name: 'playground', + component: Playground, + props: true + } + ] +}) + +export default router diff --git a/bughog/web/vue/src/views/Home.vue b/bughog/web/vue/src/views/Home.vue new file mode 100644 index 00000000..42ab088e --- /dev/null +++ b/bughog/web/vue/src/views/Home.vue @@ -0,0 +1,752 @@ + + + + diff --git a/bughog/web/vue/src/views/Playground.vue b/bughog/web/vue/src/views/Playground.vue new file mode 100644 index 00000000..2d891a92 --- /dev/null +++ b/bughog/web/vue/src/views/Playground.vue @@ -0,0 +1,337 @@ + + + diff --git a/bughog/worker.py b/bughog/worker.py index f9a7edf7..e0bfd6aa 100644 --- a/bughog/worker.py +++ b/bughog/worker.py @@ -6,10 +6,10 @@ from bughog.database.mongo.mongodb import MongoDB from bughog.evaluation.evaluation import Evaluation from bughog.exceptions import SystemError, UserError -from bughog.parameters import EvaluationParameters +from bughog.parameters import ExperimentParameters from bughog.version_control.state.base import State -# This logger argument is set explicitly so when this file is ran as a script, it will still use the logger configuration +# This logger argument is set explicitly so when this file is ran as a script, it will still use the logger configuration. logger = logging.getLogger('bughog.worker') @@ -23,7 +23,7 @@ def __run_by_worker() -> None: logger.info('Worker did not receive enough arguments.') os._exit(0) - params = EvaluationParameters.deserialize(sys.argv[1]) + params = ExperimentParameters.deserialize(sys.argv[1]) state = State.deserialize(sys.argv[2]) MongoDB().connect(params.database_params) @@ -36,7 +36,7 @@ def __run_by_worker() -> None: os._exit(0) -def run(params: EvaluationParameters, state: State): +def run(params: ExperimentParameters, state: State): """ Executes evaluation based on given parameters. """ From f3d4c8df4214c5e0e3a75404d9931db31974e99c Mon Sep 17 00:00:00 2001 From: Gertjan Date: Thu, 19 Feb 2026 21:10:24 +0000 Subject: [PATCH 05/55] refactor: rename playground to lab and enhance experiment selection - Rename Playground.vue to Lab.vue and update all routing/references to '/lab' - Sync dark mode toggle with global state and fix dark mode visibility for buttons - Enhance experiment list in Home.vue with selection counts and hover actions (Edit, Lab) - Improve experiment selection logic to correctly sync "Select All" state - Fix subject state selector to handle null inputs and out-of-range increments - Stabilize Gantt chart dropdown width with truncation - Wrap Lab page controls in styled panes to match UI consistency --- bughog/web/vue/src/components/banner.vue | 5 +- .../src/components/subject-state-selector.vue | 12 +- bughog/web/vue/src/main.js | 4 +- bughog/web/vue/src/router/index.js | 8 +- bughog/web/vue/src/style.css | 2 +- bughog/web/vue/src/views/Home.vue | 65 +++- bughog/web/vue/src/views/Lab.vue | 264 ++++++++++++++ bughog/web/vue/src/views/Playground.vue | 337 ------------------ 8 files changed, 327 insertions(+), 370 deletions(-) create mode 100644 bughog/web/vue/src/views/Lab.vue delete mode 100644 bughog/web/vue/src/views/Playground.vue diff --git a/bughog/web/vue/src/components/banner.vue b/bughog/web/vue/src/components/banner.vue index e64404b6..cf8a45b2 100644 --- a/bughog/web/vue/src/components/banner.vue +++ b/bughog/web/vue/src/components/banner.vue @@ -1,5 +1,6 @@ + + diff --git a/bughog/web/vue/src/views/Playground.vue b/bughog/web/vue/src/views/Playground.vue deleted file mode 100644 index 2d891a92..00000000 --- a/bughog/web/vue/src/views/Playground.vue +++ /dev/null @@ -1,337 +0,0 @@ - - - From 3acab2585132effb78927572987f2b92f49a128e Mon Sep 17 00:00:00 2001 From: Gertjan Date: Thu, 19 Feb 2026 21:38:12 +0000 Subject: [PATCH 06/55] feat: update Lab UI with page title and active state indicator - Set Lab page title to "Bughog - Lab" - Replace non-functional stop button in ExperimentControls with a disabled "Running experiment..." indicator - Remove unused stop experiment methods and listeners from Lab view --- .../web/vue/src/components/experiment-controls.vue | 10 +++++----- bughog/web/vue/src/views/Lab.vue | 12 +++--------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/bughog/web/vue/src/components/experiment-controls.vue b/bughog/web/vue/src/components/experiment-controls.vue index 78139062..d459f434 100644 --- a/bughog/web/vue/src/components/experiment-controls.vue +++ b/bughog/web/vue/src/components/experiment-controls.vue @@ -16,18 +16,18 @@ const props = defineProps({ } }); -const emit = defineEmits(['start', 'stop']); +const emit = defineEmits(['start']); const buttonText = computed(() => { if (props.isRunning) { - return 'Stop experiment'; + return 'Running experiment...'; } return props.hasResult ? 'Rerun experiment' : 'Run experiment'; }); const buttonClass = computed(() => { if (props.isRunning) { - return 'bg-yellow-300 hover:bg-yellow-400 dark:bg-yellow-600 dark:hover:bg-yellow-500'; + return 'bg-blue-300 dark:bg-blue-800 cursor-not-allowed text-gray-700 dark:text-gray-300'; } if (!props.canStart) { return 'bg-gray-300 dark:bg-gray-700 cursor-not-allowed text-gray-500'; @@ -39,8 +39,8 @@ const buttonClass = computed(() => {