From 958a2f61963151d88c60320fa50e28503796fa14 Mon Sep 17 00:00:00 2001 From: Dongge Liu Date: Fri, 16 Aug 2024 11:08:36 +1000 Subject: [PATCH 1/3] Disable incompatible benchmarks for cloud exp --- experiment/run_experiment.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/experiment/run_experiment.py b/experiment/run_experiment.py index e7fa3eaba..ca5e441b2 100644 --- a/experiment/run_experiment.py +++ b/experiment/run_experiment.py @@ -298,6 +298,18 @@ def get_git_hash(allow_uncommitted_changes): return '' +def _filter_incompatible_benchmarks(config: dict, + benchmarks: list[str]) -> list[str]: + """Removes benchmarks that are incompatible with build/run environment.""" + if config['local_experiment']: + return benchmarks + if 'openh264_decoder_fuzzer' in benchmarks: + benchmarks.remove('openh264_decoder_fuzzer') + if 'stb_stbi_read_fuzzer' in benchmarks: + benchmarks.remove('stb_stbi_read_fuzzer') + return benchmarks + + def start_experiment( # pylint: disable=too-many-arguments experiment_name: str, config_filename: str, @@ -322,7 +334,7 @@ def start_experiment( # pylint: disable=too-many-arguments config = read_and_validate_experiment_config(config_filename) config['fuzzers'] = fuzzers - config['benchmarks'] = benchmarks + config['benchmarks'] = _filter_incompatible_benchmarks(config, benchmarks) config['experiment'] = experiment_name config['git_hash'] = get_git_hash(allow_uncommitted_changes) config['no_seeds'] = no_seeds From 1a5cc422623ad9840b898b96d590444aef68456d Mon Sep 17 00:00:00 2001 From: Dongge Liu Date: Fri, 16 Aug 2024 11:13:45 +1000 Subject: [PATCH 2/3] Enable PR exp --- service/experiment-config.yaml | 6 +++--- service/gcbrun_experiment.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/service/experiment-config.yaml b/service/experiment-config.yaml index b9acb09f8..aadf084a6 100644 --- a/service/experiment-config.yaml +++ b/service/experiment-config.yaml @@ -2,8 +2,8 @@ # Unless you are a fuzzbench maintainer running this service, this # will not work with your setup. -trials: 20 -max_total_time: 82800 # 23 hours, the default time for preemptible experiments. +trials: 2 +max_total_time: 2100 cloud_project: fuzzbench docker_registry: gcr.io/fuzzbench cloud_compute_zone: us-central1-c @@ -15,7 +15,7 @@ preemptible_runners: true # This experiment should generate a report that is combined with other public # "production" experiments. -merge_with_nonprivate: true +merge_with_nonprivate: false # This experiment should be merged with other reports in later experiments. private: false diff --git a/service/gcbrun_experiment.py b/service/gcbrun_experiment.py index f19ab493d..fa0313ec8 100644 --- a/service/gcbrun_experiment.py +++ b/service/gcbrun_experiment.py @@ -20,6 +20,7 @@ import logging import os import sys +# dummy # pytype: disable=import-error import github # pylint: disable=import-error From 95b3a55bee45574e5eeda267c61238ac8f46dd81 Mon Sep 17 00:00:00 2001 From: Dongge Liu Date: Fri, 16 Aug 2024 11:20:14 +1000 Subject: [PATCH 3/3] FB cloud Python version is too old to understand list[str] --- experiment/run_experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiment/run_experiment.py b/experiment/run_experiment.py index ca5e441b2..6475eeb34 100644 --- a/experiment/run_experiment.py +++ b/experiment/run_experiment.py @@ -299,7 +299,7 @@ def get_git_hash(allow_uncommitted_changes): def _filter_incompatible_benchmarks(config: dict, - benchmarks: list[str]) -> list[str]: + benchmarks: List[str]) -> List[str]: """Removes benchmarks that are incompatible with build/run environment.""" if config['local_experiment']: return benchmarks