Disable incompatible benchmarks for cloud experiments

experiment/run_experiment.py

@@ -298,6 +298,18 @@ def get_git_hash(allow_uncommitted_changes):
         return ''
 
 
+def _filter_incompatible_benchmarks(config: dict,
+                                    benchmarks: List[str]) -> List[str]:
+    """Removes benchmarks that are incompatible with build/run environment."""
+    if config['local_experiment']:
+        return benchmarks
+    if 'openh264_decoder_fuzzer' in benchmarks:
+        benchmarks.remove('openh264_decoder_fuzzer')
+    if 'stb_stbi_read_fuzzer' in benchmarks:
+        benchmarks.remove('stb_stbi_read_fuzzer')
+    return benchmarks
+
+
 def start_experiment(  # pylint: disable=too-many-arguments
         experiment_name: str,
         config_filename: str,
@@ -322,7 +334,7 @@ def start_experiment(  # pylint: disable=too-many-arguments
 
     config = read_and_validate_experiment_config(config_filename)
     config['fuzzers'] = fuzzers
-    config['benchmarks'] = benchmarks
+    config['benchmarks'] = _filter_incompatible_benchmarks(config, benchmarks)
     config['experiment'] = experiment_name
     config['git_hash'] = get_git_hash(allow_uncommitted_changes)
     config['no_seeds'] = no_seeds

service/experiment-config.yaml

@@ -2,8 +2,8 @@
 # Unless you are a fuzzbench maintainer running this service, this
 # will not work with your setup.
 
-trials: 20
-max_total_time: 82800  # 23 hours, the default time for preemptible experiments.
+trials: 2
+max_total_time: 2100
 cloud_project: fuzzbench
 docker_registry: gcr.io/fuzzbench
 cloud_compute_zone: us-central1-c
@@ -15,7 +15,7 @@ preemptible_runners: true
 
 # This experiment should generate a report that is combined with other public
 # "production" experiments.
-merge_with_nonprivate: true
+merge_with_nonprivate: false
 
 # This experiment should be merged with other reports in later experiments.
 private: false

service/gcbrun_experiment.py

@@ -20,6 +20,7 @@
 import logging
 import os
 import sys
+# dummy
 
 # pytype: disable=import-error
 import github  # pylint: disable=import-error