Skip to content

Commit 1922d68

Browse files
Saumya-RSaumya-R
committed
adding python files for codeql scripts
1 parent 0bfca2d commit 1922d68

4 files changed

Lines changed: 446 additions & 3 deletions

File tree

‎.github/workflows/codeql-multiple-repo-scan.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,11 +54,11 @@ jobs:
5454
- name: Parse known_good.json and create repos.json
5555
id: parse-repos
5656
run: |
57-
scripts/workflow/parse_repos.sh
57+
python3 scripts/workflow/parse_repos.py
5858
- name: Checkout all pinned repositories
5959
id: checkout-repos
6060
run: |
61-
scripts/workflow/checkout_repos.sh
61+
python3 scripts/workflow/checkout_repos.py
6262
- name: List files in repos directory (debug)
6363
run: |
6464
echo "Listing all files in repos directory before CodeQL analysis:"
@@ -79,7 +79,7 @@ jobs:
7979
- name: Recategorize Guidelines
8080
if: always()
8181
run: |
82-
scripts/workflow/recategorize_guidelines.sh
82+
python3 scripts/workflow/recategorize_guidelines.py
8383
- name: Generate HTML Report from SARIF
8484
run: |
8585
SARIF_FILE="sarif-results/cpp.sarif"

‎scripts/workflow/checkout_repos.py‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
#!/usr/bin/env python3
2+
# *******************************************************************************
3+
# Copyright (c) 2025 Contributors to the Eclipse Foundation
4+
#
5+
# See the NOTICE file(s) distributed with this work for additional
6+
# information regarding copyright ownership.
7+
#
8+
# This program and the accompanying materials are made available under the
9+
# terms of the Apache License Version 2.0 which is available at
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# SPDX-License-Identifier: Apache-2.0
13+
# *******************************************************************************
14+
"""
15+
Checkout all pinned repositories based on repos.json configuration.
16+
"""
17+
18+
import json
19+
import sys
20+
import subprocess
21+
import re
22+
import os
23+
from pathlib import Path
24+
25+
26+
def load_repos_config(config_file="./repos.json"):
27+
"""
28+
Load repository configuration from repos.json.
29+
30+
Args:
31+
config_file: Path to repos.json file
32+
33+
Returns:
34+
List of repository configurations
35+
"""
36+
config_path = Path(config_file)
37+
38+
if not config_path.exists():
39+
print(f"Error: file not found '{config_file}'", file=sys.stderr)
40+
sys.exit(1)
41+
42+
try:
43+
with open(config_path, 'r') as f:
44+
repos = json.load(f)
45+
return repos
46+
except (json.JSONDecodeError, IOError) as e:
47+
print(f"Error: Failed to load repos.json: {e}", file=sys.stderr)
48+
sys.exit(1)
49+
50+
51+
def is_commit_hash(ref):
52+
"""
53+
Check if reference looks like a commit hash (40 hex characters for SHA-1).
54+
55+
Args:
56+
ref: Git reference (branch, tag, or hash)
57+
58+
Returns:
59+
True if ref matches commit hash pattern
60+
"""
61+
return bool(re.match(r'^[0-9a-fA-F]{40}$', ref))
62+
63+
64+
def checkout_repo(name, url, ref, path):
65+
"""
66+
Checkout a single repository.
67+
68+
Args:
69+
name: Repository name
70+
url: Repository URL
71+
ref: Git reference (branch, tag, or commit hash)
72+
path: Local path to checkout into
73+
74+
Returns:
75+
True if successful, False otherwise
76+
"""
77+
path_obj = Path(path)
78+
79+
try:
80+
# Create parent directory if needed
81+
path_obj.parent.mkdir(parents=True, exist_ok=True)
82+
83+
if is_commit_hash(ref):
84+
print(f"Checking out {name} ({ref}) to {path}")
85+
print(f" Detected commit hash. Cloning and then checking out.")
86+
87+
# Clone the repository
88+
subprocess.run(
89+
["git", "clone", url, path],
90+
check=True,
91+
capture_output=True
92+
)
93+
94+
# Checkout specific commit
95+
subprocess.run(
96+
["git", "-C", path, "checkout", ref],
97+
check=True,
98+
capture_output=True
99+
)
100+
else:
101+
print(f"Checking out {name} ({ref}) to {path}")
102+
print(f" Detected branch/tag. Cloning with --branch.")
103+
104+
# Clone with shallow copy and specific branch/tag
105+
# Add 'v' prefix if not already present (common convention)
106+
branch_ref = ref if ref.startswith('v') else f'v{ref}'
107+
subprocess.run(
108+
["git", "clone", "--depth", "1", "--branch", branch_ref, url, path],
109+
check=True,
110+
capture_output=True
111+
)
112+
113+
return True
114+
115+
except subprocess.CalledProcessError as e:
116+
print(f"Error: Failed to checkout {name}: {e}", file=sys.stderr)
117+
return False
118+
119+
120+
def main():
121+
"""Main entry point."""
122+
# Load repository configurations
123+
repos = load_repos_config('./repos.json')
124+
repo_count = len(repos)
125+
126+
# Track successfully checked out repositories
127+
repo_paths = []
128+
129+
# Checkout each repository
130+
for i, repo in enumerate(repos):
131+
name = repo.get('name', f'repo-{i}')
132+
url = repo.get('url', '')
133+
ref = repo.get('version', '')
134+
path = repo.get('path', '')
135+
136+
if not all([url, ref, path]):
137+
print(f"Warning: Skipping {name} - missing required fields", file=sys.stderr)
138+
continue
139+
140+
if checkout_repo(name, url, ref, path):
141+
repo_paths.append(path)
142+
143+
# Output all paths (comma-separated for GitHub Actions compatibility)
144+
repo_paths_output = ','.join(repo_paths)
145+
146+
# Write to GITHUB_OUTPUT if available
147+
github_output = os.environ.get('GITHUB_OUTPUT')
148+
if github_output:
149+
try:
150+
with open(github_output, 'a') as f:
151+
f.write(f"repo_paths={repo_paths_output}\n")
152+
except IOError as e:
153+
print(f"Warning: Failed to write GITHUB_OUTPUT: {e}", file=sys.stderr)
154+
155+
# Also print for debugging
156+
print(f"\nSuccessfully checked out {len(repo_paths)} of {repo_count} repositories")
157+
print(f"repo_paths={repo_paths_output}")
158+
159+
return 0 if len(repo_paths) == repo_count else 1
160+
161+
162+
if __name__ == '__main__':
163+
sys.exit(main())

‎scripts/workflow/parse_repos.py‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
#!/usr/bin/env python3
2+
# *******************************************************************************
3+
# Copyright (c) 2025 Contributors to the Eclipse Foundation
4+
#
5+
# See the NOTICE file(s) distributed with this work for additional
6+
# information regarding copyright ownership.
7+
#
8+
# This program and the accompanying materials are made available under the
9+
# terms of the Apache License Version 2.0 which is available at
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# SPDX-License-Identifier: Apache-2.0
13+
# *******************************************************************************
14+
"""
15+
Parse known_good.json and create repos.json for multi-repository CodeQL analysis.
16+
"""
17+
18+
import json
19+
import sys
20+
import subprocess
21+
from pathlib import Path
22+
23+
24+
def install_dependencies():
25+
"""Ensure jq is installed (for reference, though we use Python's json)."""
26+
try:
27+
subprocess.run(
28+
["sudo", "apt-get", "update"],
29+
check=True,
30+
capture_output=True
31+
)
32+
subprocess.run(
33+
["sudo", "apt-get", "install", "-y", "jq"],
34+
check=True,
35+
capture_output=True
36+
)
37+
except subprocess.CalledProcessError as e:
38+
print(f"Warning: Failed to install jq: {e}", file=sys.stderr)
39+
40+
41+
def parse_known_good(json_file="./known_good.json"):
42+
"""
43+
Parse known_good.json and transform modules into repository objects.
44+
45+
Args:
46+
json_file: Path to known_good.json file
47+
48+
Returns:
49+
Tuple of (repos list, module count, module outputs dict)
50+
"""
51+
json_path = Path(json_file)
52+
53+
if not json_path.exists():
54+
print(f"Error: file not found '{json_file}'", file=sys.stderr)
55+
print(f"Current directory: {Path.cwd()}", file=sys.stderr)
56+
sys.exit(1)
57+
58+
try:
59+
with open(json_path, 'r') as f:
60+
data = json.load(f)
61+
except json.JSONDecodeError as e:
62+
print(f"Error: Failed to parse JSON: {e}", file=sys.stderr)
63+
sys.exit(1)
64+
65+
# Extract target_sw modules
66+
modules = data.get('modules', {}).get('target_sw', {})
67+
68+
# Transform modules into repository objects
69+
repos = []
70+
module_outputs = {}
71+
72+
for name, config in modules.items():
73+
repo_url = config.get('repo', '')
74+
version = config.get('version', '')
75+
branch = config.get('branch', '')
76+
hash_val = config.get('hash', '')
77+
78+
# Use version, branch, or hash (in that order of preference)
79+
ref = version or branch or hash_val
80+
81+
repo_obj = {
82+
'name': name,
83+
'url': repo_url,
84+
'version': ref,
85+
'path': f'repos/{name}'
86+
}
87+
repos.append(repo_obj)
88+
89+
# Store module outputs for GITHUB_OUTPUT compatibility
90+
module_outputs[f'{name}_url'] = repo_url
91+
if version:
92+
module_outputs[f'{name}_version'] = version
93+
if branch:
94+
module_outputs[f'{name}_branch'] = branch
95+
if hash_val:
96+
module_outputs[f'{name}_hash'] = hash_val
97+
98+
return repos, len(modules), module_outputs
99+
100+
101+
def write_repos_json(repos, output_file="./repos.json"):
102+
"""Write repositories to repos.json file."""
103+
output_path = Path(output_file)
104+
105+
try:
106+
with open(output_path, 'w') as f:
107+
json.dump(repos, f, indent=2)
108+
print(f"Generated {output_file}:")
109+
print(json.dumps(repos, indent=2))
110+
print() # Add newline for readability
111+
except IOError as e:
112+
print(f"Error: Failed to write {output_file}: {e}", file=sys.stderr)
113+
sys.exit(1)
114+
115+
116+
def write_github_output(outputs):
117+
"""
118+
Write outputs to GITHUB_OUTPUT for GitHub Actions compatibility.
119+
120+
Args:
121+
outputs: Dictionary of key-value pairs to output
122+
"""
123+
github_output = Path(os.environ.get('GITHUB_OUTPUT', '/dev/null'))
124+
125+
if github_output.exists() or github_output.parent.exists():
126+
try:
127+
with open(github_output, 'a') as f:
128+
for key, value in outputs.items():
129+
f.write(f"{key}={value}\n")
130+
except IOError as e:
131+
print(f"Warning: Failed to write GITHUB_OUTPUT: {e}", file=sys.stderr)
132+
133+
134+
def main():
135+
"""Main entry point."""
136+
import os
137+
138+
# Install dependencies (optional, jq not strictly needed in Python version)
139+
# install_dependencies()
140+
141+
# Parse known_good.json
142+
repos, module_count, module_outputs = parse_known_good('./known_good.json')
143+
144+
# Write repos.json
145+
write_repos_json(repos)
146+
147+
# Write GitHub Actions outputs
148+
github_outputs = {'MODULE_COUNT': str(module_count)}
149+
github_outputs.update(module_outputs)
150+
write_github_output(github_outputs)
151+
152+
print("Parse complete!")
153+
154+
155+
if __name__ == '__main__':
156+
main()

0 commit comments

Comments
 (0)