From e46cac01660704251c25d2d0bc48e44be6a82cdb Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Sun, 19 Jan 2025 23:21:44 -0500 Subject: [PATCH 01/11] adding onSaturn command line argument and passing it down to java and pythong scaffolds calls --- saturn/cmd/saturn/main.go | 3 ++- saturn/pkg/run/java.go | 9 +++++---- saturn/pkg/run/python3.go | 9 +++++---- saturn/pkg/run/scaffold.go | 25 ++++++++++++++----------- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/saturn/cmd/saturn/main.go b/saturn/cmd/saturn/main.go index e7523ad3d..bf000e621 100644 --- a/saturn/cmd/saturn/main.go +++ b/saturn/cmd/saturn/main.go @@ -27,6 +27,7 @@ var ( monitorPort *uint = flag.Uint("port", 8005, "the port for monitoring shutdowns") scaffoldRoot *string = flag.String("scaffold", "/scaffolds", "the root directory for saving scaffolds") parallelism *uint = flag.Uint("parallel", 1, "the number of scaffolds to run in parallel") + onSaturn *bool = flag.Bool("saturn", true, "run on saturn") ) func main() { @@ -57,7 +58,7 @@ func main() { ) for i = 0; i < *parallelism; i++ { root := filepath.Join(*scaffoldRoot, strconv.FormatUint(uint64(i), 10)) - multiplexer, err := run.NewScaffoldMultiplexer(root, secret) + multiplexer, err := run.NewScaffoldMultiplexer(root, secret, *onSaturn) if err != nil { log.Ctx(ctx).Fatal().Err(err).Msg("Could not initialize scaffold multiplexer.") } diff --git a/saturn/pkg/run/java.go b/saturn/pkg/run/java.go index a170ff5aa..f88037a37 100644 --- a/saturn/pkg/run/java.go +++ b/saturn/pkg/run/java.go @@ -21,7 +21,7 @@ type JavaScaffold struct { javaEnv []string } -func NewJavaScaffold(ctx context.Context, episode saturn.Episode, repo *git.Repository, root string, javaPath string) (*JavaScaffold, error) { +func NewJavaScaffold(ctx context.Context, episode saturn.Episode, repo *git.Repository, root string, javaPath string, onSaturn bool) (*JavaScaffold, error) { s := new(JavaScaffold) s.root = root s.repo = repo @@ -43,6 +43,7 @@ func NewJavaScaffold(ctx context.Context, episode saturn.Episode, repo *git.Repo s.DetermineScores(), } s.matchOutputs = make(map[*StepArguments]string) + s.onSaturn = onSaturn return s, nil } @@ -65,7 +66,7 @@ func (s *JavaScaffold) Prepare() *Step { s.javaEnv, "./gradlew", "update", - fmt.Sprintf("-PonSaturn=%t", true), + fmt.Sprintf("-PonSaturn=%t", s.onSaturn), ) log.Ctx(ctx).Debug().Msg(out) if err != nil { @@ -145,7 +146,7 @@ func (s *JavaScaffold) VerifySubmission() *Step { "./gradlew", "verify", fmt.Sprintf("-Pteam=%s", pkg), - fmt.Sprintf("-PonSaturn=%t", true), + fmt.Sprintf("-PonSaturn=%t", s.onSaturn), ) log.Ctx(ctx).Debug().Msg(out) if err != nil { @@ -179,7 +180,7 @@ func (s *JavaScaffold) RunMatch() *Step { s.javaEnv, "./gradlew", "run", - fmt.Sprintf("-PonSaturn=%t", true), + fmt.Sprintf("-PonSaturn=%t", s.onSaturn), fmt.Sprintf("-PteamA=%s", arg.Details.(ExecuteRequest).A.TeamName), fmt.Sprintf("-PteamB=%s", arg.Details.(ExecuteRequest).B.TeamName), fmt.Sprintf("-PclassLocationA=%s", filepath.Join("data", "A")), diff --git a/saturn/pkg/run/python3.go b/saturn/pkg/run/python3.go index c97711b3d..54677c452 100644 --- a/saturn/pkg/run/python3.go +++ b/saturn/pkg/run/python3.go @@ -21,7 +21,7 @@ type Python3Scaffold struct { var pyWinnerRegex = regexp.MustCompile(`(?m)^\[server\]\s*.*\(([AB])\) wins \(round [0-9]+\)$`) -func NewPython3Scaffold(ctx context.Context, episode saturn.Episode, repo *git.Repository, root string, pyVersion string) (*Python3Scaffold, error) { +func NewPython3Scaffold(ctx context.Context, episode saturn.Episode, repo *git.Repository, root string, pyVersion string, onSaturn bool) (*Python3Scaffold, error) { s := new(Python3Scaffold) s.root = root s.repo = repo @@ -42,6 +42,7 @@ func NewPython3Scaffold(ctx context.Context, episode saturn.Episode, repo *git.R s.DetermineScores(), } s.matchOutputs = make(map[*StepArguments]string) + s.onSaturn = onSaturn return s, nil } @@ -60,7 +61,7 @@ func (s *Python3Scaffold) Prepare() *Step { s.pyVersion, "run.py", "update", - fmt.Sprintf("--on-saturn=%t", true), + fmt.Sprintf("--on-saturn=%t", s.onSaturn), ) log.Ctx(ctx).Debug().Msg(out) if err != nil { @@ -132,7 +133,7 @@ func (s *Python3Scaffold) VerifySubmission() *Step { "run.py", "verify", fmt.Sprintf("--p1=%s", pkg), - fmt.Sprintf("--on-saturn=%t", true), + fmt.Sprintf("--on-saturn=%t", s.onSaturn), ) log.Ctx(ctx).Debug().Msg(out) if err != nil { @@ -167,7 +168,7 @@ func (s *Python3Scaffold) RunMatch() *Step { s.pyVersion, "run.py", "run", - fmt.Sprintf("--on-saturn=%t", true), + fmt.Sprintf("--on-saturn=%t", s.onSaturn), fmt.Sprintf("--p1-team=%s", arg.Details.(ExecuteRequest).A.TeamName), fmt.Sprintf("--p2-team=%s", arg.Details.(ExecuteRequest).B.TeamName), fmt.Sprintf("--p1-dir=%s", filepath.Join("data", "A")), diff --git a/saturn/pkg/run/scaffold.go b/saturn/pkg/run/scaffold.go index daae58f82..e7911ab01 100644 --- a/saturn/pkg/run/scaffold.go +++ b/saturn/pkg/run/scaffold.go @@ -22,9 +22,10 @@ type ScaffoldMultiplexer struct { Root string scaffolds map[string]*Scaffold gitAuth transport.AuthMethod + onSaturn bool } -func NewScaffoldMultiplexer(root string, secret *saturn.Secret) (*ScaffoldMultiplexer, error) { +func NewScaffoldMultiplexer(root string, secret *saturn.Secret, onSaturn bool) (*ScaffoldMultiplexer, error) { gitAuth := &transportHttp.BasicAuth{ Username: "ignored", Password: secret.GitToken, @@ -33,6 +34,7 @@ func NewScaffoldMultiplexer(root string, secret *saturn.Secret) (*ScaffoldMultip Root: root, scaffolds: make(map[string]*Scaffold), gitAuth: gitAuth, + onSaturn: onSaturn, }, nil } @@ -49,7 +51,7 @@ func (m *ScaffoldMultiplexer) runTask( if err != nil { return fmt.Errorf("cloneGit: %v", err) } - scaffold, err = NewScaffold(ctx, payload.Episode, repo, root) + scaffold, err = NewScaffold(ctx, payload.Episode, repo, root, m.onSaturn) if err != nil { return fmt.Errorf("NewScaffold: %v", err) } @@ -106,18 +108,19 @@ func (m *ScaffoldMultiplexer) Execute( } type Scaffold struct { - root string - repo *git.Repository - gitAuth transport.AuthMethod - compile Recipe - execute Recipe + root string + repo *git.Repository + gitAuth transport.AuthMethod + compile Recipe + execute Recipe + onSaturn bool } -func NewScaffold(ctx context.Context, episode saturn.Episode, repo *git.Repository, root string) (*Scaffold, error) { +func NewScaffold(ctx context.Context, episode saturn.Episode, repo *git.Repository, root string, onSaturn bool) (*Scaffold, error) { switch episode.Language { case saturn.Java8: // Kept for compatibility running old episodes - s, err := NewJavaScaffold(ctx, episode, repo, root, "/usr/lib/jvm/java-8-openjdk-amd64") + s, err := NewJavaScaffold(ctx, episode, repo, root, "/usr/lib/jvm/java-8-openjdk-amd64", onSaturn) if err != nil { return nil, fmt.Errorf("NewJavaScaffold (Java8): %v", err) } @@ -125,14 +128,14 @@ func NewScaffold(ctx context.Context, episode saturn.Episode, repo *git.Reposito case saturn.Java21: // Modern java21 scaffolds store java in the 'java' subdirectory of the scaffold javaRoot := filepath.Join(root, "java") - s, err := NewJavaScaffold(ctx, episode, repo, javaRoot, "/usr/local/openjdk-21") + s, err := NewJavaScaffold(ctx, episode, repo, javaRoot, "/usr/local/openjdk-21", onSaturn) if err != nil { return nil, fmt.Errorf("NewJavaScaffold (Java21): %v", err) } return &s.Scaffold, nil case saturn.Python3: pyRoot := filepath.Join(root, "python") - s, err := NewPython3Scaffold(ctx, episode, repo, pyRoot, "python3.12") + s, err := NewPython3Scaffold(ctx, episode, repo, pyRoot, "python3.12", onSaturn) if err != nil { return nil, fmt.Errorf("NewPython3Scaffold: %v", err) } From 0df3d1dd933c9d87634c25c343c5dcda3cc4d2d6 Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Mon, 20 Jan 2025 11:05:16 -0500 Subject: [PATCH 02/11] adding local file management in google cloud storage file --- saturn/pkg/run/gcs.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/saturn/pkg/run/gcs.go b/saturn/pkg/run/gcs.go index a3e88e788..558aef364 100644 --- a/saturn/pkg/run/gcs.go +++ b/saturn/pkg/run/gcs.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "io" + "os" + "path/filepath" "cloud.google.com/go/storage" "github.com/rs/zerolog/log" @@ -22,6 +24,20 @@ func NewGCSClient(ctx context.Context) (*GCSClient, error) { } func (c *GCSClient) GetFile(ctx context.Context, f FileSpecification, w io.Writer) error { + if f.Bucket == "local" { + file, err := os.Open(f.Name) + if err != nil { + return fmt.Errorf("os.Open: %v", err) + } + defer file.Close() + + written, err := io.Copy(w, file) + if err != nil { + return fmt.Errorf("io.Copy: %v", err) + } + log.Ctx(ctx).Debug().Msgf("Read %d bytes from local file.", written) + return nil + } object := c.c.Bucket(f.Bucket).Object(f.Name) reader, err := object.NewReader(ctx) if err != nil { @@ -38,6 +54,25 @@ func (c *GCSClient) GetFile(ctx context.Context, f FileSpecification, w io.Write } func (c *GCSClient) UploadFile(ctx context.Context, f FileSpecification, r io.Reader, public bool) error { + if f.Bucket == "local" { + // Ensure the directory exists + if err := os.MkdirAll(filepath.Dir(f.Name), os.ModePerm); err != nil { + return fmt.Errorf("os.MkdirAll: %v", err) + } + + file, err := os.Create(f.Name) + if err != nil { + return fmt.Errorf("os.Create: %v", err) + } + defer file.Close() + + written, err := io.Copy(file, r) + if err != nil { + return fmt.Errorf("io.Copy: %v", err) + } + log.Ctx(ctx).Debug().Msgf("Wrote %d bytes to local file.", written) + return nil + } object := c.c.Bucket(f.Bucket).Object(f.Name) writer := object.NewWriter(ctx) defer writer.Close() From 94e71b2f597a79db96b22035b272ca1b28890ce6 Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Tue, 21 Jan 2025 13:32:48 -0500 Subject: [PATCH 03/11] modifying readSecret to read from a local secret file if locally --- saturn/cmd/saturn/main.go | 4 ++-- saturn/pkg/saturn/secret.go | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/saturn/cmd/saturn/main.go b/saturn/cmd/saturn/main.go index bf000e621..a1374e3f1 100644 --- a/saturn/cmd/saturn/main.go +++ b/saturn/cmd/saturn/main.go @@ -27,7 +27,7 @@ var ( monitorPort *uint = flag.Uint("port", 8005, "the port for monitoring shutdowns") scaffoldRoot *string = flag.String("scaffold", "/scaffolds", "the root directory for saving scaffolds") parallelism *uint = flag.Uint("parallel", 1, "the number of scaffolds to run in parallel") - onSaturn *bool = flag.Bool("saturn", true, "run on saturn") + onSaturn *bool = flag.Bool("onsaturn", true, "run on saturn") ) func main() { @@ -39,7 +39,7 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), unix.SIGINT, unix.SIGTERM) defer stop() - secret, err := saturn.ReadSecret(ctx, *gcpProjectID, *gcpSecretName) + secret, err := saturn.ReadSecret(ctx, *gcpProjectID, *gcpSecretName, *onSaturn) if err != nil { log.Ctx(ctx).Fatal().Err(err).Msg("Could not read secrets.") } diff --git a/saturn/pkg/saturn/secret.go b/saturn/pkg/saturn/secret.go index e6e68caf8..2a0f592ec 100644 --- a/saturn/pkg/saturn/secret.go +++ b/saturn/pkg/saturn/secret.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" secretmanager "cloud.google.com/go/secretmanager/apiv1" "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" @@ -13,7 +14,21 @@ type Secret struct { GitToken string `json:"git-token"` } -func ReadSecret(ctx context.Context, projectID, name string) (*Secret, error) { +func ReadSecret(ctx context.Context, projectID, name string, onSaturn bool) (*Secret, error) { + if !onSaturn { + // Read local secret file and parse as *Secret + filePath := name + content, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("ioutil.ReadFile: %v", err) + } + + var result Secret + if err := json.Unmarshal(content, &result); err != nil { + return nil, fmt.Errorf("json.Unmarshal: %v", err) + } + return &result, nil + } client, err := secretmanager.NewClient(ctx) if err != nil { return nil, fmt.Errorf("secretmanager.NewClient: %v", err) From e64ee81165789356f8f291535fb82c202f562d05 Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Tue, 21 Jan 2025 15:39:19 -0500 Subject: [PATCH 04/11] fixing local gcs --- saturn/.gitignore | 3 +++ saturn/pkg/run/gcs.go | 14 +++++++++----- saturn/pkg/run/scaffold.go | 4 ++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/saturn/.gitignore b/saturn/.gitignore index 3b735ec4a..e773fc85d 100644 --- a/saturn/.gitignore +++ b/saturn/.gitignore @@ -19,3 +19,6 @@ # Go workspace file go.work + +# Secret file for local dev +*secret.* diff --git a/saturn/pkg/run/gcs.go b/saturn/pkg/run/gcs.go index 558aef364..d81513e95 100644 --- a/saturn/pkg/run/gcs.go +++ b/saturn/pkg/run/gcs.go @@ -12,19 +12,23 @@ import ( ) type GCSClient struct { - c *storage.Client + c *storage.Client + onSaturn bool } -func NewGCSClient(ctx context.Context) (*GCSClient, error) { +func NewGCSClient(ctx context.Context, onSaturn bool) (*GCSClient, error) { + if !onSaturn { + return &GCSClient{nil, onSaturn}, nil + } client, err := storage.NewClient(ctx) if err != nil { return nil, fmt.Errorf("storage.NewClient: %v", err) } - return &GCSClient{client}, nil + return &GCSClient{client, false}, nil } func (c *GCSClient) GetFile(ctx context.Context, f FileSpecification, w io.Writer) error { - if f.Bucket == "local" { + if !c.onSaturn { file, err := os.Open(f.Name) if err != nil { return fmt.Errorf("os.Open: %v", err) @@ -54,7 +58,7 @@ func (c *GCSClient) GetFile(ctx context.Context, f FileSpecification, w io.Write } func (c *GCSClient) UploadFile(ctx context.Context, f FileSpecification, r io.Reader, public bool) error { - if f.Bucket == "local" { + if !c.onSaturn { // Ensure the directory exists if err := os.MkdirAll(filepath.Dir(f.Name), os.ModePerm); err != nil { return fmt.Errorf("os.MkdirAll: %v", err) diff --git a/saturn/pkg/run/scaffold.go b/saturn/pkg/run/scaffold.go index e7911ab01..f96c70111 100644 --- a/saturn/pkg/run/scaffold.go +++ b/saturn/pkg/run/scaffold.go @@ -70,7 +70,7 @@ func (m *ScaffoldMultiplexer) Compile( if err := mapstructure.Decode(payload.Details, &req); err != nil { return fmt.Errorf("mapstructure.Decode: %v", err) } - storage, err := NewGCSClient(ctx) + storage, err := NewGCSClient(ctx, m.onSaturn) if err != nil { return fmt.Errorf("NewGCSClient: %v", err) } @@ -93,7 +93,7 @@ func (m *ScaffoldMultiplexer) Execute( if err := mapstructure.Decode(payload.Details, &req); err != nil { return fmt.Errorf("mapstructure.Decode: %v", err) } - storage, err := NewGCSClient(ctx) + storage, err := NewGCSClient(ctx, m.onSaturn) if err != nil { return fmt.Errorf("NewGCSClient: %v", err) } From 2f8c1cb25dec559e201db3a5d76ae6042c1eda2e Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Tue, 21 Jan 2025 16:39:45 -0500 Subject: [PATCH 05/11] allowing local report --- saturn/cmd/saturn/main.go | 2 +- saturn/pkg/saturn/report.go | 57 ++++++++++++++++++++++--------------- saturn/pkg/saturn/saturn.go | 4 +-- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/saturn/cmd/saturn/main.go b/saturn/cmd/saturn/main.go index a1374e3f1..cfd44a457 100644 --- a/saturn/cmd/saturn/main.go +++ b/saturn/cmd/saturn/main.go @@ -66,7 +66,7 @@ func main() { app, err := saturn.New( ctx, saturn.WithGcpPubsubSubcriber(*gcpProjectID, *gcpPubsubSubscriptionID), - saturn.WithGcpTokenedReporter(*gcpTokenedReporterAudience, *gcpTokenedReporterUserAgent), + saturn.WithGcpTokenedReporter(*gcpTokenedReporterAudience, *gcpTokenedReporterUserAgent, *onSaturn), saturn.WithRunner("compile", multiplexer.Compile), saturn.WithRunner("execute", multiplexer.Execute), ) diff --git a/saturn/pkg/saturn/report.go b/saturn/pkg/saturn/report.go index 65011ac24..3e4159bbd 100644 --- a/saturn/pkg/saturn/report.go +++ b/saturn/pkg/saturn/report.go @@ -19,17 +19,22 @@ type Reporter interface { type GCPTokenedReporter struct { client *http.Client userAgent string + onSaturn bool } func NewGCPTokenedReporter( ctx context.Context, audience, userAgent string, + onSaturn bool, ) (*GCPTokenedReporter, error) { + if !onSaturn { + return &GCPTokenedReporter{nil, userAgent, onSaturn}, nil + } client, err := idtoken.NewClient(ctx, audience) if err != nil { return nil, fmt.Errorf("idtoken.NewClient: %v", err) } - return &GCPTokenedReporter{client, userAgent}, nil + return &GCPTokenedReporter{client, userAgent, onSaturn}, nil } func (r *GCPTokenedReporter) Report(ctx context.Context, t *Task) error { @@ -52,31 +57,37 @@ func (r *GCPTokenedReporter) Report(ctx context.Context, t *Task) error { return fmt.Errorf("json.Marshal: %v", err) } log.Ctx(ctx).Debug().RawJSON("payload", reqBody).Msg("Sending report.") + if r.onSaturn { + req, err := http.NewRequest("POST", t.Payload.Metadata.ReportURL, bytes.NewBuffer(reqBody)) + if err != nil { + return fmt.Errorf("http.NewRequestWithContext: %v", err) + } + req.Header.Set("User-Agent", r.userAgent) + req.Header.Set("Content-Type", "application/json") - req, err := http.NewRequest("POST", t.Payload.Metadata.ReportURL, bytes.NewBuffer(reqBody)) - if err != nil { - return fmt.Errorf("http.NewRequestWithContext: %v", err) - } - req.Header.Set("User-Agent", r.userAgent) - req.Header.Set("Content-Type", "application/json") - - resp, err := r.client.Do(req) - if err != nil { - return fmt.Errorf("r.client.Do: %v", err) - } - defer resp.Body.Close() + resp, err := r.client.Do(req) + if err != nil { + return fmt.Errorf("r.client.Do: %v", err) + } + defer resp.Body.Close() - respBody, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("ioutil.ReadAll: %v", err) - } - log.Ctx(ctx).Debug().Bytes("response", respBody).Msg("Report sent.") + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("ioutil.ReadAll: %v", err) + } + log.Ctx(ctx).Debug().Bytes("response", respBody).Msg("Report sent.") - if resp.StatusCode == http.StatusConflict { - t.Finish(TaskAborted, nil) - } - if !(200 <= resp.StatusCode && resp.StatusCode < 300) { - return fmt.Errorf("bad status code: %v", resp.StatusCode) + if resp.StatusCode == http.StatusConflict { + t.Finish(TaskAborted, nil) + } + if !(200 <= resp.StatusCode && resp.StatusCode < 300) { + return fmt.Errorf("bad status code: %v", resp.StatusCode) + } + } else { + filepath := t.Payload.Metadata.ReportURL + if err := ioutil.WriteFile(filepath, reqBody, 0644); err != nil { + return fmt.Errorf("ioutil.WriteFile: %v", err) + } } return nil } diff --git a/saturn/pkg/saturn/saturn.go b/saturn/pkg/saturn/saturn.go index ce7560242..74ff1454f 100644 --- a/saturn/pkg/saturn/saturn.go +++ b/saturn/pkg/saturn/saturn.go @@ -66,13 +66,13 @@ func WithGcpPubsubSubcriber(projectID, subscriptionID string) SaturnOption { } } -func WithGcpTokenedReporter(audience, userAgent string) SaturnOption { +func WithGcpTokenedReporter(audience, userAgent string, onSaturn bool) SaturnOption { return func(ctx context.Context, s *Saturn) (*Saturn, error) { if s.report != nil { return nil, fmt.Errorf("reporter already exists") } log.Ctx(ctx).Debug().Msg("Initializing outcome reporter.") - report, err := NewGCPTokenedReporter(ctx, audience, userAgent) + report, err := NewGCPTokenedReporter(ctx, audience, userAgent, onSaturn) if err != nil { return nil, fmt.Errorf("NewGCPTokenedReporter: %v", err) } From 2dfc80c312a0c497efc681d70efeda39c7c2d063 Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Tue, 21 Jan 2025 19:12:38 -0500 Subject: [PATCH 06/11] adding development pub/sub image with pubsub client to send messages and test --- saturn/development/Dockerfile | 28 +++++ saturn/development/compile.json | 23 ++++ saturn/development/execute.json | 49 ++++++++ saturn/development/go.mod | 41 +++++++ saturn/development/go.sum | 176 +++++++++++++++++++++++++++++ saturn/development/pubsubclient.go | 166 +++++++++++++++++++++++++++ saturn/development/startpubsub.sh | 26 +++++ 7 files changed, 509 insertions(+) create mode 100644 saturn/development/Dockerfile create mode 100644 saturn/development/compile.json create mode 100644 saturn/development/execute.json create mode 100644 saturn/development/go.mod create mode 100644 saturn/development/go.sum create mode 100644 saturn/development/pubsubclient.go create mode 100755 saturn/development/startpubsub.sh diff --git a/saturn/development/Dockerfile b/saturn/development/Dockerfile new file mode 100644 index 000000000..08c217274 --- /dev/null +++ b/saturn/development/Dockerfile @@ -0,0 +1,28 @@ +# Use an official Golang image as a parent image +FROM golang:1.23 + +# Install Google Cloud SDK +RUN apt-get update && apt-get install -y curl gnupg rlwrap && \ + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \ + apt-get update && apt-get install -y google-cloud-sdk google-cloud-sdk-pubsub-emulator + +# Install Pub/Sub emulator +RUN apt-get install google-cloud-cli-pubsub-emulator + +# Copy the start script +COPY startpubsub.sh /usr/local/bin/startpubsub.sh +RUN chmod +x /usr/local/bin/startpubsub.sh + +# Set the working directory +WORKDIR /app + +# Copy the Go application code +COPY . /app +RUN go mod download +RUN go build -o pubsubclient pubsubclient.go +# Expose the Pub/Sub emulator port +EXPOSE 8514 + +# Run the entry point script +ENTRYPOINT ["rlwrap", "/usr/local/bin/startpubsub.sh"] diff --git a/saturn/development/compile.json b/saturn/development/compile.json new file mode 100644 index 000000000..44694f295 --- /dev/null +++ b/saturn/development/compile.json @@ -0,0 +1,23 @@ +{ + "episode": { + "name": "bc25python", + "language": "py3", + "scaffold": "https://github.com/battlecode/battlecode25-scaffold" + }, + "metadata": { + "report-url": "/development/test/report.txt", + "task-type": "compile" + }, + "details": { + "source": { + "bucket": "local", + "name": "/development/test/python.zip" + }, + "binary": { + "bucket": "local", + "name": "/development/test/binary.zip" + }, + "team-name": "test", + "package": "examplefuncsplayer" + } +} diff --git a/saturn/development/execute.json b/saturn/development/execute.json new file mode 100644 index 000000000..bf2200166 --- /dev/null +++ b/saturn/development/execute.json @@ -0,0 +1,49 @@ +{ + "episode": { + "name": "bc25java", + "language": "java21", + "scaffold": "https://github.com/battlecode/battlecode25-scaffold" + }, + "metadata": { + "report-url": "/development/test/report.txt", + "task-type": "execute" + }, + "details": { + "maps": [ + "fix", + "galaxy", + "gridworld", + "quack", + "sierpinski" + ], + "replay": { + "bucket": "local", + "name": "/development/test/replay.bc25java" + }, + "alternate-order": true, + "a": { + "source": { + "bucket": "local", + "name": "/development/test/source1.zip" + }, + "binary": { + "bucket": "local", + "name": "/development/test/binary1.zip" + }, + "team-name": "test1", + "package": "examplefuncsplayer" + }, + "b": { + "source": { + "bucket": "local", + "name": "/development/test/source2.zip" + }, + "binary": { + "bucket": "local", + "name": "/development/test/binary2.zip" + }, + "team-name": "test2", + "package": "examplefuncsplayer" + } + } +} diff --git a/saturn/development/go.mod b/saturn/development/go.mod new file mode 100644 index 000000000..3aac77de7 --- /dev/null +++ b/saturn/development/go.mod @@ -0,0 +1,41 @@ +module github.com/battlecode/galaxy + +go 1.23 + +require ( + cloud.google.com/go/pubsub v1.45.3 + google.golang.org/api v0.217.0 +) + +require ( + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.14.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect + cloud.google.com/go/iam v1.2.2 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect + golang.org/x/crypto v0.32.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/oauth2 v0.25.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422 // indirect + google.golang.org/grpc v1.69.4 // indirect + google.golang.org/protobuf v1.36.2 // indirect +) diff --git a/saturn/development/go.sum b/saturn/development/go.sum new file mode 100644 index 000000000..cd7c1e94e --- /dev/null +++ b/saturn/development/go.sum @@ -0,0 +1,176 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.14.0 h1:A5C4dKV/Spdvxcl0ggWwWEzzP7AZMJSEIgrkngwhGYM= +cloud.google.com/go/auth v0.14.0/go.mod h1:CYsoRL1PdiDuqeQpZE0bP2pnPrGqFcOkI0nldEQis+A= +cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= +cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= +cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= +cloud.google.com/go/kms v1.20.1 h1:og29Wv59uf2FVaZlesaiDAqHFzHaoUyHI3HYp9VUHVg= +cloud.google.com/go/kms v1.20.1/go.mod h1:LywpNiVCvzYNJWS9JUcGJSVTNSwPwi0vBAotzDqn2nc= +cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= +cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= +cloud.google.com/go/pubsub v1.45.3 h1:prYj8EEAAAwkp6WNoGTE4ahe0DgHoyJd5Pbop931zow= +cloud.google.com/go/pubsub v1.45.3/go.mod h1:cGyloK/hXC4at7smAtxFnXprKEFTqmMXNNd9w+bd94Q= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= +go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.217.0 h1:GYrUtD289o4zl1AhiTZL0jvQGa2RDLyC+kX1N/lfGOU= +google.golang.org/api v0.217.0/go.mod h1:qMc2E8cBAbQlRypBTBWHklNJlaZZJBwDv81B1Iu8oSI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= +google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422 h1:3UsHvIr4Wc2aW4brOaSCmcxh9ksica6fHEr8P1XhkYw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= +google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= +google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/saturn/development/pubsubclient.go b/saturn/development/pubsubclient.go new file mode 100644 index 000000000..d5cbc4e24 --- /dev/null +++ b/saturn/development/pubsubclient.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + + "cloud.google.com/go/pubsub" + "google.golang.org/api/option" +) + +func main() { + AllCommands := []string{ + "create-topic ", + "create-pull-subscription ", + "publish ", + "publish-json ", + "subscribe ", + } + quickCommands := []string{ + "compile: to send message in compile.json", + "execute: to send message in execute.json", + "sub: to recieve message", + } + if len(os.Args) < 2 { + fmt.Println("Usage: ./pubsubclient [args]") + fmt.Println("Quick commands shortcuts:") + for _, cmd := range quickCommands { + fmt.Println(" -", cmd) + } + fmt.Println("All commands:") + for _, cmd := range AllCommands { + fmt.Println(" -", cmd) + } + os.Exit(1) + } + + command := os.Args[1] + ctx := context.Background() + projectID := "mitbattlecode" // Replace with your GCP project ID + + if os.Getenv("PUBSUB_EMULATOR_HOST") != "" { + os.Setenv("PUBSUB_PROJECT_ID", projectID) + } + + client, err := pubsub.NewClient(ctx, projectID, option.WithoutAuthentication()) + if err != nil { + log.Fatalf("Failed to create Pub/Sub client: %v", err) + } + defer client.Close() + + switch command { + case "compile": + publishJSONMessage(ctx, client, "testing-saturn", "/development/compile.json") + + case "execute": + publishJSONMessage(ctx, client, "testing-saturn", "/development/execute.json") + + case "sub": + subscribe(ctx, client, "test") + + case "create-topic": + if len(os.Args) < 3 { + log.Fatalf("Usage: ./pubsubclient create-topic ") + } + topicName := os.Args[2] + createTopic(ctx, client, topicName) + + case "create-pull-subscription": + if len(os.Args) < 4 { + log.Fatalf("Usage: ./pubsubclient create-pull-subscription ") + } + topicName := os.Args[2] + subscriptionName := os.Args[3] + createPullSubscription(ctx, client, topicName, subscriptionName) + + case "publish": + if len(os.Args) < 4 { + log.Fatalf("Usage: ./pubsubclient publish ") + } + topicName := os.Args[2] + message := os.Args[3] + publishMessage(ctx, client, topicName, message) + + case "publish-json": + if len(os.Args) < 4 { + log.Fatalf("Usage: ./pubsubclient publish-json ") + } + topicName := os.Args[2] + jsonFilePath := os.Args[3] + publishJSONMessage(ctx, client, topicName, jsonFilePath) + + case "subscribe": + if len(os.Args) < 3 { + log.Fatalf("Usage: ./pubsubclient subscribe ") + } + subscriptionName := os.Args[2] + subscribe(ctx, client, subscriptionName) + + default: + log.Fatalf("Unknown command: %s", command) + } +} + +func createTopic(ctx context.Context, client *pubsub.Client, topicName string) { + topic, err := client.CreateTopic(ctx, topicName) + if err != nil { + log.Fatalf("Failed to create topic: %v", err) + } + fmt.Printf("Topic %s created successfully\n", topic.ID()) +} + +func createPullSubscription(ctx context.Context, client *pubsub.Client, topicName, subscriptionName string) { + topic := client.Topic(topicName) + sub, err := client.CreateSubscription(ctx, subscriptionName, pubsub.SubscriptionConfig{ + Topic: topic, + }) + if err != nil { + log.Fatalf("Failed to create pull subscription: %v", err) + } + fmt.Printf("Pull subscription %s created successfully\n", sub.ID()) +} + +func publishJSONMessage(ctx context.Context, client *pubsub.Client, topicName, jsonFilePath string) { + // Read the JSON file + jsonData, err := os.ReadFile(jsonFilePath) + if err != nil { + log.Fatalf("Failed to read JSON file: %v", err) + } + + // Publish the JSON message + topic := client.Topic(topicName) + result := topic.Publish(ctx, &pubsub.Message{ + Data: jsonData, + }) + + msgID, err := result.Get(ctx) + if err != nil { + log.Fatalf("Failed to publish JSON message: %v", err) + } + fmt.Printf("JSON message published with ID: %s\n", msgID) +} +func publishMessage(ctx context.Context, client *pubsub.Client, topicName, message string) { + topic := client.Topic(topicName) + result := topic.Publish(ctx, &pubsub.Message{ + Data: []byte(message), + }) + + msgID, err := result.Get(ctx) + if err != nil { + log.Fatalf("Failed to publish message: %v", err) + } + fmt.Printf("Message published with ID: %s\n", msgID) +} + +func subscribe(ctx context.Context, client *pubsub.Client, subscriptionName string) { + sub := client.Subscription(subscriptionName) + err := sub.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { + fmt.Printf("Received message: %s\n", string(msg.Data)) + msg.Ack() + }) + if err != nil { + log.Fatalf("Failed to receive messages: %v", err) + } +} diff --git a/saturn/development/startpubsub.sh b/saturn/development/startpubsub.sh new file mode 100755 index 000000000..856acb688 --- /dev/null +++ b/saturn/development/startpubsub.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# Start the Pub/Sub emulator in the background +gcloud beta emulators pubsub start --project=mitbattlecode --host-port=127.0.0.1:8514 & + +# Wait for the emulator to start +sleep 5 + +# set env variables +export PUBSUB_EMULATOR_HOST=127.0.0.1:8514 + +# create topic +./pubsubclient create-topic testing-saturn + +# create pull subscription +./pubsubclient create-pull-subscription testing-saturn test + +# to view all commands +./pubsubclient + +# Keep the container running and accept more commands +while true; do + echo "Enter a command to run (e.g., ./pubsubclient , all paths /development/...):" + read -e -p "$ " cmd + eval "$cmd" +done From 4832f07430d3cf1948e0a7f663138ae8f2bae4d9 Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Tue, 21 Jan 2025 19:13:44 -0500 Subject: [PATCH 07/11] adding saturn readme --- saturn/README.md | 242 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 saturn/README.md diff --git a/saturn/README.md b/saturn/README.md new file mode 100644 index 000000000..1d9ad2a55 --- /dev/null +++ b/saturn/README.md @@ -0,0 +1,242 @@ +# Saturn: + + + +Saturn is a compute cluster designed to efficiently compile competitor bots and execute matches. This system is optimized for rapid job processing and seamless scalability, leveraging Google Cloud's infrastructure for dynamic resource allocation. + + + +--- + + + +## **Core Functionalities** + + + +Saturn specializes in two primary operations: + + + +1. **Compilation of Source Code** + +Converts submitted source code into executable binaries. + +2. **Match Execution** + +Executes matches between compiled binaries and generates replay file results. + + + +--- + + + +## **Data Flow** + + + +1. **Job Requests** + +Received through dedicated Pub/Sub topics. + +2. **Processing** + +Saturn uses the appropriate pipeline for job execution: compilation or match processing. + +3. **Code and Engine Retrieval** + +Pulls the scaffold and game engine from GitHub using a Git token stored in Google Secret Manager. + +4. **Result Storage** + +Results and artifacts are stored in Google Cloud Storage. + +5. **Report Delivery** + +Result reports are sent back to Siarnaq via HTTP POST. + + + +--- + + + +# **Local Development Guide for Saturn** + + + +## **Overview** + + + +Simulating Saturn’s distributed system locally can be challenging. To facilitate this, a Docker image with a Pub/Sub emulator is provided in the `development` folder. + + + +--- + + + +## **Development Environment Setup** + + + +### **Prerequisites** + + + +- Docker installed on your local machine + +- Access to Google Secret Manager + + + +--- + + + +### **Steps** + + + +#### 1. Prepare Secret File + +- Access Google Secret Manager. + +- Create a `secret.json` file in the `development` directory containing the required Git token. + + + +#### 2. Build Docker Image + +Navigate to the `development` directory and run: + +```bash + +cd development + +docker build -t pubsub-emulator . + +``` + +## **Running the System** + + + +Two processes must run simultaneously: + + + +### **1. Start Pub/Sub Emulator** + + + +In the `development` directory, execute: + +```bash + +docker run -it --rm --network host \ + +-v $(pwd):/development \ + +pubsub-emulator + +``` + +### **2. Build and Run Modified Saturn + +In the `saturn` directory: + +Build Saturn Image: + +```bash + +docker build -t saturn . + +``` + +Run Saturn Container: + +```bash + +docker run --network host \ + +-e PUBSUB_EMULATOR_HOST=localhost:8514 \ + +-v $(pwd)/development:/development \ + +saturn -subscription=test -project=mitbattlecode -onsaturn=false \ + +-secret="/development/secret.json" + +``` + + +## **Interacting with the System** + +When the system starts, the `pubsub-emulator` image runs the `./startpubsub.sh` script, which performs the following tasks: +1. Starts the Pub/Sub emulator, listening on port `8514`. +2. Creates a topic called `testing-saturn`. +3. Creates a subscription called `test` (used to pull messages). + +--- + +### **Sending Messages to Pub/Sub** + +The Pub/Sub process listens for commands. Execute `./pubsubclient` tool to view the full list of available commands. +Here are two quick and commonly used commands: + +- **`./pubsubclient pub compile`**: Sends the message defined in `compile.json`. +- **`./pubsubclient pub execute`**: Sends the message defined in `execute.json`. + + +### **Configuring Messages** + + + +- Modify the `compile.json` or `execute.json`files to specify different languages and different file paths for source code, binaries and report-url. + +- Set the report path for local storage. + +- Use `/development/` paths in both images for seamless synchronization. + + + +--- + + + +## **File Management** + + + +- The `development` directory is mounted as `/development` in both images. + +- Any changes made to the `development` directory are automatically reflected in the images. + +- You can create or modify JSON files for the Pub/Sub emulator and add source code for Saturn without rebuilding the Docker images. + + + +--- + + + +## **Security Note** + + + +Files matching `*secret.*` are included in `.gitignore` to prevent sensitive information from being accidentally committed. + + + +--- + + + +## **Example JSON Files** + + + +Refer to the `compile.json` and `execute.json` files for examples of messages sent through Pub/Sub. Adjust these files as necessary for testing and development. Don't commit any changes to these files. From f5d6f23c0f7134c990cd58fa81c1808c16611bff Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Sat, 25 Jan 2025 09:35:30 -0500 Subject: [PATCH 08/11] fixed bash with rlwrap in dockerfile --- saturn/README.md | 11 ++--------- saturn/development/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/saturn/README.md b/saturn/README.md index 1d9ad2a55..ab90c1cea 100644 --- a/saturn/README.md +++ b/saturn/README.md @@ -102,8 +102,8 @@ Simulating Saturn’s distributed system locally can be challenging. To facilita #### 1. Prepare Secret File -- Access Google Secret Manager. - +- Access our Google Cloud's Project Secret Manager. +- Copy the Git token from our production-saturn secret - Create a `secret.json` file in the `development` directory containing the required Git token. @@ -115,7 +115,6 @@ Navigate to the `development` directory and run: ```bash cd development - docker build -t pubsub-emulator . ``` @@ -137,9 +136,7 @@ In the `development` directory, execute: ```bash docker run -it --rm --network host \ - -v $(pwd):/development \ - pubsub-emulator ``` @@ -161,13 +158,9 @@ Run Saturn Container: ```bash docker run --network host \ - -e PUBSUB_EMULATOR_HOST=localhost:8514 \ - -v $(pwd)/development:/development \ - saturn -subscription=test -project=mitbattlecode -onsaturn=false \ - -secret="/development/secret.json" ``` diff --git a/saturn/development/Dockerfile b/saturn/development/Dockerfile index 08c217274..6fa161869 100644 --- a/saturn/development/Dockerfile +++ b/saturn/development/Dockerfile @@ -25,4 +25,4 @@ RUN go build -o pubsubclient pubsubclient.go EXPOSE 8514 # Run the entry point script -ENTRYPOINT ["rlwrap", "/usr/local/bin/startpubsub.sh"] +ENTRYPOINT ["rlwrap", "bash", "/usr/local/bin/startpubsub.sh"] From 37c3f71ac02e4c6f4950a3378baaca0d28735d6e Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Fri, 2 Jan 2026 22:57:48 -0500 Subject: [PATCH 09/11] development env improved --- saturn/.gitignore | 14 +- saturn/Dockerfile | 20 +- saturn/README.md | 485 +++++++++++++----- saturn/development/Dockerfile | 44 +- .../compile.template.json} | 10 +- .../execute.template.json} | 12 +- saturn/development/docker-compose.yml | 50 ++ saturn/development/pubsubclient.go | 29 +- saturn/development/startpubsub.sh | 26 - saturn/development/test-data/.gitkeep | 0 10 files changed, 499 insertions(+), 191 deletions(-) rename saturn/development/{compile.json => configs/compile.template.json} (63%) rename saturn/development/{execute.json => configs/execute.template.json} (74%) create mode 100644 saturn/development/docker-compose.yml delete mode 100755 saturn/development/startpubsub.sh create mode 100644 saturn/development/test-data/.gitkeep diff --git a/saturn/.gitignore b/saturn/.gitignore index e773fc85d..31938f073 100644 --- a/saturn/.gitignore +++ b/saturn/.gitignore @@ -20,5 +20,17 @@ # Go workspace file go.work -# Secret file for local dev +# Development environment +development/secrets/ +development/runs/ + +# Generated config files in development (templates are tracked) +development/configs/*.json +!development/configs/*.template.json + +# Keep test-data structure but ignore contents +development/test-data/* +!development/test-data/.gitkeep + +# Secret files for local dev (TODO: eliminate local secret storage) *secret.* diff --git a/saturn/Dockerfile b/saturn/Dockerfile index 2a7ab33d2..1ee15a2a5 100644 --- a/saturn/Dockerfile +++ b/saturn/Dockerfile @@ -13,15 +13,17 @@ RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /saturn -ldflags="-s -w" . # Link intermediate container for python FROM python:3.12-slim-bookworm as python -FROM openjdk:21-jdk-slim-bookworm - -# Setup -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list && \ - apt-get update - -# Install JDK8. The base image provides JDK21, but we still need JDK8 to -# run matches with java8 -RUN apt-get install -y -t unstable openjdk-8-jdk +FROM eclipse-temurin:21-jdk + +# Setup - Install JDK8 (base image provides JDK21) +# Add Ubuntu universe repository for openjdk-8 +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository ppa:openjdk-r/ppa -y && \ + apt-get update && \ + apt-get install -y openjdk-8-jdk && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Copy Python executable, libraries, standard library, site-packages, include files, binary files, and Python path COPY --from=python /usr/local/bin/python3.12 /usr/local/bin/python3.12 diff --git a/saturn/README.md b/saturn/README.md index ab90c1cea..5396af229 100644 --- a/saturn/README.md +++ b/saturn/README.md @@ -1,235 +1,454 @@ -# Saturn: +# Saturn +Saturn is a compute cluster for compiling competitor bots and executing matches. It's designed for rapid job processing and seamless scalability using Google Cloud infrastructure. +## Core Functionality -Saturn is a compute cluster designed to efficiently compile competitor bots and execute matches. This system is optimized for rapid job processing and seamless scalability, leveraging Google Cloud's infrastructure for dynamic resource allocation. +1. **Compilation** - Converts source code to executable binaries +2. **Match Execution** - Runs matches between compiled binaries and generates replay files +## Data Flow +1. Job requests arrive via Pub/Sub topics +2. Saturn processes jobs using the appropriate pipeline (Java 8, Java 21, or Python 3) +3. Code and game engine scaffolds are pulled from GitHub (using Git token from Secret Manager) +4. Results stored in Google Cloud Storage (or local filesystem in development) +5. Reports sent back to Siarnaq via HTTP POST --- +# Local Development +## Quick Start -## **Core Functionalities** +**Prerequisites:** Docker, gcloud CLI (authenticated) +```bash +cd saturn +make dev-fetch-secret # Fetch secrets from GCP +make dev-build # Build Docker images +make dev-docker-up # Start all services (creates new run) + +# In another terminal: +make dev-compile # Test compilation +make dev-execute # Test execution +``` +To stop services: +```bash +make dev-docker-down +``` -Saturn specializes in two primary operations: +**Note:** After making code changes to Saturn's Go code, rebuild with: +```bash +make dev-docker-down +make dev-build +make dev-docker-up +``` +--- +## Architecture Overview -1. **Compilation of Source Code** +The development environment consists of two Docker containers: -Converts submitted source code into executable binaries. +1. **saturn-dev** - The main Saturn worker that processes compile and execute jobs + - Runs the Saturn Go application + - Includes Java 8, Java 21 (Eclipse Temurin), and Python 3.12 + - Clones and manages battlecode scaffolds + - Executes Gradle builds and matches -2. **Match Execution** +2. **saturn-pubsub-dev** - Local Pub/Sub emulator with custom client + - Runs Google Cloud Pub/Sub emulator + - Includes custom Go client (`pubsubclient`) for sending test messages + - Provides interactive shell with command history (rlwrap) -Executes matches between compiled binaries and generates replay file results. +### Run Management +Each time you start services with `make dev-docker-up`, a new **run** is created with a unique timestamp ID (e.g., `20260102_222656`). This keeps test runs isolated: +``` +development/runs/ +├── 20260102_222656/ # Run ID (timestamp) +│ ├── scaffolds/ # Cloned game scaffolds +│ ├── logs/ # Container logs +│ │ └── combined.log +│ └── requests/ # Individual job requests +│ ├── compile_20260102_222719/ +│ │ ├── config.json +│ │ ├── report.txt +│ │ └── binary.zip +│ └── execute_20260102_222845/ +│ ├── config.json +│ ├── report.txt +│ └── replay.bc25java +└── latest # Symlink to current run +``` --- +## Development Workflow +### 1. Sending Test Jobs -## **Data Flow** - - - -1. **Job Requests** - -Received through dedicated Pub/Sub topics. +Use the Make commands to send test jobs. Each job creates a unique request directory with timestamped ID: -2. **Processing** - -Saturn uses the appropriate pipeline for job execution: compilation or match processing. +```bash +make dev-compile # Sends a compile job +make dev-execute # Sends an execute job +``` -3. **Code and Engine Retrieval** +These commands: +1. Generate a config from the template (`development/configs/*.template.json`) +2. Create a unique request directory under the current run +3. Send the job via Pub/Sub to Saturn +4. Saturn processes the job and writes results to the request directory -Pulls the scaffold and game engine from GitHub using a Git token stored in Google Secret Manager. +### 2. Viewing Logs -4. **Result Storage** +```bash +make dev-docker-logs # View logs for current run +make dev-view-run RUN_ID= # View logs for specific run +make dev-list-runs # List all test runs +``` -Results and artifacts are stored in Google Cloud Storage. +### 3. Managing Test Runs -5. **Report Delivery** +```bash +make dev-list-runs # List all runs with scaffold/log info +make dev-clean-runs # Clean old runs (keeps latest 5) +``` -Result reports are sent back to Siarnaq via HTTP POST. +### 4. Interactive Pub/Sub Client +For more control, use the interactive Pub/Sub client: +```bash +make dev-shell-pubsub # Open shell in Pub/Sub container +./pubsubclient # Show available commands +./pubsubclient list-topics # List all topics +./pubsubclient subscribe test # Listen for messages +``` --- +## Job Configuration + +Jobs are configured using JSON template files in `development/configs/`: + +### Compile Job (`compile.template.json`) + +```json +{ + "episode": { + "name": "bc25java", + "language": "java21", // java8, java21, or python3 + "scaffold": "https://github.com/battlecode/battlecode25-scaffold" + }, + "metadata": { + "report-url": "{{REPORT_PATH}}", // Auto-filled by make command + "task-type": "compile" + }, + "details": { + "source": { + "bucket": "local", + "name": "/development/test-data/source/java21.zip" + }, + "binary": { + "bucket": "local", + "name": "{{BINARY_PATH}}" // Auto-filled by make command + }, + "team-name": "test", + "package": "examplefuncsplayer" // Main package/module name + } +} +``` +### Execute Job (`execute.template.json`) + +```json +{ + "episode": { + "name": "bc25java", + "language": "java21", + "scaffold": "https://github.com/battlecode/battlecode25-scaffold" + }, + "metadata": { + "report-url": "{{REPORT_PATH}}", + "task-type": "execute" + }, + "details": { + "maps": ["fix", "galaxy", "gridworld", "quack", "sierpinski"], + "replay": { + "bucket": "local", + "name": "{{REPLAY_PATH}}" // Auto-filled by make command + }, + "alternate-order": true, + "a": { + "binary": { + "bucket": "local", + "name": "/development/test-data/binary/java21.zip" + }, + "team-name": "team-a", + "package": "examplefuncsplayer" + }, + "b": { + "binary": { + "bucket": "local", + "name": "/development/test-data/binary/java21.zip" + }, + "team-name": "team-b", + "package": "examplefuncsplayer" + } + } +} +``` -# **Local Development Guide for Saturn** - - - -## **Overview** - - +### Template Placeholders -Simulating Saturn’s distributed system locally can be challenging. To facilitate this, a Docker image with a Pub/Sub emulator is provided in the `development` folder. +When using `make dev-compile` or `make dev-execute`, the following placeholders are automatically replaced: +- `{{REPORT_PATH}}` - Path to write the result report +- `{{BINARY_PATH}}` - Path to write the compiled binary (compile jobs) +- `{{REPLAY_PATH}}` - Path to write the game replay (execute jobs) +These paths are generated based on the current run ID and request timestamp. --- +## Directory Structure - -## **Development Environment Setup** - - - -### **Prerequisites** - - - -- Docker installed on your local machine - -- Access to Google Secret Manager - - +``` +saturn/ +├── cmd/saturn/main.go # Saturn entry point +├── pkg/ # Saturn Go packages +│ ├── run/ # Job execution logic +│ ├── saturn/ # Core types and interfaces +│ └── storage/ # Storage backends (GCS, local) +├── Dockerfile # Saturn production image +├── Makefile # Development commands +└── development/ + ├── Dockerfile # Pub/Sub emulator image + ├── docker-compose.yml # Service orchestration + ├── pubsubclient.go # Custom Pub/Sub client + ├── configs/ # Job templates + │ ├── compile.template.json + │ └── execute.template.json + ├── secrets/ # GCP secrets (git-ignored) + │ └── secret.json # Git token for scaffold cloning + ├── test-data/ # Test source/binary files (git-ignored) + │ ├── source/ + │ │ ├── java21.zip + │ │ └── py3.zip + │ └── binary/ + │ └── java21.zip + └── runs/ # Test run outputs (git-ignored) + ├── latest # Points to current run + └── 20260102_222656/ # Run directories (timestamped) + ├── scaffolds/ # Cloned game scaffolds + ├── logs/ # Container logs + └── requests/ # Job request outputs +``` --- +## Make Commands Reference +### Setup Commands +```bash +make dev-fetch-secret # Fetch secrets from GCP Secret Manager +make dev-build # Build Docker images for both containers +``` -### **Steps** - - - -#### 1. Prepare Secret File - -- Access our Google Cloud's Project Secret Manager. -- Copy the Git token from our production-saturn secret -- Create a `secret.json` file in the `development` directory containing the required Git token. - - - -#### 2. Build Docker Image - -Navigate to the `development` directory and run: - +### Service Management ```bash +make dev-docker-up # Start services (creates new run, tails logs) +make dev-docker-down # Stop all services +make dev-docker-logs # View logs for current run +make dev-docker-rebuild # Rebuild images and restart services +``` -cd development -docker build -t pubsub-emulator . +### Testing Commands +```bash +make dev-compile # Send compile job to Saturn +make dev-execute # Send execute job to Saturn +``` +### Run Management +```bash +make dev-list-runs # List all test runs with info +make dev-view-run RUN_ID= # View logs for specific run +make dev-clean-runs # Clean old runs (keeps latest 5) ``` -## **Running the System** +### Container Access +```bash +make dev-shell-pubsub # Open shell in Pub/Sub container +make dev-shell-saturn # Open shell in Saturn container +make dev-pubsub-interactive # Interactive Pub/Sub client with rlwrap +``` +### Utilities +```bash +make dev-clean # Clean up containers and images +make help # Show all available commands +``` +--- -Two processes must run simultaneously: +## Supported Languages +Saturn supports three language environments: +### Java 21 (Primary) +- **Scaffold:** battlecode25-scaffold (Java track) +- **Base Image:** Eclipse Temurin 21-jdk +- **JAVA_HOME:** `/opt/java/openjdk` +- **Test File:** `test-data/source/java21.zip` -### **1. Start Pub/Sub Emulator** +### Java 8 (Legacy) +- **Scaffold:** Older battlecode scaffolds +- **Installation:** openjdk-8-jdk +- **JAVA_HOME:** `/usr/lib/jvm/java-8-openjdk-amd64` +### Python 3.12 +- **Scaffold:** battlecode25-scaffold (Python track) +- **Installation:** Custom from python:3.12-slim-bookworm +- **Test File:** `test-data/source/py3.zip` +--- -In the `development` directory, execute: +## Troubleshooting +### Services won't start ```bash - -docker run -it --rm --network host \ --v $(pwd):/development \ -pubsub-emulator - +make dev-clean +make dev-build +make dev-docker-up ``` -### **2. Build and Run Modified Saturn - -In the `saturn` directory: - -Build Saturn Image: - +### Can't fetch secrets ```bash +gcloud auth login +gcloud config set project mitbattlecode +make dev-fetch-secret +``` -docker build -t saturn . +### Code changes not showing +After modifying Saturn's Go source code: +```bash +make dev-docker-down +make dev-build +make dev-docker-up +``` +### Job fails immediately +Check the logs: +```bash +make dev-docker-logs +# or +docker logs saturn-dev --tail 100 ``` -Run Saturn Container: +Common issues: +- Missing test data files in `test-data/source/` +- Incorrect package name in config +- Java path misconfiguration (should be `/opt/java/openjdk`) +### Pub/Sub messages not received ```bash +# Check if topic exists +make dev-shell-pubsub +./pubsubclient list-topics -docker run --network host \ --e PUBSUB_EMULATOR_HOST=localhost:8514 \ --v $(pwd)/development:/development \ -saturn -subscription=test -project=mitbattlecode -onsaturn=false \ --secret="/development/secret.json" +# Check if subscription exists +docker exec saturn-pubsub-dev gcloud pubsub subscriptions list --project=mitbattlecode +# Recreate topic and subscription +./pubsubclient create-topic testing-saturn +./pubsubclient create-pull-subscription testing-saturn test ``` +### Scaffold cloning fails +Ensure `development/secrets/secret.json` contains a valid GitHub token: +```bash +make dev-fetch-secret +cat development/secrets/secret.json +``` -## **Interacting with the System** - -When the system starts, the `pubsub-emulator` image runs the `./startpubsub.sh` script, which performs the following tasks: -1. Starts the Pub/Sub emulator, listening on port `8514`. -2. Creates a topic called `testing-saturn`. -3. Creates a subscription called `test` (used to pull messages). +### Clean start from scratch +```bash +make dev-docker-down +make dev-clean +rm -rf development/runs/* +make dev-fetch-secret +make dev-build +make dev-docker-up +``` --- -### **Sending Messages to Pub/Sub** - -The Pub/Sub process listens for commands. Execute `./pubsubclient` tool to view the full list of available commands. -Here are two quick and commonly used commands: - -- **`./pubsubclient pub compile`**: Sends the message defined in `compile.json`. -- **`./pubsubclient pub execute`**: Sends the message defined in `execute.json`. - - -### **Configuring Messages** - - - -- Modify the `compile.json` or `execute.json`files to specify different languages and different file paths for source code, binaries and report-url. - -- Set the report path for local storage. - -- Use `/development/` paths in both images for seamless synchronization. +## Manual Setup (Without Make) +If you prefer not to use Make: +```bash +# 1. Fetch secrets +mkdir -p development/secrets +gcloud secrets versions access latest \ + --secret="production-saturn" \ + --project="mitbattlecode" \ + > development/secrets/secret.json + +# 2. Build Docker images +cd development +docker-compose build + +# 3. Start services +RUN_ID=$(date +%Y%m%d_%H%M%S) +mkdir -p runs/$RUN_ID/scaffolds runs/$RUN_ID/logs +echo $RUN_ID > runs/latest +SCAFFOLD_DIR=./runs/$RUN_ID/scaffolds docker-compose up -d + +# 4. Send test jobs +# For compile: +docker exec saturn-pubsub-dev ./pubsubclient publish-json testing-saturn /development/configs/compile.template.json + +# For execute: +docker exec saturn-pubsub-dev ./pubsubclient publish-json testing-saturn /development/configs/execute.template.json +``` --- +## Production Deployment +Saturn runs on Google Cloud Run, triggered by Pub/Sub messages from the `saturn-compile` and `saturn-execute` topics. -## **File Management** - - - -- The `development` directory is mounted as `/development` in both images. - -- Any changes made to the `development` directory are automatically reflected in the images. - -- You can create or modify JSON files for the Pub/Sub emulator and add source code for Saturn without rebuilding the Docker images. - +### Environment Variables +- `PUBSUB_PROJECT_ID` - GCP project ID for Pub/Sub +- `SUBSCRIPTION_NAME` - Subscription to pull jobs from +- `GCS_BUCKET` - Google Cloud Storage bucket for artifacts +- `SATURN_REVISION` - Build revision for logging/monitoring +### Secrets +- GitHub personal access token (from Secret Manager) for cloning scaffolds --- +## Contributing +When making changes: -## **Security Note** - - - -Files matching `*secret.*` are included in `.gitignore` to prevent sensitive information from being accidentally committed. - - +1. Test locally using the development environment +2. Ensure both compile and execute jobs work for all languages +3. Update this README if adding new features or changing workflow +4. Check logs for warnings or errors +5. Clean up test runs before committing: `make dev-clean-runs` --- +## Additional Resources - -## **Example JSON Files** - - - -Refer to the `compile.json` and `execute.json` files for examples of messages sent through Pub/Sub. Adjust these files as necessary for testing and development. Don't commit any changes to these files. +- [Battlecode 2025 Scaffold](https://github.com/battlecode/battlecode25-scaffold) +- [Google Cloud Pub/Sub Documentation](https://cloud.google.com/pubsub/docs) +- [Docker Compose Reference](https://docs.docker.com/compose/) diff --git a/saturn/development/Dockerfile b/saturn/development/Dockerfile index 6fa161869..870d65c2e 100644 --- a/saturn/development/Dockerfile +++ b/saturn/development/Dockerfile @@ -10,10 +10,6 @@ RUN apt-get update && apt-get install -y curl gnupg rlwrap && \ # Install Pub/Sub emulator RUN apt-get install google-cloud-cli-pubsub-emulator -# Copy the start script -COPY startpubsub.sh /usr/local/bin/startpubsub.sh -RUN chmod +x /usr/local/bin/startpubsub.sh - # Set the working directory WORKDIR /app @@ -24,5 +20,41 @@ RUN go build -o pubsubclient pubsubclient.go # Expose the Pub/Sub emulator port EXPOSE 8514 -# Run the entry point script -ENTRYPOINT ["rlwrap", "bash", "/usr/local/bin/startpubsub.sh"] +# Set Pub/Sub emulator environment variable +ENV PUBSUB_EMULATOR_HOST=0.0.0.0:8514 + +# Create startup script inline +RUN cat > /usr/local/bin/start.sh <<'EOF' +#!/bin/bash +set -e + +# Start the Pub/Sub emulator in the background +# Filter out noisy HTTP/2 and connection warnings while keeping useful logs +gcloud beta emulators pubsub start \ + --project=mitbattlecode \ + --host-port=0.0.0.0:8514 \ + 2>&1 & + +EMULATOR_PID=$! + +# Wait for the emulator to be ready +echo "Waiting for Pub/Sub emulator to start..." +sleep 5 + +# Set environment variable +export PUBSUB_EMULATOR_HOST=0.0.0.0:8514 + +# Create topic and subscription +echo "Creating topic and subscription..." +./pubsubclient create-topic testing-saturn +./pubsubclient create-pull-subscription testing-saturn test +echo "✓ Pub/Sub emulator is ready" + +# Wait for the emulator process (keeps container alive and shows logs) +wait $EMULATOR_PID +EOF + +RUN chmod +x /usr/local/bin/start.sh + +# Run the startup script +ENTRYPOINT ["/usr/local/bin/start.sh"] diff --git a/saturn/development/compile.json b/saturn/development/configs/compile.template.json similarity index 63% rename from saturn/development/compile.json rename to saturn/development/configs/compile.template.json index 44694f295..d53255cd1 100644 --- a/saturn/development/compile.json +++ b/saturn/development/configs/compile.template.json @@ -1,21 +1,21 @@ { "episode": { - "name": "bc25python", - "language": "py3", + "name": "bc25java", + "language": "java21", "scaffold": "https://github.com/battlecode/battlecode25-scaffold" }, "metadata": { - "report-url": "/development/test/report.txt", + "report-url": "{{REPORT_PATH}}", "task-type": "compile" }, "details": { "source": { "bucket": "local", - "name": "/development/test/python.zip" + "name": "/development/test-data/source/java21.zip" }, "binary": { "bucket": "local", - "name": "/development/test/binary.zip" + "name": "{{BINARY_PATH}}" }, "team-name": "test", "package": "examplefuncsplayer" diff --git a/saturn/development/execute.json b/saturn/development/configs/execute.template.json similarity index 74% rename from saturn/development/execute.json rename to saturn/development/configs/execute.template.json index bf2200166..705cdae9d 100644 --- a/saturn/development/execute.json +++ b/saturn/development/configs/execute.template.json @@ -5,7 +5,7 @@ "scaffold": "https://github.com/battlecode/battlecode25-scaffold" }, "metadata": { - "report-url": "/development/test/report.txt", + "report-url": "{{REPORT_PATH}}", "task-type": "execute" }, "details": { @@ -18,17 +18,17 @@ ], "replay": { "bucket": "local", - "name": "/development/test/replay.bc25java" + "name": "{{REPLAY_PATH}}" }, "alternate-order": true, "a": { "source": { "bucket": "local", - "name": "/development/test/source1.zip" + "name": "/development/test-data/source/java21.zip" }, "binary": { "bucket": "local", - "name": "/development/test/binary1.zip" + "name": "/development/test-data/binary/java21.zip" }, "team-name": "test1", "package": "examplefuncsplayer" @@ -36,11 +36,11 @@ "b": { "source": { "bucket": "local", - "name": "/development/test/source2.zip" + "name": "/development/test-data/source/java21.zip" }, "binary": { "bucket": "local", - "name": "/development/test/binary2.zip" + "name": "/development/test-data/binary/java21.zip" }, "team-name": "test2", "package": "examplefuncsplayer" diff --git a/saturn/development/docker-compose.yml b/saturn/development/docker-compose.yml new file mode 100644 index 000000000..74fe07a5d --- /dev/null +++ b/saturn/development/docker-compose.yml @@ -0,0 +1,50 @@ +services: + pubsub-emulator: + build: + context: . + dockerfile: Dockerfile + container_name: saturn-pubsub-dev + ports: + - "8514:8514" + volumes: + - ./:/development + networks: + - saturn-dev + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + healthcheck: + test: ["CMD", "./pubsubclient", "list-topics"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + + saturn: + build: + context: .. + dockerfile: Dockerfile + container_name: saturn-dev + environment: + - PUBSUB_EMULATOR_HOST=pubsub-emulator:8514 + - RUN_ID=${RUN_ID:-default} + volumes: + - ./:/development + - ${SCAFFOLD_DIR:-./scaffolds}:/scaffolds + command: > + -subscription=test + -project=mitbattlecode + -onsaturn=false + -secret=/development/secrets/secret.json + -scaffold=/scaffolds + depends_on: + pubsub-emulator: + condition: service_healthy + networks: + - saturn-dev + +networks: + saturn-dev: + driver: bridge diff --git a/saturn/development/pubsubclient.go b/saturn/development/pubsubclient.go index d5cbc4e24..c3454344b 100644 --- a/saturn/development/pubsubclient.go +++ b/saturn/development/pubsubclient.go @@ -19,9 +19,10 @@ func main() { "subscribe ", } quickCommands := []string{ - "compile: to send message in compile.json", - "execute: to send message in execute.json", - "sub: to recieve message", + "compile: to send message in configs/compile.json", + "execute: to send message in configs/execute.json", + "sub: to receive messages from subscription", + "list-topics: list all available topics", } if len(os.Args) < 2 { fmt.Println("Usage: ./pubsubclient [args]") @@ -52,14 +53,17 @@ func main() { switch command { case "compile": - publishJSONMessage(ctx, client, "testing-saturn", "/development/compile.json") + publishJSONMessage(ctx, client, "testing-saturn", "/development/configs/compile.json") case "execute": - publishJSONMessage(ctx, client, "testing-saturn", "/development/execute.json") + publishJSONMessage(ctx, client, "testing-saturn", "/development/configs/execute.json") case "sub": subscribe(ctx, client, "test") + case "list-topics": + listTopics(ctx, client) + case "create-topic": if len(os.Args) < 3 { log.Fatalf("Usage: ./pubsubclient create-topic ") @@ -164,3 +168,18 @@ func subscribe(ctx context.Context, client *pubsub.Client, subscriptionName stri log.Fatalf("Failed to receive messages: %v", err) } } + +func listTopics(ctx context.Context, client *pubsub.Client) { + it := client.Topics(ctx) + fmt.Println("Available topics:") + for { + topic, err := it.Next() + if err != nil { + if err.Error() == "no more items in iterator" { + break + } + log.Fatalf("Failed to list topics: %v", err) + } + fmt.Printf(" - %s\n", topic.ID()) + } +} diff --git a/saturn/development/startpubsub.sh b/saturn/development/startpubsub.sh deleted file mode 100755 index 856acb688..000000000 --- a/saturn/development/startpubsub.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -# Start the Pub/Sub emulator in the background -gcloud beta emulators pubsub start --project=mitbattlecode --host-port=127.0.0.1:8514 & - -# Wait for the emulator to start -sleep 5 - -# set env variables -export PUBSUB_EMULATOR_HOST=127.0.0.1:8514 - -# create topic -./pubsubclient create-topic testing-saturn - -# create pull subscription -./pubsubclient create-pull-subscription testing-saturn test - -# to view all commands -./pubsubclient - -# Keep the container running and accept more commands -while true; do - echo "Enter a command to run (e.g., ./pubsubclient , all paths /development/...):" - read -e -p "$ " cmd - eval "$cmd" -done diff --git a/saturn/development/test-data/.gitkeep b/saturn/development/test-data/.gitkeep new file mode 100644 index 000000000..e69de29bb From d2341777ef55438333068f0c314c56b2bdbc5a8b Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Fri, 2 Jan 2026 22:58:17 -0500 Subject: [PATCH 10/11] makefile for lcoal dev and fixing scaffoled java_home --- saturn/Makefile | 243 +++++++++++++++++++++++++++++++++++++ saturn/pkg/run/scaffold.go | 2 +- 2 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 saturn/Makefile diff --git a/saturn/Makefile b/saturn/Makefile new file mode 100644 index 000000000..4a2b3aaa5 --- /dev/null +++ b/saturn/Makefile @@ -0,0 +1,243 @@ +.PHONY: dev-fetch-secret dev-build dev-compile dev-execute dev-shell dev-shell-saturn dev-clean help +.PHONY: dev-docker-up dev-docker-down dev-docker-logs dev-docker-rebuild dev-pubsub-interactive +.PHONY: dev-list-runs dev-clean-runs dev-view-run + +help: ## Show this help message + @echo 'Usage: make [target]' + @echo '' + @echo 'Quick Start:' + @echo ' make dev-fetch-secret # Fetch secrets from GCP' + @echo ' make dev-build # Build Docker images' + @echo ' make dev-docker-up # Start all services (creates new run)' + @echo ' make dev-compile # Test compilation' + @echo ' make dev-execute # Test execution' + @echo ' make dev-pubsub-interactive # Interactive Pub/Sub client' + @echo '' + @echo 'Run Management:' + @echo ' make dev-list-runs # List all test runs' + @echo ' make dev-docker-logs # View logs for current run' + @echo ' make dev-view-run RUN_ID=X # View logs for specific run' + @echo ' make dev-clean-runs # Clean old runs (keeps 5)' + @echo '' + @echo 'All available targets:' + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-27s\033[0m %s\n", $$1, $$2}' + +dev-fetch-secret: ## Fetch secrets from Google Cloud Secret Manager + @# TODO: Eliminate storing secret locally + @echo "Fetching secret from GCP Secret Manager..." + @# Check if gcloud is installed + @if ! command -v gcloud >/dev/null 2>&1; then \ + echo "Error: gcloud CLI is not installed"; \ + echo "Install from: https://cloud.google.com/sdk/docs/install"; \ + exit 1; \ + fi + @# Check if authenticated + @if ! gcloud auth list --filter=status:ACTIVE --format="value(account)" >/dev/null 2>&1; then \ + echo "Error: Not authenticated with gcloud"; \ + echo "Run: gcloud auth login"; \ + exit 1; \ + fi + @mkdir -p development/secrets + @gcloud secrets versions access latest \ + --secret="production-saturn" \ + --project="mitbattlecode" \ + > development/secrets/secret.json + @echo "✓ Secret saved to development/secrets/secret.json" + +dev-build: ## Build all Docker images + @echo "Building Docker images..." + @cd development && docker-compose build + @echo "✓ Docker images built" + +dev-compile: ## Send a compile message + @# Check if a run is active + @if [ ! -f development/runs/latest ]; then \ + echo "Error: No active run found. Start services with 'make dev-docker-up' first."; \ + exit 1; \ + fi + @# Generate request ID + $(eval RUN_ID := $(shell cat development/runs/latest)) + $(eval REQUEST_ID := $(shell date +%Y%m%d_%H%M%S)) + @# Create request directory structure + @mkdir -p development/runs/$(RUN_ID)/requests/compile_$(REQUEST_ID) + @echo "Creating compile request: compile_$(REQUEST_ID)" + @# Generate config from template + @sed \ + -e 's|{{REPORT_PATH}}|/development/runs/$(RUN_ID)/requests/compile_$(REQUEST_ID)/report.txt|g' \ + -e 's|{{BINARY_PATH}}|/development/runs/$(RUN_ID)/requests/compile_$(REQUEST_ID)/binary.zip|g' \ + development/configs/compile.template.json \ + > development/runs/$(RUN_ID)/requests/compile_$(REQUEST_ID)/config.json + @echo "✓ Config generated: development/runs/$(RUN_ID)/requests/compile_$(REQUEST_ID)/config.json" + @# Send compile message with generated config + @docker exec saturn-pubsub-dev ./pubsubclient publish-json testing-saturn /development/runs/$(RUN_ID)/requests/compile_$(REQUEST_ID)/config.json + @echo "✓ Compile message sent" + @echo "✓ Request ID: compile_$(REQUEST_ID)" + @echo "✓ Output directory: development/runs/$(RUN_ID)/requests/compile_$(REQUEST_ID)/" + +dev-execute: ## Send an execute message + @# Check if a run is active + @if [ ! -f development/runs/latest ]; then \ + echo "Error: No active run found. Start services with 'make dev-docker-up' first."; \ + exit 1; \ + fi + @# Generate request ID + $(eval RUN_ID := $(shell cat development/runs/latest)) + $(eval REQUEST_ID := $(shell date +%Y%m%d_%H%M%S)) + @# Create request directory structure + @mkdir -p development/runs/$(RUN_ID)/requests/execute_$(REQUEST_ID) + @echo "Creating execute request: execute_$(REQUEST_ID)" + @# Generate config from template + @sed \ + -e 's|{{REPORT_PATH}}|/development/runs/$(RUN_ID)/requests/execute_$(REQUEST_ID)/report.txt|g' \ + -e 's|{{REPLAY_PATH}}|/development/runs/$(RUN_ID)/requests/execute_$(REQUEST_ID)/replay.bc25java|g' \ + development/configs/execute.template.json \ + > development/runs/$(RUN_ID)/requests/execute_$(REQUEST_ID)/config.json + @echo "✓ Config generated: development/runs/$(RUN_ID)/requests/execute_$(REQUEST_ID)/config.json" + @# Send execute message with generated config + @docker exec saturn-pubsub-dev ./pubsubclient publish-json testing-saturn /development/runs/$(RUN_ID)/requests/execute_$(REQUEST_ID)/config.json + @echo "✓ Execute message sent" + @echo "✓ Request ID: execute_$(REQUEST_ID)" + @echo "✓ Output directory: development/runs/$(RUN_ID)/requests/execute_$(REQUEST_ID)/" + +dev-shell-pubsub: ## Open shell in Pub/Sub container + @docker exec -it saturn-pubsub-dev bash + +dev-pubsub-interactive: ## Open interactive Pub/Sub client with rlwrap + @echo "Starting interactive Pub/Sub client..." + @echo "Available commands will be shown below." + @echo "Use Ctrl+D or Ctrl+C to exit." + @echo "" + @docker exec -it saturn-pubsub-dev rlwrap bash -c '\ + ./pubsubclient; \ + echo ""; \ + echo "Interactive mode - enter commands:"; \ + while true; do \ + read -e -p "$$ " cmd || break; \ + eval "$$cmd"; \ + done' + +dev-shell-saturn: ## Open shell in Saturn container + @docker exec -it saturn-dev bash + +dev-clean: ## Clean up Docker containers and images + @echo "Cleaning up development environment..." + @cd development && docker-compose down -v + @docker rmi development-pubsub-emulator development-saturn 2>/dev/null || true + @echo "✓ Cleanup complete" + +dev-docker-up: ## Start all Docker services and show logs + @# Generate run ID + $(eval RUN_ID := $(shell date +%Y%m%d_%H%M%S)) + @echo "Starting run: $(RUN_ID)" + @echo "" + @# Create run directory structure + @mkdir -p development/runs/$(RUN_ID)/scaffolds + @mkdir -p development/runs/$(RUN_ID)/logs + @echo "$(RUN_ID)" > development/runs/latest + @# Start services with run-specific scaffold directory + @echo "Starting Docker services..." + @cd development && \ + RUN_ID=$(RUN_ID) \ + SCAFFOLD_DIR=./runs/$(RUN_ID)/scaffolds \ + docker-compose up -d + @echo "✓ Services started" + @echo "✓ Run ID: $(RUN_ID)" + @echo "✓ Scaffolds: development/runs/$(RUN_ID)/scaffolds" + @echo "✓ Logs: development/runs/$(RUN_ID)/logs" + @echo "" + @# Start background log capture + @echo "Capturing logs to development/runs/$(RUN_ID)/logs/" + @(cd development && docker-compose logs -f) > development/runs/$(RUN_ID)/logs/combined.log 2>&1 & echo $$! > development/runs/$(RUN_ID)/logs/.logger.pid + @# Tail logs to terminal with colorized container names + @echo "Showing logs (Ctrl+C to exit, services and logging will keep running)..." + @echo "" + @tail -f development/runs/$(RUN_ID)/logs/combined.log | sed \ + -e 's/saturn-pubsub-dev/\x1b[36msaturn-pubsub-dev\x1b[0m/g' \ + -e 's/saturn-dev[^-]/\x1b[33msaturn-dev\x1b[0m/g' + +dev-docker-down: ## Stop all Docker services + @echo "Stopping Docker services..." + @# Stop background logger if running + @if [ -f development/runs/$$(cat development/runs/latest 2>/dev/null)/logs/.logger.pid ]; then \ + kill $$(cat development/runs/$$(cat development/runs/latest)/logs/.logger.pid) 2>/dev/null || true; \ + rm development/runs/$$(cat development/runs/latest)/logs/.logger.pid; \ + fi + @cd development && docker-compose down + @echo "✓ Services stopped" + +dev-docker-logs: ## View logs for current run + @if [ ! -f development/runs/latest ]; then \ + echo "No active run found. Start services with 'make dev-docker-up' first."; \ + exit 1; \ + fi + @RUN_ID=$$(cat development/runs/latest); \ + echo "Viewing logs for run: $$RUN_ID"; \ + echo ""; \ + tail -f development/runs/$$RUN_ID/logs/combined.log | sed \ + -e 's/saturn-pubsub-dev/\x1b[36msaturn-pubsub-dev\x1b[0m/g' \ + -e 's/saturn-dev[^-]/\x1b[33msaturn-dev\x1b[0m/g' + +dev-docker-rebuild: ## Rebuild Docker images and restart services + @echo "Rebuilding and restarting Docker services..." + @cd development && docker-compose up -d --build + @echo "✓ Rebuild complete" + +dev-list-runs: ## List all test runs + @echo "Available test runs:" + @echo "" + @if [ -d development/runs ]; then \ + for run in $$(ls -t development/runs | grep -E '^[0-9]{8}_[0-9]{6}$$'); do \ + echo " $$run"; \ + if [ -d "development/runs/$$run/scaffolds" ]; then \ + scaffold_count=$$(find development/runs/$$run/scaffolds -type f 2>/dev/null | wc -l | tr -d ' '); \ + echo " Scaffolds: $$scaffold_count files"; \ + fi; \ + if [ -f "development/runs/$$run/logs/combined.log" ]; then \ + log_size=$$(du -h development/runs/$$run/logs/combined.log | cut -f1); \ + echo " Logs: $$log_size"; \ + fi; \ + echo ""; \ + done; \ + else \ + echo " No runs found"; \ + fi + @if [ -f development/runs/latest ]; then \ + echo "Latest run: $$(cat development/runs/latest)"; \ + fi + +dev-view-run: ## View logs for a specific run (usage: make dev-view-run RUN_ID=20260102_134523) + @if [ -z "$(RUN_ID)" ]; then \ + echo "Error: Please specify RUN_ID"; \ + echo "Usage: make dev-view-run RUN_ID=20260102_134523"; \ + echo ""; \ + echo "Available runs:"; \ + ls -1 development/runs | grep -E '^[0-9]{8}_[0-9]{6}$$' || echo " No runs found"; \ + exit 1; \ + fi + @if [ ! -f "development/runs/$(RUN_ID)/logs/combined.log" ]; then \ + echo "Error: Run $(RUN_ID) not found or has no logs"; \ + exit 1; \ + fi + @echo "Viewing logs for run: $(RUN_ID)" + @echo "" + @tail -f development/runs/$(RUN_ID)/logs/combined.log | sed \ + -e 's/saturn-pubsub-dev/\x1b[36msaturn-pubsub-dev\x1b[0m/g' \ + -e 's/saturn-dev[^-]/\x1b[33msaturn-dev\x1b[0m/g' + +dev-clean-runs: ## Clean old test runs (keeps latest 5) + @echo "Cleaning old test runs..." + @if [ -d development/runs ]; then \ + runs_to_delete=$$(ls -t development/runs | grep -E '^[0-9]{8}_[0-9]{6}$$' | tail -n +6); \ + if [ -n "$$runs_to_delete" ]; then \ + echo "Deleting old runs:"; \ + for run in $$runs_to_delete; do \ + echo " $$run"; \ + rm -rf development/runs/$$run; \ + done; \ + echo "✓ Cleanup complete"; \ + else \ + echo "No old runs to clean (keeping latest 5)"; \ + fi; \ + else \ + echo "No runs directory found"; \ + fi diff --git a/saturn/pkg/run/scaffold.go b/saturn/pkg/run/scaffold.go index f96c70111..6e8ded598 100644 --- a/saturn/pkg/run/scaffold.go +++ b/saturn/pkg/run/scaffold.go @@ -128,7 +128,7 @@ func NewScaffold(ctx context.Context, episode saturn.Episode, repo *git.Reposito case saturn.Java21: // Modern java21 scaffolds store java in the 'java' subdirectory of the scaffold javaRoot := filepath.Join(root, "java") - s, err := NewJavaScaffold(ctx, episode, repo, javaRoot, "/usr/local/openjdk-21", onSaturn) + s, err := NewJavaScaffold(ctx, episode, repo, javaRoot, "/opt/java/openjdk", onSaturn) if err != nil { return nil, fmt.Errorf("NewJavaScaffold (Java21): %v", err) } From d9516d68f6d8431bc7025baf82fbe8943cff8c6c Mon Sep 17 00:00:00 2001 From: Nour Massri Date: Fri, 2 Jan 2026 23:08:41 -0500 Subject: [PATCH 11/11] adding todo --- saturn/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/saturn/Makefile b/saturn/Makefile index 4a2b3aaa5..3fe2800b4 100644 --- a/saturn/Makefile +++ b/saturn/Makefile @@ -126,6 +126,7 @@ dev-clean: ## Clean up Docker containers and images @echo "✓ Cleanup complete" dev-docker-up: ## Start all Docker services and show logs + @# TODO: Add support for setting logging level @# Generate run ID $(eval RUN_ID := $(shell date +%Y%m%d_%H%M%S)) @echo "Starting run: $(RUN_ID)"