From 57d2936a14efa193b268fb8bcb338f5c8b59d89b Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Fri, 9 Jan 2026 09:06:57 +0100 Subject: [PATCH] feat: add remote URL template loading and source tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add `workflow_dispatch_templates_urls` config option to load templates from remote URLs - Track template source type: "inline", "file", or "url" - Display source badges in UI (blue for local files, purple for URLs) - Add database migration for source_type and source_path columns - Update README and config.example.yaml with URL template documentation 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- README.md | 9 ++-- config.example.yaml | 6 ++- pkg/api/api.go | 2 + pkg/config/config.go | 86 ++++++++++++++++++++++++++++++++++++ pkg/store/postgres.go | 32 +++++++++----- pkg/store/sqlite.go | 22 +++++---- pkg/store/store.go | 2 + ui/src/pages/ApiDocsPage.tsx | 1 + ui/src/pages/GroupPage.tsx | 24 ++++++++++ ui/src/types/index.ts | 4 ++ 10 files changed, 164 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index d468ba7..2bc79d2 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ Users must be in at least one role mapping (`org_role_mapping` or `user_role_map ### Groups and Templates -Groups define pools of runners identified by labels. Each group can have multiple workflow dispatch templates defined inline or loaded from a separate file: +Groups define pools of runners identified by labels. Each group can have multiple workflow dispatch templates defined inline, loaded from local files, or fetched from remote URLs: ```yaml groups: @@ -218,10 +218,13 @@ groups: el-client: "geth" cl-client: "prysm" config: '{"network": "mainnet"}' - # Option 2: Load templates from files (paths relative to config file) + # Option 2: Load templates from local files (paths relative to config file) # workflow_dispatch_templates_files: # - templates/hoodi.yaml # - templates/mainnet.yaml + # Option 3: Load templates from remote URLs + # workflow_dispatch_templates_urls: + # - https://raw.githubusercontent.com/myorg/templates/main/sync-tests.yaml ``` Template file format (`templates/sync-tests.yaml`): @@ -247,7 +250,7 @@ Template file format (`templates/sync-tests.yaml`): cl-client: "lighthouse" ``` -Both inline templates and file templates can be used together - file templates are appended to inline templates. +All template sources can be used together - file and URL templates are appended to inline templates. The UI displays badges indicating the source of each template (inline, local file, or URL). ### Workflow Best Practices diff --git a/config.example.yaml b/config.example.yaml index 4350f49..530149c 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -70,11 +70,13 @@ groups: - self-hosted - synctest - Disk2TB - # Templates can be defined inline or loaded from separate files: + # Templates can be defined inline, loaded from local files, or fetched from remote URLs: # workflow_dispatch_templates_files: # - templates/hoodi.yaml # - templates/mainnet.yaml - # Both can be used together - file templates are appended to inline templates. + # workflow_dispatch_templates_urls: + # - https://raw.githubusercontent.com/myorg/templates/main/sync-tests.yaml + # All sources can be used together - file and URL templates are appended to inline templates. workflow_dispatch_templates: - id: sync-test-hoodi-geth-prysm name: Sync Test (Hoodi) - geth/prysm diff --git a/pkg/api/api.go b/pkg/api/api.go index d2cfdad..6010902 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -1992,6 +1992,8 @@ func SyncGroupsFromConfig(ctx context.Context, log logrus.FieldLogger, st store. DefaultInputs: tmplCfg.Inputs, Labels: tmplCfg.Labels, InConfig: true, + SourceType: tmplCfg.SourceType, + SourcePath: tmplCfg.SourcePath, CreatedAt: now, UpdatedAt: now, } diff --git a/pkg/config/config.go b/pkg/config/config.go index 64f17d1..60868d8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,6 +2,8 @@ package config import ( "fmt" + "io" + "net/http" "os" "path/filepath" "regexp" @@ -113,6 +115,7 @@ type Group struct { RunnerLabels []string `yaml:"runner_labels"` WorkflowDispatchTemplates []WorkflowDispatchTemplate `yaml:"workflow_dispatch_templates"` WorkflowDispatchTemplatesFiles []string `yaml:"workflow_dispatch_templates_files"` + WorkflowDispatchTemplatesURLs []string `yaml:"workflow_dispatch_templates_urls"` } // WorkflowDispatchTemplate represents a workflow dispatch template configuration. @@ -125,6 +128,8 @@ type WorkflowDispatchTemplate struct { Ref string `yaml:"ref"` Inputs map[string]string `yaml:"inputs"` Labels map[string]string `yaml:"labels"` + SourceType string `yaml:"-"` // "inline", "file", or "url" - set during loading + SourcePath string `yaml:"-"` // filename or URL (empty for inline) - set during loading } // Load reads and parses configuration from a YAML file. @@ -142,12 +147,20 @@ func Load(path string) (*Config, error) { return nil, fmt.Errorf("parsing config file: %w", err) } + // Mark inline templates with source type. + markInlineTemplates(&cfg) + // Load templates from external files. configDir := filepath.Dir(path) if err := loadTemplateFiles(&cfg, configDir); err != nil { return nil, fmt.Errorf("loading template files: %w", err) } + // Load templates from remote URLs. + if err := loadTemplateURLs(&cfg); err != nil { + return nil, fmt.Errorf("loading template URLs: %w", err) + } + // Apply defaults. applyDefaults(&cfg) @@ -159,6 +172,16 @@ func Load(path string) (*Config, error) { return &cfg, nil } +// markInlineTemplates marks templates defined inline in the config with source type. +func markInlineTemplates(cfg *Config) { + for i := range cfg.Groups.GitHub { + for j := range cfg.Groups.GitHub[i].WorkflowDispatchTemplates { + cfg.Groups.GitHub[i].WorkflowDispatchTemplates[j].SourceType = "inline" + cfg.Groups.GitHub[i].WorkflowDispatchTemplates[j].SourcePath = "" + } + } +} + // loadTemplateFiles loads workflow dispatch templates from external files. func loadTemplateFiles(cfg *Config, configDir string) error { for i := range cfg.Groups.GitHub { @@ -187,6 +210,12 @@ func loadTemplateFiles(cfg *Config, configDir string) error { templateFile, group.ID, err) } + // Set source type and path for each template from file. + for j := range templates { + templates[j].SourceType = "file" + templates[j].SourcePath = templateFile + } + // Append templates from file to any inline templates. group.WorkflowDispatchTemplates = append(group.WorkflowDispatchTemplates, templates...) } @@ -195,6 +224,63 @@ func loadTemplateFiles(cfg *Config, configDir string) error { return nil } +// loadTemplateURLs loads workflow dispatch templates from remote URLs. +func loadTemplateURLs(cfg *Config) error { + client := &http.Client{ + Timeout: 30 * time.Second, + } + + for i := range cfg.Groups.GitHub { + group := &cfg.Groups.GitHub[i] + + for _, templateURL := range group.WorkflowDispatchTemplatesURLs { + resp, err := client.Get(templateURL) + if err != nil { + return fmt.Errorf("fetching template URL %s for group %s: %w", + templateURL, group.ID, err) + } + + data, err := io.ReadAll(resp.Body) + + closeErr := resp.Body.Close() + if err != nil { + return fmt.Errorf("reading response from %s for group %s: %w", + templateURL, group.ID, err) + } + + if closeErr != nil { + return fmt.Errorf("closing response body from %s for group %s: %w", + templateURL, group.ID, closeErr) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("fetching template URL %s for group %s: HTTP %d", + templateURL, group.ID, resp.StatusCode) + } + + // Expand environment variables. + expanded := expandEnvVars(string(data)) + + var templates []WorkflowDispatchTemplate + if err := yaml.Unmarshal([]byte(expanded), &templates); err != nil { + return fmt.Errorf("parsing template URL %s for group %s: %w", + templateURL, group.ID, err) + } + + // Set source type and path for each template from URL. + for j := range templates { + templates[j].SourceType = "url" + templates[j].SourcePath = templateURL + } + + // Append templates from URL to any existing templates. + group.WorkflowDispatchTemplates = append(group.WorkflowDispatchTemplates, templates...) + } + } + + return nil +} + // expandEnvVars replaces ${VAR} and $VAR patterns with environment variable values. func expandEnvVars(s string) string { // Match ${VAR} pattern. diff --git a/pkg/store/postgres.go b/pkg/store/postgres.go index bc3087b..af69075 100644 --- a/pkg/store/postgres.go +++ b/pkg/store/postgres.go @@ -264,6 +264,17 @@ func (s *PostgresStore) Migrate(ctx context.Context) error { created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP )`, `CREATE INDEX IF NOT EXISTS idx_auth_codes_expires ON auth_codes(expires_at)`, + // Migration: Add source_type and source_path columns to job_templates table. + `DO $$ BEGIN + ALTER TABLE job_templates ADD COLUMN source_type TEXT NOT NULL DEFAULT 'inline'; + EXCEPTION + WHEN duplicate_column THEN NULL; + END $$`, + `DO $$ BEGIN + ALTER TABLE job_templates ADD COLUMN source_path TEXT NOT NULL DEFAULT ''; + EXCEPTION + WHEN duplicate_column THEN NULL; + END $$`, } for _, migration := range migrations { @@ -408,10 +419,11 @@ func (s *PostgresStore) CreateJobTemplate(ctx context.Context, template *JobTemp } _, err = s.db.ExecContext(ctx, ` - INSERT INTO job_templates (id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + INSERT INTO job_templates (id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) `, template.ID, template.GroupID, template.Name, template.Owner, template.Repo, - template.WorkflowID, template.Ref, string(inputsJSON), string(labelsJSON), template.InConfig, template.CreatedAt, template.UpdatedAt) + template.WorkflowID, template.Ref, string(inputsJSON), string(labelsJSON), template.InConfig, + template.SourceType, template.SourcePath, template.CreatedAt, template.UpdatedAt) if err != nil { return fmt.Errorf("inserting job_template: %w", err) @@ -427,11 +439,11 @@ func (s *PostgresStore) GetJobTemplate(ctx context.Context, id string) (*JobTemp var inputsJSON, labelsJSON sql.NullString err := s.db.QueryRowContext(ctx, ` - SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at + SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at FROM job_templates WHERE id = $1 `, id).Scan(&template.ID, &template.GroupID, &template.Name, &template.Owner, &template.Repo, &template.WorkflowID, &template.Ref, &inputsJSON, &labelsJSON, - &template.InConfig, &template.CreatedAt, &template.UpdatedAt) + &template.InConfig, &template.SourceType, &template.SourcePath, &template.CreatedAt, &template.UpdatedAt) if err == sql.ErrNoRows { return nil, nil @@ -459,7 +471,7 @@ func (s *PostgresStore) GetJobTemplate(ctx context.Context, id string) (*JobTemp // ListJobTemplatesByGroup retrieves all job templates for a group. func (s *PostgresStore) ListJobTemplatesByGroup(ctx context.Context, groupID string) ([]*JobTemplate, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at + SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at FROM job_templates WHERE group_id = $1 ORDER BY name `, groupID) if err != nil { @@ -477,7 +489,7 @@ func (s *PostgresStore) ListJobTemplatesByGroup(ctx context.Context, groupID str if err := rows.Scan(&template.ID, &template.GroupID, &template.Name, &template.Owner, &template.Repo, &template.WorkflowID, &template.Ref, &inputsJSON, &labelsJSON, - &template.InConfig, &template.CreatedAt, &template.UpdatedAt); err != nil { + &template.InConfig, &template.SourceType, &template.SourcePath, &template.CreatedAt, &template.UpdatedAt); err != nil { return nil, fmt.Errorf("scanning job_template: %w", err) } @@ -514,10 +526,10 @@ func (s *PostgresStore) UpdateJobTemplate(ctx context.Context, template *JobTemp template.UpdatedAt = time.Now() _, err = s.db.ExecContext(ctx, ` - UPDATE job_templates SET name = $1, owner = $2, repo = $3, workflow_id = $4, ref = $5, default_inputs = $6, labels = $7, in_config = $8, updated_at = $9 - WHERE id = $10 + UPDATE job_templates SET name = $1, owner = $2, repo = $3, workflow_id = $4, ref = $5, default_inputs = $6, labels = $7, in_config = $8, source_type = $9, source_path = $10, updated_at = $11 + WHERE id = $12 `, template.Name, template.Owner, template.Repo, template.WorkflowID, template.Ref, - string(inputsJSON), string(labelsJSON), template.InConfig, template.UpdatedAt, template.ID) + string(inputsJSON), string(labelsJSON), template.InConfig, template.SourceType, template.SourcePath, template.UpdatedAt, template.ID) if err != nil { return fmt.Errorf("updating job_template: %w", err) diff --git a/pkg/store/sqlite.go b/pkg/store/sqlite.go index a7b16f5..eb32b42 100644 --- a/pkg/store/sqlite.go +++ b/pkg/store/sqlite.go @@ -201,6 +201,9 @@ func (s *SQLiteStore) Migrate(ctx context.Context) error { created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )`, `CREATE INDEX IF NOT EXISTS idx_auth_codes_expires ON auth_codes(expires_at)`, + // Migration: Add source_type and source_path columns to job_templates table. + `ALTER TABLE job_templates ADD COLUMN source_type TEXT NOT NULL DEFAULT 'inline'`, + `ALTER TABLE job_templates ADD COLUMN source_path TEXT NOT NULL DEFAULT ''`, } for _, migration := range migrations { @@ -467,10 +470,11 @@ func (s *SQLiteStore) CreateJobTemplate(ctx context.Context, template *JobTempla } _, err = s.db.ExecContext(ctx, ` - INSERT INTO job_templates (id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO job_templates (id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, template.ID, template.GroupID, template.Name, template.Owner, template.Repo, - template.WorkflowID, template.Ref, string(inputsJSON), string(labelsJSON), template.InConfig, template.CreatedAt, template.UpdatedAt) + template.WorkflowID, template.Ref, string(inputsJSON), string(labelsJSON), template.InConfig, + template.SourceType, template.SourcePath, template.CreatedAt, template.UpdatedAt) if err != nil { return fmt.Errorf("inserting job_template: %w", err) @@ -488,11 +492,11 @@ func (s *SQLiteStore) GetJobTemplate(ctx context.Context, id string) (*JobTempla var inConfig int err := s.db.QueryRowContext(ctx, ` - SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at + SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at FROM job_templates WHERE id = ? `, id).Scan(&template.ID, &template.GroupID, &template.Name, &template.Owner, &template.Repo, &template.WorkflowID, &template.Ref, &inputsJSON, &labelsJSON, - &inConfig, &template.CreatedAt, &template.UpdatedAt) + &inConfig, &template.SourceType, &template.SourcePath, &template.CreatedAt, &template.UpdatedAt) if err == sql.ErrNoRows { return nil, nil @@ -522,7 +526,7 @@ func (s *SQLiteStore) GetJobTemplate(ctx context.Context, id string) (*JobTempla // ListJobTemplatesByGroup retrieves all job templates for a group. func (s *SQLiteStore) ListJobTemplatesByGroup(ctx context.Context, groupID string) ([]*JobTemplate, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at + SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at FROM job_templates WHERE group_id = ? ORDER BY name `, groupID) if err != nil { @@ -542,7 +546,7 @@ func (s *SQLiteStore) ListJobTemplatesByGroup(ctx context.Context, groupID strin if err := rows.Scan(&template.ID, &template.GroupID, &template.Name, &template.Owner, &template.Repo, &template.WorkflowID, &template.Ref, &inputsJSON, &labelsJSON, - &inConfig, &template.CreatedAt, &template.UpdatedAt); err != nil { + &inConfig, &template.SourceType, &template.SourcePath, &template.CreatedAt, &template.UpdatedAt); err != nil { return nil, fmt.Errorf("scanning job_template: %w", err) } @@ -580,10 +584,10 @@ func (s *SQLiteStore) UpdateJobTemplate(ctx context.Context, template *JobTempla template.UpdatedAt = time.Now() _, err = s.db.ExecContext(ctx, ` - UPDATE job_templates SET name = ?, owner = ?, repo = ?, workflow_id = ?, ref = ?, default_inputs = ?, labels = ?, in_config = ?, updated_at = ? + UPDATE job_templates SET name = ?, owner = ?, repo = ?, workflow_id = ?, ref = ?, default_inputs = ?, labels = ?, in_config = ?, source_type = ?, source_path = ?, updated_at = ? WHERE id = ? `, template.Name, template.Owner, template.Repo, template.WorkflowID, template.Ref, - string(inputsJSON), string(labelsJSON), template.InConfig, template.UpdatedAt, template.ID) + string(inputsJSON), string(labelsJSON), template.InConfig, template.SourceType, template.SourcePath, template.UpdatedAt, template.ID) if err != nil { return fmt.Errorf("updating job_template: %w", err) diff --git a/pkg/store/store.go b/pkg/store/store.go index 0b2e559..aecf78c 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -115,6 +115,8 @@ type JobTemplate struct { DefaultInputs map[string]string `json:"default_inputs"` Labels map[string]string `json:"labels"` InConfig bool `json:"in_config"` + SourceType string `json:"source_type"` // "inline", "file", or "url" + SourcePath string `json:"source_path"` // filename or URL (empty for inline) CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } diff --git a/ui/src/pages/ApiDocsPage.tsx b/ui/src/pages/ApiDocsPage.tsx index cd2f7a7..0a8ca46 100644 --- a/ui/src/pages/ApiDocsPage.tsx +++ b/ui/src/pages/ApiDocsPage.tsx @@ -22,6 +22,7 @@ export function ApiDocsPage() { setSidebarCollapsed(false); } }; + // eslint-disable-next-line react-hooks/exhaustive-deps -- intentionally run only on mount/unmount }, []); const config: ReferenceProps['configuration'] = { diff --git a/ui/src/pages/GroupPage.tsx b/ui/src/pages/GroupPage.tsx index f990177..7a119c4 100644 --- a/ui/src/pages/GroupPage.tsx +++ b/ui/src/pages/GroupPage.tsx @@ -1554,6 +1554,30 @@ export function GroupPage() { Not in config )} + {template.source_type === 'file' && ( + + + + + {template.source_path.split('/').pop()} + + )} + {template.source_type === 'url' && ( + + + + + URL + + )} {/* Template labels */} {template.labels && Object.keys(template.labels).length > 0 && ( diff --git a/ui/src/types/index.ts b/ui/src/types/index.ts index 1e4e9f9..3883d40 100644 --- a/ui/src/types/index.ts +++ b/ui/src/types/index.ts @@ -28,6 +28,8 @@ export interface GroupWithStats extends Group { template_count: number; } +export type TemplateSourceType = 'inline' | 'file' | 'url'; + export interface JobTemplate { id: string; group_id: string; @@ -39,6 +41,8 @@ export interface JobTemplate { default_inputs: Record; labels?: Record; in_config: boolean; + source_type: TemplateSourceType; + source_path: string; created_at: string; updated_at: string; }