diff --git a/Makefile b/Makefile index fa3f87f..8ce3795 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL = /bin/sh -VERSION=1.7.1 +VERSION=1.8.0 BUILD=`git rev-parse HEAD` LDFLAGS=-ldflags "-w -s \ diff --git a/README.md b/README.md index b4b14b8..ceeef01 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ If you have a large organization, you may wish to tag ownership of splits to a s ### Syncing split assignments -If you want to ensure that your local split assignments are in sync with your remote (production) assignments, you can run `TESTTRACK_CLI_URL= testtrack sync` (e.g. `TESTTRACK_CLI_URL=https://tt.example.com testtrack sync`) from your project directory to pull the assignments from your remote server into your local `schema.yml` file. +If you want to ensure that your local split assignments are in sync with your remote (production) assignments, you can run `TESTTRACK_CLI_URL= testtrack sync` (e.g. `TESTTRACK_CLI_URL=https://tt.example.com testtrack sync`) from your project directory to pull the assignments from your remote server into your local `schema.{json,yml}` file. ## How to Contribute diff --git a/cmds/schema_generate.go b/cmds/schema_generate.go index 845e964..2f73897 100644 --- a/cmds/schema_generate.go +++ b/cmds/schema_generate.go @@ -7,10 +7,10 @@ import ( var schemaGenerateDoc = ` Reads the migrations in testtrack/migrate and writes the resulting schema state -to testtrack/schema.yml, overwriting the file if it already exists. Generate +to testtrack/schema.{json,yml}, overwriting the file if it already exists. Generate makes no TestTrack API calls. -In addition to refreshing a schema.yml file that may have been corrupted due to +In addition to refreshing a schema file that may have been corrupted due to a bad merge or bug that produced incorrect schema state, 'schema generate' will also validate that migrations merged from multiple development branches don't logically conflict, or else it will fail with errors. @@ -30,7 +30,7 @@ func init() { var schemaGenerateCmd = &cobra.Command{ Use: "generate", - Short: "Generate schema.yml from migration files", + Short: "Generate schema.{json,yml} from migration files", Long: schemaGenerateDoc, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { diff --git a/cmds/schema_load.go b/cmds/schema_load.go index 11b54e0..87fbb46 100644 --- a/cmds/schema_load.go +++ b/cmds/schema_load.go @@ -6,7 +6,7 @@ import ( ) var schemaLoadDoc = ` -Loads the testtrack/schema.yml state into TestTrack server. This operation is +Loads the testtrack/schema.{json,yml} state into TestTrack server. This operation is idempotent with a valid, consistent schema file, though might fail if your schema file became invalid due to a bad merge or a bug. @@ -27,7 +27,7 @@ func init() { var schemaLoadCmd = &cobra.Command{ Use: "load", - Short: "Load schema.yml state into TestTrack server", + Short: "Load schema.{json,yml} state into TestTrack server", Long: schemaLoadDoc, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { diff --git a/cmds/server.go b/cmds/server.go index 9504dc8..9e70971 100644 --- a/cmds/server.go +++ b/cmds/server.go @@ -6,7 +6,7 @@ import ( ) var serverDoc = ` -Run a fake TestTrack server for local development, backed by schema.yml files +Run a fake TestTrack server for local development, backed by schema.{json,yml} files and nonsense. ` diff --git a/cmds/sync.go b/cmds/sync.go index b3491e2..0977b58 100644 --- a/cmds/sync.go +++ b/cmds/sync.go @@ -48,7 +48,7 @@ func Sync() error { remoteSplit, exists := splitRegistry.Splits[localSplit.Name] if exists { remoteWeights := splits.Weights(remoteSplit.Weights) - localSchema.Splits[ind].Weights = remoteWeights.ToYAML() + localSchema.Splits[ind].Weights = remoteWeights } } diff --git a/fakeserver/routes.go b/fakeserver/routes.go index 4730be6..49acea7 100644 --- a/fakeserver/routes.go +++ b/fakeserver/routes.go @@ -192,7 +192,7 @@ func getV1SplitRegistry() (interface{}, error) { } splitRegistry := map[string]*splits.Weights{} for _, split := range schema.Splits { - splitRegistry[split.Name], err = splits.WeightsFromYAML(split.Weights) + splitRegistry[split.Name], err = splits.NewWeights(split.Weights) if err != nil { return nil, err } @@ -208,7 +208,7 @@ func getV2PlusSplitRegistry() (interface{}, error) { splitRegistry := map[string]*v2Split{} for _, split := range schema.Splits { isFeatureGate := splits.IsFeatureGateFromName(split.Name) - weights, err := splits.WeightsFromYAML(split.Weights) + weights, err := splits.NewWeights(split.Weights) if err != nil { return nil, err } @@ -231,7 +231,7 @@ func getV4SplitRegistry() (interface{}, error) { v4Splits := make([]v4Split, 0, len(schema.Splits)) for _, split := range schema.Splits { isFeatureGate := splits.IsFeatureGateFromName(split.Name) - weights, err := splits.WeightsFromYAML(split.Weights) + weights, err := splits.NewWeights(split.Weights) if err != nil { return nil, err } diff --git a/fakeserver/server_test.go b/fakeserver/server_test.go index 72e32a0..a0a554a 100644 --- a/fakeserver/server_test.go +++ b/fakeserver/server_test.go @@ -33,6 +33,17 @@ splits: treatment: 40 ` +var otherTestSchema = `{ + "serializer_version": 1, + "schema_version": "2020011774023", + "splits": [ + { + "name": "test.json_experiment", + "weights": { "control": 50, "treatment": 50 } + } + ] +}` + var testAssignments = ` something_something_enabled: "true" ` @@ -52,7 +63,12 @@ func TestMain(m *testing.M) { } schemaContent := []byte(testSchema) - if err := os.WriteFile(filepath.Join(schemasDir, "test.yml"), schemaContent, 0644); err != nil { + if err := os.WriteFile(filepath.Join(schemasDir, "a.yml"), schemaContent, 0644); err != nil { + log.Fatal(err) + } + + otherSchemaContent := []byte(otherTestSchema) + if err := os.WriteFile(filepath.Join(schemasDir, "b.json"), otherSchemaContent, 0644); err != nil { log.Fatal(err) } @@ -140,6 +156,22 @@ func TestSplitRegistry(t *testing.T) { require.Equal(t, 40, treatment.Weight) require.Equal(t, false, split.FeatureGate) }) + + t.Run("it loads JSON schemas from home directory", func(t *testing.T) { + w := httptest.NewRecorder() + h := createHandler() + + h.ServeHTTP(w, httptest.NewRequest("GET", "/api/v2/split_registry", nil)) + + require.Equal(t, http.StatusOK, w.Code) + + registry := v2SplitRegistry{} + err := json.Unmarshal(w.Body.Bytes(), ®istry) + require.Nil(t, err) + + require.Equal(t, 50, registry.Splits["test.json_experiment"].Weights["control"]) + require.Equal(t, 50, registry.Splits["test.json_experiment"].Weights["treatment"]) + }) } func TestVisitorConfig(t *testing.T) { diff --git a/schema/schema.go b/schema/schema.go index cdc66e0..9916da8 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -1,6 +1,7 @@ package schema import ( + "encoding/json" "errors" "fmt" "os" @@ -15,12 +16,24 @@ import ( "gopkg.in/yaml.v2" ) +// Finds the path to the schema file (preferring JSON), or returns testtrack/schema.json +func findSchemaPath() (string, bool) { + if _, err := os.Stat("testtrack/schema.json"); err == nil { + return "testtrack/schema.json", true + } + if _, err := os.Stat("testtrack/schema.yml"); err == nil { + return "testtrack/schema.yml", true + } + return "testtrack/schema.json", false +} + // Read a schema from disk or generate one func Read() (*serializers.Schema, error) { - if _, err := os.Stat("testtrack/schema.yml"); os.IsNotExist(err) { + schemaPath, exists := findSchemaPath() + if !exists { return Generate() } - schemaBytes, err := os.ReadFile("testtrack/schema.yml") + schemaBytes, err := os.ReadFile(schemaPath) if err != nil { return nil, err } @@ -53,12 +66,21 @@ func Generate() (*serializers.Schema, error) { // Write a schema to disk after alpha-sorting its resources func Write(schema *serializers.Schema) error { SortAlphabetically(schema) - out, err := yaml.Marshal(schema) + + schemaPath, _ := findSchemaPath() + + var out []byte + var err error + if filepath.Ext(schemaPath) == ".yml" { + out, err = yaml.Marshal(schema) + } else { + out, err = json.MarshalIndent(schema, "", " ") + } if err != nil { return err } - err = os.WriteFile("testtrack/schema.yml", out, 0644) + err = os.WriteFile(schemaPath, out, 0644) if err != nil { return err } @@ -68,8 +90,9 @@ func Write(schema *serializers.Schema) error { // Link a schema to the user's home dir func Link(force bool) error { - if _, err := os.Stat("testtrack/schema.yml"); os.IsNotExist(err) { - return errors.New("testtrack/schema.yml does not exist. Are you in your app root dir? If so, call testtrack init_project first") + schemaPath, exists := findSchemaPath() + if !exists { + return errors.New("testtrack/schema.{json,yml} does not exist. Are you in your app root dir? If so, call testtrack init_project first") } dir, err := os.Getwd() if err != nil { @@ -84,11 +107,12 @@ func Link(force bool) error { if err != nil { return err } - path := fmt.Sprintf("%s/schemas/%s.yml", *configDir, dirname) + ext := filepath.Ext(schemaPath) + path := fmt.Sprintf("%s/schemas/%s%s", *configDir, dirname, ext) if force { os.Remove(path) // If this fails it might just not exist, we'll error on the next line if something else is up } - return os.Symlink(dir+"/testtrack/schema.yml", path) + return os.Symlink(dir+"/"+schemaPath, path) } // ReadMerged merges schemas linked at ~/testtrack/schemas into a single virtual schema @@ -97,28 +121,20 @@ func ReadMerged() (*serializers.Schema, error) { if err != nil { return nil, err } - paths, err := filepath.Glob(*configDir + "/schemas/*.yml") + paths, err := filepath.Glob(*configDir + "/schemas/*.*") if err != nil { return nil, err } var mergedSchema serializers.Schema for _, path := range paths { - // Deref symlink - fi, err := os.Lstat(path) - if err != nil { - return nil, err - } - if fi.Mode()&os.ModeSymlink != 0 { - path, err = os.Readlink(path) - if err != nil { - continue // It's OK if this symlink isn't traversable (e.g. app was uninstalled), we'll just skip it. - } - } - // Read file schemaBytes, err := os.ReadFile(path) if err != nil { + if os.IsNotExist(err) { + continue // It's OK if this file doesn't exist (e.g. broken symlink, app was uninstalled), we'll just skip it. + } return nil, err } + var schema serializers.Schema err = yaml.Unmarshal(schemaBytes, &schema) if err != nil { @@ -156,18 +172,18 @@ func mergeLegacySchema(schema *serializers.Schema) error { if !ok { return fmt.Errorf("expected split name, got %v", mapSlice.Key) } - weightsYAML, ok := mapSlice.Value.(yaml.MapSlice) + weightsYAML, ok := mapSlice.Value.(map[string]int) if !ok { return fmt.Errorf("expected weights, got %v", mapSlice.Value) } - weights, err := splits.WeightsFromYAML(weightsYAML) + weights, err := splits.NewWeights(weightsYAML) if err != nil { return err } schema.Splits = append(schema.Splits, serializers.SchemaSplit{ Name: name, - Weights: weights.ToYAML(), + Weights: *weights, Decided: false, }) } diff --git a/schemaloaders/schemaloaders.go b/schemaloaders/schemaloaders.go index 222ce08..cbf5d22 100644 --- a/schemaloaders/schemaloaders.go +++ b/schemaloaders/schemaloaders.go @@ -98,7 +98,7 @@ func schemaSplitMigrations(schemaSplit serializers.SchemaSplit) ([]migrations.IM if schemaSplit.Decided { var decision *string - weights, err := splits.WeightsFromYAML(schemaSplit.Weights) + weights, err := splits.NewWeights(schemaSplit.Weights) if err != nil { return nil, fmt.Errorf("schema split %s invalid: %w", schemaSplit.Name, err) } diff --git a/serializers/serializers.go b/serializers/serializers.go index 7188161..34f06ae 100644 --- a/serializers/serializers.go +++ b/serializers/serializers.go @@ -40,9 +40,9 @@ type RemoteKill struct { // SplitYAML is the YAML-marshalable representation of a Split type SplitYAML struct { - Name string `yaml:"name"` - Weights yaml.MapSlice `yaml:"weights"` - Owner string `yaml:"owner,omitempty"` + Name string `yaml:"name"` + Weights map[string]int `yaml:"weights"` + Owner string `yaml:"owner,omitempty"` } // SplitJSON is the JSON-marshalabe representation of a Split @@ -80,21 +80,21 @@ type IdentifierType struct { // SchemaSplit is the schema-file YAML-marshalable representation of a split's state type SchemaSplit struct { - Name string `yaml:"name"` - Weights yaml.MapSlice `yaml:"weights"` - Decided bool `yaml:"decided,omitempty"` - Owner string `yaml:"owner,omitempty"` + Name string `yaml:"name" json:"name"` + Weights map[string]int `yaml:"weights" json:"weights"` + Decided bool `yaml:"decided,omitempty" json:"decided,omitempty"` + Owner string `yaml:"owner,omitempty" json:"owner,omitempty"` } // Schema is the YAML-marshalable representation of the TestTrack schema for // migration validation and bootstrapping of new ecosystems type Schema struct { - SerializerVersion int `yaml:"serializer_version"` - SchemaVersion string `yaml:"schema_version"` - Splits []SchemaSplit `yaml:"splits,omitempty"` - IdentifierTypes []IdentifierType `yaml:"identifier_types,omitempty"` - RemoteKills []RemoteKill `yaml:"remote_kills,omitempty"` - FeatureCompletions []FeatureCompletion `yaml:"feature_completions,omitempty"` + SerializerVersion int `yaml:"serializer_version" json:"serializer_version"` + SchemaVersion string `yaml:"schema_version" json:"schema_version"` + Splits []SchemaSplit `yaml:"splits,omitempty" json:"splits,omitempty"` + IdentifierTypes []IdentifierType `yaml:"identifier_types,omitempty" json:"identifier_types,omitempty"` + RemoteKills []RemoteKill `yaml:"remote_kills,omitempty" json:"remote_kills,omitempty"` + FeatureCompletions []FeatureCompletion `yaml:"feature_completions,omitempty" json:"feature_completions,omitempty"` } // LegacySchema represents the Rails migration-piggybacked testtrack schema files of old diff --git a/splitdecisions/splitdecisions.go b/splitdecisions/splitdecisions.go index cf8dfe1..3d9fb8d 100644 --- a/splitdecisions/splitdecisions.go +++ b/splitdecisions/splitdecisions.go @@ -97,7 +97,7 @@ func (s *SplitDecision) ApplyToSchema(schema *serializers.Schema, migrationRepo for i, candidate := range schema.Splits { if candidate.Name == *s.split { schema.Splits[i].Decided = true - weights, err := splits.WeightsFromYAML(candidate.Weights) + weights, err := splits.NewWeights(candidate.Weights) if err != nil { return err } @@ -105,7 +105,7 @@ func (s *SplitDecision) ApplyToSchema(schema *serializers.Schema, migrationRepo if err != nil { return fmt.Errorf("in split %s in schema: %w", *s.split, err) } - schema.Splits[i].Weights = weights.ToYAML() + schema.Splits[i].Weights = *weights return nil } } @@ -119,7 +119,7 @@ func (s *SplitDecision) ApplyToSchema(schema *serializers.Schema, migrationRepo } schema.Splits = append(schema.Splits, serializers.SchemaSplit{ Name: *s.split, - Weights: weights.ToYAML(), + Weights: *weights, Decided: true, }) return nil diff --git a/splitretirements/splitretirements.go b/splitretirements/splitretirements.go index a0f18fc..c9c5b2e 100644 --- a/splitretirements/splitretirements.go +++ b/splitretirements/splitretirements.go @@ -96,7 +96,7 @@ func (s *SplitRetirement) SameResourceAs(other migrations.IMigration) bool { func (s *SplitRetirement) ApplyToSchema(schema *serializers.Schema, _ migrations.Repository, _idempotently bool) error { for i, candidate := range schema.Splits { if candidate.Name == *s.split { - weights, err := splits.WeightsFromYAML(candidate.Weights) + weights, err := splits.NewWeights(candidate.Weights) if err != nil { return err } diff --git a/splits/splits.go b/splits/splits.go index 703d8ba..ddd6316 100644 --- a/splits/splits.go +++ b/splits/splits.go @@ -83,13 +83,14 @@ func IsFeatureGateFromName(name string) bool { // FromFile reifies a migration from the yaml serializable representation func FromFile(migrationVersion *string, serializable *serializers.SplitYAML) (migrations.IMigration, error) { - weights, err := WeightsFromYAML(serializable.Weights) + weights, err := NewWeights(serializable.Weights) if err != nil { return nil, err } return &Split{ migrationVersion: migrationVersion, name: &serializable.Name, + owner: &serializable.Owner, weights: weights, }, nil } @@ -111,7 +112,7 @@ func (s *Split) File() *serializers.MigrationFile { SerializerVersion: serializers.SerializerVersion, Split: &serializers.SplitYAML{ Name: *s.name, - Weights: s.weights.ToYAML(), + Weights: *s.weights, Owner: *s.owner, }, } @@ -152,13 +153,13 @@ func (s *Split) SameResourceAs(other migrations.IMigration) bool { func (s *Split) ApplyToSchema(schema *serializers.Schema, migrationRepo migrations.Repository, _idempotently bool) error { for i, candidate := range schema.Splits { // Replace if candidate.Name == *s.name { - schemaWeights, err := WeightsFromYAML(candidate.Weights) + schemaWeights, err := NewWeights(candidate.Weights) if err != nil { return err } schemaWeights.Merge(*s.weights) schema.Splits[i].Decided = false - schema.Splits[i].Weights = schemaWeights.ToYAML() + schema.Splits[i].Weights = *schemaWeights return nil } } @@ -169,7 +170,7 @@ func (s *Split) ApplyToSchema(schema *serializers.Schema, migrationRepo migratio weights.Merge(*s.weights) schema.Splits = append(schema.Splits, serializers.SchemaSplit{ Name: *s.name, - Weights: weights.ToYAML(), + Weights: *weights, Decided: false, }) return nil @@ -177,7 +178,7 @@ func (s *Split) ApplyToSchema(schema *serializers.Schema, migrationRepo migratio } schemaSplit := serializers.SchemaSplit{ // Create Name: *s.name, - Weights: s.weights.ToYAML(), + Weights: *s.weights, Decided: false, Owner: *s.owner, } diff --git a/splits/weights.go b/splits/weights.go index a27be76..d4f9931 100644 --- a/splits/weights.go +++ b/splits/weights.go @@ -2,51 +2,25 @@ package splits import ( "fmt" - "sort" - - "gopkg.in/yaml.v2" ) // Weights represents the weightings of a split type Weights map[string]int -// WeightsFromYAML converts YAML-serializable weights to a weights map -func WeightsFromYAML(yamlWeights yaml.MapSlice) (*Weights, error) { - weights := make(Weights) +// NewWeights creates a Weights instance from a map, validating that weights sum to 100 +func NewWeights(weights map[string]int) (*Weights, error) { cumulativeWeight := 0 - for _, item := range yamlWeights { - variant, ok := item.Key.(string) - if !ok { - return nil, fmt.Errorf("variant %v is not a string", item.Key) - } - weight, ok := item.Value.(int) - if !ok { - return nil, fmt.Errorf("weighting %v is not an int", item.Value) - } + for _, weight := range weights { if weight < 0 { return nil, fmt.Errorf("weight %d is less than zero", weight) } cumulativeWeight += weight - weights[variant] = weight } if cumulativeWeight != 100 { return nil, fmt.Errorf("weights must sum to 100, got %d", cumulativeWeight) } - return &weights, nil -} - -// ToYAML converts weights to a YAML-serializable representation -func (w *Weights) ToYAML() yaml.MapSlice { - var variants = make([]string, 0, len(*w)) - for variant := range *w { - variants = append(variants, variant) - } - sort.Strings(variants) - weightsYaml := make(yaml.MapSlice, 0, len(variants)) - for _, variant := range variants { - weightsYaml = append(weightsYaml, yaml.MapItem{Key: variant, Value: (*w)[variant]}) - } - return weightsYaml + w := Weights(weights) + return &w, nil } // Merge newWeights over weights diff --git a/validations/validations.go b/validations/validations.go index 79e4185..7000995 100644 --- a/validations/validations.go +++ b/validations/validations.go @@ -259,11 +259,7 @@ func VariantExistsInSchema(paramName string, variant *string, split string, sche } for _, schemaSplit := range schema.Splits { if schemaSplit.Name == split { - for _, item := range schemaSplit.Weights { - v, ok := item.Key.(string) - if !ok { - return fmt.Errorf("variant %v is not a string", item.Key) - } + for v := range schemaSplit.Weights { if v == *variant { return nil }