Dev (#348)

* cleanup

* fix yaml not recognizing relative paths or vscode workspaceFolder var

* refactor and lint spec

* refactor

* discriminator note

* oapi custom update

* update oapi and vendor

* wget retries

* fix readme setup

* alternative awk script compatible with mawk

Author: danicc097

.github/actions/setup/action.yaml

@@ -15,7 +15,7 @@ runs:
     - name: Setup Go
       uses: actions/setup-go@v3
       with:
-        go-version: "^1.21.6"
+        go-version: "^1.22.0"
 
     # - name: Setup Python
     #   uses: actions/setup-python@v4

.golangci.yml

@@ -63,7 +63,7 @@ linters-settings:
         json: goCamel
         yaml: goCamel
   staticcheck:
-    go: "1.21"
+    go: "1.22"
     checks:
       - "all"
   gci:

.vscode/anySchema.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "additionalProperties": true
+}

.vscode/settings.json

@@ -97,4 +97,9 @@
   "files.associations": {
     "*.sql": "postgres"
   },
+  "redoclyOpenAPI.trace.server": "off",
+  "yaml.schemas": {
+    "https://raw.githubusercontent.com/docker/cli/master/cli/compose/schema/data/config_schema_v3.8.json": "*compose*.yml",
+    ".vscode/openapi-schema.json": "openapi.yaml"
+  },
 }

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21-alpine3.18 AS build
+FROM golang:1.22-alpine3.18 AS build
 
 ARG DOCKER_UID
 ARG DOCKER_GID

Dockerfile.oidc-server

@@ -1,4 +1,4 @@
-FROM golang:1.21-alpine3.18 AS build
+FROM golang:1.22-alpine3.18 AS build
 
 WORKDIR /go/src
 COPY ./vendor/ ./vendor/

README.md

@@ -103,8 +103,8 @@ direnv allow # you can also customize direnv with .envrc.local as you would a re
 cp openapi-go.code-workspace.example openapi-go.code-workspace # edit as desired
 
 project bootstrap # dependency and tools interactive installation
-project gen
 project recreate-shared-services
+project gen
 project run.all
 ```
 

bin/.helpers.sh

@@ -599,7 +599,7 @@ docker.images.load_latest() {
 AWK_REMOVE_GO_COMMENTS='
      /\/\*/ { f=1 } # set flag that is a block comment
 
-     f==0 && !/^\s*(\/\/|\/\*)/ { # skip // or /*
+     f==0 && !/^[ \t]*(\/\/|\/\*)/ { # skip // or /*
         print  # print non-commented lines
      }
      /\*\// { f=0 } # reset flag at end of block comment

bin/.project.dependencies.sh

@@ -44,7 +44,7 @@ check.bin.column() {
 
 install.bin.column() {
   { { {
-    wget https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v2.36/util-linux-2.36.2.tar.gz
+    wget --tries=10 https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v2.36/util-linux-2.36.2.tar.gz
     tar -xf util-linux-2.36.2.tar.gz
     cd util-linux-2.36.2 || exit 1
     ./configure
@@ -72,7 +72,7 @@ install.bin.protoc() {
   { { {
     VERSION=3.19.4
     id="protoc-$VERSION-linux-x86_64"
-    wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/$id.zip
+    wget --tries=10 https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/$id.zip
     unzip -q "$id".zip -d "$id"
     mv "$id"/bin/protoc ./bin/tools/
     rm -rf "$id"
@@ -145,7 +145,7 @@ check.bin.pg_format() {
 install.bin.pg_format() {
   { { {
     local version="5.2"
-    wget https://github.com/darold/pgFormatter/archive/refs/tags/v"$version".tar.gz
+    wget --tries=10 https://github.com/darold/pgFormatter/archive/refs/tags/v"$version".tar.gz
     tar xzf v"$version".tar.gz
     cd pgFormatter-"$version"/ || exit 1
     perl Makefile.PL
@@ -279,7 +279,7 @@ install.bin.mkcert() {
   { { {
     VERSION="1.4.3"
     sudo apt-get install libnss3-tools -y
-    wget https://github.com/FiloSottile/mkcert/releases/download/v"$VERSION"/mkcert-v"$VERSION"-linux-amd64 -O mkcert
+    wget --tries=10 https://github.com/FiloSottile/mkcert/releases/download/v"$VERSION"/mkcert-v"$VERSION"-linux-amd64 -O mkcert
     chmod +x mkcert
     mv mkcert ./bin/tools/
     mkcert -install

bin/project

@@ -298,6 +298,8 @@ x.install-tools() {
       "go install github.com/maxbrunsfeld/counterfeiter/v6@latest"
       "go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1"
       "go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2"
+      # vacuum lint openapi.exploded.yaml -ed
+      "go install github.com/daveshanley/vacuum@latest"
     )
     for command in "${commands[@]}"; do
       $command &
@@ -343,48 +345,13 @@ x.gen.pregen() {
     echo "Running generation"
 
     # NOTE: swaggest won't generate for arrays of structs. use a struct with array fields instead.
-    generate_structs_map # structs might have been removed in the meantime
+    generate_structs_map # spec structs or their content in rest package might have changed
 
-    go build -o $BUILD_DIR/codegen cmd/codegen/main.go || mark_failed_tool_build # xo gen may create/update models.
+    go build -o $BUILD_DIR/codegen cmd/codegen/main.go || mark_failed_tool_build
 
     local rest_structs=() refs=() spec_schemas=()
-    declare -A rest_schema_lookup spec_schema_lookup
 
-    go-utils.find_structs rest_structs "$REST_MODELS"
-    for schema in "${rest_structs[@]}"; do # only interested in generating rest models.spec.go schemas. let the rest fail normally.
-      rest_schema_lookup["$schema"]=1
-    done
-
-    mapfile -t spec_schemas < <(yq eval '.components.schemas[] | key' "$SPEC_PATH")
-    for schema in "${spec_schemas[@]}"; do # only interested in generating rest models.spec.go schemas. let the rest fail.
-      spec_schema_lookup["$schema"]=1
-    done
-
-    local missing_rest_models=()
-    mapfile -t refs < <(sed -nr "s/[^#]?.*\\$ref: '#\/components\/schemas\/(.*)'/\1/p" $SPEC_PATH | sort | uniq)
-    for ref in "${refs[@]}"; do
-      if [[ -n "${rest_schema_lookup["$ref"]}" && -z "${spec_schema_lookup["$ref"]}" ]]; then
-        missing_rest_models+=("$ref")
-      fi
-    done
-
-    if [[ ${#missing_rest_models[@]} -gt 0 ]]; then
-      echo "${YELLOW}Generating missing rest package \$ref's:"
-      printf "${YELLOW}  - %s\n" "${missing_rest_models[@]}"
-      echo "${OFF}"
-
-      local new_schemas=""
-      for ref in "${missing_rest_models[@]}"; do
-        new_schemas+="\"$ref\": {
-            \"x-gen-struct\": \"$ref\",
-            \"x-is-generated\": true
-          },"
-      done
-
-      yq eval-all ".components.schemas += {
-        ${new_schemas%,}
-      }" -i "$SPEC_PATH"
-    fi
+    generate_missing_spec_schemas
 
     codegen validate-spec -env=".env.$X_ENV"
     # useless after merging models and db
@@ -394,40 +361,87 @@ x.gen.pregen() {
 
     ######## Ensure consistent style for future codegen
 
-    echo "Applying PascalCase to operation IDs in $SPEC_PATH"
+    pascal_case_spec_operation_ids
+
+    update_roles_and_scopes
+
+    validate_spec
+    codegen pre -env=".env.$X_ENV" -op-id-auth="$OPID_AUTH_PATH"
+    update_spec_with_structs
+    remove_schemas_marked_for_deletion
+  } 2>&4 | xlog >&3; } 4>&1 | xerr >&3; } 3>&1
+  xsetup.backup.cleanup
+}
+
+pascal_case_spec_operation_ids() {
+  echo "Applying PascalCase to operation IDs"
 
-    # outputs safe double-quoted paths for yq
-    # https://github.com/mikefarah/yq/issues/1295
-    mapfile -t opid_paths < <(yq e '
+  # outputs safe double-quoted paths for yq
+  # https://github.com/mikefarah/yq/issues/1295
+  mapfile -t opid_paths < <(yq e '
       .paths[][].operationId
       | path
       | with(.[] | select(contains(".") or contains("/") or contains("{")); . = "\"" + . + "\"")
       | join(".")
     ' $SPEC_PATH)
-    mapfile -t opids < <(yq e ".paths[][].operationId" $SPEC_PATH)
+  mapfile -t opids < <(yq e ".paths[][].operationId" $SPEC_PATH)
 
-    local ops=""
-    for i in ${!opids[@]}; do
-      local new_opid=""
-      to_pascal new_opid "${opids[$i]}"
-      ops+=".${opid_paths[$i]}=\"${new_opid}\" |" # cant have space
-    done
+  local ops=""
+  for i in ${!opids[@]}; do
+    local new_opid=""
+    to_pascal new_opid "${opids[$i]}"
+    ops+=".${opid_paths[$i]}=\"${new_opid}\" |" # cant have space
+  done
+
+  yq_op="${ops%"|"}"
 
-    yq_op="${ops%"|"}"
+  yq e "$yq_op" -i "$SPEC_PATH"
+}
 
-    yq e "$yq_op" -i "$SPEC_PATH"
+# ext vars: spec_schemas, rest_structs
+generate_missing_spec_schemas() {
+  declare -A rest_schema_lookup spec_schema_lookup
 
-    update_roles_and_scopes
+  go-utils.find_structs rest_structs "$REST_MODELS"
+  for schema in "${rest_structs[@]}"; do # only interested in generating rest models.spec.go schemas. let the rest fail normally.
+    rest_schema_lookup["$schema"]=1
+  done
 
-    validate_spec
-    codegen pre -env=".env.$X_ENV" -op-id-auth="$OPID_AUTH_PATH" # should not edit spec
-    update_spec_with_structs
-    remove_schemas_marked_for_deletion
-  } 2>&4 | xlog >&3; } 4>&1 | xerr >&3; } 3>&1
-  xsetup.backup.cleanup
+  mapfile -t spec_schemas < <(yq eval '.components.schemas[] | key' "$SPEC_PATH")
+  for schema in "${spec_schemas[@]}"; do # only interested in generating rest models.spec.go schemas. let the rest fail.
+    spec_schema_lookup["$schema"]=1
+  done
+
+  local missing_rest_models=()
+  mapfile -t refs < <(sed -nr "s/[^#]?.*\\$ref: '#\/components\/schemas\/(.*)'/\1/p" $SPEC_PATH | sort | uniq)
+  for ref in "${refs[@]}"; do
+    if [[ -n "${rest_schema_lookup["$ref"]}" && -z "${spec_schema_lookup["$ref"]}" ]]; then
+      missing_rest_models+=("$ref")
+    fi
+  done
+
+  if [[ ${#missing_rest_models[@]} -gt 0 ]]; then
+    echo "${YELLOW}Generating missing rest package \$ref's:"
+    printf "${YELLOW}  - %s\n" "${missing_rest_models[@]}"
+    echo "${OFF}"
+
+    local new_schemas=""
+    for ref in "${missing_rest_models[@]}"; do
+      new_schemas+="\"$ref\": {
+            \"x-gen-struct\": \"$ref\",
+            \"x-is-generated\": true
+          },"
+    done
+
+    yq eval-all ".components.schemas += {
+        ${new_schemas%,}
+      }" -i "$SPEC_PATH"
+  fi
 }
 
 validate_spec() {
+  echo "Validating spec $SPEC_PATH"
+
   mapfile -t invalid_schemas < <(yq e '.components.schemas[] | select(has("x-gen-struct") and (has("x-is-generated") | not)) | key' $SPEC_PATH)
   [[ ${#invalid_schemas[@]} -gt 0 ]] && err "x-gen-struct can only be used in generated schemas (x-is-generated). Found in:
 $(join_by $'\n' "${invalid_schemas[@]}")"
@@ -465,7 +479,7 @@ $(join_by $'\n' "${services_prefix_clashes[@]}")"
 }
 
 update_roles_and_scopes() {
-  echo "Updating roles and scopes in $SPEC_PATH"
+  echo "Updating roles and scopes"
   ######## Sync spec enums with external policy sources and validate existing schema enums.
   ######## External json files are the source of truth, indexed by enum name
   # arrays can't be nested in bash
@@ -959,7 +973,7 @@ additional-imports:
 EOF
     )
     # IMPORTANT: additional-imports packages affect x-go extensions
-    # see https://github.com/deepmap/oapi-codegen?tab=readme-ov-file#openapi-extensions
+    # see https://github.com/oapi-codegen/oapi-codegen?tab=readme-ov-file#openapi-extensions
 
     # TODO: we already have structs.gen.go indexed by "Models(models struct)" or plain rest type...
     # however this might be faster if the build is cached (templates unchanged). could construct internally as map set anyway
@@ -1086,7 +1100,7 @@ x.gen.frontend() {
     mkdir -p frontend/src/types
     rm -rf $FRONTEND_GEN
 
-    yq 'explode(.)' $SPEC_PATH >$exploded_spec # js-lib won't support anchors - used by most gen libs
+    yq 'explode(.)' $SPEC_PATH >$exploded_spec # js-yaml wont support explicit tags - used by most gen libs
 
     {
       node frontend/scripts/generate-client-validator.js
@@ -1160,8 +1174,6 @@ x.gen() {
 
     wait_without_error || err Failed jobs
 
-    x.test.xo
-
     {
       x.gen.pregen
       x.gen.client-server
@@ -1189,7 +1201,9 @@ x.gen() {
 # Build code generation custom tools.
 x.gen.build-tools() {
   { { {
-    generate_structs_map # openapi-go requires structs already compiled
+    # openapi-go via codegen cli requires structs already compiled, but we will rebuild
+    # right before codegen gen-schema since PublicStructs is not used anywhere else inside codegen
+    # generate_structs_map
 
     out_dir=$BUILD_DIR
 
@@ -2536,6 +2550,7 @@ find_deleted_pkg_schemas() {
   local pkg="$2"
   local pkg_prefix=""
   to_pascal pkg_prefix "$pkg"
+
   echo "Finding deleted structs or enums from package '$pkg_prefix'..."
   mapfile -t spec_schemas < <(yq eval '.components.schemas[] | key' "$SPEC_PATH" | grep -E "^${pkg_prefix}" || true)
 
@@ -2552,6 +2567,8 @@ find_deleted_pkg_schemas() {
 }
 
 update_spec_with_structs() {
+  echo "Updating spec with rest package structs"
+
   mapfile -t structs_to_generate < <(yq e '.components.schemas[] | select(has("x-gen-struct")).x-gen-struct' $SPEC_PATH)
 
   local rest_structs=()
@@ -2635,6 +2652,8 @@ update_spec_with_structs() {
 }
 
 remove_schemas_marked_for_deletion() {
+  echo "Removing schemas marked for deletion"
+
   local paths_arr paths
   paths_arr=$(yq e '..
       | select(has("x-TO-BE-DELETED"))