Error handling updates in quota management usage (#367)

* Small error handling updates.

* Small formatting updates
Improved error handling.

* Updates to the gitignore.

* Changes to allow for quota trees to present

* Addressed PR comment.
Removed un-used code
Fixed other linter warnings.

* Fixed issues with correct interpretation of quanty
in QM trees and requests

* Cleanup e2e script.
Added rules to update the deployment crds.

* Changed regex for quota specification

* Fixed error in qm backend struct
More error handling updates

* Fixed test case error
Minor cleanup in the Fits function.

* First pass at fixing the concurency issues

* Added explicit quota release steps for errors in ScheduleNext
Logging improvements
Additional test cases in quotaforest manager

* Added sleep to allow for the test namespace to be cleaned up.

* Small changes to unit test

Author: z103cb

.gitignore

@@ -127,4 +127,8 @@ kubernetes.tar.gz
 
 # .devcontainer files
 .devcontainer
-
+#local debug files
+cmd/kar-controllers/__debug_bin
+apiserver.local.config
+kubeconfig
+cover.out

Makefile

@@ -72,7 +72,7 @@ ${BIN_DIR}/deepcopy-gen:
 	$(info Compiling deepcopy-gen...)
 	go build -o ${BIN_DIR}/deepcopy-gen ./cmd/deepcopy-gen/
 
-images: verify-tag-name generate-code
+images: verify-tag-name generate-code update-deployment-crds
 	$(info List executable directory)
 	$(info repo id: ${git_repository_id})
 	$(info branch: ${GIT_BRANCH})
@@ -83,7 +83,7 @@ else
 	docker build --no-cache --tag mcad-controller:${TAG} --build-arg GO_BUILD_ARGS=$(GO_BUILD_ARGS) -f ${CURRENT_DIR}/Dockerfile  ${CURRENT_DIR}
 endif		
 
-images-podman: verify-tag-name generate-code
+images-podman: verify-tag-name generate-code update-deployment-crds
 	$(info List executable directory)
 	$(info repo id: ${git_repository_id})
 	$(info branch: ${GIT_BRANCH})
@@ -115,9 +115,9 @@ endif
 
 run-test:
 	$(info Running unit tests...)
-	hack/make-rules/test.sh $(WHAT) $(TESTS)
+	go test -v -coverprofile cover.out -race -parallel 8  ./pkg/...
 
-run-e2e: verify-tag-name
+run-e2e: verify-tag-name update-deployment-crds
 ifeq ($(strip $(quay_repository)),)
 	echo "Running e2e with MCAD local image: mcad-controller ${TAG} IfNotPresent."
 	hack/run-e2e-kind.sh mcad-controller ${TAG} IfNotPresent
@@ -131,3 +131,22 @@ coverage:
 
 clean:
 	rm -rf _output/
+
+#CRD file maintenance rules
+DEPLOYMENT_CRD_DIR=deployment/mcad-controller/crds
+CRD_BASE_DIR=config/crd/bases
+MCAD_CRDS= ${DEPLOYMENT_CRD_DIR}/ibm.com_quotasubtree-v1.yaml  \
+		   ${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_appwrappers.yaml \
+		   ${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_queuejobs.yaml \
+		   ${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_schedulingspecs.yaml
+
+update-deployment-crds: ${MCAD_CRDS}
+
+${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_schedulingspecs.yaml : ${CRD_BASE_DIR}/mcad.ibm.com_schedulingspecs.yaml
+${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_appwrappers.yaml : ${CRD_BASE_DIR}/mcad.ibm.com_appwrappers.yaml
+${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_queuejobs.yaml : ${CRD_BASE_DIR}/mcad.ibm.com_queuejobs.yaml
+${DEPLOYMENT_CRD_DIR}/mcad.ibm.com_schedulingspecs.yaml : ${CRD_BASE_DIR}/mcad.ibm.com_schedulingspecs.yaml
+
+
+$(DEPLOYMENT_CRD_DIR)/%: ${CRD_BASE_DIR}/%
+	cp $< $@

config/crd/bases/ibm.com_quotasubtree-v1.yaml

@@ -53,10 +53,10 @@ spec:
                           properties: 
                             cpu:
                               x-kubernetes-int-or-string: true
-                              pattern: '^[0-9]*(m)*$'
+                              pattern: '^[0-9]*(m)?$'
                             memory:
                               x-kubernetes-int-or-string: true
-                              pattern: '^[0-9]*(Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)*$'
+                              pattern: '^[0-9]*(Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)?$'
                             nvidia.com/gpu:
                               x-kubernetes-int-or-string: true
-                              pattern: '^[0-9]*$'
+                              pattern: '^[0-9]?$'

config/crd/bases/mcad.ibm.com_appwrappers.yaml

@@ -5,7 +5,6 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.4.1
-  creationTimestamp: null
   name: appwrappers.mcad.ibm.com
 spec:
   group: mcad.ibm.com
@@ -68,6 +67,8 @@ spec:
                             The associated item's level .status.conditions[].type field is monitored for any one of these conditions.  Once all items with this 
                             option is set and the conditionstatus is met the entire appwrapper state will be changed to one of the valid appwrapper completion state. Note :- this is an AND 
                             operation for all items where this option is set.  See the list of appwrapper states for a list of valid complete states.
+                          type: string
+
                         custompodresources:
                           description: Optional section that specifies resource requirements
                             for non-standard k8s resources, follows same format as

config/crd/bases/mcad.ibm.com_queuejobs.yaml

@@ -5,7 +5,6 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.4.1
-  creationTimestamp: null
   name: queuejobs.mcad.ibm.com
 spec:
   group: mcad.ibm.com

config/crd/bases/mcad.ibm.com_schedulingspecs.yaml

@@ -5,7 +5,6 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.4.1
-  creationTimestamp: null
   name: schedulingspecs.mcad.ibm.com
 spec:
   group: mcad.ibm.com

deployment/mcad-controller/crds/ibm.com_quotasubtree-v1.yaml

@@ -0,0 +1,62 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: quotasubtrees.ibm.com
+  finalizers: []
+spec:
+  group: ibm.com
+  scope: Namespaced
+  names:
+    kind: QuotaSubtree
+    singular: quotasubtree
+    plural: quotasubtrees
+    shortNames:
+    - qst
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    subresources:
+      status: {}
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          status:
+            type: object
+            x-kubernetes-preserve-unknown-fields: true
+          spec:
+            type: object
+            properties:    
+              parent: 
+                type: string
+              parentNamespace:
+                type: string
+              children:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    namespace:
+                      type: string
+                    quotas:
+                      type: object
+                      properties:
+                        disabled: 
+                          type: boolean
+                        hardLimit: 
+                          type: boolean
+                        requests:
+                          type: object
+                          properties: 
+                            cpu:
+                              x-kubernetes-int-or-string: true
+                              pattern: '^[0-9]*(m)?$'
+                            memory:
+                              x-kubernetes-int-or-string: true
+                              pattern: '^[0-9]*(Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)?$'
+                            nvidia.com/gpu:
+                              x-kubernetes-int-or-string: true
+                              pattern: '^[0-9]?$'