ATLAS-4988: BusinessMetadata with attribute of data type Array takes time to Delete.

client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java

@@ -349,11 +349,33 @@ public AtlasTypesDef updateAtlasTypeDefs(AtlasTypesDef typesDef) throws AtlasSer
      * @param typesDef A composite object that captures all types to be deleted
      */
     public void deleteAtlasTypeDefs(AtlasTypesDef typesDef) throws AtlasServiceException {
-        callAPI(API_V2.DELETE_TYPE_DEFS, (Class<?>) null, AtlasType.toJson(typesDef));
+        deleteAtlasTypeDefs(typesDef, false);
+    }
+
+    public void deleteAtlasTypeDefs(AtlasTypesDef typesDef, boolean forceDelete) throws AtlasServiceException {
+        MultivaluedMap<String, String> queryParams = null;
+
+        if (forceDelete) {
+            queryParams = new MultivaluedMapImpl();
+            queryParams.add("force", "true");
+        }
+
+        callAPI(API_V2.DELETE_TYPE_DEFS, (Class<?>) null, AtlasType.toJson(typesDef), queryParams);
     }
 
     public void deleteTypeByName(String typeName) throws AtlasServiceException {
-        callAPI(API_V2.DELETE_TYPE_DEF_BY_NAME, (Class) null, null, typeName);
+        callAPI(API_V2.DELETE_TYPE_DEF_BY_NAME, (Class<?>) null, null, typeName);
+    }
+
+    public void deleteTypeByName(String typeName, boolean forceDelete) throws AtlasServiceException {
+        MultivaluedMap<String, String> queryParams = null;
+
+        if (forceDelete) {
+            queryParams = new MultivaluedMapImpl();
+            queryParams.add("force", "true");
+        }
+
+        callAPI(API_V2.DELETE_TYPE_DEF_BY_NAME, (Class<?>) null, queryParams, typeName);
     }
 
     // Entity APIs

client/client-v2/src/test/java/org/apache/atlas/AtlasClientV2Test.java

@@ -82,6 +82,7 @@
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 
@@ -808,6 +809,7 @@ private static class TestableAtlasClientV2 extends AtlasClientV2 {
         private Object mockResponse;
         private Class<?> expectedReturnType;
         private boolean shouldThrowException;
+        private javax.ws.rs.core.MultivaluedMap<String, String> lastQueryParams;
 
         public TestableAtlasClientV2() {
             super(mock(WebResource.class), mock(Configuration.class));
@@ -823,8 +825,13 @@ public void setShouldThrowException(boolean shouldThrow) {
             this.shouldThrowException = shouldThrow;
         }
 
+        public javax.ws.rs.core.MultivaluedMap<String, String> getLastQueryParams() {
+            return lastQueryParams;
+        }
+
         @Override
         public <T> T callAPI(API api, Class<T> responseType, Object requestObject, String... params) throws AtlasServiceException {
+            lastQueryParams = null;
             return handleCallAPI(responseType);
         }
 
@@ -838,11 +845,13 @@ public <T> T callAPI(API api, GenericType<T> responseType, Object requestObject,
 
         @Override
         public <T> T callAPI(API api, Class<T> responseType, javax.ws.rs.core.MultivaluedMap<String, String> queryParams, String... params) throws AtlasServiceException {
+            lastQueryParams = queryParams;
             return handleCallAPI(responseType);
         }
 
         @Override
         public <T> T callAPI(API api, Class<T> responseType, Object requestObject, javax.ws.rs.core.MultivaluedMap<String, String> queryParams, String... params) throws AtlasServiceException {
+            lastQueryParams = queryParams;
             return handleCallAPI(responseType);
         }
 
@@ -1053,6 +1062,19 @@ public void testDeleteAtlasTypeDefsRealExecution() throws Exception {
         // Should not throw exception
         client.deleteAtlasTypeDefs(input);
         assertTrue(true);
+        assertNull(client.getLastQueryParams());
+    }
+
+    @Test
+    public void testDeleteAtlasTypeDefsRealExecutionWithForceDelete() throws Exception {
+        TestableAtlasClientV2 client = new TestableAtlasClientV2();
+        client.setMockResponse(null, Object.class);
+
+        AtlasTypesDef input = new AtlasTypesDef();
+        client.deleteAtlasTypeDefs(input, true);
+
+        assertNotNull(client.getLastQueryParams());
+        assertEquals(client.getLastQueryParams().getFirst("force"), "true");
     }
 
     @Test

intg/src/main/java/org/apache/atlas/AtlasErrorCode.java

@@ -183,6 +183,7 @@ public enum AtlasErrorCode {
     BLANK_NAME_ATTRIBUTE(400, "ATLAS-400-00-104", "Name Attribute can't be empty!"),
     BLANK_VALUE_ATTRIBUTE(400, "ATLAS-400-00-105", "Value Attribute can't be empty!"),
     INVALID_RELATIONSHIP_LABEL(400, "ATLAS-400-00-106", "Invalid relationship label {0}. The referenced entity type {1} could not be resolved from the type registry."),
+    NON_INDEXABLE_BM_DELETE_NOT_ALLOWED(400, "ATLAS-400-00-107", "Deletion not allowed for non-indexable Business Metadata ''{0}'' without force=true. Non-indexable attributes cannot be validated efficiently for references; use force=true to skip validation and delete (warning: orphaned references may remain)."),
 
     UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
 

intg/src/main/java/org/apache/atlas/type/AtlasBusinessMetadataType.java

@@ -44,12 +44,18 @@ public class AtlasBusinessMetadataType extends AtlasStructType {
 
     private final AtlasBusinessMetadataDef businessMetadataDef;
 
+    private boolean hasNonIndexableAttributes;
+
     public AtlasBusinessMetadataType(AtlasBusinessMetadataDef businessMetadataDef) {
         super(businessMetadataDef);
 
         this.businessMetadataDef = businessMetadataDef;
     }
 
+    public boolean hasNonIndexableAttributes() {
+        return hasNonIndexableAttributes;
+    }
+
     @Override
     public AtlasStruct createDefaultValue() {
         return null;  // there is no runtime instance for businessMetadataDef, so return null
@@ -71,6 +77,8 @@ void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException
 
         Map<String, AtlasBusinessAttribute> a = new HashMap<>();
 
+        hasNonIndexableAttributes = false;
+
         for (AtlasAttribute attribute : super.allAttributes.values()) {
             AtlasAttributeDef attributeDef = attribute.getAttributeDef();
             String            attrName     = attribute.getName();
@@ -115,10 +123,16 @@ void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException
                 bmAttribute = new AtlasBusinessAttribute(attribute, entityTypes);
             }
 
+            if (!hasNonIndexableAttributes && !Boolean.TRUE.equals(attributeDef.getIsIndexable())) {
+                hasNonIndexableAttributes = true;
+            }
+
             a.put(attrName, bmAttribute);
         }
 
         super.allAttributes = Collections.unmodifiableMap(a);
+        LOG.debug("CACHE-POPULATE [Phase1] BM='{}' hasNonIndexableAttributes={} totalAttributes={}",
+                getTypeName(), hasNonIndexableAttributes, a.size());
     }
 
     @Override
@@ -137,6 +151,29 @@ void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseExc
         }
     }
 
+    @Override
+    void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+        super.resolveReferencesPhase3(typeRegistry);
+
+        for (AtlasAttribute attribute : super.allAttributes.values()) {
+            AtlasBusinessAttribute bmAttribute = (AtlasBusinessAttribute) attribute;
+            Set<AtlasEntityType>   entityTypes = bmAttribute.getApplicableEntityTypes();
+
+            if (CollectionUtils.isNotEmpty(entityTypes)) {
+                Set<String> expandedTypeNames = new HashSet<>();
+
+                for (AtlasEntityType entityType : entityTypes) {
+                    expandedTypeNames.add(entityType.getTypeName());
+                    expandedTypeNames.addAll(entityType.getAllSubTypes());
+                }
+
+                bmAttribute.applicableEntityTypeNamesWithSubTypes = Collections.unmodifiableSet(expandedTypeNames);
+                LOG.debug("CACHE-POPULATE [Phase3] BM='{}' attr='{}' expandedTypes={}",
+                        getTypeName(), bmAttribute.getName(), expandedTypeNames);
+            }
+        }
+    }
+
     public AtlasBusinessMetadataDef getBusinessMetadataDef() {
         return businessMetadataDef;
     }
@@ -146,6 +183,8 @@ public static class AtlasBusinessAttribute extends AtlasAttribute {
         private final int                  maxStringLength;
         private final String               validPattern;
 
+        private Set<String> applicableEntityTypeNamesWithSubTypes = Collections.emptySet();
+
         public AtlasBusinessAttribute(AtlasAttribute attribute, Set<AtlasEntityType> applicableEntityTypes) {
             super(attribute);
 
@@ -171,6 +210,10 @@ public Set<AtlasEntityType> getApplicableEntityTypes() {
             return applicableEntityTypes;
         }
 
+        public Set<String> getApplicableEntityTypeNamesWithSubTypes() {
+            return applicableEntityTypeNamesWithSubTypes;
+        }
+
         public String getValidPattern() {
             return validPattern;
         }

repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasDefStore.java

@@ -20,13 +20,17 @@
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.List;
 
 /**
  * Interface for graph persistence store for AtlasTypeDef
  */
 public interface AtlasDefStore<T extends AtlasBaseTypeDef> {
+    Logger LOG = LoggerFactory.getLogger(AtlasDefStore.class);
+
     AtlasVertex preCreate(T typeDef) throws AtlasBaseException;
 
     T create(T typeDef, AtlasVertex preCreateResult) throws AtlasBaseException;
@@ -45,9 +49,33 @@ public interface AtlasDefStore<T extends AtlasBaseTypeDef> {
 
     AtlasVertex preDeleteByName(String name) throws AtlasBaseException;
 
-    void deleteByName(String name, AtlasVertex preDeleteResult) throws AtlasBaseException;
-
     AtlasVertex preDeleteByGuid(String guid) throws AtlasBaseException;
 
+    void deleteByName(String name, AtlasVertex preDeleteResult) throws AtlasBaseException;
+
     void deleteByGuid(String guid, AtlasVertex preDeleteResult) throws AtlasBaseException;
+
+    default AtlasVertex preDeleteByName(String name, boolean forceDelete) throws AtlasBaseException {
+        if (forceDelete) {
+            LOG.debug("Force-delete flag ignored in {}.preDeleteByName() for type '{}'; feature is implemented only for BusinessMetadata.", getClass().getSimpleName(), name);
+        }
+
+        return preDeleteByName(name);
+    }
+
+    default void deleteByName(String name, AtlasVertex preDeleteResult, boolean forceDelete) throws AtlasBaseException {
+        deleteByName(name, preDeleteResult);
+    }
+
+    default AtlasVertex preDeleteByGuid(String guid, boolean forceDelete) throws AtlasBaseException {
+        if (forceDelete) {
+            LOG.debug("Force-delete flag ignored in {}.preDeleteByGuid() for guid '{}'; feature is implemented only for BusinessMetadata.", getClass().getSimpleName(), guid);
+        }
+
+        return preDeleteByGuid(guid);
+    }
+
+    default void deleteByGuid(String guid, AtlasVertex preDeleteResult, boolean forceDelete) throws AtlasBaseException {
+        deleteByGuid(guid, preDeleteResult);
+    }
 }

repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java

@@ -539,15 +539,19 @@ public AtlasTypesDef createUpdateTypesDef(AtlasTypesDef typesToCreate, AtlasType
     @Override
     @GraphTransaction
     public void deleteTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException {
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("==> AtlasTypeDefGraphStore.deleteTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={}, businessMetadataDefs={})",
-                    CollectionUtils.size(typesDef.getEnumDefs()),
-                    CollectionUtils.size(typesDef.getStructDefs()),
-                    CollectionUtils.size(typesDef.getClassificationDefs()),
-                    CollectionUtils.size(typesDef.getEntityDefs()),
-                    CollectionUtils.size(typesDef.getRelationshipDefs()),
-                    CollectionUtils.size(typesDef.getBusinessMetadataDefs()));
-        }
+        deleteTypesDef(typesDef, false);
+    }
+
+    @GraphTransaction
+    public void deleteTypesDef(AtlasTypesDef typesDef, boolean forceDelete) throws AtlasBaseException {
+        LOG.debug("==> AtlasTypeDefGraphStore.deleteTypesDef(enums={}, structs={}, classfications={}, entities={}, relationships={}, businessMetadataDefs={}, forceDelete={})",
+                CollectionUtils.size(typesDef.getEnumDefs()),
+                CollectionUtils.size(typesDef.getStructDefs()),
+                CollectionUtils.size(typesDef.getClassificationDefs()),
+                CollectionUtils.size(typesDef.getEntityDefs()),
+                CollectionUtils.size(typesDef.getRelationshipDefs()),
+                CollectionUtils.size(typesDef.getBusinessMetadataDefs()),
+                forceDelete);
 
         AtlasTransientTypeRegistry ttr = lockTypeRegistryAndReleasePostCommit();
 
@@ -678,9 +682,9 @@ public void deleteTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException {
         if (CollectionUtils.isNotEmpty(typesDef.getBusinessMetadataDefs())) {
             for (AtlasBusinessMetadataDef businessMetadataDef : typesDef.getBusinessMetadataDefs()) {
                 if (StringUtils.isNotBlank(businessMetadataDef.getGuid())) {
-                    businessMetadataDefStore.deleteByGuid(businessMetadataDef.getGuid(), null);
+                    businessMetadataDefStore.deleteByGuid(businessMetadataDef.getGuid(), null, forceDelete);
                 } else {
-                    businessMetadataDefStore.deleteByName(businessMetadataDef.getName(), null);
+                    businessMetadataDefStore.deleteByName(businessMetadataDef.getName(), null, forceDelete);
                 }
             }
         }
@@ -777,7 +781,7 @@ public AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException {
 
     @Override
     @GraphTransaction
-    public void deleteTypeByName(String typeName) throws AtlasBaseException {
+    public void deleteTypeByName(String typeName, boolean forceDelete) throws AtlasBaseException {
         AtlasType atlasType = typeRegistry.getType(typeName);
 
         if (atlasType == null) {
@@ -801,7 +805,7 @@ public void deleteTypeByName(String typeName) throws AtlasBaseException {
             typesDef.setStructDefs(Collections.singletonList((AtlasStructDef) baseTypeDef));
         }
 
-        deleteTypesDef(typesDef);
+        deleteTypesDef(typesDef, forceDelete);
     }
 
     @Override

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasAbstractDefStoreV2.java

@@ -148,24 +148,46 @@ public boolean isValidName(String typeName) {
 
     @Override
     public void deleteByName(String name, AtlasVertex preDeleteResult) throws AtlasBaseException {
-        LOG.debug("==> AtlasAbstractDefStoreV1.deleteByName({}, {})", name, preDeleteResult);
+        LOG.debug("==> AtlasAbstractDefStoreV2.deleteByName({}, {})", name, preDeleteResult);
 
         AtlasVertex vertex = (preDeleteResult == null) ? preDeleteByName(name) : preDeleteResult;
 
         typeDefStore.deleteTypeVertex(vertex);
 
-        LOG.debug("<== AtlasAbstractDefStoreV1.deleteByName({}, {})", name, preDeleteResult);
+        LOG.debug("<== AtlasAbstractDefStoreV2.deleteByName({}, {})", name, preDeleteResult);
     }
 
     @Override
     public void deleteByGuid(String guid, AtlasVertex preDeleteResult) throws AtlasBaseException {
-        LOG.debug("==> AtlasAbstractDefStoreV1.deleteByGuid({}, {})", guid, preDeleteResult);
+        LOG.debug("==> AtlasAbstractDefStoreV2.deleteByGuid({}, {})", guid, preDeleteResult);
 
         AtlasVertex vertex = (preDeleteResult == null) ? preDeleteByGuid(guid) : preDeleteResult;
 
         typeDefStore.deleteTypeVertex(vertex);
 
-        LOG.debug("<== AtlasAbstractDefStoreV1.deleteByGuid({}, {})", guid, preDeleteResult);
+        LOG.debug("<== AtlasAbstractDefStoreV2.deleteByGuid({}, {})", guid, preDeleteResult);
+    }
+
+    @Override
+    public void deleteByName(String name, AtlasVertex preDeleteResult, boolean forceDelete) throws AtlasBaseException {
+        LOG.debug("==> AtlasAbstractDefStoreV2.deleteByName({}, {}, {})", name, preDeleteResult, forceDelete);
+
+        AtlasVertex vertex = (preDeleteResult == null) ? preDeleteByName(name, forceDelete) : preDeleteResult;
+
+        typeDefStore.deleteTypeVertex(vertex);
+
+        LOG.debug("<== AtlasAbstractDefStoreV2.deleteByName({}, {}, {})", name, preDeleteResult, forceDelete);
+    }
+
+    @Override
+    public void deleteByGuid(String guid, AtlasVertex preDeleteResult, boolean forceDelete) throws AtlasBaseException {
+        LOG.debug("==> AtlasAbstractDefStoreV2.deleteByGuid({}, {}, {})", guid, preDeleteResult, forceDelete);
+
+        AtlasVertex vertex = (preDeleteResult == null) ? preDeleteByGuid(guid, forceDelete) : preDeleteResult;
+
+        typeDefStore.deleteTypeVertex(vertex);
+
+        LOG.debug("<== AtlasAbstractDefStoreV2.deleteByGuid({}, {}, {})", guid, preDeleteResult, forceDelete);
     }
 
     public boolean isInvalidTypeDefName(String typeName) {