Skip to content

Commit

Permalink
ATLAS-3110: added REST API to get multiple entities given their uniqu…
Browse files Browse the repository at this point in the history 
…e attribute values

Signed-off-by: Madhan Neethiraj <[email protected]>
  • Loading branch information
@ayushmnnit @mneethiraj
ayushmnnit authored and mneethiraj committed Apr 11, 2019
1 parent f070d92 commit 9d4380e
Show file tree
Hide file tree
Showing 6 changed files with 220 additions and 26 deletions.
50 changes: 41 additions & 9 deletions client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.model.instance.EntityMutationResponse;
Expand Down Expand Up @@ -56,6 +55,7 @@ public class AtlasClientV2 extends AtlasBaseClient {
// Entity APIs
public static final String ENTITY_API = BASE_URI + "v2/entity/";
private static final String PREFIX_ATTR = "attr:";
private static final String PREFIX_ATTR_ = "attr_";
private static final String TYPEDEFS_API = TYPES_API + "typedefs/";
private static final String TYPEDEF_BY_NAME = TYPES_API + "typedef/name/";
private static final String TYPEDEF_BY_GUID = TYPES_API + "typedef/guid/";
Expand Down Expand Up @@ -271,30 +271,44 @@ public AtlasEntityWithExtInfo getEntityByGuid(String guid, boolean minExtInfo, b
return callAPI(API_V2.GET_ENTITY_BY_GUID, AtlasEntityWithExtInfo.class, queryParams, guid);
}

public AtlasEntityWithExtInfo getEntityByAttribute(String type, Map<String, String> attributes) throws AtlasServiceException {
return getEntityByAttribute(type, attributes, false, false);
public AtlasEntityWithExtInfo getEntityByAttribute(String type, Map<String, String> uniqAttributes) throws AtlasServiceException {
return getEntityByAttribute(type, uniqAttributes, false, false);
}

public AtlasEntityWithExtInfo getEntityByAttribute(String type, Map<String, String> attributes, boolean minExtInfo, boolean ignoreRelationship) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(attributes);
public AtlasEntityWithExtInfo getEntityByAttribute(String type, Map<String, String> uniqAttributes, boolean minExtInfo, boolean ignoreRelationship) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(uniqAttributes);

queryParams.add("minExtInfo", String.valueOf(minExtInfo));
queryParams.add("ignoreRelationships", String.valueOf(ignoreRelationship));

return callAPI(API_V2.GET_ENTITY_BY_ATTRIBUTE, AtlasEntityWithExtInfo.class, queryParams, type);
}

public EntityMutationResponse updateEntityByAttribute(String type, Map<String, String> attributes, AtlasEntityWithExtInfo entityInfo)
public AtlasEntitiesWithExtInfo getEntitiesByAttribute(String type, List<Map<String,String>> uniqAttributesList) throws AtlasServiceException {
return getEntitiesByAttribute(type, uniqAttributesList, false, false);
}

public AtlasEntitiesWithExtInfo getEntitiesByAttribute(String type, List<Map<String, String>> uniqAttributesList, boolean minExtInfo, boolean ignoreRelationship) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(uniqAttributesList, null);

queryParams.add("minExtInfo", String.valueOf(minExtInfo));
queryParams.add("ignoreRelationships", String.valueOf(ignoreRelationship));

return callAPI(API_V2.GET_ENTITIES_BY_ATTRIBUTES, AtlasEntitiesWithExtInfo.class, queryParams, type);
}


public EntityMutationResponse updateEntityByAttribute(String type, Map<String, String> uniqAttributes, AtlasEntityWithExtInfo entityInfo)
throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(attributes);
MultivaluedMap<String, String> queryParams = attributesToQueryParams(uniqAttributes);

return callAPI(API_V2.UPDATE_ENTITY_BY_ATTRIBUTE, EntityMutationResponse.class, entityInfo, queryParams, type);
}

/* Lineage Calls */

public EntityMutationResponse deleteEntityByAttribute(String type, Map<String, String> attributes) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(attributes);
public EntityMutationResponse deleteEntityByAttribute(String type, Map<String, String> uniqAttributes) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = attributesToQueryParams(uniqAttributes);

return callAPI(API_V2.DELETE_ENTITY_BY_ATTRIBUTE, EntityMutationResponse.class, queryParams, type);
}
Expand Down Expand Up @@ -462,6 +476,23 @@ private MultivaluedMap<String, String> attributesToQueryParams(Map<String, Strin
return queryParams;
}

private MultivaluedMap<String, String> attributesToQueryParams(List<Map<String, String>> attributesList,
MultivaluedMap<String, String> queryParams) {
if (queryParams == null) {
queryParams = new MultivaluedMapImpl();
}

for (int i = 0; i < attributesList.size(); i++) {
Map<String, String> attributes = attributesList.get(i);

for (Map.Entry<String, String> entry : attributes.entrySet()) {
queryParams.putSingle(PREFIX_ATTR_ + i + ":" + entry.getKey(), entry.getValue());
}
}

return queryParams;
}

private <T> T getTypeDefByName(final String name, Class<T> typeDefClass) throws AtlasServiceException {
String atlasPath = getAtlasPath(typeDefClass);
API api = new API(String.format(GET_BY_NAME_TEMPLATE, atlasPath, name), HttpMethod.GET, Response.Status.OK);
Expand Down Expand Up @@ -502,6 +533,7 @@ public static class API_V2 extends API {
public static final API_V2 UPDATE_ENTITY_BY_ATTRIBUTE = new API_V2(ENTITY_API + "uniqueAttribute/type/", HttpMethod.PUT, Response.Status.OK);
public static final API_V2 DELETE_ENTITY_BY_GUID = new API_V2(ENTITY_API + "guid/", HttpMethod.DELETE, Response.Status.OK);
public static final API_V2 DELETE_ENTITY_BY_ATTRIBUTE = new API_V2(ENTITY_API + "uniqueAttribute/type/", HttpMethod.DELETE, Response.Status.OK);
public static final API_V2 GET_ENTITIES_BY_ATTRIBUTES = new API_V2(ENTITY_BULK_API + "uniqueAttribute/type/", HttpMethod.GET, Response.Status.OK);
public static final API_V2 GET_ENTITIES_BY_GUIDS = new API_V2(ENTITY_BULK_API, HttpMethod.GET, Response.Status.OK);
public static final API_V2 CREATE_ENTITIES = new API_V2(ENTITY_BULK_API, HttpMethod.POST, Response.Status.OK);
public static final API_V2 UPDATE_ENTITIES = new API_V2(ENTITY_BULK_API, HttpMethod.POST, Response.Status.OK);
Expand Down
11 changes: 11 additions & 0 deletions repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,17 @@ public interface AtlasEntityStore {
*/
AtlasEntitiesWithExtInfo getByIds(List<String> guid, boolean isMinExtInfo, boolean ignoreRelationships) throws AtlasBaseException;

/**
* Batch GET to retrieve entities by their uniqueIds
* @param entityType
* @param uniqueAttributes
* @param isMinExtInfo
* @param ignoreRelationships
* @return
* @throws AtlasBaseException
*/
AtlasEntitiesWithExtInfo getEntitiesByUniqueAttributes(AtlasEntityType entityType, List<Map<String, Object>> uniqueAttributes, boolean isMinExtInfo, boolean ignoreRelationships) throws AtlasBaseException;

/**
*
* Get an eneity by its unique attribute
Expand Down
24 changes: 24 additions & 0 deletions repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,30 @@ public AtlasEntitiesWithExtInfo getByIds(List<String> guids, boolean isMinExtInf
return ret;
}

@Override
@GraphTransaction
public AtlasEntitiesWithExtInfo getEntitiesByUniqueAttributes(AtlasEntityType entityType, List<Map<String, Object>> uniqueAttributes , boolean isMinExtInfo, boolean ignoreRelationships) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getEntitiesByUniqueAttributes({}, {})", entityType.getTypeName(), uniqueAttributes);
}

EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry, ignoreRelationships);

AtlasEntitiesWithExtInfo ret = entityRetriever.getEntitiesByUniqueAttributes(entityType.getTypeName(), uniqueAttributes, isMinExtInfo);

if (ret != null && ret.getEntities() != null) {
for (AtlasEntity entity : ret.getEntities()) {
AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_READ, new AtlasEntityHeader(entity)), "read entity: typeName=", entityType.getTypeName(), ", guid=", entity.getGuid());
}
}

if (LOG.isDebugEnabled()) {
LOG.debug("<== getEntitiesByUniqueAttributes({}, {}): {}", entityType.getTypeName(), uniqueAttributes, ret);
}

return ret;
}

@Override
@GraphTransaction
public AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes)
Expand Down
31 changes: 27 additions & 4 deletions ...sitory/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import org.apache.atlas.model.instance.AtlasRelationship.AtlasRelationshipWithExtInfo;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
Expand All @@ -45,7 +44,6 @@
import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType;
Expand Down Expand Up @@ -119,8 +117,6 @@
import static org.apache.atlas.repository.graph.GraphHelper.getReferenceMap;
import static org.apache.atlas.repository.graph.GraphHelper.getOutGoingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagatedClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagationEnabledClassificationVertices;
import static org.apache.atlas.repository.graph.GraphHelper.getRelationshipGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getRemovePropagations;
import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
Expand Down Expand Up @@ -375,6 +371,33 @@ public Map<String, Object> getEntityUniqueAttribute(AtlasVertex entityVertex) th
return ret;
}

public AtlasEntitiesWithExtInfo getEntitiesByUniqueAttributes(String typeName, List<Map<String, Object>> uniqueAttributesList, boolean isMinExtInfo) throws AtlasBaseException {
AtlasEntitiesWithExtInfo ret = new AtlasEntitiesWithExtInfo();
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);

if (entityType != null) {
for (Map<String, Object> uniqAttributes : uniqueAttributesList) {
try {
AtlasVertex vertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(entityType, uniqAttributes);

if (vertex != null) {
AtlasEntity entity = mapVertexToAtlasEntity(vertex, ret, isMinExtInfo);

ret.addEntity(entity);
}
} catch(AtlasBaseException e) {
if (e.getAtlasErrorCode() != AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND) {
throw e;
}
}
}
}

ret.compact();

return ret;
}

private AtlasVertex getEntityVertex(AtlasObjectId objId) throws AtlasBaseException {
AtlasVertex ret = null;

Expand Down
92 changes: 91 additions & 1 deletion webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,9 @@ public class EntityREST {
private static final Logger LOG = LoggerFactory.getLogger(EntityREST.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.EntityREST");

public static final String PREFIX_ATTR = "attr:";
public static final String PREFIX_ATTR = "attr:";
public static final String PREFIX_ATTR_ = "attr_";


private final AtlasTypeRegistry typeRegistry;
private final AtlasEntityStore entitiesStore;
Expand Down Expand Up @@ -159,6 +161,8 @@ public AtlasEntityHeader getHeaderById(@PathParam("guid") String guid) throws At
* GET /v2/entity/uniqueAttribute/type/aType?attr:aTypeAttribute=someValue
*
* @param typeName
* @param minExtInfo
* @param ignoreRelationships
* @return AtlasEntityWithExtInfo
* @throws AtlasBaseException
*/
Expand Down Expand Up @@ -581,6 +585,54 @@ public void deleteClassification(@PathParam("guid") String guid,
/** Bulk API operations **/
/******************************************************************/

/**
* Bulk API to retrieve list of entities identified by its unique attributes.
*
* In addition to the typeName path parameter, attribute key-value pair(s) can be provided in the following format
*
* typeName=<typeName>&attr_1:<attrName>=<attrValue>&attr_2:<attrName>=<attrValue>&attr_3:<attrName>=<attrValue>
*
* NOTE: The attrName should be an unique attribute for the given entity-type
*
* The REST request would look something like this
*
* GET /v2/entity/bulk/uniqueAttribute/type/hive_db?attrs_0:qualifiedName=db1@cl1&attrs_2:qualifiedName=db2@cl1
*
* @param typeName
* @param minExtInfo
* @param ignoreRelationships
* @return AtlasEntitiesWithExtInfo
* @throws AtlasBaseException
*/
@GET
@Path("/bulk/uniqueAttribute/type/{typeName}")
public AtlasEntitiesWithExtInfo getEntitiesByUniqueAttributes(@PathParam("typeName") String typeName,
@QueryParam("minExtInfo") @DefaultValue("false") boolean minExtInfo,
@QueryParam("ignoreRelationships") @DefaultValue("false") boolean ignoreRelationships,
@Context HttpServletRequest servletRequest) throws AtlasBaseException {
Servlets.validateQueryParamLength("typeName", typeName);

AtlasPerfTracer perf = null;

try {
List<Map<String, Object>> uniqAttributesList = getAttributesList(servletRequest);

if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.getEntitiesByUniqueAttributes(" + typeName + "," + uniqAttributesList + ")");
}

AtlasEntityType entityType = ensureEntityType(typeName);

for (Map<String, Object> uniqAttributes : uniqAttributesList) {
validateUniqueAttribute(entityType, uniqAttributes);
}

return entitiesStore.getEntitiesByUniqueAttributes(entityType, uniqAttributesList, minExtInfo, ignoreRelationships);
} finally {
AtlasPerfTracer.log(perf);
}
}

/**
* Bulk API to retrieve list of entities identified by its GUIDs.
*/
Expand Down Expand Up @@ -781,6 +833,7 @@ private AtlasClassificationType ensureClassificationType(String typeName) throws
return ret;
}

// attr:qualifiedName=db1@cl1 ==> { qualifiedName:db1@cl1 }
private Map<String, Object> getAttributes(HttpServletRequest request) {
Map<String, Object> attributes = new HashMap<>();

Expand All @@ -800,6 +853,43 @@ private Map<String, Object> getAttributes(HttpServletRequest request) {
return attributes;
}

// attrs_1:qualifiedName=db1@cl1&attrs_2:qualifiedName=db2@cl1 ==> [ { qualifiedName:db1@cl1 }, { qualifiedName:db2@cl1 } ]
private List<Map<String, Object>> getAttributesList(HttpServletRequest request) {
Map<String, Map<String, Object>> ret = new HashMap<>();

if (MapUtils.isNotEmpty(request.getParameterMap())) {
for (Map.Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
String key = entry.getKey();

if (key == null || !key.startsWith(PREFIX_ATTR_)) {
continue;
}

int sepPos = key.indexOf(':', PREFIX_ATTR_.length());
String[] values = entry.getValue();
String value = values != null && values.length > 0 ? values[0] : null;

if (sepPos == -1 || value == null) {
continue;
}

String attrName = key.substring(sepPos + 1);
String listIdx = key.substring(PREFIX_ATTR_.length(), sepPos);
Map<String, Object> attributes = ret.get(listIdx);

if (attributes == null) {
attributes = new HashMap<>();

ret.put(listIdx, attributes);
}

attributes.put(attrName, value);
}
}

return new ArrayList<>(ret.values());
}

/**
* Validate that each attribute given is an unique attribute
* @param entityType the entity type
Expand Down
Loading

0 comments on commit 9d4380e

Please sign in to comment.