Skip to content

Commit

Permalink
NIFI-8206: Added identifiesExternalResource() method to PropertyDescr…
Browse files Browse the repository at this point in the history
…iptor.Builder and implemented functionality.

- Updated components to make use of new feature

NIFI-8206: Added a ResourceType of TEXT. This requires that the ResourceReferenceFactory know which types are allowed in order to create the ResourceReference. PropertyValue needs to then have the PropertyDescriptor available to it. This resulted in highlighting many bugs in unit tests where components were not exposing property descriptors via getSupportedPropertyDescriptors() or were evaluating Expression Language using the wrong scope, so fixed many unit tests/components to properly declare Expression Language scope when using it

NIFI-8206: Removed problematic unit test that required directory names with special characters that are not allowed on some operating systems

This closes apache#4890.

Signed-off-by: Bryan Bende <[email protected]>
  • Loading branch information
markap14 authored and bbende committed Apr 12, 2021
1 parent 052c60d commit 7d1d536
Show file tree
Hide file tree
Showing 136 changed files with 2,720 additions and 1,821 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,23 @@
*/
package org.apache.nifi.components;

import org.apache.nifi.components.resource.ResourceCardinality;
import org.apache.nifi.components.resource.ResourceDefinition;
import org.apache.nifi.components.resource.ResourceReference;
import org.apache.nifi.components.resource.StandardResourceReferenceFactory;
import org.apache.nifi.components.resource.ResourceType;
import org.apache.nifi.components.resource.StandardResourceDefinition;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.expression.ExpressionLanguageScope;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.expression.ExpressionLanguageScope;

/**
* An immutable object for holding information about a type of component
Expand Down Expand Up @@ -110,6 +118,10 @@ public final class PropertyDescriptor implements Comparable<PropertyDescriptor>
*/
private final Set<PropertyDependency> dependencies;

/**
* The definition of the resource(s) that this property references
*/
private final ResourceDefinition resourceDefinition;

protected PropertyDescriptor(final Builder builder) {
this.displayName = builder.displayName == null ? builder.name : builder.displayName;
Expand All @@ -126,6 +138,7 @@ protected PropertyDescriptor(final Builder builder) {
this.controllerServiceDefinition = builder.controllerServiceDefinition;
this.validators = Collections.unmodifiableList(new ArrayList<>(builder.validators));
this.dependencies = builder.dependencies == null ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(builder.dependencies));
this.resourceDefinition = builder.resourceDefinition;
}

@Override
Expand Down Expand Up @@ -159,6 +172,17 @@ public ValidationResult validate(final String input, final ValidationContext con
}
}

final ResourceDefinition resourceDefinition = getResourceDefinition();
if (resourceDefinition != null) {
final Validator validator = new ResourceDefinitionValidator(resourceDefinition, this.expressionLanguageScope);
final ValidationResult result = validator.validate(this.name, input, context);
if (!result.isValid()) {
return result;
}

lastResult = result;
}

for (final Validator validator : validators) {
lastResult = validator.validate(this.name, input, context);
if (!lastResult.isValid()) {
Expand Down Expand Up @@ -204,6 +228,7 @@ public static final class Builder {
private boolean dynamic = false;
private boolean dynamicallyModifiesClasspath = false;
private Class<? extends ControllerService> controllerServiceDefinition;
private ResourceDefinition resourceDefinition;
private List<Validator> validators = new ArrayList<>();

public Builder fromPropertyDescriptor(final PropertyDescriptor specDescriptor) {
Expand All @@ -221,6 +246,7 @@ public Builder fromPropertyDescriptor(final PropertyDescriptor specDescriptor) {
this.controllerServiceDefinition = specDescriptor.getControllerServiceDefinition();
this.validators = new ArrayList<>(specDescriptor.validators);
this.dependencies = new HashSet<>(specDescriptor.dependencies);
this.resourceDefinition = specDescriptor.resourceDefinition;
return this;
}

Expand Down Expand Up @@ -334,6 +360,8 @@ public Builder dynamic(final boolean dynamic) {
* to load required classes on an instance-by-instance basis
* (by calling {@link Class#forName(String, boolean, ClassLoader)} for example).
*
* Any property descriptor that dynamically modifies the classpath should also make use of the {@link #identifiesExternalResource(ResourceCardinality, ResourceType, ResourceType...)} method
* to indicate that the property descriptor references external resources and optionally restrict which types of resources and how many resources the property allows.
*
* @param dynamicallyModifiesClasspath whether or not this property should be used by the framework to modify the classpath
* @return the builder
Expand Down Expand Up @@ -453,6 +481,38 @@ private boolean isValueAllowed(final String value) {
return false;
}

/**
* Specifies that this property references one or more resources that are external to NiFi that the component is meant to consume.
* Any property descriptor that identifies an external resource will be automatically validated against the following rules:
* <ul>
* <li>If the ResourceCardinality is SINGLE, the given property value must be a file, a directory, or a URL that uses a protocol of http/https/file.</li>
* <li>The given resourceTypes dictate which types of input are allowed. For example, if <code>identifiesExternalResource(ResourceCardinality.SINGLE, ResourceType.FILE)</code>
* is used, the input must be a regular file. If <code>identifiesExternalResource(ResourceCardinality.SINGLE, ResourceType.FILE, ResourceType.DIRECTORY)</code> is used, then the input
* must be exactly one file OR directory.
* </li>
* <li>If the ResourceCardinality is MULTIPLE, the given property value may consist of one or more resources, each separted by a comma and optional white space.</li>
* </ul>
*
* Generally, any property descriptor that makes use of the {@link #dynamicallyModifiesClasspath(boolean)} method to dynamically update its classpath should also
* make use of this method, specifying which types of resources are allowed and how many.
*
* @param cardinality specifies how many resources the property should allow
* @param resourceType the type of resource that is allowed
* @param additionalResourceTypes if more than one type of resource is allowed, any resource type in addition to the given resource type may be provided
* @return the builder
*/
public Builder identifiesExternalResource(final ResourceCardinality cardinality, final ResourceType resourceType, final ResourceType... additionalResourceTypes) {
Objects.requireNonNull(cardinality);
Objects.requireNonNull(resourceType);

final Set<ResourceType> resourceTypes = new HashSet<>();
resourceTypes.add(resourceType);
resourceTypes.addAll(Arrays.asList(additionalResourceTypes));

this.resourceDefinition = new StandardResourceDefinition(cardinality, resourceTypes);
return this;
}

/**
* Establishes a relationship between this Property and the given property by declaring that this Property is only relevant if the given Property has a non-null value.
* Furthermore, if one or more explicit Allowable Values are provided, this Property will not be relevant unless the given Property's value is equal to one of the given Allowable Values.
Expand Down Expand Up @@ -592,6 +652,10 @@ public Set<PropertyDependency> getDependencies() {
return dependencies;
}

public ResourceDefinition getResourceDefinition() {
return resourceDefinition;
}

@Override
public boolean equals(final Object other) {
if (other == null) {
Expand Down Expand Up @@ -665,4 +729,99 @@ public ValidationResult validate(final String subject, final String input, final
return builder.build();
}
}

private static class ResourceDefinitionValidator implements Validator {
private final ResourceDefinition resourceDefinition;
private final ExpressionLanguageScope expressionLanguageScope;

public ResourceDefinitionValidator(final ResourceDefinition resourceDefinition, final ExpressionLanguageScope expressionLanguageScope) {
this.resourceDefinition = resourceDefinition;
this.expressionLanguageScope = expressionLanguageScope;
}

@Override
public ValidationResult validate(final String subject, final String configuredInput, final ValidationContext context) {
final ValidationResult.Builder resultBuilder = new ValidationResult.Builder()
.input(configuredInput)
.subject(subject);

if (configuredInput == null) {
return resultBuilder.valid(false)
.explanation("No value specified")
.build();
}

// If Expression Language is supported and is used in the property value, we cannot perform validation against the configured
// input unless the Expression Language is expressly limited to only variable registry. In that case, we can evaluate it and then
// validate the value after evaluating the Expression Language.
String input = configuredInput;
if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(configuredInput)) {
if (expressionLanguageScope != null && expressionLanguageScope == ExpressionLanguageScope.VARIABLE_REGISTRY) {
input = context.newPropertyValue(configuredInput).evaluateAttributeExpressions().getValue();
resultBuilder.input(input);
} else {
return resultBuilder.valid(true)
.explanation("Expression Language is present, so validation of property value cannot be performed")
.build();
}
}

// If the property can be text, then there's nothing to validate. Anything that is entered may be valid.
// This will be improved in the future, by allowing the user to specify the type of resource that is being referenced.
// Until then, we will simply require that the component perform any necessary validation.
final boolean allowsText = resourceDefinition.getResourceTypes().contains(ResourceType.TEXT);
if (allowsText) {
return resultBuilder.valid(true)
.explanation("Property allows for Resource Type of Text, so validation of property value cannot be performed")
.build();
}

final String[] splits = input.split(",");
if (resourceDefinition.getCardinality() == ResourceCardinality.SINGLE && splits.length > 1) {
return resultBuilder.valid(false)
.explanation("Property only supports a single Resource but " + splits.length + " resources were specified")
.build();
}

final Set<ResourceType> resourceTypes = resourceDefinition.getResourceTypes();
final List<String> nonExistentResources = new ArrayList<>();

int count = 0;
for (final String split : splits) {
final ResourceReference resourceReference = new StandardResourceReferenceFactory().createResourceReference(split, resourceDefinition);
if (resourceReference == null) {
continue;
}

count++;

final boolean accessible = resourceReference.isAccessible();
if (!accessible) {
nonExistentResources.add(resourceReference.getLocation());
continue;
}

if (!resourceTypes.contains(resourceReference.getResourceType())) {
return resultBuilder.valid(false)
.explanation("Specified Resource is a " + resourceReference.getResourceType().name() + " but this property does not allow this type of resource")
.build();
}
}

if (count == 0) {
return resultBuilder.valid(false)
.explanation("No resources were specified")
.build();
}

if (!nonExistentResources.isEmpty()) {
return resultBuilder.valid(false)
.explanation("The specified resource(s) do not exist or could not be accessed: " + nonExistentResources)
.build();
}

return resultBuilder.valid(true)
.build();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,18 @@
*/
package org.apache.nifi.components;

import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.nifi.components.resource.ResourceReference;
import org.apache.nifi.components.resource.ResourceReferences;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.expression.AttributeValueDecorator;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.DataUnit;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.registry.VariableRegistry;

import java.util.Map;
import java.util.concurrent.TimeUnit;

/**
* <p>
* A PropertyValue provides a mechanism whereby the currently configured value
Expand Down Expand Up @@ -112,6 +115,18 @@ public interface PropertyValue {
*/
<T extends ControllerService> T asControllerService(Class<T> serviceType) throws IllegalArgumentException;

/**
* @return a ResourceReference for the configured property value, or <code>null</code> if no value was specified, or if the property references multiple resources.
* @see #asResources()
*/
ResourceReference asResource();

/**
* @return a ResourceReferences for the configured property value. If no property value is set, a ResourceRferences will be returned that references no resources.
* I.e., this method will never return <code>null</code>.
*/
ResourceReferences asResources();

/**
* @return <code>true</code> if the user has configured a value, or if the
* {@link PropertyDescriptor} for the associated property has a default
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.nifi.components.resource;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Objects;

public class FileResourceReference implements ResourceReference {
private final File file;
private final ResourceType resourceType;

public FileResourceReference(final File file) {
this.file = Objects.requireNonNull(file);
this.resourceType = file.isDirectory() ? ResourceType.DIRECTORY : ResourceType.FILE;
}

@Override
public File asFile() {
return file;
}

@Override
public URL asURL() {
try {
return file.toURI().toURL();
} catch (final MalformedURLException e) {
throw new AssertionError("File " + file.getAbsolutePath() + " cannot be represented as a URL"); // we won't encounter this.
}
}

@Override
public InputStream read() throws IOException {
if (resourceType != ResourceType.FILE) {
throw new FileNotFoundException("Could not read from file with name " + file.getAbsolutePath() + " because that references a directory");
}

return new FileInputStream(file);
}

@Override
public boolean isAccessible() {
return file.exists() && file.canRead();
}

@Override
public String getLocation() {
return file.getAbsolutePath();
}

@Override
public ResourceType getResourceType() {
return resourceType;
}

@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}

if (o == null || getClass() != o.getClass()) {
return false;
}

final FileResourceReference that = (FileResourceReference) o;
return Objects.equals(file, that.file)
&& resourceType == that.resourceType;
}

@Override
public int hashCode() {
return Objects.hash(file, resourceType);
}

@Override
public String toString() {
return "FileResourceReference[file=" + file + ", resourceType=" + resourceType + "]";
}
}
Loading

0 comments on commit 7d1d536

Please sign in to comment.