作用:可以自定义组件, 以下是模拟 LookupAttrbution 的功能.
from org.apache.nifi.processor import Processor
from org.apache.nifi.processor import Relationship
from org.apache.nifi.components import PropertyDescriptor
from org.apache.nifi.expression import *
from org.apache.nifi.lookup import LookupFailureException;
from org.apache.nifi.lookup import LookupService;
from org.apache.nifi.lookup import StringLookupService;
from org.apache.nifi.processor import ProcessContext
from org.apache.nifi.processor import ProcessSession
from org.apache.nifi.processor import ProcessorInitializationContext
from org.apache.nifi.processor.exception import ProcessException
from org.apache.nifi.processor.util import StandardValidators
from org.apache.nifi.expression import ExpressionLanguageScope
class ReadContentAndStoreAsAttribute(Processor) :
__lookup_service = PropertyDescriptor.Builder().name("lookup-service").displayName("Lookup Service").description("The lookup service to use for attribute lookups").identifiesControllerService(StringLookupService).required(True).build()
__token_url = PropertyDescriptor.Builder().name("Token Url").description("get Token http Url").required(True).defaultValue("contentListener").addValidator(StandardValidators.URI_VALIDATOR).build()
__query_key = PropertyDescriptor.Builder().name("Query Key").description("query redis key").required(True).expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).build()
__rel_matched = Relationship.Builder().description("FlowFiles with matching lookups are routed to this relationship").name("matched").build()
__rel_unmatched = Relationship.Builder().description("FlowFiles with missing lookups are routed to this relationship").name("unmatched").build();
__rel_failure = Relationship.Builder().description("FlowFiles with failing lookups are routed to this relationship").name("failure").build();
def customValidate(self, context) :
errors = []
dynamicProperties = context.getProperties().keySet().stream().filter(lambda prop: prop.isDynamic()).collect(Collectors.toSet())
if dynamicProperties == null or dynamicProperties.size() < 1 :
errors.append(ValidationResult.Builder().subject("User-Defined Properties").valid(False).explanation("At least one user-defined property must be specified.").build())
requiredKeys = validationContext.getProperty(self.__lookup_service).asControllerService(LookupService).getRequiredKeys()
if requiredKeys == null or requiredKeys.size() != 1 :
errors.append(ValidationResult.Builder().subject(self.__lookup_service.getDisplayName()).valid(False).explanation("LookupAttribute requires a key-value lookup service supporting exactly one required key.").build())
return errors
def __init__(self) :
pass
def initialize(self, context) :
pass
def getRelationships(self) :
return set([self.__rel_matched, self.__rel_unmatched, self.__rel_failure])
def validate(self, context) :
pass
def getSupportedDynamicPropertyDescriptor(self, propertyDescriptorName) :
return PropertyDescriptor.Builder().name(propertyDescriptorName).required(False).addValidator(StandardValidators.createAttributeExpressionLanguageValidator(AttributeExpression.ResultType.STRING, true)).addValidator(StandardValidators.ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR).expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES).dynamic(True).build()
def getPropertyDescriptors(self) :
descriptors = [];
descriptors.append(self.__lookup_service);
descriptors.append(self.__token_url)
descriptors.append(self.__query_key)
return descriptors
def onPropertyModified(self, descriptor, newValue, oldValue) :
pass
def onTrigger(self, context, sessionFactory) :
session = sessionFactory.createSession()
lookupService = context.getProperty(self.__lookup_service).asControllerService(LookupService)
tokenHttp = context.getProperty(self.__token_url).getValue()
queryKey = context.getProperty(self.__query_key).getValue()
# ensure there is work to do
flowfile = session.get()
if flowfile is None :
return
requiredKeys = lookupService.getRequiredKeys()
coordinateKey = requiredKeys.iterator().next()
attributeValue = lookupService.lookup({coordinateKey: queryKey},flowfile.getAttributes())
matched = False
if attributeValue.isPresent() and attributeValue.get()!="" :
matched = True
queryValue = flowfile.getAttribute("queryValue")
flowfile = session.putAttribute(flowfile, 'requiredKeys',"".join(attributeValue.get()))
session.transfer(flowfile, self.__rel_matched if matched else self.__rel_unmatched)
session.commit()
processor = ReadContentAndStoreAsAttribute()
以上程序使用LookupService仅能读redis, 以下程序使用DistributedMapCacheClient可对Redis 进行读写。
from org.apache.nifi.processor import Processor
from org.apache.nifi.processor import Relationship
from org.apache.nifi.components import PropertyDescriptor
from org.apache.nifi.expression import AttributeExpression
from org.apache.nifi.expression import ExpressionLanguageScope
from org.apache.nifi.components import PropertyValue
from org.apache.nifi.lookup import StringLookupService;
from org.apache.nifi.processor import ProcessContext
from org.apache.nifi.processor import ProcessSession
from org.apache.nifi.processor import ProcessorInitializationContext
from org.apache.nifi.processor.exception import ProcessException
from org.apache.nifi.processor.util import StandardValidators
from org.apache.nifi.expression import ExpressionLanguageScope
from org.apache.nifi.distributed.cache.client import AtomicDistributedMapCacheClient
from org.apache.nifi.distributed.cache.client import DistributedMapCacheClient
from org.apache.nifi.distributed.cache.client import Serializer
from org.apache.nifi.distributed.cache.client import Deserializer
from org.python.core.util import StringUtil
import requests
# Define a subclass of Serializer for use in the client's get() method
class StringSerializer(Serializer):
def __init__(self):
pass
def serialize(self, value, out):
out.write(value)
# Define a subclass of Deserializer for use in the client's get() method
class StringDeserializer(Deserializer):
def __init__(self):
pass
def deserialize(self, bytes):
if bytes is None:
return None
return StringUtil.fromBytes(bytes)
class ReadContentAndStoreAsAttribute(Processor) :
__lookup_service = PropertyDescriptor.Builder().name("lookup-service").displayName("Lookup Service").description("The lookup service to use for attribute lookups").identifiesControllerService(AtomicDistributedMapCacheClient).required(True).build()
__token_url = PropertyDescriptor.Builder().name("Request TokenUrl").description("HTTP request for token").required(True).defaultValue("contentListener").addValidator(StandardValidators.URI_VALIDATOR).build()
__query_key = PropertyDescriptor.Builder().name("Query Key").description("query redis key").required(True).expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).build()
__compare_value = PropertyDescriptor.Builder().name("Compare Value").description("Need to be compared with redis value").required(True).expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).build()
__rel_matched = Relationship.Builder().description("FlowFiles with matching lookups are routed to this relationship").name("matched").build()
__rel_unmatched = Relationship.Builder().description("FlowFiles with missing lookups are routed to this relationship").name("unmatched").build()
__rel_failure = Relationship.Builder().description("FlowFiles with failing lookups are routed to this relationship").name("failure").build()
def validate(self, context) :
pass
def __init__(self) :
pass
def initialize(self, context) :
pass
def getRelationships(self) :
return set([self.__rel_matched, self.__rel_unmatched, self.__rel_failure])
def getPropertyDescriptors(self) :
descriptors = [];
descriptors.append(self.__lookup_service);
descriptors.append(self.__token_url)
descriptors.append(self.__query_key)
descriptors.append(self.__compare_value)
return descriptors
def onPropertyModified(self, descriptor, newValue, oldValue) :
pass
def onTrigger(self, context, sessionFactory) :
session = sessionFactory.createSession()
cacheClient = context.getProperty(self.__lookup_service).asControllerService(DistributedMapCacheClient)
tokenHttp = context.getProperty(self.__token_url).getValue()
flowfile = session.get()
if flowfile is None :
return
keySerializer = StringSerializer()
valueDeserializer = StringDeserializer()
loopupKey = context.getProperty(self.__query_key).evaluateAttributeExpressions(flowfile).getValue()
compareAttributeValue = context.getProperty(self.__compare_value).evaluateAttributeExpressions(flowfile).getValue()
redisAttributeValue = cacheClient.get(loopupKey, keySerializer, valueDeserializer) #从redis获取到的token
matched = False
httpToken = ""
statusCode = "200"
resp = "-----"
if redisAttributeValue is None:
#resp= requests.get('https://api.github.com')
httpToken = "huoge123"
if(httpToken == compareAttributeValue):
cacheClient.putIfAbsent(loopupKey,httpToken,keySerializer,keySerializer)
matched = True
else:
statusCode = "401"
matched = False
elif compareAttributeValue == redisAttributeValue:
matched = True
else:
statusCode = "402"
matched = False
flowfile = session.putAttribute(flowfile, 'resp=====',resp)
flowfile = session.putAttribute(flowfile, 'statusCode',statusCode)
session.transfer(flowfile, self.__rel_matched if matched else self.__rel_unmatched)
session.commit()
processor = ReadContentAndStoreAsAttribute()
如果需要引入外界包,可在Module Directory 中配置:
/Users/huoge/Applications/nifi/nifi-1.8.0/work/nar/extensions/nifi-lookup-services-nar-1.8.0.nar-unpacked/NAR-INF/bundled-dependencies/nifi-lookup-services-1.8.0.jar,/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages