Processor | InvokeScriptedProcessor 使用

最新推荐文章于 2022-05-24 10:27:57 发布

野狼e族

最新推荐文章于 2022-05-24 10:27:57 发布

阅读量693

点赞数

CC 4.0 BY-SA版权

分类专栏： Nifi 文章标签： InvokeScriptedProcessor

本文链接：https://blog.youkuaiyun.com/qq_24505127/article/details/85542037

Nifi 专栏收录该内容

6 篇文章

订阅专栏

本文介绍如何在NiFi中自定义处理器以实现属性查找功能，包括使用LookupService进行读取Redis的操作，以及利用DistributedMapCacheClient进行Redis的读写操作。通过具体代码示例展示了处理器的定义、属性设置及触发处理流程。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

作用：可以自定义组件, 以下是模拟 LookupAttrbution 的功能.

from org.apache.nifi.processor import Processor
from org.apache.nifi.processor import Relationship
from org.apache.nifi.components import PropertyDescriptor
from org.apache.nifi.expression import *
from org.apache.nifi.lookup import LookupFailureException;
from org.apache.nifi.lookup import LookupService;
from org.apache.nifi.lookup import StringLookupService;
from org.apache.nifi.processor import ProcessContext
from org.apache.nifi.processor import ProcessSession
from org.apache.nifi.processor import ProcessorInitializationContext
from org.apache.nifi.processor.exception import ProcessException
from org.apache.nifi.processor.util import StandardValidators
from org.apache.nifi.expression import ExpressionLanguageScope
class ReadContentAndStoreAsAttribute(Processor) :
    __lookup_service = PropertyDescriptor.Builder().name("lookup-service").displayName("Lookup Service").description("The lookup service to use for attribute lookups").identifiesControllerService(StringLookupService).required(True).build()
    __token_url = PropertyDescriptor.Builder().name("Token Url").description("get Token http Url").required(True).defaultValue("contentListener").addValidator(StandardValidators.URI_VALIDATOR).build()
    __query_key = PropertyDescriptor.Builder().name("Query Key").description("query redis key").required(True).expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).build()
    
    __rel_matched =  Relationship.Builder().description("FlowFiles with matching lookups are routed to this relationship").name("matched").build()
    __rel_unmatched =  Relationship.Builder().description("FlowFiles with missing lookups are routed to this relationship").name("unmatched").build();
    __rel_failure =  Relationship.Builder().description("FlowFiles with failing lookups are routed to this relationship").name("failure").build();
    


    def customValidate(self, context) :
        errors = []
        dynamicProperties = context.getProperties().keySet().stream().filter(lambda prop: prop.isDynamic()).collect(Collectors.toSet())
        if dynamicProperties == null or dynamicProperties.size() < 1 :
            errors.append(ValidationResult.Builder().subject("User-Defined Properties").valid(False).explanation("At least one user-defined property must be specified.").build())
        
        requiredKeys = validationContext.getProperty(self.__lookup_service).asControllerService(LookupService).getRequiredKeys()
        if requiredKeys == null or requiredKeys.size() != 1 :
            errors.append(ValidationResult.Builder().subject(self.__lookup_service.getDisplayName()).valid(False).explanation("LookupAttribute requires a key-value lookup service supporting exactly one required key.").build())
        
        return errors

    def __init__(self) :
        pass

    def initialize(self, context) :
        pass

    def getRelationships(self) :
        return set([self.__rel_matched, self.__rel_unmatched, self.__rel_failure])

    def validate(self, context) :
        pass

    def getSupportedDynamicPropertyDescriptor(self, propertyDescriptorName) :
        return  PropertyDescriptor.Builder().name(propertyDescriptorName).required(False).addValidator(StandardValidators.createAttributeExpressionLanguageValidator(AttributeExpression.ResultType.STRING, true)).addValidator(StandardValidators.ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR).expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES).dynamic(True).build()

    def getPropertyDescriptors(self) :
        descriptors = [];
        descriptors.append(self.__lookup_service);
        descriptors.append(self.__token_url)
        descriptors.append(self.__query_key)
        return descriptors

    def onPropertyModified(self, descriptor, newValue, oldValue) :
        pass

    def onTrigger(self, context, sessionFactory) :
        session = sessionFactory.createSession()
        lookupService = context.getProperty(self.__lookup_service).asControllerService(LookupService)
        tokenHttp = context.getProperty(self.__token_url).getValue()
        queryKey = context.getProperty(self.__query_key).getValue()
     
        # ensure there is work to do
        flowfile = session.get()
        if flowfile is None :
             return
        requiredKeys = lookupService.getRequiredKeys()
        coordinateKey = requiredKeys.iterator().next()
        attributeValue = lookupService.lookup({coordinateKey: queryKey},flowfile.getAttributes())
        matched = False
        if attributeValue.isPresent() and attributeValue.get()!="" :
            matched = True
            
        queryValue = flowfile.getAttribute("queryValue")
        flowfile = session.putAttribute(flowfile, 'requiredKeys',"".join(attributeValue.get()))
        session.transfer(flowfile, self.__rel_matched if matched else self.__rel_unmatched)
session.commit()
        

processor = ReadContentAndStoreAsAttribute()

以上程序使用LookupService仅能读redis，以下程序使用DistributedMapCacheClient可对Redis 进行读写。

from org.apache.nifi.processor import Processor
from org.apache.nifi.processor import Relationship
from org.apache.nifi.components import PropertyDescriptor
from org.apache.nifi.expression import AttributeExpression
from org.apache.nifi.expression import ExpressionLanguageScope
from org.apache.nifi.components import PropertyValue
from org.apache.nifi.lookup import StringLookupService;
from org.apache.nifi.processor import ProcessContext
from org.apache.nifi.processor import ProcessSession
from org.apache.nifi.processor import ProcessorInitializationContext
from org.apache.nifi.processor.exception import ProcessException
from org.apache.nifi.processor.util import StandardValidators
from org.apache.nifi.expression import ExpressionLanguageScope
from org.apache.nifi.distributed.cache.client import AtomicDistributedMapCacheClient
from org.apache.nifi.distributed.cache.client import DistributedMapCacheClient
from org.apache.nifi.distributed.cache.client import Serializer
from org.apache.nifi.distributed.cache.client import Deserializer
from org.python.core.util import StringUtil
import requests

# Define a subclass of Serializer for use in the client's get() method
class StringSerializer(Serializer):
    def __init__(self):
        pass
    def serialize(self, value, out):
        out.write(value)

# Define a subclass of Deserializer for use in the client's get() method
class StringDeserializer(Deserializer):
    def __init__(self):
        pass
    def deserialize(self, bytes):
        if bytes is None:
            return None
        return StringUtil.fromBytes(bytes)

class ReadContentAndStoreAsAttribute(Processor) :
    __lookup_service = PropertyDescriptor.Builder().name("lookup-service").displayName("Lookup Service").description("The lookup service to use for attribute lookups").identifiesControllerService(AtomicDistributedMapCacheClient).required(True).build()
    __token_url = PropertyDescriptor.Builder().name("Request TokenUrl").description("HTTP request for token").required(True).defaultValue("contentListener").addValidator(StandardValidators.URI_VALIDATOR).build()
    __query_key = PropertyDescriptor.Builder().name("Query Key").description("query redis key").required(True).expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).build()
    __compare_value = PropertyDescriptor.Builder().name("Compare Value").description("Need to be compared with redis value").required(True).expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).build()

    __rel_matched =  Relationship.Builder().description("FlowFiles with matching lookups are routed to this relationship").name("matched").build()
    __rel_unmatched =  Relationship.Builder().description("FlowFiles with missing lookups are routed to this relationship").name("unmatched").build()
    __rel_failure =  Relationship.Builder().description("FlowFiles with failing lookups are routed to this relationship").name("failure").build()
    

    def validate(self, context) :
        pass

    def __init__(self) :
        pass

    def initialize(self, context) :
        pass

    def getRelationships(self) :
        return set([self.__rel_matched, self.__rel_unmatched, self.__rel_failure])

    def getPropertyDescriptors(self) :
        descriptors = [];
        descriptors.append(self.__lookup_service);
        descriptors.append(self.__token_url)
        descriptors.append(self.__query_key)
        descriptors.append(self.__compare_value)
        return descriptors

    def onPropertyModified(self, descriptor, newValue, oldValue) :
        pass

    def onTrigger(self, context, sessionFactory) :
        session = sessionFactory.createSession()
        cacheClient = context.getProperty(self.__lookup_service).asControllerService(DistributedMapCacheClient)
        tokenHttp = context.getProperty(self.__token_url).getValue()
        flowfile = session.get()
        if flowfile is None :
             return
        keySerializer = StringSerializer()
        valueDeserializer = StringDeserializer()
             
        loopupKey = context.getProperty(self.__query_key).evaluateAttributeExpressions(flowfile).getValue()
        
        compareAttributeValue = context.getProperty(self.__compare_value).evaluateAttributeExpressions(flowfile).getValue()
        redisAttributeValue = cacheClient.get(loopupKey, keySerializer, valueDeserializer) #从redis获取到的token

        matched = False
        httpToken = ""
        statusCode = "200"
        resp = "-----"
        if redisAttributeValue is None:
            #resp= requests.get('https://api.github.com')
            httpToken = "huoge123"
            if(httpToken == compareAttributeValue):
                cacheClient.putIfAbsent(loopupKey,httpToken,keySerializer,keySerializer)
                matched = True
            else:
                statusCode = "401"
                matched = False 
        elif compareAttributeValue == redisAttributeValue:
            matched = True
        else:
            statusCode = "402"
            matched = False 
        flowfile = session.putAttribute(flowfile, 'resp=====',resp)   
        flowfile = session.putAttribute(flowfile, 'statusCode',statusCode)
        session.transfer(flowfile, self.__rel_matched if matched else self.__rel_unmatched)
session.commit()

processor = ReadContentAndStoreAsAttribute()

如果需要引入外界包，可在Module Directory 中配置：

/Users/huoge/Applications/nifi/nifi-1.8.0/work/nar/extensions/nifi-lookup-services-nar-1.8.0.nar-unpacked/NAR-INF/bundled-dependencies/nifi-lookup-services-1.8.0.jar,/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages