flink Transformation算子（更新中）

flink Transformation算子部分

Transformation算子

map

该方法是将一个DataStream调用map方法返回一个新的DataStream。本质是将该DataStream中对应的每一条数据依次迭代出来，应用map方法传入的计算逻辑，返回一个新的DataStream。原来的DataStream中对应的每一条数据，与新生成的DataStream中数据是一一对应的，也可以说是存在着映射关系的。

package com.lyj.sx.flink.day03;

import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class MapDemo  {
   
   
    public static void main(String[] args) throws  Exception {
   
   
         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
         DataStreamSource<String> dataSource = env.socketTextStream("192.168.25.62", 8899);
         dataSource.map(new QueryCategoryNameFromMySQLFunction()).print();
         env.execute();

    }
}

 package com.lyj.sx.flink.day03;

import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple4;
import org.apache.flink.configuration.Configuration;


import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;

public class QueryCategoryNameFromMySQLFunction extends RichMapFunction<String, Tuple4<String, String, String, Double>> {
   
   
    private Connection connection;
    private PreparedStatement preparedStatement;
    @Override
    public void open(Configuration parameters) throws Exception {
   
   
        connection=DriverManager.getConnection("jdbc:mysql://localhost:3306/dy_flink?characterEncoding=utf-8","root","");
        preparedStatement=connection.prepareStatement("select name from t_category where id = ?");
        super.open(parameters);
    }

    @Override
    public Tuple4<String, String, String, Double> map(String s) throws Exception {
   
   
         String[] fields = s.split(",");
         String cid = fields[0];
         preparedStatement.setInt(1,Integer.parseInt(cid));
         ResultSet resultSet = preparedStatement.executeQuery();
        String name = "未知";
         while (resultSet.next()){
   
   
               name = resultSet.getString(1);
         }
         resultSet.close();
        return  Tuple4.of(fields[0], fields[1], name, Double.parseDouble(fields[3]));
    }
    
    @Override
    public void close() throws Exception {
   
   
        if(connection!=null){
   
   
            connection.close();
        }
        if(preparedStatement!=null){
   
   
            preparedStatement.close();
        }
    }
}

flatMap扁平化映射（DataStream → DataStream）

- 该方法是将一个DataStream调用flatMap方法返回一个新的DataStream，本质上是将该DataStream中的对应的每一条数据依次迭代出来，应用flatMap方法传入的计算逻辑，返回一个新的DataStream。原来的DataStream中输入的一条数据经过flatMap方法传入的计算逻辑后，会返回零到多条数据。所谓的扁平化即将原来的数据压平，返回多条数据。

package com.lyj.sx.flink.day03;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

public class FlatMapDemo1 {
   
   
    public static void main(String[] args) throws Exception {
   
   
         StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
         DataStreamSource<String> source = env.socketTextStream("192.168.25.62", 8889);
         SingleOutputStreamOperator<Tuple2<String, Integer>> flatMapData = source.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
   
   
            @Override
            public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
   
   
                String[] strings = value.split(" ");
                for (String string : strings) {
   
   
                    out.collect(Tuple2.of(string, 1));
                }
            }
        });
         flatMapData.print();
         env.execute("pxj");
    }
}

package com.lyj.sx.flink.day03;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

public class FlatMapDemo2 {
   
   
    public static void main(String[] args) throws Exception {
   
   
         StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
         DataStreamSource<String> source = env.socketTextStream("192.168.25.62", 8887);
         SingleOutputStreamOperator<Tuple2<String, Integer>> myflatMap = source.transform("MyflatMap", TypeInformation.of(new TypeHint<Tuple2<String, Integer>>() {
   
   

        }), new MyflatMap());
         myflatMap.print();
        env.execute("pxj");
    }
}

package com.lyj.sx.flink.day03;

import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;

public  class MyflatMap extends AbstractStreamOperator<Tuple2<String, Integer>> implements OneInputStreamOperator<String, Tuple2<String, Integer>> {
   
   
    @Override
    public void processElement(StreamRecord<String> element) throws Exception {
   
   
         String[] split = element.getValue().split(",");
        for (String s : split) {
   
   
            output.collect(element.replace(Tuple2.of(s,1)));
        }
    }

    @Override
    public void setKeyContextElement(StreamRecord<String> record) throws Exception {
   
   
        System.out.println("StreamRecord..............");
        OneInputStreamOperator.super.setKeyContextElement(record);
    
    }
}

keyBy按key分区（DataStream → KeyedStream）

package com.lyj.sx.flink.day03;


import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class KeyByDemo1 {
   
   
    public static void main(String[] args) throws Exception {
   
   
         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
         DataStreamSource<String> source = env.socketTextStream("192.168.25.62", 8888);
         MapFunction<String, Tuple2<String, Integer>> mapFunction = new MapFunction<String, Tuple2<String, Integer>>() {
   
   
            Tuple2<String, Integer> t;

            @Override
            public Tuple2<String, Integer> map(String s) throws Exception {
   
   
                String[] strings = s.split(" ");
                for (String string : strings) {
   
   
                    t = Tuple2.of(string, 1);
                }
                return t;
            }
        };
         source.map(mapFunction).print();
         env.execute("pxj");
    }
}

 package com.lyj.sx.flink.day03;


import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class KeyByDemo2 {
   
   
    public static void main(String[] args) throws  Exception {
   
   
         StreamExecutionEnvironment env= StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
         DataStreamSource<String> source = env.socketTextStream("192.168.25.62", 8886);
         SingleOutputStreamOperator<Tuple3<String, String, Double>> tpStream = source.map(new MapFunction<String, Tuple3<String, String, Double>>() {
   
   
            @Override
            public Tuple3<String, String, Double> map(String s) throws Exception {
   
   
                String[] fields = s.split(",");
                return Tuple3.of(fields[0], fields[1], Double.parseDouble(fields[2]));
            }
        });
        KeyedStream<Tuple3<String, String, Double>, Tuple> tuple3TupleKeyedStream = tpStream.keyBy("f0", "f1");
        tuple3TupleKeyedStream.print();
        env.execute("pxj");
    }
}

package com.lyj.sx.flink.day03;

import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class KeyByDemo3 {
   
   
    public static void main