spark streaming sql demo

最新推荐文章于 2021-02-28 23:06:32 发布

仲景武

最新推荐文章于 2021-02-28 23:06:32 发布

阅读量641

点赞数

CC 4.0 BY-SA版权

分类专栏： hadoop

本文链接：https://blog.youkuaiyun.com/zjw11763/article/details/49131149

hadoop 专栏收录该内容

12 篇文章

订阅专栏

本文介绍了一个使用Apache Kafka和Apache Spark进行实时数据流处理的应用案例。该应用从Kafka接收消息，通过Spark Streaming进行处理，并将结果存储为DataFrame以便进一步分析。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

demo

package com.hxzq.sjzx.realdata;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.VoidFunction2;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.Time;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.kafka.KafkaUtils;
import org.json.JSONObject;
import scala.Tuple2;

public class KafkaStreamingManager {

	private static final Pattern SPACE = Pattern.compile(" ");

	private KafkaStreamingManager() {
	}

	public static void main(String[] args) throws Exception {

		// SparkContext sc = new SparkContext("local[2]","testsparkcontext");

		// final SQLContext sqlCtx = new SQLContext(sc);

		JavaStreamingContext jssc = new JavaStreamingContext("local[2]",
				"appname", new Duration(10000));
		final SQLContext sqlCtx = new SQLContext(jssc.sparkContext());

		Map<String, Integer> map = new HashMap<String, Integer>();
		map.put("message", 5);

		JavaPairReceiverInputDStream<String, String> lines = KafkaUtils
				.createStream(jssc, "192.168.20.237:2181", "message", map);
		// lines.foreachRDD(foreachFunc);

		JavaDStream<String> newline = lines
				.map(new Function<Tuple2<String, String>, String>() {
					public String call(Tuple2<String, String> tuple2) {
						return tuple2._2;

					}
				});

		lines.foreachRDD(new VoidFunction2<JavaPairRDD<String, String>, Time>() {

			public void call(JavaPairRDD<String, String> paramT1, Time paramT2)
					throws Exception {
				// TODO Auto-generated method stub
				System.out.println("======" + paramT2);

				JavaRDD<ServerRecord> rowRDD = paramT1
						.map(new Function<Tuple2<String, String>, ServerRecord>() {
							public ServerRecord call(
									Tuple2<String, String> paramT1)
									throws Exception {
								// TODO Auto-generated method stub
								JSONObject ret = new JSONObject(paramT1._2);
								String serverid = ret.getString("serverid");
								String requestId = ((JSONObject) ret
										.get("traceinfo")).get("requestid")
										.toString();

								String uuid = ((JSONObject) ret
										.get("traceinfo")).get("uuid")
										.toString();

								java.util.Random random = new java.util.Random();
								String testData = String.valueOf(random
										.nextInt(10));
								ServerRecord sr = new ServerRecord();
								sr.setAllTime(testData);
								sr.setErrorCode(testData);
								sr.setSeverId(testData);
								sr.setStatus(testData);
								sr.setNum(testData);
								return sr;
							}
						});

				DataFrame sServerFrame = sqlCtx.createDataFrame(rowRDD,
						ServerRecord.class);
				sServerFrame.registerTempTable("test000");

				// private String severId;
				// private String status;
				// private String errorCode;
				// private String allTime;
				// private String num;
				DataFrame teenagers = sqlCtx
						.sql("SELECT count(*) from test000 group by severId,status,errorCode,allTime,num");

				List<Row> list = teenagers.collectAsList();

				if (list != null && list.size() > 0) {

					for (int i = 0; i < list.size(); i++) {
						Row row = list.get(i);

						System.out.println("lllllll========" + list.get(i));
					}
				}

			}
		});

		// lines.print();
		// lines.print();
		// lineLengths.print();
		jssc.start();
		jssc.awaitTermination();

	}
}

package com.hxzq.sjzx.realdata;

public class ServerRecord {

	private String severId;
	private String status;
	private String errorCode;
	private String allTime;
	private String num;
	public String getSeverId() {
		return severId;
	}
	public void setSeverId(String severId) {
		this.severId = severId;
	}
	public String getStatus() {
		return status;
	}
	public void setStatus(String status) {
		this.status = status;
	}
	public String getErrorCode() {
		return errorCode;
	}
	public void setErrorCode(String errorCode) {
		this.errorCode = errorCode;
	}
	public String getAllTime() {
		return allTime;
	}
	public void setAllTime(String allTime) {
		this.allTime = allTime;
	}
	public String getNum() {
		return num;
	}
	public void setNum(String num) {
		this.num = num;
	}
	
	
}

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>com.hxzq.sjzx</groupId>
	<artifactId>common</artifactId>
	<version>0.0.1-SNAPSHOT</version>

	<dependencies>

		<dependency>
			<groupId>org.apache.kafka</groupId>
			<artifactId>kafka-clients</artifactId>
			<version>0.10.1.0</version>
		</dependency>
		<dependency>
			<groupId>com.googlecode.json-simple</groupId>
			<artifactId>json-simple</artifactId>
			<version>1.1.1</version>
		</dependency>


		<dependency>
			<groupId>org.json</groupId>
			<artifactId>json</artifactId>
			<version>20160810</version>
		</dependency>


		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-streaming_2.10</artifactId>
			<version>1.6.2</version>
		</dependency>

		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-streaming-kafka_2.10</artifactId>
			<version>1.6.2</version>
		</dependency>

		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-sql_2.10</artifactId>
			<version>1.6.2</version>
		</dependency>

		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-core_2.10</artifactId>
			<version>1.6.2</version>
		</dependency>
	</dependencies>
</project>