作业9：通过curator存储和读取偏移量

最新推荐文章于 2025-01-15 10:55:53 发布

烙痕

最新推荐文章于 2025-01-15 10:55:53 发布

阅读量477

点赞数

CC 4.0 BY-SA版权

分类专栏：作业

本文链接：https://blog.youkuaiyun.com/qq_37408712/article/details/81570047

作业专栏收录该内容

12 篇文章

订阅专栏

本文介绍如何使用Scala编程语言结合Curator客户端库来实现Zookeeper中偏移量的存储和读取功能。示例代码展示了依赖配置、连接设置及核心逻辑实现。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

作业要求：以Scala代码编程，通过curator存储和读取偏移量（存储到ZK，从ZK读取）
依赖包：

//为方便展示，就不把版本号抽取出来了。
<dependency>
    <groupId>org.apache.curator</groupId>
    <artifactId>curator-recipes</artifactId>
    <version>2.7.0</version>
</dependency>
<dependency>
    <groupId>org.slf4j</groupId>
    <artifactId>slf4j-log4j12</artifactId>
    <version>1.8.0-alpha2</version>
</dependency>

源码：

package com.ruozedata.zk

import org.apache.curator.retry.RetryNTimes
import org.apache.curator.framework.CuratorFrameworkFactory
import scala.collection.JavaConversions._
import org.slf4j.LoggerFactory

object Curator_zk {
  private val logger = LoggerFactory.getLogger(Curator_zk.getClass)
  val OffsetPath = s"/zktest"
  val client = CuratorFrameworkFactory.newClient("192.168.137.252:2181", new RetryNTimes(10,5000))

  def main(args: Array[String]): Unit = {
    client.start()
    logger.warn("zk client status:{}",client.getState)

    val ZK_PATH = "/zktest"
//    val data1 = "hello"
//    print("create", ZK_PATH, data1)
//    client.create.creatingParentsIfNeeded.forPath(ZK_PATH, data1.getBytes)

    val offsets : Map[TopicAndPartition, Long] = Map(TopicAndPartition("web",1) ->4444, TopicAndPartition("web",2)->2222 )
    storeOffsets(offsets)

    val topicSet = Set("web")
    val offsetstoMap:Map[TopicAndPartition, Long]= obtainOffsets(topicSet,Map(TopicAndPartition("web",1) ->0))
    offsetstoMap.keySet.foreach(line=>{
      val  topicName = line.topic
      val  topicPartition = line.partition
      val data = client.getData.forPath(s"$OffsetPath/$topicName/$topicPartition")
      val offset = java.lang.Long.valueOf(new String(data)).toLong
      println("偏移量读取: "+"路径"+s"$OffsetPath/$topicName/$topicPartition"+"的值为:"+ offset)
    })
  }

//  /consumers/G301/offsetsze_offset_topic/partition/0
//  /consumers/G301/offsetsze_offset_topic/partition/1
//  /consumers/G301/offsetsze_offset_topic/partition/2

  def obtainOffsets(topicSet: Set[String], defaultOffset: Map[TopicAndPartition, Long])
  : Map[TopicAndPartition, Long] = {

    // TODO...
    val offsets = for {
      //t就是路径webstatistic/Offsets下面的子目录遍历
      t <- client.getChildren.forPath(OffsetPath)
      if topicSet.contains(t)
      //p就是新路径   /webstatistic/Offsets/donews_website
      p <- client.getChildren.forPath(s"$OffsetPath/$t")
    } yield {
      //遍历路径下面的partition中的offset
      val data = client.getData.forPath(s"$OffsetPath/$t/$p")
      //将data变成Long类型
      val offset = java.lang.Long.valueOf(new String(data)).toLong
      (TopicAndPartition(t, Integer.parseInt(p)), offset)
    }
    defaultOffset ++ offsets.toMap
  }

  case class TopicAndPartition(topic:String, partition:Int)

  def storeOffsets(offsets: Map[TopicAndPartition, Long]): Unit = {
    // TODO...
    val OffsetPath = s"/zktest"
    if (client.checkExists().forPath(OffsetPath) == null) {
      client.create().creatingParentsIfNeeded().forPath(OffsetPath)
    }
    for ((tp, offset) <- offsets) {
      val data = String.valueOf(offset).getBytes
      val path = s"$OffsetPath/${tp.topic}/${tp.partition}"
      if(client.checkExists().forPath(path) == null)
        client.create().creatingParentsIfNeeded().forPath(path)
      client.setData().forPath(path, data)
      println("偏移量储存成功")
    }
  }

}

操作结果：

[INFO ] 18-08-10 21:05:04 method:org.apache.curator.framework.imps.CuratorFrameworkImpl.start(CuratorFrameworkImpl.java:238)
Starting
[DEBUG] 18-08-10 21:05:04 method:org.apache.curator.CuratorZookeeperClient.start(CuratorZookeeperClient.java:181)
Starting
[DEBUG] 18-08-10 21:05:04 method:org.apache.curator.ConnectionState.start(ConnectionState.java:100)
Starting
[DEBUG] 18-08-10 21:05:04 method:org.apache.curator.ConnectionState.reset(ConnectionState.java:211)
reset
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:host.name=LAPTOP-UOVPHEV2.mshome.net
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:java.version=1.8.0_151
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:java.vendor=Oracle Corporation
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:java.home=C:\Program Files\Java\jdk1.8.0_151\jre
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:java.class.path=C:\Program Files\Java\jdk1.8.0_151\jre\lib\charsets.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\deploy.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\access-bridge-64.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\cldrdata.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\dnsns.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\jaccess.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\jfxrt.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\localedata.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\nashorn.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\sunec.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\sunjce_provider.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\sunmscapi.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\sunpkcs11.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\ext\zipfs.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\javaws.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\jce.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\jfr.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\jfxswt.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\jsse.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\management-agent.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\plugin.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\resources.jar;C:\Program Files\Java\jdk1.8.0_151\jre\lib\rt.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\IDEA_workspace\ZKhomework\target\classes;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\org\scala-lang\scala-library\2.11.8\scala-library-2.11.8.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\org\apache\curator\curator-recipes\2.7.0\curator-recipes-2.7.0.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\org\apache\curator\curator-framework\2.7.0\curator-framework-2.7.0.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\org\apache\curator\curator-client\2.7.0\curator-client-2.7.0.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\org\apache\zookeeper\zookeeper\3.4.6\zookeeper-3.4.6.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\jline\jline\0.9.94\jline-0.9.94.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\io\netty\netty\3.7.0.Final\netty-3.7.0.Final.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\com\google\guava\guava\16.0.1\guava-16.0.1.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\org\slf4j\slf4j-log4j12\1.8.0-alpha2\slf4j-log4j12-1.8.0-alpha2.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\org\slf4j\slf4j-api\1.8.0-alpha2\slf4j-api-1.8.0-alpha2.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\apache-maven-3.3.9\log4j\log4j\1.2.17\log4j-1.2.17.jar;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\IntelliJ IDEA 2018.1.6\lib\idea_rt.jar
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:java.library.path=C:\Program Files\Java\jdk1.8.0_151\bin;C:\WINDOWS\Sun\Java\bin;C:\WINDOWS\system32;C:\WINDOWS;%ANDROID_SDK_HOME%\platform-tools;%ANDROID_SDK_HOME%\tools;C:\Program Files\Java\jdk1.8.0_151\bin;C:\Program Files\Java\jdk1.8.0_151\jre\bin;C:\windows\system32;C:\windows;C:\windows\System32\Wbem;C:\Program Files (x86)\Intel\Intel(R) Management Engine Components\DAL;C:\Program Files\Intel\Intel(R) Management Engine Components\DAL;C:\Program Files (x86)\Intel\Intel(R) Management Engine Components\IPT;C:\Program Files\Intel\Intel(R) Management Engine Components\IPT;C:\Program Files (x86)\NVIDIA Corporation\PhysX\Common;G:\eclipse\android-sdk-windows\platform-tools;C:\Android;C:\Windows\System32;C:\Program Files (x86)\Microsoft SQL Server\100\Tools\Binn\;C:\Program Files\Microsoft SQL Server\100\Tools\Binn\;C:\Program Files\Microsoft SQL Server\100\DTS\Binn\;C:\Program Files (x86)\Microsoft SQL Server\100\Tools\Binn\VSShell\Common7\IDE\;C:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PrivateAssemblies\;C:\Program Files (x86)\Microsoft SQL Server\100\DTS\Binn\;C:\WINDOWS\system32;C:\WINDOWS;C:\WINDOWS\System32\Wbem;C:\WINDOWS\System32\WindowsPowerShell\v1.0\;C:\Program Files\Intel\WiFi\bin\;C:\Program Files\Common Files\Intel\WirelessCommon\;G:\git\Git\cmd;C:\WINDOWS\System32\OpenSSH\;G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\Git\Git\cmd;C:\Program Files\TortoiseGit\bin;C:\Program Files\Microsoft VS Code\bin;G:\bigdata\software\Anaconda3;G:\bigdata\software\Anaconda3\Library\mingw-w64\bin;G:\bigdata\software\Anaconda3\Library\usr\bin;G:\bigdata\software\Anaconda3\Library\bin;G:\bigdata\software\Anaconda3\Scripts;D:\vc++6.0\Tools\WinNT;D:\vc++6.0\MSDev98\Bin;D:\vc++6.0\Tools;D:\公用\bin;C:\Users\\AppData\Local\Microsoft\WindowsApps;c:\Program Files\Intel\WiFi\bin\;c:\Program Files\Common Files\Intel\WirelessCommon\;.
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:java.io.tmpdir=C:\Users\\AppData\Local\Temp\
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:java.compiler=<NA>
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:os.name=Windows 10
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:os.arch=amd64
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:os.version=10.0
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:user.name=
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:user.home=C:\Users\
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.Environment.logEnv(Environment.java:100)
Client environment:user.dir=G:\bigdata\[201805发布版]若泽数据-零基础大数据第4期\软件\IDEA_workspace\ZKhomework
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:438)
Initiating client connection, connectString=192.168.137.252:2181 sessionTimeout=60000 watcher=org.apache.curator.ConnectionState@9a7504c
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn.<clinit>(ClientCnxn.java:102)
zookeeper.disableAutoWatchReset is false
[WARN ] 18-08-10 21:05:04 method:com.ruozedata.zk.Curator_zk$.main(Curator_zk.scala:15)
zk client status:STARTED
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.logStartConnect(ClientCnxn.java:975)
Opening socket connection to server 192.168.137.252/192.168.137.252:2181. Will not attempt to authenticate using SASL (unknown error)
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.primeConnection(ClientCnxn.java:852)
Socket connection established to 192.168.137.252/192.168.137.252:2181, initiating session
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.primeConnection(ClientCnxn.java:892)
Session establishment request sent on 192.168.137.252/192.168.137.252:2181
[INFO ] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.onConnected(ClientCnxn.java:1235)
Session establishment complete on server 192.168.137.252/192.168.137.252:2181, sessionid = 0x16523e29171000b, negotiated timeout = 40000
[INFO ] 18-08-10 21:05:04 method:org.apache.curator.framework.state.ConnectionStateManager.postState(ConnectionStateManager.java:228)
State change: CONNECTED
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 1,3  replyHeader:: 1,132,0  request:: '/zktest,F  response:: s{5,5,1533809972887,1533809972887,0,1,0,0,5,1,9} 
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 2,3  replyHeader:: 2,132,0  request:: '/zktest/web/1,F  response:: s{10,130,1533811066879,1533906283086,30,0,0,0,4,0,10} 
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 3,5  replyHeader:: 3,133,0  request:: '/zktest/web/1,#34343434,-1  response:: s{10,133,1533811066879,1533906304146,31,0,0,0,4,0,10} 
偏移量储存成功
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 4,3  replyHeader:: 4,133,0  request:: '/zktest/web/2,F  response:: s{12,131,1533811066888,1533906283092,30,0,0,0,4,0,12} 
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 5,5  replyHeader:: 5,134,0  request:: '/zktest/web/2,#32323232,-1  response:: s{12,134,1533811066888,1533906304152,31,0,0,0,4,0,12} 
偏移量储存成功
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 6,12  replyHeader:: 6,134,0  request:: '/zktest,F  response:: v{'web},s{5,5,1533809972887,1533809972887,0,1,0,0,5,1,9} 
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 7,12  replyHeader:: 7,134,0  request:: '/zktest/web,F  response:: v{'1,'2},s{9,9,1533811066877,1533811066877,0,2,0,0,0,2,12} 
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 8,4  replyHeader:: 8,134,0  request:: '/zktest/web/1,F  response:: #34343434,s{10,133,1533811066879,1533906304146,31,0,0,0,4,0,10} 
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 9,4  replyHeader:: 9,134,0  request:: '/zktest/web/2,F  response:: #32323232,s{12,134,1533811066888,1533906304152,31,0,0,0,4,0,12} 
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 10,4  replyHeader:: 10,134,0  request:: '/zktest/web/1,F  response:: #34343434,s{10,133,1533811066879,1533906304146,31,0,0,0,4,0,10} 
偏移量读取: 路径/zktest/web/1的值为:4444
[DEBUG] 18-08-10 21:05:04 method:org.apache.zookeeper.ClientCnxn$SendThread.readResponse(ClientCnxn.java:818)
Reading reply sessionid:0x16523e29171000b, packet:: clientPath:null serverPath:null finished:false header:: 11,4  replyHeader:: 11,134,0  request:: '/zktest/web/2,F  response:: #32323232,s{12,134,1533811066888,1533906304152,31,0,0,0,4,0,12} 
偏移量读取: 路径/zktest/web/2的值为:2222