spark中读取hdfs中文件出错

最新推荐文章于 2024-08-06 10:36:56 发布

原创最新推荐文章于 2024-08-06 10:36:56 发布 · 3.6k 阅读

0 ·

CC 4.0 BY-SA版权

spark 专栏收录该内容

6 篇文章

订阅专栏

在尝试使用Scala的Spark从HDFS读取文件时遇到VerifyError，问题源于protobuf版本冲突。解决方法是检查并更新Spark的mesos依赖至0.18.0以上，或者重新编译Spark以解决潜在的Maven编译问题。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

scala> val rdd1 = sc.textFile("hdfs://master:9001/spark/spark02/directory/")
14/07/19 17:09:36 INFO MemoryStore: ensureFreeSpace(138763) called with curMem=0, maxMem=309225062
14/07/19 17:09:36 INFO MemoryStore: Block broadcast_0 stored as values to memory (estimated size 135.5 KB, free 294.8 MB)
rdd1: org.apache.spark.rdd.RDD[String] = MappedRDD[1] at textFile at <console>:12

scala> 14/07/19 17:09:45 INFO SparkDeploySchedulerBackend: Registered executor: Actor[akka.tcp://sparkExecutor@slave01:42733/user/Executor#-2006581551] with ID 1
14/07/19 17:09:48 INFO BlockManagerInfo: Registering block manager slave01:60074 with 593.9 MB RAM

scala> rdd1.toDebugString
java.lang.VerifyError: class org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$SetOwnerRequestProto overrides final method getUnknownFields.()Lcom/google/protobuf/UnknownFieldSet;
   at java.lang.ClassLoader.defineClass1(Native Method)
   at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
   at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
   at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
   at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
   at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
   at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
   at java.security.AccessController.doPrivileged(Native Method)
   at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
   at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
   at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
   at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
   at java.lang.Class.getDeclaredMethods0(Native Method)
   at java.lang.Class.privateGetDeclaredMethods(Class.java:2531)
   at java.lang.Class.privateGetPublicMethods(Class.java:2651)
   at java.lang.Class.privateGetPublicMethods(Class.java:2661)
   at java.lang.Class.getMethods(Class.java:1467)
   at sun.misc.ProxyGenerator.generateClassFile(ProxyGenerator.java:426)
   at sun.misc.ProxyGenerator.generateProxyClass(ProxyGenerator.java:323)
   at java.lang.reflect.Proxy.getProxyClass0(Proxy.java:636)
   at java.lang.reflect.Proxy.newProxyInstance(Proxy.java:722)
   at org.apache.hadoop.ipc.ProtobufRpcEngine.getProxy(ProtobufRpcEngine.java:92)
   at org.apache.hadoop.ipc.RPC.getProtocolProxy(RPC.java:537)
   at org.apache.hadoop.hdfs.NameNodeProxies.createNNProxyWithClientProtocol(NameNodeProxies.java:328)
   at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:235)
   at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:139)
   at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:510)
   at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:453)
   at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:136)
   at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2433)
   at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:88)
   at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2467)
   at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2449)
   at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:367)
   at org.apache.hadoop.fs.Path.getFileSystem(Path.java:287)
   at org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:221)
   at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:270)
   at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:172)
   at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
   at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
   at scala.Option.getOrElse(Option.scala:120)
   at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
   at org.apache.spark.rdd.MappedRDD.getPartitions(MappedRDD.scala:28)
   at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
   at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
   at scala.Option.getOrElse(Option.scala:120)
   at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
   at org.apache.spark.rdd.RDD.org$apache$spark$rdd$RDD$$debugString$1(RDD.scala:1194)
   at org.apache.spark.rdd.RDD.toDebugString(RDD.scala:1197)
   at $iwC$$iwC$$iwC$$iwC.<init>(<console>:15)
   at $iwC$$iwC$$iwC.<init>(<console>:20)
   at $iwC$$iwC.<init>(<console>:22)
   at $iwC.<init>(<console>:24)
   at <init>(<console>:26)
   at .<init>(<console>:30)
   at .<clinit>(<console>)
   at .<init>(<console>:7)
   at .<clinit>(<console>)
   at $print(<console>)
   at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
   at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   at java.lang.reflect.Method.invoke(Method.java:606)
   at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:788)
   at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1056)
   at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:614)
   at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:645)
   at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:609)
   at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:796)
   at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
   at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:753)
   at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:601)
   at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:608)
   at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:611)
   at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply$mcZ$sp(SparkILoop.scala:936)
   at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
   at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
   at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
   at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:884)
   at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:982)
   at org.apache.spark.repl.Main$.main(Main.scala:31)
   at org.apache.spark.repl.Main.main(Main.scala)
   at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
   at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   at java.lang.reflect.Method.invoke(Method.java:606)
   at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:292)
   at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:55)
   at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

google搜到了

I've received the same error with Spark built using Maven. It turns out that mesos-0.13.0 depends on protobuf-2.4.1 which is causing the clash at runtime. Protobuf included by Akka is shaded and doesn't cause any problems.

The solution is to update the mesos dependency to 0.18.0 in spark's pom.xml. Rebuilding the JAR with this configuration solves the issue.

-Anant

我看了一下pom.xml文件，spark1.0.0里面的mesos的确已经是0.18.1版本了，所以应该不是mesos版本低的问题。我在pom.xml找了一下确实发现使用了两个版本的protobuf。

有个地方确实使用了2.4.1.

我这里的spark-1.0.0是使用maven编译的,我怀疑是我编译的时候出了问题，导致上面的错误。因为没有找到解决办法，我只好通过使用make-distribution.sh又重新编译spark，。漫长的等待后终于重新编译了一份，重新解压分发部署。再次执行同样的命令，上面的错误消失了。

可见maven编译的spark可能会存在问题。