org.apache.hadoop.io.compress源码解读

本文详细介绍了Hadoop中压缩器接口Compressor的功能与使用方法,包括设置输入数据、检查输入缓冲区状态、设定预设词典等操作,并阐述了如何获取压缩数据及管理压缩流程。
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.io.compress;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

/**
 * Specification of a stream-based 'compressor' which can be  
 * plugged into a {@link CompressionOutputStream} to compress data.
 * This is modelled after {@link java.util.zip.Deflater}
 *
 */
public interface Compressor {
  /**
   * Sets input data for compression.
   * This should be called whenever #needsInput() returns
   * <code>true</code> indicating that more input data is required.
   *
   * @param b Input data
   * @param off Start offset
   * @param len Length
   */
    //通过setInput()接收数据到内部缓存区  可以多次调用该方法
  public void setInput(byte[] b, int off, int len);
 
  /**
   * Returns true if the input data buffer is empty and
   * #setInput() should be called to provide more input.
   *
   * @return <code>true</code> if the input data buffer is empty and
   * #setInput() should be called in order to provide more input.
   */
 
  //返回false表示内部缓存已经满了  此时必须通过compress()方法获取压缩后的数据
  public boolean needsInput();
 
  /**
   * Sets preset dictionary for compression. A preset dictionary
   * is used when the history buffer can be predetermined.
   *
   * @param b Dictionary data bytes
   * @param off Start offset
   * @param len Length
   */
  public void setDictionary(byte[] b, int off, int len);

  /**
   * Return number of uncompressed bytes input so far.
   */
  //获得compressor()输入没有压缩字节的总数
  public long getBytesRead();

  /**
   * Return number of compressed bytes output so far.
   */
  //输出压缩自己的总数
  public long getBytesWritten();

  /**
   * When called, indicates that compression should end
   * with the current contents of the input buffer.
   */
  //调用finish()就开始压缩  
  public void finish();
 
  /**
   * Returns true if the end of the compressed
   * data output stream has been reached.
   * @return <code>true</code> if the end of the compressed
   * data output stream has been reached.
   */
  //判断压缩器里是不是还有没有压缩的数据
  public boolean finished();
 
  /**
   * Fills specified buffer with compressed data. Returns actual number
   * of bytes of compressed data. A return value of 0 indicates that
   * needsInput() should be called in order to determine if more input
   * data is required.
   *
   * @param b Buffer for the compressed data
   * @param off Start offset of the data
   * @param len Size of the buffer
   * @return The actual number of bytes of compressed data.
   */
  public int compress(byte[] b, int off, int len) throws IOException;
 
  /**
   * Resets compressor so that a new set of input data can be processed.
   */
  //用与重置压缩器 以处理新的输入数据集合
  public void reset();
 
  /**
   * Closes the compressor and discards any unprocessed input.
   */
  //关闭解压缩器并放弃所有没有处理的输入
  public void end();

  /**
   * Prepare the compressor to be used in a new stream with settings defined in
   * the given Configuration
   *
   * @param conf Configuration from which new setting are fetched
   */
  //更进一步允许使用hadoop的配置系统 重新配置压缩器
  public void reinit(Configuration conf);
}

输入代码 import java.io.IOException; import java.util.Iterator; import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; public class WordCount { public WordCount() { } public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "word count"); job.setJarByClass(WordCount.class); job.setMapperClass(WordCount.TokenizerMapper.class); job.setCombinerClass(WordCount.IntSumReducer.class); job.setReducerClass(WordCount.IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path("hdfs://10.23.175.100:9000/user/wordcount.txt")); FileOutputFormat.setOutputPath(job, new Path("hdfs://10.23.175.100:9000/user/result15")); System.exit(job.waitForCompletion(true)?0:1); } public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> { private static final IntWritable one = new IntWritable(1); private Text word = new Text(); public TokenizerMapper() { } public void map(Object key, Text value, Mapper<Object, Text, Text, IntWritable>.Context context) throws IOException, InterruptedException { StringTokenizer itr = new StringTokenizer(value.toString()); while(itr.hasMoreTokens()) { this.word.set(itr.nextToken()); context.write(this.word, one); } } } public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> { private IntWritable result = new IntWritable(); public IntSumReducer() { } public void reduce(Text key, Iterable<IntWritable> values, Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException { int sum = 0; IntWritable val; for(Iterator i$ = values.iterator(); i$.hasNext(); sum += val.get()) { val = (IntWritable)i$.next(); } this.result.set(sum); context.write(key, this.result); } } } 后报错 D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\bin\java.exe "-javaagent:D:\2025.9\JAVA\IDEA 2023.1\IntelliJ IDEA 2023.1\lib\idea_rt.jar=4323:D:\2025.9\JAVA\IDEA 2023.1\IntelliJ IDEA 2023.1\bin" -Dfile.encoding=UTF-8 -classpath D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\charsets.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\deploy.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\access-bridge-64.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\cldrdata.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\dnsns.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\jaccess.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\jfxrt.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\localedata.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\nashorn.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\sunec.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\sunjce_provider.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\sunmscapi.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\sunpkcs11.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\ext\zipfs.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\javaws.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\jce.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\jfr.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\jfxswt.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\jsse.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\management-agent.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\plugin.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\resources.jar;D:\2025.9\Hadoop\test\4\jdk1.8\jdk1.8\jdk1.8.0_181\jre\lib\rt.jar;D:\2025.9\Hadoop\java\hadoop_project\out\production\hadoop_project;D:\2025.9\Hadoop\test\4\lib2\fst-2.50.jar;D:\2025.9\Hadoop\test\4\lib2\re2j-1.1.jar;D:\2025.9\Hadoop\test\4\lib2\asm-5.0.4.jar;D:\2025.9\Hadoop\test\4\lib2\guice-4.0.jar;D:\2025.9\Hadoop\test\4\lib2\jna-5.2.0.jar;D:\2025.9\Hadoop\test\4\lib2\avro-1.7.7.jar;D:\2025.9\Hadoop\test\4\lib2\gson-2.8.9.jar;D:\2025.9\Hadoop\test\4\lib2\okio-2.8.0.jar;D:\2025.9\Hadoop\test\4\lib2\jline-3.9.0.jar;D:\2025.9\Hadoop\test\4\lib2\jsch-0.1.55.jar;D:\2025.9\Hadoop\test\4\lib2\asm-tree-9.1.jar;D:\2025.9\Hadoop\test\4\lib2\jettison-1.1.jar;D:\2025.9\Hadoop\test\4\lib2\jsr305-3.0.2.jar;D:\2025.9\Hadoop\test\4\lib2\log4j-1.2.17.jar;D:\2025.9\Hadoop\test\4\lib2\okhttp-4.9.3.jar;D:\2025.9\Hadoop\test\4\lib2\dnsjava-2.1.7.jar;D:\2025.9\Hadoop\test\4\lib2\ehcache-3.3.1.jar;D:\2025.9\Hadoop\test\4\lib2\json-io-2.5.1.jar;D:\2025.9\Hadoop\test\4\lib2\objenesis-2.6.jar;D:\2025.9\Hadoop\test\4\lib2\paranamer-2.3.jar;D:\2025.9\Hadoop\test\4\lib2\guava-27.0-jre.jar;D:\2025.9\Hadoop\test\4\lib2\javax.inject-1.jar;D:\2025.9\Hadoop\test\4\lib2\snakeyaml-1.26.jar;D:\2025.9\Hadoop\test\4\lib2\aopalliance-1.0.jar;D:\2025.9\Hadoop\test\4\lib2\asm-commons-9.1.jar;D:\2025.9\Hadoop\test\4\lib2\commons-cli-1.2.jar;D:\2025.9\Hadoop\test\4\lib2\commons-net-3.6.jar;D:\2025.9\Hadoop\test\4\lib2\httpcore-4.4.13.jar;D:\2025.9\Hadoop\test\4\lib2\java-util-1.9.0.jar;D:\2025.9\Hadoop\test\4\lib2\jaxb-api-2.2.11.jar;D:\2025.9\Hadoop\test\4\lib2\kerb-core-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\kerb-util-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\kerby-xdr-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\reload4j-1.2.22.jar;D:\2025.9\Hadoop\test\4\lib2\stax2-api-4.2.1.jar;D:\2025.9\Hadoop\test\4\lib2\zookeeper-3.5.6.jar;D:\2025.9\Hadoop\test\4\lib2\asm-analysis-9.1.jar;D:\2025.9\Hadoop\test\4\lib2\commons-io-2.8.0.jar;D:\2025.9\Hadoop\test\4\lib2\commons-text-1.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-kms-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-nfs-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\jersey-core-1.19.jar;D:\2025.9\Hadoop\test\4\lib2\jersey-json-1.19.jar;D:\2025.9\Hadoop\test\4\lib2\json-smart-2.4.7.jar;D:\2025.9\Hadoop\test\4\lib2\jsr311-api-1.1.1.jar;D:\2025.9\Hadoop\test\4\lib2\kerb-admin-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\kerby-asn1-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\kerby-pkix-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\kerby-util-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\slf4j-api-1.7.36.jar;D:\2025.9\Hadoop\test\4\lib2\failureaccess-1.0.jar;D:\2025.9\Hadoop\test\4\lib2\guice-servlet-4.0.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-auth-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-hdfs-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\httpclient-4.5.13.jar;D:\2025.9\Hadoop\test\4\lib2\jackson-xc-1.9.13.jar;D:\2025.9\Hadoop\test\4\lib2\jaxb-impl-2.2.3-1.jar;D:\2025.9\Hadoop\test\4\lib2\jersey-guice-1.19.jar;D:\2025.9\Hadoop\test\4\lib2\json-simple-1.1.1.jar;D:\2025.9\Hadoop\test\4\lib2\kerb-client-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\kerb-common-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\kerb-crypto-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\kerb-server-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\checker-qual-2.5.2.jar;D:\2025.9\Hadoop\test\4\lib2\commons-codec-1.15.jar;D:\2025.9\Hadoop\test\4\lib2\jersey-client-1.19.jar;D:\2025.9\Hadoop\test\4\lib2\jersey-server-1.19.jar;D:\2025.9\Hadoop\test\4\lib2\kerby-config-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\leveldbjni-all-1.8.jar;D:\2025.9\Hadoop\test\4\lib2\metrics-core-3.2.4.jar;D:\2025.9\Hadoop\test\4\lib2\netty-3.10.6.Final.jar;D:\2025.9\Hadoop\test\4\lib2\bcpkix-jdk15on-1.60.jar;D:\2025.9\Hadoop\test\4\lib2\bcprov-jdk15on-1.60.jar;D:\2025.9\Hadoop\test\4\lib2\commons-math3-3.1.1.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-common-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\jackson-core-2.12.7.jar;D:\2025.9\Hadoop\test\4\lib2\jersey-servlet-1.19.jar;D:\2025.9\Hadoop\test\4\lib2\kerb-identity-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\protobuf-java-2.5.0.jar;D:\2025.9\Hadoop\test\4\lib2\snappy-java-1.1.8.2.jar;D:\2025.9\Hadoop\test\4\lib2\woodstox-core-5.3.0.jar;D:\2025.9\Hadoop\test\4\lib2\commons-lang3-3.12.0.jar;D:\2025.9\Hadoop\test\4\lib2\curator-client-4.2.0.jar;D:\2025.9\Hadoop\test\4\lib2\jackson-jaxrs-1.9.13.jar;D:\2025.9\Hadoop\test\4\lib2\kerb-simplekdc-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\kotlin-stdlib-1.4.10.jar;D:\2025.9\Hadoop\test\4\lib2\token-provider-1.0.1.jar;D:\2025.9\Hadoop\test\4\lib2\zookeeper-jute-3.5.6.jar;D:\2025.9\Hadoop\test\4\lib2\accessors-smart-2.4.7.jar;D:\2025.9\Hadoop\test\4\lib2\commons-compress-1.21.jar;D:\2025.9\Hadoop\test\4\lib2\commons-daemon-1.0.13.jar;D:\2025.9\Hadoop\test\4\lib2\commons-logging-1.1.3.jar;D:\2025.9\Hadoop\test\4\lib2\curator-recipes-4.2.0.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-hdfs-nfs-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-hdfs-rbf-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-registry-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-api-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\HikariCP-java7-2.4.12.jar;D:\2025.9\Hadoop\test\4\lib2\mssql-jdbc-6.2.1.jre7.jar;D:\2025.9\Hadoop\test\4\lib2\nimbus-jose-jwt-9.8.1.jar;D:\2025.9\Hadoop\test\4\lib2\slf4j-reload4j-1.7.36.jar;D:\2025.9\Hadoop\test\4\lib2\j2objc-annotations-1.1.jar;D:\2025.9\Hadoop\test\4\lib2\jcip-annotations-1.0-1.jar;D:\2025.9\Hadoop\test\4\lib2\netty-all-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\commons-beanutils-1.9.4.jar;D:\2025.9\Hadoop\test\4\lib2\curator-framework-4.2.0.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-hdfs-3.3.4-tests.jar;D:\2025.9\Hadoop\test\4\lib2\jackson-core-asl-1.9.13.jar;D:\2025.9\Hadoop\test\4\lib2\jackson-databind-2.12.7.jar;D:\2025.9\Hadoop\test\4\lib2\javax.servlet-api-3.1.0.jar;D:\2025.9\Hadoop\test\4\lib2\javax.websocket-api-1.0.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-annotations-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-hdfs-client-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-hdfs-httpfs-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-client-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-common-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\commons-collections-3.2.2.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-common-3.3.4-tests.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-shaded-guava-1.1.1.jar;D:\2025.9\Hadoop\test\4\lib2\jackson-jaxrs-base-2.12.7.jar;D:\2025.9\Hadoop\test\4\lib2\jackson-mapper-asl-1.9.13.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-io-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\netty-buffer-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-common-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\swagger-annotations-1.5.4.jar;D:\2025.9\Hadoop\test\4\lib2\audience-annotations-0.5.0.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-registry-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\jackson-annotations-2.12.7.jar;D:\2025.9\Hadoop\test\4\lib2\jakarta.xml.bind-api-2.3.2.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-xml-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\netty-handler-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-hdfs-rbf-3.3.4-tests.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-http-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-jndi-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-plus-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-util-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\kotlin-stdlib-common-1.4.10.jar;D:\2025.9\Hadoop\test\4\lib2\netty-resolver-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\commons-configuration2-2.1.1.jar;D:\2025.9\Hadoop\test\4\lib2\jakarta.activation-api-1.2.1.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-dns-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-xml-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-client-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-server-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-webapp-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-http-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-mqtt-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-smtp-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-hdfs-client-3.3.4-tests.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-server-tests-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-services-api-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\javax.websocket-client-api-1.0.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-servlet-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-http2-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-redis-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-socks-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-stomp-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\websocket-api-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\animal-sniffer-annotations-1.17.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-hdfs-native-client-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-examples-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-server-common-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-server-router-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-services-core-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-security-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\netty-resolver-dns-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-client-hs-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-shaded-protobuf_3_7-1.1.1.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-util-ajax-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-haproxy-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-handler-proxy-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-udt-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-client-app-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\netty-codec-memcache-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-rxtx-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-sctp-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\websocket-client-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\websocket-common-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\websocket-server-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-client-core-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-server-web-proxy-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\jackson-jaxrs-json-provider-2.12.7.jar;D:\2025.9\Hadoop\test\4\lib2\jetty-annotations-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\websocket-servlet-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\geronimo-jcache_1.0_spec-1.0-alpha-1.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-client-common-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-server-nodemanager-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-hdfs-native-client-3.3.4-tests.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-client-shuffle-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-client-uploader-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\jackson-module-jaxb-annotations-2.12.7.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-client-jobclient-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-client-hs-plugins-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-client-nativetask-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-applications-mawo-core-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-server-resourcemanager-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-classes-epoll-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-server-sharedcachemanager-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-classes-kqueue-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\javax-websocket-client-impl-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\javax-websocket-server-impl-9.4.43.v20210629.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-mapreduce-client-jobclient-3.3.4-tests.jar;D:\2025.9\Hadoop\test\4\lib2\netty-resolver-dns-classes-macos-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-applications-distributedshell-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-server-timeline-pluginstorage-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-native-unix-common-4.1.77.Final.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-server-applicationhistoryservice-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\hadoop-yarn-applications-unmanaged-am-launcher-3.3.4.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-native-kqueue-4.1.77.Final-osx-x86_64.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-native-epoll-4.1.77.Final-linux-x86_64.jar;D:\2025.9\Hadoop\test\4\lib2\netty-resolver-dns-native-macos-4.1.77.Final-osx-x86_64.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-native-kqueue-4.1.77.Final-osx-aarch_64.jar;D:\2025.9\Hadoop\test\4\lib2\netty-transport-native-epoll-4.1.77.Final-linux-aarch_64.jar;D:\2025.9\Hadoop\test\4\lib2\netty-resolver-dns-native-macos-4.1.77.Final-osx-aarch_64.jar;D:\2025.9\Hadoop\test\4\lib2\listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar WordCount 2025-10-30 01:03:31,804 WARN [main] util.NativeCodeLoader (NativeCodeLoader.java:<clinit>(60)) - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2025-10-30 01:03:37,414 WARN [main] impl.MetricsConfig (MetricsConfig.java:loadFirst(136)) - Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2025-10-30 01:03:37,456 INFO [main] impl.MetricsSystemImpl (MetricsSystemImpl.java:startTimer(378)) - Scheduled Metric snapshot period at 10 second(s). 2025-10-30 01:03:37,456 INFO [main] impl.MetricsSystemImpl (MetricsSystemImpl.java:start(191)) - JobTracker metrics system started Exception in thread "main" ExitCodeException exitCode=-1073741515: at org.apache.hadoop.util.Shell.runCommand(Shell.java:1007) at org.apache.hadoop.util.Shell.run(Shell.java:900) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1212) at org.apache.hadoop.util.Shell.execCommand(Shell.java:1306) at org.apache.hadoop.util.Shell.execCommand(Shell.java:1288) at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:978) at org.apache.hadoop.fs.RawLocalFileSystem.mkOneDirWithMode(RawLocalFileSystem.java:660) at org.apache.hadoop.fs.RawLocalFileSystem.mkdirsWithOptionalPermission(RawLocalFileSystem.java:700) at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:672) at org.apache.hadoop.fs.RawLocalFileSystem.mkdirsWithOptionalPermission(RawLocalFileSystem.java:699) at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:677) at org.apache.hadoop.fs.FilterFileSystem.mkdirs(FilterFileSystem.java:336) at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:162) at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:113) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:148) at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1571) at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1568) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1878) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1568) at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1589) at WordCount.main(WordCount.java:28) 进程已结束,退出代码1
最新发布
10-31
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值