HDPCD-Java-复习笔记（3）-lab

Java Lab Booklet

Lab: Understanding Block Storage

1.The block size needs to be at least （1,048,576 bytes） according to the dfs.namenode.fs - limits.min -block-size property。

2.The block size must be a multiple of 512 bytes (the checksum size).

View the number of blocks

hdfs fsck /user/root/stocks.csv 

-files --- the names of the files on the DataNodes.

-blocks --- the block IDs of the file.

-locations --- the IP addresses of the DataNodes.

Lab: Configuring a Hadoop Development Environment

hdfs dfsadmin -report --- Verify DataNodes in cluster.

yarn node -list  -- Verify NodeManagers in cluster.

Lab: Putting Files in HDFS with Java

Configuration configuration = new Configuration();

FileSystem fs = FileSystem.get(configuration);
Path path = new Path("counties");
Path localPath = null;
Path destPath = null;
if (!fs.exists(path)) {
fs.mkdirs(path);
}
String filename = null;
for (int i = 1; i <= 4; i++) {
filename = "counties_" + i + ".csv";
localPath = new Path("counties/" + filename);
destPath = new Path("counties/" + filename);
fs.copyFromLocalFile(localPath, destPath);
}

build.gradle:

project.ext.mainclass = 'hdfs.InputCounties'
project.ext.archiveName = 'inputcounties.jar'
apply from: '/root/java/labs/build.gradle'

yarn jar inputcounties.jar hdfs.InputCounties

Demo: Understanding MapReduce

Why are the words sorted alphabetically?

The words are the keys, and keys get sorted during the shuffle/sort phase.

Lab: Word Count

WordCountMapper

package wordcount;

import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;


public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
private static final IntWritable ONE = new IntWritable(1);
private Text outputKey = new Text();
@Override
protected void map(LongWritable key, Text value,
Mapper<LongWritable, Text, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
String lineStr = value.toString();
String[] words = StringUtils.split(lineStr, ' ');
//super.map(key, value, context);
for (String word : words) {
outputKey.set(word);
context.write(outputKey, ONE);
}
}
@Override
protected void setup(
Mapper<LongWritable, Text, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
super.setup(context);
}

@Override
protected void cleanup(
Mapper<LongWritable, Text, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
super.cleanup(context);
}

}

WordCountReducer

package wordcount;

import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
private IntWritable outputValue = new IntWritable();
@Override
protected void reduce(Text key, Iterable<IntWritable> values,
Reducer<Text, IntWritable, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable value : values) {
sum += value.get();
}
outputValue.set(sum);
context.write(key, outputValue);
}

}

WordCountJob

package wordcount;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class WordCountJob extends Configured implements Tool {

public static void main(String[] args) throws Exception {
int result = ToolRunner.run(new Configuration(), new WordCountJob(), args);
System.exit(result);
}

@Override
public int run(String[] args) throws Exception {
Job job = Job.getInstance(getConf(), "WordCountJob");
Configuration configuration = job.getConfiguration();
job.setJarByClass(getClass());
Path in = new Path(args[0]);
Path out = new Path(args[1]);
out.getFileSystem(configuration).delete(out, true);
FileInputFormat.setInputPaths(job, in);
FileOutputFormat.setOutputPath(job, out);
job.setMapperClass(WordCountMapper.class);
job.setReducerClass(WordCountReducer.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
return job.waitForCompletion(true) ? 0 : 1;
}
}