单线程、多线程完成wordcount统计

单线程完成wordcount统计

package wc;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;

/**
 * 单线程完成数据wordcount的统计
 */
public class WordCountTest {
    public static void main(String[] args) {
        long start = System.currentTimeMillis();
        Map<String,Integer> wordMap = new TreeMap<>();
        FileReader fr = null;
        BufferedReader br = null;
        try {
            fr = new FileReader("word.txt");
            br = new BufferedReader(fr);
            String tmpStr = "";
            int num = 0;
            while ((tmpStr = br.readLine())!=null){
                num++;
                String[] split = tmpStr.trim().split("\\s+");
                for (int i = 0; i < split.length; i++) {
                    if (wordMap.containsKey(split[i])){
                        Integer count = wordMap.get(split[i]);
                        wordMap.put(split[i],count+1);
                    }else {
                        wordMap.put(split[i],1);
                    }
                }
            }
            System.out.println("word.txt中总行数："+num);
            Set<String> words = wordMap.keySet();
            Iterator<String> iterator = words.iterator();
            while (iterator.hasNext()) {
                String word = iterator.next();
                Integer count = wordMap.get(word);
                System.out.println("单词:"+word+":"+count);
            }
            long end = System.currentTimeMillis();
            System.out.println("程序总用时："+(end-start)+"ms");
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

多线程完成wordcount

package wc;

import java.util.Map;
import java.util.TreeMap;

/**
 * 多线程完成wordcount
 */
public class WordCount implements Runnable{
    String content = "";
    Map<String,Integer> wordMap = new TreeMap<>();

    public WordCount(String content) {
        this.content = content;
    }

    @Override
    public void run() {
        String[] split = content.trim().split("\\s+");
        for (int i = 0; i < split.length; i++) {
            if (wordMap.containsKey(split[i])){
                Integer count = wordMap.get(split[i]);
                wordMap.put(split[i],count+1);
            }else {
                wordMap.put(split[i],1);
            }
        }
    }
}

package wc;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 *多线程完成数据 wordcount的统计
 */
public class WordCountTest1 {
    static ExecutorService executorService = Executors.newCachedThreadPool();

    // String 每个线程的名字
    // Map<String, Integer> 存放 每一个线程计算结束后 的结果
    static TreeMap<String, Map<String, Integer>> threadMap = new TreeMap<>();

    public static void threadSet(String content, Integer count){
        WordCount wordCount = new WordCount(content);
        Thread thread = new Thread(wordCount);
        executorService.execute(thread);
        threadMap.put("thread- "+ count,wordCount.wordMap);
    }

    public static void main(String[] args) {
        long start = System.currentTimeMillis();

        FileReader fr = null;
        BufferedReader br = null;

        try {
            fr = new FileReader("word.txt");
            br = new BufferedReader(fr);
            String tmpStr="";
            String content = "";
            StringBuffer lineStringBuf = new StringBuffer();
            int num = 0;
            while ( (tmpStr = br.readLine())!=null){
                num++;   // 没30万条数据启动一个线程   hello world
                //                                 hello
                //                                  hello world hello
                lineStringBuf.append(tmpStr.trim()+" ");
                if(num%300000==0){
                    threadSet(lineStringBuf.toString(),num/300000);
                    // 清空buffer 三种方法如下
                    lineStringBuf.setLength(0);
//                    lineStringBuf.delete(0,lineStringBuf.length());
//                lineStringBuf = new StringBuffer();
                }
            }
            if(lineStringBuf.length()>0){
                threadSet(lineStringBuf.toString(),0);
                // 清空buffer 三种方法如下
                lineStringBuf.setLength(0);
//                    lineStringBuf.delete(0,lineStringBuf.length());
//                lineStringBuf = new StringBuffer();
            }
            System.out.println("word.txt 总行数为："+num);

            executorService.shutdown();

            while (true){
                if(executorService.isTerminated()){  // 所有子线程全部执行完毕
                    HashMap<String, Integer> wordMap = new HashMap<>();
                    for (Map<String,Integer> map :    // map 每个子线程的结果集
                            threadMap.values()) {
                        Iterator iterator = map.keySet().iterator();
                        while (iterator.hasNext()){
                            String word = iterator.next().toString();
                            if(wordMap.containsKey(word)){
                                wordMap.put(word, wordMap.get(word)+map.get(word));
                            }else{
                                wordMap.put(word, map.get(word));
                            }
                        }
                    }
                    for (String word :
                            wordMap.keySet()) {
                        System.out.println("单词："+word+" : "+ wordMap.get(word));
                    }
                    break;
                }
            }
            long end = System.currentTimeMillis();
            System.out.println("程序用时："+ (end-start)+" ms");
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}