Java环形缓冲区+生产消费模型及同步开销测试

最新推荐文章于 2022-08-09 12:56:28 发布

飞火流云

最新推荐文章于 2022-08-09 12:56:28 发布

阅读量1.9k

点赞数

CC 4.0 BY-SA版权

分类专栏： Java多线程

本文链接：https://blog.youkuaiyun.com/cloudeagle_bupt/article/details/43909771

Java多线程专栏收录该内容

95 篇文章

订阅专栏

本文对比了两种环形缓冲区实现方式：一种通过Map作为中间缓存，另一种直接操作缓冲区。实验结果显示，直接操作缓冲区的方式在时间和内存效率上更优。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

代码是为了使用环形缓冲区实现消费者生产者功能:

package ringBuffer.parallel;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.LinkedHashMap;
import java.util.Map;

public class Producer {
 
	public static void main(String[] args) {
		long start = System.currentTimeMillis() ;
		Producer mc = new Producer() ;
		Consumer cr = mc.new Consumer() ;
		cr.start() ;
		
		for (Integer j = 0; j < 5; j++) {
			LinkedHashMap<Integer, String> msgs = new LinkedHashMap<Integer, String>() ;
			for (Integer i = 0; i < 100000; i++) {
				msgs.put(i,j.toString()) ;
			}
			cr.addToMsgBuf(msgs);
		}
		
		try {
//				cr.waitUntilEmpty();
				cr.interrupt() ;				
				cr.join() ;   //这里主线程会否比子线程先跑完? 加上join
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		System.out.println(" last : "+ ( System.currentTimeMillis() -start));
		System.exit(0);
	}
	
	class Consumer extends Thread {
		int bufferSize = 2 * 1024 * 1024;
		private byte[] buffer = new byte[bufferSize];
		private Integer pos = 0;
		private Integer prePos = 0;
		String outputFile = "F:\\test\\ringBuf.txt";
    	RandomAccessFile raf;
    	StringBuffer sb = null;
    	LinkedHashMap<Integer, String> cMsgs = new LinkedHashMap<Integer, String>() ;    ;

		public Consumer(){
		    try {
		    	raf = new RandomAccessFile(outputFile, "rw");
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		
		public void run() {
			try {
				   while (!Thread.interrupted()) {
					   synchronized (cMsgs){
						   while (cMsgs.isEmpty()) {
							   cMsgs.wait();
			               }
						   sb = new StringBuffer();
						   for (Map.Entry<Integer, String> v : cMsgs.entrySet()) {
							   sb.append(v.getKey()).append("-").append(v.getValue()).append("V") ;
						   }
						   cMsgs.clear() ;
						   cMsgs.notify() ;
					   }
					   sb.append("S");  
					   byte[] buf = sb.toString().getBytes();
					   int length = buf.length ;
					   pos = prePos + length ;
					   int bufPos = 0;
					   if (pos > bufferSize && prePos < bufferSize) {   
						    bufPos = bufferSize - prePos;
							System.arraycopy(buf, 0, buffer, prePos, bufPos);  
							prePos = pos = 0;
							long fileLength = raf.length();  
							raf.seek(fileLength);  
							raf.write(buffer);  
					   }else 
						    System.arraycopy(buf, 0, buffer, prePos, length);  
					   if(bufPos==0){   
						   prePos = pos;
					   }else{    
						   System.arraycopy(buf, bufPos, buffer, prePos, length-bufPos); 
						   prePos = prePos +  length-bufPos ;
						   pos =prePos ;
					   }
				   }
				   long fileLength = raf.length();  
				   raf.seek(fileLength);  
				   raf.write(buffer, 0, prePos);  
				   raf.close();
		    }catch (IOException e) {
				e.printStackTrace();
			} catch (InterruptedException e) {
				e.printStackTrace();
			}  
		}
		
		public void addToMsgBuf(LinkedHashMap<Integer, String> currentStepMsgs) {
			 synchronized (cMsgs) {
				 	try {
				 	    while(!cMsgs.isEmpty()){
				 	    	cMsgs.wait() ;
				 	    }
				 	    cMsgs.putAll(currentStepMsgs);
					 	cMsgs.notify() ;
					} catch (InterruptedException e) {
						e.printStackTrace();
					}
		     }
		}
		
		public void waitUntilEmpty() throws InterruptedException{
			synchronized (cMsgs) {
				 while(!cMsgs.isEmpty()){
					  cMsgs.wait();
				 }
	        }
		}
	}
}

package ringBuffer.parallel;

import java.io.IOException;
import java.io.RandomAccessFile;

public class Producer2 {
 
	public static void main(String[] args) {
		long start = System.currentTimeMillis() ;
		Producer2 pr = new Producer2() ;
		Consumer cr = pr.new Consumer() ;
		cr.start() ;
		for (Integer j = 0; j < 5; j++) {
			StringBuffer sb = new StringBuffer();
			for (Integer i = 0; i < 100000; i++) {
				sb.append(i.toString()).append("-").append(j.toString() + "V");
			}
			sb.append("S");
			byte[] msgs = sb.toString().getBytes();
//			System.out.println(msgs.length);
			cr.addToMsgBuf(msgs);
		}
		try {
			cr.interrupt() ; 
			cr.join() ;  //这里主线程会否比子线程先跑完? 加上join
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		System.out.println(" last : "+ ( System.currentTimeMillis() -start));
	}
	
	class Consumer extends Thread {
		int bufferSize = 2 * 1024 * 1024;
		private byte[] buffer = new byte[bufferSize];
		private Integer pos = 0;
		private Integer prePos = 0;
		String outputFile = "F:\\test\\ringBuf.txt";
        private int totalLength = 0;
    	RandomAccessFile raf;

		public Consumer(){
		    try {
		    	raf = new RandomAccessFile(outputFile, "rw");
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		
		public void run() {
			try {
				while (!Thread.interrupted()) {
						synchronized (buffer) {
							while (pos > bufferSize ) { 
								totalLength = totalLength + prePos ;
						    	long fileLength = raf.length();     
						    	raf.seek(fileLength);     
						    	raf.write(buffer) ;  
								prePos = prePos % bufferSize ;
								pos = prePos;  
								buffer.notify() ;   
							}
						}
				}
				totalLength = totalLength + prePos ;
				long fileLength = raf.length();    
		    	raf.seek(fileLength);    
		    	raf.write(buffer, 0, prePos) ;  
				raf.close() ;
			} catch (IOException e) {
				e.printStackTrace();
			}
		}

		public void addToMsgBuf(byte[] buf) {
			if (buf == null || buf.length == 0)
				return;
			try {
				synchronized (buffer) {
					pos = prePos + buf.length ;
					int bufPos = 0 ;
					while (pos > bufferSize && prePos<bufferSize) {  
						bufPos = bufferSize-prePos ;
						System.arraycopy(buf, 0, buffer, prePos, bufPos);  
						prePos = bufferSize;
						System.out.println("Wait Until there is enough space! ");
						buffer.wait();
					}
					System.arraycopy(buf,bufPos, buffer, prePos, buf.length - bufPos);
					prePos = pos ;
					if(pos==0&&pos==prePos){  //当buffer写满时
						prePos = prePos + (buf.length - bufPos) ;
						pos = prePos ;
					}
				}
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
	}
}

以上两种实现方式，从代码功能上看，两者差不多，

空间开销分析:

从理论上看，Producer比Producer2多使用了 new LinkedHashMap<Integer, String>() ; 可能导致比其多占用内存，但实际测试显示， Producer2比Producer内存的占用略高，使用jdk自带的Jconsole(两者均为0次gc收集时)，Producer2的堆上已使用内存较高(同一个程序不同次执行时堆内存的使用情况不同，因此需要均为0次gc时比较)，

从YongGc Times(没有发生oldGc)看，也是Producer2明显多( jstat -gc pid interval(ms) 次数 > 1.txt Producer2为289次,远高于Producer的114次，这是我在内存较小的机器上的测试结果，换成大内存机器后，两者的YGC次数相同，均为6次)，

时间开销分析:

时间上猜测原因是由于Producer并行粒度太小，

producer2直接写环形缓冲区(写缓冲区时同步一次)，而producer 数据先写入map(每次循环同步一次)，再到缓冲区

这里用了int指针代替空的判断，提高了性能.

我机器配置为

为了使结果明显区别，我将main中的j 改为10:

Producer:

last : 544

Producer2:

Wait Until there is enough space!
last : 426

这里加上直接用顺序实现的，发现同步的开销确实很大，因此环形缓冲区适用于内存不足的情形，且同步开销真是大啊！

package Producer;

import java.io.IOException;
import java.io.RandomAccessFile;


public class ProduceSeq {

	public static void main(String[] args) {
		long start = System.currentTimeMillis() ;
		StringBuffer sb = new StringBuffer();
		for (Integer j = 0; j < 10; j++) {
			for (Integer i = 0; i < 100000; i++) {
				sb.append(i.toString()).append("-").append(j.toString() + "V");
			}
			sb.append("S");
		}
		 
	    try {
	    	RandomAccessFile raf = new RandomAccessFile("F:\\test\\ringBuf.txt", "rw");
	    	raf.write(sb.toString().getBytes());
	    	raf.close() ;
		 } catch (IOException e) {
			e.printStackTrace();
		 }
		System.out.println(" last : "+ ( System.currentTimeMillis() -start));
	}
}

last : 317

此外，对cuda的同步实现和java做了比较，发现cuda只有共享存储等简单的同步方式(函数)，没有java这种更高级的锁的实现方式处理数据一致性。