Hibernate Search大数据量手动建立索引

最新推荐文章于 2021-03-10 14:36:44 发布

cai555

最新推荐文章于 2021-03-10 14:36:44 发布

阅读量171

点赞数

分类专栏： java 文章标签： Hibernate Java junit Eclipse JPA

java 专栏收录该内容

55 篇文章

订阅专栏

本文介绍了一个HibernateSearch分页索引过程中遇到的内存溢出问题及其解决方案。通过定期提交事务和清理EntityManager缓存，有效避免了内存溢出错误。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

刚刚看了Hibernate Search，想要写个程序试着使用JPA建立一个索引，由于数据量比较大，所以是用分页来取得数据，但是老是报outOfMemory错误，最后解决了，所以把关键的代码贴出来和大家分享：

public void index(){
		EntityManager em = EntityManagerHolder.getEntityManager();
		//wrap a EntityManager object
		
		Query query = em.createQuery(" from A").setMaxResults(1000);
		for(int i=0, j=0; i < 1000; i++, j+=1000){
			FullTextEntityManager ftem = 
						org.hibernate.search.jpa.Search.getFullTextEntityManager(em);
			
			ftem.getTransaction().begin();
			
			
			@SuppressWarnings("unchecked")
			List<Item> items = query.setFirstResult(j).getResultList();
			
			for (Item item : items) {
			    ftem.index(item);  //manually index an item instance
			}
			ftem.flush();
			//ftem.clear();
			
			/*必须每次都commit一次，否则会内存溢出*/
			ftem.getTransaction().commit(); //index are written at commit time
			
			/* 1. EntityManager.clear()必须在ftem.index之后调用，
			 * 否则ftem无法再context中找不到相应的对象。
			 * 
			 * 2. 因为EntityManager中有个Context用于缓存事务中query到
			 * 以及update,insert到数据库的对象，这样可以减少数据库的读写
			 * 次数，所以web应用的时候每个请求的线程一个EntityManager
			 * 对象。如果要用同一个EntityManager分页获得大数据量的数据，
			 * 就必须定期clear 一次。否则缓存写满了就会报outOfMemery错误。
			 * */
			em.clear();
		}
	}

EntityManagerHolder：

public class EntityManagerHolder {
	private static ThreadLocal<EntityManager> entityManagerHolder = 
										new ThreadLocal<EntityManager>();
	
	public static EntityManager getEntityManager() {
		return entityManagerHolder.get();
	}
	
	public static void setEntityManager(EntityManager session) {
		entityManagerHolder.set(session);
	}
}

初始化EntityManagerHolder：

public class TestCase {

	protected EntityManagerFactory factory;
	
	@Before
	public void setUp() throws Exception {
		factory = Persistence.createEntityManagerFactory("test");
		EntityManagerHolder.setEntityManager(factory.createEntityManager());
	}

	@After
	public void tearDown() throws Exception {
		EntityManagerHolder.getEntityManager().close();
		EntityManagerHolder.setEntityManager(null);
		factory.close();
	}

}

下面是我之前碰到的outOfMemory错误的详细信息：

java.lang.OutOfMemoryError: Java heap space
	at org.hibernate.util.IdentityMap.entryArray(IdentityMap.java:224)
	at org.hibernate.util.IdentityMap.concurrentEntries(IdentityMap.java:89)
	at org.hibernate.event.def.AbstractFlushingEventListener.prepareEntityFlushes(AbstractFlushingEventListener.java:136)
	at org.hibernate.event.def.AbstractFlushingEventListener.flushEverythingToExecutions(AbstractFlushingEventListener.java:88)
	at org.hibernate.event.def.DefaultFlushEventListener.onFlush(DefaultFlushEventListener.java:49)
	at org.hibernate.impl.SessionImpl.flush(SessionImpl.java:1027)
	at org.hibernate.impl.SessionImpl.managedFlush(SessionImpl.java:365)
	at org.hibernate.transaction.JDBCTransaction.commit(JDBCTransaction.java:137)
	at org.hibernate.ejb.TransactionImpl.commit(TransactionImpl.java:54)
	at com.chaizj.search.proptotype.ManuallyIndex.index(ManuallyIndex.java:26)
	at com.chaizj.search.proptotype.test.MannuallyIndexTest.postSetUp(MannuallyIndexTest.java:12)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:585)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:44)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:41)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:31)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:73)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:46)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:180)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:41)
	at org.junit.runners.ParentRunner$1.evaluate(ParentRunner.java:173)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:31)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:220)
	at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:46)
	at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
	at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:467)
	
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>	
java.lang.OutOfMemoryError: Java heap space
	at java.util.HashMap.addEntry(HashMap.java:797)
	at java.util.HashMap.put(HashMap.java:431)
	at org.hibernate.engine.StatefulPersistenceContext.addEntity(StatefulPersistenceContext.java:336)
	at org.hibernate.engine.StatefulPersistenceContext.addEntity(StatefulPersistenceContext.java:431)
	at org.hibernate.engine.TwoPhaseLoad.addUninitializedEntity(TwoPhaseLoad.java:260)
	at org.hibernate.loader.Loader.loadFromResultSet(Loader.java:1390)
	at org.hibernate.loader.Loader.instanceNotYetLoaded(Loader.java:1332)
	at org.hibernate.loader.Loader.getRow(Loader.java:1230)
	at org.hibernate.loader.Loader.getRowFromResultSet(Loader.java:603)
	at org.hibernate.loader.Loader.doQuery(Loader.java:724)
	at org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:259)
	at org.hibernate.loader.Loader.doList(Loader.java:2228)
	at org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2125)
	at org.hibernate.loader.Loader.list(Loader.java:2120)
	at org.hibernate.loader.hql.QueryLoader.list(QueryLoader.java:401)
	at org.hibernate.hql.ast.QueryTranslatorImpl.list(QueryTranslatorImpl.java:361)
	at org.hibernate.engine.query.HQLQueryPlan.performList(HQLQueryPlan.java:196)
	at org.hibernate.impl.SessionImpl.list(SessionImpl.java:1148)
	at org.hibernate.impl.QueryImpl.list(QueryImpl.java:102)
	at org.hibernate.ejb.QueryImpl.getResultList(QueryImpl.java:67)
	at com.chaizj.search.proptotype.ManuallyIndex.index(ManuallyIndex.java:21)
	at com.chaizj.search.proptotype.test.MannuallyIndexTest.postSetUp(MannuallyIndexTest.java:12)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:585)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:44)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:41)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:31)