上一篇里面有增量更新索引
执行完就可以看到D:\search\solr\home\ask\conf下自动生成了一个dataimport.properties文件,里面是最后更新索引时间,就是根据这个时间来达到增量更新的
定时更新索引,还是借鉴以前有位哥们的
http://www.cnblogs.com/ezhangliang/archive/2012/04/11/2441945.html
solr6.0的只是引入资源文件的地方改一下,很多地方源码都不全,这里给出全部的
SolrDataImportProperties
package org.apache.solr.handler.dataimport.scheduler;
import org.apache.solr.core.SolrResourceLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Properties;
public class SolrDataImportProperties {
private Properties properties;
public static final String SYNC_ENABLED = "syncEnabled";
public static final String SYNC_CORES = "syncCores";
public static final String SERVER = "server";
public static final String PORT = "port";
public static final String WEBAPP = "webapp";
public static final String PARAMS = "params";
public static final String INTERVAL = "interval";
public static final String REBUILDINDEXPARAMS = "reBuildIndexParams";
public static final String REBUILDINDEXBEGINTIME = "reBuildIndexBeginTime";
public static final String REBUILDINDEXINTERVAL = "reBuildIndexInterval";
private static final Logger logger = LoggerFactory.getLogger(SolrDataImportProperties.class);
public void loadProperties(boolean force) {
try {
SolrResourceLoader loader = new SolrResourceLoader();
logger.info("Instance dir = " + loader.getInstancePath());
String configDir = loader.getConfigDir();
configDir = SolrResourceLoader.normalizeDir(configDir);
if ((force) || (this.properties == null)) {
this.properties = new Properties();
String dataImportPropertiesPath = configDir + "dataimport.properties";
FileInputStream fis = new FileInputStream(dataImportPropertiesPath);
this.properties.load(fis);
}
} catch (FileNotFoundException fnfe) {
logger.error(
"Error locating DataImportScheduler dataimport.properties file",
fnfe);
} catch (IOException ioe) {
logger.error(
"Error reading DataImportScheduler dataimport.properties file",
ioe);
} catch (Exception e) {
logger.error("Error loading DataImportScheduler properties", e);
}
}
public String getProperty(String key) {
return this.properties.getProperty(key);
}
}
BaseTimerTask
package org.apache.solr.handler.dataimport.scheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Timer;
import java.util.TimerTask;
public abstract class BaseTimerTask extends TimerTask {
protected String syncEnabled;
protected String[] syncCores;
protected String server;
protected String port;
protected String webapp;
protected String params;
protected String interval;
protected String cores;
protected SolrDataImportProperties p;
protected boolean singleCore;
protected String reBuildIndexParams;
protected String reBuildIndexBeginTime;
protected String reBuildIndexInterval;
protected static final Logger logger = LoggerFactory
.getLogger(BaseTimerTask.class);
public BaseTimerTask(String webAppName, Timer t) throws Exception {
// load properties from global dataimport.properties
p = new SolrDataImportProperties();
reloadParams();
fixParams(webAppName);
if (!syncEnabled.equals("1"))
throw new Exception("Schedule disabled");
if (syncCores == null
|| (syncCores.length == 1 && syncCores[0].isEmpty())) {
singleCore = true;
logger.info("<index update process> Single core identified in dataimport.properties");
} else {
singleCore = false;
logger.info("<index update process> Multiple cores identified in dataimport.properties. Sync active for: "
+ cores);
}
}
protected void reloadParams() {
p.loadProperties(true);
syncEnabled = p.getProperty(SolrDataImportProperties.SYNC_ENABLED);
cores = p.getProperty(SolrDataImportProperties.SYNC_CORES);
server = p.getProperty(SolrDataImportProperties.SERVER);
port = p.getProperty(SolrDataImportProperties.PORT);
webapp = p.getProperty(SolrDataImportProperties.WEBAPP);
params = p.getProperty(SolrDataImportProperties.PARAMS);
interval = p.getProperty(SolrDataImportProperties.INTERVAL);
syncCores = cores != null ? cores.split(",") : null;
reBuildIndexParams = p
.getProperty(SolrDataImportProperties.REBUILDINDEXPARAMS);
reBuildIndexBeginTime = p
.getProperty(SolrDataImportProperties.REBUILDINDEXBEGINTIME);
reBuildIndexInterval = p
.getProperty(SolrDataImportProperties.REBUILDINDEXINTERVAL);
}
protected void fixParams(String webAppName) {
if (server == null || server.isEmpty())
server = "localhost";
if (port == null || port.isEmpty())
port = "8080";
if (webapp == null || webapp.isEmpty())
webapp = webAppName;
if (interval == null || interval.isEmpty() || getIntervalInt() <= 0)
interval = "30";
if (reBuildIndexBeginTime == null || reBuildIndexBeginTime.isEmpty())
interval = "00:00:00";
if (reBuildIndexInterval == null || reBuildIndexInterval.isEmpty()
|| getReBuildIndexIntervalInt() <= 0)
reBuildIndexInterval = "0";
}
protected void prepUrlSendHttpPost(String params) {
String coreUrl = "http://" + server + ":" + port + "/" + webapp
+ params;
sendHttpPost(coreUrl, null);
}
protected void prepUrlSendHttpPost(String coreName, String params) {
String coreUrl = "http://" + server + ":" + port + "/" + webapp + "/"
+ coreName + params;
sendHttpPost(coreUrl, coreName);
}
protected void sendHttpPost(String completeUrl, String coreName) {
DateFormat df = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss SSS");
Date startTime = new Date();
// prepare the core var
String core = coreName == null ? "" : "[" + coreName + "] ";
logger.info(core
+ "<index update process> Process started at .............. "
+ df.format(startTime));
try {
URL url = new URL(completeUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setRequestProperty("type", "submit");
conn.setDoOutput(true);
// Send HTTP POST
conn.connect();
logger.info(core + "<index update process> Full URL\t\t\t\t"
+ conn.getURL());
logger.info(core + "<index update process> Response message\t\t\t"
+ conn.getResponseMessage());
logger.info(core + "<index update process> Response code\t\t\t"
+ conn.getResponseCode());
// listen for change in properties file if an error occurs
if (conn.getResponseCode() != 200) {
reloadParams();
}
conn.disconnect();
logger.info(core
+ "<index update process> Disconnected from server\t\t"
+ server);
Date endTime = new Date();
logger.info(core
+ "<index update process> Process ended at ................ "
+ df.format(endTime));
} catch (MalformedURLException mue) {
logger.error("Failed to assemble URL for HTTP POST", mue);
} catch (IOException ioe) {
logger.error(
"Failed to connect to the specified URL while trying to send HTTP POST",
ioe);
} catch (Exception e) {
logger.error("Failed to send HTTP POST", e);
}
}
public int getIntervalInt() {
try {
return Integer.parseInt(interval);
} catch (NumberFormatException e) {
logger.warn(
"Unable to convert 'interval' to number. Using default value (30) instead",
e);
return 30; // return default in case of error
}
}
public int getReBuildIndexIntervalInt() {
try {
return Integer.parseInt(reBuildIndexInterval);
} catch (NumberFormatException e) {
logger.info(
"Unable to convert 'reBuildIndexInterval' to number. do't rebuild index.",
e);
return 0; // return default in case of error
}
}
public Date getReBuildIndexBeginTime() {
Date beginDate = null;
try {
SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd");
String dateStr = sdfDate.format(new Date());
beginDate = sdfDate.parse(dateStr);
if (reBuildIndexBeginTime == null
|| reBuildIndexBeginTime.isEmpty()) {
return beginDate;
}
if (reBuildIndexBeginTime.matches("\\d{2}:\\d{2}:\\d{2}")) {
SimpleDateFormat sdf = new SimpleDateFormat(
"yyyy-MM-dd HH:mm:ss");
beginDate = sdf.parse(dateStr + " " + reBuildIndexBeginTime);
} else if (reBuildIndexBeginTime
.matches("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}")) {
SimpleDateFormat sdf = new SimpleDateFormat(
"yyyy-MM-dd HH:mm:ss");
beginDate = sdf.parse(reBuildIndexBeginTime);
}
return beginDate;
} catch (ParseException e) {
logger.warn(
"Unable to convert 'reBuildIndexBeginTime' to date. use now time.",
e);
return beginDate;
}
}
}
DeltaImportHTTPPostScheduler
package org.apache.solr.handler.dataimport.scheduler;
import java.util.Timer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DeltaImportHTTPPostScheduler extends BaseTimerTask {
private static final Logger logger = LoggerFactory
.getLogger(DeltaImportHTTPPostScheduler.class);
public DeltaImportHTTPPostScheduler(String webAppName, Timer t)
throws Exception {
super(webAppName, t);
logger.info("<index update process> DeltaImportHTTPPostScheduler init");
}
public void run() {
try {
// check mandatory params
if (server.isEmpty() || webapp.isEmpty() || params == null
|| params.isEmpty()) {
logger.warn("<index update process> Insuficient info provided for data import");
logger.info("<index update process> Reloading global dataimport.properties");
reloadParams();
// single-core
} else if (singleCore) {
prepUrlSendHttpPost(params);
// multi-core
} else if (syncCores.length == 0
|| (syncCores.length == 1 && syncCores[0].isEmpty())) {
logger.warn("<index update process> No cores scheduled for data import");
logger.info("<index update process> Reloading global dataimport.properties");
reloadParams();
} else {
for (String core : syncCores) {
prepUrlSendHttpPost(core, params);
}
}
} catch (Exception e) {
logger.error("Failed to prepare for sendHttpPost", e);
reloadParams();
}
}
}
FullImportHTTPPostScheduler
package org.apache.solr.handler.dataimport.scheduler;
import java.util.Timer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FullImportHTTPPostScheduler extends BaseTimerTask {
private static final Logger logger = LoggerFactory
.getLogger(FullImportHTTPPostScheduler.class);
public FullImportHTTPPostScheduler(String webAppName, Timer t)
throws Exception {
super(webAppName, t);
logger.info("<index update process> DeltaImportHTTPPostScheduler init");
}
public void run() {
try {
// check mandatory params
if (server.isEmpty() || webapp.isEmpty()
|| reBuildIndexParams == null
|| reBuildIndexParams.isEmpty()) {
logger.warn("<index update process> Insuficient info provided for data import, reBuildIndexParams is null");
logger.info("<index update process> Reloading global dataimport.properties");
reloadParams();
// single-core
} else if (singleCore) {
prepUrlSendHttpPost(reBuildIndexParams);
// multi-core
} else if (syncCores.length == 0
|| (syncCores.length == 1 && syncCores[0].isEmpty())) {
logger.warn("<index update process> No cores scheduled for data import");
logger.info("<index update process> Reloading global dataimport.properties");
reloadParams();
} else {
for (String core : syncCores) {
prepUrlSendHttpPost(core, reBuildIndexParams);
}
}
} catch (Exception e) {
logger.error("Failed to prepare for sendHttpPost", e);
reloadParams();
}
}
}
ApplicationListener
package org.apache.solr.handler.dataimport.scheduler;
import java.util.Calendar;
import java.util.Date;
import java.util.Timer;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ApplicationListener implements ServletContextListener {
private static final Logger logger = LoggerFactory
.getLogger(ApplicationListener.class);
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
ServletContext servletContext = servletContextEvent.getServletContext();
// get our timer from the context
Timer timer = (Timer) servletContext.getAttribute("timer");
Timer fullImportTimer = (Timer) servletContext
.getAttribute("fullImportTimer");
// cancel all active tasks in the timers queue
if (timer != null)
timer.cancel();
if (fullImportTimer != null)
fullImportTimer.cancel();
// remove the timer from the context
servletContext.removeAttribute("timer");
servletContext.removeAttribute("fullImportTimer");
}
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
ServletContext servletContext = servletContextEvent.getServletContext();
try {
// 增量更新任务计划
// create the timer and timer task objects
Timer timer = new Timer();
DeltaImportHTTPPostScheduler task = new DeltaImportHTTPPostScheduler(
servletContext.getServletContextName(), timer);
// get our interval from HTTPPostScheduler
int interval = task.getIntervalInt();
// get a calendar to set the start time (first run)
Calendar calendar = Calendar.getInstance();
// set the first run to now + interval (to avoid fireing while the
// app/server is starting)
calendar.add(Calendar.MINUTE, interval);
Date startTime = calendar.getTime();
// schedule the task
timer.scheduleAtFixedRate(task, startTime, 1000 * 60 * interval);
// save the timer in context
servletContext.setAttribute("timer", timer);
// 重做索引任务计划
Timer fullImportTimer = new Timer();
FullImportHTTPPostScheduler fullImportTask = new FullImportHTTPPostScheduler(
servletContext.getServletContextName(), fullImportTimer);
int reBuildIndexInterval = fullImportTask
.getReBuildIndexIntervalInt();
if (reBuildIndexInterval <= 0) {
logger.warn("Full Import Schedule disabled");
return;
}
Calendar fullImportCalendar = Calendar.getInstance();
Date beginDate = fullImportTask.getReBuildIndexBeginTime();
fullImportCalendar.setTime(beginDate);
fullImportCalendar.add(Calendar.MINUTE, reBuildIndexInterval);
Date fullImportStartTime = fullImportCalendar.getTime();
// schedule the task
fullImportTimer.scheduleAtFixedRate(fullImportTask,
fullImportStartTime, 1000 * 60 * reBuildIndexInterval);
// save the timer in context
servletContext.setAttribute("fullImportTimer", fullImportTimer);
} catch (Exception e) {
if (e.getMessage().endsWith("disabled")) {
logger.warn("Schedule disabled");
} else {
logger.error("Problem initializing the scheduled task: ", e);
}
}
}
}
打完jar表直接丢到D:\search\apache-tomcat-8.0.36\webapps\solr\WEB-INF\lib下
然后修改web.xml,增加
<listener>
<listener-class>org.apache.solr.handler.dataimport.scheduler.ApplicationListener</listener-class>
</listener>
在D:\search\solr\home下新建conf增加dataimport.properties文件
# to sync or not to sync
# 1 - active; anything else - inactive
syncEnabled=1
# which cores to schedule
# in a multi-core environment you can decide which cores you want syncronized
# leave empty or comment it out if using single-core deployment
syncCores=ask,topic
# solr server name or IP address
# [defaults to localhost if empty]
server=localhost
# solr server port
# [defaults to 80 if empty]
port=8080
# application name/context
# [defaults to current ServletContextListener's context (app) name]
webapp=solr
params=/dataimport?command=delta-import&clean=false&commit=true&wt=json
# number of minutes between two runs
# [defaults to 30 if empty]
interval=1
# 重做索引的时间间隔,单位分钟,默认7200,即5天;
# 为空,为0,或者注释掉:表示永不重做索引
reBuildIndexInterval=7200
reBuildIndexParams=/dataimport?command=full-import&clean=true&commit=true&wt=json
# 两种格式:2016-08-08 03:10:00 或者 03:10:00,后一种会自动补全日期部分为服务启动时的日期
reBuildIndexBeginTime=03:10:00
这里是多个cores分别为ask,topic
为了做测试interval设为每1分钟增量更新一次索引
接下来就可以启动tomcat 查看结果了