solr6.0 -MySQL定时增量、全量更新索引

上一篇里面有增量更新索引
这里写图片描述

执行完就可以看到D:\search\solr\home\ask\conf下自动生成了一个dataimport.properties文件,里面是最后更新索引时间,就是根据这个时间来达到增量更新的
定时更新索引,还是借鉴以前有位哥们的
http://www.cnblogs.com/ezhangliang/archive/2012/04/11/2441945.html
solr6.0的只是引入资源文件的地方改一下,很多地方源码都不全,这里给出全部的
这里写图片描述

SolrDataImportProperties

package org.apache.solr.handler.dataimport.scheduler;

import org.apache.solr.core.SolrResourceLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Properties;

public class SolrDataImportProperties {
    private Properties properties;
    public static final String SYNC_ENABLED = "syncEnabled";
    public static final String SYNC_CORES = "syncCores";
    public static final String SERVER = "server";
    public static final String PORT = "port";
    public static final String WEBAPP = "webapp";
    public static final String PARAMS = "params";
    public static final String INTERVAL = "interval";
    public static final String REBUILDINDEXPARAMS = "reBuildIndexParams";
    public static final String REBUILDINDEXBEGINTIME = "reBuildIndexBeginTime";
    public static final String REBUILDINDEXINTERVAL = "reBuildIndexInterval";
    private static final Logger logger = LoggerFactory.getLogger(SolrDataImportProperties.class);

    public void loadProperties(boolean force) {
        try {
            SolrResourceLoader loader = new SolrResourceLoader();
            logger.info("Instance dir = " + loader.getInstancePath());
            String configDir = loader.getConfigDir();
            configDir = SolrResourceLoader.normalizeDir(configDir);
            if ((force) || (this.properties == null)) {
                this.properties = new Properties();
                String dataImportPropertiesPath = configDir + "dataimport.properties";
                FileInputStream fis = new FileInputStream(dataImportPropertiesPath);
                this.properties.load(fis);
            }
        } catch (FileNotFoundException fnfe) {
            logger.error(
                    "Error locating DataImportScheduler dataimport.properties file",
                    fnfe);
        } catch (IOException ioe) {
            logger.error(
                    "Error reading DataImportScheduler dataimport.properties file",
                    ioe);
        } catch (Exception e) {
            logger.error("Error loading DataImportScheduler properties", e);
        }
    }

    public String getProperty(String key) {
        return this.properties.getProperty(key);
    }
}

BaseTimerTask

package org.apache.solr.handler.dataimport.scheduler;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Timer;
import java.util.TimerTask;

public abstract class BaseTimerTask extends TimerTask {
    protected String syncEnabled;
    protected String[] syncCores;
    protected String server;
    protected String port;
    protected String webapp;
    protected String params;
    protected String interval;
    protected String cores;
    protected SolrDataImportProperties p;
    protected boolean singleCore;

    protected String reBuildIndexParams;
    protected String reBuildIndexBeginTime;
    protected String reBuildIndexInterval;

    protected static final Logger logger = LoggerFactory
            .getLogger(BaseTimerTask.class);

    public BaseTimerTask(String webAppName, Timer t) throws Exception {
        // load properties from global dataimport.properties
        p = new SolrDataImportProperties();
        reloadParams();
        fixParams(webAppName);

        if (!syncEnabled.equals("1"))
            throw new Exception("Schedule disabled");

        if (syncCores == null
                || (syncCores.length == 1 && syncCores[0].isEmpty())) {
            singleCore = true;
            logger.info("<index update process> Single core identified in dataimport.properties");
        } else {
            singleCore = false;
            logger.info("<index update process> Multiple cores identified in dataimport.properties. Sync active for: "
                    + cores);
        }
    }

    protected void reloadParams() {
        p.loadProperties(true);
        syncEnabled = p.getProperty(SolrDataImportProperties.SYNC_ENABLED);
        cores = p.getProperty(SolrDataImportProperties.SYNC_CORES);
        server = p.getProperty(SolrDataImportProperties.SERVER);
        port = p.getProperty(SolrDataImportProperties.PORT);
        webapp = p.getProperty(SolrDataImportProperties.WEBAPP);
        params = p.getProperty(SolrDataImportProperties.PARAMS);
        interval = p.getProperty(SolrDataImportProperties.INTERVAL);
        syncCores = cores != null ? cores.split(",") : null;

        reBuildIndexParams = p
                .getProperty(SolrDataImportProperties.REBUILDINDEXPARAMS);
        reBuildIndexBeginTime = p
                .getProperty(SolrDataImportProperties.REBUILDINDEXBEGINTIME);
        reBuildIndexInterval = p
                .getProperty(SolrDataImportProperties.REBUILDINDEXINTERVAL);

    }

    protected void fixParams(String webAppName) {
        if (server == null || server.isEmpty())
            server = "localhost";
        if (port == null || port.isEmpty())
            port = "8080";
        if (webapp == null || webapp.isEmpty())
            webapp = webAppName;
        if (interval == null || interval.isEmpty() || getIntervalInt() <= 0)
            interval = "30";
        if (reBuildIndexBeginTime == null || reBuildIndexBeginTime.isEmpty())
            interval = "00:00:00";
        if (reBuildIndexInterval == null || reBuildIndexInterval.isEmpty()
                || getReBuildIndexIntervalInt() <= 0)
            reBuildIndexInterval = "0";
    }

    protected void prepUrlSendHttpPost(String params) {
        String coreUrl = "http://" + server + ":" + port + "/" + webapp
                + params;
        sendHttpPost(coreUrl, null);
    }

    protected void prepUrlSendHttpPost(String coreName, String params) {
        String coreUrl = "http://" + server + ":" + port + "/" + webapp + "/"
                + coreName + params;
        sendHttpPost(coreUrl, coreName);
    }

    protected void sendHttpPost(String completeUrl, String coreName) {
        DateFormat df = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss SSS");
        Date startTime = new Date();

        // prepare the core var
        String core = coreName == null ? "" : "[" + coreName + "] ";

        logger.info(core
                + "<index update process> Process started at .............. "
                + df.format(startTime));

        try {

            URL url = new URL(completeUrl);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();

            conn.setRequestMethod("POST");
            conn.setRequestProperty("type", "submit");
            conn.setDoOutput(true);

            // Send HTTP POST
            conn.connect();

            logger.info(core + "<index update process> Full URL\t\t\t\t"
                    + conn.getURL());
            logger.info(core + "<index update process> Response message\t\t\t"
                    + conn.getResponseMessage());
            logger.info(core + "<index update process> Response code\t\t\t"
                    + conn.getResponseCode());

            // listen for change in properties file if an error occurs
            if (conn.getResponseCode() != 200) {
                reloadParams();
            }

            conn.disconnect();
            logger.info(core
                    + "<index update process> Disconnected from server\t\t"
                    + server);
            Date endTime = new Date();
            logger.info(core
                    + "<index update process> Process ended at ................ "
                    + df.format(endTime));
        } catch (MalformedURLException mue) {
            logger.error("Failed to assemble URL for HTTP POST", mue);
        } catch (IOException ioe) {
            logger.error(
                    "Failed to connect to the specified URL while trying to send HTTP POST",
                    ioe);
        } catch (Exception e) {
            logger.error("Failed to send HTTP POST", e);
        }
    }

    public int getIntervalInt() {
        try {
            return Integer.parseInt(interval);
        } catch (NumberFormatException e) {
            logger.warn(
                    "Unable to convert 'interval' to number. Using default value (30) instead",
                    e);
            return 30; // return default in case of error
        }
    }

    public int getReBuildIndexIntervalInt() {
        try {
            return Integer.parseInt(reBuildIndexInterval);
        } catch (NumberFormatException e) {
            logger.info(
                    "Unable to convert 'reBuildIndexInterval' to number. do't rebuild index.",
                    e);
            return 0; // return default in case of error
        }
    }

    public Date getReBuildIndexBeginTime() {
        Date beginDate = null;
        try {
            SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd");
            String dateStr = sdfDate.format(new Date());
            beginDate = sdfDate.parse(dateStr);
            if (reBuildIndexBeginTime == null
                    || reBuildIndexBeginTime.isEmpty()) {
                return beginDate;
            }
            if (reBuildIndexBeginTime.matches("\\d{2}:\\d{2}:\\d{2}")) {
                SimpleDateFormat sdf = new SimpleDateFormat(
                        "yyyy-MM-dd HH:mm:ss");
                beginDate = sdf.parse(dateStr + " " + reBuildIndexBeginTime);
            } else if (reBuildIndexBeginTime
                    .matches("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}")) {
                SimpleDateFormat sdf = new SimpleDateFormat(
                        "yyyy-MM-dd HH:mm:ss");
                beginDate = sdf.parse(reBuildIndexBeginTime);
            }
            return beginDate;
        } catch (ParseException e) {
            logger.warn(
                    "Unable to convert 'reBuildIndexBeginTime' to date. use now time.",
                    e);
            return beginDate;
        }
    }

}

DeltaImportHTTPPostScheduler

package org.apache.solr.handler.dataimport.scheduler;

import java.util.Timer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class DeltaImportHTTPPostScheduler extends BaseTimerTask {

    private static final Logger logger = LoggerFactory
            .getLogger(DeltaImportHTTPPostScheduler.class);

    public DeltaImportHTTPPostScheduler(String webAppName, Timer t)
            throws Exception {
        super(webAppName, t);
        logger.info("<index update process> DeltaImportHTTPPostScheduler init");
    }

    public void run() {
        try {
            // check mandatory params
            if (server.isEmpty() || webapp.isEmpty() || params == null
                    || params.isEmpty()) {
                logger.warn("<index update process> Insuficient info provided for data import");
                logger.info("<index update process> Reloading global dataimport.properties");
                reloadParams();
                // single-core
            } else if (singleCore) {
                prepUrlSendHttpPost(params);

                // multi-core
            } else if (syncCores.length == 0
                    || (syncCores.length == 1 && syncCores[0].isEmpty())) {
                logger.warn("<index update process> No cores scheduled for data import");
                logger.info("<index update process> Reloading global dataimport.properties");
                reloadParams();

            } else {
                for (String core : syncCores) {
                    prepUrlSendHttpPost(core, params);
                }
            }
        } catch (Exception e) {
            logger.error("Failed to prepare for sendHttpPost", e);
            reloadParams();
        }
    }
}

FullImportHTTPPostScheduler

package org.apache.solr.handler.dataimport.scheduler;

import java.util.Timer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class FullImportHTTPPostScheduler extends BaseTimerTask {

    private static final Logger logger = LoggerFactory
            .getLogger(FullImportHTTPPostScheduler.class);

    public FullImportHTTPPostScheduler(String webAppName, Timer t)
            throws Exception {
        super(webAppName, t);
        logger.info("<index update process> DeltaImportHTTPPostScheduler init");
    }

    public void run() {
        try {
            // check mandatory params
            if (server.isEmpty() || webapp.isEmpty()
                    || reBuildIndexParams == null
                    || reBuildIndexParams.isEmpty()) {
                logger.warn("<index update process> Insuficient info provided for data import, reBuildIndexParams is null");
                logger.info("<index update process> Reloading global dataimport.properties");
                reloadParams();
                // single-core
            } else if (singleCore) {
                prepUrlSendHttpPost(reBuildIndexParams);

                // multi-core
            } else if (syncCores.length == 0
                    || (syncCores.length == 1 && syncCores[0].isEmpty())) {
                logger.warn("<index update process> No cores scheduled for data import");
                logger.info("<index update process> Reloading global dataimport.properties");
                reloadParams();

            } else {
                for (String core : syncCores) {
                    prepUrlSendHttpPost(core, reBuildIndexParams);
                }
            }
        } catch (Exception e) {
            logger.error("Failed to prepare for sendHttpPost", e);
            reloadParams();
        }
    }
}

ApplicationListener

package org.apache.solr.handler.dataimport.scheduler;

import java.util.Calendar;
import java.util.Date;
import java.util.Timer;

import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class ApplicationListener implements ServletContextListener {

    private static final Logger logger = LoggerFactory
            .getLogger(ApplicationListener.class);

    @Override
    public void contextDestroyed(ServletContextEvent servletContextEvent) {
        ServletContext servletContext = servletContextEvent.getServletContext();

        // get our timer from the context
        Timer timer = (Timer) servletContext.getAttribute("timer");
        Timer fullImportTimer = (Timer) servletContext
                .getAttribute("fullImportTimer");

        // cancel all active tasks in the timers queue
        if (timer != null)
            timer.cancel();
        if (fullImportTimer != null)
            fullImportTimer.cancel();

        // remove the timer from the context
        servletContext.removeAttribute("timer");
        servletContext.removeAttribute("fullImportTimer");

    }

    @Override
    public void contextInitialized(ServletContextEvent servletContextEvent) {
        ServletContext servletContext = servletContextEvent.getServletContext();
        try {
            // 增量更新任务计划
            // create the timer and timer task objects
            Timer timer = new Timer();
            DeltaImportHTTPPostScheduler task = new DeltaImportHTTPPostScheduler(
                    servletContext.getServletContextName(), timer);

            // get our interval from HTTPPostScheduler
            int interval = task.getIntervalInt();

            // get a calendar to set the start time (first run)
            Calendar calendar = Calendar.getInstance();

            // set the first run to now + interval (to avoid fireing while the
            // app/server is starting)
            calendar.add(Calendar.MINUTE, interval);
            Date startTime = calendar.getTime();

            // schedule the task
            timer.scheduleAtFixedRate(task, startTime, 1000 * 60 * interval);

            // save the timer in context
            servletContext.setAttribute("timer", timer);

            // 重做索引任务计划
            Timer fullImportTimer = new Timer();
            FullImportHTTPPostScheduler fullImportTask = new FullImportHTTPPostScheduler(
                    servletContext.getServletContextName(), fullImportTimer);

            int reBuildIndexInterval = fullImportTask
                    .getReBuildIndexIntervalInt();
            if (reBuildIndexInterval <= 0) {
                logger.warn("Full Import Schedule disabled");
                return;
            }

            Calendar fullImportCalendar = Calendar.getInstance();
            Date beginDate = fullImportTask.getReBuildIndexBeginTime();
            fullImportCalendar.setTime(beginDate);
            fullImportCalendar.add(Calendar.MINUTE, reBuildIndexInterval);
            Date fullImportStartTime = fullImportCalendar.getTime();

            // schedule the task
            fullImportTimer.scheduleAtFixedRate(fullImportTask,
                    fullImportStartTime, 1000 * 60 * reBuildIndexInterval);

            // save the timer in context
            servletContext.setAttribute("fullImportTimer", fullImportTimer);

        } catch (Exception e) {
            if (e.getMessage().endsWith("disabled")) {
                logger.warn("Schedule disabled");
            } else {
                logger.error("Problem initializing the scheduled task: ", e);
            }
        }

    }

}

打完jar表直接丢到D:\search\apache-tomcat-8.0.36\webapps\solr\WEB-INF\lib下
然后修改web.xml,增加

<listener> 
       <listener-class>org.apache.solr.handler.dataimport.scheduler.ApplicationListener</listener-class>
</listener>

在D:\search\solr\home下新建conf增加dataimport.properties文件

# to sync or not to sync
# 1 - active; anything else - inactive
syncEnabled=1
# which cores to schedule
# in a multi-core environment you can decide which cores you want syncronized
# leave empty or comment it out if using single-core deployment
syncCores=ask,topic
# solr server name or IP address
# [defaults to localhost if empty]
server=localhost
# solr server port
# [defaults to 80 if empty]
port=8080
# application name/context
# [defaults to current ServletContextListener's context (app) name]
webapp=solr
params=/dataimport?command=delta-import&clean=false&commit=true&wt=json
# number of minutes between two runs
# [defaults to 30 if empty]
interval=1
# 重做索引的时间间隔,单位分钟,默认7200,即5天;
# 为空,为0,或者注释掉:表示永不重做索引
reBuildIndexInterval=7200
reBuildIndexParams=/dataimport?command=full-import&clean=true&commit=true&wt=json
# 两种格式:2016-08-08 03:10:00 或者 03:10:00,后一种会自动补全日期部分为服务启动时的日期
reBuildIndexBeginTime=03:10:00

这里是多个cores分别为ask,topic
为了做测试interval设为每1分钟增量更新一次索引

接下来就可以启动tomcat 查看结果了

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值