Azkaban源码阅读之AzkabanApplication-优快云博客

本文详细解析了Azkaban批处理调度系统的初始化流程，包括关键组件的配置、工作流缓存的设置以及邮件通知等功能。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

Azkaban是一个Hadoop批处理调度器，可以通过访问其前台展示页面来进行部署、运行job等操作。

[java] view plain copy

//IndexServlet.java

   @Override
    protected void doPost(HttpServletRequest req, HttpServletResponse resp)
            throws ServletException, IOException {

        /* set runtime properties from request and response */
        super.setRuntimeProperties(req, resp);

        AzkabanApplication app = getApplication();
        String action = getParam(req, "action");
        if ("loadjobs".equals(action)) {
            resp.setContentType("application/json");
            String folder = getParam(req, "folder");
            resp.getWriter().print(getJSONJobsForFolder(app.getAllFlows(), folder));
            resp.getWriter().flush();
            return;
        }
        else if("unschedule".equals(action)) {
            String jobid = getParam(req, "job");
            app.getScheduleManager().removeScheduledJob(jobid);
        } else if("cancel".equals(action)) {
            cancelJob(app, req);
        } else if("schedule".equals(action)) {
            String redirect = scheduleJobs(app, req, resp);
            if (!redirect.isEmpty()) {
                resp.sendRedirect(redirect);
                return;
            }
        } else {
            throw new ServletException("Unknown action: " + action);
        }
        resp.sendRedirect(req.getContextPath());
    }

看文件名就可以看出了这个是整个系统运行的源头，一切从这里开始. . . . . . .

-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

AzkabanApplication中是一系列用于整个系统初始化的代码，接下来就来看一下吧~

-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

[java] view plain copy

public AzkabanApplication(List<File> jobDirs, File logDir, File tempDir, boolean enableDevMode) throws IOException {
    this._jobDirs = Utils.nonNull(jobDirs);
    this._logsDir = Utils.nonNull(logDir);
    this._tempDir = Utils.nonNull(tempDir);

    if(!this._logsDir.exists())
        this._logsDir.mkdirs();

    if(!this._tempDir.exists())
        this._tempDir.mkdirs();

    for(File jobDir: _jobDirs) {
        if(!jobDir.exists()) {
            logger.warn("Job directory " + jobDir + " does not exist. Creating.");
            jobDir.mkdirs();
        }
    }

    if(jobDirs.size() < 1)
        throw new IllegalArgumentException("No job directory given.");
    /**
     * 加载默认的属性文件
     * 实际上只用.properties为后缀的文件
     */
    Props defaultProps = PropsUtils.loadPropsInDirs(_jobDirs, ".properties", ".schema");

    /**
     * 主要功能就是加载HADOOP_HOME
     */
    _baseClassLoader = getBaseClassloader();

    String defaultTimezoneID = defaultProps.getString(DEFAULT_TIMEZONE_ID, null);
    if (defaultTimezoneID != null) {
        DateTimeZone.setDefault(DateTimeZone.forID(defaultTimezoneID));
        TimeZone.setDefault(TimeZone.getTimeZone(defaultTimezoneID));
    }

    /**
     * 由于Azkaban支持锁shezhi,在这里进行锁管理的初始化
     */
    NamedPermitManager permitManager = getNamedPermitManager(defaultProps);
    /**
     * 初始化Azkaban支持的job类型
     */
    JobWrappingFactory factory = new JobWrappingFactory(
            permitManager,
            new ReadWriteLockManager(),
            _logsDir.getAbsolutePath(),
            "java",
            new ImmutableMap.Builder<String, Class<? extends Job>>()
             .put("java", JavaJob.class)
             .put("command", ProcessJob.class)
             .put("javaprocess", JavaProcessJob.class)
             .put("pig", PigProcessJob.class)
             .put("propertyPusher", NoopJob.class)
             .put("python", PythonJob.class)
             .put("ruby", RubyJob.class)
             .put("script", ScriptJob.class).build());

    _hdfsUrl = defaultProps.getString("hdfs.instance.url", null);
    _jobManager = new JobManager(factory,
                                 _logsDir.getAbsolutePath(),
                                 defaultProps,
                                 _jobDirs,
                                 _baseClassLoader);

    _mailer = new Mailman(defaultProps.getString("mail.host", "localhost"),
                          defaultProps.getString("mail.user", ""),
                          defaultProps.getString("mail.password", ""));

    String failureEmail = defaultProps.getString("job.failure.email", null);
    String successEmail = defaultProps.getString("job.success.email", null);
    int schedulerThreads = defaultProps.getInt("scheduler.threads", 50);
    _instanceName = defaultProps.getString(INSTANCE_NAME, "");

    final File initialJobDir = _jobDirs.get(0);
    File schedule = getScheduleFile(defaultProps, initialJobDir);
    File backup = getBackupFile(defaultProps, initialJobDir);
    File executionsStorageDir = new File(
            defaultProps.getString("azkaban.executions.storage.dir", initialJobDir.getAbsolutePath() + "/executions")
    );
    if (! executionsStorageDir.exists()) executionsStorageDir.mkdirs();
    /**
     * 获得上次启动Azkaban时执行的最大jobId
     */
    long lastExecutionId = getLastExecutionId(executionsStorageDir);
    logger.info(String.format("Using path[%s] for storing executions.", executionsStorageDir));
    logger.info(String.format("Last known execution id was [%s]", lastExecutionId));

    final ExecutableFlowSerializer flowSerializer = new DefaultExecutableFlowSerializer();
    final ExecutableFlowDeserializer flowDeserializer = new DefaultExecutableFlowDeserializer(_jobManager, factory);

    FlowExecutionSerializer flowExecutionSerializer = new FlowExecutionSerializer(flowSerializer);
    FlowExecutionDeserializer flowExecutionDeserializer = new FlowExecutionDeserializer(flowDeserializer);

    /**
     * 初始化管理工作流的缓存,默认将cache大小设置为1000
     * <span style="color:#ff0000;">这里是整个初始化过程中的重点!!!</span>
     */
    _allFlows = new CachingFlowManager(
            new RefreshableFlowManager(
                    _jobManager,
                    flowExecutionSerializer,
                    flowExecutionDeserializer,
                    executionsStorageDir,
                    lastExecutionId
            ),
            defaultProps.getInt("azkaban.flow.cache.size", 1000)
    );
    _jobManager.setFlowManager(_allFlows);

    _jobExecutorManager = new JobExecutorManager(
                _allFlows,
                _jobManager,
                _mailer,
                failureEmail,
                successEmail,
                schedulerThreads
            );

    this._schedulerManager = new ScheduleManager(_jobExecutorManager, new LocalFileScheduleLoader(schedule, backup));

    /* set predefined log url prefix
    */
    String server_url = defaultProps.getString("server.url", null) ;
    if (server_url != null) {
        if (server_url.endsWith("/"))
            _jobExecutorManager.setRuntimeProperty(AppCommon.DEFAULT_LOG_URL_PREFIX, server_url + "logs?file=" );
        else
            _jobExecutorManager.setRuntimeProperty(AppCommon.DEFAULT_LOG_URL_PREFIX, server_url + "/logs?file=" );
    }

    this._velocityEngine = configureVelocityEngine(enableDevMode);
}

这部分将系统所需的各个部分，各个属性都进行了初始化的操作。