azkaban整合mysql和邮件

Infernal Affairs

于 2022-08-15 07:53:19 发布

阅读量378

点赞数 1

文章标签： mysql 数据库 java

本文链接：https://blog.youkuaiyun.com/Smple_BOY/article/details/126339555

版权

yum -y install git
yum -y install wget
wget https://github.com/azkaban/azkaban/archive/master.zip
yum -y install zip unzip
unzip azkaban-master.zip
cd azkaban-master ./gradlew build installDist

=====================================================

Getting started with the Multi Executor Server
修改mysql配置
cd /usr/share/mysql
cp my-default.cnf /etc/my.cnf

vi /etc/my.cnf
[mysqld]
...
max_allowed_packet=1024M

systemctl restart mysql

cd /root/azkaban-master/azkaban-db/build/distributions
tar -zxvf azkaban-db-0.1.0-SNAPSHOT.tar.gz
cd azkaban-db-0.1.0-SNAPSHOT

mysql -uroot -p123456
create database azkaban;
use azkaban;
source source create-all-sql-0.1.0-SNAPSHOT.sql;

-----------------------------------------------------

启动executor
cd /root/azkaban-master/azkaban-exec-server
cd conf

vi azkaban.properties
default.timezone.id=Asia/Shanghai
# Mysql Configs
mysql.user=<username>
mysql.password=<password>

bin/start-exec.sh
激活executor
cd azkaban-exec-server/build/install/azkaban-exec-server
curl -G "hadoop3:33442/executor?action=activate" && echo

----------------------------------------------------------

cd /root/azkaban-master/azkaban-web-server/build/install/azkaban-web-server
cd conf

vi azkaban.properties
default.timezone.id=Asia/Shanghai
# Mysql Configs
mysql.user=<username>
mysql.password=<password>

#将azkaban.executorselector.filters=StaticRemainingFlowSize,MinimumFreeMemory,CpuStatus中的MinimumFreeMemory去掉
azkaban.executorselector.filters=StaticRemainingFlowSize,CpuStatus

bin/start
bin/start-web.sh

==================================================================

给web模块添加hdfs浏览插件
下载https://github.com/azkaban/azkaban-plugins/archive/3.0.0.zip
或者下载国内镜像https://gitee.com/jiuweikandianying/azkaban-plugins.git

在linux上安装ant
解压，配置环境变量

cd azkaban-plugins/plugins/hdfsviewer
输入ant编译
cd azkaban-plugins/plugins/hadoopsecuritymanager-yarn
输入ant编译

在azkaban-web-server目录下建立plugins/viewer文件夹
cd azkaban-plugins/plugins
cp -r hdfsviewer/ azkaban-web-server/plugins/viewer
cd azkaban-web-server/plugins/viewer
mv hdfsviewer hdfs

cd azkaban-plugins/dist/hdfsviewer/jars
cp azkaban-hdfs-viewer-3.0.0.jar azkaban-web-server/plugins/viewer/hdfs/lib

cd azkaban-web-server/plugins/viewer/hdfs/conf
vi plugin.properties
viewer.name=HDFS
viewer.path=hdfs
viewer.order=1
viewer.hidden=false
viewer.external.classpaths=extlib/
viewer.servlet.class=azkaban.viewer.hdfs.HdfsBrowserServlet
hadoop.security.manager.class=azkaban.security.HadoopSecurityManager_H_2_0
azkaban.should.proxy=false
proxy.user=root
proxy.keytab.location=
allow.group.proxy=false
file.max.lines=1000

cd azkaban-web-server
mkdir extlib
向extlib目录中添加
commons-cli-1.2.jar
commons-configuration-1.6.jar
hadoop-auth-2.7.3.jar
hadoop-common-2.7.3.jar
hadoop-hdfs-2.7.3.jar
hadoop-mapreduce-client-common-2.7.3.jar
hadoop-mapreduce-client-core-2.7.3.jar
hadoop-yarn-api-2.7.3.jar
hadoop-yarn-common-2.7.3.jar
htrace-core-3.1.0-incubating.jar
protobuf-java-2.5.0.jar

cd azkaban-plugins/dist/hadoopsecuritymanager-yarn/jars
cp azkaban-hadoopsecuritymanageryarn-3.0.0.jar azkaban-web-server/extlib

cd azkaban-web-server/conf
vi azkaban.properties
azkaban.hdfsviewer.plugin.dir=plugins/viewer/hdfs/

重启azkaban-web-server

==============================================================

给azkaban添加hadoop和hive的jobTypes
cd /root/azkaban-plugins/plugins/jobtype/jobtypes

vi common.properties
## everything that the user job can know

hadoop.home=/usr/local/hadoop-2.7.3
hive.home=/usr/local/apache-hive-2.3.6-bin
#pig.home=
#spark.home=

#azkaban.should.proxy=
jobtype.global.classpath=${hadoop.home}/etc/hadoop,${hadoop.home}/share/hadoop/common/*,${hadoop.home}/share/hadoop/common/lib/*,${hadoop.home}/share/hadoop/hdfs/*,${hadoop.home}/share/hadoop/hdfs/lib/*,${hadoop.home}/share/hadoop/yarn/*,${hadoop.home}/share/hadoop/yarn/lib/*,${hadoop.home}/share/hadoop/mapreduce/*,${hadoop.home}/share/hadoop/mapreduce/lib/*

------------------

vi commonprivate.properties
## hadoop security manager setting common to all hadoop jobs
hadoop.security.manager.class=azkaban.security.HadoopSecurityManager_H_2_0

## hadoop security related settings

# proxy.keytab.location=
# proxy.user=

azkaban.should.proxy=false
obtain.binary.token=false
# obtain.namenode.token=true
# obtain.jobtracker.token=true

# global classpath items for all jobs. e.g. hadoop-core jar, hadoop conf
#jobtype.global.classpath=${hadoop.home}/*,${hadoop.home}/etc/hadoop

# global jvm args for all jobs. e.g. java.io.temp.dir, java.library.path
#jobtype.global.jvm.args=

# hadoop
hadoop.home=/usr/local/hadoop-2.7.3
#pig.home=
hive.home=/usr/local/apache-hive-2.3.6-bin
#spark.home=

# configs for jobtype security settings
execute.as.user=false
azkaban.native.lib=false

---------------------
cd azkaban-exec-server/plugins
rm -rf jobtypes
cd /root/azkaban-plugins/plugins/jobtype
cp -r jobtypes azkaban-exec-server/plugins

删除jobtypes中的pig，spark等不需要的plugin

==================================================================

编写上传txt到hdfs，然后maprduce执行wordcount的流程

vi createdir.job
#createdir.job
type=command
command=/usr/local/hadoop-2.7.3/bin/hadoop fs -mkdir /input

vi upload.job
#upload.job
type=command
dependencies=createdir
command=/usr/local/hadoop-2.7.3/bin/hadoop fs -put /root/wc.txt /input

vi wordcount.job
#wordcount.job
type=command
dependencies=upload
command=/usr/local/hadoop-2.7.3/bin/hadoop jar /usr/local/hadoop-2.7.3/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar wordcount /input /output

zip finish.zip ./*

=============================================================

执行java程序

写java程序
package com.bawei.demo;

import java.io.FileOutputStream;
import java.io.IOException;

public class AzkabanTest {
    public static void main(String[] args) throws IOException {
        FileOutputStream fos = new FileOutputStream("/root/output.txt");
        fos.write("this is a java progress".getBytes());
        fos.close();
    }
}
然后打成jar包

写flow文件
vi azkabanjava.job
#azkabanjava.job
type=javaprocess
java.class=com.bawei.demo.AzkabanTest
classpath=/root/TestAZ-1.0-SNAPSHOT.jar

=========================================================

azkaban整合发邮件
/usr/local/src/azkaban-3.90.0/azkaban-common/src/main/java/azkaban/utils/

vi EmailMessage.java

import java.security.Security;
##############2##################
#在sendEmail()方法里props的配置代码下加入以下几行代码:
Security.addProvider(new com.sun.net.ssl.internal.ssl.Provider());
final String SSL_FACTORY = "javax.net.ssl.SSLSocketFactory";
props.setProperty("mail.smtp.socketFactory.class", SSL_FACTORY);
props.setProperty("mail.smtp.socketFactory.fallback", "false");
props.setProperty("mail.smtp.socketFactory.port", "465");

public void sendEmail() throws MessagingException {
    checkSettings();
    final Properties props = new Properties();
   //在这里添加
    Security.addProvider(new com.sun.net.ssl.internal.ssl.Provider());
    final String SSL_FACTORY = "javax.net.ssl.SSLSocketFactory";
    props.setProperty("mail.smtp.socketFactory.class", SSL_FACTORY);
    props.setProperty("mail.smtp.socketFactory.fallback", "false");
    props.setProperty("mail.smtp.socketFactory.port", "465");
    if (this._usesAuth) {
      props.put("mail.smtp.auth", "true");
      props.put("mail.user", this._mailUser);
      props.put("mail.password", this._mailPassword);
    } else {
      props.put("mail.smtp.auth", "false");
    }
    props.put("mail.smtp.host", this._mailHost);
    props.put("mail.smtp.port", this._mailPort);
    props.put("mail.smtp.timeout", _mailTimeout);
    props.put("mail.smtp.connectiontimeout", _connectionTimeout);
    props.put("mail.smtp.starttls.enable", this._tls);
    props.put("mail.smtp.ssl.trust", this._mailHost);

vi JavaxMailSender.java

import javax.mail.Transport;

private final Transport t;

public JavaxMailSender(final Properties props)
      throws NoSuchProviderException {
    this.session = Session.getInstance(props, null);
    this.t = this.session.getTransport(props.getProperty("mail.transport.protocol",PROTOCOL));
}

vi EmailMessageCreator.java

@Inject
public EmailMessageCreator(final Props props) {
    this.mailHost = props.getString("mail.host", "localhost");
    this.mailPort = props.getInt("mail.port", DEFAULT_SMTP_PORT);
    this.mailUser = props.getString("mail.user", "");
    this.mailPassword = props.getString("mail.password", "");
    this.protocol = props.getString("mail.transport.protocol", "smtp"); //添加这一行
    this.mailSender = props.getString("mail.sender", "");
    this.tls = props.getString("mail.tls", "false");
    this.usesAuth = props.getBoolean("mail.useAuth", true);
}

重新编译

在azkaban.properties中配置

mail.transport.protocol=smtps
mail.sender=xxxx@qq.com
mail.host=smtp.qq.com
mail.port=465
mail.user=xxxx@qq.com
mail.password=xxxx
mail.useAuth=true