DolphinScheduler集群部署容器部署

动物管理员sun

已于 2023-09-10 16:40:35 修改

阅读量437

点赞数

分类专栏：大数据数仓文章标签：大数据

于 2023-08-18 20:04:00 首次发布

本文链接：https://blog.youkuaiyun.com/weixin_46676071/article/details/132368440

版权

大数据数仓专栏收录该内容

3 篇文章

订阅专栏

本文详细描述了如何在Linux环境下使用docker部署Dolphinscheduler，包括创建数据库、设置用户权限、拉取镜像、配置环境变量以及在主节点和worker节点上运行Docker容器的过程。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

主机
10.0.0.90
10.0.0.91
10.0.0.92

mysql

#创建数据库
CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

#创建用户付权限
ALTER USER 'dolphinscheduler'@'%' IDENTIFIED WITH mysql_native_password BY 'dolphinscheduler';
CREATE USER 'dolphinscheduler'@'%' IDENTIFIED BY 'dolphinscheduler';
GRANT ALL ON dolphinscheduler.* TO 'dolphinscheduler'@'%';
flush privileges;

zookeeper

同之前

dolphinscheduler

官网：https://dolphinscheduler.apache.org/zh-cn/docs/

#拉去镜像到主节点
docker pull apache/dolphinscheduler-master:3.1.7
docker pull apache/dolphinscheduler-api:3.1.7
docker pull apache/dolphinscheduler-alert-server:3.1.7
docker pull apache/dolphinscheduler-tools:3.1.7
#拉镜像到从节点
docker pull apache/dolphinscheduler-worker:3.1.7
#创建目录主节点
mkdir -p /data/dolphinscheduler/logs/{master,api,alert}
#全部节点下载mysql启动器
wget -o  /data/dolphinscheduler/  https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
#worker节点创建目录
mkdir -p /data/dolphinscheduler/logs/
#woker节点下载插件datax
官网：https://github.com/alibaba/DataX/blob/master/userGuid.md
wget https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/202308/datax.tar.gz

部署主节点

#下载mysql驱动包
wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar

#初始化数据    
docker run -d --name dolphinscheduler-tools \
    -e DATABASE="mysql" \
    -e SPRING_DATASOURCE_URL="jdbc:mysql://10.0.0.77:53306/dolphinscheduler" \
    -e SPRING_DATASOURCE_USERNAME="dolphinscheduler" \
    -e SPRING_DATASOURCE_PASSWORD="dolphinscheduler" \
    -v /data/dolphinscheduler/mysql-connector-java-8.0.16.jar:/opt/dolphinscheduler/tools/libs/mysql-connector-java-8.0.16.jar\
    --net host \
    apache/dolphinscheduler-tools:3.1.7  tools/bin/upgrade-schema.sh

#api
docker run -d --name dolphinscheduler-api \
    -e DATABASE="mysql" \
    -e SPRING_DATASOURCE_URL="jdbc:mysql://10.0.0.77:53306/dolphinscheduler" \
    -e SPRING_DATASOURCE_USERNAME="dolphinscheduler" \
    -e SPRING_DATASOURCE_PASSWORD="dolphinscheduler" \
    -e REGISTRY_ZOOKEEPER_CONNECT_STRING="192.168.1.1:2181,192.168.1.2:2181,192.168.1.3:2181"\
    -v /data/dolphinscheduler/mysql-connector-java-8.0.16.jar:/opt/dolphinscheduler/libs/mysql-connector-java-8.0.16.jar\
    -v /data/dolphinscheduler/logs/api/:/opt/dolphinscheduler/logs\
    --net host \
    apache/dolphinscheduler-api:3.1.7 

#alter
docker run -d --name dolphinscheduler-alert-server \
    -e DATABASE="mysql" \
    -e SPRING_DATASOURCE_URL="jdbc:mysql://10.0.0.77:53306/dolphinscheduler" \
    -e SPRING_DATASOURCE_USERNAME="dolphinscheduler" \
    -e SPRING_DATASOURCE_PASSWORD="dolphinscheduler" \
    -e REGISTRY_ZOOKEEPER_CONNECT_STRING="192.168.1.1:2181,192.168.1.2:2181,192.168.1.3:2181"\
    -v /data/dolphinscheduler/mysql-connector-java-8.0.16.jar:/opt/dolphinscheduler/libs/mysql-connector-java-8.0.16.jar\
    -v /data/dolphinscheduler/logs/alter/:/opt/dolphinscheduler/logs\
    --net host \
    apache/dolphinscheduler-alert-server:3.1.7 

# 主节点   
docker run -d --name dolphinscheduler-master \
    -e DATABASE="mysql" \
    -e SPRING_DATASOURCE_URL="jdbc:mysql://10.0.0.77:53306/dolphinscheduler" \
    -e SPRING_DATASOURCE_USERNAME="dolphinscheduler" \
    -e SPRING_DATASOURCE_PASSWORD="dolphinscheduler" \
    -e REGISTRY_ZOOKEEPER_CONNECT_STRING="192.168.1.1:2181,192.168.1.2:2181,192.168.1.3:2181"\
    -v /data/dolphinscheduler/mysql-connector-java-8.0.16.jar:/opt/dolphinscheduler/libs/mysql-connector-java-8.0.16.jar\
    -v /data/dolphinscheduler/logs/master/:/opt/dolphinscheduler/logs\
    --net host \
    apache/dolphinscheduler-master:3.1.7

部署worker节点

#启动
docker run -d --name dolphinscheduler-worker \
    -e DATABASE="mysql" \
    -e SPRING_DATASOURCE_URL="jdbc:mysql://10.0.0.77:53306/dolphinscheduler" \
    -e SPRING_DATASOURCE_USERNAME="dolphinscheduler" \
    -e SPRING_DATASOURCE_PASSWORD="dolphinscheduler" \
    -e REGISTRY_ZOOKEEPER_CONNECT_STRING="192.168.1.1:2181,192.168.1.2:2181,192.168.1.3:2181"\
    -v /data/dolphinscheduler/mysql-connector-java-8.0.16.jar:/opt/dolphinscheduler/libs/mysql-connector-java-8.0.16.jar\
    -v /data/dolphinscheduler/logs/:/opt/dolphinscheduler/logs\
    --net host \
    apache/dolphinscheduler-worker:3.1.7 
#部署插件datax
docker exec -it dolphinscheduler-worker bash
mkdir -p /opt/soft/
docker cp datax.tar.gz dolphinscheduler-worker:/opt/soft/
docker exec -it dolphinscheduler-worker bash
tar xf /opt/soft/datax.tar.gz
apt-get update
apt-get install python2 -y
mkdir -p /opt/soft/python/bin/
cp /usr/bin/python2 /opt/soft/python/bin/python2.7
#重启容器
docker restart dolphinscheduler-worker

编写worker的dockerfile含有datax

cat Dockerfile
FROM centos:7
# 维护者信息JDK8
# 将 jdk8添加到镜像centos的/usr/local/目录下，ADD 命令会自动解压 将tar包解压到这个路
ENV LANG en_US.utf8
ENV LC_ALL en_US.utf8
ENV TZ Asia/Shanghai
######jdk######
ADD ./softwares/jdk-8u381-linux-x64.tar.gz /opt/java/
ADD ./datax.tar.gz /opt/soft/
######jdk######
RUN  ln -sv /opt/java/jdk1.8.0_381  /opt/java/openjdk
######python##
RUN yum -y  update\
    &&  yum install -y python2\
    &&  mkdir -p /opt/soft/python/bin/\
    &&  cp /usr/bin/python2 /opt/soft/python/bin/python2.7
#####datax###
#####api#####
RUN  mkdir -p /opt/dolphinscheduler\
     && yum install -y sudo\
     && chmod 755 /etc/sudoers\
     && echo "%admin ALL=(ALL) ALL" >> /etc/sudoers\
     && echo "%sudo	ALL=(ALL:ALL) ALL" >> /etc/sudoers\
     && chmod 0440  /etc/sudoers\
     && visudo -c
COPY ./apache-dolphinscheduler-3.1.7-bin/worker-server  /opt/dolphinscheduler
COPY ./mysql-connector-java-8.0.16.jar  /opt/dolphinscheduler/libs
WORKDIR /opt/dolphinscheduler
EXPOSE 12345
CMD [ "/bin/bash", "./bin/start.sh" ]


####下载好datax，java，mysql启动jar包,二进制部署的时候的worker的路径
##在工作流使用时回在worker节点创建用户，所以一定要安装sudo命令并给用户提权

登陆

不管你是用那种方式启动的服务，只要服务启动后，你都可以通过 http://localhost:12345/dolphinscheduler/ui 访问 DolphinScheduler。访问上述链接后会跳转到登陆页面，DolphinScheduler 默认的用户和密码分别为 admin 和 dolphinscheduler123。