银河麒麟—利用docker搭建hadoop集群（完全分布式）

最新推荐文章于 2025-10-28 10:02:02 发布

原创

最新推荐文章于 2025-10-28 10:02:02 发布 · 994 阅读

4 ·

CC 4.0 BY-SA版权

文章标签：

#hadoop #docker

这篇博客详细介绍了如何利用docker构建一个Hadoop的完全分布式集群。首先，需要制作符合要求的镜像，然后进行Hadoop集群的搭建。集群规划包括确定namenode、datanode、secnamenode、resourcemanager和nodemanager的角色分配。通过这个过程，成功完成了docker环境中的Hadoop集群部署。

一、制作符合要求的镜像

$ docker pull ubuntu:16.04
$ docker run -it ubuntu:16.04

### 1.在容器内安装jdk和hadoop
$ ls
hadoop-2.7.7.tar.gz  jdk-8u241-linux-x64.tar.gz
$ docker exec -it 05272c80eb5b /bin/bash
root@05272c80eb5b:/opt# mkdir /opt/software 
#将宿主机中的软件包拷贝到指定容器中    
$ docker cp jdk-8u241-linux-x64.tar.gz 05272c80eb5b:/opt/software
$ docker cp hadoop-2.7.7.tar.gz 05272c80eb5b:/opt/software
root@05272c80eb5b:/opt# cd software
root@05272c80eb5b:/opt/software# ls
hadoop-2.7.7.tar.gz  jdk-8u241-linux-x64.tar.gz
#解压
root@05272c80eb5b:/opt/software# tar -zxvf jdk-8u241-linux-x64.tar.gz
root@05272c80eb5b:/opt/software# tar -zxvf hadoop-2.7.7.tar.gz
#将压缩包转移到moudle文件夹下管理
root@05272c80eb5b:/opt# mkdir moudle
root@05272c80eb5b:/opt# mv jdk-8u241-linux-x64.tar.gz /opt/moudle/
root@05272c80eb5b:/opt# mv hadoop-2.7.7.tar.gz /opt/moudle/
#重命名安装包
root@05272c80eb5b:/opt/software# mv hadoop-2.7.7 hadoop2.7
root@05272c80eb5b:/opt/software# mv jdk1.8.0_241 jdk1.8

### 2.踩坑
#现在已经不能用PPA方式安装jdk，因为这个端口已经停止使用了，里面什么也没有，现在从oracle官网下载jdk必须注册账号，以下方法完全不行
root@05272c80eb5b:/# apt update
root@05272c80eb5b:/# apt install python-software-properties
root@05272c80eb5b:/# apt install software-properties-common
root@05272c80eb5b:/# add-apt-repository ppa:webupd8team/java
root@05272c80eb5b:/# apt update
root@05272c80eb5b:/# apt install oracle-java8-installer
#看一下是什么东西
root@05272c80eb5b:/etc/apt# ls
apt.conf.d   preferences.d  sources.list.d     trusted.gpg
auth.conf.d  sources.list   sources.list.save  trusted.gpg.d
root@05272c80eb5b:/etc/apt/sources.list.d# ls
webupd8team-ubuntu-java-xenial.list
root@05272c80eb5b:/etc/apt/sources.list.d# cat webupd8team-ubuntu-java-xenial.list 
deb http://ppa.launchpad.net/webupd8team/java/ubuntu xenial main
# deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu xenial main
root@05272c80eb5b:/etc/apt/sources.list.d# rm webupd8team-ubuntu-java-xenial.list

### 3.更换apt下载源
root@05272c80eb5b:/etc/apt# cp sources.list sources.list.backup
root@05272c80eb5b:/etc/apt# vim sources.list
root@05272c80eb5b:/etc/apt$ cat sources.list
# 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释
deb http://mirrors.aliyun.com/ubuntu/ xenial main
#deb-src http://mirrors.aliyun.com/ubuntu/ xenial main

deb http://mirrors.aliyun.com/ubuntu/ xenial-updates main
#deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates main

deb http://mirrors.aliyun.com/ubuntu/ xenial universe
#deb-src http://mirrors.aliyun.com/ubuntu/ xenial universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates universe
#deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates universe

deb http://mirrors.aliyun.com/ubuntu/ xenial-security main
#deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security main
deb http://mirrors.aliyun.com/ubuntu/ xenial-security universe
#deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security universe
deb http://archive.ubuntu.com/ubuntu/ trusty main universe restricted multiverse

### 4.安装一些必要的工具
#安装sshd,实现无密码登录，访问集群机器更方便
test@hadoop101:~/桌面$ docker exec -it 05272c80eb5b /bin/bash
root@05272c80eb5b:/# apt install ssh
root@05272c80eb5b:/# /etc/init.d/ssh start
root@05272c80eb5b:/# cd ~/ #进入用户主目录，这里就是进入/root/
root@05272c80eb5b:~# ssh-keygen -t rsa  #生成秘钥
root@05272c80eb5b:~# cd .ssh/
root@05272c80eb5b:~/.ssh# ll
total 16
drwx------ 2 root root 4096 Apr  7 10:21 ./
drwx------ 1 root root 4096 Apr  7 10:17 ../
-rw------- 1 root root 1675 Apr  7