编译Hadoop3.3.6编译

最新推荐文章于 2024-01-20 15:40:22 发布

过去的曾经^_^

最新推荐文章于 2024-01-20 15:40:22 发布

阅读量618

点赞数 1

文章标签： hadoop

本文链接：https://blog.youkuaiyun.com/weixin_43838444/article/details/132565809

版权

该篇文章详细描述了在Windows环境下安装和配置Hadoop开发环境的过程，包括安装Java、Maven、CMake、依赖库（如ProtocolBuffers、snappy等）、PMDK以及前端环境（Node.js和Yarn），最后成功编译了Hadoop3.3.6版本。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

编译Hadoop

在win上解压打开一个Hadoop源码
查看BUILDING.txt，阅读Requirements 确定依赖版本根据Building on CentOS 8 提示安装

安装JAVA环境根据本文安装jdk来安装

# 查找安装的jdk文件以及一些其他文件
rpm -qa | grep java
# 卸载 OpenJDK文件
rpm -e --nodeps java-1.8.0-openjdk-headless-1.8.0.191.b12-1.el7_6.x86_64
# 软件包放置的目录
mkdir -p /export/software
# 软件安装的目录
mkdir -p /export/server
# 解压安装
tar -zxvf /export/software/jdk-8u241-linux-x64.tar.gz -C /export/server
# 配置jdk的环境变量
vim /etc/profile

#set java environment 
export JAVA_HOME=/export/server/jdk1.8.0_241 
export JRE_HOME=/export/server/jdk1.8.0_241/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib:$CLASSPATH
export PATH=$JAVA_HOME/bin:$PATH

安装Maven编译环境根据本文安装来安装

tar xvf /export/software/apache-maven-3.9.4-bin.tar.gz  -C /export/server/
# 添加环境变量
vim /etc/profile

# set maven enviroment
export MAVEN_HOME=/export/server/apache-maven-3.9.4
export PATH=$PATH:$JAVA_HOME/bin:$MAVEN_HOME/bin
# 验证是否安装成功
source /etc/profile
mvn -v

# 修改Maven镜像
# 设置本地存放目录
<localRepository>/export/server/maven/repository</localRepository>

# 删除原来的镜像保护
<mirror>
	<id>maven-default-http-blocker</id>
	<mirrorOf>external:http:*</mirrorOf>
	<name>Pseudo repository to mirror external repositories initially using HTTP.</name>
	<url>http://0.0.0.0/</url>
	<blocked>true</blocked>
</mirror>
# 添加新镜像
<mirror>
   <id>alimaven-central</id>
   <mirrorOf>central</mirrorOf>
   <name>aliyun maven</name>
   <url>https://maven.aliyun.com/nexus/content/repositories/central/</url>
</mirror>
<mirror>
   <id>alimaven</id>
   <name>aliyun maven</name>
   <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
   <mirrorOf>central</mirrorOf>
</mirror>
<mirror>
   <id>jboss-public-repository-group</id>
   <mirrorOf>central</mirrorOf>
   <name>JBoss Public Repository Group</name>
   <url>http://repository.jboss.org/nexus/content/groups/public</url>
</mirror>

安装依赖和压缩环境

yum install  libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel lzo-devel zlib-devel gcc gcc-c++

安装C环境根据要求去githup上下载CMake 3.1

tar -zxvf /export/software/cmake-3.27.1.tar.gz -C /export/server/
cd /export/server/cmake-3.27.1
./bootstrap
./configure 
gmake
gmake install

安装 Protocol Buffers v3.7.1

yum -y install autoconf automake libtool
tar -zxvf protobuf-3.7.1.tar.gz -C ../server/
cd /export/server/protobuf-3.7.1
./autogen.sh
./configure
make &  make install

安装snappy

tar -zxvf snappy-1.1.3.tar.gz -C ../server/
cd /export/server/snappy-1.1.3
./autogen.sh
./configure
make &  make install
sudo yum install -y snappy-devel

安装isa-l

sudo yum install -y epel-release
sudo yum install -y libzstd-devel

yum install -y yasm
yum install -y nasm
tar -zxvf isa-l-2.30.0.tar.gz -C ../server/
cd /export/server/isa-l-2.30.0
./autogen.sh
./configure
make &  make install

安装pmdk

yum install libpmem-devel librpmem-devel libpmemblk-devel libpmemlog-devel libpmemobj-devel libpmemobj++-devel libpmempool-devel
yum install -y bash-completion*
yum install -y json-c*
yum install -y libudev*
yum install -y ndctl*
yum install -y autoconf* 
tar -zxvf pmdk-2.0.0.tar.gz -C /export/server/
cd /export/server/pmdk-2.0.0
make -j
make install

安装前端环境

安装nodejs 和 yarn
yum -y install nodejs
yum install npm
curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo
yum -y install yarn
yarn config set ignore-engines true

Hadoop编译

tar -zxvf /export/software/hadoop-3.3.6-src.tar.gz  -C /export/server/

mvn package -DskipTests -Pdist,native -Dtar -Drequire.snappy -Drequire.pmdk -Drequire.isal -Drequire.openssl -Drequire.zstd  -e -X

查看编译成的样子

[root@node00 bin]# ./hadoop checknative
2023-08-29 04:52:39,162 INFO bzip2.Bzip2Factory: Successfully loaded & initialized native-bzip2 library system-native
2023-08-29 04:52:39,164 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library
2023-08-29 04:52:39,198 INFO nativeio.NativeIO: The native code was built with PMDK support, and PMDK libs were loaded successfully.
Native library checking:
hadoop:  true /export/server/hadoop-3.3.6-src/hadoop-dist/target/hadoop-3.3.6/lib/native/libhadoop.so.1.0.0
zlib:    true /lib64/libz.so.1
zstd  :  true /lib64/libzstd.so.1
bzip2:   true /lib64/libbz2.so.1
openssl: true /lib64/libcrypto.so
ISA-L:   true /lib/libisal.so.2
PMDK:    true /usr/local/lib64/libpmem.so.1.0.0