制作编译镜像事例

背景说明

制作编译镜像，基础的编译依赖，ssh服务，docker服务等等。

Dockerfile如下

FROM ubuntu:16.04

COPY ./sources.list       /etc/apt/sources.list

RUN  echo 'Asia/Shanghai' >/etc/timezone \
  && apt-get update \
`# ------ # Tools (工具类)# ------ #` \
  `## ------ ## Basic common tools (基础工具)## ------ ##` \
    && apt-get -y install sudo=1.8.16-0ubuntu1.10 \
    && apt-get install -y vim=2:7.4.1689-3ubuntu1.5 \
    && apt-get install -y tree=1.7.0-3 \ 
    && apt-get install -y zip=3.0-11 \
    && apt-get install -y unzip=6.0-20ubuntu1.1 \
    && apt-get install -y wget=1.17.1-1ubuntu1.5 \
    && apt-get install -y net-tools=1.60-26ubuntu1 \
    && apt-get install -y sysstat=11.2.0-1ubuntu0.3 \
    && apt-get install -y nload=0.7.4-1build1 \
    && apt-get install -y iputils-ping=3:20121221-5ubuntu2 \
    && apt-get install -y expect=5.45-7 \
    && apt-get install -y curl=7.47.0-1ubuntu2.19 \
  `## ------ ## Code management tools (代码管理管工具)## ------ ##` \
    && apt-get install -y git=1:2.7.4-0ubuntu1.10 \
  `## ------ ## Code compilation tools (代码编译工具)## ------ ##` \
    `## ------ ## C/C++ ## ------ ##` \
      && apt-get install -y g++=4:5.3.1-1ubuntu1 \
      && apt-get install -y cmake=3.5.1-1ubuntu3 \
      && apt-get install -y autoconf=2.69-9 \
    `## ------ ## Python ## ------ ##` \
      && apt-get install -y python3=3.5.1-3 \
    `## ------ ## Java ## ------ ##` \
      && apt-get install -y default-jdk=2:1.8-56ubuntu2 \
      && apt-get install -y maven=3.3.9-3 \
      && cd /opt \
        && wget http://10.0.5.5:8100/install/build_java_depend.zip \
        && unzip build_java_depend.zip \
    `## ------ ## JavaScript(web) ## ------ ##` \
      && cd /opt/ \
        && wget http://10.0.5.5:8100/install/nodejs/8.11.1/util-node-node-v8.11.1.tar.gz \
        && mkdir -p /opt/node \
        && tar -zxvf /opt/util-node-node-v8.11.1.tar.gz  --strip 1  -C  /opt/node/ \
        && rm -rf /opt/util-node-node-v8.11.1.tar.gz \
`# ------ # Services (服务类)# ------ #` \
  `## ------ ## Docker (操作系统虚拟化服务) ## ------ ##` \
    && apt-get install -y apt-transport-https=1.2.35 \
    && apt-get install -y ca-certificates=20210119~16.04.1 \
    && apt-get install -y gnupg-agent=2.1.11-6ubuntu2.1 \
    && apt-get install -y iptables=1.6.0-2ubuntu3 \
    && apt-get install -y libnfnetlink0=1.0.1-3 \
    && cd /opt \
      && curl -fsSL http://10.0.1.189:8081/repository/Ubuntu16.04-docker2/gpg | apt-key add - \
      && apt-key fingerprint 0EBFCD88 \
      && apt-get update \
      && apt-get install -y \
        docker-ce=5:20.10.7~3-0~ubuntu-xenial \
        docker-ce-cli=5:20.10.7~3-0~ubuntu-xenial \
        containerd.io=1.4.6-1 \
  `## ------ ## Install ssh (SSH远程服务)## ------ ##` \
    && apt-get install -y openssh-server=1:7.2p2-4ubuntu2.10 \
    && sed 's/PermitRootLogin prohibit-password/PermitRootLogin yes/g; s/PubkeyAuthentication yes/PubkeyAuthentication no/g; s/#PasswordAuthentication yes/PasswordAuthentication yes/g' -i  /etc/ssh/sshd_config \
`# ------ # Third dependecy (第三方依赖类)# ------ #` \
  `## ------ ## C/C++ dependecy (C/C++编译程序依赖) ## ------ ##` \
    `### ------ ### Install OpenGL ### ------ ###` \
      && apt-get install -y libgl1-mesa-dev=18.0.5-0ubuntu0~16.04.1 \
      && apt-get install -y build-essential=12.1ubuntu2 \
    `### ------ ### Install ZMQ ### ------ ###` \
      && apt-get install -y libtool=2.4.6-0.1 \
      && cd /opt \
        && wget http://10.0.5.5:8100/install/code/zmq/libzmq-4.zip \
        && unzip libzmq-4.zip \
        && cd libzmq-4 \
        && cp include/* /usr/local/include/ \
        && cp lib/libzmq.so.5.0.3 /usr/local/lib/ \
        && cd /usr/local/lib/ \
        && ln -s libzmq.so.5.0.3 libzmq.so.5 \
        && ln -s libzmq.so.5.0.3 libzmq.so \
        && rm -rf /opt/libzmq-4.zip \
    `### ------ ### Install googletest (C++单元测试库)### ------ ###` \
      && cd /opt \
        && wget http://10.0.5.5:8100/install/code/googletest/googletest-release-1.10.0.zip \
        && unzip googletest-release-1.10.0.zip \
        && cd googletest-release-1.10.0 \
        && mkdir build \
        && cd build \
        && cmake .. \
        && make \
        && make install \
        && rm -rf /opt/googletest-release-1.10.0.zip \
    `### ------ ### Install jsoncpp (json操作库)### ------ ###` \
      && cd /opt/ \
        && wget http://10.0.5.5:8100/install/code/jsoncpp/jsoncpp-1.8.2.zip \
        && unzip jsoncpp-1.8.2.zip \
        && cd jsoncpp-1.8.2 \
        && cmake . \
        && make -j4 \
        && make install \
        && rm -rf /opt/jsoncpp-1.8.2.zip \
    `### ------ ### Install fastdfs (分布式存储库)### ------ ###` \
      && cd /opt/ \
        && wget http://10.0.5.5:8100/install/code/fastdfs/libfastcommon-V1.0.56.zip \
        && unzip libfastcommon-V1.0.56.zip \
        && cd libfastcommon-V1.0.56 \
        && ./make.sh clean \
        && ./make.sh \
        && ./make.sh install \
        && rm -rf /opt/libfastcommon-V1.0.56.zip \
      && cd /opt/ \
        && wget http://10.0.5.5:8100/install/code/fastdfs/fastdfs-V6.08.zip \
        && unzip fastdfs-V6.08.zip \
        && cd fastdfs-V6.08 \
        && ./make.sh clean \
        && ./make.sh \
        && ./make.sh install \
        && rm -rf /opt/fastdfs-V6.08.zip \
    `### ------ ### Install libb64 (Base64操作库)### ------ ###` \
      && cd /opt/ \
        && wget http://10.0.5.5:8100/install/code/libb64/libb64-1.2.1.zip \
        && unzip libb64-1.2.1.zip \
        && cd libb64-1.2.1 \
        && make \
        && cd /opt/libb64-1.2.1 \
        && cp -rf ./include/b64 /usr/include/ \
        && cp -rf ./src/libb64.a /usr/lib/ \
        && rm -rf /opt/libb64-1.2.1.zip \
    `### ------ ### Install OpenCV ### ------ ###` \
      && cd /opt/ \
        && wget http://10.0.5.5:8100/install/code/opencv/opencv-2.4.9.zip \
        && unzip opencv-2.4.9.zip \
        && cd opencv-2.4.9 \
        && cmake . \
        && make -j24 \
        && make install -j24 \
        && rm -rf /opt/opencv-2.4.9 /opt/opencv-2.4.9.zip \
    `### ------ ### Install tbb2(算法依赖) ### ------ ###` \
        && apt-get install -y libtbb2=4.4~20151115-0ubuntu3 \
    `### ------ ### Install cuda(算法依赖) ### ------ ###` \
      && cd /opt/ \
      && wget http://10.0.5.5:8100/install/cuda/10.2/cuda-ubuntu1604.pin \
      && wget http://10.0.5.5:8100/install/cuda/10.2/cuda-repo-ubuntu1604-10-2-local-10.2.89-440.33.01_1.0-1_amd64.deb \
      && cp cuda-ubuntu1604.pin /etc/apt/preferences.d/cuda-repository-pin-600 \
      && dpkg -i cuda-repo-ubuntu1604-10-2-local-10.2.89-440.33.01_1.0-1_amd64.deb \
      && apt-key add /var/cuda-repo-10-2-local-10.2.89-440.33.01/7fa2af80.pub \
      && apt-get -y update \
      && apt-get -y install cuda \
      && rm -rf /opt/cuda-ubuntu1604.pin /opt/cuda-repo-ubuntu1604-10-2-local-10.2.89-440.33.01_1.0-1_amd64.deb \
    `### ------ ### Install tensorrt(算法依赖) ### ------ ###` \
      && cd /opt \
        && wget http://10.0.5.5:8100/install/tensorRT/7.1.3.4/nv-tensorrt-repo-ubuntu1604-cuda10.2-trt7.1.3.4-ga-20200617_1-1_amd64.deb \
        && dpkg -i nv-tensorrt-repo-ubuntu1604-cuda10.2-trt7.1.3.4-ga-20200617_1-1_amd64.deb \
        && cp /var/nv-tensorrt-repo-cuda10.2-trt7.1.3.4-ga-20200617/*.gpg /usr/share/keyrings/ \
        && apt-get -y update \
        && apt-get -y install tensorrt \
        && rm -rf /opt/nv-tensorrt-repo-ubuntu1604-cuda10.2-trt7.1.3.4-ga-20200617_1-1_amd64.deb \



# ------ # config (配置相关)# ------ #
  ## ------ ## nodeJS config PATH ## ------ ##
  ENV PATH="/opt/node/bin:${PATH}"
  ## ------ ## maven config ## ------ ##
  COPY ./settings.xml /etc/maven/settings.xml

上述文件中的source.list 采用阿里的源。

上述setting.xml

<?xml version="1.0" encoding="UTF-8"?>

<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
  <pluginGroups>
  </pluginGroups>

  <proxies>
  </proxies>

  <servers>
    <server>
      <id>dilu-release</id>
      <username>admin</username>
      <password>Admin123</password>
    </server>
    <server>
      <id>dilu-snapshot</id>
      <username>admin</username>
      <password>Admin123</password>
    </server>
  </servers>

  <mirrors>    
    <mirror>
      <id>dilu-group</id>
      <name>nexus public repositories</name>
      <url>http://10.0.1.189:8081/repository/dilu-group/</url>
      <!-- 匹配该镜像下所有的仓库。 此时pom中定义的仓库都不生效 -->
      <mirrorOf>*</mirrorOf>
    </mirror>
  </mirrors>

  <profiles>
    <profile>
      <!-- id必须唯一 -->
      <id>dilu-release</id>
      <repositories>
        <repository>
          <!-- id必须唯一 -->
          <id>dilu-release-1</id>
          <!-- 仓库的url地址 -->
          <url>http://10.0.1.189:8081/repository/dilu-release/</url>
          <releases>
            <enabled>true</enabled>
          </releases>
        </repository>
      </repositories>
    </profile>

    <profile>
      <!-- id必须唯一 -->
      <id>dilu-snapshot</id>
      <repositories>
        <repository>
          <!-- id必须唯一 -->
          <id>dilu-snapshot-1</id>
          <!-- 仓库的url地址 -->
          <url>http://10.0.1.189:8081/repository/dilu-snapshot/</url>
          <releases>
            <enabled>true</enabled>
          </releases>
          <snapshots>
            <enabled>true</enabled>
            <updatePolicy>always</updatePolicy>
          </snapshots>
        </repository>
      </repositories>
    </profile>

  </profiles>

  <activeProfiles>
    <activeProfile>dilu-snapshot</activeProfile>
    <activeProfile>dilu-release</activeProfile>
  </activeProfiles>

</settings>

启动容器脚本如下

3.1 start.sh

该脚本在/data/下存储数据。也可以修改其他路径。该脚本的父文件夹名称会在容器内创建用户名。注意：该容器采用--net=none模式启动。

#!/bin/bash

usr=`pwd | awk -F "/" '{print $NF}'`
echo "usr is $usr"

user_path=/data/userdata/$usr
mkdir -p $user_path
cp ./home/.bash_logout $user_path/ -rf
cp ./home/.bashrc $user_path/ -rf
cp ./home/.cache/ $user_path/ -rf
cp ./home/.profile $user_path/ -rf
cp ./home/.viminfo $user_path/ -rf

image_name=compilation_platform:1.0.1


export POSTGRES_usr=$usr

mkdir -p ./tmp
cp ./run.sh ./tmp/ -f

docker run -itd \
    --privileged=true \
    --net=none \
    -v /etc/localtime:/etc/localtime:ro \
    -v $user_path:/home/$usr \
    -v $PWD/tmp/:/usr/sbin/$usr \
    -e POSTGRES_usr \
    --restart=on-failure \
    --hostname cmp-$usr \
    --name $usr \
    $image_name \
    sh -c 'cd /usr/sbin/$POSTGRES_usr/ && ./run.sh'

3.2、run.sh

#!/bin/bash

usr=`pwd | awk -F "/" '{print $NF}'`

# sudo
echo `date` "start put $usr into sudoers file"  
chmod +w /etc/sudoers
echo "$usr ALL=(ALL) ALL" >> /etc/sudoers
if [ $? -ne 0 ]; then
  echo `date` "start put $usr into sudoers file failed!"  
else
  echo `date` "start put $usr into sudoers file succ."  
fi
chmod -w /etc/sudoers
#resolve:sudo -i , sudo: /usr/bin/sudo must be owned by uid 0 and have the setuid bit set
chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo

# resolve use sudo tips error
echo "127.0.0.1 cmp-$usr" >> /etc/hosts
if [ $? -ne 0 ]; then
  echo `date` "put into hosts file failed!"  
else
  echo `date` "put into hosts file succ."  
fi

# add user
useradd -m $usr -s /bin/bash -d /home/$usr
echo "ua is : useradd -m $usr -s /bin/bash -d /home/$usr"
if [ $? -ne 0 ]; then
  echo `date` "useradd failed!"  
else
  echo `date` "useradd succ."  
fi

# add user's password
expect -c "
set timeout -1
spawn bash -c \"passwd $usr\"
expect {
 \"Enter new UNIX password:\" {send \"Admin123\r\";exp_continue}
 \"Retype new UNIX password:\" {send \"Admin123\r\"}
}
expect eof
"
if [ $? -ne 0 ]; then
  echo `date` "expect failed!"  
else
  echo `date` "expect succ."  
fi
chown taoxp:taoxp /home/$usr -R

# start service
service docker start
service ssh start;

# add docker right
gpasswd -a $usr docker
newgrp docker

echo "start successfully !"

/bin/bash

自启动配置

#!/bin/bash

#
mount  /dev/sda1 /sda/

### wait 30 s
echo "wait 30s ..."
sleep 30

### stop docker service
systemctl stop docker
## systemctl stop docker.socket
#stop docker0 network card and delete it
## ip link set dev docker0 down
## brctl delbr docker0

### create bridge
#host ip
ip=10.1.2.62/24 
#gateway
gateway=10.1.2.1
#bridge
br_name=jbr1
#physical netwok card
pys_net=eth0
#broadcast address
brd=10.0.1.255

brctl addbr $br_name; \
ip link set dev $br_name up; \
ip addr del $ip dev $pys_net; \
ip addr add $ip brd $brd dev $br_name; \
brctl addif $br_name $pys_net; \
ip route add default via $gateway dev $br_name

# vim /usr/lib/systemd/system/docker.service
# modify the "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
#        to "ExecStart=/usr/bin/dockerd -b jbr1 -H fd:// --containerd=/run/containerd/containerd.sock"
# then excute the command "systemctl daemon-reload; systemctl restart docker"

systemctl start docker

cd /opt/mysql_workdir
docker start mysql
./start.sh
pipework jbr1 mysql   10.1.2.242/24@10.1.2.1

cd /opt/filesvr_workdir
docker start filesvr
./start.sh
pipework jbr1 filesvr 10.1.2.241/24@10.1.2.1

cd /opt/harbor_workdir
#./stop.sh
#./start.sh
pipework jbr1 harbor  10.1.2.243/24@10.1.2.1

cd /opt/compile_workdir
docker start compile
./start.sh
pipework jbr1 compile 10.1.2.244/24@10.1.2.1

cd /opt/gitlab_workdir
docker start gitlab
./start.sh
pipework jbr1 gitlab 10.1.2.246/24@10.1.2.1

# ---------------------------------------------------

usr=taoxp
docker start $usr
pipework jbr1 $usr  10.1.2.100/24@10.1.2.1

usr=zhangqj
docker start $usr
pipework jbr1 $usr  10.1.2.101/24@10.1.2.1

usr=liujing
docker start $usr
pipework jbr1 $usr  10.1.2.102/24@10.1.2.1

usr=lina
docker start $usr
pipework jbr1 $usr  10.1.2.103/24@10.1.2.1

usr=fuxq
docker start $usr
pipework jbr1 $usr  10.1.2.104/24@10.1.2.1


/usr/bin/auto_assign_ip.sh &

上述auto_assign_ip.sh内容如下：

#!/bin/bash

#sub thread
function a_sub 
{
 dist=$1
 ping -c 1 $dist
 if [ $? -eq 0 ]; then
   echo "The ip $dist is used!"
 else
   echo "----------------------------------------" $dist
   echo $dist > /tmp/canuseip
 fi
}

function mk_tmp_fifo_file
{
 tmp_fifofile="/tmp/tmp.fifo"
 mkfifo $tmp_fifofile
 exec 6<>$tmp_fifofile
 rm $tmp_fifofile
}

function init_threads
{
 thread=50
 for ((i=0;i<$thread;i++));do
 echo "$i"
 done >&6
}

function start_thread
{
 for ((i=200;i<233;i++));do
 read -u6
 {
  a_sub 10.1.2.$i $i && {
   tmp_tmp=tmp
  } || {
   echo "===="
  }
  echo >&6
 }&
 done

 wait
 exec 6>&-
}

function assign_ip
{
 can_use_ip=`cat /tmp/canuseip|head -1`
 rm -f /tmp/canuseip

 if [ "$can_use_ip" != "" ];then
  pipework jbr1 $1 $can_use_ip/24@10.1.2.1
  echo "end, assign $can_use_ip to $1"
 else
  echo "no ip"
 fi
}

read_dir(){
 for file in `ls $1`
  do
   if [ -f $1"/"$file ];then
    echo "begin " `date` " -- $file"
    mk_tmp_fifo_file
    init_threads
    start_thread

    rm -f $1"/"$file
    echo "rm the $file."

    assign_ip $file

   fi
 done
}


mkdir -p /data/structworkbench/ctn/
while [ 1 ]
do
 read_dir /data/structworkbench/ctn/
 sleep 1
done