caffe学习笔记1.1-- caffe的Makefile文件

在本文中，描述一下caffe的Makefile文件，这个文件用于caffe的编译；这里不会把所有文件都考过来，有些脚本相似的会省略掉

文件在caffe根目录下，因为用到makefile.config中的变量，因此，在用到的时候我会给下说明

首先，是项目名称

PROJECT := caffe #项目名称，就是当前的caffe目录

接下来，这个文件要用到了Makefile.config中的一些变量，要包含这个文件：

CONFIG_FILE := Makefile.config #配置文件名
# Explicitly check for the config file, otherwise make -k will proceed anyway.
ifeq ($(wildcard $(CONFIG_FILE)),) #检查配置问价是否存在，如果不存在，使用config.example
$(error $(CONFIG_FILE) not found. See $(CONFIG_FILE).example.)
endif
include $(CONFIG_FILE) #将文件包含进来

编译后文件的存放目录

BUILD_DIR_LINK := $(BUILD_DIR) #BUILD_DIR在Makefile.config文件中定义　caffe中的build目录

编译方式：

ifeq ($(RELEASE_BUILD_DIR),) #是编译方式，使用debug还是release编译，如果用debug生成./build_debug目录，release生成./build_release目录
	RELEASE_BUILD_DIR := .$(BUILD_DIR)_release
endif
ifeq ($(DEBUG_BUILD_DIR),)
	DEBUG_BUILD_DIR := .$(BUILD_DIR)_debug
endif

#检查debug是否为０,debug的定义在.config文件中
DEBUG ?= 0
ifeq ($(DEBUG), 1)
	BUILD_DIR := $(DEBUG_BUILD_DIR)
	OTHER_BUILD_DIR := $(RELEASE_BUILD_DIR)
else
	BUILD_DIR := $(RELEASE_BUILD_DIR)
	OTHER_BUILD_DIR := $(DEBUG_BUILD_DIR)
endif

链接库生成

# All of the directories containing code.找到所有.cpp .proto
SRC_DIRS := $(shell find * -type d -exec bash -c "find {} -maxdepth 1 \
	\( -name '*.cpp' -o -name '*.proto' \) | grep -q ." \; -print)

# The target shared library name
LIB_BUILD_DIR := $(BUILD_DIR)/lib #build/lib存放生成的链接库
STATIC_NAME := $(LIB_BUILD_DIR)/lib$(PROJECT).a #生成静态链接库
DYNAMIC_NAME := $(LIB_BUILD_DIR)/lib$(PROJECT).so　#动态链接库

获取所有源文件：

##############################
# Get all source files
##############################
# CXX_SRCS are the source files excluding the test ones.
CXX_SRCS := $(shell find src/$(PROJECT) ! -name "test_*.cpp" -name "*.cpp") #排除test_*.cpp
# CU_SRCS are the cuda source files
CU_SRCS := $(shell find src/$(PROJECT) ! -name "test_*.cu" -name "*.cu") #排除test_*.cu
# TEST_SRCS are the test source files
TEST_MAIN_SRC := src/$(PROJECT)/test/test_caffe_main.cpp #~/src/caffe/test/test_caffe_main.cpp
TEST_SRCS := $(shell find src/$(PROJECT) -name "test_*.cpp")
TEST_SRCS := $(filter-out $(TEST_MAIN_SRC), $(TEST_SRCS))
TEST_CU_SRCS := $(shell find src/$(PROJECT) -name "test_*.cu")
GTEST_SRC := src/gtest/gtest-all.cpp
# TOOL_SRCS are the source files for the tool binaries
TOOL_SRCS := $(shell find tools -name "*.cpp") #tools目录下所有.cpp文件
# EXAMPLE_SRCS are the source files for the example binaries
EXAMPLE_SRCS := $(shell find examples -name "*.cpp")
# BUILD_INCLUDE_DIR contains any generated header files we want to include.
BUILD_INCLUDE_DIR := $(BUILD_DIR)/src　#build/src, 包含include的头文件。
# PROTO_SRCS are the protocol buffer definitions
PROTO_SRC_DIR := src/$(PROJECT)/proto　	#~/src/caffe/proto
PROTO_SRCS := $(wildcard $(PROTO_SRC_DIR)/*.proto)
# PROTO_BUILD_DIR will contain the .cc and obj files generated from
# PROTO_SRCS; PROTO_BUILD_INCLUDE_DIR will contain the .h header files
PROTO_BUILD_DIR := $(BUILD_DIR)/$(PROTO_SRC_DIR) #build/proto
PROTO_BUILD_INCLUDE_DIR := $(BUILD_INCLUDE_DIR)/$(PROJECT)/proto
# NONGEN_CXX_SRCS includes all source/header files except those generated
# NONGEN_CXX_SRCS　所有排除的文件
# automatically (e.g., by proto).
NONGEN_CXX_SRCS := $(shell find \
	src/$(PROJECT) \
	include/$(PROJECT) \
	python/$(PROJECT) \
	matlab/+$(PROJECT)/private \
	examples \
	tools \
	-name "*.cpp" -or -name "*.hpp" -or -name "*.cu" -or -name "*.cuh")

后面一段LINT貌似是关于静态代码解析的，还有一些是用于python和matlab接口的，这里不做说明

导出生成文件：

这里需要说一下编译caffe的过程：从.cpp文件，生成.o文件，由.o文件生成.bin文件，同时，在生成.d的过程需要依赖.d文件

因此，在build文件中会由这三种后缀的文件

##############################
# Derive generated files
##############################
# The generated files for protocol buffers
#caffe.pb.h，caffe.pb.cc在build/proto中
PROTO_GEN_HEADER_SRCS := $(addprefix $(PROTO_BUILD_DIR)/, \ 
		$(notdir ${PROTO_SRCS:.proto=.pb.h}))
PROTO_GEN_HEADER := $(addprefix $(PROTO_BUILD_INCLUDE_DIR)/, \
		$(notdir ${PROTO_SRCS:.proto=.pb.h}))
PROTO_GEN_CC := $(addprefix $(BUILD_DIR)/, ${PROTO_SRCS:.proto=.pb.cc})
PY_PROTO_BUILD_DIR := python/$(PROJECT)/proto
PY_PROTO_INIT := python/$(PROJECT)/proto/__init__.py
PROTO_GEN_PY := $(foreach file,${PROTO_SRCS:.proto=_pb2.py}, \
		$(PY_PROTO_BUILD_DIR)/$(notdir $(file)))
# The objects corresponding to the source files
# These objects will be linked into the final shared library, so we
# exclude the tool, example, and test objects.
CXX_OBJS := $(addprefix $(BUILD_DIR)/, ${CXX_SRCS:.cpp=.o})
CU_OBJS := $(addprefix $(BUILD_DIR)/cuda/, ${CU_SRCS:.cu=.o})
PROTO_OBJS := ${PROTO_GEN_CC:.cc=.o}
OBJS := $(PROTO_OBJS) $(CXX_OBJS) $(CU_OBJS)
# tool, example, and test objects
TOOL_OBJS := $(addprefix $(BUILD_DIR)/, ${TOOL_SRCS:.cpp=.o}) #定义生成规则，tools下的.cpp文件生成.o文件;文件存放在build/tools中
#下面类似
TOOL_BUILD_DIR := $(BUILD_DIR)/tools
TEST_CXX_BUILD_DIR := $(BUILD_DIR)/src/$(PROJECT)/test
TEST_CU_BUILD_DIR := $(BUILD_DIR)/cuda/src/$(PROJECT)/test
TEST_CXX_OBJS := $(addprefix $(BUILD_DIR)/, ${TEST_SRCS:.cpp=.o})
TEST_CU_OBJS := $(addprefix $(BUILD_DIR)/cuda/, ${TEST_CU_SRCS:.cu=.o})
TEST_OBJS := $(TEST_CXX_OBJS) $(TEST_CU_OBJS)
GTEST_OBJ := $(addprefix $(BUILD_DIR)/, ${GTEST_SRC:.cpp=.o})
EXAMPLE_OBJS := $(addprefix $(BUILD_DIR)/, ${EXAMPLE_SRCS:.cpp=.o})
# Output files for automatic dependency generation
#　生成的依赖文件是.d的
DEPS := ${CXX_OBJS:.o=.d} ${CU_OBJS:.o=.d} ${TEST_CXX_OBJS:.o=.d} \
	${TEST_CU_OBJS:.o=.d} $(BUILD_DIR)/${MAT$(PROJECT)_SO:.$(MAT_SO_EXT)=.d}
# tool, example, and test bins
TOOL_BINS := ${TOOL_OBJS:.o=.bin}　 #.o文件生成.bin文件
EXAMPLE_BINS := ${EXAMPLE_OBJS:.o=.bin}
# symlinks to tool bins without the ".bin" extension
TOOL_BIN_LINKS := ${TOOL_BINS:.bin=}
# Put the test binaries in build/test for convenience.
TEST_BIN_DIR := $(BUILD_DIR)/test
TEST_CU_BINS := $(addsuffix .testbin,$(addprefix $(TEST_BIN_DIR)/, \
		$(foreach obj,$(TEST_CU_OBJS),$(basename $(notdir $(obj))))))
TEST_CXX_BINS := $(addsuffix .testbin,$(addprefix $(TEST_BIN_DIR)/, \
		$(foreach obj,$(TEST_CXX_OBJS),$(basename $(notdir $(obj))))))
TEST_BINS := $(TEST_CXX_BINS) $(TEST_CU_BINS)
# TEST_ALL_BIN is the test binary that links caffe dynamically.
TEST_ALL_BIN := $(TEST_BIN_DIR)/test_all.testbin

再下来，生成警告文件

##############################
# Derive compiler warning dump locations
# 警告文件，.o.warnings.txt；编译没有问题生警告文件为０字符
##############################
WARNS_EXT := warnings.txt
CXX_WARNS := $(addprefix $(BUILD_DIR)/, ${CXX_SRCS:.cpp=.o.$(WARNS_EXT)})
CU_WARNS := $(addprefix $(BUILD_DIR)/cuda/, ${CU_SRCS:.cu=.o.$(WARNS_EXT)})
TOOL_WARNS := $(addprefix $(BUILD_DIR)/, ${TOOL_SRCS:.cpp=.o.$(WARNS_EXT)})
EXAMPLE_WARNS := $(addprefix $(BUILD_DIR)/, ${EXAMPLE_SRCS:.cpp=.o.$(WARNS_EXT)})
TEST_WARNS := $(addprefix $(BUILD_DIR)/, ${TEST_SRCS:.cpp=.o.$(WARNS_EXT)})
TEST_CU_WARNS := $(addprefix $(BUILD_DIR)/cuda/, ${TEST_CU_SRCS:.cu=.o.$(WARNS_EXT)})
ALL_CXX_WARNS := $(CXX_WARNS) $(TOOL_WARNS) $(EXAMPLE_WARNS) $(TEST_WARNS)
ALL_CU_WARNS := $(CU_WARNS) $(TEST_CU_WARNS)
ALL_WARNS := $(ALL_CXX_WARNS) $(ALL_CU_WARNS)

EMPTY_WARN_REPORT := $(BUILD_DIR)/.$(WARNS_EXT)
NONEMPTY_WARN_REPORT := $(BUILD_DIR)/$(WARNS_EXT)

添加cuda的路径

##############################
# Derive include and lib directories
# 导出include和lib目录
##############################
CUDA_INCLUDE_DIR := $(CUDA_DIR)/include

CUDA_LIB_DIR :=
# add <cuda>/lib64 only if it exists
#　添加cuda/lib64文件路径：如果你已经安装了cuda
ifneq ("$(wildcard $(CUDA_DIR)/lib64)","")
	CUDA_LIB_DIR += $(CUDA_DIR)/lib64
endif
CUDA_LIB_DIR += $(CUDA_DIR)/lib

INCLUDE_DIRS += $(BUILD_INCLUDE_DIR) ./src ./include
ifneq ($(CPU_ONLY), 1)
	INCLUDE_DIRS += $(CUDA_INCLUDE_DIR)
	LIBRARY_DIRS += $(CUDA_LIB_DIR)
	LIBRARIES := cudart cublas curand
endif
LIBRARIES += glog gflags protobuf leveldb snappy \
	lmdb boost_system hdf5_hl hdf5 m \
	opencv_core opencv_highgui opencv_imgproc
PYTHON_LIBRARIES := boost_python python2.7
WARNINGS := -Wall -Wno-sign-compare

build文档目录等

##############################
# Set build directories
# 设置build目录
##############################

DISTRIBUTE_DIR ?= distribute
DISTRIBUTE_SUBDIRS := $(DISTRIBUTE_DIR)/bin $(DISTRIBUTE_DIR)/lib
DIST_ALIASES := dist
ifneq ($(strip $(DISTRIBUTE_DIR)),distribute)
		DIST_ALIASES += distribute
endif

ALL_BUILD_DIRS := $(sort $(BUILD_DIR) $(addprefix $(BUILD_DIR)/, $(SRC_DIRS)) \
	$(addprefix $(BUILD_DIR)/cuda/, $(SRC_DIRS)) \
	$(LIB_BUILD_DIR) $(TEST_BIN_DIR) $(PY_PROTO_BUILD_DIR) $(LINT_OUTPUT_DIR) \
	$(DISTRIBUTE_SUBDIRS) $(PROTO_BUILD_INCLUDE_DIR))

##############################
# Set directory for Doxygen-generated documentation
# 注释文档什么的
##############################
DOXYGEN_CONFIG_FILE ?= ./.Doxyfile
# should be the same as OUTPUT_DIRECTORY in the .Doxyfile
DOXYGEN_OUTPUT_DIR ?= ./doxygen
DOXYGEN_COMMAND ?= doxygen
# All the files that might have Doxygen documentation.
DOXYGEN_SOURCES := $(shell find \
	src/$(PROJECT) \
	include/$(PROJECT) \
	python/ \
	matlab/ \
	examples \
	tools \
	-name "*.cpp" -or -name "*.hpp" -or -name "*.cu" -or -name "*.cuh" -or \
        -name "*.py" -or -name "*.m")
DOXYGEN_SOURCES += $(DOXYGEN_CONFIG_FILE)

操作系统（只贴下linux部分的）

##############################
# Configure build
# 操作系统
##############################

# Determine platform
UNAME := $(shell uname -s)
ifeq ($(UNAME), Linux) #对于linux系统
	LINUX := 1
else ifeq ($(UNAME), Darwin) #ios
	OSX := 1
endif

# Linux
ifeq ($(LINUX), 1)
	CXX ?= /usr/bin/g++ #是否有g++
	GCCVERSION := $(shell $(CXX) -dumpversion | cut -f1,2 -d.)
	# older versions of gcc are too dumb to build boost with -Wuninitalized
	ifeq ($(shell echo | awk '{exit $(GCCVERSION) < 4.6;}'), 1) #如果g++版本<4.6需要添加一些配置
		WARNINGS += -Wno-uninitialized
	endif
	# boost::thread is reasonably called boost_thread (compare OS X)
	# We will also explicitly add stdc++ to the link target.
	LIBRARIES += boost_thread stdc++ #添加boost_thread和stdc++
endif

后面一大堆配置，如自定义编译，矩阵运算包之类；

# Custom compiler
# 自定义编译
ifdef CUSTOM_CXX
	CXX := $(CUSTOM_CXX)
endif

# Static linking
# 静态链接库
ifneq (,$(findstring clang++,$(CXX)))
	STATIC_LINK_COMMAND := -Wl,-force_load $(STATIC_NAME)
else ifneq (,$(findstring g++,$(CXX)))
	STATIC_LINK_COMMAND := -Wl,--whole-archive $(STATIC_NAME) -Wl,--no-whole-archive
else
  # The following line must not be indented with a tab, since we are not inside a target
  $(error Cannot static link with the $(CXX) compiler)
endif

# Debugging
ifeq ($(DEBUG), 1)
	COMMON_FLAGS += -DDEBUG -g -O0
	NVCCFLAGS += -G
else
	COMMON_FLAGS += -DNDEBUG -O2
endif

# cuDNN acceleration configuration.
ifeq ($(USE_CUDNN), 1)
	LIBRARIES += cudnn
	COMMON_FLAGS += -DUSE_CUDNN
endif

# CPU-only configuration
# 只有CPU下的配置
ifeq ($(CPU_ONLY), 1)
	OBJS := $(PROTO_OBJS) $(CXX_OBJS)
	TEST_OBJS := $(TEST_CXX_OBJS)
	TEST_BINS := $(TEST_CXX_BINS)
	ALL_WARNS := $(ALL_CXX_WARNS)
	TEST_FILTER := --gtest_filter="-*GPU*"
	COMMON_FLAGS += -DCPU_ONLY
endif

# Python layer support
ifeq ($(WITH_PYTHON_LAYER), 1)
	COMMON_FLAGS += -DWITH_PYTHON_LAYER
	LIBRARIES += $(PYTHON_LIBRARIES)
endif

# BLAS configuration (default = ATLAS)
# 矩阵运算库，默认ATLAS
BLAS ?= atlas
ifeq ($(BLAS), mkl)
	# MKL
	LIBRARIES += mkl_rt
	COMMON_FLAGS += -DUSE_MKL
	MKL_DIR ?= /opt/intel/mkl
	BLAS_INCLUDE ?= $(MKL_DIR)/include
	BLAS_LIB ?= $(MKL_DIR)/lib $(MKL_DIR)/lib/intel64
else ifeq ($(BLAS), open)
	# OpenBLAS
	LIBRARIES += openblas
else
	# ATLAS
	ifeq ($(LINUX), 1)
		ifeq ($(BLAS), atlas)
			# Linux simply has cblas and atlas
			LIBRARIES += cblas atlas
		endif
	else ifeq ($(OSX), 1)
		# OS X packages atlas as the vecLib framework
		LIBRARIES += cblas
		# 10.10 has accelerate while 10.9 has veclib
		XCODE_CLT_VER := $(shell pkgutil --pkg-info=com.apple.pkg.CLTools_Executables | grep -o 'version: 6')
		ifneq (,$(findstring version: 6,$(XCODE_CLT_VER)))
			BLAS_INCLUDE ?= /System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Headers/
			LDFLAGS += -framework Accelerate
		else
			BLAS_INCLUDE ?= /System/Library/Frameworks/vecLib.framework/Versions/Current/Headers/
			LDFLAGS += -framework vecLib
		endif
	endif
endif
INCLUDE_DIRS += $(BLAS_INCLUDE)
LIBRARY_DIRS += $(BLAS_LIB)

LIBRARY_DIRS += $(LIB_BUILD_DIR)

后面一句是为了写.d文件

# Automatic dependency generation (nvcc is handled separately)
CXXFLAGS += -MMD -MP　#生成依赖
##
# build文件夹的子目录中会有.d文件，MMD生成这个文件的前半段：.build.......
# MP生成了后面include ...:　的内容
# 第一次编译时，会编译所有文件，而如果include中的内容由改变时，不会重新生成.d.
##

以build/src/bolb.b为例：

MMD生成了：

.build_debug/src/caffe/blob.o: src/caffe/blob.cpp include/caffe/blob.hpp \
 include/caffe/common.hpp include/caffe/util/device_alternate.hpp \
 /usr/local/cuda/include/cublas_v2.h /usr/local/cuda/include/cublas_api.h \
 /usr/local/cuda/include/driver_types.h \.......................

MP生了：

include/caffe/blob.hpp:
include/caffe/common.hpp:
include/caffe/util/device_alternate.hpp:...................................

然后直到下面定义编译目标：

这里，就是make xx的东西了

##############################
# Define build targets
#　定义编译目标，就是make运行的内容
##############################
# make
#如果make后面什么都不加，就运行所有的all test clean.....
.PHONY: all test clean docs linecount lint lintclean tools examples $(DIST_ALIASES) \
	py mat py$(PROJECT) mat$(PROJECT) proto runtest \
	superclean supercleanlist supercleanfiles warn everything

# make all
all: $(STATIC_NAME) $(DYNAMIC_NAME) tools examples
....................

以tools为例子，解释一下这部分做的事情：

tools: $(TOOL_BINS) $(TOOL_BIN_LINKS)

这里的意思是说，如果你运行make tools，会做：右边的两件事情，$(TOOL_BIN)和$(TOOL_BIN_LINKS)其实是两个规则，具体定义如下：

$(TOOL_BINS): %.bin : %.o | $(DYNAMIC_NAME)
	@ echo CXX/LD -o $@
	$(Q)$(CXX) $< -o $@ $(LINKFLAGS) -l$(PROJECT) $(LDFLAGS) \
		-Wl,-rpath,$(ORIGIN)/../lib

文件中的标识中：$@--目标文件，$^--所有的依赖文件，$<--第一个依赖文件。

echo后面的内容，是make后在命令窗口中打印出来的内容

$(Q)在Makefile.config的最后定义：默认不打印后面的内容，将这里注释掉，就可以打印出所有内容

# enable pretty build (comment to see full commands)
Q ?= @

%.o：是个通配符，表示所有的.o文件，这几步是把.o文件生成.bin文件

| : 是一种依赖规则：当不存在目标文件时，所有的依赖都会被执行，生成目标文件，当存在目标时，若修改依赖中的内容，目标文件不会被重新创建。即｜后面的依赖是一种很弱的依赖，

这部分可以参考：http://blog.youkuaiyun.com/jingsuxuyilq/article/details/8194536的博客中的内容

关于makefile的一些规则，可以参考：http://www.codeproject.com/Articles/31488/Makefiles-in-Linux-An-Overview

Makefile中的内容大致就是这样，现在，注释掉Makefile.config中的$(Q),运行：

make 2>&1 >out.log　

２是错误的输出，１为正确的输出，这句的意思是将所有编译的内容全部输出到out.log中

将所有内容打印到out.log文件中，看一下caffe.cpp文件的执行过程：

CXX tools/caffe.cpp
g++ tools/caffe.cpp -MMD -MP -pthread -fPIC -DDEBUG -g -O0 -DUSE_OPENCV -DUSE_LEVELDB -DUSE_LMDB -I/usr/include/python2.7 -I/usr/lib/python2.7/dist-packages/numpy/core/include -I/usr/local/include -I.build_debug/src -I./src -I./include -I/usr/local/cuda-7.5/include -Wall -Wno-sign-compare -c -o .build_debug/tools/caffe.o 2> .build_debug/tools/caffe.o.warnings.txt \
		|| (cat .build_debug/tools/caffe.o.warnings.txt; exit 1)
CXX/LD -o .build_debug/tools/caffe.bin
g++ .build_debug/tools/caffe.o -o .build_debug/tools/caffe.bin -pthread -fPIC -DDEBUG -g -O0 -DUSE_OPENCV -DUSE_LEVELDB -DUSE_LMDB -I/usr/include/python2.7 -I/usr/lib/python2.7/dist-packages/numpy/core/include -I/usr/local/include -I.build_debug/src -I./src -I./include -I/usr/local/cuda-7.5/include -Wall -Wno-sign-compare -lcaffe -L/home/sindyz/anaconda/lib -L/usr/local/lib -L/usr/lib -L/usr/local/cuda-7.5/lib64 -L/usr/local/cuda-7.5/lib -L.build_debug/lib  -lcudart -lcublas -lcurand -lglog -lgflags -lprotobuf -lboost_system -lboost_filesystem -lm -lhdf5_hl -lhdf5 -lleveldb -lsnappy -llmdb -lopencv_core -lopencv_highgui -lopencv_imgproc -lboost_thread -lstdc++ -lcblas -latlas \
		-Wl,-rpath,\$ORIGIN/../lib

这里，第一个g++后面的内容，是从.cpp文件生成.o文件的过程，可以看到相应的链接库和配置信息

第二个g++后面的内容是.o文件到.bin的生成过程。

由此，我们可以自己写caffe的代码并且配置Makefile了。

初学caffe，由一些地方说明的不清楚的不对的，希望大家一起讨论，共同进步。