1. 模板配置
- 创建公共仓库,如
ci-config
- 修改目标仓库配置,填写模板文件地址
ci-config/test.yml
当前仓库流水线配置文件实际位置(可跨仓库)
- 流水线已定义变量 https://gitlab.cn/docs/jh/ci/variables/predefined_variables.html
stages:
- test
variables:
#dind config disable TLS
DOCKER_TLS_CERTDIR: "" # 禁用docker
Filename: $Filename
# Use workflow to ensure pipeline only runs for manual or specific conditions
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "web" # 手动触发
- if: $CI_PIPELINE_SOURCE == "schedule" # 定时触发
- if: $CI_PIPELINE_SOURCE == "api" # api触发
- when: never # Avoid running on commits by default
test_case:
stage: test
rules:
- if: '$Filename && $Filename != ""' # 触发条件
when: always
- when: never
trigger: # 触发子流水线
include: '$Filename' # 触发子流水线
strategy: depend # 根据子流水线返回状态来确定父流水线状态
2. 常用.gitlab-ci.yml
可设置中文
image: python:latest
cache:
paths:
- deps_cache
- venv/
before_script:
- echo -e "准备虚拟环境"
- python --version
- python -m venv venv
- source venv/bin/activate
# - pip install -r test-requirements.txt --cache-dir deps_cache
- pip install --cache-dir deps_cache mypy pylint pytest-cov
stages:
- 合规静态检查
- 执行测试
- 生成报告
- 部署
静态检查:
stage: 合规静态检查
# only:
# - master
# - merge_requests
allow_failure: true
script:
- echo "开始静态分析pylint"
- pylint --fail-under=8 auto-data-api
mypy:
stage: 合规静态检查
# only: # 执行条件
# - master # 主分支
# - merge_requests # 合并请求
allow_failure: true
script:
- echo "开始静态分析mypy"
- mypy --ignore-missing-imports auto-data-api
测试过程:
stage: 执行测试
# only:
# - master
# - merge_requests
script:
- echo "开始测试"
- cd auto-data-api
- echo 4 | python runner.py
- ls -la
coverage: '/^TOTAL.+?(\d+\%)$/'
artifacts:
paths:
- auto-data-api/result
- auto-data-api/.coverage
- auto-data-api/xml
- auto-data-api/lib
when: always
allow_failure: true # 如果静态检查失败,不影响后续阶段
报告:
stage: 生成报告
dependencies:
- 测试过程
script:
- cd auto-data-api/
- ls -la xml
- python mearge_xml.py
- ls
- cat merged_report.xml
- pwd
artifacts:
paths:
- $CI_PROJECT_DIR/auto-data-api/merged_report.xml
when: always
reports:
junit: $CI_PROJECT_DIR/auto-data-api/merged_report.xml
allow_failure: true # 如果静态检查失败,不影响后续阶段
pages: # 发布配置页面
dependencies:
- 测试过程
stage: 部署
script:
- mkdir .public
- cp -r auto-data-api/result .public
- rm -rf public
- mv .public public
artifacts:
paths:
- public
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- 测试报告 覆盖率
# 测试报告
test_and_report:
stage: report
script:
- pip install -r requirements.txt # 假设你的项目依赖在requirements.txt中
- mkdir reports
- pytest --cov=. --alluredir=reports/allure --junitxml=reports/junitxml.xml # 执行测试并生成覆盖率和allure报告
# - coverage report -m ; coverage html ; coverage xml # 生成覆盖率报告 \
# - echo "复制报告"
# - cp coverage.* reports
# - cp -r reports/* public
# - cp -r htmlcov public/htmlcov
coverage: '/^TOTAL.+?(\d+\%)$/' # 覆盖率展示
artifacts: # 缓存工件够后续使用
paths:
- reports/* # 保存allure报告供后续阶段使用
# - coverage.xml
# - htmlcov/*
- .coverage
when: always
reports:
junit: reports/junitxml.xml
requirements: requirements.txt
allow_failure: true # 如果静态检查失败,不影响后续阶段
2.1 父子流水线触发
可一直嵌套
# 父
next-job:
stage: deploy
trigger:
include:
- project: 'study/tmp_test/tmp002'
# ref: 'master'
file: '.gitlab-ci.yml'
strategy: depend #根据子流水线返回状态来确定父流水线状态
2.2 需手动或延迟
stages:
- build
- test
- deploy
- manual_approval # 手动确认阶段
- post_deploy # 手动确认后的后续阶段
# 构建阶段
build_project:
stage: build
script:
- echo "Building project..."
# - # 执行构建项目的命令
# 测试阶段
run_tests:
stage: test
script:
- echo "Running tests..."
# - # 执行测试命令
# when: manual
# 部署准备阶段(可选,根据实际需求添加)
prepare_deploy:
stage: deploy
script:
- echo "Preparing deployment..."
- echo "$token"
# - # 执行部署前的准备工作
# only:
# - master # 或其他你希望触发的分支
# 手动确认阶段
manual_approval:
stage: manual_approval
script:
- echo "Waiting for manual approval..."
when: manual # 指定此阶段为手动执行
allow_failure: false # 不允许此阶段失败(可选)
# only:
# - master # 或其他你希望触发的分支
# 手动确认后的部署阶段
deploy_app:
stage: post_deploy
script:
- echo "Deploying application..."
# - # 执行部署命令
# only:
# - master # 确保只有在手动确认阶段通过后,此阶段才会执行
needs: [ "manual_approval" ] # 确保此阶段在 manual_approval 阶段之后执行(GitLab 13.9+ 支持)
# 后续操作(可选,根据实际需求添加)
post_deploy_tasks:
stage: post_deploy
script:
- echo "Running post-deploy tasks..."
# - # 执行部署后的清理或其他任务
# only:
# - master
needs: [ "deploy_app" ] # 确保此阶段在 deploy_app 阶段之后执行(GitLab 13.9+ 支持)
2.3 根据条件判断是否执行 job2
image: python:latest
stages:
- build
- test
- deploy
build-job:
stage: build
script:
- echo 'aaa'
test-job1:
stage: test
script:
- echo 'aaa'
test-job2:
stage: test
script:
- echo 'aaa'
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # 如果是默认分支,则执行此任务
deploy-job:
stage: deploy
script:
- echo 'aaa'
needs:
- job: test-job2
optional: true # 如果 test-job2 不通过,则跳过此任务
- job: test-job1
review-job:
stage: deploy
script:
- echo 'aaa'
needs:
- job: test-job2
optional: true # 如果 test-job2 不通过,则跳过此任务
environment: review
2.4 漏扫配置
default:
image:
name: ${docker_image}
entrypoint: [ "" ]
include:
- template: Jobs/Code-Quality.gitlab-ci.yml
- template: Security/SAST.gitlab-ci.yml
- template: Security/Secret-Detection.gitlab-ci.yml
- template: Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/License-Scanning.gitlab-ci.yml
stages:
- cleanup
- test
- sec_check
- build_pkg_info
- upload_pkg_info
clean_work_dir:
stage: cleanup
script:
- rm -fr $CI_PROJECT_DIR
tags:
- shell04
rules:
- when: on_success
code_quality:
stage: test
variables:
SECURE_LOG_LEVEL: "debug"
TIMEOUT_SECONDS: 7200
tags:
- shell23
before_script:
- |
if [[ ! -f ".codeclimate.yml" ]] ;then
echo "========================================================================================================================"
echo "代码质量扫描配置文件.codeclimate.yml不存在,请在仓库根目录下提交该文件"
echo "codeclimate配置可参考官网:https://docs.codeclimate.com/docs/default-analysis-configuration#sample-codeclimateyml"
echo "========================================================================================================================"
exit 1
fi
rules:
- when: on_success
allow_failure: false
gemnasium-dependency_scanning:
stage: test
variables:
SECURE_LOG_LEVEL: "debug"
GEMNASIUM_DB_REMOTE_URL: "$GITLAB_URL/pubgroup/gitlab-org/security-products/gemnasium-db.git"
tags:
- docker14
rules:
- when: on_success
semgrep-sast:
stage: test
variables:
SECURE_LOG_LEVEL: "debug"
tags:
- docker23
rules:
- when: on_success
flawfinder-sast:
stage: test
variables:
SECURE_LOG_LEVEL: "debug"
tags:
- docker04
rules:
- when: on_success
secret_detection:
stage: test
variables:
SECURE_LOG_LEVEL: "debug"
tags:
- docker24
rules:
- when: on_success
vul_stat:
stage: sec_check
variables:
SECURE_LOG_LEVEL: "debug"
script:
- |
api_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/vulnerability_findings?pipeline_id=${CI_PIPELINE_ID}&page=1&per_page=100&severity=critical"
# 发送API请求获取漏洞信息,并过滤出包含关键词"critical"的行
api_response=$(curl -s -H "PRIVATE-TOKEN: ${PRIVATETOKEN}" "$api_url")
has_critical_bugs=$(echo "$api_response" | grep -q "critical" && echo "true" || echo "false")
if [ "$has_critical_bugs" = "true" ]; then
# 统计严重漏洞个数
critical_bugs=$(echo "$api_response" | jq '.' | grep "critical" | wc -l)
# 判断严重漏洞个数是否超过阈值
# if [ "$critical_bugs" -gt 10 ]; then
# echo -e "严重漏洞个数为${critical_bugs},大于10,流水线终止,请先修复漏洞,严重漏洞个数为0后,方可继续"
# exit 1
# fi
echo "========================================================================================================================"
echo -e "严重漏洞个数为${critical_bugs},流水线终止,请先修复漏洞,严重漏洞个数为0后,方可继续"
echo "========================================================================================================================"
exit 1
else
echo -e "未检测到严重漏洞! 继续~"
fi
tags:
- shell13
rules:
- when: on_success
build_dev:
stage: build_pkg_info
needs: [ "vul_stat" ]
script:
- |
# install dependency
if [ -f package/depends ];then
rm -rf /var/lib/pkg_info/info/*
cat $dev_pkg_info_conf > /var/lib/pkg_info/pkg_info.conf
pkg_info update
for i in $(cat package/depends); do pkg_info uninstall $i || true ; pkg_info install $i ; done
fi
bash -x build.sh
i=1
for d in *.pkg_info; do
echo "pkg_info${i}=$d" >> pkg_info.env
i=$((i + 1))
done
artifacts:
paths:
- $CI_PROJECT_DIR/*.pkg_info
- $CI_PROJECT_DIR/pkg_info.env
expire_in: 1 week
tags:
- docker05
rules:
- when: on_success
upload_dev:
stage: upload_pkg_info
needs: [ "build_dev" ]
variables:
SECURE_LOG_LEVEL: "debug"
script:
- |
cat $dev_pkg_info_conf > /var/lib/pkg_info/pkg_info.conf
for pkg_info_name in $(cat pkg_info.env | awk -F'=' '{print $2}')
do
UpResult=$(pkg_info upload ${pkg_info_name})
set +e
echo ${UpResult} | grep 'upload is completed'
if [ $? -ne 0 ];then
echo "上传失败,可能原因为${pkg_info_name}在dev环境中已存在,请检查后重试!"
exit 127
fi
set -e
done
tags:
- docker05
rules:
- when: on_success