import os
import datetime
import shutil
import subprocess
import re
import platform
import hashlib
def get_commit_id():
cmd = ['git', 'rev-parse', 'HEAD']
try:
result = subprocess.check_output(cmd)
commit_id = result.decode().strip()
if not commit_id:
raise Exception('commit id not found')
return commit_id
except Exception as e:
print(e)
raise e
def remove_pycache(path):
for root, dirs, files in os.walk(path):
for dir in dirs:
if dir == '__pycache__':
pycache_dir = os.path.join(root, dir)
shutil.rmtree(pycache_dir)
def use_shell():
if platform.system() == "Linux":
return False
return True
class SdkPacker:
def __init__(self):
self.starttime = datetime.datetime.now()
self.record('*' * 10 + 'SDK pack init' + '*' * 10)
# pip
self.pipUrl = os.getenv('SDK_PIP_URL')
self.pipTrustHost = os.getenv('SDK_PIP_TRUST_HOST')
self.pipArgs = os.getenv('SDK_PIP_ARGS')
self.record(f'pipUrl: {self.pipUrl}')
self.record(f'pipTrustHost: {self.pipTrustHost}')
self.record(f'pipArgs: {self.pipArgs}')
# sdk path
self.sdkPath = os.path.dirname(os.path.abspath(__file__))
self.outPath = os.path.join(self.sdkPath, 'out')
self.enginePath = os.path.join(self.outPath, 'engine')
self.remove_path(os.path.join(self.sdkPath, 'build'))
self.remove_path(os.path.join(self.sdkPath, 'rpa', 'build'))
self.remove_path(self.enginePath)
# commit id
self.commitId = get_commit_id()
self.record(f"commit id：{self.commitId}")
# cache path
self.cachePath = os.path.join(self.sdkPath, 'out', 'cache')
self.cacheZipPath = os.path.join(self.cachePath, f'{self.commitId}.7z')
# env
self.RPA_PACK_PLATFORM = self.get_env('RPA_PACK_PLATFORM')
self.RPA_PACK_ARCH = self.get_env('RPA_PACK_ARCH')
self.RPA_VERSION = self.get_env('RPA_PACK_VERSION')
if not self.RPA_VERSION or not re.search(r"\d", self.RPA_VERSION):
self.RPA_VERSION = "15.0.0"
self.RPA_FORCE_REBUILD = self.get_env('RPA_PACK_FORCE_REBUILD')
self.platform = platform.system()
self.record(f"System: {self.platform}")
# tools path
self.sdkToolsPath = os.path.join(self.sdkPath, 'out', 'sdk_tools')
# output path
self.python_out = os.path.join(self.enginePath)
if self.RPA_PACK_PLATFORM == 'windows':
self.reqsPath = os.path.join(self.sdkPath, 'rpa', 'requirements.txt')
self.site_packages = os.path.join(self.enginePath, 'Lib', 'site-packages')
elif self.RPA_PACK_PLATFORM in ['linux', 'UOS', 'kylinOS']:
self.reqsPath = os.path.join(self.sdkPath, 'rpa', 'requirements_uos.txt')
self.site_packages = os.path.join(self.enginePath, 'lib', 'python3.7', 'site-packages')
else:
raise Exception(f'not support platform: {self.RPA_PACK_PLATFORM} and arch: {self.RPA_PACK_ARCH}')
self.seven_zip_out = os.path.join(self.site_packages, 'rpa', 'file_folder')
self.ffmpeg_out = os.path.join(self.site_packages, 'rpa', 'win32')
self.db2_out = os.path.join(self.site_packages)
self.db2_cli_out = os.path.join(self.site_packages, 'ibm_db-3.1.4')
self.pip_args = []
if self.pipUrl:
self.pip_args.extend(['-i', self.pipUrl])
if self.pipTrustHost:
self.pip_args.extend(['--trusted-host', self.pipTrustHost])
if self.pipArgs:
self.pip_args.extend(self.pipArgs.split(','))
# self.pip_args.extend(['--no-cache-dir', '--no-warn-script-location'])
self.record("sdk pack init end")
def run_command(self, command, cwd=None):
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd,
shell=use_shell())
while True:
output = process.stdout.readline()
if output == b'' and process.poll() is not None:
break
if output:
print(output.strip().decode('utf-8', errors='replace'))
process.wait()
if process.returncode is not 0:
raise Exception(f'run command {command} error, return code: {process.returncode}')
self.record(f"run command: {command}")
def get_env(self, env_key):
env_value = os.getenv(env_key)
if env_value:
self.record(f'{env_key}: {env_value}')
else:
raise Exception(f'{env_key} not found')
return env_value
def remove_path(self, path):
if os.path.exists(path):
try:
if os.path.isfile(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
self.record(f"remove {path}: successfully")
except Exception as e:
self.record(f'remove {path}: {e} error')
raise e
else:
self.record(f"remove {path}: not exists")
def record(self, title):
end_time = datetime.datetime.now()
diff = (end_time - self.starttime)
print(f"[{end_time.time()} - {diff.seconds}] {title}")
def unzip(self, src, dst):
# self.run_command(['7z', 'x', src, '-o' + dst, '-bb0'], cwd=os.path.join(self.sdkToolsPath, '7z'))
os.system(f"7z x {src} -o{dst}")
self.record(f"unzip {src} to {dict}")
def calculate_md5(self, file_path):
with open(file_path, "rb") as f:
md5_hash = hashlib.md5()
for chunk in iter(lambda: f.read(4096), b""):
md5_hash.update(chunk)
return md5_hash.hexdigest()
def copy(self, package_name, *rpa_dir):
package = os.path.join(self.sdkToolsPath, package_name)
package_out = os.path.join(self.site_packages, 'rpa', *rpa_dir)
for file in os.listdir(package):
package_file = os.path.join(package, file)
shutil.copy(package_file, package_out)
self.record(f"{package_file} >> {package_out}")
self.record(f"copy {package_name}")
def pack(self):
# encrypt sdk
self.record('*' * 10 + 'SDK pack' + '*' * 10)
if self.RPA_FORCE_REBUILD == 'false' and os.path.exists(self.cacheZipPath):
self.record('SDK use cache')
self.unzip(self.cacheZipPath, self.outPath)
else:
self.encrypt_sdk()
# add version
version_path = os.path.join(self.site_packages, 'rpa', 'version', 'version')
content = f'{self.RPA_VERSION}\n{datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")}\n{self.commitId}'
with open(version_path, 'w') as f:
f.write(content)
shutil.copy(version_path, self.enginePath)
with open(os.path.join(self.enginePath, 'version'), 'r') as f:
self.record(f.read())
# remove cache
remove_pycache(self.enginePath)
self.record(f"remove engine pycache: {self.enginePath}")
self.record("SDK pack end")
def link_python(self):
pass
def is_linux(self):
return self.RPA_PACK_PLATFORM in ['linux', 'UOS', 'kylinOS']
def is_windows(self):
return self.RPA_PACK_PLATFORM == 'windows'
def encrypt_sdk(self):
def fix_tk():
# remove mouseinfo tk sys.exit
mouseinfo_init_path = os.path.join(self.site_packages, 'mouseinfo', '__init__.py')
command = """sed -i "s/sys.exit.*/pass/g" """ + mouseinfo_init_path
ret = os.system(command)
msg = f"remove mouseinfo tk sys.exit code: {ret}"
if ret:
raise SystemError(msg)
self.record(msg)
def install_pywpsrpc():
wheel_path = 'pywpsrpc-2.3.3-cp37-cp37m-manylinux_2_5_x86_64.whl'
if self.RPA_PACK_ARCH == "arm64":
wheel_path = 'pywpsrpc-2.3.3-cp37-cp37m-manylinux_2_28_aarch64.whl'
pywpsrpc_path = os.path.join(self.sdkToolsPath, 'pywpsrpc', wheel_path)
self.run_command([python_path, '-m', 'pip', 'install', pywpsrpc_path])
def copy_depends():
shutil.copytree(os.path.join(self.sdkToolsPath, 'deps', 'at-spi2-core'), os.path.join(self.enginePath, 'deps', 'at-spi2-core'))
shutil.copytree(os.path.join(self.sdkToolsPath, 'deps', 'wps'), os.path.join(self.enginePath, 'deps', 'wps'))
shutil.copytree(os.path.join(self.sdkToolsPath, 'deps', 'xclip'), os.path.join(self.site_packages, 'xclip'))
# move python
self.record('SDK encrypt')
use_cache = False
requirements_md5 = self.calculate_md5(self.reqsPath)
requirements_cache_path = os.path.join(self.sdkPath, 'out', 'cache', f'{requirements_md5}.7z')
if self.RPA_FORCE_REBUILD == 'false':
if os.path.exists(requirements_cache_path):
use_cache = True
python_source = ""
python_path = ""
if self.is_windows():
python_source = os.path.join(self.sdkToolsPath, 'python')
python_path = os.path.join(self.enginePath, "python.exe")
elif self.is_linux():
python_source = "/opt/python3.7"
python_path = os.path.join(self.enginePath, "bin", "python")
if not use_cache:
shutil.copytree(python_source, self.enginePath)
self.record(f"{python_source} >> {self.enginePath}")
if self.is_linux():
os.system(f'apt-get install -y libcairo2-dev libgirepository1.0-dev unixodbc-dev')
current_cwd = os.getcwd()
self.record(f"current cwd:{current_cwd}")
bin_path = os.path.join(self.enginePath, "bin")
os.chdir(bin_path)
os.system(f'ln -s python3.7 python')
os.system(f'ln -s python3.7 python3')
self.record("link python3.7 to python python3")
os.chdir(current_cwd)
# install requirements
# comtypes<1.1.11 need 2to3, setuptools<58 support 2to3
self.run_command([python_path, '-m', 'pip', 'install', '--upgrade', 'pip'] + self.pip_args)
self.run_command([python_path, '-m', 'pip', 'install', '--upgrade', 'setuptools < 58'] + self.pip_args)
self.run_command([python_path, '-m', 'pip', 'install', '-r', self.reqsPath] + self.pip_args)
if self.is_windows():
# install db2
shutil.copytree(os.path.join(self.sdkToolsPath, 'db2', 'ibm_db-3.1.4'),
os.path.join(self.enginePath, 'Lib', 'site-packages', 'ibm_db-3.1.4'))
shutil.copytree(os.path.join(self.sdkToolsPath, 'db2_cli', 'clidriver'),
os.path.join(self.enginePath, 'Lib', 'site-packages', 'ibm_db-3.1.4', 'clidriver'))
self.run_command([os.path.join(self.enginePath, 'python'), 'setup.py', 'install'],
cwd=os.path.join(self.enginePath, 'Lib', 'site-packages', 'ibm_db-3.1.4'))
elif self.is_linux():
# install db2
# shutil.copytree(os.path.join(self.sdkToolsPath, 'db2', 'ibm_db-3.1.4'),
# os.path.join(self.site_packages, 'ibm_db-3.1.4'))
# shutil.copytree(os.path.join(self.sdkToolsPath, 'db2_cli', 'clidriver'),
# os.path.join(self.site_packages, 'ibm_db-3.1.4', 'clidriver'))
# self.run_command([python_path, 'setup.py', 'install'],
# cwd=os.path.join(self.site_packages, 'ibm_db-3.1.4'))
fix_tk()
install_pywpsrpc()
copy_depends()
# install cython
self.run_command([python_path, '-m', 'pip', 'install', 'cython==0.29.24'] + self.pip_args)
self.remove_path(requirements_cache_path)
self.run_command(['7z', 'a', '-mx1', requirements_cache_path, self.enginePath], cwd=os.path.join(self.sdkToolsPath, '7z'))
else:
self.record("requirements use cache")
self.unzip(requirements_cache_path, self.outPath)
build_path = os.path.join(self.sdkPath, 'build', 'rpa')
# encrypt sdk
self.run_command([python_path, 'setup.py'], cwd=os.path.join(self.sdkPath, 'rpa'))
# uninstall cython
self.run_command([python_path, '-m', 'pip', 'uninstall', 'cython', '-y'])
# remove pycache
remove_pycache(build_path)
self.record(f"remove rpa pycache: {build_path}")
# copy sdk
rpa_path = os.path.join(self.site_packages, 'rpa')
shutil.move(build_path, rpa_path)
self.record(f"move {build_path} >> {rpa_path}")
if self.RPA_PACK_PLATFORM == 'windows':
self.copy('activexinput', 'uia', 'activexinput')
self.copy("7z", "file_folder")
self.copy("ffmpeg", "win32")
# save cache
self.remove_path(self.cacheZipPath)
self.run_command(['7z', 'a', '-mx1', self.cacheZipPath, self.enginePath], cwd=os.path.join(self.sdkToolsPath, '7z'))
# self.run_command(['7z', 'a', '-tzip', self.cacheZipPath, '-r', self.enginePath, '-y', '-bb0'],
# cwd=os.path.join(self.sdkToolsPath, '7z'))
# remove paths
self.remove_path(os.path.join(self.sdkPath, 'build'))
self.remove_path(build_path)
self.record("SDK encrypt end")
if __name__ == '__main__':
import sys
if sys.platform == "win32":
# for tests
os.environ['RPA_PACK_PLATFORM'] = 'windows'
os.environ['RPA_PACK_ARCH'] = 'x64'
os.environ['RPA_TARGET_FORMAT'] = 'zip'
os.environ['RPA_PACK_GITOKEN'] = 'pack_gitoken'
os.environ['RPA_VERSION'] = '1.0.0'
os.environ['RPA_GIT_TOKEN'] = 'git_token'
os.environ['RPA_FORCE_REBUILD'] = 'false'
os.environ['RPA_TOOLS_HOME'] = 'C:\\Repos\\tools'
elif sys.platform == "darwin":
sys.exit(0)
else:
os.environ['RPA_PACK_PLATFORM'] = 'linux'
os.environ['RPA_PACK_ARCH'] = 'x64'
os.environ['RPA_TARGET_FORMAT'] = 'deb'
os.environ['RPA_PACK_GITOKEN'] = 'pack_gitoken'
os.environ['RPA_VERSION'] = '1.0.0'
os.environ['RPA_GIT_TOKEN'] = 'git_token'
os.environ['RPA_FORCE_REBUILD'] = 'false'
os.environ['RPA_TOOLS_HOME'] = '/home/uos/tools'
os.environ['SDK_PIP_URL'] = 'https://repo.datagrand.com/repository/py/simple'
packer = SdkPacker()
packer.pack()
分解一下项目打包的流程，介绍如何实现打包，如何配置环境，依赖等等，以及是否实现可执行文件，将整个流程用图表表示
最新发布
本文档提供了IBM DB2数据库管理系统中各种管理命令的详细列表，并指导如何在Linux环境中卸载DB2软件。包括创建、迁移、更新DB2管理服务器及实例的操作指令。
扫一扫
4033
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
