# fog.bbclass, add new fake fetcher fog (means file or git)
# fog://[src repo path];[params same as git fetcher];prebuilt=[prebuilt repo path];licrepo=[src/prebuilt/all]
#
# fog will be tranformed to file:// or git:// by below rules:
# 1. if [src repo path] is existed on local, using local code directly by file://,ignore prebuilt & SRCREV
# 2. else if prebuilt param is set and [prebuilt repo path] is existed on local, using local prebuilt directly by file://, igonre src & SRCREV
# 3. else if no prebuilt param and [src repo path] is existed in manifest, using corresponding repo name in manifest, add left params to git://
# 3.1 if manifest include revisionId, set it to SRCREV and ignore SRCREV in recipe
# 3.2 if manifest not include revisionId but include revision branch, add it to git:// param
# 4. else if prebuilt param is set and [prebuilt repo path] is existed in manifest, using corresponding repo name in manifest, add left params to git://
# 4.1 if manifest include revisionId, set it to SRCREV and ignore SRCREV in recipe
# 4.2 if manifest not include revisionId but include revision branch, add it to git:// param
# 5. else construct git uri git://[CODE_SERVER_PREFIX][src/prebuilt repo path]
# 5.1 branch using CODE_BRANCH, SRCREV using recipe setting, git params append without modification
#
# if use prebuilt path, OVERRIDE "prebuilt" will be set
# if one fog uri prebuilt is "none", this fog uri will be SKIPED if no src repo can be found
# if repo is license, please set "licrepo=src", "licrepo=prebuilt" or "licrepo=all"
#
# FOG_FORCE_GIT : force to use git fetcher and ignore local source code repo
# FOG_KEEP_DOTGIT : .git is not copied default, set to "1" will copy .git folder to workdir
#
DEPENDS_append = " rsync-native"
do_unpack[depends] += "rsync-native:do_populate_sysroot"
CODE_SERVER_PREFIX ?= ""
CODE_BRANCH ?= ""
SOURCE_INFO_FOLDER = "${TMPDIR}/source-info"
do_unpack[dirs] += "${SOURCE_INFO_FOLDER}"
def find_root_dir(d):
if not hasattr(find_root_dir, "rootdir"):
current = d.getVar("FILE", True)
while True:
if current == "/":
find_root_dir.rootdir = "/NOTEXISTED"
break
current = os.path.dirname(current)
if os.path.exists(os.path.join(current, ".repo")):
find_root_dir.rootdir = current
break
return find_root_dir.rootdir
def get_manifest(d):
try:
if not hasattr(get_manifest, "manifest"):
import sys
import importlib
rootdir = find_root_dir(d)
sys.path.insert(0,os.path.join(rootdir, ".repo/repo"))
manifest_xml = importlib.import_module("manifest_xml")
try:
get_manifest.manifest = manifest_xml.XmlManifest(
os.path.join(rootdir, ".repo"), os.path.join(rootdir, ".repo/manifest.xml"))
except:
get_manifest.manifest = manifest_xml.XmlManifest(
os.path.join(rootdir, ".repo"))
return get_manifest.manifest
except:
return None
def parse_srcuri(perms):
items = {}
for perm in perms:
if perm == "":
continue
k,v = perm.split("=")
items[k.strip()] = v.strip()
return items
def check_connection(uri, d):
try:
fetcher = bb.fetch2.Fetch([uri], d)
fetcher.checkstatus()
print(f"check_connection {uri} success!")
return True
except Exception as err:
print(f"check_connection {uri} fail!")
return False
def get_target_platform(d):
target_platform = d.getVar('TARGET_PLATFORM', True)
if target_platform is not None:
return target_platform
target_platform = d.getVar('BACKUP_TARGET_PLATFORM', True)
if target_platform is not None:
return target_platform
bb.fatal(f'[fog]Could not get target platform information')
def get_annotation_value(manifest, path, name):
for annotation in manifest.paths[path].annotations:
if annotation.name == name:
return annotation.value
return ""
def fog_santity_check(uri, d):
perms = uri.split(";")
perms_dict = parse_srcuri(perms[1:])
src_path = perms[0].replace("fog://","")
platform_src_path = src_path.replace("prebuilt",f"prebuilt/{get_target_platform(d)}") if src_path.startswith("prebuilt/") else src_path
prebuilt_path = perms_dict["prebuilt"] if "prebuilt" in perms_dict else "NOTEXISTED"
platform_prebuilt_path = prebuilt_path.replace("prebuilt",f"prebuilt/{get_target_platform(d)}")
if "name" not in perms_dict:
bb.fatal(f'[fog]{uri} not has "name" param, please set name for each fog uri')
if d.getVar(f'SRCREV_{perms_dict["name"]}') is None:
bb.fatal(f'[fog][{perms_dict["name"]} not has SRCREV_{perms_dict["name"]}, please set SRCREV_{perms_dict["name"]} for {perms_dict["name"]}')
if src_path.startswith("/"):
bb.fatal(f'[fog]absolute path is forbidden used in fog fetcher, please use path relative to codebase root')
manifest = get_manifest(d)
for path in [src_path, platform_src_path]:
if path in manifest.paths and "prebuilt" not in perms_dict:
for annotation in manifest.paths[path].annotations:
if get_annotation_value(manifest, path, "RELEASE_POLICY") != "source":
bb.fatal(f"[fog]{path} is {annotation.value} release, please set prebuilt parameter")
break
src_package_group = ""
prebuilt_package_group = ""
src_repo_existed = False
prebuilt_repo_existed = False
for path in [src_path, platform_src_path]:
if path in manifest.paths:
src_repo_existed = True
src_package_group = get_annotation_value(manifest, path, "PACKAGE_GROUP")
if src_package_group:
break
for path in [prebuilt_path, platform_prebuilt_path]:
if path in manifest.paths:
prebuilt_repo_existed = True
prebuilt_package_group = get_annotation_value(manifest, path, "PACKAGE_GROUP")
if prebuilt_package_group:
break
licrepo = None
if src_package_group.startswith("license.") and prebuilt_package_group.startswith("license."):
licrepo = "all"
if src_package_group.startswith("license.") and not prebuilt_package_group.startswith("license."):
licrepo = "src"
if not src_package_group.startswith("license.") and prebuilt_package_group.startswith("license."):
licrepo = "prebuilt"
if licrepo and "licrepo" not in perms_dict:
bb.fatal(f"[fog]{uri} includes license repo, please set 'licrepo' in fog paramter")
if licrepo and licrepo != perms_dict["licrepo"]:
if perms_dict["licrepo"] == "all" and licrepo == "src" and not prebuilt_repo_existed:
pass
elif perms_dict["licrepo"] == "all" and licrepo == "prebuilt" and not src_repo_existed:
pass
else:
bb.fatal(f"[fog]{uri} includes license repo, plase set 'licrepo' in fog paramter")
if not licrepo and src_package_group and (licrepo == "all" or licrepo == "src"):
bb.fatal(f"[fog]{uri} does not include license src repo, please do not set 'licrepo=src' or 'licrepo=all' in fog paramter")
if not licrepo and prebuilt_package_group and (licrepo == "all" or licrepo == "prebuilt"):
bb.fatal(f"[fog]{uri} does not include license prebuilt repo, please do not set 'licrepo=prebuilt' or 'licrepo=all' in fog paramter")
def fog_get_file_fetcher(uri, d):
if d.getVar('FOG_FORCE_GIT', True) == '1':
return (None, None, None)
rootdir = find_root_dir(d)
perms = uri.split(";")
perms_dict = parse_srcuri(perms[1:])
src_path = perms[0].replace("fog://","")
platform_src_path = src_path.replace("prebuilt",f"prebuilt/{get_target_platform(d)}") if src_path.startswith("prebuilt/") else src_path
prebuilt_path = perms_dict["prebuilt"] if "prebuilt" in perms_dict else "NOTEXISTED"
platform_prebuilt_path = prebuilt_path.replace("prebuilt",f"prebuilt/{get_target_platform(d)}")
for path in [src_path, platform_src_path, prebuilt_path, platform_prebuilt_path]:
full_path = os.path.join(rootdir, path)
if not os.path.exists(full_path):
continue
if not os.path.exists(os.path.join(full_path,".git")):
bb.fatal(f"[fog]{repo} is existed but not a git repo, please check it")
d.setVar("FILESEXTRAPATHS_prepend", rootdir + ":")
print(f"[fog]from {uri} to file://{path}")
return (f"file://{path}", perms_dict, path.startswith("prebuilt"))
return (None, None, None)
def fog_get_manifest_git_fetcher(uri, d):
manifest = get_manifest(d)
perms = uri.split(";")
perms_dict = parse_srcuri(perms[1:])
src_path = perms[0].replace("fog://","")
platform_src_path = src_path.replace("prebuilt",f"prebuilt/{get_target_platform(d)}") if src_path.startswith("prebuilt/") else src_path
prebuilt_path = perms_dict["prebuilt"] if "prebuilt" in perms_dict else "NOTEXISTED"
platform_prebuilt_path = prebuilt_path.replace("prebuilt",f"prebuilt/{get_target_platform(d)}")
for path in [src_path, platform_src_path, prebuilt_path, platform_prebuilt_path]:
if path in manifest.paths:
url = manifest.paths[path].remote.url
if manifest.paths[path].revisionId is None:
branch = manifest.paths[path].revisionExpr.replace("refs/heads/","")
branch_str = f"branch={branch}"
else:
branch_str = "nobranch=1"
d.setVar(f'SRCREV_{perms_dict["name"]}',manifest.paths[path].revisionId)
protocol, baseurl = url.split("://")
if not baseurl.endswith("/"):
baseurl += "/"
url = f"git://{baseurl};protocol={protocol};{branch_str};"+";".join(perms[1:])
if check_connection(url, d):
print(f"[fog]from {uri} to {url}")
d.setVar("__BBSEENSRCREV", "1")
d.setVar("BB_NO_NETWORK", "0")
d.setVar("BB_FETCH_PREMIRRORONLY", "0")
return (url, perms_dict, path.startswith("prebuilt"))
return (None, None, None)
def fog_get_remote_git_fetcher(uri, d):
manifest = get_manifest(d)
perms = uri.split(";")
perms_dict = parse_srcuri(perms[1:])
src_path = perms[0].replace("fog://","")
platform_src_path = src_path.replace("prebuilt",f"prebuilt/{get_target_platform(d)}") if src_path.startswith("prebuilt/") else src_path
prebuilt_path = perms_dict["prebuilt"] if "prebuilt" in perms_dict else "NOTEXISTED"
platform_prebuilt_path = prebuilt_path.replace("prebuilt",f"prebuilt/{get_target_platform(d)}")
code_server_prefix = d.getVar("CODE_SERVER_PREFIX",True)
if code_server_prefix == "": # guess current server prefix
print("CODE_SERVER_PREFIX is not set, skip remote git fetcher!")
return (None, None, None)
if not code_server_prefix.endswith("/"):
code_server_prefix += "/"
branch = d.getVar("CODE_BRANCH",True)
if branch is None:
branch = d.getVar("BACKUP_CODE_BRANCH",True)
if branch == "": # guess curret branch
branch = manifest.default.revisionExpr.replace("refs/heads/","")
branch_str = f"branch={branch}"
licrepo = perms_dict["licrepo"] if "licrepo" in perms_dict else None
for path in [src_path, platform_src_path, prebuilt_path, platform_prebuilt_path]:
origin_code_server_prefix = code_server_prefix
if licrepo == "all" or (licrepo == "src" and (path in [src_path, platform_src_path])) or (licrepo == "prebuilt" and (path == prebuilt_path or path == platform_prebuilt_path)):
code_server_prefix += "license/"
url = code_server_prefix + path
protocol, baseurl = url.split("://")
if not baseurl.endswith("/"):
baseurl += "/"
url = f"git://{baseurl};protocol={protocol};{branch_str};"+";".join(perms[1:])
if check_connection(url, d):
print(f"[fog]from {uri} to {url}")
d.setVar("__BBSEENSRCREV", "1")
d.setVar("BB_NO_NETWORK", "0")
d.setVar("BB_FETCH_PREMIRRORONLY", "0")
return (url, perms_dict, path.startswith("prebuilt"))
else:
code_server_prefix = origin_code_server_prefix
return (None, None, None)
python () {
rootdir = find_root_dir(d)
manifest = get_manifest(d)
src_uri = d.getVar("SRC_URI", False)
if src_uri is None:
bb.fatal("[fog][{pn}] SRC_URI is not set!")
pn = d.getVar("PN",True)
fog_force_git = d.getVar('FOG_FORCE_GIT', True)
uris = src_uri.split()
new_uris = []
fog_file_info = {}
fog_git_info = {}
use_prebuilt = False
for uri in uris:
uri = d.expand(uri)
if not uri.startswith("fog://"):
new_uris.append(uri)
continue
fog_santity_check(uri, d)
furi,perm_dict,prebuilt = fog_get_file_fetcher(uri,d)
if furi is not None:
new_uris.append(furi)
fog_file_info[furi] = perm_dict
use_prebuilt = prebuilt
continue
furi,perm_dict,prebuilt = fog_get_manifest_git_fetcher(uri,d)
if furi is not None:
new_uris.append(furi)
fog_git_info[furi] = perm_dict
use_prebuilt = prebuilt
continue
furi,perm_dict,prebuilt = fog_get_remote_git_fetcher(uri,d)
if furi is not None:
new_uris.append(furi)
fog_git_info[furi] = perm_dict
use_prebuilt = prebuilt
continue
d.setVar("SRC_URI", " ".join(new_uris))
d.setVar("FOG_FILE_INFO", fog_file_info)
d.setVar("FOG_GIT_INFO", fog_git_info)
if use_prebuilt:
d.setVar("OVERRIDES", "prebuilt:"+d.getVar('OVERRIDES'))
}
# for local repo case, skip ".git" folder
python do_unpack() {
import subprocess
import json
rootdir = find_root_dir(d)
src_uri = (d.getVar('SRC_URI') or "").split()
if len(src_uri) == 0:
return
fog_keep_dotgit = d.getVar("FOG_KEEP_DOTGIT")
fog_file_info = d.getVar("FOG_FILE_INFO")
fog_info = {}
bpn = d.getVar("BPN",True)
pn = d.getVar("PN",True)
fog_info["bpn"] = bpn
fog_info["pn"] = pn
fog_info["source"] = {}
try:
fetcher = bb.fetch2.Fetch(src_uri, d)
for uri in src_uri:
ud = fetcher.ud[uri]
ud.setup_localpath(fetcher.d)
if uri not in fog_file_info:
fetcher.unpack(d.getVar('WORKDIR'),[uri])
continue
# make param behavior same as git fetcher
destdir = d.getVar('WORKDIR')
perm_dict = fog_file_info[uri]
def_destsuffix = "git"
subpath = "" if "subpath" not in perm_dict else perm_dict["subpath"]
subdir = "" if "subdir" not in perm_dict else perm_dict["subdir"]
if subpath:
def_destsuffix = subpath
if subdir:
if os.path.isabs(subdir):
if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)):
raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s"%destdir, ud.url)
destdir = subdir
else:
destdir = os.path.join(destdir, subdir)
def_destsuffix = ""
destsuffix = def_destsuffix if "destsuffix" not in perm_dict else perm_dict["destsuffix"]
destdir = os.path.join(destdir, destsuffix)
srcdir = os.path.join(ud.localpath,subpath)
fog_info["source"][os.path.relpath(srcdir,rootdir)] = os.path.relpath(destdir,rootdir)
if ud.lockfile:
lf = bb.utils.lockfile(ud.lockfile)
bb.note(f"[fog]Unpacking {srcdir} to {destdir}")
if os.path.exists(destdir):
bb.utils.prunedir(destdir)
bb.utils.mkdirhier(f"{destdir}")
#cmd = 'cd %s ; cp -fpPRH -t %s `ls -A | grep -v ".git"`' % (srcdir, destdir)
cmd = f"rsync -a --exclude=.git {srcdir.rstrip('/')}/ {destdir.rstrip('/')}/;"
src_dotgit = os.path.join(srcdir.rstrip('/'),'.git')
dest_dotgit = os.path.join(destdir.rstrip('/'),'.git')
if fog_keep_dotgit == "1" and os.path.exists(src_dotgit):
broken_files = []
for file in os.listdir(src_dotgit):
if not os.access(os.path.join(src_dotgit,file),os.F_OK):
broken_files.append(file)
exculde_args = " ".join(["--exclude=" + file for file in broken_files])
cmd += f"rsync -a -L {exculde_args} --ignore-missing-args {src_dotgit}/ {dest_dotgit}/"
path = fetcher.d.getVar('PATH')
if path:
cmd = "PATH=\"%s\" %s" % (path, cmd)
ret = subprocess.call(cmd, shell=True)
if ret != 0:
bb.fatal("[fog][{pn}] Unpack command %s failed with return value %s" % (cmd, ret))
if ud.lockfile:
bb.utils.unlockfile(lf)
except bb.fetch2.BBFetchException as e:
bb.fatal(str(e))
source_info_folder = d.getVar('SOURCE_INFO_FOLDER', True)
with open(os.path.join(source_info_folder,bpn+".json"),"w") as f:
json.dump(fog_info, f, indent = 4)
}