lbmk/include/git.sh

215 lines
5.9 KiB
Bash
Raw Normal View History

# SPDX-License-Identifier: GPL-3.0-or-later
# Copyright (c) 2020-2021,2023-2024 Leah Rowe <leah@libreboot.org>
# Copyright (c) 2022 Caleb La Grange <thonkpeasant@protonmail.com>
eval "$(setvars "" _target rev _xm loc url bkup_url depend tree_depend xtree \
mdir subrev subrepo subrepo_bkup)"
fetch_project_trees()
{
_target="$target"
[ ! -d "src/$project/$project" ] && x_ mkdir -p "src/$project" \
&& fetch_project_repo "$project"
fetch_config
e "src/$project/$tree" d && return 0
prepare_new_tree
}
fetch_config()
{
rm -f "$cfgsdir/"*/seen || $err "fetch_config $cfgsdir: !rm seen"
eval "$(setvars "" xtree tree_depend)"
while true; do
eval "$(setvars "" rev tree)"
_xm="fetch_config $project/$_target"
load_target_config "$_target"
[ "$_target" = "$tree" ] && break
_target="$tree"
done
[ -n "$tree_depend" ] && [ "$tree_depend" != "$tree" ] && \
x_ ./update trees -f "$project" "$tree_depend"; return 0
}
load_target_config()
{
safer, simpler error handling in lbmk in shell scripts, a function named the same as a program included in the $PATH will override that program. for example, you could make a function called ls() and this would override the standand "ls". in lbmk, a part of it was first trying to run the "fail" command, deferring to "err", because some scripts call fail() which does some minor cleanup before calling err. in most cases, fail() is not defined, and it's possible that the user could have a program called "fail" in their $PATH, the behaviour of which we could not determine, and it could have disastrous effects. lbmk error handling has been re-engineered in such a way that the err function is defined in a variable, which defaults to err_ which calls err_, so defined under include/err.sh. in functions that require cleanup prior to error handling, a fail() function is still defined, and err is overridden, thus: err="fail" this change has made xx_() obsolete, so now only x_ is used. the x_ function is a wrapper that can be used to run a command and exit with non-zero status (from lbmk) if the command fails. the xx_ command did the same thing, but called fail() which would have called err(); now everything is $err example: rm -f "$filename" || err "could not delete file" this would now be: rm -f "$filename" || $err "could not delete file" overriding of err= must be done *after* including err.sh. for example: err="fail" . "include/err.sh" ^ this is wrong. instead, one must do: . "include/err.sh" err="fail" this is because err is set as a global variable under err.sh the new error handling is much cleaner, and safer. it also reduces the chance of mistakes such as: calling err when you meant to call fail. this is because the standard way is now to call $err, so you set err="fail" at the top of the script and all is well. Signed-off-by: Leah Rowe <leah@libreboot.org>
2024-03-27 01:19:39 +00:00
[ -f "$cfgsdir/$1/target.cfg" ] || $err "$1: target.cfg missing"
[ -f "$cfgsdir/$1/seen" ] && $err "$_xm cfg: infinite loop in trees"
safer, simpler error handling in lbmk in shell scripts, a function named the same as a program included in the $PATH will override that program. for example, you could make a function called ls() and this would override the standand "ls". in lbmk, a part of it was first trying to run the "fail" command, deferring to "err", because some scripts call fail() which does some minor cleanup before calling err. in most cases, fail() is not defined, and it's possible that the user could have a program called "fail" in their $PATH, the behaviour of which we could not determine, and it could have disastrous effects. lbmk error handling has been re-engineered in such a way that the err function is defined in a variable, which defaults to err_ which calls err_, so defined under include/err.sh. in functions that require cleanup prior to error handling, a fail() function is still defined, and err is overridden, thus: err="fail" this change has made xx_() obsolete, so now only x_ is used. the x_ function is a wrapper that can be used to run a command and exit with non-zero status (from lbmk) if the command fails. the xx_ command did the same thing, but called fail() which would have called err(); now everything is $err example: rm -f "$filename" || err "could not delete file" this would now be: rm -f "$filename" || $err "could not delete file" overriding of err= must be done *after* including err.sh. for example: err="fail" . "include/err.sh" ^ this is wrong. instead, one must do: . "include/err.sh" err="fail" this is because err is set as a global variable under err.sh the new error handling is much cleaner, and safer. it also reduces the chance of mistakes such as: calling err when you meant to call fail. this is because the standard way is now to call $err, so you set err="fail" at the top of the script and all is well. Signed-off-by: Leah Rowe <leah@libreboot.org>
2024-03-27 01:19:39 +00:00
. "$cfgsdir/$1/target.cfg" || $err "load_target_config !$cfgsdir/$1"
touch "$cfgsdir/$1/seen" || $err "load_config $cfgsdir/$1: !mk seen"
}
prepare_new_tree()
{
printf "Creating %s tree %s (%s)\n" "$project" "$tree" "$_target"
cp -R "src/$project/$project" "$tmpgit" || \
$err "prepare_new_tree $project/$tree: can't make tmpclone"
git_prep "$PWD/$cfgsdir/$tree/patches" "src/$project/$tree" "update"
nuke "$project/$tree" "$project/$tree"
}
fetch_project_repo()
{
eval "$(setvars "" xtree tree_depend)"
scan_config "$project" "config/git"
safer, simpler error handling in lbmk in shell scripts, a function named the same as a program included in the $PATH will override that program. for example, you could make a function called ls() and this would override the standand "ls". in lbmk, a part of it was first trying to run the "fail" command, deferring to "err", because some scripts call fail() which does some minor cleanup before calling err. in most cases, fail() is not defined, and it's possible that the user could have a program called "fail" in their $PATH, the behaviour of which we could not determine, and it could have disastrous effects. lbmk error handling has been re-engineered in such a way that the err function is defined in a variable, which defaults to err_ which calls err_, so defined under include/err.sh. in functions that require cleanup prior to error handling, a fail() function is still defined, and err is overridden, thus: err="fail" this change has made xx_() obsolete, so now only x_ is used. the x_ function is a wrapper that can be used to run a command and exit with non-zero status (from lbmk) if the command fails. the xx_ command did the same thing, but called fail() which would have called err(); now everything is $err example: rm -f "$filename" || err "could not delete file" this would now be: rm -f "$filename" || $err "could not delete file" overriding of err= must be done *after* including err.sh. for example: err="fail" . "include/err.sh" ^ this is wrong. instead, one must do: . "include/err.sh" err="fail" this is because err is set as a global variable under err.sh the new error handling is much cleaner, and safer. it also reduces the chance of mistakes such as: calling err when you meant to call fail. this is because the standard way is now to call $err, so you set err="fail" at the top of the script and all is well. Signed-off-by: Leah Rowe <leah@libreboot.org>
2024-03-27 01:19:39 +00:00
[ -z "${loc+x}" ] && $err "fetch_project_repo $project: loc not set"
[ -z "${url+x}" ] && $err "fetch_project_repo $project: url not set"
clone_project
[ -z "$depend" ] || for d in $depend ; do
x_ ./update trees -f $d
done
rm -Rf "$tmpgit" || $err "fetch_repo: !rm -Rf $tmpgit"
for x in config/git/*; do
[ -f "$x" ] && nuke "${x##*/}" "src/${x##*/}"; continue
done
}
clone_project()
{
loc="${loc#src/}"
loc="src/$loc"
e "$loc" d && return 0
git clone $url "$tmpgit" || git clone $bkup_url "$tmpgit" \
|| $err "clone_project: could not download $project"
git_prep "$PWD/config/$project/patches" "$loc"
}
git_prep()
{
_patchdir="$1"
_loc="$2"
safer, simpler error handling in lbmk in shell scripts, a function named the same as a program included in the $PATH will override that program. for example, you could make a function called ls() and this would override the standand "ls". in lbmk, a part of it was first trying to run the "fail" command, deferring to "err", because some scripts call fail() which does some minor cleanup before calling err. in most cases, fail() is not defined, and it's possible that the user could have a program called "fail" in their $PATH, the behaviour of which we could not determine, and it could have disastrous effects. lbmk error handling has been re-engineered in such a way that the err function is defined in a variable, which defaults to err_ which calls err_, so defined under include/err.sh. in functions that require cleanup prior to error handling, a fail() function is still defined, and err is overridden, thus: err="fail" this change has made xx_() obsolete, so now only x_ is used. the x_ function is a wrapper that can be used to run a command and exit with non-zero status (from lbmk) if the command fails. the xx_ command did the same thing, but called fail() which would have called err(); now everything is $err example: rm -f "$filename" || err "could not delete file" this would now be: rm -f "$filename" || $err "could not delete file" overriding of err= must be done *after* including err.sh. for example: err="fail" . "include/err.sh" ^ this is wrong. instead, one must do: . "include/err.sh" err="fail" this is because err is set as a global variable under err.sh the new error handling is much cleaner, and safer. it also reduces the chance of mistakes such as: calling err when you meant to call fail. this is because the standard way is now to call $err, so you set err="fail" at the top of the script and all is well. Signed-off-by: Leah Rowe <leah@libreboot.org>
2024-03-27 01:19:39 +00:00
[ -z "${rev+x}" ] && $err "git_prep $_loc: rev not set"
git -C "$tmpgit" reset --hard $rev || $err "git -C $_loc: !reset $rev"
git_am_patches "$tmpgit" "$_patchdir" || $err "!am $_loc $_patchdir"
if [ "$project" != "coreboot" ] || [ $# -gt 2 ]; then
prep_submodules "$_loc"
fi
[ "$project" = "coreboot" ] && [ -n "$xtree" ] && [ $# -gt 2 ] && \
[ "$xtree" != "$tree" ] && link_crossgcc "$_loc"
[ "$xbmk_release" = "y" ] && [ "$_loc" != "src/$project/$project" ] \
&& rmgit "$tmpgit"
move_repo "$_loc"
}
prep_submodules()
{
mdir="$PWD/config/submodule/$project"
[ -n "$tree" ] && mdir="$mdir/$tree"
if [ -f "$mdir/module.list" ]; then
cat "$mdir/module.list" > "$tmpdir/modules" || \
$err "!cp $mdir/module.list $tmpdir/modules"
make GRUB multi-tree and re-add xhci patches Re-add xHCI only on haswell and broadwell machines, where they are needed. Otherwise, keep the same GRUB code. The xHCI patches were removed because they caused issues on Sandybridge-based Dell Latitude laptops. See: https://codeberg.org/libreboot/lbmk/issues/216 The issue was not reported elsewhere, including on the Haswell/Broadwell hardware where they are needed, but the build system could only build one version of GRUB. The older machines do not need xHCI patches, because they either do not have xHCI patches, or work (in GRUB) because they're in EHCI mode when running the payload. So, the problem is that we need the xHCI patches for GRUB on Haswell/Broadwell hardware, but the patches break Sandybridge hardware, and we only had the one build of GRUB. To mitigate this problem, the build system now supports building multiple revisions of GRUB, with different patches, and each given coreboot target can say which GRUB tree to use by setting this in target.cfg: grubtree="xhci" In the above example, the "xhci" tree would be used. Some generic GRUB config has been moved to config/data/grub/ and config/grub/ now looks like config/coreboot/ - also, the grub.cfg file (named "payload" in each tree) is copied to the GRUB source tree as ".config", then added to GRUB's memdisk in the same way, as grub.cfg. Several other design changes had to be made because of this: * grub.cfg in memdisk no longer automatically jumps to one in CBFS, but now shows a menuentry for it if available * Certain commands in script/trees are disabled for GRUB, such as *config make commands. * gnulib is now defined in config/submodule/grub/, instead of config/git/grub - and this mitigates an existing bug where downloading gnulib first would make grub no longer possible to download in lbmk. The coreboot option CONFIG_FINALIZE_USB_ROUTE_XHCI has been re-enabled on: Dell OptiPlex 9020 MT, Dell OptiPlex 9020 SFF, Lenovo ThinkPad T440p and Lenovo ThinkPad W541 - now USB should work again in GRUB. The GRUB payload has been re-enabled on HP EliteBook 820 G2. This change will enable per-board GRUB optimisation in the future. For example, we hardcode what partitions and LVMs GRUB scans because * is slow on ICH7-based machines, due to GRUB's design. On other machines, * is reasonably fast, for automatically enumerating the list of devices for boot. Use of * (and other wildcards) could enable our GRUB payload to automatically boot more distros, with minimal fuss. This can be done at a later date, in subsequent revisions. Signed-off-by: Leah Rowe <leah@libreboot.org>
2024-06-01 22:01:30 +00:00
elif [ -f "$tmpgit/.gitmodules" ]; then
git -C "$tmpgit" submodule status | awk '{print $2}' > \
"$tmpdir/modules" || $err "$mdir: cannot list submodules"
make GRUB multi-tree and re-add xhci patches Re-add xHCI only on haswell and broadwell machines, where they are needed. Otherwise, keep the same GRUB code. The xHCI patches were removed because they caused issues on Sandybridge-based Dell Latitude laptops. See: https://codeberg.org/libreboot/lbmk/issues/216 The issue was not reported elsewhere, including on the Haswell/Broadwell hardware where they are needed, but the build system could only build one version of GRUB. The older machines do not need xHCI patches, because they either do not have xHCI patches, or work (in GRUB) because they're in EHCI mode when running the payload. So, the problem is that we need the xHCI patches for GRUB on Haswell/Broadwell hardware, but the patches break Sandybridge hardware, and we only had the one build of GRUB. To mitigate this problem, the build system now supports building multiple revisions of GRUB, with different patches, and each given coreboot target can say which GRUB tree to use by setting this in target.cfg: grubtree="xhci" In the above example, the "xhci" tree would be used. Some generic GRUB config has been moved to config/data/grub/ and config/grub/ now looks like config/coreboot/ - also, the grub.cfg file (named "payload" in each tree) is copied to the GRUB source tree as ".config", then added to GRUB's memdisk in the same way, as grub.cfg. Several other design changes had to be made because of this: * grub.cfg in memdisk no longer automatically jumps to one in CBFS, but now shows a menuentry for it if available * Certain commands in script/trees are disabled for GRUB, such as *config make commands. * gnulib is now defined in config/submodule/grub/, instead of config/git/grub - and this mitigates an existing bug where downloading gnulib first would make grub no longer possible to download in lbmk. The coreboot option CONFIG_FINALIZE_USB_ROUTE_XHCI has been re-enabled on: Dell OptiPlex 9020 MT, Dell OptiPlex 9020 SFF, Lenovo ThinkPad T440p and Lenovo ThinkPad W541 - now USB should work again in GRUB. The GRUB payload has been re-enabled on HP EliteBook 820 G2. This change will enable per-board GRUB optimisation in the future. For example, we hardcode what partitions and LVMs GRUB scans because * is slow on ICH7-based machines, due to GRUB's design. On other machines, * is reasonably fast, for automatically enumerating the list of devices for boot. Use of * (and other wildcards) could enable our GRUB payload to automatically boot more distros, with minimal fuss. This can be done at a later date, in subsequent revisions. Signed-off-by: Leah Rowe <leah@libreboot.org>
2024-06-01 22:01:30 +00:00
else
return 0
fi
while read -r msrcdir; do
fetch_submodule "$msrcdir"
patch_submodule "$msrcdir"
done < "$tmpdir/modules"
}
fetch_submodule()
{
mcfgdir="$mdir/${1##*/}"
eval "$(setvars "" subrev subrepo subrepo_bkup)"
[ ! -f "$mcfgdir/module.cfg" ] || . "$mcfgdir/module.cfg" || \
$err "! . $mcfgdir/module.cfg"
if [ -n "$subrepo" ] || [ -n "$subrepo_bkup" ]; then
[ -n "$subrev" ] || $err "$1, $mdir: subrev not defined"
rm -Rf "$tmpgit/$1" || $err "!rm '$mdir' '$1'"
for mod in "$subrepo" "$subrepo_bkup"; do
[ -z "$mod" ] && continue
git clone "$mod" "$tmpgit/$1" || rm -Rf "$tmpgit/$1" \
|| $err "!rm $mod $project $cfgdir $1"
[ -d "$tmpgit/$1" ] && break
done
[ -d "$tmpgit/$1" ] || $err "!clone $mod $project $mcfgdir $1"
else
git -C "$tmpgit" submodule update --init --checkout -- "$1" \
|| $err "$mdir: !update $1"
fi
}
patch_submodule()
{
[ -z "$subrev" ] || git -C "$tmpgit/$1" reset --hard "$subrev" || \
$err "$mdir $1: cannot reset git revision"
git_am_patches "$tmpgit/$1" "$mdir/${1##*/}/patches"
}
git_am_patches()
{
for _patch in "$2/"*; do
[ -L "$_patch" ] || [ ! -f "$_patch" ] || git -C "$1" am \
"$_patch" || $err "$1 $2: !git am $_patch"; continue
done
for _patches in "$2/"*; do
[ ! -L "$_patches" ] && [ -d "$_patches" ] && \
git_am_patches "$1" "$_patches"; continue
done
}
link_crossgcc()
{
(
cd "$tmpgit/util" || $err "prep $1: !cd $tmpgit/util"
rm -Rf crossgcc || $err "prep $1: !rm xgcc"
ln -s "../../$xtree/util/crossgcc" crossgcc || $err "$1: !xgcc link"
) || $err "$1: !xgcc link"
}
move_repo()
{
[ "$1" = "${1%/*}" ] || x_ mkdir -p "${1%/*}"
mv "$tmpgit" "$1" || $err "git_prep: !mv $tmpgit $1"
[ -n "$xtree" ] && [ ! -d "src/coreboot/$xtree" ] && \
x_ ./update trees -f coreboot "$xtree"; return 0
}
# can delete from multi- and single-tree projects.
# called from script/trees when downloading sources.
nuke()
{
del="n"
pjcfgdir="${1%/}"
pjsrcdir="${2%/}"
pjsrcdir="${pjsrcdir#src/}"
[ ! -f "config/$pjcfgdir/nuke.list" ] && return 0
while read -r nukefile; do
rmf="$(realpath "src/$pjsrcdir/$nukefile" 2>/dev/null)" || \
continue
[ -L "$rmf" ] && continue # we will delete the actual file
[ "${rmf#"$PWD/src/$pjsrcdir"}" = "$rmf" ] && continue
[ "${rmf#"$PWD/src/"}" = "$pjsrcdir" ] && continue
rmf="${rmf#"$PWD/"}"
[ -e "$rmf" ] || continue
del="y"
rm -Rf "$rmf" || $err "$nuke pjcfgdir: can't rm \"$nukefile\""
printf "nuke %s: deleted \"%s\"\n" "$pjcfgdir" "$rmf"
done < "config/$pjcfgdir/nuke.list"
[ "${del}" = "y" ] && return 0
printf "nuke %s: no defined files exist in dir, src/%s\n" 1>&2 \
"$pjcfgdir" "$pjsrcdir"
printf "(this is not an error)\n" 1>&2
}