update/release/*: merge to update/project/release

The logic has been re-written, where source archives are
concerned. This clones the current repository, and starts
a new build from scratch. A custom release directory is
possible, by passing -d

This eliminates a step during build-testing, saving hours
of time, because it builds the release archive *inside* the
release archive, with git files removed, thus replicating
the same setup that the user would have.

This also makes everything a bit more consistent, because
it's guaranteed that a release archive will always have
the same files; previously, the release build script would
only copy what was already built, without building anything.

Now, this script builds everything itself.
The script also builds serprog images, not just coreboot.

Usage:

./update project release

If -d is not passed, release/ is used inside lbmk.

Otherwise, you could do:

./update project release -d /path/to/directory

If the directory exists, this script will exit (error).

Other minor fixes: build/fw/coreboot: make version in
coreboot-version (file) not contain hyphens, to work
around a quirk in coreboot's build system when not building
on regular libreboot releases. this quirk only appears
when lbmk is not being compiled under git.

The other main benefit of this change is that the new
script will probably require a lot less maintenance.

Signed-off-by: Leah Rowe <leah@libreboot.org>
btrfsvols
Leah Rowe 2023-10-08 02:34:50 +01:00
parent e7a77b50e8
commit d1ba94ea08
4 changed files with 302 additions and 289 deletions

299
script/update/project/release Executable file
View File

@ -0,0 +1,299 @@
#!/usr/bin/env sh
# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: 2020-2023 Leah Rowe <leah@libreboot.org>
[ "x${DEBUG+set}" = 'xset' ] && set -v
set -u -e
. "include/err.sh"
eval "$(setvars "" vdir basename src_dirname srcdir _nogit _xm target romdir \
microcode_required CONFIG_HAVE_MRC CONFIG_HAVE_ME_BIN \
CONFIG_KBC1126_FIRMWARE CONFIG_INCLUDE_SMSC_SCH5545_EC_FW)"
_f="coreboot u-boot seabios flashrom grub memtest86plus me_cleaner uefitool"
_f="${_f} bios_extract biosutilities pico-serprog stm32-vserprog"
_f="${_f} rpi-pico-serprog"
ifdtool="cbutils/default/ifdtool"
cbfstool="cbutils/default/cbfstool"
main()
{
vdir="release"
while getopts d: option
do
case "${1}" in
-d) vdir="${2}" ;;
*) err "Invalid option" ;;
esac
done
[ -z "${vdir}" ] && err "Empty directory names not allowed"
vdir="${vdir}/${version}"
basename="${projectname}-${version}"
src_dirname="${basename}_src"
srcdir="${vdir}/${src_dirname}"
[ -e "${vdir}" ] && err "already exists: \"${vdir}\""
mkvdir
build_release
}
mkvdir()
{
mkdir -p "${vdir}" || err "mkvdir: !mkdir -p \"${vdir}\""
git_init || err "mkvdir: !git_init \"${vdir}\""
git clone . "${srcdir}" || err "mkdir: !gitclone \"${srcdir}\""
[ -z "${_nogit}" ] || x_ rm -Rf ".git"
rm -Rf "${srcdir}/.git"
insert_version_files "${srcdir}" || err "mkvdir ${srcdir}: versionfile"
}
# release archives contain .gitignore, but not .git.
# this script can be run from lbmk.git, or an archive.
git_init()
{
[ -L ".git" ] && err "Reference .git is a symlink"
[ -e ".git" ] && return 0
eval "$(setvars "$(date -Rd @${versiondate})" cdate _nogit)"
git init || return 1
git add -A . || return 1
git commit -m "${projectname} ${version}" --date "${cdate}" || return 1
git tag -a "${version}" -m "${projectname} ${version}" || return 1
}
build_release()
{
_xm="build_release ${vdir}"
(
cd "${srcdir}" || err "${_xm}: !cd \"${srcdir}\""
fetch_trees
mkrom_images
clean_release
)
(
cd "${srcdir%/*}" || err "${_xm}: mktarball \"${srcdir}\""
mktarball "${srcdir##*/}" "${srcdir##*/}.tar.xz" || err "${_xm}: mksrc"
)
rm -Rf "${srcdir}" || err "${_xm}: !rm -Rf \"${srcdir}\""
}
fetch_trees()
{
for x in ${_f}; do
./update project trees -f "${x}" || err "${_xm}: fetch ${x}"
done
x_ rm -Rf src/coreboot/coreboot src/u-boot/u-boot src/seabios/seabios
# by doing this first, we build-test under the same conditions as
# the user, when they extract and build from the src tarball. this
# saves time on build-testing, because we build once, not twice. thus:
rm -Rf */.git* */*/.git* */*/*/.git* */*/*/*/.git* */*/*/*/*/.git* \
*/*/*/*/*/*/.git* */*/*/*/*/*/*/.git* */*/*/*/*/*/*/*/.git* || \
err "${_xm}: rm-dotgit"
}
mkrom_images()
{
./build fw coreboot all || err "${_xm}: roms-all"
./build fw serprog rp2040 || err "${_xm}: rp2040"
./build fw serprog stm32 || err "${_xm}: stm32"
for rombuild in bin/*; do
[ -d "${rombuild}" ] || continue
handle_rom_archive "${rombuild}"
done
mv "release/${version}/roms/" ../roms || err "${_xm}: copy roms/"
}
handle_rom_archive()
{
builddir="${1}"
romdir="tmp/romdir"
x_ rm -Rf "${romdir}"
target="${builddir##*/}"
if [ ! -f "config/coreboot/${target}/target.cfg" ]; then
# No config, just make a tarball
tarball="release/${version}/roms/${basename}_${target}.tar.xz"
insert_copying_files "${builddir}"
mktarball "${builddir}" "${tarball}"
return 0
fi
romdir="${romdir}/bin/${target}"
x_ mkdir -p "${romdir}"
x_ cp "${builddir}/"* "${romdir}"
handle_vendorfiles
printf "Generating release/%s/roms/%s-%s_%s.tar.xz\n" \
"${version}" "${projectname}" "${version}" "${target##*/}"
insert_version_files "${romdir}" || \
err "mkrom_tarball ${romdir}: versionfile"
insert_copying_files "${romdir}"
mkrom_tarball
}
handle_vendorfiles()
{
microcode_required="y"
. "config/coreboot/${target}/target.cfg"
if [ "${microcode_required}" != "y" ] && \
[ "${microcode_required}" != "n" ]; then microcode_required="y"; fi
for x in CONFIG_HAVE_MRC CONFIG_HAVE_ME_BIN CONFIG_KBC1126_FIRMWARE \
CONFIG_INCLUDE_SMSC_SCH5545_EC_FW; do
eval "${x}=\"y\""
grep "${x}=y" "config/coreboot/${target}/config/"* || \
eval "${x}=\"n\""
done
# remove ME/MRC/EC firmware from ROM images
if [ "${CONFIG_HAVE_ME_BIN}" = "y" ] || \
[ "${target}" = "e6400_4mb" ] || \
[ "${microcode_required}" = "n" ]; then
strip_archive
fi
}
strip_archive()
{
if [ "${microcode_required}" = "n" ]; then
for romfile in "${romdir}"/*.rom; do
[ -f "${romfile}" ] || continue
strip_ucode "${romfile}"
done
for romfile in "${romdir}"/*.tmprom; do
[ -f "${romfile}" ] || continue
x_ mv "${romfile}" "${romfile%.tmprom}.rom"
done
fi
# Hash the rom before removing vendor files
x_ rm -f "${romdir}/vendorhashes"
x_ touch "${romdir}/vendorhashes"
(
x_ cd "${romdir}"
x_ sha512sum *.rom >> vendorhashes
)
for romfile in "${romdir}"/*.rom; do
[ -f "${romfile}" ] || continue
strip_rom_image "${romfile}"
done
}
strip_ucode()
{
romfile=${1}
_newrom_b="${romfile%.rom}_nomicrocode.tmprom"
x_ cp "${romfile}" "${_newrom_b}"
microcode_present="y"
"${cbfstool}" "${_newrom_b}" remove -n \
cpu_microcode_blob.bin 2>/dev/null || microcode_present="n"
[ "${microcode_present}" = "n" ] || return 0
printf "REMARK: '%s' already lacks microcode\n" "${romfile}" 1>&2
printf "Renaming default ROM file instead.\n" 1>&2
x_ mv "${romfile}" "${_newrom_b}"
}
strip_rom_image()
{
[ -f "${1}" ] || return 0
[ "${CONFIG_HAVE_ME_BIN}" != "y" ] || \
x_ "${ifdtool}" --nuke me "${1}" -O "${1}"
[ "${CONFIG_HAVE_MRC}" != "y" ] || \
x_ "${cbfstool}" "${1}" remove -n mrc.bin
[ "${CONFIG_INCLUDE_SMSC_SCH5545_EC_FW}" != "y" ] || \
x_ "${cbfstool}" "${1}" remove -n sch5545_ecfw.bin
if [ "${CONFIG_KBC1126_FIRMWARE}" = "y" ]; then
x_ "${cbfstool}" "${1}" remove -n ecfw1.bin
x_ "${cbfstool}" "${1}" remove -n ecfw2.bin
fi
# TODO: replace this board-specific hack
[ "${target}" = "e6400_4mb" ] || return 0
x_ "${cbfstool}" "${1}" remove -n "pci10de,06eb.rom"
}
insert_copying_files()
{
x_ rm -Rf "${1}/licenses"
x_ mkdir -p "${1}/licenses"
l="${1}/licenses"
# copy licenses to rom image archive, for completion
x_ cp "src/grub/COPYING" "${l}/COPYING.grub"
x_ cp "src/coreboot/default/COPYING" "${l}/COPYING.coreboot"
x_ cp -R "src/coreboot/default/LICENSES" "${l}/LICENSES.coreboot"
x_ cp "src/seabios/default/COPYING" "${l}/COPYING.coreboot"
x_ cp "src/seabios/default/COPYING.LESSER" "${l}/COPYING.LESSER.seabios"
x_ cp -R "src/u-boot/default/Licenses" "${l}/COPYING.u-boot"
}
mkrom_tarball()
{
archivename="${basename}_${target##*/}"
f="release/${version}/roms/${archivename}"
x_ mkdir -p "${f%/*}"
(
x_ cd "${romdir%/bin/${target}}"
mktarball "bin/${target}" "${archivename}.tar.xz"
)
x_ mv "${romdir%/bin/${target}}/${archivename}.tar.xz"* "${f%/*}"
printf "Created ROM archive: ${f%/*}/${archivename}.tar.xz"
}
clean_release()
{
for x in "src/coreboot/default/util/kbc1126" util/*; do
[ ! -f "${x}/Makefile" ] && continue
x_ make clean -C "${x}"
done
for x in ${_f}; do
./update project trees -c "${x}" || err "${_xm}: pclean ${x}"
done
./update project trees -x coreboot || err "${_xm}: clean-crossgcc"
./update project trees -c "stm32-vserprog/libopencm3"
# make absolutely sure crossgcc was purged. for some reason,
# crossgcc-clean isn't always reliable on every coreboot tree
for xgcc in "build-" "binutils-" "gcc-" "gmp-" "mpc-" "mpfr-" \
"llvm-" "clang-tools-" "cfe-" "compiler-rt-" "acpica-" \
"getopt" "xgcc"; do
x_ rm -Rf src/coreboot/*/util/crossgcc/${xgcc}*
done
rm -Rf vendor mrc ec pciroms || err "${_xm}: vendor files"
rm -Rf elf tmp cbutils src/pico-serprog/build release bin || \
err "${_xm}: elf, tmp, cbutils, pico-serprog/build"
}
insert_version_files()
{
printf "%s\n" "${version}" > "${1}/version" || return 1
printf "%s\n" "${versiondate}" > "${1}/versiondate" || return 1
printf "%s\n" "${projectname}" > "${1}/projectname" || return 1
}
mktarball()
{
[ "${2%/*}" = "${2}" ] || x_ mkdir -p "${2%/*}"
x_ tar -c "${1}" | xz -T0 -9e > "${2}"
(
[ "${2%/*}" != "${2}" ] && x_ cd "${2%/*}"
x_ sha512sum "${2##*/}" > "${2##*/}.sha512"
)
}
main $@

View File

@ -11,7 +11,7 @@ set -u -e
. "include/option.sh"
. "include/git.sh"
export LOCALVERSION="-${projectname}-${version}"
export LOCALVERSION="-${projectname}-${version%%-*}"
eval "$(setvars "" arch cfgsdir codedir config config_name crossgcc_ada mode \
elfdir listfile project romtype target target_dir targets tree cbfstool _f \
@ -271,9 +271,9 @@ run_make_command()
[ -f "${codedir}/Makefile" ] || [ -f "${codedir}/makefile" ] || \
[ -f "${codedir}/GNUmakefile" ] || return 1
[ "${project}" = "coreboot" ] && [ -z "${mode}" ] && \
x_ printf "%s\n" "${version}" >"${codedir}/.coreboot-version"
x_ printf "%s\n" "${version%%-*}" >"${codedir}/.coreboot-version"
x_ make -C "${codedir}" -j$(nproc) ${mode}
x_ make ${mode} -j$(nproc) -C "${codedir}"
[ "${mode}" != "clean" ] && return 0
make -C "${codedir}" distclean 2>/dev/null || :
}

View File

@ -1,175 +0,0 @@
#!/usr/bin/env sh
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2020,2021,2022,2023 Leah Rowe <leah@libreboot.org>
[ "x${DEBUG+set}" = 'xset' ] && set -v
set -u -e
. "include/err.sh"
tree="default"
eval "$(setvars "" target CONFIG_HAVE_MRC CONFIG_HAVE_ME_BIN \
CONFIG_KBC1126_FIRMWARE CONFIG_INCLUDE_SMSC_SCH5545_EC_FW)"
ifdtool="cbutils/${tree}/ifdtool"
cbfstool="cbutils/${tree}/cbfstool"
microcode_required="y"
main()
{
printf "Building ROM image archives for version %s\n" "${version}"
init_check
for romdir in bin/*; do
[ -d "${romdir}" ] || continue
make_archive "${romdir}"
done
printf "\nROM archives available at release/%s/roms/\n\n" "${version}"
}
init_check()
{
[ ! -d "bin/" ] && \
err "init_check: no ROMs built yet (error)"
[ -d "release/" ] || \
x_ mkdir -p release/
[ -d "release/${version}/" ] || \
x_ mkdir -p "release/${version}/"
[ ! -d "release/${version}/roms/" ] || \
x_ rm -Rf "release/${version}/roms/"
[ -d "release/${version}/roms/" ] && return 0
x_ mkdir -p "release/${version}/roms/"
}
make_archive()
{
builddir="${1}"
romdir="tmp/romdir"
x_ rm -Rf "${romdir}"
target="${builddir##*/}"
if [ ! -f "config/coreboot/${target}/target.cfg" ]; then
# No config, just make a tarball
tarball=release/${version}/roms/${target}_${version}.tar.xz
x_ tar -c "${builddir}" | xz -T0 -6 > ${tarball} || \
x_ rm -f ${tarball}
return 0
fi
romdir="${romdir}/bin/${target}"
x_ mkdir -p "${romdir}"
x_ cp "${builddir}"/* "${romdir}"
printf "%s\n" "${target}"
microcode_required="y"
. "config/coreboot/${target}/target.cfg"
if [ "${microcode_required}" != "y" ] && \
[ "${microcode_required}" != "n" ]; then microcode_required="y"; fi
for x in CONFIG_HAVE_MRC CONFIG_HAVE_ME_BIN CONFIG_KBC1126_FIRMWARE \
CONFIG_INCLUDE_SMSC_SCH5545_EC_FW; do
eval "${x}=\"y\""
grep "${x}=y" "config/coreboot/${target}/config/"* || \
eval "${x}=\"n\""
done
# remove ME/MRC/EC firmware from ROM images
if [ "${CONFIG_HAVE_ME_BIN}" = "y" ] || \
[ "${target}" = "e6400_4mb" ] || \
[ "${microcode_required}" = "n" ]; then
strip_archive "${romdir}"
fi
printf "Generating release/%s/roms/%s-%s_%s.tar.xz\n" \
"${version}" "${projectname}" "${version}" "${target##*/}"
x_ printf "%s\n" "${version}" > "${romdir}/version"
x_ printf "%s\n" "${versiondate}" > "${romdir}/versiondate"
x_ printf "%s\n" "${projectname}" > "${romdir}/projectname"
for x in u-boot seabios coreboot; do
[ -d "src/${x}/default" ] || \
x_ ./update project trees -f ${x} default
done
[ -d src/grub ] || x_ ./update project trees -f grub
# copy licenses to rom image archive, for completion
x_ cp "src/grub/COPYING" "${romdir}/COPYING.grub"
x_ cp "src/coreboot/default/COPYING" "${romdir}/COPYING.coreboot"
x_ cp "src/seabios/default/COPYING" "${romdir}/COPYING.coreboot"
x_ cp -R "src/u-boot/default/Licenses" "${romdir}/COPYING.u-boot"
archivename="${projectname}-${version}_${target##*/}"
f="release/${version}/roms/${archivename}"
(
x_ cd "${romdir%/bin/${target}}"
x_ tar -c "bin/${target}/" | xz -T0 -9e > "${archivename}.tar.xz"
)
x_ mv "${romdir%/bin/${target}}/${archivename}.tar.xz" "${f}.tar.xz"
x_ rm -Rf "${romdir%/bin/${target}}"
}
strip_archive()
{
romdir=${1}
[ -d "src/coreboot/${tree}" ] || \
x_ ./update project trees -f coreboot ${tree}
x_ ./update project trees -b coreboot utils ${tree}
if [ "${microcode_required}" = "n" ]; then
for romfile in "${romdir}"/*.rom; do
[ -f "${romfile}" ] || continue
strip_ucode "${romfile}"
done
for romfile in "${romdir}"/*.tmprom; do
[ -f "${romfile}" ] || continue
x_ mv "${romfile}" "${romfile%.tmprom}.rom"
done
fi
# Hash the rom before removing vendor files
x_ rm -f "${romdir}/vendorhashes"
x_ touch "${romdir}/vendorhashes"
(
x_ cd "${romdir}"
x_ sha512sum *.rom >> vendorhashes
)
for romfile in "${romdir}"/*.rom; do
[ -f "${romfile}" ] || continue
strip_rom_image "${romfile}"
done
}
strip_ucode()
{
romfile=${1}
_newrom_b="${romfile%.rom}_nomicrocode.tmprom"
x_ cp "${romfile}" "${_newrom_b}"
microcode_present="y"
"${cbfstool}" "${_newrom_b}" remove -n \
cpu_microcode_blob.bin 2>/dev/null || microcode_present="n"
[ "${microcode_present}" = "n" ] || return 0
printf "REMARK: '%s' already lacks microcode\n" "${romfile}" 1>&2
printf "Renaming default ROM file instead.\n" 1>&2
x_ mv "${romfile}" "${_newrom_b}"
}
strip_rom_image()
{
[ -f "${1}" ] || return 0
[ "${CONFIG_HAVE_ME_BIN}" != "y" ] || \
x_ "${ifdtool}" --nuke me "${1}" -O "${1}"
[ "${CONFIG_HAVE_MRC}" != "y" ] || \
x_ "${cbfstool}" "${1}" remove -n mrc.bin
[ "${CONFIG_INCLUDE_SMSC_SCH5545_EC_FW}" != "y" ] || \
x_ "${cbfstool}" "${1}" remove -n sch5545_ecfw.bin
if [ "${CONFIG_KBC1126_FIRMWARE}" = "y" ]; then
x_ "${cbfstool}" "${1}" remove -n ecfw1.bin
x_ "${cbfstool}" "${1}" remove -n ecfw2.bin
fi
# TODO: replace this board-specific hack
[ "${target}" = "e6400_4mb" ] || return 0
x_ "${cbfstool}" "${1}" remove -n "pci10de,06eb.rom"
}
main $@

View File

@ -1,111 +0,0 @@
#!/usr/bin/env sh
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2020,2021,2023 Leah Rowe <leah@libreboot.org>
[ "x${DEBUG+set}" = 'xset' ] && set -v
set -u -e
. "include/err.sh"
trees_fetch_list="coreboot u-boot seabios"
simple_fetch_list="flashrom grub memtest86plus me_cleaner uefitool"
simple_fetch_list="${simple_fetch_list} bios_extract biosutilities"
download_only_list="pico-serprog stm32-vserprog"
dirlist="config util script include"
linklist="update" # symlinks in main directory, to script: lbmk
filelist="build README.md COPYING version versiondate projectname .gitignore"
eval "$(setvars "" reldir dirname srcdir)"
printf "Building source code archive, version %s\n" "${version}"
main()
{
create_release_directory
download_modules
copy_files
purge_files
create_release_archive
printf "Source code archive available at %s.tar.xz\n\n" "${srcdir}"
}
create_release_directory()
{
reldir="release/${version}"
dirname="${projectname}-${version}_src"
srcdir="${reldir}/${dirname}"
xx_ mkdir -p "${reldir}"
xx_ rm -Rf "${srcdir}" "${srcdir}.tar.xz"
xx_ mkdir -p "${srcdir}"
}
download_modules()
{
for modname in ${trees_fetch_list} ${simple_fetch_list} \
${download_only_list}; do
[ -d "src/${modname}/" ] || \
xx_ ./update project trees -f ${modname}
done
}
copy_files()
{
xx_ cp -R "src" "${srcdir}/src"
for dir in ${trees_fetch_list}; do
xx_ rm -Rf "${srcdir}/src/${dir}/${dir}"
done
for dir in ${dirlist}; do
xx_ cp -R "${dir}/" "${srcdir}/"
done
for i in ${filelist}; do
[ -f "${i}" ] || fail "copy_files: '${i}' does not exist"
xx_ cp "${i}" "${srcdir}/"
done
(
xx_ cd "${srcdir}/"
for i in ${linklist}; do
xx_ ln -s build "${i}"
done
)
}
purge_files()
{
(
xx_ cd "${srcdir}"
[ ! -d "src/coreboot/default/util/kbc1126" ] || \
xx_ ./update project trees -c "src/coreboot/default/util/kbc1126"
xx_ ./update project trees -x coreboot
for p in u-boot seabios coreboot; do
xx_ ./update project trees -c "${p}"
done
xx_ ./update project trees -c bios_extract flashrom grub uefitool \
stm32-vserprog stm32-vserprog/libopencm3 util/* memtest86plus
xx_ rm -Rf .git */.git* */*/.git* */*/*/.git* */*/*/*/.git* \
*/*/*/*/*/.git* */*/*/*/*/*/.git* */*/*/*/*/*/*/.git* \
*/*/*/*/*/*/*/*/.git*
xx_ rm -Rf cbutils elf src/pico-serprog/build
)
}
create_release_archive()
{
(
xx_ cd "${reldir}/"
xx_ tar -c "${dirname}/" | xz -T0 -9e >"${dirname}.tar.xz"
xx_ rm -Rf "${dirname}/"
)
}
fail()
{
[ -z "${srcdir}" ] || rm -Rf "${srcdir}" 1>/dev/null 2>/dev/null || :
fail "${1}"
}
main $@