diff options
| -rw-r--r-- | meta/classes/packaged-staging.bbclass | 322 |
1 files changed, 322 insertions, 0 deletions
diff --git a/meta/classes/packaged-staging.bbclass b/meta/classes/packaged-staging.bbclass new file mode 100644 index 000000000..bb166cb2f --- /dev/null +++ b/meta/classes/packaged-staging.bbclass @@ -0,0 +1,322 @@ +# +# Populate builds using prebuilt packages where possible to speed up builds +# and allow staging to be reconstructed. +# +# To use it add that line to conf/local.conf: +# +# INHERIT += "packaged-staging" +# + + +# +# bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active +# +PSTAGE_PKGVERSION = "${PV}-${PR}" +PSTAGE_PKGARCH = "${BUILD_SYS}" +PSTAGE_EXTRAPATH ?= "" +PSTAGE_PKGPATH = "${DISTRO}${PSTAGE_EXTRAPATH}" +PSTAGE_PKGPN = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}" +PSTAGE_PKGNAME = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk" +PSTAGE_PKG = "${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}" + +# multimachine.bbclass will override this but add a default in case we're not using it +MULTIMACH_ARCH ?= "${PACKAGE_ARCH}" + +PSTAGE_NATIVEDEPENDS = "\ + pkgconfig-native \ + autoconf-native \ + automake-native \ + curl-native \ + zlib-native \ + libtool-native \ + gnu-config-native \ + shasum-native \ + libtool-native \ + automake-native \ + update-alternatives-cworth-native \ + ipkg-utils-native \ + opkg-native \ + m4-native \ + quilt-native \ + stagemanager-native \ + " + +python () { + import bb + pstage_allowed = True + + # These classes encode staging paths into the binary data so can only be + # reused if the path doesn't change/ + if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d): + path = bb.data.getVar('PSTAGE_PKGPATH', d, 1) + path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-') + bb.data.setVar('PSTAGE_PKGPATH', path, d) + + # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged + # staging. There will always be some packages we depend on. + if bb.data.inherits_class('native', d): + pn = bb.data.getVar('PN', d, True) + nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split() + if pn in nativedeps: + pstage_allowed = False + + # Images aren't of interest to us + if bb.data.inherits_class('image', d): + pstage_allowed = False + + # Add task dependencies if we're active, otherwise mark packaged staging + # as inactive. + if pstage_allowed: + deps = bb.data.getVarFlag('do_populate_staging', 'depends', d) or "" + deps += " stagemanager-native:do_populate_staging" + bb.data.setVarFlag('do_populate_staging', 'depends', deps, d) + + deps = bb.data.getVarFlag('do_prepackaged_stage', 'depends', d) or "" + deps += " opkg-native:do_populate_staging ipkg-utils-native:do_populate_staging" + bb.data.setVarFlag('do_prepackaged_stage', 'depends', deps, d) + bb.data.setVar("PSTAGING_ACTIVE", "1", d) + else: + bb.data.setVar("PSTAGING_ACTIVE", "0", d) +} + +DEPLOY_DIR_PSTAGE = "${DEPLOY_DIR}/pstage" +PSTAGE_MACHCONFIG = "${DEPLOY_DIR_PSTAGE}/opkg.conf" + +PSTAGE_BUILD_CMD = "${IPKGBUILDCMD}" +PSTAGE_INSTALL_CMD = "opkg-cl install -force-depends -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}" +PSTAGE_UPDATE_CMD = "opkg-cl update -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}" +PSTAGE_REMOVE_CMD = "opkg-cl remove -force-depends -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}" +PSTAGE_LIST_CMD = "opkg-cl list_installed -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}" + +PSTAGE_TMPDIR_STAGE = "${WORKDIR}/staging-pkg" + +do_clean_append() { + """ + Clear the build and temp directories + """ + bb.note("Uninstalling package from staging...") + path = bb.data.getVar("PATH", d, 1) + removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1) + removepkg = bb.data.expand("${PSTAGE_PKGPN}", d) + ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, removepkg)) + if ret != 0: + bb.note("Failure removing staging package") + + stagepkg = bb.data.expand("${PSTAGE_PKG}", d) + bb.note("Removing staging package %s" % stagepkg) + os.system('rm -rf ' + stagepkg) +} + +staging_helper () { + # Assemble appropriate opkg.conf + conffile=${PSTAGE_MACHCONFIG} + mkdir -p ${DEPLOY_DIR_PSTAGE}/pstaging_lists + if [ ! -e $conffile ]; then + ipkgarchs="${BUILD_SYS}" + priority=1 + for arch in $ipkgarchs; do + echo "arch $arch $priority" >> $conffile + priority=$(expr $priority + 5) + done + fi +} + +PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_staging package_write_deb package_write_ipk package_write package_stage qa_staging" + +python do_prepackaged_stage () { + import os + + if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0": + bb.build.make_stamp("do_prepackaged_stage", d) + return + + bb.note("Uninstalling any existing package from staging...") + path = bb.data.getVar("PATH", d, 1) + removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1) + removepkg = bb.data.expand("${PSTAGE_PKGPN}", d) + lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d)) + ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, removepkg)) + bb.utils.unlockfile(lf) + if ret != 0: + bb.note("Failure attempting to remove staging package") + + stagepkg = bb.data.expand("${PSTAGE_PKG}", d) + + if os.path.exists(stagepkg): + bb.note("Following speedup\n") + path = bb.data.getVar("PATH", d, 1) + installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1) + + bb.build.exec_func("staging_helper", d) + + bb.debug(1, "Staging stuff already packaged, using that instead") + lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d)) + ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg)) + bb.utils.unlockfile(lf) + if ret != 0: + bb.note("Failure installing prestage package") + + #bb.build.make_stamp("do_prepackaged_stage", d) + #for task in bb.data.getVar("PSTAGE_TASKS_COVERED", d, 1).split(): + # bb.build.make_stamp("do_" + task, d) + bb.build.make_stamp("do_stage_package_populated", d) + + else: + bb.build.make_stamp("do_prepackaged_stage", d) +} +do_prepackaged_stage[cleandirs] = "${PSTAGE_TMPDIR_STAGE}" +do_prepackaged_stage[selfstamp] = "1" +addtask prepackaged_stage before do_fetch + +addhandler packagedstage_stampfixing_eventhandler +python packagedstage_stampfixing_eventhandler() { + from bb.event import getName + import os + + if getName(e) == "StampUpdate": + taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split() + for (fn, task) in e.targets: + # strip off 'do_' + task = task[3:] + if task in taskscovered: + stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn] + if os.path.exists(stamp): + # We're targetting a task which was skipped with packaged staging + # so we need to remove the autogenerated stamps. + for task in taskscovered: + dir = "%s.do_%s" % (e.stampPrefix[fn], task) + os.system('rm -f ' + dir) + os.system('rm -f ' + stamp) + + return NotHandled +} + +populate_staging_preamble () { + if [ "$PSTAGING_ACTIVE" = "1" ]; then + stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u || true + stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u || true + fi +} + +populate_staging_postamble () { + if [ "$PSTAGING_ACTIVE" = "1" ]; then + # list the packages currently installed in staging + ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${DEPLOY_DIR_PSTAGE}/installed-list + + set +e + stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/staging + stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross + set -e + fi +} + +do_populate_staging[lockfiles] = "${STAGING_DIR}/staging.lock" +do_populate_staging[dirs] =+ "${DEPLOY_DIR_PSTAGE}" +python do_populate_staging_prepend() { + bb.build.exec_func("populate_staging_preamble", d) +} + +python do_populate_staging_append() { + bb.build.exec_func("populate_staging_postamble", d) +} + + +staging_packager () { + + mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL + mkdir -p ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH} + + echo "Package: ${PSTAGE_PKGPN}" > ${PSTAGE_TMPDIR_STAGE}/CONTROL/control + echo "Version: ${PSTAGE_PKGVERSION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control + echo "Description: ${DESCRIPTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control + echo "Section: ${SECTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control + echo "Priority: Optional" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control + echo "Maintainer: ${MAINTAINER}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control + echo "Architecture: ${PSTAGE_PKGARCH}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control + + # Protect against empty SRC_URI + if [ "${SRC_URI}" != "" ] ; then + echo "Source: ${SRC_URI}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control + else + echo "Source: OpenEmbedded" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control + fi + + ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH} + ${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG} +} + +staging_package_installer () { + ${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG} +} + +python do_package_stage () { + if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1": + return + + # + # Handle deploy/ packages + # + bb.build.exec_func("read_subpackage_metadata", d) + stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1) + tmpdir = bb.data.getVar("TMPDIR", d, True) + packages = (bb.data.getVar('PACKAGES', d, 1) or "").split() + if len(packages) > 0: + if bb.data.inherits_class('package_ipk', d): + ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath) + if bb.data.inherits_class('package_deb', d): + debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath) + + for pkg in packages: + pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1) + if not pkgname: + pkgname = pkg + arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1) + if not arch: + arch = bb.data.getVar('PACKAGE_ARCH', d, 1) + pr = bb.data.getVar('PR_%s' % pkg, d, 1) + if not pr: + pr = bb.data.getVar('PR', d, 1) + if not packaged(pkg, d): + continue + if bb.data.inherits_class('package_ipk', d): + srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_" + arch + ".ipk", d) + srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d) + if not os.path.exists(srcfile): + bb.fatal("Package %s does not exist yet it should" % srcfile) + destpath = ipkpath + "/" + arch + "/" + bb.mkdirhier(destpath) + bb.copyfile(srcfile, destpath + srcname) + + if bb.data.inherits_class('package_deb', d): + if arch == 'all': + srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_all.deb", d) + else: + srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_${DPKG_ARCH}.deb", d) + srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d) + if not os.path.exists(srcfile): + bb.fatal("Package %s does not exist yet it should" % srcfile) + destpath = debpath + "/" + arch + "/" + bb.mkdirhier(destpath) + bb.copyfile(srcfile, destpath + srcname) + + # + # Handle stamps/ files + # + stampfn = bb.data.getVar("STAMP", d, True) + destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath)) + bb.mkdirhier(destdir) + # We need to include the package_stage stamp in the staging package so create one + bb.build.make_stamp("do_package_stage", d) + os.system("cp %s.do_* %s/" % (stampfn, destdir)) + + bb.build.exec_func("staging_helper", d) + bb.build.exec_func("staging_packager", d) + lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d)) + bb.build.exec_func("staging_package_installer", d) + bb.utils.unlockfile(lf) +} + +# +# Note an assumption here is that do_deploy runs before do_package_write/do_populate_staging +# +addtask package_stage after do_package_write do_populate_staging before do_build |