From 90b24172fe8668fe3297ccf71b9c2e12d02b0076 Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Fri, 18 Oct 2019 21:05:25 -0500 Subject: Add dependency handling Dependencies for any project can be determined by calling project_dependencies() with a given project and (optional) arguments. This is the function you will almost always want to call when you're wanting to collect a list of dependencies. The output of the topological sort is only one possible ordering of those dependencies. 'dependencies' files may now contain any number of spaces or tabs between each argument. --- libs/project | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 4 deletions(-) diff --git a/libs/project b/libs/project index b9d33114..cfb886ed 100755 --- a/libs/project +++ b/libs/project @@ -60,12 +60,89 @@ project_dependencies() { local project=$1 shift - local -a dependencies - mapfile -t dependencies < <(project_file_contents_herit "$project" "$CONFIGS" "$DEPENDENCIES" "$@") + if [[ -n "${PROJECTS_FORCE[*]}" ]]; then + local expanded + local project_force - if [[ -n ${dependencies[*]} ]]; then - printf '%s\n' "${dependencies[@]}" + for project_force in "${PROJECTS_FORCE[@]}"; do + project_arguments_expand_recursive $project_force | while IFS='' read -r expanded; do + project_dependencies_sort $expanded + done + done fi + + project_dependencies_sort "$project" "$@" | head -n -1 +} + +project_dependencies_collect() { + local project=$1 + shift + + local argument + local path + + for argument in "" "$@"; do + if [[ -z $argument ]]; then + path=$CONFIGS + else + path=$path/$argument + fi + + project_file_contents "$project" "$path" "$DEPENDENCIES" + done + + project_dependencies_collect_recursive "$project" "$@" +} + +project_dependencies_collect_recursive() { + local project=$1 + shift + + local argument + local path + + for argument in "" "$@"; do + if [[ -z $argument ]]; then + path=$CONFIGS + else + path=$path/$argument + fi + done + + project_action_helper arguments "$project" "$@" | while IFS='' read -r argument; do + project_file_contents "$project" "$path/$argument" "$DEPENDENCIES" + project_dependencies_collect_recursive "$project" "$@" "$argument" + done +} + +project_dependencies_encode_recursive() { + local project=$1 + shift + + local project_arguments=$(arguments_concat ' ' "$project" "$@") + local project_arguments_encoded=$(base64 -w0 <<< "$project_arguments") + + local dependency + + project_dependencies_collect "$project" "$@" | while IFS='' read -r dependency; do + local dependency_arguments=$(arguments_concat ' ' $dependency) + local dependency_arguments_encoded=$(base64 -w0 <<< "$dependency_arguments") + + printf '%s\n' "$project_arguments_encoded $dependency_arguments_encoded" + + project_dependencies_encode_recursive $dependency + done +} + +project_dependencies_sort() { + local project=$1 + shift + + ( + set -o pipefail + + project_dependencies_encode_recursive "$project" "$@" | tsort | base64 -d -w0 | tac + ) } project_dependencies_check() { @@ -705,6 +782,26 @@ project_arguments_targets() { fi } +project_arguments_expand_recursive() { + local project=$1 + shift + + local -a arguments + readarray -t arguments < <(project_action_helper arguments "$project" "$@") + + if [[ -z "${arguments[*]}" ]]; then + echo "$project" "$@" + + return 0 + fi + + local argument + + for argument in "${arguments[@]}"; do + project_arguments_expand_recursive "$project" "$@" "$argument" + done +} + project_usage_actions() { local project="$1" shift -- cgit v1.2.3-70-g09d2 From 250e8b28511a789a96eec2055219f04495fd45f5 Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Thu, 17 Oct 2019 20:01:10 -0500 Subject: Create new meta target working with dependencies The specified action will be taken on each of the dependencies returned by project_dependencies() where $project is the first argument to the meta-target, e.g. ./libreboot download libreboot-dependencies coreboot x200 The above command would download only the minimum projects necessary for building all x200 image configurations. To then build those dependencies: ./libreboot build libreboot-dependencies coreboot x200 And to get the final images: ./libreboot build coreboot x200 --- .../libreboot-dependencies/libreboot-dependencies | 52 ++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100755 projects/libreboot-dependencies/libreboot-dependencies diff --git a/projects/libreboot-dependencies/libreboot-dependencies b/projects/libreboot-dependencies/libreboot-dependencies new file mode 100755 index 00000000..ae50406c --- /dev/null +++ b/projects/libreboot-dependencies/libreboot-dependencies @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Copyright (C) 2019 Andrew Robbins +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +usage() { + project_usage_actions "$project" +} + +collect() { + project_dependencies "$@" +} + +download() { + project_dependencies_action_arguments "$action" "$@" +} + +extract() { + project_dependencies_action_arguments "$action" "$@" +} + +update() { + project_dependencies_action_arguments "$action" "$@" +} + +build() { + project_dependencies_action_arguments "$action" "$@" +} + +install() { + project_dependencies_action_arguments "$action" "$@" +} + +release() { + project_dependencies_action_arguments "$action" "$@" +} + +clean() { + project_dependencies_action_arguments "$action" "$@" +} -- cgit v1.2.3-70-g09d2 From 436971aa5a2cd6ba3662ee29b83f06424bc466ce Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Sat, 19 Oct 2019 17:32:48 -0500 Subject: libs/project: Simplify acting on dependencies --- libs/project | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/libs/project b/libs/project index cfb886ed..adc738fb 100755 --- a/libs/project +++ b/libs/project @@ -149,15 +149,11 @@ project_dependencies_check() { local project=$1 shift - local -a dependencies - mapfile -t dependencies < <(project_dependencies "$project" "$@") - + local dependency local -i missing=0 - for ((i = 0, nodeps = ${#dependencies[@]}; i < nodeps; ++i)); do - local -a dependency=(${dependencies[i]}) - - project_check "${dependency[0]}" || let ++missing + project_dependencies "$project" "$@" | while read -r dependency; do + project_check $dependency || let ++missing done return $missing @@ -167,16 +163,11 @@ project_dependencies_sources_check() { local project=$1 shift - local -a dependencies - mapfile -t dependencies < <(project_dependencies "$project" "$@") - + local dependency local -i missing=0 - for ((i = 0, nodeps = ${#dependencies[@]}; i < nodeps; ++i)); do - local -a dependency=(${dependencies[i]}) - - project_sources_directory_filled_check "${dependency[0]}" || - let ++missing + project_dependencies "$project" "$@" | while read -r dependency; do + project_sources_directory_filled_check $dependency || let ++missing done return $missing @@ -187,12 +178,9 @@ project_dependencies_action_arguments() { local project=$2 shift 2 - local -a dependencies - mapfile -t dependencies < <(project_dependencies "$project" "$@") - - for ((i = 0, nodeps = ${#dependencies[@]}; i < nodeps; ++i)); do - local -a dependency=(${dependencies[i]}) + local -a dependency + project_dependencies "$project" "$@" | while read -ra dependency; do if project_function_check "${dependency[0]}" "$action"; then project_action_arguments "$action" "${dependency[@]}" fi -- cgit v1.2.3-70-g09d2 From 4b8a27ade3f4cdb8b415972124e868bc2c0dd480 Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Thu, 17 Oct 2019 19:50:17 -0500 Subject: libs/project: Perform project action in subshell Whenever project_include() is called, it should be done within a subshell in order to avoid clobbering function definitions. --- libs/project | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libs/project b/libs/project index adc738fb..0bcc4d9d 100755 --- a/libs/project +++ b/libs/project @@ -282,10 +282,12 @@ project_action_arguments() { local project="$1" shift - project_include "$project" + ( + project_include "$project" - project_action_arguments_verify_recursive "$action" "$project" "$@" - project_action_arguments_recursive "$action" "$project" "$@" + project_action_arguments_verify_recursive "$action" "$project" "$@" + project_action_arguments_recursive "$action" "$project" "$@" + ) } project_action_arguments_verify_recursive() { -- cgit v1.2.3-70-g09d2 From ddce689d0582e5c3cfd353645c7d7bbb1f74f5ae Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Thu, 17 Oct 2019 20:14:42 -0500 Subject: Remove dependencies_check() from project scripts Removing the check function allows dependencies() to be executed, printing the list of dependencies. It was not executed before due to the way check functions are handled; check functions are executed before functions with the same name (minus the "_check" suffix) in order to determine whether calling the function is necessary. --- projects/coreboot/coreboot | 4 ---- projects/cros-ec/cros-ec | 4 ---- projects/dejavu-fonts/dejavu-fonts | 4 ---- projects/depthcharge/depthcharge | 4 ---- projects/grub/grub | 4 ---- projects/mosys/mosys | 4 ---- 6 files changed, 24 deletions(-) diff --git a/projects/coreboot/coreboot b/projects/coreboot/coreboot index 046a4de1..e224cccc 100755 --- a/projects/coreboot/coreboot +++ b/projects/coreboot/coreboot @@ -28,10 +28,6 @@ dependencies() { project_dependencies "$project" "$@" } -dependencies_check() { - project_dependencies_check "$project" "$@" -} - download() { local repository=$project diff --git a/projects/cros-ec/cros-ec b/projects/cros-ec/cros-ec index 1df44feb..55dc676e 100755 --- a/projects/cros-ec/cros-ec +++ b/projects/cros-ec/cros-ec @@ -23,10 +23,6 @@ dependencies() { project_dependencies "$project" "$@" } -dependencies_check() { - project_dependencies_check "$project" "$@" -} - usage() { project_usage_actions "$project" project_usage_arguments "$project" "$@" diff --git a/projects/dejavu-fonts/dejavu-fonts b/projects/dejavu-fonts/dejavu-fonts index c3d76cf9..1e91362f 100755 --- a/projects/dejavu-fonts/dejavu-fonts +++ b/projects/dejavu-fonts/dejavu-fonts @@ -23,10 +23,6 @@ dependencies() { project_dependencies "$project" "$@" } -dependencies_check() { - project_dependencies_check "$project" "$@" -} - usage() { project_usage_actions "$project" project_usage_arguments "$project" "$@" diff --git a/projects/depthcharge/depthcharge b/projects/depthcharge/depthcharge index 6a58d305..17057e69 100755 --- a/projects/depthcharge/depthcharge +++ b/projects/depthcharge/depthcharge @@ -23,10 +23,6 @@ dependencies() { project_dependencies "$project" "$@" } -dependencies_check() { - project_dependencies_check "$project" "$@" -} - usage() { project_usage_actions "$project" project_usage_arguments "$project" "$@" diff --git a/projects/grub/grub b/projects/grub/grub index 3daca888..75d29380 100755 --- a/projects/grub/grub +++ b/projects/grub/grub @@ -23,10 +23,6 @@ dependencies() { project_dependencies "$project" "$@" } -dependencies_check() { - project_dependencies_check "$project" "$@" -} - usage() { project_usage_actions "$project" project_usage_arguments "$project" "$@" diff --git a/projects/mosys/mosys b/projects/mosys/mosys index fecf9267..4407d784 100755 --- a/projects/mosys/mosys +++ b/projects/mosys/mosys @@ -23,10 +23,6 @@ dependencies() { project_dependencies "$project" "$@" } -dependencies_check() { - project_dependencies_check "$project" "$@" -} - download() { local repository=$project -- cgit v1.2.3-70-g09d2 From 70fc19fb1c86c8a9d3fdad26db2c5009f8db3565 Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Thu, 17 Oct 2019 20:51:07 -0500 Subject: Add missing QEMU payload dependencies files --- .../configs/qemu_i440fx_piix4/16MiB/corebootfb/grub/dependencies | 1 + .../configs/qemu_i440fx_piix4/16MiB/corebootfb/seabios/dependencies | 1 + .../coreboot/configs/qemu_i440fx_piix4/16MiB/textmode/grub/dependencies | 1 + .../configs/qemu_i440fx_piix4/16MiB/textmode/seabios/dependencies | 1 + .../coreboot/configs/qemu_i440fx_piix4/8MiB/corebootfb/grub/dependencies | 1 + .../configs/qemu_i440fx_piix4/8MiB/corebootfb/seabios/dependencies | 1 + .../coreboot/configs/qemu_i440fx_piix4/8MiB/textmode/grub/dependencies | 1 + .../configs/qemu_i440fx_piix4/8MiB/textmode/seabios/dependencies | 1 + .../coreboot/configs/qemu_q35_ich9/16MiB/corebootfb/grub/dependencies | 1 + .../coreboot/configs/qemu_q35_ich9/16MiB/corebootfb/seabios/dependencies | 1 + projects/coreboot/configs/qemu_q35_ich9/16MiB/textmode/grub/dependencies | 1 + .../coreboot/configs/qemu_q35_ich9/16MiB/textmode/seabios/dependencies | 1 + .../coreboot/configs/qemu_q35_ich9/8MiB/corebootfb/grub/dependencies | 1 + .../coreboot/configs/qemu_q35_ich9/8MiB/corebootfb/seabios/dependencies | 1 + projects/coreboot/configs/qemu_q35_ich9/8MiB/textmode/grub/dependencies | 1 + .../coreboot/configs/qemu_q35_ich9/8MiB/textmode/seabios/dependencies | 1 + 16 files changed, 16 insertions(+) create mode 100644 projects/coreboot/configs/qemu_i440fx_piix4/16MiB/corebootfb/grub/dependencies create mode 100644 projects/coreboot/configs/qemu_i440fx_piix4/16MiB/corebootfb/seabios/dependencies create mode 100644 projects/coreboot/configs/qemu_i440fx_piix4/16MiB/textmode/grub/dependencies create mode 100644 projects/coreboot/configs/qemu_i440fx_piix4/16MiB/textmode/seabios/dependencies create mode 100644 projects/coreboot/configs/qemu_i440fx_piix4/8MiB/corebootfb/grub/dependencies create mode 100644 projects/coreboot/configs/qemu_i440fx_piix4/8MiB/corebootfb/seabios/dependencies create mode 100644 projects/coreboot/configs/qemu_i440fx_piix4/8MiB/textmode/grub/dependencies create mode 100644 projects/coreboot/configs/qemu_i440fx_piix4/8MiB/textmode/seabios/dependencies create mode 100644 projects/coreboot/configs/qemu_q35_ich9/16MiB/corebootfb/grub/dependencies create mode 100644 projects/coreboot/configs/qemu_q35_ich9/16MiB/corebootfb/seabios/dependencies create mode 100644 projects/coreboot/configs/qemu_q35_ich9/16MiB/textmode/grub/dependencies create mode 100644 projects/coreboot/configs/qemu_q35_ich9/16MiB/textmode/seabios/dependencies create mode 100644 projects/coreboot/configs/qemu_q35_ich9/8MiB/corebootfb/grub/dependencies create mode 100644 projects/coreboot/configs/qemu_q35_ich9/8MiB/corebootfb/seabios/dependencies create mode 100644 projects/coreboot/configs/qemu_q35_ich9/8MiB/textmode/grub/dependencies create mode 100644 projects/coreboot/configs/qemu_q35_ich9/8MiB/textmode/seabios/dependencies diff --git a/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/corebootfb/grub/dependencies b/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/corebootfb/grub/dependencies new file mode 100644 index 00000000..6640cb8d --- /dev/null +++ b/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/corebootfb/grub/dependencies @@ -0,0 +1 @@ +grub coreboot diff --git a/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/corebootfb/seabios/dependencies b/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/corebootfb/seabios/dependencies new file mode 100644 index 00000000..41f077c0 --- /dev/null +++ b/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/corebootfb/seabios/dependencies @@ -0,0 +1 @@ +seabios diff --git a/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/textmode/grub/dependencies b/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/textmode/grub/dependencies new file mode 100644 index 00000000..6640cb8d --- /dev/null +++ b/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/textmode/grub/dependencies @@ -0,0 +1 @@ +grub coreboot diff --git a/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/textmode/seabios/dependencies b/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/textmode/seabios/dependencies new file mode 100644 index 00000000..41f077c0 --- /dev/null +++ b/projects/coreboot/configs/qemu_i440fx_piix4/16MiB/textmode/seabios/dependencies @@ -0,0 +1 @@ +seabios diff --git a/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/corebootfb/grub/dependencies b/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/corebootfb/grub/dependencies new file mode 100644 index 00000000..6640cb8d --- /dev/null +++ b/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/corebootfb/grub/dependencies @@ -0,0 +1 @@ +grub coreboot diff --git a/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/corebootfb/seabios/dependencies b/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/corebootfb/seabios/dependencies new file mode 100644 index 00000000..41f077c0 --- /dev/null +++ b/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/corebootfb/seabios/dependencies @@ -0,0 +1 @@ +seabios diff --git a/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/textmode/grub/dependencies b/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/textmode/grub/dependencies new file mode 100644 index 00000000..6640cb8d --- /dev/null +++ b/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/textmode/grub/dependencies @@ -0,0 +1 @@ +grub coreboot diff --git a/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/textmode/seabios/dependencies b/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/textmode/seabios/dependencies new file mode 100644 index 00000000..41f077c0 --- /dev/null +++ b/projects/coreboot/configs/qemu_i440fx_piix4/8MiB/textmode/seabios/dependencies @@ -0,0 +1 @@ +seabios diff --git a/projects/coreboot/configs/qemu_q35_ich9/16MiB/corebootfb/grub/dependencies b/projects/coreboot/configs/qemu_q35_ich9/16MiB/corebootfb/grub/dependencies new file mode 100644 index 00000000..6640cb8d --- /dev/null +++ b/projects/coreboot/configs/qemu_q35_ich9/16MiB/corebootfb/grub/dependencies @@ -0,0 +1 @@ +grub coreboot diff --git a/projects/coreboot/configs/qemu_q35_ich9/16MiB/corebootfb/seabios/dependencies b/projects/coreboot/configs/qemu_q35_ich9/16MiB/corebootfb/seabios/dependencies new file mode 100644 index 00000000..41f077c0 --- /dev/null +++ b/projects/coreboot/configs/qemu_q35_ich9/16MiB/corebootfb/seabios/dependencies @@ -0,0 +1 @@ +seabios diff --git a/projects/coreboot/configs/qemu_q35_ich9/16MiB/textmode/grub/dependencies b/projects/coreboot/configs/qemu_q35_ich9/16MiB/textmode/grub/dependencies new file mode 100644 index 00000000..6640cb8d --- /dev/null +++ b/projects/coreboot/configs/qemu_q35_ich9/16MiB/textmode/grub/dependencies @@ -0,0 +1 @@ +grub coreboot diff --git a/projects/coreboot/configs/qemu_q35_ich9/16MiB/textmode/seabios/dependencies b/projects/coreboot/configs/qemu_q35_ich9/16MiB/textmode/seabios/dependencies new file mode 100644 index 00000000..41f077c0 --- /dev/null +++ b/projects/coreboot/configs/qemu_q35_ich9/16MiB/textmode/seabios/dependencies @@ -0,0 +1 @@ +seabios diff --git a/projects/coreboot/configs/qemu_q35_ich9/8MiB/corebootfb/grub/dependencies b/projects/coreboot/configs/qemu_q35_ich9/8MiB/corebootfb/grub/dependencies new file mode 100644 index 00000000..6640cb8d --- /dev/null +++ b/projects/coreboot/configs/qemu_q35_ich9/8MiB/corebootfb/grub/dependencies @@ -0,0 +1 @@ +grub coreboot diff --git a/projects/coreboot/configs/qemu_q35_ich9/8MiB/corebootfb/seabios/dependencies b/projects/coreboot/configs/qemu_q35_ich9/8MiB/corebootfb/seabios/dependencies new file mode 100644 index 00000000..41f077c0 --- /dev/null +++ b/projects/coreboot/configs/qemu_q35_ich9/8MiB/corebootfb/seabios/dependencies @@ -0,0 +1 @@ +seabios diff --git a/projects/coreboot/configs/qemu_q35_ich9/8MiB/textmode/grub/dependencies b/projects/coreboot/configs/qemu_q35_ich9/8MiB/textmode/grub/dependencies new file mode 100644 index 00000000..6640cb8d --- /dev/null +++ b/projects/coreboot/configs/qemu_q35_ich9/8MiB/textmode/grub/dependencies @@ -0,0 +1 @@ +grub coreboot diff --git a/projects/coreboot/configs/qemu_q35_ich9/8MiB/textmode/seabios/dependencies b/projects/coreboot/configs/qemu_q35_ich9/8MiB/textmode/seabios/dependencies new file mode 100644 index 00000000..41f077c0 --- /dev/null +++ b/projects/coreboot/configs/qemu_q35_ich9/8MiB/textmode/seabios/dependencies @@ -0,0 +1 @@ +seabios -- cgit v1.2.3-70-g09d2