#!/usr/bin/env bash ## Implements JSON generation for GitHub Actions matrixes. ## nb: this is an example of what not to use bash for. ## The same code in Python would be 10x shorter and run infinitely faster. ## Please be patient while reading this. # TAG selection, using environment variables: # - CI_TAGS should be a space-separated list of tags to include in the matrixes # - Those will be matched against each flavor's TAG # - Any flavor with a matching tag will be included in the matrixes # GH Runner selection, using environment variables (in order of specificity, for AMD64, same applies to ARM64 variant): ## Kernels: (1) # - CI_RUNNER_KERNEL_SOURCE_AMD64 is the runner to use for amd64 kernel builds for kernels built from source # - CI_RUNNER_KERNEL_EXTERNAL_AMD64 is the runner to use for amd64 kernel builds for kernels obtained externally # - CI_RUNNER_KERNEL_AMD64 is the runner to use for amd64 kernel builds # - CI_RUNNER_KERNEL_SOURCE is the runner to use for kernels built from source # - CI_RUNNER_KERNEL_EXTERNAL is the runner to use for kernels obtained externally # - CI_RUNNER_KERNEL is the runner to use for all kernels # - CI_RUNNER_AMD64 is the runner to use for all AMD64 things # - CI_RUNNER is the runner to use for everything ## LinuxKit/Hook: (2) # - CI_RUNNER_LK_AMD64 is the runner to use for amd64 linuxkit builds # - CI_RUNNER_LK is the runner to use for all linuxkit builds # - CI_RUNNER_AMD64 is the runner to use for all AMD64 things # - CI_RUNNER is the runner to use for everything ## LK containers (hook-bootkit, hook-docker, hook-udev): (3): # - CI_RUNNER_LK_CONTAINERS_AMD64 is the runner to use for amd64 linuxkit containers builds # - CI_RUNNER_LK_CONTAINERS is the runner to use for all linuxkit containers builds # - CI_RUNNER_AMD64 is the runner to use for all AMD64 things # - CI_RUNNER is the runner to use for everything function output_gha_matrixes() { declare -r CI_TAGS="${CI_TAGS:-"standard dev armbian-sbc armbian-uefi"}" # shellcheck disable=SC2206 # yes, we want to split declare -g -a -r CI_TAGS_ARRAY=(${CI_TAGS}) log info "CI_TAGS: ${CI_TAGS_ARRAY[*]}" declare -A all_arches=() # accumulator for every arch that is selected to be built declare full_json="" prepare_json_matrix "KERNEL" # sets full_json and adds to all_arches declare kernels_json="${full_json}" declare full_json="" prepare_json_matrix "LK" # sets full_json and adds to all_arches declare lk_hooks_json="${full_json}" declare full_json="" prepare_json_matrix_lkcontainers "LK_CONTAINERS" # reads all_arches's keys and sets full_json declare lkcontainers_json="${full_json}" declare full_json="" prepare_json_matrix_bootable "BOOTABLE" # lists boards under bootables declare bootable_json="${full_json}" log info "kernels_json to: ${kernels_json}" log info "lk_hooks_json to: ${lk_hooks_json}" log info "lkcontainers_json to: ${lkcontainers_json}" log info "bootable_json to: ${bootable_json}" # If under GHA, set a GHA output variable. if [[ -n "${GITHUB_OUTPUT}" ]]; then # shellcheck disable=SC2129 # no, thanks, shellcheck. echo "kernels_json=${kernels_json}" >> "${GITHUB_OUTPUT}" echo "lk_hooks_json=${lk_hooks_json}" >> "${GITHUB_OUTPUT}" echo "lkcontainers_json=${lkcontainers_json}" >> "${GITHUB_OUTPUT}" echo "bootable_json=${bootable_json}" >> "${GITHUB_OUTPUT}" fi echo -n "${bootable_json}" # output the hooks matrix to stdout, for cli/jq etc #echo -n "${lk_hooks_json}" # output the hooks matrix to stdout, for cli/jq etc } function prepare_json_matrix() { declare -r matrix_type="${1}" declare -a json_items=() declare kernel for kernel in "${inventory_ids[@]}"; do declare -A kernel_info get_kernel_info_dict "${kernel}" declare -a find_runner_params=("${matrix_type}" "${kernel_info[DOCKER_ARCH]}") if [[ "${matrix_type}" == "KERNEL" ]]; then # special case for kernel builds # if TYPE is set, add it as extra param to find_runner if [[ -n "${kernel_info[TYPE]}" ]]; then log debug "Adding TYPE='${kernel_info[TYPE]}' to runner search for kernel '${kernel}'" find_runner_params+=("${kernel_info[TYPE]}") fi if [[ -n "${kernel_info[USE_KERNEL_ID]}" ]]; then # if USE_KERNEL_ID is set, skip this kernel log info "Skipping build of kernel '${kernel}' due to it having USE_KERNEL_ID set to '${kernel_info[USE_KERNEL_ID]}'" continue fi fi if json_matrix_tag_match "${kernel_info[TAG]}"; then declare runner="unknown-runner" runner="$(json_matrix_find_runner "${find_runner_params[@]}")" declare gha_cache="yes" # always use GH cache; hitting DockerHub for linuxkit images is prone to rate limiting all_arches["${kernel_info[DOCKER_ARCH]}"]=1 json_items+=("{\"kernel\":\"${kernel}\",\"arch\":\"${kernel_info[ARCH]}\",\"docker_arch\":\"${kernel_info[DOCKER_ARCH]}\",\"build_iso\":\"${kernel_info[SUPPORTS_ISO]}\",\"runner\":${runner},\"gha_cache\":\"${gha_cache}\"}") fi done prepare_json_array_to_json return 0 } function prepare_json_matrix_bootable() { declare -r matrix_type="${1}" declare -a json_items=() declare bootable_id for bootable_id in "${bootable_inventory_ids[@]}"; do declare -g -A bootable_info=() get_bootable_info_dict "${bootable_id}" declare -g -A kernel_info=() get_kernel_info_dict "${bootable_info['INVENTORY_ID']}" if json_matrix_tag_match "${bootable_info[TAG]}"; then declare runner="unknown-runner" runner="$(json_matrix_find_runner "${matrix_type}" "${kernel_info[DOCKER_ARCH]}")" declare bootable_list_func="${bootable_info['BOOTABLE_LIST_FUNC']}" if [[ -z "${bootable_list_func}" ]]; then log error "No BOOTABLE_LIST_FUNC found for bootable '${bootable_id}'" exit 1 fi declare -A bootable_boards=() log debug "Calling bootable list function: ${bootable_list_func}" "${bootable_list_func}" declare -a bootable_board_ids=("${!bootable_boards[@]}") declare bootable_board_id for bootable_board_id in "${bootable_board_ids[@]}"; do declare board_opts="${bootable_boards[${bootable_board_id}]}" json_items+=("{\"kernel\":\"${bootable_info['INVENTORY_ID']}\",\"bootable\":\"${bootable_id}\",\"arch\":\"${kernel_info[ARCH]}\",\"docker_arch\":\"${kernel_info[DOCKER_ARCH]}\",\"board_id\":\"${bootable_board_id}\",\"board_opts\":\"${board_opts}\",\"runner\":${runner}}") done fi done prepare_json_array_to_json return 0 } function prepare_json_matrix_lkcontainers() { declare -r matrix_type="${1}" declare -a unique_arches=("${!all_arches[@]}") # get an array with the KEYS of all_arches dict declare -a json_items=() declare kernel for one_arch in "${unique_arches[@]}"; do declare runner="unknown-runner" runner="$(json_matrix_find_runner "${matrix_type}" "${one_arch}")" json_items+=("{\"docker_arch\":\"${one_arch}\",\"runner\":${runner}}") done prepare_json_array_to_json return 0 } # takes json_items array, outputs full_json single-line string; massage the array into JSON (comma handling) function prepare_json_array_to_json() { declare this_json="[" declare -i counter=0 declare json_item for json_item in "${json_items[@]}"; do this_json+="${json_item}" [[ $counter -lt $((${#json_items[@]} - 1)) ]] && this_json+="," # append a comma if not the last element counter+=1 done this_json+="]" if [[ "${skip_jq:-"no"}" == "yes" ]]; then full_json="${this_json}" return 0 fi log debug "Raw json before jq: ${this_json}" full_json="$(echo "${this_json}" | jq -c .)" # Pass it through jq for correctness check & compaction return 0 } # This is probably the slowest bash code ever written function json_matrix_tag_match() { declare current_tags="${1}" # shellcheck disable=SC2206 # we want to split the string into an array, thanks declare -a current_tags_array=(${current_tags}) # if any of current_tags_array in in CI_TAGS_ARRAY, we've a match for tag in "${current_tags_array[@]}"; do for ci_tag in "${CI_TAGS_ARRAY[@]}"; do if [[ "${tag}" == "${ci_tag}" ]]; then log debug "Tag '${tag}' matches CI_TAG '${ci_tag}'..." return 0 fi done done log debug "No tags matched." return 1 } function json_matrix_find_runner() { declare matrix_type="${1}" declare docker_arch="${2}" declare matrix_extra="${3:-""}" declare runner="ubuntu-latest" if [[ "${docker_arch}" == "arm64" ]]; then runner="ubuntu-24.04-arm" fi log debug "Finding runner for matrix type '${matrix_type}' and docker arch '${docker_arch}'" declare -a vars_to_try=( "CI_RUNNER_${matrix_type^^}_${docker_arch^^}" "CI_RUNNER_${matrix_type^^}" "CI_RUNNER_${docker_arch^^}" "CI_RUNNER" ) if [[ -n "${matrix_extra}" ]]; then log debug "Including matrix extra '${matrix_extra}' in runner search" vars_to_try=( "CI_RUNNER_${matrix_type^^}_${matrix_extra^^}_${docker_arch^^}" "CI_RUNNER_${matrix_type^^}_${matrix_extra^^}" "CI_RUNNER_${matrix_type^^}_${docker_arch^^}" "CI_RUNNER_${matrix_type^^}" "CI_RUNNER_${docker_arch^^}" "CI_RUNNER" ) fi for var in "${vars_to_try[@]}"; do log debug "Checking var '${var}'" if [[ -n "${!var}" && "${!var}" != "" ]]; then # if var is set, and not empty... log debug "Found runner '${!var}' for matrix type '${matrix_type}' and docker arch '${docker_arch}' via var '${var}'" runner="${!var}" break fi done log debug "Found runner '${runner}' for matrix type '${matrix_type}' and docker arch '${docker_arch}'" # shellcheck disable=SC2206 # split by spaces, make it a json array declare -a json_items_bare=(${runner}) # wrap each json_items array item in double quotes declare -a json_items=() if [[ "${runner}" == "ubuntu-"* ]]; then # if not using a GH-hosted runner, auto-add the "self-hosted" member : else if [[ "${CI_RUNNERS_SELF_HOSTED_TAG:-"self-hosted"}" != "none" ]]; then log debug "Adding 'self-hosted' tag to runner '${runner}'" json_items+=("\"${CI_RUNNERS_SELF_HOSTED_TAG:-"self-hosted"}\"") else log debug "Not adding 'self-hosted' tag to runner '${runner}' due to CI_RUNNERS_SELF_HOSTED_TAG being set to 'none'" fi fi for item in "${json_items_bare[@]}"; do json_items+=("\"${item}\"") done declare full_json="" skip_jq="yes" prepare_json_array_to_json # skip jq; this is only a json fragment echo -n "${full_json}" return 0 }