diff --git a/scripts/release/README.md b/scripts/release/README.md new file mode 100644 index 0000000000000000000000000000000000000000..631fa72b2f689d8a6cb5f7898eed07b6a4db09de --- /dev/null +++ b/scripts/release/README.md @@ -0,0 +1,183 @@ +# NOOBS + +
+ Recalbox entries for os_list_v3.json + + > ℹ️ This can only be added to the [official `os_list_v3.json`](https://downloads.raspberrypi.org/os_list_v3.json) by Raspberry Foundation engineers + + ```json + { + "os_list": [ + { + "os_name": "Recalbox - Pi0/1", + "description": "The official retro-gaming OS! Turn your Raspberry Pi into an all-in-one and plug-n-play retro-gaming console, supporting 100+ gaming systems!", + "nominal_size": 4096, + "icon": "https://download.recalbox.com/noobs/recalboxOS.png", + "marketing_info": "https://download.recalbox.com/noobs/marketing.tar", + "partition_setup": "https://download.recalbox.com/noobs/partition_setup.sh", + "partitions_info": "https://download.recalbox.com/noobs/rpi1/partitions.json", + "os_info": "https://download.recalbox.com/noobs/os.json", + "supported_models": [ + "Pi Model", + "Pi Compute Module Rev", + "Pi Zero" + ], + "tarballs": [ + "https://recalbox-releases.s3.nl-ams.scw.cloud/stable/v1/noobs/rpi1/boot.tar.xz", + "https://recalbox-releases.s3.nl-ams.scw.cloud/stable/v1/noobs/rpi1/root.tar.xz" + ] + }, + { + "os_name": "Recalbox - Pi2", + "description": "The official retro-gaming OS! Turn your Raspberry Pi into an all-in-one and plug-n-play retro-gaming console, supporting 100+ gaming systems!", + "nominal_size": 4096, + "icon": "https://download.recalbox.com/noobs/recalboxOS.png", + "marketing_info": "https://download.recalbox.com/noobs/marketing.tar", + "partition_setup": "https://download.recalbox.com/noobs/partition_setup.sh", + "partitions_info": "https://download.recalbox.com/noobs/rpi2/partitions.json", + "os_info": "https://download.recalbox.com/noobs/os.json", + "supported_models": [ + "Pi 2" + ], + "tarballs": [ + "https://recalbox-releases.s3.nl-ams.scw.cloud/stable/v1/noobs/rpi2/boot.tar.xz", + "https://recalbox-releases.s3.nl-ams.scw.cloud/stable/v1/noobs/rpi2/root.tar.xz" + ] + }, + { + "os_name": "Recalbox - Pi3", + "description": "The official retro-gaming OS! Turn your Raspberry Pi into an all-in-one and plug-n-play retro-gaming console, supporting 100+ gaming systems!", + "nominal_size": 4096, + "icon": "https://download.recalbox.com/noobs/recalboxOS.png", + "marketing_info": "https://download.recalbox.com/noobs/marketing.tar", + "partition_setup": "https://download.recalbox.com/noobs/partition_setup.sh", + "partitions_info": "https://download.recalbox.com/noobs/rpi3/partitions.json", + "os_info": "https://download.recalbox.com/noobs/os.json", + "supported_models": [ + "Pi 3", + "Pi Compute Module 3" + ], + "tarballs": [ + "https://recalbox-releases.s3.nl-ams.scw.cloud/stable/v1/noobs/rpi3/boot.tar.xz", + "https://recalbox-releases.s3.nl-ams.scw.cloud/stable/v1/noobs/rpi3/root.tar.xz" + ] + } + ] + } + ``` +
+ +*TL;DR* NOOBS configuration is completely undocumented. Here is our attempt to document knowledge we gathered by digging into NOOBS source code, reading other OS entries in the [official `os_list_v3.json`](https://downloads.raspberrypi.org/os_list_v3.json) and experimenting. + +## `os_list_v3.json` + +* `os_name` ⇒ used as base name for the OS +* `nominal_size` ⇒ used by NOOBS to determine if the OS can be installed based on the available disk space +* `os_info` ⇒ URL of file downloaded as `os.json` +* `partitions_info` ⇒ URL of file downloaded as `partitions.json` +* `marketing_info` ⇒ URL of file downloaded as `marketing.tar` (images in it should be at least 400x200 pixels) +* `partition_setup` ⇒ URL of file downloaded as `partition_setup.sh` +* `icon` ⇒ URL of file downloaded as `icon.png` (must be 40x40 pixels) +* `flavours` ⇒ array of OS flavours, each item can contain (those fields will be merged with top-level fields and each flavour will be available as a separate OS in NOOBS): + * `name` + * `description` + * `icon` + * `feature_level` ⇒ ??? +* `supported_models` ⇒ array of strings which at least one must be present in `/proc/device-tree/model` for the OS to be considered compatible with the hardware + * `Pi Zero` matches all Pi Zero models (Zero, Zero W, Zero WH) + * `Pi Model` matches all Pi1 models (A, B, A+, B+) + * `Pi Model A` matches Pi1 A models (A, A+) + * `Pi Model A+` matches Pi1 A+ model specifically + * `Pi Model B` matches Pi1 B models (B, B+) + * `Pi Model B+` matches Pi1 B+ model specifically + * `Pi 2` matches all Pi2 models (there's only one at the moment, actually: B) + * `Pi 3` matches all Pi3 models (A+, B, B+) + * `Pi 3 Model A` matches Pi3 A models (there's only one at the moment, actually: A+) + * `Pi 3 Model B` matches Pi3 B models (Pi3B, Pi3B+) + * `Pi 3 Model B+` matches Pi3B+ model specifically + * `Pi 4` matches all Pi4 models (there's only one at the moment, actually: B) + * `Pi Compute Module` matches all Pi CM models (CM1, CM3, CM3 Lite, CM3+, CM3+ Lite) + * `Pi Compute Module Rev` matches Pi CM1 model specifically + * `Pi Compute Module 3` matches Pi CM3 models (CM3, CM3+, maybe Lite variants too) + * `Pi Compute Module 3+` matches PiCM3+ model specifically (maybe CM3+ Lite variant too) + * not sure how to match CM Lite variants specifically: no one does that and I don't own one to see what `/proc/device-tree/model` contains + +*
+ 💀 Legacy fields + + * `group` ⇒ seen in existing entries of `os_list_v3.json` but there's no sign of it in NOOBS code +
+ +## `partitions.json` + +This file is downloaded from URL defined by the `partitions_info` field in `os_list_v3.json` and is used to configure how `parted` and `mkfs` will partition the disk. + +Fields read from `partitioninfo.cpp`: +* `filesystem_type` ⇒ `fat`/`swap`/`ntfs` (default is for "Linux native", _a.k.a_ ext3/4) +* `mkfs_options` ⇒ additional options passed to `mkfs` +* `label` ⇒ partition label, must match a tarball filename (from the `tarballs` field in `os_list_v3.json`) +* `tarball` +* `want_maximised` ⇒ take all available space +* `empty_fs` ⇒ created empty +* `offset_in_sectors` +* `partition_size_nominal` ⇒ partition size in MB +* `requires_partition_number` +* `uncompressed_tarball_size` ⇒ uncompressed tarball size in MB (used to display accurante download and installation progress) +* `active` + +## `os.json` + +This file is downloaded from URL defined by the `os_info` field in `os_list_v3.json` and is used to display some additional information in NOOBS graphical interface. + +Fields read from `osinfo.cpp`: +* `name` ⇒ name of the OS +* `version` ⇒ version number or version string +* `description` ⇒ description of the OS, appended to the name as a new line +* `release_date` ⇒ release date in `YYYY-MM-DD` format +* `bootable` +* `riscos_offset` +*
+ 💀 Legacy fields + + * `supported_revisions` ⇒ used for hardware compatibility detection ([replaced in 2016 by `supported_models`](https://github.com/raspberrypi/noobs/commit/11ba0b51db6fa48c85b5c8cd13ca757f64a6bb96)) + * `supported_hex_revisions` ⇒ used for hardware compatibility detection ([replaced in 2016 by `supported_models`](https://github.com/raspberrypi/noobs/commit/11ba0b51db6fa48c85b5c8cd13ca757f64a6bb96)) + * `kernel` ⇒ seen in existing entries of `os_list_v3.json` but there's no sign of it in NOOBS code + * `url` ⇒ seen in existing entries of `os_list_v3.json` but there's no sign of it in NOOBS code +
+ +# Raspberry Pi Imager + +
+ Recalbox entry for os_list_imagingutility.json + + > ℹ️ This can only be added to the [official `os_list_imagingutility.json`](https://downloads.raspberrypi.org/os_list_imagingutility.json) by Raspberry Foundation engineers + + ```json + { + "os_list": [ + { + "name": "Recalbox", + "description": "The official retro-gaming OS! Turn your Raspberry Pi into an all-in-one and plug-n-play retro-gaming console, supporting 100+ gaming systems!", + "icon": "https://download.recalbox.com/raspi-imager/recalbox.svg", + "subitems_url": "https://download.recalbox.com/raspi-imager/os_list_imagingutility_recalbox.json" + } + ] + } + ``` +
+ +Raspberry Pi Imager is much more flexible than NOOBS' one, because it allows us to host (and thus dynamically update!) our own configuration file via `subitems_url`. + +## `os_list_imagingutility.json` + +Each entry in this file represents an OS. Here is the list of fields each entry can define: + * `name` ⇒ name of the OS + * `description` ⇒ longer description of the OS + * `icon` ⇒ logo of the OS + * `extract_size` ⇒ size of image in bytes, once uncompressed + * `extract_sha256` ⇒ SHA256 hash of image, once uncompressed + * `image_download_size` ⇒ size of downloaded image (compressed) + * `image_download_sha256` ⇒ SHA256 hash of image (compressed) + * `release_date` ⇒ release date in `YYY-MM-DD` format + * `subitems_url` ⇒ URL to a file describing "flavours" of this OS: they can define exactly the same fields and will replace the entry declaring `subitems_url` + diff --git a/scripts/release/generate_external_installer_assets.sh b/scripts/release/generate_external_installer_assets.sh new file mode 100755 index 0000000000000000000000000000000000000000..1f33b907d2492744cb843420f6940604871d1bcf --- /dev/null +++ b/scripts/release/generate_external_installer_assets.sh @@ -0,0 +1,151 @@ +#!/bin/bash + +## FUNCTIONS ## + +function exitWithUsage { + echo "Usage: $0 --images-dir --destination-dir " + echo + echo "This script generates assets for external OS installers (imagers)" + echo "such as NOOBS, PINN or Raspberry Pi Imager." + echo + echo " --images-dir path to Recalbox images" + echo " --destination-dir path where assets will be generated" + echo + echo "The expects the following file hierarchy:" + echo + echo " /" + echo " ├─ rpi1/" + echo " │ └─ boot.tar.xz" + echo " │ └─ root.tar.xz" + echo " │ └─ recalbox-rpi1.img.xz" + echo " ├─ rpi2/" + echo " │ └─ boot.tar.xz" + echo " │ └─ root.tar.xz" + echo " │ └─ recalbox-rpi2.img.xz" + echo " └─ rpi3/" + echo " └─ boot.tar.xz" + echo " └─ root.tar.xz" + echo " └─ recalbox-rpi3.img.xz" + echo + exit 64 +} + +function generateNoobsAssets { + local templateDir="$(dirname $(readlink -f $0))/templates/noobs" + local destinationDir="${params[destinationDir]}/noobs" + declare -A metadata + + echo ">>> Generating assets for NOOBS (in ${destinationDir})" + + # Gather required information from images directory + + metadata[version]=${CI_COMMIT_REF_NAME} + metadata[releaseDate]=$(date +%Y-%m-%d) + + for arch in rpi1 rpi2 rpi3; do + # Fetch info regarding extracted boot and root images (boot.tar and root.tar, after XZ decompression) + unxz --keep "${params[imagesDir]}/${arch}/boot.tar.xz" + unxz --keep "${params[imagesDir]}/${arch}/root.tar.xz" + metadata["${arch}BootUncompressedTarballSize"]=$(du -m "${params[imagesDir]}/${arch}/boot.tar" | cut -f 1) + metadata["${arch}RootUncompressedTarballSize"]=$(du -m "${params[imagesDir]}/${arch}/root.tar" | cut -f 1) + rm "${params[imagesDir]}/${arch}/boot.tar" + rm "${params[imagesDir]}/${arch}/root.tar" + done + + # Create assets in destination directory + + mkdir -p ${destinationDir} + + cp -n "${templateDir}/recalbox.png" "${destinationDir}/recalbox.png" + cp -n "${templateDir}/marketing.tar" "${destinationDir}/marketing.tar" + cp -n "${templateDir}/partition_setup.sh" "${destinationDir}/partition_setup.sh" + + cat "${templateDir}/os.json" \ + | sed -e "s|{{version}}|${metadata[version]}|" \ + -e "s|{{releaseDate}}|${metadata[releaseDate]}|" \ + > "${destinationDir}/os.json" + + for arch in rpi1 rpi2 rpi3; do + mkdir -p "${destinationDir}/${arch}" + cat "${templateDir}/partitions.json" \ + | sed -e "s|{{bootUncompressedTarballSize}}|${metadata["${arch}BootUncompressedTarballSize"]}|" \ + -e "s|{{rootUncompressedTarballSize}}|${metadata["${arch}RootUncompressedTarballSize"]}|" \ + > "${destinationDir}/${arch}/partitions.json" + done +} + +function generateRaspberryPiImagerAssets { + local templateDir="$(dirname $(readlink -f $0))/templates/raspi_imager" + local destinationDir="${params[destinationDir]}/raspi-imager" + declare -A metadata + + echo ">>> Generating assets for Raspberry Pi Imager (in ${destinationDir})" + + # Gather required information from images directory + + metadata[version]=${CI_COMMIT_REF_NAME} + metadata[releaseDate]=$(date +%Y-%m-%d) + + for arch in rpi1 rpi2 rpi3; do + # Fetch info regarding image downloads (XZ-compressed Recalbox image) + metadata["${arch}ImageDownloadSize"]=$(stat --format=%s "${params[imagesDir]}/${arch}/recalbox-${arch}.img.xz") + metadata["${arch}ImageDownloadSha256"]=$(sha256sum "${params[imagesDir]}/${arch}/recalbox-${arch}.img.xz" | cut -d' ' -f1) + # Fetch info regarding extracted images (raw Recalbox image, after XZ decompression) + unxz --keep "${params[imagesDir]}/${arch}/recalbox-${arch}.img.xz" + metadata["${arch}ExtractSize"]=$(stat --format=%s "${params[imagesDir]}/${arch}/recalbox-${arch}.img") + metadata["${arch}ExtractSha256"]=$(sha256sum "${params[imagesDir]}/${arch}/recalbox-${arch}.img" | cut -d' ' -f1) + rm "${params[imagesDir]}/${arch}/recalbox-${arch}.img" + done + + # Create assets in destination directory + + mkdir -p ${destinationDir} + + cat "${templateDir}/os_list_imagingutility_recalbox.json" \ + | sed -e "s|{{version}}|${metadata[version]}|" \ + -e "s|{{releaseDate}}|${metadata[releaseDate]}|" \ + -e "s|{{rpi1ExtractSize}}|${metadata[rpi1ExtractSize]}|" \ + -e "s|{{rpi2ExtractSize}}|${metadata[rpi2ExtractSize]}|" \ + -e "s|{{rpi3ExtractSize}}|${metadata[rpi3ExtractSize]}|" \ + -e "s|{{rpi1ExtractSha256}}|${metadata[rpi1ExtractSha256]}|" \ + -e "s|{{rpi2ExtractSha256}}|${metadata[rpi2ExtractSha256]}|" \ + -e "s|{{rpi3ExtractSha256}}|${metadata[rpi3ExtractSha256]}|" \ + -e "s|{{rpi1ImageDownloadSize}}|${metadata[rpi1ImageDownloadSize]}|" \ + -e "s|{{rpi2ImageDownloadSize}}|${metadata[rpi2ImageDownloadSize]}|" \ + -e "s|{{rpi3ImageDownloadSize}}|${metadata[rpi3ImageDownloadSize]}|" \ + -e "s|{{rpi1ImageDownloadSha256}}|${metadata[rpi1ImageDownloadSha256]}|" \ + -e "s|{{rpi2ImageDownloadSha256}}|${metadata[rpi2ImageDownloadSha256]}|" \ + -e "s|{{rpi3ImageDownloadSha256}}|${metadata[rpi3ImageDownloadSha256]}|" \ + > "${destinationDir}/os_list_imagingutility_recalbox.json" + cp -n "${templateDir}/recalbox.svg" "${destinationDir}/recalbox.svg" +} + +## PARAMETERS PARSING ## + +declare -A params + +while [ -n "$1" ]; do + case "$1" in + --images-dir) + shift + [ -n "$1" ] && params[imagesDir]=$(readlink -f "$1") || exitWithUsage + ;; + --destination-dir) + shift + [ -n "$1" ] && params[destinationDir]=$(readlink -f "$1") || exitWithUsage + ;; + *) + exitWithUsage + ;; + esac + shift +done + +if [[ ! -d ${params[imagesDir]} || ! -d ${params[destinationDir]} ]]; then + exitWithUsage +fi + +## MAIN ## + +generateNoobsAssets +generateRaspberryPiImagerAssets diff --git a/scripts/release/templates/noobs/marketing.tar b/scripts/release/templates/noobs/marketing.tar new file mode 100644 index 0000000000000000000000000000000000000000..e47b4fa37b5e642b0ede4f03a7826be100400c5d Binary files /dev/null and b/scripts/release/templates/noobs/marketing.tar differ diff --git a/scripts/release/templates/noobs/os.json b/scripts/release/templates/noobs/os.json new file mode 100644 index 0000000000000000000000000000000000000000..64f5ac5f9c982e61e454f1db992255b9fe7f7e4f --- /dev/null +++ b/scripts/release/templates/noobs/os.json @@ -0,0 +1,4 @@ +{ + "version": "{{version}}", + "release_date": "{{releaseDate}}" +} diff --git a/scripts/release/templates/noobs/partition_setup.sh b/scripts/release/templates/noobs/partition_setup.sh new file mode 100644 index 0000000000000000000000000000000000000000..d0209322afa002b396f4dab5365c292bb8e1ed94 --- /dev/null +++ b/scripts/release/templates/noobs/partition_setup.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -ex + +if [ -z "$part1" ] || [ -z "$part2" ] || [ -z "$part3" ]; then + printf "Error: missing environment variable part1 or part2 or part3\n" 1>&2 + exit 1 +fi + +mkdir -p /tmp/1 /tmp/2 + +mount "$part1" /tmp/1 +mount "$part2" /tmp/2 + +sed /tmp/1/cmdline.txt -i -e "s|root=/dev/[^ ]*|root=${part2}|" + +umount /tmp/1 +umount /tmp/2 diff --git a/scripts/release/templates/noobs/partitions.json b/scripts/release/templates/noobs/partitions.json new file mode 100644 index 0000000000000000000000000000000000000000..0ec8dd8f3abbbfc6576a0a6561ac9eda63eee492 --- /dev/null +++ b/scripts/release/templates/noobs/partitions.json @@ -0,0 +1,26 @@ +{ + "partitions": [ + { + "label": "boot", + "filesystem_type": "fat", + "partition_size_nominal": 64, + "want_maximised": false, + "mkfs_options": "-F 32", + "uncompressed_tarball_size": {{bootUncompressedTarballSize}} + }, + { + "label": "root", + "filesystem_type": "ext4", + "partition_size_nominal": 2048, + "want_maximised": false, + "mkfs_options": "-O ^huge_file", + "uncompressed_tarball_size": {{rootUncompressedTarballSize}} + }, + { + "label": "share", + "filesystem_type": "ext4", + "want_maximised": true, + "empty_fs": true + } + ] +} diff --git a/scripts/release/templates/noobs/recalbox.png b/scripts/release/templates/noobs/recalbox.png new file mode 100644 index 0000000000000000000000000000000000000000..e4e325fb79c0d41a841ac2c1388f8e48cdc305e0 Binary files /dev/null and b/scripts/release/templates/noobs/recalbox.png differ diff --git a/scripts/release/templates/raspi_imager/os_list_imagingutility_recalbox.json b/scripts/release/templates/raspi_imager/os_list_imagingutility_recalbox.json new file mode 100644 index 0000000000000000000000000000000000000000..b97f8ca89d0764f1eb77367d68e2ecd7712b8d46 --- /dev/null +++ b/scripts/release/templates/raspi_imager/os_list_imagingutility_recalbox.json @@ -0,0 +1,37 @@ +{ + "os_list": [ + { + "name": "Recalbox {{version}} (Pi 0/1)", + "description": "The official retro-gaming OS! Turn your Raspberry Pi into an all-in-one and plug-n-play retro-gaming console, supporting 100+ gaming systems!", + "url": "https://recalbox-releases.s3.nl-ams.scw.cloud/stable/v2/upgrade/rpi1/recalbox-rpi1.img.xz", + "icon": "https://download.recalbox.com/recalbox.svg", + "extract_size": {{rpi1ExtractSize}}, + "extract_sha256": "{{rpi1ExtractSha256}}", + "image_download_size": {{rpi1ImageDownloadSize}}, + "image_download_sha256": "{{rpi1ImageDownloadSha256}}", + "release_date": "{{releaseDate}}" + }, + { + "name": "Recalbox {{version}} (Pi 2)", + "description": "The official retro-gaming OS! Turn your Raspberry Pi into an all-in-one and plug-n-play retro-gaming console, supporting 100+ gaming systems!", + "url": "https://recalbox-releases.s3.nl-ams.scw.cloud/stable/v2/upgrade/rpi2/recalbox-rpi2.img.xz", + "icon": "https://download.recalbox.com/recalbox.svg", + "extract_size": {{rpi2ExtractSize}}, + "extract_sha256": "{{rpi2ExtractSha256}}", + "image_download_size": {{rpi2ImageDownloadSize}}, + "image_download_sha256": "{{rpi2ImageDownloadSha256}}", + "release_date": "{{releaseDate}}" + }, + { + "name": "Recalbox {{version}} (Pi 3)", + "description": "The official retro-gaming OS! Turn your Raspberry Pi into an all-in-one and plug-n-play retro-gaming console, supporting 100+ gaming systems!", + "url": "https://recalbox-releases.s3.nl-ams.scw.cloud/stable/v2/upgrade/rpi3/recalbox-rpi3.img.xz", + "icon": "https://download.recalbox.com/recalbox.svg", + "extract_size": {{rpi3ExtractSize}}, + "extract_sha256": "{{rpi3ExtractSha256}}", + "image_download_size": {{rpi3ImageDownloadSize}}, + "image_download_sha256": "{{rpi3ImageDownloadSha256}}", + "release_date": "{{releaseDate}}" + } + ] +} diff --git a/scripts/release/templates/raspi_imager/recalbox.svg b/scripts/release/templates/raspi_imager/recalbox.svg new file mode 100644 index 0000000000000000000000000000000000000000..c292e5ce7941d1ed1f67db46ea273f8d9c94dd34 --- /dev/null +++ b/scripts/release/templates/raspi_imager/recalbox.svg @@ -0,0 +1,5991 @@ + + + +