From c8ffadc7bc3e166bbe645e2378106e469d52df34 Mon Sep 17 00:00:00 2001 From: Ark74 <ark@switnet.org> Date: Mon, 12 Sep 2022 19:27:16 -0500 Subject: [PATCH 1/7] aramo-udeb: add template and package list for rebuild. --- helpers/DATA/aramo-udeb-rebuild-list | 73 ++++++++++++++++++++++++++++ helpers/DATA/aramo-udeb-template | 32 ++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 helpers/DATA/aramo-udeb-rebuild-list create mode 100644 helpers/DATA/aramo-udeb-template diff --git a/helpers/DATA/aramo-udeb-rebuild-list b/helpers/DATA/aramo-udeb-rebuild-list new file mode 100644 index 00000000..51fbe5a4 --- /dev/null +++ b/helpers/DATA/aramo-udeb-rebuild-list @@ -0,0 +1,73 @@ +argon2 +at-spi2-atk +bogl +brltty +btrfs-progs +busybox +ca-certificates +cdebconf +cdebconf-terminal +console-setup +cryptsetup +disk-detect +dmidecode +dmraid +dosfstools +e2fsprogs +espeakup +fonts-android +fonts-dejavu +fonts-farsiweb +fonts-freefont +fonts-gubbi +fonts-khmeros +fonts-lao +fonts-lohit-guru +fonts-lohit-telu +fonts-noto +fonts-samyak +fonts-sil-abyssinica +fonts-sil-padauk +fonts-smc-rachana +fonts-thai-tlwg +fonts-tibetan-machine +fonts-ukij-uyghur +fribidi +fuse +gtk+2.0 +gtk2-engines +jfsutils +json-c +kbd +libaio +libbsd +libdebian-installer +libzstd +localechooser +lzo2 +mdadm +mountmedia +multipath-tools +nano +ntfs-3g +open-iscsi +open-isns +openssh +os-prober +partconf +parted +pciutils +pcmciautils +pcre3 +popt +rdate +reiserfsprogs +screen +systemd +user-setup +util-linux +wget +xfsprogs +xorg-server +xserver-xorg-input-libinput +xserver-xorg-video-fbdev diff --git a/helpers/DATA/aramo-udeb-template b/helpers/DATA/aramo-udeb-template new file mode 100644 index 00000000..041f2861 --- /dev/null +++ b/helpers/DATA/aramo-udeb-template @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Copyright (C) 2022 Luis Guzmán <ark@switnet.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# + +## +# This helper is part of a series of packages that are now droppped on upstream +# Ubuntu, therefore in order to build a working debian-installer we now fetch +# such packages from Debian in order to build the required udeb packages. +## + +VERSION=0 +#NETINST=true +. ./config + + +changelog "Fetch and build as udeb dependency for debian-installer." +compile -- GitLab From a06243dc56e5d336ce389a0a062e3b3e6eb03ae4 Mon Sep 17 00:00:00 2001 From: Ark74 <ark@switnet.org> Date: Wed, 5 Oct 2022 12:02:31 -0500 Subject: [PATCH 2/7] aramo-udeb-repository: update list and process. --- helpers/DATA/udeb-repo/README.mkd | 106 ++++++++++++++++++ .../udeb-repo/aramo/aramo-udeb-rebuild-list | 41 +++++++ .../DATA/udeb-repo/aramo/aramo-udeb-template | 32 ++++++ 3 files changed, 179 insertions(+) create mode 100644 helpers/DATA/udeb-repo/README.mkd create mode 100644 helpers/DATA/udeb-repo/aramo/aramo-udeb-rebuild-list create mode 100644 helpers/DATA/udeb-repo/aramo/aramo-udeb-template diff --git a/helpers/DATA/udeb-repo/README.mkd b/helpers/DATA/udeb-repo/README.mkd new file mode 100644 index 00000000..38b9db10 --- /dev/null +++ b/helpers/DATA/udeb-repo/README.mkd @@ -0,0 +1,106 @@ +## udeb repository + +Starting on trisquel 11.0, codename aramo, it is required to provide the necesary udeb repository for the debian-installer, we mainly get it by fetching missing packages from Debian and also by rebuilding them from upstream ones. + +Those new packages should be added on top of the helpers mentioned at: +[https://gitlab.trisquel.org/trisquel/package-helpers/-/tree/aramo#netinstall](https://gitlab.trisquel.org/trisquel/package-helpers/-/tree/aramo#netinstall) + +In order to popullate this repository, there is a couple of places to get the packages required on the udeb repository, + +### 1) debian-installer pkg-list + +The debian-installer package has a fixed list of packages from which we can get the names, make sure to avoik the `gtk-common` file as we don't require graphical packages. + +``` +$ cd debian-installer-*/build/pkg-lists +$ cat $(find . -type f ! -name 'gtk-common') | \ + grep -v "\#" | \ + grep -v "\-\$" | \ + grep -v "\?\$" | \ + grep -v 'kernel:Version' | \ + grep -v 'kfreebsd' | \ + sed 's|\ \[linux\]||g' | \ + awk 'NF' > ~/d-i_pkg-list + +``` +--- +**Dictionary** + +* `grep -v "\#"` - remove comments on files +* `grep -v "\-\$"` - remove unpromoted packages +* `grep -v "\?\$"` - remove undefined packages +* `grep -v 'kernel:Version'` - remove kernel packages, this should be taken care of by the kernel helper. +* `grep -v 'kfreebsd'` - trisquel kernel is linux-libre based so removing other kernel specific packages. +* `sed 's|\ \[linux\]||g'` - remove explicit linux(-libre) kernel definition, all should be linux(-libre) specific. +* `awk 'NF'` - remove empty lines +--- + + +Then we can parse the `d-i_pkg-list` using, + +``` +$ for i in $(cat ~/d-i_pkg-list) ; \ + do apt-cache madison $i | \ + grep Sources | awk '{print$1}' ; done | \ + awk '!seen[$0]++' | sort +``` + +Note that as the list is a big one to parse, it will take some time. + +This will provide a list of source packages necessary to build for our udeb repository, be ware that some helpers for these source packages might already exist, errors finding the source package mean source packages required from debian. + +### 2) anna list (nabia) + +The second place is from a dinamically generated list by anna (*anna's not nearly apt*) which is generated from the repository itself. + +For this list we relay on a previous release like a nabia netinstall. There is no *simple* way to get this list, as it requires a second system to serve as a **proxy cache server** or as a **tftp server** for it to get the output of the access.log (proxy server) or syslog (on the installed system) file. + +#### Proxy cache - squid +The *"easiest"* way of getting a hand of such package list is to parse the output of a cache proxy like squid once a nabia netinstallation is complete. + +In this process a nabia system needs to be installed and on the "Choose the Triquel mirror" step set the address of the squid proxy cache server (e.g. *http://192.168.1.100:3128*) so the nabia installation fetches the packages through the proxy server for them to be logged at the `access.log` on the proxy server. + +We can parse the access.log on the proxy server or through ssh, + +``` +SQUID_LOG="/var/log/squid/access.log" + +# for i in $(cat $SQUID_LOG | \ + awk '/htt.*udeb/{print$7}' | awk -F'[/_]' '{print$10}') ; \ + do apt-cache madison $i | grep Sources | awk '{print$1}' ; done | \ + awk '!seen[$0]++' | sort + +``` + +#### tftp server - syslog + +Another way is to export the syslog of the netinstall through tftp to a server. + +While installing the nabia system at the kernel it is possible to chage the tty and send the syslog file for further parsing using tftp, as busybox has no limited capabilities, to the tftp server (e.g. 192.168.1.100). + +``` +cd /var/log/ +tftp -l syslog -r syslog_netinstall -p 192.168.1.100 +``` +On the tftp server it will be possible to parse the package list just like the proxy cache one. + +``` +SYSLOG_NETINST="path-to/syslog_netinstall" +# for i in $(cat $SYSLOG_NETINST | \ + awk '/htt.*udeb/{print$7}' | awk -F'[/_]' '{print$10}') ; \ + do apt-cache madison $i | grep Sources | awk '{print$1}' ; done | \ + awk '!seen[$0]++' | sort + +``` + +Just like before, this will provide a list of source packages necessary to build for our udeb repository, be ware that some helpers for these source packages might already exist, errors finding the source package mean source packages required from debian. + +## Set upstream source + +Once finding source packages required from debian as several debian-installer packages has been droped from upstream Ubuntu, targetting the use of debian **stable** as source repository. + +After the pkg-list+anna list is cleaned from duplicated or existing helpers, that would result on a base packages list to rebuild the udeb required repository for aramo. + +Further packages could be added based on trial-an-error and further debian-installer developement. + +Debian's [netboot mailing list](https://lists.debian.org/debian-boot/) would be a good place to check further developement. \ No newline at end of file diff --git a/helpers/DATA/udeb-repo/aramo/aramo-udeb-rebuild-list b/helpers/DATA/udeb-repo/aramo/aramo-udeb-rebuild-list new file mode 100644 index 00000000..6fd1c2ed --- /dev/null +++ b/helpers/DATA/udeb-repo/aramo/aramo-udeb-rebuild-list @@ -0,0 +1,41 @@ +bogl +busybox +console-setup +cryptsetup +debian-ports-archive-keyring +dmraid +fribidi +fuse +haveged +kbd +kmod +libdebian-installer +libzstd +lowmem +lvm2 +lzo2 +mdadm +micro-evtd +mountmedia +multipath-tools +nano +ntfs-3g +open-iscsi +os-prober +partconf +parted +pciutils +pcmciautils +pcre3 +popt +qcontrol +rdate +reiserfsprogs +s390-netdevice +screen +systemd +user-setup +util-linux +wget +wireless-regdb +xfsprogs diff --git a/helpers/DATA/udeb-repo/aramo/aramo-udeb-template b/helpers/DATA/udeb-repo/aramo/aramo-udeb-template new file mode 100644 index 00000000..ee74ecb2 --- /dev/null +++ b/helpers/DATA/udeb-repo/aramo/aramo-udeb-template @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Copyright (C) 2022 Luis Guzmán <ark@switnet.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# + +## +# This helper is part of a series of packages that are now droppped on upstream +# Ubuntu, therefore in order to build a working debian-installer we now fetch +# such packages from Debian in order to build the required udeb packages. +## + +VERSION=0 +NETINST=true +. ./config + + +changelog "Fetch and build as udeb dependency for debian-installer." +compile -- GitLab From eacc92870d64d1d5c2432b83ae5344e9ef87607d Mon Sep 17 00:00:00 2001 From: Ark74 <ark@switnet.org> Date: Wed, 5 Oct 2022 12:05:19 -0500 Subject: [PATCH 3/7] aramo-udeb-repository: update README and remove old files --- helpers/DATA/aramo-udeb-rebuild-list | 73 ---------------------------- helpers/DATA/aramo-udeb-template | 32 ------------ helpers/DATA/udeb-repo/README.mkd | 3 +- 3 files changed, 2 insertions(+), 106 deletions(-) delete mode 100644 helpers/DATA/aramo-udeb-rebuild-list delete mode 100644 helpers/DATA/aramo-udeb-template diff --git a/helpers/DATA/aramo-udeb-rebuild-list b/helpers/DATA/aramo-udeb-rebuild-list deleted file mode 100644 index 51fbe5a4..00000000 --- a/helpers/DATA/aramo-udeb-rebuild-list +++ /dev/null @@ -1,73 +0,0 @@ -argon2 -at-spi2-atk -bogl -brltty -btrfs-progs -busybox -ca-certificates -cdebconf -cdebconf-terminal -console-setup -cryptsetup -disk-detect -dmidecode -dmraid -dosfstools -e2fsprogs -espeakup -fonts-android -fonts-dejavu -fonts-farsiweb -fonts-freefont -fonts-gubbi -fonts-khmeros -fonts-lao -fonts-lohit-guru -fonts-lohit-telu -fonts-noto -fonts-samyak -fonts-sil-abyssinica -fonts-sil-padauk -fonts-smc-rachana -fonts-thai-tlwg -fonts-tibetan-machine -fonts-ukij-uyghur -fribidi -fuse -gtk+2.0 -gtk2-engines -jfsutils -json-c -kbd -libaio -libbsd -libdebian-installer -libzstd -localechooser -lzo2 -mdadm -mountmedia -multipath-tools -nano -ntfs-3g -open-iscsi -open-isns -openssh -os-prober -partconf -parted -pciutils -pcmciautils -pcre3 -popt -rdate -reiserfsprogs -screen -systemd -user-setup -util-linux -wget -xfsprogs -xorg-server -xserver-xorg-input-libinput -xserver-xorg-video-fbdev diff --git a/helpers/DATA/aramo-udeb-template b/helpers/DATA/aramo-udeb-template deleted file mode 100644 index 041f2861..00000000 --- a/helpers/DATA/aramo-udeb-template +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2022 Luis Guzmán <ark@switnet.org> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -# - -## -# This helper is part of a series of packages that are now droppped on upstream -# Ubuntu, therefore in order to build a working debian-installer we now fetch -# such packages from Debian in order to build the required udeb packages. -## - -VERSION=0 -#NETINST=true -. ./config - - -changelog "Fetch and build as udeb dependency for debian-installer." -compile diff --git a/helpers/DATA/udeb-repo/README.mkd b/helpers/DATA/udeb-repo/README.mkd index 38b9db10..6b9bd11c 100644 --- a/helpers/DATA/udeb-repo/README.mkd +++ b/helpers/DATA/udeb-repo/README.mkd @@ -13,7 +13,7 @@ The debian-installer package has a fixed list of packages from which we can get ``` $ cd debian-installer-*/build/pkg-lists -$ cat $(find . -type f ! -name 'gtk-common') | \ +$ cat $(find . -type f ! -name 'gtk-common' -not -path "*/gtk/*") | \ grep -v "\#" | \ grep -v "\-\$" | \ grep -v "\?\$" | \ @@ -26,6 +26,7 @@ $ cat $(find . -type f ! -name 'gtk-common') | \ --- **Dictionary** +* *find* - `! -name 'gtk-common' -not -path "*/gtk/*"` - avoid file gtk-common and directories specific to gtk packages. * `grep -v "\#"` - remove comments on files * `grep -v "\-\$"` - remove unpromoted packages * `grep -v "\?\$"` - remove undefined packages -- GitLab From 8ab2de9fda6eb7485546526fbe5b92471014b9f7 Mon Sep 17 00:00:00 2001 From: Ark74 <ark@switnet.org> Date: Wed, 5 Oct 2022 13:04:04 -0500 Subject: [PATCH 4/7] aramo-udeb-repository: update README --- helpers/DATA/udeb-repo/README.mkd | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/helpers/DATA/udeb-repo/README.mkd b/helpers/DATA/udeb-repo/README.mkd index 6b9bd11c..a989f2b6 100644 --- a/helpers/DATA/udeb-repo/README.mkd +++ b/helpers/DATA/udeb-repo/README.mkd @@ -48,7 +48,7 @@ $ for i in $(cat ~/d-i_pkg-list) ; \ Note that as the list is a big one to parse, it will take some time. -This will provide a list of source packages necessary to build for our udeb repository, be ware that some helpers for these source packages might already exist, errors finding the source package mean source packages required from debian. +This will provide a list of source packages necessary to build for our udeb repository, be ware that some helpers for these source packages might already exist, errors finding the source package mean source packages are required from debian. ### 2) anna list (nabia) @@ -72,12 +72,13 @@ SQUID_LOG="/var/log/squid/access.log" awk '!seen[$0]++' | sort ``` +* acces.log might will likely require sudo or root to read. #### tftp server - syslog Another way is to export the syslog of the netinstall through tftp to a server. -While installing the nabia system at the kernel it is possible to chage the tty and send the syslog file for further parsing using tftp, as busybox has no limited capabilities, to the tftp server (e.g. 192.168.1.100). +While installing the nabia system at the kernel selection step, it is possible to chage the tty and send the syslog file to the tftp server (e.g. 192.168.1.100) for further parsing using tftp, as busybox has limited capabilities. ``` cd /var/log/ @@ -94,7 +95,7 @@ SYSLOG_NETINST="path-to/syslog_netinstall" ``` -Just like before, this will provide a list of source packages necessary to build for our udeb repository, be ware that some helpers for these source packages might already exist, errors finding the source package mean source packages required from debian. +Just like before, this will provide a list of source packages necessary to build for our udeb repository, be ware that some helpers for these source packages might already exist, errors finding the source package mean source packages are required from debian. ## Set upstream source -- GitLab From 5b2f3d347ea614606974cdcad4f92fd2bb2b2d9a Mon Sep 17 00:00:00 2001 From: Ark74 <ark@switnet.org> Date: Wed, 5 Oct 2022 23:08:59 -0500 Subject: [PATCH 5/7] aramo-udeb-repository: fix typos README --- helpers/DATA/udeb-repo/README.mkd | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/helpers/DATA/udeb-repo/README.mkd b/helpers/DATA/udeb-repo/README.mkd index a989f2b6..2878afbf 100644 --- a/helpers/DATA/udeb-repo/README.mkd +++ b/helpers/DATA/udeb-repo/README.mkd @@ -1,11 +1,11 @@ ## udeb repository -Starting on trisquel 11.0, codename aramo, it is required to provide the necesary udeb repository for the debian-installer, we mainly get it by fetching missing packages from Debian and also by rebuilding them from upstream ones. +Starting on trisquel 11.0, codename aramo, it is required to provide the necessary udeb repository for the debian-installer, we mainly get it by fetching missing packages from Debian and also by rebuilding them from upstream ones. Those new packages should be added on top of the helpers mentioned at: [https://gitlab.trisquel.org/trisquel/package-helpers/-/tree/aramo#netinstall](https://gitlab.trisquel.org/trisquel/package-helpers/-/tree/aramo#netinstall) -In order to popullate this repository, there is a couple of places to get the packages required on the udeb repository, +In order to populate this repository, there is a couple of places to get the packages required on the udeb repository, ### 1) debian-installer pkg-list @@ -52,7 +52,7 @@ This will provide a list of source packages necessary to build for our udeb repo ### 2) anna list (nabia) -The second place is from a dinamically generated list by anna (*anna's not nearly apt*) which is generated from the repository itself. +The second place is from a dynamically generated list by anna (*anna's not nearly apt*) which is generated from the repository itself. For this list we relay on a previous release like a nabia netinstall. There is no *simple* way to get this list, as it requires a second system to serve as a **proxy cache server** or as a **tftp server** for it to get the output of the access.log (proxy server) or syslog (on the installed system) file. @@ -99,10 +99,10 @@ Just like before, this will provide a list of source packages necessary to build ## Set upstream source -Once finding source packages required from debian as several debian-installer packages has been droped from upstream Ubuntu, targetting the use of debian **stable** as source repository. +Once finding source packages required from debian as several debian-installer packages has been droped from upstream Ubuntu, targeting the use of debian **stable** as source repository. After the pkg-list+anna list is cleaned from duplicated or existing helpers, that would result on a base packages list to rebuild the udeb required repository for aramo. -Further packages could be added based on trial-an-error and further debian-installer developement. +Further packages could be added based on trial and error and further debian-installer developement. -Debian's [netboot mailing list](https://lists.debian.org/debian-boot/) would be a good place to check further developement. \ No newline at end of file +Debian's [netboot mailing list](https://lists.debian.org/debian-boot/) would be a good place to check further developement. -- GitLab From 69207f03a9e2f897343a327e8d9e960a6d3c3c1c Mon Sep 17 00:00:00 2001 From: Ark74 <ark@switnet.org> Date: Fri, 7 Oct 2022 18:10:17 -0500 Subject: [PATCH 6/7] set correct installation step name README. --- helpers/DATA/udeb-repo/README.mkd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/DATA/udeb-repo/README.mkd b/helpers/DATA/udeb-repo/README.mkd index 2878afbf..7218d13f 100644 --- a/helpers/DATA/udeb-repo/README.mkd +++ b/helpers/DATA/udeb-repo/README.mkd @@ -59,7 +59,7 @@ For this list we relay on a previous release like a nabia netinstall. There is n #### Proxy cache - squid The *"easiest"* way of getting a hand of such package list is to parse the output of a cache proxy like squid once a nabia netinstallation is complete. -In this process a nabia system needs to be installed and on the "Choose the Triquel mirror" step set the address of the squid proxy cache server (e.g. *http://192.168.1.100:3128*) so the nabia installation fetches the packages through the proxy server for them to be logged at the `access.log` on the proxy server. +In this process a nabia system needs to be installed and on the *"Choose a mirror of the Triquel archive"* step set the address of the squid proxy cache server (e.g. *http://192.168.1.100:3128*) so the nabia installation fetches the packages through the proxy server for them to be logged at the `access.log` on the proxy server. We can parse the access.log on the proxy server or through ssh, -- GitLab From cd6c7eb9d3b434911d0f59de04316b42be83be40 Mon Sep 17 00:00:00 2001 From: Ark74 <ark@switnet.org> Date: Wed, 12 Oct 2022 13:11:55 -0500 Subject: [PATCH 7/7] apply suggestions. --- helpers/DATA/udeb-repo/README.mkd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/DATA/udeb-repo/README.mkd b/helpers/DATA/udeb-repo/README.mkd index 7218d13f..cc2bd3b8 100644 --- a/helpers/DATA/udeb-repo/README.mkd +++ b/helpers/DATA/udeb-repo/README.mkd @@ -1,11 +1,11 @@ ## udeb repository -Starting on trisquel 11.0, codename aramo, it is required to provide the necessary udeb repository for the debian-installer, we mainly get it by fetching missing packages from Debian and also by rebuilding them from upstream ones. +Starting on trisquel 11.0, codename aramo, it is necessary to build all udeb packages related to debian-installer. Missing packages (like debian-installer itself) are fetched and built from Debian, and others are produced by rebuilding their source packages. Those new packages should be added on top of the helpers mentioned at: [https://gitlab.trisquel.org/trisquel/package-helpers/-/tree/aramo#netinstall](https://gitlab.trisquel.org/trisquel/package-helpers/-/tree/aramo#netinstall) -In order to populate this repository, there is a couple of places to get the packages required on the udeb repository, +In order to make a list of all the source packages that would produce the required udeb binary packages, we look in these places: ### 1) debian-installer pkg-list -- GitLab