Merge pull request #377 from mlocati/mark-used-packages-as-used

Mark pre-installed required APT/APK packages as used
2021-07-02 14:39:28 +02:00 · 2021-07-02 14:39:28 +02:00 · 8889295860
parent 57528a4e76 a226dc2b5e
commit 8889295860
4 changed files with 111 additions and 24 deletions
--- a/.github/workflows/test-extensions.yml
+++ b/.github/workflows/test-extensions.yml
@ -35,9 +35,7 @@ jobs:
      - name: Install shfmt
        run: GO111MODULE=on go install mvdan.cc/sh/v3/cmd/shfmt
      - name: Checkout
-        uses: actions/checkout@v1
-        with:
-          fetch-depth: 1
+        uses: actions/checkout@v2
      - name: Check coding style
        run: |
          export PATH=$PATH:$(go env GOPATH)/bin
@ -83,7 +81,9 @@ jobs:
      IPETEST_DOCKER_DISTRO: ${{ matrix.distro }}
    steps:
      - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
      - name: Test extensions
        run: ./scripts/ci-test-extensions from-commits "${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}"
  test_restoring_packages:
@ -95,9 +95,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
-        uses: actions/checkout@v1
-        with:
-          fetch-depth: 1
+        uses: actions/checkout@v2
      - run: docker run --rm --volume "$(pwd):/app" --workdir /app php:7.3-stretch ./scripts/test-restore-apt
  test_custom_version:
    name: Test installing specific versions
@ -116,9 +114,7 @@ jobs:
          - ^2.8
    steps:
      - name: Checkout
-        uses: actions/checkout@v1
-        with:
-          fetch-depth: 1
+        uses: actions/checkout@v2
      - run: docker run --rm --volume "$(pwd):/app" --workdir /app php:7.4-alpine ./scripts/test-installversion "${{ matrix.xdebug_version }}"
  test_composer:
    name: Test installing composer
@ -138,4 +134,19 @@ jobs:
        uses: actions/checkout@v2
      - run: |
          docker run --rm --volume "$(pwd):/app" --workdir /app php:7.4-alpine ./scripts/test-installcomposer "${{ matrix.composer_version }}"
-
+  test_marking_packages:
+    name: Test marking pre-installed packages
+    needs:
+      - check_syntax_data
+      - check_syntax_shell
+      - check_syntax_php
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        distro:
+          - alpine
+          - buster
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - run: docker run --rm --volume "$(pwd):/app" --workdir /app "php:7.4-${{ matrix.distro }}" "./scripts/ci-markused-${{ matrix.distro }}"
--- a/74
+++ b/74
@ -392,9 +392,10 @@ sortModulesToInstall() {
 #   $@: the PHP module handles
 #
 # Set:
-#   PACKAGES_PERSISTENT
-#   PACKAGES_VOLATILE
-#   PACKAGES_PREVIOUS
+#   PACKAGES_PERSISTENT_NEW the list of packages required at runtume that must be installed
+#   PACKAGES_PERSISTENT_PRE the list of packages required at runtume that are already installed
+#   PACKAGES_VOLATILE the list of packages required at compile time that must be installed
+#   PACKAGES_PREVIOUS the list of packages (with their version) that are installed right now (calculated only on Debian and only if PACKAGES_PERSISTENT_NEW or PACKAGES_VOLATILE are not empty)
 buildRequiredPackageLists() {
 	buildRequiredPackageLists_persistent=''
 	buildRequiredPackageLists_volatile=''
@ -983,7 +984,8 @@ buildRequiredPackageLists() {
 		esac
 		shift
 	done
-	PACKAGES_PERSISTENT=''
+	PACKAGES_PERSISTENT_NEW=''
+	PACKAGES_PERSISTENT_PRE=''
 	PACKAGES_VOLATILE=''
 	PACKAGES_PREVIOUS=''
 	if test -z "$buildRequiredPackageLists_persistent$buildRequiredPackageLists_volatile"; then
@ -995,10 +997,18 @@ buildRequiredPackageLists() {
 			;;
 	esac
 	if test -n "$buildRequiredPackageLists_persistent"; then
-		PACKAGES_PERSISTENT="$(expandPackagesToBeInstalled $buildRequiredPackageLists_persistent)"
+		PACKAGES_PERSISTENT_NEW="$(expandPackagesToBeInstalled $buildRequiredPackageLists_persistent)"
 		if test -s "$IPE_ERRFLAG_FILE"; then
 			exit 1
 		fi
+		resetIFS
+		for buildRequiredPackageLists_package in $buildRequiredPackageLists_persistent; do
+			buildRequiredPackageLists_package="$(expandInstalledSystemPackageName "$buildRequiredPackageLists_package")"
+			if test -n "$buildRequiredPackageLists_package"; then
+				PACKAGES_PERSISTENT_PRE="$PACKAGES_PERSISTENT_PRE $buildRequiredPackageLists_package"
+			fi
+		done
+		PACKAGES_PERSISTENT_PRE="${PACKAGES_PERSISTENT_PRE# }"
 	fi
 	if test -n "$buildRequiredPackageLists_volatile"; then
 		buildRequiredPackageLists_packages="$(expandPackagesToBeInstalled $buildRequiredPackageLists_volatile)"
@ -1007,13 +1017,13 @@ buildRequiredPackageLists() {
 		fi
 		resetIFS
 		for buildRequiredPackageLists_package in $buildRequiredPackageLists_packages; do
-			if ! stringInList "$buildRequiredPackageLists_package" "$PACKAGES_PERSISTENT"; then
+			if ! stringInList "$buildRequiredPackageLists_package" "$PACKAGES_PERSISTENT_NEW"; then
 				PACKAGES_VOLATILE="$PACKAGES_VOLATILE $buildRequiredPackageLists_package"
 			fi
 		done
 		PACKAGES_VOLATILE="${PACKAGES_VOLATILE# }"
 	fi
-	if test -n "$PACKAGES_PERSISTENT$PACKAGES_VOLATILE"; then
+	if test -n "$PACKAGES_PERSISTENT_NEW$PACKAGES_VOLATILE"; then
 		case "$DISTRO" in
 			debian)
 				PACKAGES_PREVIOUS="$(dpkg --get-selections | grep -E '\sinstall$' | awk '{ print $1 }')"
@ -1084,7 +1094,28 @@ expandPackagesToBeInstalled() {
 	printf '%s' "${expandPackagesToBeInstalled_result# }"
 }

+# Check if a system package is installed; if so we prints its name.
+#
+# Arguments:
+#   $1: the name of the package to be checked (regular expressions accepted: they must start with a ^)
+expandInstalledSystemPackageName() {
+	if test "$1" = "${1#^}"; then
+		expandInstalledSystemPackageName_grepflags='-Fx'
+	else
+		expandInstalledSystemPackageName_grepflags='-E'
+	fi
+	case "$DISTRO" in
+		alpine)
+			apk info | grep $expandInstalledSystemPackageName_grepflags -- "$1" || test $? -eq 1
+			;;
+		debian)
+			dpkg --get-selections | grep -E '\sinstall$' | awk '{ print $1 }' | cut -d: -f1 | grep $expandInstalledSystemPackageName_grepflags -- "$1" || test $? -eq 1
+			;;
+	esac
+}
+
 # Retrieve the number of available cores (alternative to nproc if not available)
+#
 # Output:
 #   The number of processor cores available
 getProcessorCount() {
@ -1128,21 +1159,35 @@ getCompilationProcessorCount() {
 	esac
 }

+# Mark the pre-installed APT/APK packages as used
+# that way they won't be uninstalled by accident
+markPreinstalledPackagesAsUsed() {
+	printf '### MARKING PRE-INSTALLED PACKAGES AS IN-USE ###\n'
+	case "$DISTRO" in
+		alpine)
+			printf '# Packages: %s\n' "$PACKAGES_PERSISTENT_PRE"
+			apk add $PACKAGES_PERSISTENT_PRE
+			;;
+		debian)
+			DEBIAN_FRONTEND=noninteractive apt-mark manual $PACKAGES_PERSISTENT_PRE
+			;;
+	esac
+}
+
 # Install the required APT/APK packages
 #
 # Arguments:
 #   $@: the list of APT/APK packages to be installed
 installRequiredPackages() {
 	printf '### INSTALLING REQUIRED PACKAGES ###\n'
-	printf '# Packages to be kept after installation: %s\n' "$PACKAGES_PERSISTENT"
+	printf '# Packages to be kept after installation: %s\n' "$PACKAGES_PERSISTENT_NEW"
 	printf '# Packages to be used only for installation: %s\n' "$PACKAGES_VOLATILE"
-
 	case "$DISTRO" in
 		alpine)
-			apk add $IPE_APK_FLAGS $PACKAGES_PERSISTENT $PACKAGES_VOLATILE
+			apk add $IPE_APK_FLAGS $PACKAGES_PERSISTENT_NEW $PACKAGES_VOLATILE
 			;;
 		debian)
-			DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qq -y $PACKAGES_PERSISTENT $PACKAGES_VOLATILE
+			DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qq -y $PACKAGES_PERSISTENT_NEW $PACKAGES_VOLATILE
 			;;
 	esac
 }
@ -1571,7 +1616,7 @@ installRemoteModule() {
 			if ! test -e /usr/local/lib/libcmark.so && ! test -e /usr/local/lib64/libcmark.so && ! test -e /usr/lib/libcmark.so && ! test -e /usr/lib64/libcmark.so && ! test -e /lib/libcmark.so; then
 				case "$DISTRO_VERSION" in
 					debian@8)
-						# cmark library version 0.30.0 doesn't work with cmake 3.0 (debian jessie)
+						# cmark library version 0.30.0 requires cmake 3.3+ (debian jessie comes with cmake 3.0)
 						installRemoteModule_tmp=0.29.0
 						;;
 					*)
@ -2527,7 +2572,10 @@ BUNDLED_MODULES="$(find /usr/src/php/ext -mindepth 2 -maxdepth 2 -type f -name '
 configureInstaller

 buildRequiredPackageLists $PHP_MODULES_TO_INSTALL
-if test -n "$PACKAGES_PERSISTENT$PACKAGES_VOLATILE"; then
+if test -n "$PACKAGES_PERSISTENT_PRE"; then
+	markPreinstalledPackagesAsUsed
+fi
+if test -n "$PACKAGES_PERSISTENT_NEW$PACKAGES_VOLATILE"; then
 	installRequiredPackages
 fi
 if test "$PHP_MODULES_TO_INSTALL" != '@composer'; then
--- a/scripts/ci-markused-alpine
+++ b/scripts/ci-markused-alpine
@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Let's set a sane environment
+set -o errexit
+set -o nounset
+
+apk update
+apk del libzip
+apk add ebook-tools # <- uses libzip
+CI=true ./install-php-extensions zip # <- uses libzip
+apk del ebook-tools
+php --ri zip
--- a/scripts/ci-markused-buster
+++ b/scripts/ci-markused-buster
@ -0,0 +1,16 @@
+#!/bin/sh
+
+# Let's set a sane environment
+set -o errexit
+set -o nounset
+
+DEBIAN_FRONTEND=noninteractive
+export DEBIAN_FRONTEND
+
+apt-get update -q
+apt-get remove -qy --purge '^libzip[0-9]*$'
+apt-get install -qy libepub0 # <- uses libzip
+CI=true ./install-php-extensions zip # <- uses libzip
+apt-get remove -qy --purge libepub0
+apt-get autoremove -qy --purge
+php --ri zip