Ability to install extensions from source code/specific commits (#538)

.github/workflows/test-extensions.yml

@@ -162,3 +162,31 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
       - run: IPE_INSTANTCLIENT_BASIC=1 ./install-php-extensions oci8 pdo_oci
+  test_install_fromsource:
+    name: Test installing from source
+    needs:
+      - check_syntax_data
+      - check_syntax_shell
+      - check_syntax_php
+    runs-on: ubuntu-latest
+    container: ${{ matrix.container }}
+    strategy:
+      matrix:
+        container:
+          - php:8.1-cli-alpine
+        source:
+          - php-memcached-dev/php-memcached@8f106564e6bb005ca6100b12ccc89000daafa9d8
+          - php-memcached-dev/php-memcached@8f106564e6bb
+          - php-memcached-dev/php-memcached@v3.2.0RC2
+          - php-memcached-dev/php-memcached@refs/tags/v3.2.0RC2
+        include:
+          -
+            container: php:8.1-cli-bullseye
+            source: php-memcached-dev/php-memcached@v3.2.0RC2
+          -
+            container: php:7.4-cli-alpine
+            source: php-memcached-dev/php-memcached@v3.2.0RC2
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - run: ./install-php-extensions ${{ matrix.source }}

README.md

@@ -95,6 +95,45 @@ For example:
 install-php-extensions mongodb-stable
 ```
 
+### Installing from source code
+
+You can also install PHP extensions from source code (provided that it comes with a `package.xml` or a `package2.xml` file).
+
+Accepted formats are:
+
+- A short version for repositories hosted on GitHub.
+  For example, for the [php-memcached-dev/php-memcached](https://github.com/php-memcached-dev/php-memcached) GitHub repository,
+  you can simply write:
+  ```sh
+  # Install from a specific commit (full commit SHA-1)
+  install-php-extensions php-memcached-dev/php-memcached@8f106564e6bb005ca6100b12ccc89000daafa9d8
+  # Install from a specific commit (short commit SHA-1)
+  install-php-extensions php-memcached-dev/php-memcached@8f106564e6bb
+  # Install from tag v3.2.0RC2
+  install-php-extensions php-memcached-dev/php-memcached@v3.2.0RC2
+  install-php-extensions php-memcached-dev/php-memcached@refs/tags/v3.2.0RC2
+  # Install from branch master
+  install-php-extensions php-memcached-dev/php-memcached@master
+  install-php-extensions php-memcached-dev/php-memcached@refs/heads/master
+  ```
+- An URL providing an archive containing the source code.
+  Examples:
+  ```sh
+  # tgz archive for commit 8f106564e6bb005ca6100b12ccc89000daafa9d8
+  install-php-extensions https://codeload.github.com/php-memcached-dev/php-memcached/tar.gz/8f106564e6bb005ca6100b12ccc89000daafa9d8
+  # tgz archive for tag v3.1.5
+  install-php-extensions https://codeload.github.com/php-memcached-dev/php-memcached/tar.gz/refs/tags/v3.1.5
+  # tgz archive for branch master
+  install-php-extensions https://codeload.github.com/php-memcached-dev/php-memcached/tar.gz/refs/heads/master
+  ```
+- The absolute path of a local directory
+  Examples:
+  ```sh
+  # Download the source code
+  curl -o /tmp/source.tgz https://codeload.github.com/php-memcached-dev/php-memcached/tar.gz/refs/tags/v3.1.5
+  tar xzf /tmp/source.tgz -C /tmp
+  install-php-extensions /tmp/php-memcached-3.1.5
+
 ### Installing composer
 
 You can also install [composer](https://getcomposer.org/), and you also can specify a major version of it, or a full version.

install-php-extensions

@@ -183,9 +183,51 @@ extractPackageVersionFromXML() {
 		printf 'Unable to find the file\n%s\n' >&2
 		return 1
 	fi
-	extractPackageVersionFromXML_xml="$(cat "$1" | tr '\n' ' ' | sed -E 's/<!--.*?-->//')"
+	extractPackageVersionFromXML_code="$(
-	printf '\extractPackageVersionFromXML_xml=\n%s\n' "$extractPackageVersionFromXML_xml" | sed -E 's/^.*?'
+		cat <<'EOT'
-	exit 1
+$doc = new DOMDocument();
+if (!$doc->load($argv[1])) {
+	fwrite(STDERR, "Failed to load XML file\n");
+	exit(1);
+}
+set_error_handler(
+	static function($errno, $errstr) {
+		fwrite(STDERR, trim((string) $errstr) . "\n");
+		exit(1);
+	},
+	-1
+);
+$xpath = new DOMXpath($doc);
+$xpath->registerNamespace('v20', 'http://pear.php.net/dtd/package-2.0');
+$xpath->registerNamespace('v21', 'http://pear.php.net/dtd/package-2.1');
+if ($xpath->query('/v20:package/v20:dependencies')->count() === 1) {
+	$ns = 'v20:';
+} elseif ($xpath->query('/v21:package/v21:dependencies')->count() === 1) {
+	$ns = 'v21:';
+} elseif ($xpath->query('/package')->count() === 1) {
+	$ns = '';
+} else {
+	fwrite(STDERR, "Unsupported namespace of the XML of package version details\n");
+}
+$nodes = $xpath->query("/{$ns}package/{$ns}name");
+$name = trim((string) $nodes[0]->nodeValue);
+if ($ns === '') {
+	$nodes = $xpath->query("/{$ns}package/{$ns}version");
+} else {
+	$nodes = $xpath->query("/{$ns}package/{$ns}version/{$ns}release");
+}
+$version = trim((string) $nodes[0]->nodeValue);
+echo "EXTRACTPACKAGEVERSIONFROMXML_NAME='{$name}'\n";
+echo "EXTRACTPACKAGEVERSIONFROMXML_VERSION='{$version}'\n";
+exit(0);
+EOT
+	)"
+	extractPackageVersionFromXML_vars="$(php -n -d display_errors=stderr -r "$extractPackageVersionFromXML_code" "$1")"
+	if test -z "$extractPackageVersionFromXML_vars"; then
+		return 1
+	fi
+	eval "$extractPackageVersionFromXML_vars"
+	return 0
 }
 
 # Parse a module name (and optionally version) as received via command arguments, extracting the version and normalizing it
@@ -202,6 +244,7 @@ extractPackageVersionFromXML() {
 #
 # Optionally set these variables:
 # - PHP_WANTEDMODULEVERSION_<...> (where <...> is the normalized module name)
+# - PHP_MODULESOURCECODEPATH_<...> (where <...> is the normalized module name)
 #
 # Output:
 #   Nothing
@@ -212,7 +255,7 @@ processPHPModuleArgument() {
 	# php-memcached-dev/php-memcached@8f106564e6bb005ca6100b12ccc89000daafa9d8
 	# to
 	# https://codeload.github.com/php-memcached-dev/php-memcached/tar.gz/8f106564e6bb005ca6100b12ccc89000daafa9d8
-	processPHPModuleArgument_arg="$(printf '%s' "$processPHPModuleArgument_arg" | sed -E 's/^([a-zA-Z0-9_.\-]+\/[a-zA-Z0-9_.\-]+)@([a-zA-Z0-9]{40}$)/https:\/\/codeload.github.com\/\1\/tar.gz\/\2/')"
+	processPHPModuleArgument_arg="$(printf '%s' "$processPHPModuleArgument_arg" | sed -E 's/^([a-zA-Z0-9_.\-]+\/[a-zA-Z0-9_.\-]+)@(.+$)/https:\/\/codeload.github.com\/\1\/tar.gz\/\2/')"
 	# Let's check if $processPHPModuleArgument_arg is an URL
 	if printf '%s' "$processPHPModuleArgument_arg" | grep -Eq '^https?://[^ ]+/[^ ]+$'; then
 		printf 'Downloading source from %s\n' "$processPHPModuleArgument_arg"
@@ -234,7 +277,16 @@ processPHPModuleArgument() {
 			printf 'Unable to find the package.xml file in the directory\n%s\n' "$processPHPModuleArgument_arg"
 			return 1
 		fi
+		printf 'good (name: %s, version: %s)\n' "$EXTRACTPACKAGEVERSIONFROMXML_NAME" "$EXTRACTPACKAGEVERSIONFROMXML_VERSION"
+		PROCESSED_PHP_MODULE_ARGUMENT="$(normalizePHPModuleName "$EXTRACTPACKAGEVERSIONFROMXML_NAME")"
+		processPHPModuleArgument_version="$EXTRACTPACKAGEVERSIONFROMXML_VERSION"
+		if printf '%s' "$PROCESSED_PHP_MODULE_ARGUMENT" | grep -Eq '^[a-zA-Z0-9_]+$'; then
+			eval PHP_MODULESOURCECODEPATH_$PROCESSED_PHP_MODULE_ARGUMENT="$processPHPModuleArgument_arg"
+		else
+			printf 'Unable to parse the following module name:\n%s\n' "$PROCESSED_PHP_MODULE_ARGUMENT" >&2
+			exit 1
 		fi
+	else
 		PROCESSED_PHP_MODULE_ARGUMENT="${processPHPModuleArgument_arg%%-*}"
 		if test -n "$PROCESSED_PHP_MODULE_ARGUMENT" && test "$PROCESSED_PHP_MODULE_ARGUMENT" != "$processPHPModuleArgument_arg"; then
 			processPHPModuleArgument_version="${processPHPModuleArgument_arg#*-}"
@@ -242,6 +294,7 @@ processPHPModuleArgument() {
 			processPHPModuleArgument_version=''
 		fi
 		PROCESSED_PHP_MODULE_ARGUMENT="$(normalizePHPModuleName "$PROCESSED_PHP_MODULE_ARGUMENT")"
+	fi
 	if test -n "$processPHPModuleArgument_version"; then
 		if printf '%s' "$PROCESSED_PHP_MODULE_ARGUMENT" | grep -Eq '^[a-zA-Z0-9_]+$'; then
 			eval PHP_WANTEDMODULEVERSION_$PROCESSED_PHP_MODULE_ARGUMENT="$processPHPModuleArgument_version"
@@ -268,6 +321,19 @@ getWantedPHPModuleVersion() {
 	fi
 }
 
+# Get source code path of a PHP module version, as specified in the command line arguments.
+#
+# Arguments:
+#   $1: the name of the module to be normalized
+#
+# Output:
+#   The wanted version (if any)
+getModuleSourceCodePath() {
+	if printf '%s' "$1" | grep -Eq '^[a-zA-Z0-9_]+$'; then
+		eval printf '%s' "\${PHP_MODULESOURCECODEPATH_$1:-}"
+	fi
+}
+
 # Get the wanted PHP module version, resolving it if it starts with '^'
 #
 # Arguments:
@@ -1943,6 +2009,9 @@ installBundledModule() {
 	if test -n "$(getWantedPHPModuleVersion "$1")"; then
 		printf '### WARNING the module "%s" is bundled with PHP, you can NOT specify a version for it\n' "$1" >&2
 	fi
+	if test -n "$(getModuleSourceCodePath "$1")"; then
+		printf '### WARNING the module "%s" is bundled with PHP, you can NOT specify a source code path for it\n' "$1" >&2
+	fi
 	case "$1" in
 		dba)
 			if test -e /usr/lib/$TARGET_TRIPLET/libdb-5.3.so && ! test -e /usr/lib/libdb-5.3.so; then
@@ -2199,6 +2268,7 @@ installRemoteModule() {
 	installRemoteModule_module="$1"
 	printf '### INSTALLING REMOTE MODULE %s ###\n' "$installRemoteModule_module"
 	installRemoteModule_version="$(resolveWantedPHPModuleVersion "$installRemoteModule_module")"
+	installRemoteModule_path="$(getModuleSourceCodePath "$installRemoteModule_module")"
 	rm -rf "$CONFIGURE_FILE"
 	installRemoteModule_manuallyInstalled=0
 	installRemoteModule_cppflags=''
@@ -2389,6 +2459,7 @@ installRemoteModule() {
 			fi
 			;;
 		geos)
+			if test -z "$installRemoteModule_path"; then
 				if test -z "$installRemoteModule_version"; then
 					installRemoteModule_version=71b5f9001512e16d3cf4657b517e8a051d6ef36f
 				fi
@@ -2399,6 +2470,7 @@ installRemoteModule() {
 				make -j$(getProcessorCount) install
 				cd - >/dev/null
 				installRemoteModule_manuallyInstalled=1
+			fi
 			;;
 		geospatial)
 			if test -z "$installRemoteModule_version"; then
@@ -2738,6 +2810,7 @@ installRemoteModule() {
 			fi
 			;;
 		snuffleupagus)
+			if test -z "$installRemoteModule_path"; then
 				if test -z "$installRemoteModule_version"; then
 					installRemoteModule_version=0.7.1
 				fi
@@ -2748,6 +2821,7 @@ installRemoteModule() {
 				make -j$(getProcessorCount) install
 				cd - >/dev/null
 				cp -a "$installRemoteModule_src/config/default.rules" "$PHP_INI_DIR/conf.d/snuffleupagus.rules"
+			fi
 			installRemoteModule_manuallyInstalled=1
 			;;
 		solr)
@@ -2762,6 +2836,7 @@ installRemoteModule() {
 			installRemoteModule_manuallyInstalled=1
 			;;
 		spx)
+			if test -z "$installRemoteModule_path"; then
 				if test -z "$installRemoteModule_version"; then
 					installRemoteModule_version=6eba1a5839d0ce8dc71148ed723c72fe06c69600
 				fi
@@ -2778,6 +2853,7 @@ installRemoteModule() {
 				make -j$(getProcessorCount) install
 				cd - >/dev/null
 				installRemoteModule_manuallyInstalled=1
+			fi
 			;;
 		sqlsrv | pdo_sqlsrv)
 			isMicrosoftSqlServerODBCInstalled || installMicrosoftSqlServerODBC
@@ -2967,6 +3043,7 @@ installRemoteModule() {
 				make install
 				cd - >/dev/null
 			fi
+			if test -z "$installRemoteModule_path"; then
 				installRemoteModule_tmp="$(mktemp -p /tmp/src -d)"
 				git clone --depth=1 --recurse-submodules https://github.com/yaroslavche/phptdlib.git "$installRemoteModule_tmp"
 				mkdir "$installRemoteModule_tmp/build"
@@ -2977,6 +3054,7 @@ installRemoteModule() {
 				cd - >/dev/null
 				rm "$PHP_INI_DIR/conf.d/tdlib.ini"
 				installRemoteModule_manuallyInstalled=1
+			fi
 			;;
 		tensor)
 			if test -z "$installRemoteModule_version"; then
@@ -3135,10 +3213,12 @@ installRemoteModule() {
 			;;
 	esac
 	if test $installRemoteModule_manuallyInstalled -eq 0; then
-		if test -n "$installRemoteModule_version"; then
+		if test -n "$installRemoteModule_path"; then
+			printf '  (installing version %s from %s)\n' "$installRemoteModule_version" "$installRemoteModule_path"
+		elif test -n "$installRemoteModule_version"; then
 			printf '  (installing version %s)\n' "$installRemoteModule_version"
 		fi
-		installPeclPackage "$installRemoteModule_module" "$installRemoteModule_version" "$installRemoteModule_cppflags"
+		installPeclPackage "$installRemoteModule_module" "$installRemoteModule_version" "$installRemoteModule_cppflags" "$installRemoteModule_path"
 	fi
 	postProcessModule "$installRemoteModule_module"
 	checkModuleWorking "$installRemoteModule_module"
@@ -3263,7 +3343,8 @@ addConfigureOption() {
 # Arguments:
 #   $1: the package to be installed
 #   $2: the package version to be installed (optional)
-#   $3: the value of the CPPFLAGS variable
+#   $3: the value of the CPPFLAGS variable (optional)
+#   $4: the path of the local package to be installed (optional, downloaded from PECL if omitted/empty)
 installPeclPackage() {
 	if ! test -f "$CONFIGURE_FILE"; then
 		printf '\n' >"$CONFIGURE_FILE"
@@ -3274,10 +3355,21 @@ installPeclPackage() {
 	else
 		installPeclPackage_fullname="$installPeclPackage_name-$2"
 	fi
+	installPeclPackage_path="${4:-}"
+	if test -z "$installPeclPackage_path"; then
+		installPeclPackage_path="$installPeclPackage_fullname"
+	fi
 	if test $USE_PICKLE -eq 0; then
-		cat "$CONFIGURE_FILE" | MAKE="make -j$(getCompilationProcessorCount $1)" CPPFLAGS="${3:-}" pecl install "$installPeclPackage_fullname"
+		if test -n "${4:-}"; then
+			if test -f "$installPeclPackage_path/package2.xml"; then
+				installPeclPackage_path="$installPeclPackage_path/package2.xml"
 			else
-		MAKE="make -j$(getCompilationProcessorCount $1)" CPPFLAGS="${3:-}" /tmp/pickle install --tmp-dir=/tmp/pickle.tmp --no-interaction --version-override='' --with-configure-options "$CONFIGURE_FILE" -- "$installPeclPackage_fullname"
+				installPeclPackage_path="$installPeclPackage_path/package.xml"
+			fi
+		fi
+		cat "$CONFIGURE_FILE" | MAKE="make -j$(getCompilationProcessorCount $1)" CPPFLAGS="${3:-}" pecl install "$installPeclPackage_path"
+	else
+		MAKE="make -j$(getCompilationProcessorCount $1)" CPPFLAGS="${3:-}" /tmp/pickle install --tmp-dir=/tmp/pickle.tmp --no-interaction --version-override='' --with-configure-options "$CONFIGURE_FILE" -- "$installPeclPackage_path"
 	fi
 }