public-mirrors/composer
1
0
Fork 0
mirror of https://github.com/composer/composer synced 2025-05-09 00:22:53 +00:00
Code Activity

Implemented PoolOptimizer

Browse source
This commit is contained in:
Yanick Witschi 2020-10-01 16:42:02 +02:00 • committed by Jordi Boggiano
parent 7eca450d9b
commit 34183f49f9
No known key found for this signature in database
GPG key ID: 7BBD42C429EC80BC
30 changed files with 1492 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/Composer/DependencyResolver/Pool.php
View file

@ -221,6 +221,14 @@ class Pool implements \Countable
return \in_array($package, $this->unacceptableFixedOrLockedPackages, true);
}
/**
* @return BasePackage[]
*/
public function getUnacceptableFixedOrLockedPackages()
{
return $this->unacceptableFixedOrLockedPackages;
}
public function __toString()
{
$str = "Pool:\n";

38
src/Composer/DependencyResolver/PoolBuilder.php
View file

@ -61,6 +61,10 @@ class PoolBuilder
* @var ?EventDispatcher
*/
private $eventDispatcher;
/**
* @var PoolOptimizer|null
*/
private $poolOptimizer;
/**
* @var IOInterface
*/
@ -128,13 +132,14 @@ class PoolBuilder
* @param string[] $rootReferences an array of package name => source reference
* @phpstan-param array<string, string> $rootReferences
*/
public function __construct(array $acceptableStabilities, array $stabilityFlags, array $rootAliases, array $rootReferences, IOInterface $io, EventDispatcher $eventDispatcher = null)
public function __construct(array $acceptableStabilities, array $stabilityFlags, array $rootAliases, array $rootReferences, IOInterface $io, EventDispatcher $eventDispatcher = null, PoolOptimizer $poolOptimizer = null)
{
$this->acceptableStabilities = $acceptableStabilities;
$this->stabilityFlags = $stabilityFlags;
$this->rootAliases = $rootAliases;
$this->rootReferences = $rootReferences;
$this->eventDispatcher = $eventDispatcher;
$this->poolOptimizer = $poolOptimizer;
$this->io = $io;
}
@ -259,6 +264,8 @@ class PoolBuilder
$this->skippedLoad = array();
$this->indexCounter = 0;
$pool = $this->runOptimizer($request, $pool);
Intervals::clear();
return $pool;
@ -572,4 +579,33 @@ class PoolBuilder
unset($this->aliasMap[spl_object_hash($package)]);
}
}
/**
* @return Pool
*/
private function runOptimizer(Request $request, Pool $pool)
{
if (null === $this->poolOptimizer) {
return $pool;
}
$total = \count($pool->getPackages());
$pool = $this->poolOptimizer->optimize($request, $pool);
$filtered = $total - \count($pool->getPackages());
if (0 === $filtered) {
return $pool;
}
$this->io->write(sprintf(
'<info>Found %s package versions referenced in your dependency graph. %s (%d%%) were optimized away.</info>',
number_format($total),
number_format($filtered),
round(100/$total*$filtered)
), true, IOInterface::VERY_VERBOSE);
return $pool;
}
}

345
src/Composer/DependencyResolver/PoolOptimizer.php Normal file
View file

@ -0,0 +1,345 @@
<?php
/*
* This file is part of Composer.
*
* (c) Nils Adermann <naderman@naderman.de>
* Jordi Boggiano <j.boggiano@seld.be>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Composer\DependencyResolver;
use Composer\Package\AliasPackage;
use Composer\Package\BasePackage;
use Composer\Semver\CompilingMatcher;
use Composer\Semver\Constraint\ConstraintInterface;
use Composer\Semver\Constraint\Constraint;
use Composer\Semver\Constraint\MultiConstraint;
use Composer\Semver\Intervals;
/**
* Optimizes a given pool
*
* @author Yanick Witschi <yanick.witschi@terminal42.ch>
*/
class PoolOptimizer
{
/**
* @var PolicyInterface
*/
private $policy;
/**
* @var array<int, true>
*/
private $irremovablePackages = array();
/**
* @var array<string, array<string, ConstraintInterface>>
*/
private $requireConstraintsPerPackage = array();
/**
* @var array<string, array<string, ConstraintInterface>>
*/
private $conflictConstraintsPerPackage = array();
/**
* @var array<int, true>
*/
private $packagesToRemove = array();
/**
* @var array<int, BasePackage[]>
*/
private $aliasesPerPackage = array();
public function __construct(PolicyInterface $policy)
{
$this->policy = $policy;
}
/**
* @return Pool
*/
public function optimize(Request $request, Pool $pool)
{
$this->prepare($request, $pool);
$optimizedPool = $this->optimizeByIdenticalDependencies($pool);
// No need to run this recursively at the moment
// because the current optimizations cannot provide
// even more gains when ran again. Might change
// in the future with additional optimizations.
$this->irremovablePackages = array();
$this->requireConstraintsPerPackage = array();
$this->conflictConstraintsPerPackage = array();
$this->packagesToRemove = array();
$this->aliasesPerPackage = array();
return $optimizedPool;
}
/**
* @return void
*/
private function prepare(Request $request, Pool $pool)
{
$irremovablePackageConstraintGroups = array();
// Mark fixed or locked packages as irremovable
foreach ($request->getFixedOrLockedPackages() as $package) {
$irremovablePackageConstraintGroups[$package->getName()][] = new Constraint('==', $package->getVersion());
}
// Extract requested package requirements
foreach ($request->getRequires() as $require => $constraint) {
$constraint = Intervals::compactConstraint($constraint);
$this->requireConstraintsPerPackage[$require][(string) $constraint] = $constraint;
}
// First pass over all packages to extract information and mark package constraints irremovable
foreach ($pool->getPackages() as $package) {
// Extract package requirements
foreach ($package->getRequires() as $link) {
$constraint = Intervals::compactConstraint($link->getConstraint());
$this->requireConstraintsPerPackage[$link->getTarget()][(string) $constraint] = $constraint;
}
// Extract package conflicts
foreach ($package->getConflicts() as $link) {
$constraint = Intervals::compactConstraint($link->getConstraint());
$this->conflictConstraintsPerPackage[$link->getTarget()][(string) $constraint] = $constraint;
}
// Keep track of alias packages for every package so if either the alias or aliased is kept
// we keep the others as they are a unit of packages really
if ($package instanceof AliasPackage) {
$this->aliasesPerPackage[$package->getAliasOf()->id][] = $package;
}
}
$irremovablePackageConstraints = array();
foreach ($irremovablePackageConstraintGroups as $packageName => $constraints) {
$irremovablePackageConstraints[$packageName] = 1 === \count($constraints) ? $constraints[0] : new MultiConstraint($constraints, false);
}
unset($irremovablePackageConstraintGroups);
// Mark the packages as irremovable based on the constraints
foreach ($pool->getPackages() as $package) {
if (!isset($irremovablePackageConstraints[$package->getName()])) {
continue;
}
if (CompilingMatcher::match($irremovablePackageConstraints[$package->getName()], Constraint::OP_EQ, $package->getVersion())) {
$this->markPackageIrremovable($package);
}
}
}
/**
* @return void
*/
private function markPackageIrremovable(BasePackage $package)
{
$this->irremovablePackages[$package->id] = true;
if ($package instanceof AliasPackage) {
// recursing here so aliasesPerPackage for the aliasOf can be checked
// and all its aliases marked as irremovable as well
$this->markPackageIrremovable($package->getAliasOf());
}
if (isset($this->aliasesPerPackage[$package->id])) {
foreach ($this->aliasesPerPackage[$package->id] as $aliasPackage) {
$this->irremovablePackages[$aliasPackage->id] = true;
}
}
}
/**
* @return Pool Optimized pool
*/
private function applyRemovalsToPool(Pool $pool)
{
$packages = array();
foreach ($pool->getPackages() as $package) {
if (!isset($this->packagesToRemove[$package->id])) {
$packages[] = $package;
}
}
$optimizedPool = new Pool($packages, $pool->getUnacceptableFixedOrLockedPackages());
// Reset package removals
$this->packagesToRemove = array();
return $optimizedPool;
}
/**
* @return Pool
*/
private function optimizeByIdenticalDependencies(Pool $pool)
{
$identicalDefinitionPerPackage = array();
$packageIdsToRemove = array();
foreach ($pool->getPackages() as $package) {
// If that package was already marked irremovable, we can skip
// the entire process for it
if (isset($this->irremovablePackages[$package->id])) {
continue;
}
$packageIdsToRemove[$package->id] = true;
$dependencyHash = $this->calculateDependencyHash($package);
foreach ($package->getNames(false) as $packageName) {
if (!isset($this->requireConstraintsPerPackage[$packageName])) {
continue;
}
foreach ($this->requireConstraintsPerPackage[$packageName] as $requireConstraint) {
$groupHashParts = array();
if (CompilingMatcher::match($requireConstraint, Constraint::OP_EQ, $package->getVersion())) {
$groupHashParts[] = 'require:' . (string) $requireConstraint;
}
if ($package->getReplaces()) {
foreach ($package->getReplaces() as $link) {
if (CompilingMatcher::match($link->getConstraint(), Constraint::OP_EQ, $package->getVersion())) {
// Use the same hash part as the regular require hash because that's what the replacement does
$groupHashParts[] = 'require:' . (string) $link->getConstraint();
}
}
}
if (isset($this->conflictConstraintsPerPackage[$packageName])) {
foreach ($this->conflictConstraintsPerPackage[$packageName] as $conflictConstraint) {
if (CompilingMatcher::match($conflictConstraint, Constraint::OP_EQ, $package->getVersion())) {
$groupHashParts[] = 'conflict:' . (string) $conflictConstraint;
}
}
}
if (!$groupHashParts) {
continue;
}
$identicalDefinitionPerPackage[$packageName][implode('', $groupHashParts)][$dependencyHash][] = $package;
}
}
}
$keepPackage = function (BasePackage $package, $aliasesPerPackage) use (&$packageIdsToRemove, &$keepPackage) {
unset($packageIdsToRemove[$package->id]);
if ($package instanceof AliasPackage) {
// recursing here so aliasesPerPackage for the aliasOf can be checked
// and all its aliases marked to be kept as well
$keepPackage($package->getAliasOf(), $aliasesPerPackage);
}
if (isset($aliasesPerPackage[$package->id])) {
foreach ($aliasesPerPackage[$package->id] as $aliasPackage) {
unset($packageIdsToRemove[$aliasPackage->id]);
}
}
};
foreach ($identicalDefinitionPerPackage as $package => $constraintGroups) {
foreach ($constraintGroups as $constraintGroup) {
foreach ($constraintGroup as $hash => $packages) {
// Only one package in this constraint group has the same requirements, we're not allowed to remove that package
if (1 === \count($packages)) {
$keepPackage($packages[0], $this->aliasesPerPackage);
continue;
}
// Otherwise we find out which one is the preferred package in this constraint group which is
// then not allowed to be removed either
$literals = array();
foreach ($packages as $package) {
$literals[] = $package->id;
}
foreach ($this->policy->selectPreferredPackages($pool, $literals) as $preferredLiteral) {
$keepPackage($pool->literalToPackage($preferredLiteral), $this->aliasesPerPackage);
}
}
}
}
foreach ($packageIdsToRemove as $id => $dummy) {
$this->markPackageForRemoval($id);
}
// Apply removals
return $this->applyRemovalsToPool($pool);
}
/**
* @return string
*/
private function calculateDependencyHash(BasePackage $package)
{
$hash = '';
$hashRelevantLinks = array(
'requires' => $package->getRequires(),
'conflicts' => $package->getConflicts(),
'replaces' => $package->getReplaces(),
'provides' => $package->getProvides()
);
foreach ($hashRelevantLinks as $key => $links) {
if (0 === \count($links)) {
continue;
}
// start new hash section
$hash .= $key . ':';
$subhash = array();
foreach ($links as $link) {
// To get the best dependency hash matches we should use Intervals::compactConstraint() here.
// However, the majority of projects are going to specify their constraints already pretty
// much in the best variant possible. In other words, we'd be wasting time here and it would actually hurt
// performance more than the additional few packages that could be filtered out would benefit the process.
$subhash[$link->getTarget()] = (string) $link->getConstraint();
}
// Sort for best result
ksort($subhash);
foreach ($subhash as $target => $constraint) {
$hash .= $target . '@' . $constraint;
}
}
return $hash;
}
/**
* @param int $id
* @return void
*/
private function markPackageForRemoval($id)
{
// We are not allowed to remove packages if they have been marked as irremovable
if (isset($this->irremovablePackages[$id])) {
throw new \LogicException('Attempted removing a package which was previously marked irremovable');
}
$this->packagesToRemove[$id] = true;
}
}

20
src/Composer/Installer.php
View file

@ -20,6 +20,7 @@ use Composer\DependencyResolver\LockTransaction;
use Composer\DependencyResolver\Operation\UpdateOperation;
use Composer\DependencyResolver\Operation\InstallOperation;
use Composer\DependencyResolver\Operation\UninstallOperation;
use Composer\DependencyResolver\PoolOptimizer;
use Composer\DependencyResolver\Pool;
use Composer\DependencyResolver\Request;
use Composer\DependencyResolver\Solver;
@ -430,7 +431,7 @@ class Installer
$request->setUpdateAllowList($this->updateAllowList, $this->updateAllowTransitiveDependencies);
}
$pool = $repositorySet->createPool($request, $this->io, $this->eventDispatcher);
$pool = $repositorySet->createPool($request, $this->io, $this->eventDispatcher, $this->createPoolOptimizer($policy));
$this->io->writeError('<info>Updating dependencies</info>');
@ -999,6 +1000,23 @@ class Installer
);
}
/**
* @return PoolOptimizer|null
*/
private function createPoolOptimizer(PolicyInterface $policy)
{
// Not the best architectural decision here, would need to be able
// to configure from the outside of Installer but this is only
// a debugging tool and should never be required in any other use case
if ('0' === Platform::getEnv('COMPOSER_POOL_OPTIMIZER')) {
$this->io->write('Pool Optimizer was disabled for debugging purposes.', true, IOInterface::DEBUG);
return null;
}
return new PoolOptimizer($policy);
}
/**
* Create Installer
*

6
src/Composer/Repository/RepositorySet.php
View file

@ -12,6 +12,8 @@
namespace Composer\Repository;
use Composer\DependencyResolver\PoolOptimizer;
use Composer\DependencyResolver\PolicyInterface;
use Composer\DependencyResolver\Pool;
use Composer\DependencyResolver\PoolBuilder;
use Composer\DependencyResolver\Request;
@ -244,9 +246,9 @@ class RepositorySet
*
* @return Pool
*/
public function createPool(Request $request, IOInterface $io, EventDispatcher $eventDispatcher = null)
public function createPool(Request $request, IOInterface $io, EventDispatcher $eventDispatcher = null, PoolOptimizer $poolOptimizer = null)
{
$poolBuilder = new PoolBuilder($this->acceptableStabilities, $this->stabilityFlags, $this->rootAliases, $this->rootReferences, $io, $eventDispatcher);
$poolBuilder = new PoolBuilder($this->acceptableStabilities, $this->stabilityFlags, $this->rootAliases, $this->rootReferences, $io, $eventDispatcher, $poolOptimizer);
foreach ($this->repositories as $repo) {
if (($repo instanceof InstalledRepositoryInterface || $repo instanceof InstalledRepository) && !$this->allowInstalledRepositories) {