Avoid loading same packages multiple times

src/Composer/DependencyResolver/PoolBuilder.php

@@ -70,6 +70,10 @@ class PoolBuilder
      * @psalm-var array<string, ConstraintInterface>
      */
     private $loadedPackages = array();
+    /**
+     * @psalm-var array<int, array<string, array<string, PackageInterface>>>
+     */
+    private $loadedPerRepo = array();
     /**
      * @psalm-var Package[]
      */
@@ -227,8 +231,11 @@ class PoolBuilder
         $this->aliasMap = array();
         $this->packagesToLoad = array();
         $this->loadedPackages = array();
+        $this->loadedPerRepo = array();
         $this->packages = array();
         $this->unacceptableFixedPackages = array();
+        $this->maxExtendedReqs = array();
+        $this->skippedLoad = array();
         $this->indexCounter = 0;
 
         return $pool;
@@ -287,20 +294,13 @@ class PoolBuilder
     private function loadPackagesMarkedForLoading(Request $request, $repositories)
     {
         foreach ($this->packagesToLoad as $name => $constraint) {
-            // remove all already-loaded packages matching those to be loaded to avoid duplicates
-            foreach ($this->packages as $index => $pkg) {
-                if ($pkg->getName() === $name) {
-                    $this->removeLoadedPackage($request, $pkg, $index);
-                }
-            }
-
             $this->loadedPackages[$name] = $constraint;
         }
 
         $packageBatch = $this->packagesToLoad;
         $this->packagesToLoad = array();
 
-        foreach ($repositories as $repository) {
+        foreach ($repositories as $repoIndex => $repository) {
             if (empty($packageBatch)) {
                 break;
             }
@@ -310,13 +310,14 @@ class PoolBuilder
             if ($repository instanceof PlatformRepository || $repository === $request->getLockedRepository()) {
                 continue;
             }
-            $result = $repository->loadPackages($packageBatch, $this->acceptableStabilities, $this->stabilityFlags);
+            $result = $repository->loadPackages($packageBatch, $this->acceptableStabilities, $this->stabilityFlags, isset($this->loadedPerRepo[$repoIndex]) ? $this->loadedPerRepo[$repoIndex] : array());
 
             foreach ($result['namesFound'] as $name) {
                 // avoid loading the same package again from other repositories once it has been found
                 unset($packageBatch[$name]);
             }
             foreach ($result['packages'] as $package) {
+                $this->loadedPerRepo[$repoIndex][$package->getName()][$package->getVersion()] = $package;
                 $this->loadPackage($request, $package);
             }
         }

src/Composer/Repository/ArrayRepository.php

@@ -50,7 +50,7 @@ class ArrayRepository implements RepositoryInterface
     /**
      * {@inheritDoc}
      */
-    public function loadPackages(array $packageMap, array $acceptableStabilities, array $stabilityFlags)
+    public function loadPackages(array $packageMap, array $acceptableStabilities, array $stabilityFlags, array $alreadyLoaded = array())
     {
         $packages = $this->getPackages();
 
@@ -61,6 +61,7 @@ class ArrayRepository implements RepositoryInterface
                 if (
                     (!$packageMap[$package->getName()] || $packageMap[$package->getName()]->matches(new Constraint('==', $package->getVersion())))
                     && StabilityFilter::isPackageAcceptable($acceptableStabilities, $stabilityFlags, $package->getNames(), $package->getStability())
+                    && !isset($alreadyLoaded[$package->getName()][$package->getVersion()])
                 ) {
                     // add selected packages which match stability requirements
                     $result[spl_object_hash($package)] = $package;

src/Composer/Repository/ComposerRepository.php

@@ -340,13 +340,13 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
         return $names;
     }
 
-    public function loadPackages(array $packageNameMap, array $acceptableStabilities, array $stabilityFlags)
+    public function loadPackages(array $packageNameMap, array $acceptableStabilities, array $stabilityFlags, array $alreadyLoaded = array())
     {
         // this call initializes loadRootServerFile which is needed for the rest below to work
         $hasProviders = $this->hasProviders();
 
         if (!$hasProviders && !$this->hasPartialPackages() && !$this->lazyProvidersUrl) {
-            return parent::loadPackages($packageNameMap, $acceptableStabilities, $stabilityFlags);
+            return parent::loadPackages($packageNameMap, $acceptableStabilities, $stabilityFlags, $alreadyLoaded);
         }
 
         $packages = array();
@@ -362,12 +362,13 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
                     continue;
                 }
 
-                $candidates = $this->whatProvides($name, $acceptableStabilities, $stabilityFlags);
+                $candidates = $this->whatProvides($name, $acceptableStabilities, $stabilityFlags, $alreadyLoaded);
                 foreach ($candidates as $candidate) {
                     if ($candidate->getName() !== $name) {
                         throw new \LogicException('whatProvides should never return a package with a different name than the requested one');
                     }
                     $namesFound[$name] = true;
+
                     if (!$constraint || $constraint->matches(new Constraint('==', $candidate->getVersion()))) {
                         $matches[spl_object_hash($candidate)] = $candidate;
                         if ($candidate instanceof AliasPackage && !isset($matches[spl_object_hash($candidate->getAliasOf())])) {
@@ -400,7 +401,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
                 }
             }
 
-            $result = $this->loadAsyncPackages($packageNameMap, $acceptableStabilities, $stabilityFlags);
+            $result = $this->loadAsyncPackages($packageNameMap, $acceptableStabilities, $stabilityFlags, $alreadyLoaded);
             $packages = array_merge($packages, $result['packages']);
             $namesFound = array_merge($namesFound, $result['namesFound']);
         }
@@ -530,7 +531,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
      * @param string $name package name
      * @return array|mixed
      */
-    private function whatProvides($name, array $acceptableStabilities = null, array $stabilityFlags = null)
+    private function whatProvides($name, array $acceptableStabilities = null, array $stabilityFlags = null, array $alreadyLoaded = array())
     {
         if (!$this->hasPartialPackages() || !isset($this->partialPackagesByName[$name])) {
             // skip platform packages, root package and composer-plugin-api
@@ -622,6 +623,11 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
                         $version['version_normalized'] = $this->versionParser->normalize($version['version']);
                     }
 
+                    // avoid loading packages which have already been loaded
+                    if (isset($alreadyLoaded[$name][$version['version_normalized']])) {
+                        continue;
+                    }
+
                     if ($this->isVersionAcceptable(null, $normalizedName, $version, $acceptableStabilities, $stabilityFlags)) {
                         $versionsToLoad[$version['uid']] = $version;
                     }
@@ -679,7 +685,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
     /**
      * @param array $packageNames array of package name => ConstraintInterface|null - if a constraint is provided, only packages matching it will be loaded
      */
-    private function loadAsyncPackages(array $packageNames, array $acceptableStabilities = null, array $stabilityFlags = null)
+    private function loadAsyncPackages(array $packageNames, array $acceptableStabilities = null, array $stabilityFlags = null, array $alreadyLoaded = array())
     {
         $this->loadRootServerFile();
 
@@ -722,7 +728,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
             }
 
             $promises[] = $this->asyncFetchFile($url, $cacheKey, $lastModified)
-                ->then(function ($response) use (&$packages, &$namesFound, $contents, $realName, $constraint, $repo, $acceptableStabilities, $stabilityFlags) {
+                ->then(function ($response) use (&$packages, &$namesFound, $contents, $realName, $constraint, $repo, $acceptableStabilities, $stabilityFlags, $alreadyLoaded) {
                     if (true === $response) {
                         $response = $contents;
                     }
@@ -747,6 +753,11 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
                             $version['version_normalized'] = $repo->versionParser->normalize($version['version']);
                         }
 
+                        // avoid loading packages which have already been loaded
+                        if (isset($alreadyLoaded[$realName][$version['version_normalized']])) {
+                            continue;
+                        }
+
                         if ($repo->isVersionAcceptable($constraint, $realName, $version, $acceptableStabilities, $stabilityFlags)) {
                             $versionsToLoad[] = $version;
                         }

src/Composer/Repository/CompositeRepository.php

@@ -102,13 +102,13 @@ class CompositeRepository implements RepositoryInterface
     /**
      * {@inheritDoc}
      */
-    public function loadPackages(array $packageMap, array $acceptableStabilities, array $stabilityFlags)
+    public function loadPackages(array $packageMap, array $acceptableStabilities, array $stabilityFlags, array $alreadyLoaded = array())
     {
         $packages = array();
         $namesFound = array();
         foreach ($this->repositories as $repository) {
             /* @var $repository RepositoryInterface */
-            $result = $repository->loadPackages($packageMap, $acceptableStabilities, $stabilityFlags);
+            $result = $repository->loadPackages($packageMap, $acceptableStabilities, $stabilityFlags, $alreadyLoaded);
             $packages[] = $result['packages'];
             $namesFound[] = $result['namesFound'];
         }

src/Composer/Repository/FilterRepository.php

@@ -108,7 +108,7 @@ class FilterRepository implements RepositoryInterface
     /**
      * {@inheritDoc}
      */
-    public function loadPackages(array $packageMap, array $acceptableStabilities, array $stabilityFlags)
+    public function loadPackages(array $packageMap, array $acceptableStabilities, array $stabilityFlags, array $alreadyLoaded = array())
     {
         foreach ($packageMap as $name => $constraint) {
             if (!$this->isAllowed($name)) {
@@ -120,7 +120,7 @@ class FilterRepository implements RepositoryInterface
             return array('namesFound' => array(), 'packages' => array());
         }
 
-        $result = $this->repo->loadPackages($packageMap, $acceptableStabilities, $stabilityFlags);
+        $result = $this->repo->loadPackages($packageMap, $acceptableStabilities, $stabilityFlags, $alreadyLoaded);
         if (!$this->canonical) {
             $result['namesFound'] = array();
         }

src/Composer/Repository/RepositoryInterface.php

@@ -75,11 +75,13 @@ interface RepositoryInterface extends \Countable
      * @psalm-param array<string, BasePackage::STABILITY_*> $acceptableStabilities
      * @param int[] $stabilityFlags an array of package name => BasePackage::STABILITY_* value
      * @psalm-param array<string, BasePackage::STABILITY_*> $stabilityFlags
+     * @param array[] $alreadyLoaded an array of package name => package version => package
+     * @psalm-param array<string, array<string, PackageInterface>> $alreadyLoaded
      *
      * @return array [namesFound => string[], packages => PackageInterface[]]
      * @psalm-return array{namesFound: string[], packages: PackageInterface[]}
      */
-    public function loadPackages(array $packageNameMap, array $acceptableStabilities, array $stabilityFlags);
+    public function loadPackages(array $packageNameMap, array $acceptableStabilities, array $stabilityFlags, array $alreadyLoaded = array());
 
     /**
      * Searches the repository for packages containing the query

tests/Composer/Test/DependencyResolver/Fixtures/poolbuilder/constraint-expansion-works-with-exact-versions.test

@@ -23,7 +23,7 @@ Tests if version constraint is expanded. If not, dep/dep 3.0.0 would not be load
 --EXPECT--
 [
     "root/req-1.0.0.0",
-    "dep/dep2-2.3.4.0",
     "dep/dep-2.3.4.0",
+    "dep/dep2-2.3.4.0",
     "dep/dep-2.3.5.0"
 ]

tests/Composer/Test/DependencyResolver/Fixtures/poolbuilder/package-versions-are-not-loaded-if-not-required-expansion.test

@@ -25,8 +25,8 @@ Tests if version constraint is expanded. If not, dep/dep 3.0.0 would not be load
 --EXPECT--
 [
     "root/req-1.0.0.0",
-    "dep/dep2-2.3.4.0",
     "dep/dep-2.3.4.0",
     "dep/dep-2.3.5.0",
+    "dep/dep2-2.3.4.0",
     "dep/dep-3.0.0.0"
 ]