From 7259630cbe8c38caa35772e481150f05a4be2b7d Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Wed, 30 Nov 2016 22:34:59 +0100 Subject: [PATCH 1/4] Fix support for partial provider repos --- .../Repository/ComposerRepository.php | 127 +++++++++++------- 1 file changed, 82 insertions(+), 45 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 4cf661517..e2a73beb0 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -59,6 +59,8 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito protected $distMirrors; private $degradedMode = false; private $rootData; + private $hasPartialPackages; + private $partialPackagesByName; public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null) { @@ -280,69 +282,84 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $this->providers[$name]; } - // skip platform packages, root package and composer-plugin-api - if (preg_match(PlatformRepository::PLATFORM_PACKAGE_REGEX, $name) || '__root__' === $name || 'composer-plugin-api' === $name) { - return array(); + if ($this->hasPartialPackages && null === $this->partialPackagesByName) { + $this->initializePartialPackages(); } - if (null === $this->providerListing) { - $this->loadProviderListings($this->loadRootServerFile()); - } - - $useLastModifiedCheck = false; - if ($this->lazyProvidersUrl && !isset($this->providerListing[$name])) { - $hash = null; - $url = str_replace('%package%', $name, $this->lazyProvidersUrl); - $cacheKey = 'provider-'.strtr($name, '/', '$').'.json'; - $useLastModifiedCheck = true; - } elseif ($this->providersUrl) { - // package does not exist in this repo - if (!isset($this->providerListing[$name])) { + if (!$this->hasPartialPackages || !isset($this->partialPackagesByName[$name])) { + // skip platform packages, root package and composer-plugin-api + if (preg_match(PlatformRepository::PLATFORM_PACKAGE_REGEX, $name) || '__root__' === $name || 'composer-plugin-api' === $name) { return array(); } - $hash = $this->providerListing[$name]['sha256']; - $url = str_replace(array('%package%', '%hash%'), array($name, $hash), $this->providersUrl); - $cacheKey = 'provider-'.strtr($name, '/', '$').'.json'; - } else { - return array(); - } + if (null === $this->providerListing) { + $this->loadProviderListings($this->loadRootServerFile()); + } - $packages = null; - if ($cacheKey) { - if (!$useLastModifiedCheck && $hash && $this->cache->sha256($cacheKey) === $hash) { - $packages = json_decode($this->cache->read($cacheKey), true); - } elseif ($useLastModifiedCheck) { - if ($contents = $this->cache->read($cacheKey)) { - $contents = json_decode($contents, true); - if (isset($contents['last-modified'])) { - $response = $this->fetchFileIfLastModified($url, $cacheKey, $contents['last-modified']); - if (true === $response) { - $packages = $contents; - } elseif ($response) { - $packages = $response; + $useLastModifiedCheck = false; + if ($this->lazyProvidersUrl && !isset($this->providerListing[$name])) { + $hash = null; + $url = str_replace('%package%', $name, $this->lazyProvidersUrl); + $cacheKey = 'provider-'.strtr($name, '/', '$').'.json'; + $useLastModifiedCheck = true; + } elseif ($this->providersUrl) { + // package does not exist in this repo + if (!isset($this->providerListing[$name])) { + return array(); + } + + $hash = $this->providerListing[$name]['sha256']; + $url = str_replace(array('%package%', '%hash%'), array($name, $hash), $this->providersUrl); + $cacheKey = 'provider-'.strtr($name, '/', '$').'.json'; + } else { + return array(); + } + + $packages = null; + if ($cacheKey) { + if (!$useLastModifiedCheck && $hash && $this->cache->sha256($cacheKey) === $hash) { + $packages = json_decode($this->cache->read($cacheKey), true); + } elseif ($useLastModifiedCheck) { + if ($contents = $this->cache->read($cacheKey)) { + $contents = json_decode($contents, true); + if (isset($contents['last-modified'])) { + $response = $this->fetchFileIfLastModified($url, $cacheKey, $contents['last-modified']); + if (true === $response) { + $packages = $contents; + } elseif ($response) { + $packages = $response; + } } } } } - } - if (!$packages) { - try { - $packages = $this->fetchFile($url, $cacheKey, $hash, $useLastModifiedCheck); - } catch (TransportException $e) { - // 404s are acceptable for lazy provider repos - if ($e->getStatusCode() === 404 && $this->lazyProvidersUrl) { - $packages = array('packages' => array()); - } else { - throw $e; + if (!$packages) { + try { + $packages = $this->fetchFile($url, $cacheKey, $hash, $useLastModifiedCheck); + } catch (TransportException $e) { + // 404s are acceptable for lazy provider repos + if ($e->getStatusCode() === 404 && $this->lazyProvidersUrl) { + $packages = array('packages' => array()); + } else { + throw $e; + } } } + + $loadingPartialPackage = false; + } else { + $packages = array('packages' => array('versions' => $this->partialPackagesByName[$name])); + $loadingPartialPackage = true; } $this->providers[$name] = array(); foreach ($packages['packages'] as $versions) { foreach ($versions as $version) { + if (!$loadingPartialPackage && $this->hasPartialPackages && isset($this->partialPackagesByName[$version['name']])) { + continue; + } + // avoid loading the same objects twice if (isset($this->providersByUid[$version['uid']])) { // skip if already assigned @@ -492,6 +509,8 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito if (!empty($data['providers-lazy-url'])) { $this->lazyProvidersUrl = $this->canonicalizeUrl($data['providers-lazy-url']); $this->hasProviders = true; + + $this->hasPartialPackages = !empty($data['packages']) && is_array($data['packages']); } if ($this->allowSslDowngrade) { @@ -754,4 +773,22 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } } } + + /** + * This initializes the packages key of a partial packages.json that contain some packages inlined + a providers-lazy-url + * + * This should only be called once + */ + private function initializePartialPackages() + { + $rootData = $this->loadRootServerFile(); + + $this->partialPackagesByName = array(); + foreach ($rootData['packages'] as $package => $versions) { + $this->partialPackagesByName[strtolower($package)] = $versions; + } + + // wipe rootData as it is fully consumed at this point and this saves some memory + $this->rootData = true; + } } From 91e7372c6eab3d480d49f73f7861d3448b4845d3 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Thu, 3 Nov 2016 10:59:59 +0100 Subject: [PATCH 2/4] Backport update packagist default repo to be packagist.org --- doc/05-repositories.md | 4 ++-- src/Composer/Config.php | 21 +++++++++++++++---- .../Test/Json/JsonManipulatorTest.php | 4 ++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/doc/05-repositories.md b/doc/05-repositories.md index f29d4187f..d48ec4939 100644 --- a/doc/05-repositories.md +++ b/doc/05-repositories.md @@ -312,7 +312,7 @@ VCS repository provides `dist`s for them that fetch the packages as zips. * **BitBucket:** [bitbucket.org](https://bitbucket.org) (Git and Mercurial) The VCS driver to be used is detected automatically based on the URL. However, -should you need to specify one for whatever reason, you can use `fossil`, `git`, +should you need to specify one for whatever reason, you can use `fossil`, `git`, `svn` or `hg` as the repository type instead of `vcs`. If you set the `no-api` key to `true` on a github repository it will clone the @@ -686,7 +686,7 @@ You can disable the default Packagist repository by adding this to your { "repositories": [ { - "packagist": false + "packagist.org": false } ] } diff --git a/src/Composer/Config.php b/src/Composer/Config.php index 757811141..c25fee6db 100644 --- a/src/Composer/Config.php +++ b/src/Composer/Config.php @@ -67,7 +67,7 @@ class Config ); public static $defaultRepositories = array( - 'packagist' => array( + 'packagist.org' => array( 'type' => 'composer', 'url' => 'https?://packagist.org', 'allow_ssl_downgrade' => true, @@ -157,13 +157,13 @@ class Config foreach ($newRepos as $name => $repository) { // disable a repository by name if (false === $repository) { - unset($this->repositories[$name]); + $this->disableRepoByName($name); continue; } // disable a repository with an anonymous {"name": false} repo if (is_array($repository) && 1 === count($repository) && false === current($repository)) { - unset($this->repositories[key($repository)]); + $this->disableRepoByName(key($repository)); continue; } @@ -171,7 +171,11 @@ class Config if (is_int($name)) { $this->repositories[] = $repository; } else { - $this->repositories[$name] = $repository; + if ($name === 'packagist') { // BC support for default "packagist" named repo + $this->repositories[$name . '.org'] = $repository; + } else { + $this->repositories[$name] = $repository; + } } } $this->repositories = array_reverse($this->repositories, true); @@ -404,6 +408,15 @@ class Config return false; } + private function disableRepoByName($name) + { + if (isset($this->repositories[$name])) { + unset($this->repositories[$name]); + } else if ($name === 'packagist') { // BC support for default "packagist" named repo + unset($this->repositories['packagist.org']); + } + } + /** * Validates that the passed URL is allowed to be used by current config, or throws an exception. * diff --git a/tests/Composer/Test/Json/JsonManipulatorTest.php b/tests/Composer/Test/Json/JsonManipulatorTest.php index 2652ad0b5..893350a3f 100644 --- a/tests/Composer/Test/Json/JsonManipulatorTest.php +++ b/tests/Composer/Test/Json/JsonManipulatorTest.php @@ -292,7 +292,7 @@ class JsonManipulatorTest extends \PHPUnit_Framework_TestCase }, "repositories": [ { - "packagist": false + "packagist.org": false }, { "type": "package", @@ -773,7 +773,7 @@ class JsonManipulatorTest extends \PHPUnit_Framework_TestCase }, "repositories": [ { - "packagist": false + "packagist.org": false }, { "type": "package", From ce4d941b3ea261fa539bce0eae1a3bac9cd7226b Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Wed, 30 Nov 2016 23:56:38 +0100 Subject: [PATCH 3/4] Prepare 1.2.3 changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b45cc6ca..a89ac7422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +### [1.2.3] - 2016-12-01 + + * Fixed bug in HgDriver failing to identify BitBucket repositories + * Fixed support for loading partial provider repositories + ### [1.2.2] - 2016-11-03 * Fixed selection of packages based on stability to be independent from package repository order From 8a2f597db34d293c90da94d67e789e1b49c10af3 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Thu, 3 Nov 2016 11:30:35 +0100 Subject: [PATCH 4/4] Fix tests --- tests/Composer/Test/ConfigTest.php | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/Composer/Test/ConfigTest.php b/tests/Composer/Test/ConfigTest.php index 81d7646dd..d7df18fc5 100644 --- a/tests/Composer/Test/ConfigTest.php +++ b/tests/Composer/Test/ConfigTest.php @@ -35,12 +35,19 @@ class ConfigTest extends \PHPUnit_Framework_TestCase $data = array(); $data['local config inherits system defaults'] = array( array( - 'packagist' => array('type' => 'composer', 'url' => 'https?://packagist.org', 'allow_ssl_downgrade' => true), + 'packagist.org' => array('type' => 'composer', 'url' => 'https?://packagist.org', 'allow_ssl_downgrade' => true), ), array(), ); $data['local config can disable system config by name'] = array( + array(), + array( + array('packagist.org' => false), + ), + ); + + $data['local config can disable system config by name bc'] = array( array(), array( array('packagist' => false), @@ -51,7 +58,7 @@ class ConfigTest extends \PHPUnit_Framework_TestCase array( 1 => array('type' => 'vcs', 'url' => 'git://github.com/composer/composer.git'), 0 => array('type' => 'pear', 'url' => 'http://pear.composer.org'), - 'packagist' => array('type' => 'composer', 'url' => 'https?://packagist.org', 'allow_ssl_downgrade' => true), + 'packagist.org' => array('type' => 'composer', 'url' => 'https?://packagist.org', 'allow_ssl_downgrade' => true), ), array( array('type' => 'vcs', 'url' => 'git://github.com/composer/composer.git'), @@ -62,7 +69,7 @@ class ConfigTest extends \PHPUnit_Framework_TestCase $data['system config adds above core defaults'] = array( array( 'example.com' => array('type' => 'composer', 'url' => 'http://example.com'), - 'packagist' => array('type' => 'composer', 'url' => 'https?://packagist.org', 'allow_ssl_downgrade' => true), + 'packagist.org' => array('type' => 'composer', 'url' => 'https?://packagist.org', 'allow_ssl_downgrade' => true), ), array(), array( @@ -76,7 +83,7 @@ class ConfigTest extends \PHPUnit_Framework_TestCase 'example.com' => array('type' => 'composer', 'url' => 'http://example.com'), ), array( - array('packagist' => false), + array('packagist.org' => false), array('type' => 'composer', 'url' => 'http://packagist.org'), ), array( @@ -86,11 +93,11 @@ class ConfigTest extends \PHPUnit_Framework_TestCase $data['local config can override by name to bring a repo above system config'] = array( array( - 'packagist' => array('type' => 'composer', 'url' => 'http://packagistnew.org'), + 'packagist.org' => array('type' => 'composer', 'url' => 'http://packagistnew.org'), 'example.com' => array('type' => 'composer', 'url' => 'http://example.com'), ), array( - 'packagist' => array('type' => 'composer', 'url' => 'http://packagistnew.org'), + 'packagist.org' => array('type' => 'composer', 'url' => 'http://packagistnew.org'), ), array( 'example.com' => array('type' => 'composer', 'url' => 'http://example.com'), @@ -99,7 +106,7 @@ class ConfigTest extends \PHPUnit_Framework_TestCase $data['incorrect local config does not cause ErrorException'] = array( array( - 'packagist' => array('type' => 'composer', 'url' => 'https?://packagist.org', 'allow_ssl_downgrade' => true), + 'packagist.org' => array('type' => 'composer', 'url' => 'https?://packagist.org', 'allow_ssl_downgrade' => true), 'type' => 'vcs', 'url' => 'http://example.com', ),