From 56805ecafe8597a07b4576b45e5532b9862fd65a Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Wed, 12 Sep 2018 18:58:54 +0200 Subject: [PATCH 01/34] Add HttpDownloader to wrap/replace RemoteFilesystem with a new curl multi implementation --- composer.json | 3 +- composer.lock | 46 ++- src/Composer/Downloader/FileDownloader.php | 20 +- src/Composer/Downloader/GzipDownloader.php | 6 +- src/Composer/Downloader/RarDownloader.php | 6 +- src/Composer/Downloader/XzDownloader.php | 6 +- src/Composer/Downloader/ZipDownloader.php | 6 +- src/Composer/Factory.php | 14 +- src/Composer/Plugin/PreFileDownloadEvent.php | 20 +- .../Repository/ComposerRepository.php | 220 +++++++++++--- src/Composer/Repository/RepositoryFactory.php | 6 +- src/Composer/Repository/RepositoryManager.php | 6 +- src/Composer/Util/Http/CurlDownloader.php | 282 ++++++++++++++++++ src/Composer/Util/Http/Response.php | 75 +++++ src/Composer/Util/HttpDownloader.php | 246 +++++++++++++++ src/Composer/Util/RemoteFilesystem.php | 108 +------ src/Composer/Util/StreamContextFactory.php | 105 +++++++ 17 files changed, 985 insertions(+), 190 deletions(-) create mode 100644 src/Composer/Util/Http/CurlDownloader.php create mode 100644 src/Composer/Util/Http/Response.php create mode 100644 src/Composer/Util/HttpDownloader.php diff --git a/composer.json b/composer.json index 41048903b..1b75131bc 100644 --- a/composer.json +++ b/composer.json @@ -34,7 +34,8 @@ "symfony/console": "^2.7 || ^3.0 || ^4.0", "symfony/filesystem": "^2.7 || ^3.0 || ^4.0", "symfony/finder": "^2.7 || ^3.0 || ^4.0", - "symfony/process": "^2.7 || ^3.0 || ^4.0" + "symfony/process": "^2.7 || ^3.0 || ^4.0", + "react/promise": "^1.2" }, "conflict": { "symfony/console": "2.8.38" diff --git a/composer.lock b/composer.lock index 957382bc6..63b8033b9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e46280c4cfd37bf3ec8be36095feb20e", + "content-hash": "d356b92e869790db1e9d2c0f4b10935e", "packages": [ { "name": "composer/ca-bundle", @@ -342,6 +342,50 @@ ], "time": "2018-11-20T15:27:04+00:00" }, + { + "name": "react/promise", + "version": "v1.2.1", + "source": { + "type": "git", + "url": "https://github.com/reactphp/promise.git", + "reference": "eefff597e67ff66b719f8171480add3c91474a1e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/reactphp/promise/zipball/eefff597e67ff66b719f8171480add3c91474a1e", + "reference": "eefff597e67ff66b719f8171480add3c91474a1e", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1-dev" + } + }, + "autoload": { + "psr-0": { + "React\\Promise": "src/" + }, + "files": [ + "src/React/Promise/functions_include.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jan Sorgalla", + "email": "jsorgalla@gmail.com" + } + ], + "description": "A lightweight implementation of CommonJS Promises/A for PHP", + "time": "2016-03-07T13:46:50+00:00" + }, { "name": "seld/jsonlint", "version": "1.7.1", diff --git a/src/Composer/Downloader/FileDownloader.php b/src/Composer/Downloader/FileDownloader.php index 6596d9c8b..6b1349bb8 100644 --- a/src/Composer/Downloader/FileDownloader.php +++ b/src/Composer/Downloader/FileDownloader.php @@ -24,7 +24,7 @@ use Composer\Plugin\PluginEvents; use Composer\Plugin\PreFileDownloadEvent; use Composer\EventDispatcher\EventDispatcher; use Composer\Util\Filesystem; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Util\Url as UrlUtil; /** @@ -39,7 +39,7 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface { protected $io; protected $config; - protected $rfs; + protected $httpDownloader; protected $filesystem; protected $cache; protected $outputProgress = true; @@ -52,16 +52,16 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface * @param IOInterface $io The IO instance * @param Config $config The config * @param EventDispatcher $eventDispatcher The event dispatcher - * @param Cache $cache Optional cache instance - * @param RemoteFilesystem $rfs The remote filesystem + * @param Cache $cache Cache instance + * @param HttpDownloader $httpDownloader The remote filesystem * @param Filesystem $filesystem The filesystem */ - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, RemoteFilesystem $rfs = null, Filesystem $filesystem = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher, Cache $cache, HttpDownloader $httpDownloader, Filesystem $filesystem = null) { $this->io = $io; $this->config = $config; $this->eventDispatcher = $eventDispatcher; - $this->rfs = $rfs ?: Factory::createRemoteFilesystem($this->io, $config); + $this->httpDownloader = $httpDownloader; $this->filesystem = $filesystem ?: new Filesystem(); $this->cache = $cache; @@ -125,13 +125,12 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface $fileName = $this->getFileName($package, $path); $processedUrl = $this->processUrl($package, $url); - $hostname = parse_url($processedUrl, PHP_URL_HOST); - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->rfs, $processedUrl); + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $processedUrl); if ($this->eventDispatcher) { $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); } - $rfs = $preFileDownloadEvent->getRemoteFilesystem(); + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); try { $checksum = $package->getDistSha1Checksum(); @@ -150,7 +149,8 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface $retries = 3; while ($retries--) { try { - $rfs->copy($hostname, $processedUrl, $fileName, $this->outputProgress, $package->getTransportOptions()); + // TODO handle this->outputProgress + $httpDownloader->copy($processedUrl, $fileName, $package->getTransportOptions()); break; } catch (TransportException $e) { // if we got an http response with a proper code, then requesting again will probably not help, abort diff --git a/src/Composer/Downloader/GzipDownloader.php b/src/Composer/Downloader/GzipDownloader.php index 19e4a45e1..bb86f6267 100644 --- a/src/Composer/Downloader/GzipDownloader.php +++ b/src/Composer/Downloader/GzipDownloader.php @@ -18,7 +18,7 @@ use Composer\EventDispatcher\EventDispatcher; use Composer\Package\PackageInterface; use Composer\Util\Platform; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; /** @@ -30,10 +30,10 @@ class GzipDownloader extends ArchiveDownloader { protected $process; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $rfs); + parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); } protected function extract($file, $path) diff --git a/src/Composer/Downloader/RarDownloader.php b/src/Composer/Downloader/RarDownloader.php index 40cd09896..c1ed8d60b 100644 --- a/src/Composer/Downloader/RarDownloader.php +++ b/src/Composer/Downloader/RarDownloader.php @@ -18,7 +18,7 @@ use Composer\EventDispatcher\EventDispatcher; use Composer\Util\IniHelper; use Composer\Util\Platform; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; use RarArchive; @@ -33,10 +33,10 @@ class RarDownloader extends ArchiveDownloader { protected $process; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $rfs); + parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); } protected function extract($file, $path) diff --git a/src/Composer/Downloader/XzDownloader.php b/src/Composer/Downloader/XzDownloader.php index 4a9b854d3..1f5947997 100644 --- a/src/Composer/Downloader/XzDownloader.php +++ b/src/Composer/Downloader/XzDownloader.php @@ -17,7 +17,7 @@ use Composer\Cache; use Composer\EventDispatcher\EventDispatcher; use Composer\Package\PackageInterface; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; /** @@ -30,11 +30,11 @@ class XzDownloader extends ArchiveDownloader { protected $process; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $rfs); + parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); } protected function extract($file, $path) diff --git a/src/Composer/Downloader/ZipDownloader.php b/src/Composer/Downloader/ZipDownloader.php index 6534db3d8..10518e026 100644 --- a/src/Composer/Downloader/ZipDownloader.php +++ b/src/Composer/Downloader/ZipDownloader.php @@ -19,7 +19,7 @@ use Composer\Package\PackageInterface; use Composer\Util\IniHelper; use Composer\Util\Platform; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; use Symfony\Component\Process\ExecutableFinder; use ZipArchive; @@ -36,10 +36,10 @@ class ZipDownloader extends ArchiveDownloader protected $process; private $zipArchiveObject; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $rfs); + parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); } /** diff --git a/src/Composer/Factory.php b/src/Composer/Factory.php index 1aac934a1..97d507b10 100644 --- a/src/Composer/Factory.php +++ b/src/Composer/Factory.php @@ -23,7 +23,7 @@ use Composer\Repository\WritableRepositoryInterface; use Composer\Util\Filesystem; use Composer\Util\Platform; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Util\Silencer; use Composer\Plugin\PluginEvents; use Composer\EventDispatcher\Event; @@ -325,7 +325,7 @@ class Factory $io->loadConfiguration($config); } - $rfs = self::createRemoteFilesystem($io, $config); + $rfs = self::createHttpDownloader($io, $config); // initialize event dispatcher $dispatcher = new EventDispatcher($composer, $io); @@ -451,7 +451,7 @@ class Factory * @param EventDispatcher $eventDispatcher * @return Downloader\DownloadManager */ - public function createDownloadManager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null) + public function createDownloadManager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null) { $cache = null; if ($config->get('cache-files-ttl') > 0) { @@ -579,10 +579,10 @@ class Factory /** * @param IOInterface $io IO instance * @param Config $config Config instance - * @param array $options Array of options passed directly to RemoteFilesystem constructor - * @return RemoteFilesystem + * @param array $options Array of options passed directly to HttpDownloader constructor + * @return HttpDownloader */ - public static function createRemoteFilesystem(IOInterface $io, Config $config = null, $options = array()) + public static function createHttpDownloader(IOInterface $io, Config $config = null, $options = array()) { static $warned = false; $disableTls = false; @@ -607,7 +607,7 @@ class Factory $remoteFilesystemOptions = array_replace_recursive($remoteFilesystemOptions, $options); } try { - $remoteFilesystem = new RemoteFilesystem($io, $config, $remoteFilesystemOptions, $disableTls); + $remoteFilesystem = new HttpDownloader($io, $config, $remoteFilesystemOptions, $disableTls); } catch (TransportException $e) { if (false !== strpos($e->getMessage(), 'cafile')) { $io->write('Unable to locate a valid CA certificate file. You must set a valid \'cafile\' option.'); diff --git a/src/Composer/Plugin/PreFileDownloadEvent.php b/src/Composer/Plugin/PreFileDownloadEvent.php index 7ae6821ce..076449484 100644 --- a/src/Composer/Plugin/PreFileDownloadEvent.php +++ b/src/Composer/Plugin/PreFileDownloadEvent.php @@ -13,7 +13,7 @@ namespace Composer\Plugin; use Composer\EventDispatcher\Event; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; /** * The pre file download event. @@ -23,7 +23,7 @@ use Composer\Util\RemoteFilesystem; class PreFileDownloadEvent extends Event { /** - * @var RemoteFilesystem + * @var HttpDownloader */ private $rfs; @@ -36,10 +36,10 @@ class PreFileDownloadEvent extends Event * Constructor. * * @param string $name The event name - * @param RemoteFilesystem $rfs + * @param HttpDownloader $rfs * @param string $processedUrl */ - public function __construct($name, RemoteFilesystem $rfs, $processedUrl) + public function __construct($name, HttpDownloader $rfs, $processedUrl) { parent::__construct($name); $this->rfs = $rfs; @@ -47,21 +47,17 @@ class PreFileDownloadEvent extends Event } /** - * Returns the remote filesystem - * - * @return RemoteFilesystem + * @return HttpDownloader */ - public function getRemoteFilesystem() + public function getHttpDownloader() { return $this->rfs; } /** - * Sets the remote filesystem - * - * @param RemoteFilesystem $rfs + * @param HttpDownloader $rfs */ - public function setRemoteFilesystem(RemoteFilesystem $rfs) + public function setHttpDownloader(HttpDownloader $rfs) { $this->rfs = $rfs; } diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 09e2179d8..c4a570c75 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -21,7 +21,7 @@ use Composer\Cache; use Composer\Config; use Composer\Factory; use Composer\IO\IOInterface; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Plugin\PluginEvents; use Composer\Plugin\PreFileDownloadEvent; use Composer\EventDispatcher\EventDispatcher; @@ -40,7 +40,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito protected $url; protected $baseUrl; protected $io; - protected $rfs; + protected $httpDownloader; protected $cache; protected $notifyUrl; protected $searchUrl; @@ -60,8 +60,9 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito private $rootData; private $hasPartialPackages; private $partialPackagesByName; + private $versionParser; - public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null) + public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $eventDispatcher, HttpDownloader $httpDownloader) { parent::__construct(); if (!preg_match('{^[\w.]+\??://}', $repoConfig['url'])) { @@ -98,12 +99,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->baseUrl = rtrim(preg_replace('{(?:/[^/\\\\]+\.json)?(?:[?#].*)?$}', '', $this->url), '/'); $this->io = $io; $this->cache = new Cache($io, $config->get('cache-repo-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', $this->url), 'a-z0-9.$'); - $this->loader = new ArrayLoader(); - if ($rfs && $this->options) { - $rfs = clone $rfs; - $rfs->setOptions($this->options); + $this->versionParser = new VersionParser(); + $this->loader = new ArrayLoader($this->versionParser); + if ($httpDownloader && $this->options) { + // TODO solve this somehow - should be sent a request time not on the instance + $httpDownloader = clone $httpDownloader; + $httpDownloader->setOptions($this->options); } - $this->rfs = $rfs ?: Factory::createRemoteFilesystem($this->io, $this->config, $this->options); + $this->httpDownloader = $httpDownloader; $this->eventDispatcher = $eventDispatcher; $this->repoConfig = $repoConfig; } @@ -129,8 +132,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $name = strtolower($name); if (!$constraint instanceof ConstraintInterface) { - $versionParser = new VersionParser(); - $constraint = $versionParser->parseConstraints($constraint); + $constraint = $this->versionParser->parseConstraints($constraint); } foreach ($this->getProviderNames() as $providerName) { @@ -161,8 +163,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $name = strtolower($name); if (null !== $constraint && !$constraint instanceof ConstraintInterface) { - $versionParser = new VersionParser(); - $constraint = $versionParser->parseConstraints($constraint); + $constraint = $this->versionParser->parseConstraints($constraint); } $packages = array(); @@ -196,8 +197,10 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito public function loadPackages(array $packageNameMap, $isPackageAcceptableCallable) { + if ($this->lazyProvidersUrl) { + return $this->loadAsyncPackages($packageNameMap, $isPackageAcceptableCallable); + } if (!$this->hasProviders()) { - // TODO build more efficient version of this return parent::loadPackages($packageNameMap, $isPackageAcceptableCallable); } @@ -235,9 +238,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito if ($this->searchUrl && $mode === self::SEARCH_FULLTEXT) { $url = str_replace(array('%query%', '%type%'), array($query, $type), $this->searchUrl); - $hostname = parse_url($url, PHP_URL_HOST) ?: $url; - $json = $this->rfs->getContents($hostname, $url, false); - $search = JsonFile::parseJson($json, $url); + $search = $this->httpDownloader->get($url)->decodeJson(); if (empty($search['results'])) { return array(); @@ -496,6 +497,85 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->configurePackageTransportOptions($package); } + private function loadAsyncPackages(array $packageNames, $isPackageAcceptableCallable) + { + $this->loadRootServerFile(); + + $packages = array(); + $repo = $this; + + $createPackageIfAcceptable = function ($version, $constraint) use (&$packages, $isPackageAcceptableCallable, $repo) { + if (!call_user_func($isPackageAcceptableCallable, strtolower($version['name']), VersionParser::parseStability($version['version']))) { + return; + } + + if (isset($version['version_normalized']) && $constraint && !$constraint->matches(new Constraint('==', $version['version_normalized']))) { + return; + } + + // load acceptable packages in the providers + $package = $this->createPackage($version, 'Composer\Package\CompletePackage'); + $package->setRepository($repo); + + // if there was no version_normalized, then we need to check now for the constraint + if (!$constraint || isset($version['version_normalized']) || $constraint->matches(new Constraint('==', $package->getVersion()))) { + $packages[spl_object_hash($package)] = $package; + if ($package instanceof AliasPackage && !isset($packages[spl_object_hash($package->getAliasOf())])) { + $packages[spl_object_hash($package->getAliasOf())] = $package->getAliasOf(); + } + } + }; + + if ($this->lazyProvidersUrl) { + foreach ($packageNames as $name => $constraint) { + $url = str_replace('%package%', $name, $this->lazyProvidersUrl); + $cacheKey = 'provider-'.strtr($name, '/', '$').'.json'; + + $lastModified = null; + if ($contents = $this->cache->read($cacheKey)) { + $contents = json_decode($contents, true); + $lastModified = isset($contents['last-modified']) ? $contents['last-modified'] : null; + } + + $this->asyncFetchFile($url, $cacheKey, $lastModified) + ->then(function ($response) use (&$packages, $contents, $name, $constraint, $createPackageIfAcceptable) { + if (true === $response) { + $response = $contents; + } + + $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); + foreach ($response['packages'][$name] as $version) { + if (isset($version['versions'])) { + $baseVersion = $version; + foreach ($uniqKeys as $key) { + unset($baseVersion[$key.'s']); + } + + foreach ($version['versions'] as $index => $dummy) { + $unpackedVersion = $baseVersion; + foreach ($uniqKeys as $key) { + $unpackedVersion[$key] = $version[$key.'s'][$index]; + } + + $createPackageIfAcceptable($unpackedVersion, $constraint); + } + } else { + $createPackageIfAcceptable($version, $constraint); + } + } + }, function ($e) { + // TODO use ->done() above instead with react/promise 2.0 + var_dump('Uncaught Ex', $e->getMessage()); + }); + } + } + + $this->httpDownloader->wait(); + + return $packages; + // RepositorySet should call loadMetadata, getMetadata when all promises resolved, then metadataComplete when done so we can GC the loaded json and whatnot then as needed + } + protected function loadRootServerFile() { if (null !== $this->rootData) { @@ -691,15 +771,13 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $retries = 3; while ($retries--) { try { - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->rfs, $filename); - if ($this->eventDispatcher) { - $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - } + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); + $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - $hostname = parse_url($filename, PHP_URL_HOST) ?: $filename; - $rfs = $preFileDownloadEvent->getRemoteFilesystem(); + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); - $json = $rfs->getContents($hostname, $filename, false); + $response = $httpDownloader->get($filename); + $json = $response->getBody(); if ($sha256 && $sha256 !== hash('sha256', $json)) { // undo downgrade before trying again if http seems to be hijacked or modifying content somehow if ($this->allowSslDowngrade) { @@ -718,7 +796,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito throw new RepositorySecurityException('The contents of '.$filename.' do not match its signature. This could indicate a man-in-the-middle attack or e.g. antivirus software corrupting files. Try running composer again and report this if you think it is a mistake.'); } - $data = JsonFile::parseJson($json, $filename); + $data = $response->decodeJson(); if (!empty($data['warning'])) { $this->io->writeError('Warning from '.$this->url.': '.$data['warning'].''); } @@ -728,7 +806,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito if ($cacheKey) { if ($storeLastModifiedTime) { - $lastModifiedDate = $rfs->findHeaderValue($rfs->getLastHeaders(), 'last-modified'); + $lastModifiedDate = $response->getHeader('last-modified'); if ($lastModifiedDate) { $data['last-modified'] = $lastModifiedDate; $json = json_encode($data); @@ -737,8 +815,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->cache->write($cacheKey, $json); } + $response->collect(); + break; } catch (\Exception $e) { + if ($e instanceof \LogicException) { + throw $e; + } + if ($e instanceof TransportException && $e->getStatusCode() === 404) { throw $e; } @@ -775,20 +859,18 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $retries = 3; while ($retries--) { try { - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->rfs, $filename); - if ($this->eventDispatcher) { - $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - } + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); + $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - $hostname = parse_url($filename, PHP_URL_HOST) ?: $filename; - $rfs = $preFileDownloadEvent->getRemoteFilesystem(); + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); $options = array('http' => array('header' => array('If-Modified-Since: '.$lastModifiedTime))); - $json = $rfs->getContents($hostname, $filename, false, $options); - if ($json === '' && $rfs->findStatusCode($rfs->getLastHeaders()) === 304) { + $response = $httpDownloader->get($filename, $options); + $json = $response->getBody(); + if ($json === '' && $response->getStatusCode() === 304) { return true; } - $data = JsonFile::parseJson($json, $filename); + $data = $response->decodeJson(); if (!empty($data['warning'])) { $this->io->writeError('Warning from '.$this->url.': '.$data['warning'].''); } @@ -796,7 +878,8 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->io->writeError('Info from '.$this->url.': '.$data['info'].''); } - $lastModifiedDate = $rfs->findHeaderValue($rfs->getLastHeaders(), 'last-modified'); + $lastModifiedDate = $response->getHeader('last-modified'); + $response->collect(); if ($lastModifiedDate) { $data['last-modified'] = $lastModifiedDate; $json = json_encode($data); @@ -805,6 +888,10 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $data; } catch (\Exception $e) { + if ($e instanceof \LogicException) { + throw $e; + } + if ($e instanceof TransportException && $e->getStatusCode() === 404) { throw $e; } @@ -825,6 +912,69 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } } + protected function asyncFetchFile($filename, $cacheKey, $lastModifiedTime = null) + { + $retries = 3; + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); + $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); + + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); + $options = $lastModifiedTime ? array('http' => array('header' => array('If-Modified-Since: '.$lastModifiedTime))) : array(); + + $io = $this->io; + $url = $this->url; + $cache = $this->cache; + $degradedMode =& $this->degradedMode; + + $accept = function ($response) use ($io, $url, $cache, $cacheKey) { + $json = $response->getBody(); + if ($json === '' && $response->getStatusCode() === 304) { + return true; + } + + $data = $response->decodeJson(); + if (!empty($data['warning'])) { + $io->writeError('Warning from '.$url.': '.$data['warning'].''); + } + if (!empty($data['info'])) { + $io->writeError('Info from '.$url.': '.$data['info'].''); + } + + $lastModifiedDate = $response->getHeader('last-modified'); + $response->collect(); + if ($lastModifiedDate) { + $data['last-modified'] = $lastModifiedDate; + $json = JsonFile::encode($data, JsonFile::JSON_UNESCAPED_SLASHES | JsonFile::JSON_UNESCAPED_UNICODE); + } + $cache->write($cacheKey, $json); + + return $data; + }; + + $reject = function ($e) use (&$retries, $httpDownloader, $filename, $options, &$reject, $accept, $io, $url, $cache, &$degradedMode) { + var_dump('Caught8', $e->getMessage()); + if ($e instanceof TransportException && $e->getStatusCode() === 404) { + return false; + } + + if (--$retries) { + usleep(100000); + + return $httpDownloader->add($filename, $options)->then($accept, $reject); + } + + if (!$degradedMode) { + $io->writeError(''.$e->getMessage().''); + $io->writeError(''.$url.' could not be fully loaded, package information was loaded from the local cache and may be out of date'); + } + $degradedMode = true; + + return true; + }; + + return $httpDownloader->add($filename, $options)->then($accept, $reject); + } + /** * This initializes the packages key of a partial packages.json that contain some packages inlined + a providers-lazy-url * diff --git a/src/Composer/Repository/RepositoryFactory.php b/src/Composer/Repository/RepositoryFactory.php index ca479a7fd..5737a5359 100644 --- a/src/Composer/Repository/RepositoryFactory.php +++ b/src/Composer/Repository/RepositoryFactory.php @@ -16,7 +16,7 @@ use Composer\Factory; use Composer\IO\IOInterface; use Composer\Config; use Composer\EventDispatcher\EventDispatcher; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Json\JsonFile; /** @@ -108,10 +108,10 @@ class RepositoryFactory * @param IOInterface $io * @param Config $config * @param EventDispatcher $eventDispatcher - * @param RemoteFilesystem $rfs + * @param HttpDownloader $rfs * @return RepositoryManager */ - public static function manager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null) + public static function manager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null) { $rm = new RepositoryManager($io, $config, $eventDispatcher, $rfs); $rm->setRepositoryClass('composer', 'Composer\Repository\ComposerRepository'); diff --git a/src/Composer/Repository/RepositoryManager.php b/src/Composer/Repository/RepositoryManager.php index 87b82d14d..64568514c 100644 --- a/src/Composer/Repository/RepositoryManager.php +++ b/src/Composer/Repository/RepositoryManager.php @@ -16,7 +16,7 @@ use Composer\IO\IOInterface; use Composer\Config; use Composer\EventDispatcher\EventDispatcher; use Composer\Package\PackageInterface; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; /** * Repositories manager. @@ -35,7 +35,7 @@ class RepositoryManager private $eventDispatcher; private $rfs; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null) { $this->io = $io; $this->config = $config; @@ -127,7 +127,7 @@ class RepositoryManager $reflMethod = new \ReflectionMethod($class, '__construct'); $params = $reflMethod->getParameters(); - if (isset($params[4]) && $params[4]->getClass() && $params[4]->getClass()->getName() === 'Composer\Util\RemoteFilesystem') { + if (isset($params[4]) && $params[4]->getClass() && $params[4]->getClass()->getName() === 'Composer\Util\HttpDownloader') { return new $class($config, $this->io, $this->config, $this->eventDispatcher, $this->rfs); } diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php new file mode 100644 index 000000000..846c41883 --- /dev/null +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -0,0 +1,282 @@ + + * Jordi Boggiano + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Composer\Util\Http; + +use Composer\Config; +use Composer\IO\IOInterface; +use Composer\Downloader\TransportException; +use Composer\CaBundle\CaBundle; +use Psr\Log\LoggerInterface; +use React\Promise\Promise; + +/** + * @author Jordi Boggiano + * @author Nicolas Grekas + */ +class CurlDownloader +{ + private $multiHandle; + private $shareHandle; + private $jobs = array(); + private $io; + private $selectTimeout = 5.0; + protected $multiErrors = array( + CURLM_BAD_HANDLE => array('CURLM_BAD_HANDLE', 'The passed-in handle is not a valid CURLM handle.'), + CURLM_BAD_EASY_HANDLE => array('CURLM_BAD_EASY_HANDLE', "An easy handle was not good/valid. It could mean that it isn't an easy handle at all, or possibly that the handle already is in used by this or another multi handle."), + CURLM_OUT_OF_MEMORY => array('CURLM_OUT_OF_MEMORY', 'You are doomed.'), + CURLM_INTERNAL_ERROR => array('CURLM_INTERNAL_ERROR', 'This can only be returned if libcurl bugs. Please report it to us!') + ); + + private static $options = array( + 'http' => array( + 'method' => CURLOPT_CUSTOMREQUEST, + 'content' => CURLOPT_POSTFIELDS, + 'proxy' => CURLOPT_PROXY, + ), + 'ssl' => array( + 'ciphers' => CURLOPT_SSL_CIPHER_LIST, + 'cafile' => CURLOPT_CAINFO, + 'capath' => CURLOPT_CAPATH, + ), + ); + + private static $timeInfo = array( + 'total_time' => true, + 'namelookup_time' => true, + 'connect_time' => true, + 'pretransfer_time' => true, + 'starttransfer_time' => true, + 'redirect_time' => true, + ); + + public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false) + { + $this->io = $io; + + $this->multiHandle = $mh = curl_multi_init(); + if (function_exists('curl_multi_setopt')) { + curl_multi_setopt($mh, CURLMOPT_PIPELINING, /*CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX*/ 3); + if (defined('CURLMOPT_MAX_HOST_CONNECTIONS')) { + curl_multi_setopt($mh, CURLMOPT_MAX_HOST_CONNECTIONS, 8); + } + } + + if (function_exists('curl_share_init')) { + $this->shareHandle = $sh = curl_share_init(); + curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); + curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); + curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); + } + } + + public function download($resolve, $reject, $origin, $url, $options, $copyTo = null) + { + $ch = curl_init(); + $hd = fopen('php://temp/maxmemory:32768', 'w+b'); + + // TODO auth & other context + // TODO cleanup + + if ($copyTo && !$fd = @fopen($copyTo.'~', 'w+b')) { + // TODO throw here probably? + $copyTo = null; + } + if (!$copyTo) { + $fd = @fopen('php://temp/maxmemory:524288', 'w+b'); + } + + if (!isset($options['http']['header'])) { + $options['http']['header'] = array(); + } + + $headers = array_diff($options['http']['header'], array('Connection: close')); + + // TODO + $degradedMode = false; + if ($degradedMode) { + curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); + } else { + $headers[] = 'Connection: keep-alive'; + $version = curl_version(); + $features = $version['features']; + if (0 === strpos($url, 'https://') && \defined('CURL_VERSION_HTTP2') && \defined('CURL_HTTP_VERSION_2_0') && (CURL_VERSION_HTTP2 & $features)) { + curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + } + } + + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + //curl_setopt($ch, CURLOPT_DNS_USE_GLOBAL_CACHE, false); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + curl_setopt($ch, CURLOPT_TIMEOUT, 10); // TODO increase + curl_setopt($ch, CURLOPT_WRITEHEADER, $hd); + curl_setopt($ch, CURLOPT_FILE, $fd); + if (function_exists('curl_share_init')) { + curl_setopt($ch, CURLOPT_SHARE, $this->shareHandle); + } + + foreach (self::$options as $type => $curlOptions) { + foreach ($curlOptions as $name => $curlOption) { + if (isset($options[$type][$name])) { + curl_setopt($ch, $curlOption, $options[$type][$name]); + } + } + } + + $progress = array_diff_key(curl_getinfo($ch), self::$timeInfo); + + $this->jobs[(int) $ch] = array( + 'progress' => $progress, + 'ch' => $ch, + //'callback' => $params['notification'], + 'file' => $copyTo, + 'hd' => $hd, + 'fd' => $fd, + 'resolve' => $resolve, + 'reject' => $reject, + ); + + $this->io->write('Downloading '.$url, true, IOInterface::DEBUG); + + $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $ch)); + //$params['notification'](STREAM_NOTIFY_RESOLVE, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, 0, false); + } + + public function tick() + { + // TODO check we have active handles before doing this + if (!$this->jobs) { + return; + } + + $active = true; + try { + $this->checkCurlResult(curl_multi_exec($this->multiHandle, $active)); + if (-1 === curl_multi_select($this->multiHandle, $this->selectTimeout)) { + // sleep in case select returns -1 as it can happen on old php versions or some platforms where curl does not manage to do the select + usleep(150); + } + + while ($progress = curl_multi_info_read($this->multiHandle)) { + $h = $progress['handle']; + $i = (int) $h; + if (!isset($this->jobs[$i])) { + continue; + } + $progress = array_diff_key(curl_getinfo($h), self::$timeInfo); + $job = $this->jobs[$i]; + unset($this->jobs[$i]); + curl_multi_remove_handle($this->multiHandle, $h); + $error = curl_error($h); + $errno = curl_errno($h); + curl_close($h); + + try { + //$this->onProgress($h, $job['callback'], $progress, $job['progress']); + if ('' !== $error) { + throw new TransportException(curl_error($h)); + } + + if ($job['file']) { + if (CURLE_OK === $errno) { + fclose($job['fd']); + rename($job['file'].'~', $job['file']); + call_user_func($job['resolve'], true); + } + // TODO otherwise show error? + } else { + rewind($job['hd']); + $headers = explode("\r\n", rtrim(stream_get_contents($job['hd']))); + fclose($job['hd']); + rewind($job['fd']); + $contents = stream_get_contents($job['fd']); + fclose($job['fd']); + $this->io->writeError('['.$progress['http_code'].'] '.$progress['url'], true, IOInterface::DEBUG); + call_user_func($job['resolve'], new Response(array('url' => $progress['url']), $progress['http_code'], $headers, $contents)); + } + } catch (TransportException $e) { + fclose($job['hd']); + fclose($job['fd']); + if ($job['file']) { + @unlink($job['file'].'~'); + } + call_user_func($job['reject'], $e); + } + } + + foreach ($this->jobs as $i => $h) { + if (!isset($this->jobs[$i])) { + continue; + } + $h = $this->jobs[$i]['ch']; + $progress = array_diff_key(curl_getinfo($h), self::$timeInfo); + + if ($this->jobs[$i]['progress'] !== $progress) { + $previousProgress = $this->jobs[$i]['progress']; + $this->jobs[$i]['progress'] = $progress; + try { + //$this->onProgress($h, $this->jobs[$i]['callback'], $progress, $previousProgress); + } catch (TransportException $e) { + var_dump('Caught '.$e->getMessage());die; + unset($this->jobs[$i]); + curl_multi_remove_handle($this->multiHandle, $h); + curl_close($h); + + fclose($job['hd']); + fclose($job['fd']); + if ($job['file']) { + @unlink($job['file'].'~'); + } + call_user_func($job['reject'], $e); + } + } + } + } catch (\Exception $e) { + var_dump('Caught2', get_class($e), $e->getMessage(), $e);die; + } + +// TODO finalize / resolve +// if ($copyTo && !isset($this->exceptions[(int) $ch])) { +// $fd = fopen($copyTo, 'rb'); +// } +// + } + + private function onProgress($ch, callable $notify, array $progress, array $previousProgress) + { + if (300 <= $progress['http_code'] && $progress['http_code'] < 400) { + return; + } + if (!$previousProgress['http_code'] && $progress['http_code'] && $progress['http_code'] < 200 || 400 <= $progress['http_code']) { + $code = 403 === $progress['http_code'] ? STREAM_NOTIFY_AUTH_RESULT : STREAM_NOTIFY_FAILURE; + $notify($code, STREAM_NOTIFY_SEVERITY_ERR, curl_error($ch), $progress['http_code'], 0, 0, false); + } + if ($previousProgress['download_content_length'] < $progress['download_content_length']) { + $notify(STREAM_NOTIFY_FILE_SIZE_IS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, (int) $progress['download_content_length'], false); + } + if ($previousProgress['size_download'] < $progress['size_download']) { + $notify(STREAM_NOTIFY_PROGRESS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, (int) $progress['size_download'], (int) $progress['download_content_length'], false); + } + } + + private function checkCurlResult($code) + { + if ($code != CURLM_OK && $code != CURLM_CALL_MULTI_PERFORM) { + throw new \RuntimeException(isset($this->multiErrors[$code]) + ? "cURL error: {$code} ({$this->multiErrors[$code][0]}): cURL message: {$this->multiErrors[$code][1]}" + : 'Unexpected cURL error: ' . $code + ); + } + } +} diff --git a/src/Composer/Util/Http/Response.php b/src/Composer/Util/Http/Response.php new file mode 100644 index 000000000..ff48fdb40 --- /dev/null +++ b/src/Composer/Util/Http/Response.php @@ -0,0 +1,75 @@ + + * Jordi Boggiano + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Composer\Util\Http; + +use Composer\Json\JsonFile; + +class Response +{ + private $request; + private $code; + private $headers; + private $body; + + public function __construct(array $request, $code, array $headers, $body) + { + $this->request = $request; + $this->code = $code; + $this->headers = $headers; + $this->body = $body; + } + + public function getStatusCode() + { + return $this->code; + } + + public function getHeaders() + { + return $this->headers; + } + + public function getHeader($name) + { + $value = null; + foreach ($this->headers as $header) { + if (preg_match('{^'.$name.':\s*(.+?)\s*$}i', $header, $match)) { + $value = $match[1]; + } elseif (preg_match('{^HTTP/}i', $header)) { + // TODO ideally redirects would be handled in CurlDownloader/RemoteFilesystem and this becomes unnecessary + // + // In case of redirects, http_response_headers contains the headers of all responses + // so we reset the flag when a new response is being parsed as we are only interested in the last response + $value = null; + } + } + + return $value; + } + + + public function getBody() + { + return $this->body; + } + + public function decodeJson() + { + return JsonFile::parseJson($this->body, $this->request['url']); + } + + public function collect() + { + $this->request = $this->code = $this->headers = $this->body = null; + } +} diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php new file mode 100644 index 000000000..31c615e0c --- /dev/null +++ b/src/Composer/Util/HttpDownloader.php @@ -0,0 +1,246 @@ + + * Jordi Boggiano + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Composer\Util; + +use Composer\Config; +use Composer\IO\IOInterface; +use Composer\Downloader\TransportException; +use Composer\CaBundle\CaBundle; +use Psr\Log\LoggerInterface; +use React\Promise\Promise; + +/** + * @author Jordi Boggiano + */ +class HttpDownloader +{ + const STATUS_QUEUED = 1; + const STATUS_STARTED = 2; + const STATUS_COMPLETED = 3; + const STATUS_FAILED = 4; + + private $io; + private $config; + private $jobs = array(); + private $index; + private $progress; + private $lastProgress; + private $disableTls = false; + private $curl; + private $rfs; + private $idGen = 0; + + /** + * Constructor. + * + * @param IOInterface $io The IO instance + * @param Config $config The config + * @param array $options The options + * @param bool $disableTls + */ + public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false) + { + $this->io = $io; + + // Setup TLS options + // The cafile option can be set via config.json + if ($disableTls === false) { + $logger = $io instanceof LoggerInterface ? $io : null; + $this->options = StreamContextFactory::getTlsDefaults($options, $logger); + } else { + $this->disableTls = true; + } + + // handle the other externally set options normally. + $this->options = array_replace_recursive($this->options, $options); + $this->config = $config; + + if (extension_loaded('curl')) { + $this->curl = new Http\CurlDownloader($io, $config, $options, $disableTls); + } + + $this->rfs = new RemoteFilesystem($io, $config, $options, $disableTls); + } + + public function get($url, $options = array()) + { + list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => false), true); + $this->wait($job['id']); + + return $this->getResponse($job['id']); + } + + public function add($url, $options = array()) + { + list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => false)); + + return $promise; + } + + public function copy($url, $to, $options = array()) + { + list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => $to), true); + $this->wait($job['id']); + + return $this->getResponse($job['id']); + } + + public function addCopy($url, $to, $options = array()) + { + list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => $to)); + + return $promise; + } + + private function addJob($request, $sync = false) + { + $job = array( + 'id' => $this->idGen++, + 'status' => self::STATUS_QUEUED, + 'request' => $request, + 'sync' => $sync, + ); + + $curl = $this->curl; + $rfs = $this->rfs; + $io = $this->io; + + $origin = $this->getOrigin($job['request']['url']); + + // TODO only send http/https through curl + if ($curl) { + $resolver = function ($resolve, $reject) use (&$job, $curl, $origin) { + // start job + $url = $job['request']['url']; + $options = $job['request']['options']; + + $job['status'] = HttpDownloader::STATUS_STARTED; + + if ($job['request']['copyTo']) { + $curl->download($resolve, $reject, $origin, $url, $options, $job['request']['copyTo']); + } else { + $curl->download($resolve, $reject, $origin, $url, $options); + } + }; + } else { + $resolver = function ($resolve, $reject) use (&$job, $rfs, $curl, $origin) { + // start job + $url = $job['request']['url']; + $options = $job['request']['options']; + + $job['status'] = HttpDownloader::STATUS_STARTED; + + if ($job['request']['copyTo']) { + if ($curl) { + $result = $curl->download($origin, $url, $options, $job['request']['copyTo']); + } else { + $result = $rfs->copy($origin, $url, $job['request']['copyTo'], false /* TODO progress */, $options); + } + + $resolve($result); + } else { + $body = $rfs->getContents($origin, $url, false /* TODO progress */, $options); + $headers = $rfs->getLastHeaders(); + $response = new Http\Response($job['request'], $rfs->findStatusCode($headers), $headers, $body); + + $resolve($response); + } + }; + } + + $canceler = function () {}; + + $promise = new Promise($resolver, $canceler); + $promise->then(function ($response) use (&$job) { + $job['status'] = HttpDownloader::STATUS_COMPLETED; + $job['response'] = $response; + // TODO look for more jobs to start once we throttle to max X jobs + }, function ($e) use ($io, &$job) { + var_dump(__CLASS__ . __LINE__); + var_dump(gettype($e)); + var_dump($e->getMessage()); + die; + $job['status'] = HttpDownloader::STATUS_FAILED; + $job['exception'] = $e; + }); + $this->jobs[$job['id']] =& $job; + + return array($job, $promise); + } + + public function wait($index = null, $progress = false) + { + while (true) { + if ($this->curl) { + $this->curl->tick(); + } + + if (null !== $index) { + if ($this->jobs[$index]['status'] === self::STATUS_COMPLETED || $this->jobs[$index]['status'] === self::STATUS_FAILED) { + return; + } + } else { + $done = true; + foreach ($this->jobs as $job) { + if (!in_array($job['status'], array(self::STATUS_COMPLETED, self::STATUS_FAILED), true)) { + $done = false; + break; + } elseif (!$job['sync']) { + unset($this->jobs[$job['id']]); + } + } + if ($done) { + return; + } + } + + usleep(1000); + } + } + + private function getResponse($index) + { + if (!isset($this->jobs[$index])) { + throw new \LogicException('Invalid request id'); + } + + if ($this->jobs[$index]['status'] === self::STATUS_FAILED) { + throw $this->jobs[$index]['exception']; + } + + if (!isset($this->jobs[$index]['response'])) { + throw new \LogicException('Response not available yet, call wait() first'); + } + + $resp = $this->jobs[$index]['response']; + + unset($this->jobs[$index]); + + return $resp; + } + + private function getOrigin($url) + { + $origin = parse_url($url, PHP_URL_HOST); + + if ($origin === 'api.github.com') { + return 'github.com'; + } + + if ($origin === 'repo.packagist.org') { + return 'packagist.org'; + } + + return $origin ?: $url; + } +} diff --git a/src/Composer/Util/RemoteFilesystem.php b/src/Composer/Util/RemoteFilesystem.php index ea18a9e30..f4a211acb 100644 --- a/src/Composer/Util/RemoteFilesystem.php +++ b/src/Composer/Util/RemoteFilesystem.php @@ -60,7 +60,8 @@ class RemoteFilesystem // Setup TLS options // The cafile option can be set via config.json if ($disableTls === false) { - $this->options = $this->getTlsDefaults($options); + $logger = $io instanceof LoggerInterface ? $io : null; + $this->options = StreamContextFactory::getTlsDefaults($options, $logger); } else { $this->disableTls = true; } @@ -891,111 +892,6 @@ class RemoteFilesystem return false; } - /** - * @param array $options - * - * @return array - */ - private function getTlsDefaults(array $options) - { - $ciphers = implode(':', array( - 'ECDHE-RSA-AES128-GCM-SHA256', - 'ECDHE-ECDSA-AES128-GCM-SHA256', - 'ECDHE-RSA-AES256-GCM-SHA384', - 'ECDHE-ECDSA-AES256-GCM-SHA384', - 'DHE-RSA-AES128-GCM-SHA256', - 'DHE-DSS-AES128-GCM-SHA256', - 'kEDH+AESGCM', - 'ECDHE-RSA-AES128-SHA256', - 'ECDHE-ECDSA-AES128-SHA256', - 'ECDHE-RSA-AES128-SHA', - 'ECDHE-ECDSA-AES128-SHA', - 'ECDHE-RSA-AES256-SHA384', - 'ECDHE-ECDSA-AES256-SHA384', - 'ECDHE-RSA-AES256-SHA', - 'ECDHE-ECDSA-AES256-SHA', - 'DHE-RSA-AES128-SHA256', - 'DHE-RSA-AES128-SHA', - 'DHE-DSS-AES128-SHA256', - 'DHE-RSA-AES256-SHA256', - 'DHE-DSS-AES256-SHA', - 'DHE-RSA-AES256-SHA', - 'AES128-GCM-SHA256', - 'AES256-GCM-SHA384', - 'AES128-SHA256', - 'AES256-SHA256', - 'AES128-SHA', - 'AES256-SHA', - 'AES', - 'CAMELLIA', - 'DES-CBC3-SHA', - '!aNULL', - '!eNULL', - '!EXPORT', - '!DES', - '!RC4', - '!MD5', - '!PSK', - '!aECDH', - '!EDH-DSS-DES-CBC3-SHA', - '!EDH-RSA-DES-CBC3-SHA', - '!KRB5-DES-CBC3-SHA', - )); - - /** - * CN_match and SNI_server_name are only known once a URL is passed. - * They will be set in the getOptionsForUrl() method which receives a URL. - * - * cafile or capath can be overridden by passing in those options to constructor. - */ - $defaults = array( - 'ssl' => array( - 'ciphers' => $ciphers, - 'verify_peer' => true, - 'verify_depth' => 7, - 'SNI_enabled' => true, - 'capture_peer_cert' => true, - ), - ); - - if (isset($options['ssl'])) { - $defaults['ssl'] = array_replace_recursive($defaults['ssl'], $options['ssl']); - } - - $caBundleLogger = $this->io instanceof LoggerInterface ? $this->io : null; - - /** - * Attempt to find a local cafile or throw an exception if none pre-set - * The user may go download one if this occurs. - */ - if (!isset($defaults['ssl']['cafile']) && !isset($defaults['ssl']['capath'])) { - $result = CaBundle::getSystemCaRootBundlePath($caBundleLogger); - - if (is_dir($result)) { - $defaults['ssl']['capath'] = $result; - } else { - $defaults['ssl']['cafile'] = $result; - } - } - - if (isset($defaults['ssl']['cafile']) && (!is_readable($defaults['ssl']['cafile']) || !CaBundle::validateCaFile($defaults['ssl']['cafile'], $caBundleLogger))) { - throw new TransportException('The configured cafile was not valid or could not be read.'); - } - - if (isset($defaults['ssl']['capath']) && (!is_dir($defaults['ssl']['capath']) || !is_readable($defaults['ssl']['capath']))) { - throw new TransportException('The configured capath was not valid or could not be read.'); - } - - /** - * Disable TLS compression to prevent CRIME attacks where supported. - */ - if (PHP_VERSION_ID >= 50413) { - $defaults['ssl']['disable_compression'] = true; - } - - return $defaults; - } - /** * Fetch certificate common name and fingerprint for validation of SAN. * diff --git a/src/Composer/Util/StreamContextFactory.php b/src/Composer/Util/StreamContextFactory.php index 8dfd6624a..72d12115d 100644 --- a/src/Composer/Util/StreamContextFactory.php +++ b/src/Composer/Util/StreamContextFactory.php @@ -13,6 +13,8 @@ namespace Composer\Util; use Composer\Composer; +use Composer\CaBundle\CaBundle; +use Psr\Log\LoggerInterface; /** * Allows the creation of a basic context supporting http proxy @@ -153,6 +155,109 @@ final class StreamContextFactory return stream_context_create($options, $defaultParams); } + /** + * @param array $options + * + * @return array + */ + public static function getTlsDefaults(array $options, LoggerInterface $logger = null) + { + $ciphers = implode(':', array( + 'ECDHE-RSA-AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES256-GCM-SHA384', + 'ECDHE-ECDSA-AES256-GCM-SHA384', + 'DHE-RSA-AES128-GCM-SHA256', + 'DHE-DSS-AES128-GCM-SHA256', + 'kEDH+AESGCM', + 'ECDHE-RSA-AES128-SHA256', + 'ECDHE-ECDSA-AES128-SHA256', + 'ECDHE-RSA-AES128-SHA', + 'ECDHE-ECDSA-AES128-SHA', + 'ECDHE-RSA-AES256-SHA384', + 'ECDHE-ECDSA-AES256-SHA384', + 'ECDHE-RSA-AES256-SHA', + 'ECDHE-ECDSA-AES256-SHA', + 'DHE-RSA-AES128-SHA256', + 'DHE-RSA-AES128-SHA', + 'DHE-DSS-AES128-SHA256', + 'DHE-RSA-AES256-SHA256', + 'DHE-DSS-AES256-SHA', + 'DHE-RSA-AES256-SHA', + 'AES128-GCM-SHA256', + 'AES256-GCM-SHA384', + 'AES128-SHA256', + 'AES256-SHA256', + 'AES128-SHA', + 'AES256-SHA', + 'AES', + 'CAMELLIA', + 'DES-CBC3-SHA', + '!aNULL', + '!eNULL', + '!EXPORT', + '!DES', + '!RC4', + '!MD5', + '!PSK', + '!aECDH', + '!EDH-DSS-DES-CBC3-SHA', + '!EDH-RSA-DES-CBC3-SHA', + '!KRB5-DES-CBC3-SHA', + )); + + /** + * CN_match and SNI_server_name are only known once a URL is passed. + * They will be set in the getOptionsForUrl() method which receives a URL. + * + * cafile or capath can be overridden by passing in those options to constructor. + */ + $defaults = array( + 'ssl' => array( + 'ciphers' => $ciphers, + 'verify_peer' => true, + 'verify_depth' => 7, + 'SNI_enabled' => true, + 'capture_peer_cert' => true, + ), + ); + + if (isset($options['ssl'])) { + $defaults['ssl'] = array_replace_recursive($defaults['ssl'], $options['ssl']); + } + + /** + * Attempt to find a local cafile or throw an exception if none pre-set + * The user may go download one if this occurs. + */ + if (!isset($defaults['ssl']['cafile']) && !isset($defaults['ssl']['capath'])) { + $result = CaBundle::getSystemCaRootBundlePath($logger); + + if (is_dir($result)) { + $defaults['ssl']['capath'] = $result; + } else { + $defaults['ssl']['cafile'] = $result; + } + } + + if (isset($defaults['ssl']['cafile']) && (!is_readable($defaults['ssl']['cafile']) || !CaBundle::validateCaFile($defaults['ssl']['cafile'], $logger))) { + throw new TransportException('The configured cafile was not valid or could not be read.'); + } + + if (isset($defaults['ssl']['capath']) && (!is_dir($defaults['ssl']['capath']) || !is_readable($defaults['ssl']['capath']))) { + throw new TransportException('The configured capath was not valid or could not be read.'); + } + + /** + * Disable TLS compression to prevent CRIME attacks where supported. + */ + if (PHP_VERSION_ID >= 50413) { + $defaults['ssl']['disable_compression'] = true; + } + + return $defaults; + } + /** * A bug in PHP prevents the headers from correctly being sent when a content-type header is present and * NOT at the end of the array From 713bc4de1d506a83f341e033569ded22bff7d228 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Wed, 31 Oct 2018 12:44:54 +0100 Subject: [PATCH 02/34] Minor fixes and updated the rest of the code/tests to use HttpDownloader --- doc/articles/plugins.md | 4 +- src/Composer/Command/ArchiveCommand.php | 2 +- src/Composer/Downloader/FileDownloader.php | 4 +- src/Composer/Downloader/GzipDownloader.php | 4 +- src/Composer/Downloader/RarDownloader.php | 4 +- src/Composer/Downloader/XzDownloader.php | 4 +- src/Composer/Downloader/ZipDownloader.php | 4 +- src/Composer/Factory.php | 32 +++--- src/Composer/Package/Loader/ArrayLoader.php | 3 +- .../Repository/ComposerRepository.php | 17 ++- .../Repository/Pear/BaseChannelReader.php | 16 ++- .../Repository/Pear/ChannelReader.php | 10 +- .../Repository/Pear/ChannelRest10Reader.php | 5 +- .../Repository/Pear/ChannelRest11Reader.php | 6 +- src/Composer/Repository/PearRepository.php | 10 +- src/Composer/Repository/RepositoryFactory.php | 6 +- src/Composer/Repository/RepositoryManager.php | 8 +- .../Repository/Vcs/BitbucketDriver.php | 25 +++-- .../Repository/Vcs/GitBitbucketDriver.php | 2 +- src/Composer/Repository/Vcs/GitHubDriver.php | 53 +++++---- src/Composer/Repository/Vcs/GitLabDriver.php | 55 ++++----- .../Repository/Vcs/HgBitbucketDriver.php | 2 +- src/Composer/Repository/Vcs/VcsDriver.php | 17 +-- src/Composer/Util/Bitbucket.php | 12 +- src/Composer/Util/GitHub.php | 10 +- src/Composer/Util/GitLab.php | 12 +- src/Composer/Util/Http/Response.php | 3 + src/Composer/Util/HttpDownloader.php | 10 +- src/Composer/Util/RemoteFilesystem.php | 4 +- .../Test/Downloader/ArchiveDownloaderTest.php | 6 +- .../Test/Downloader/FileDownloaderTest.php | 6 +- .../Test/Downloader/XzDownloaderTest.php | 4 +- .../Test/Downloader/ZipDownloaderTest.php | 38 +++---- tests/Composer/Test/InstallerTest.php | 5 +- ...esystemMock.php => HttpDownloaderMock.php} | 12 +- .../Package/Archiver/ArchiveManagerTest.php | 9 +- .../Repository/ComposerRepositoryTest.php | 32 ++++-- .../Test/Repository/PathRepositoryTest.php | 2 +- .../Repository/Pear/ChannelReaderTest.php | 10 +- .../Pear/ChannelRest10ReaderTest.php | 4 +- .../Pear/ChannelRest11ReaderTest.php | 4 +- .../Test/Repository/PearRepositoryTest.php | 4 +- .../Test/Repository/RepositoryFactoryTest.php | 4 +- .../Test/Repository/RepositoryManagerTest.php | 6 +- .../Repository/Vcs/GitBitbucketDriverTest.php | 68 ++++++----- .../Test/Repository/Vcs/GitHubDriverTest.php | 85 +++++++------- .../Test/Repository/Vcs/GitLabDriverTest.php | 106 +++++++----------- .../Repository/Vcs/PerforceDriverTest.php | 14 +-- tests/Composer/Test/Util/BitbucketTest.php | 72 ++++++------ tests/Composer/Test/Util/GitHubTest.php | 31 +++-- tests/Composer/Test/Util/GitLabTest.php | 31 +++-- 51 files changed, 461 insertions(+), 436 deletions(-) rename tests/Composer/Test/Mock/{RemoteFilesystemMock.php => HttpDownloaderMock.php} (73%) diff --git a/doc/articles/plugins.md b/doc/articles/plugins.md index 228cbac9e..59e2a2f15 100644 --- a/doc/articles/plugins.md +++ b/doc/articles/plugins.md @@ -176,8 +176,8 @@ class AwsPlugin implements PluginInterface, EventSubscriberInterface if ($protocol === 's3') { $awsClient = new AwsClient($this->io, $this->composer->getConfig()); - $s3RemoteFilesystem = new S3RemoteFilesystem($this->io, $event->getRemoteFilesystem()->getOptions(), $awsClient); - $event->setRemoteFilesystem($s3RemoteFilesystem); + $s3Downloader = new S3Downloader($this->io, $event->getRemoteFilesystem()->getOptions(), $awsClient); + $event->setHttpdownloader($s3Downloader); } } } diff --git a/src/Composer/Command/ArchiveCommand.php b/src/Composer/Command/ArchiveCommand.php index 29858c6fc..f893ed679 100644 --- a/src/Composer/Command/ArchiveCommand.php +++ b/src/Composer/Command/ArchiveCommand.php @@ -104,7 +104,7 @@ EOT $archiveManager = $composer->getArchiveManager(); } else { $factory = new Factory; - $downloadManager = $factory->createDownloadManager($io, $config); + $downloadManager = $factory->createDownloadManager($io, $config, $factory->createHttpDownloader($io, $config)); $archiveManager = $factory->createArchiveManager($config, $downloadManager); } diff --git a/src/Composer/Downloader/FileDownloader.php b/src/Composer/Downloader/FileDownloader.php index 6b1349bb8..8b196e60c 100644 --- a/src/Composer/Downloader/FileDownloader.php +++ b/src/Composer/Downloader/FileDownloader.php @@ -51,12 +51,12 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface * * @param IOInterface $io The IO instance * @param Config $config The config + * @param HttpDownloader $httpDownloader The remote filesystem * @param EventDispatcher $eventDispatcher The event dispatcher * @param Cache $cache Cache instance - * @param HttpDownloader $httpDownloader The remote filesystem * @param Filesystem $filesystem The filesystem */ - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher, Cache $cache, HttpDownloader $httpDownloader, Filesystem $filesystem = null) + public function __construct(IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, Filesystem $filesystem = null) { $this->io = $io; $this->config = $config; diff --git a/src/Composer/Downloader/GzipDownloader.php b/src/Composer/Downloader/GzipDownloader.php index bb86f6267..f65fcf27d 100644 --- a/src/Composer/Downloader/GzipDownloader.php +++ b/src/Composer/Downloader/GzipDownloader.php @@ -30,10 +30,10 @@ class GzipDownloader extends ArchiveDownloader { protected $process; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) + public function __construct(IOInterface $io, Config $config, HttpDownloader $downloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); + parent::__construct($io, $config, $downloader, $eventDispatcher, $cache); } protected function extract($file, $path) diff --git a/src/Composer/Downloader/RarDownloader.php b/src/Composer/Downloader/RarDownloader.php index c1ed8d60b..6fe4cf27c 100644 --- a/src/Composer/Downloader/RarDownloader.php +++ b/src/Composer/Downloader/RarDownloader.php @@ -33,10 +33,10 @@ class RarDownloader extends ArchiveDownloader { protected $process; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) + public function __construct(IOInterface $io, Config $config, HttpDownloader $downloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); + parent::__construct($io, $config, $downloader, $eventDispatcher, $cache); } protected function extract($file, $path) diff --git a/src/Composer/Downloader/XzDownloader.php b/src/Composer/Downloader/XzDownloader.php index 1f5947997..bd7b028e2 100644 --- a/src/Composer/Downloader/XzDownloader.php +++ b/src/Composer/Downloader/XzDownloader.php @@ -30,11 +30,11 @@ class XzDownloader extends ArchiveDownloader { protected $process; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) + public function __construct(IOInterface $io, Config $config, HttpDownloader $downloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); + parent::__construct($io, $config, $downloader, $eventDispatcher, $cache); } protected function extract($file, $path) diff --git a/src/Composer/Downloader/ZipDownloader.php b/src/Composer/Downloader/ZipDownloader.php index 10518e026..9eceab250 100644 --- a/src/Composer/Downloader/ZipDownloader.php +++ b/src/Composer/Downloader/ZipDownloader.php @@ -36,10 +36,10 @@ class ZipDownloader extends ArchiveDownloader protected $process; private $zipArchiveObject; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) + public function __construct(IOInterface $io, Config $config, HttpDownloader $downloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); + parent::__construct($io, $config, $downloader, $eventDispatcher, $cache); } /** diff --git a/src/Composer/Factory.php b/src/Composer/Factory.php index 97d507b10..00aa499d0 100644 --- a/src/Composer/Factory.php +++ b/src/Composer/Factory.php @@ -325,14 +325,14 @@ class Factory $io->loadConfiguration($config); } - $rfs = self::createHttpDownloader($io, $config); + $httpDownloader = self::createHttpDownloader($io, $config); // initialize event dispatcher $dispatcher = new EventDispatcher($composer, $io); $composer->setEventDispatcher($dispatcher); // initialize repository manager - $rm = RepositoryFactory::manager($io, $config, $dispatcher, $rfs); + $rm = RepositoryFactory::manager($io, $config, $httpDownloader, $dispatcher); $composer->setRepositoryManager($rm); // load local repository @@ -357,7 +357,7 @@ class Factory if ($fullLoad) { // initialize download manager - $dm = $this->createDownloadManager($io, $config, $dispatcher, $rfs); + $dm = $this->createDownloadManager($io, $config, $httpDownloader, $dispatcher); $composer->setDownloadManager($dm); // initialize autoload generator @@ -451,7 +451,7 @@ class Factory * @param EventDispatcher $eventDispatcher * @return Downloader\DownloadManager */ - public function createDownloadManager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null) + public function createDownloadManager(IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $eventDispatcher = null) { $cache = null; if ($config->get('cache-files-ttl') > 0) { @@ -484,14 +484,14 @@ class Factory $dm->setDownloader('fossil', new Downloader\FossilDownloader($io, $config, $executor, $fs)); $dm->setDownloader('hg', new Downloader\HgDownloader($io, $config, $executor, $fs)); $dm->setDownloader('perforce', new Downloader\PerforceDownloader($io, $config)); - $dm->setDownloader('zip', new Downloader\ZipDownloader($io, $config, $eventDispatcher, $cache, $executor, $rfs)); - $dm->setDownloader('rar', new Downloader\RarDownloader($io, $config, $eventDispatcher, $cache, $executor, $rfs)); - $dm->setDownloader('tar', new Downloader\TarDownloader($io, $config, $eventDispatcher, $cache, $rfs)); - $dm->setDownloader('gzip', new Downloader\GzipDownloader($io, $config, $eventDispatcher, $cache, $executor, $rfs)); - $dm->setDownloader('xz', new Downloader\XzDownloader($io, $config, $eventDispatcher, $cache, $executor, $rfs)); - $dm->setDownloader('phar', new Downloader\PharDownloader($io, $config, $eventDispatcher, $cache, $rfs)); - $dm->setDownloader('file', new Downloader\FileDownloader($io, $config, $eventDispatcher, $cache, $rfs)); - $dm->setDownloader('path', new Downloader\PathDownloader($io, $config, $eventDispatcher, $cache, $rfs)); + $dm->setDownloader('zip', new Downloader\ZipDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $executor)); + $dm->setDownloader('rar', new Downloader\RarDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $executor)); + $dm->setDownloader('tar', new Downloader\TarDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache)); + $dm->setDownloader('gzip', new Downloader\GzipDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $executor)); + $dm->setDownloader('xz', new Downloader\XzDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $executor)); + $dm->setDownloader('phar', new Downloader\PharDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache)); + $dm->setDownloader('file', new Downloader\FileDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache)); + $dm->setDownloader('path', new Downloader\PathDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache)); return $dm; } @@ -501,14 +501,8 @@ class Factory * @param Downloader\DownloadManager $dm Manager use to download sources * @return Archiver\ArchiveManager */ - public function createArchiveManager(Config $config, Downloader\DownloadManager $dm = null) + public function createArchiveManager(Config $config, Downloader\DownloadManager $dm) { - if (null === $dm) { - $io = new IO\NullIO(); - $io->loadConfiguration($config); - $dm = $this->createDownloadManager($io, $config); - } - $am = new Archiver\ArchiveManager($dm); $am->addArchiver(new Archiver\ZipArchiver); $am->addArchiver(new Archiver\PharArchiver); diff --git a/src/Composer/Package/Loader/ArrayLoader.php b/src/Composer/Package/Loader/ArrayLoader.php index 303cc3c13..49ba45aa8 100644 --- a/src/Composer/Package/Loader/ArrayLoader.php +++ b/src/Composer/Package/Loader/ArrayLoader.php @@ -18,7 +18,6 @@ use Composer\Package\Link; use Composer\Package\RootAliasPackage; use Composer\Package\RootPackageInterface; use Composer\Package\Version\VersionParser; -use Composer\Semver\VersionParser as SemverVersionParser; /** * @author Konstantin Kudryashiv @@ -29,7 +28,7 @@ class ArrayLoader implements LoaderInterface protected $versionParser; protected $loadOptions; - public function __construct(SemverVersionParser $parser = null, $loadOptions = false) + public function __construct(VersionParser $parser = null, $loadOptions = false) { if (!$parser) { $parser = new VersionParser; diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index c4a570c75..a0036e4fd 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -102,7 +102,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->versionParser = new VersionParser(); $this->loader = new ArrayLoader($this->versionParser); if ($httpDownloader && $this->options) { - // TODO solve this somehow - should be sent a request time not on the instance + // TODO solve this somehow - should be sent at request time not on the instance $httpDownloader = clone $httpDownloader; $httpDownloader->setOptions($this->options); } @@ -543,6 +543,10 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $response = $contents; } + if (!isset($response['packages'][$name])) { + return; + } + $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); foreach ($response['packages'][$name] as $version) { if (isset($version['versions'])) { @@ -566,6 +570,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito }, function ($e) { // TODO use ->done() above instead with react/promise 2.0 var_dump('Uncaught Ex', $e->getMessage()); + throw $e; }); } } @@ -644,6 +649,11 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->hasProviders = true; } + // TODO this is for testing only, remove once packagist reports v2 protocol support + if (preg_match('{^https?://repo\.packagist\.org/?$}i', $this->url)) { + $this->repoConfig['force-lazy-providers'] = true; + } + // force values for packagist if (preg_match('{^https?://repo\.packagist\.org/?$}i', $this->url) && !empty($this->repoConfig['force-lazy-providers'])) { $this->url = 'https://repo.packagist.org'; @@ -927,6 +937,11 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $degradedMode =& $this->degradedMode; $accept = function ($response) use ($io, $url, $cache, $cacheKey) { + // package not found is acceptable for a v2 protocol repository + if ($response->getStatusCode() === 404) { + return array('packages' => array()); + } + $json = $response->getBody(); if ($json === '' && $response->getStatusCode() === 304) { return true; diff --git a/src/Composer/Repository/Pear/BaseChannelReader.php b/src/Composer/Repository/Pear/BaseChannelReader.php index 9b26eb9db..b778bf08b 100644 --- a/src/Composer/Repository/Pear/BaseChannelReader.php +++ b/src/Composer/Repository/Pear/BaseChannelReader.php @@ -12,7 +12,7 @@ namespace Composer\Repository\Pear; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; /** * Base PEAR Channel reader. @@ -33,12 +33,12 @@ abstract class BaseChannelReader const ALL_RELEASES_NS = 'http://pear.php.net/dtd/rest.allreleases'; const PACKAGE_INFO_NS = 'http://pear.php.net/dtd/rest.package'; - /** @var RemoteFilesystem */ - private $rfs; + /** @var HttpDownloader */ + private $httpDownloader; - protected function __construct(RemoteFilesystem $rfs) + protected function __construct(HttpDownloader $httpDownloader) { - $this->rfs = $rfs; + $this->httpDownloader = $httpDownloader; } /** @@ -52,7 +52,11 @@ abstract class BaseChannelReader protected function requestContent($origin, $path) { $url = rtrim($origin, '/') . '/' . ltrim($path, '/'); - $content = $this->rfs->getContents($origin, $url, false); + try { + $content = $this->httpDownloader->get($url)->getBody(); + } catch (\Exception $e) { + throw new \UnexpectedValueException('The PEAR channel at ' . $url . ' did not respond.', 0, $e); + } if (!$content) { throw new \UnexpectedValueException('The PEAR channel at ' . $url . ' did not respond.'); } diff --git a/src/Composer/Repository/Pear/ChannelReader.php b/src/Composer/Repository/Pear/ChannelReader.php index 73cc9152e..14d48ad86 100644 --- a/src/Composer/Repository/Pear/ChannelReader.php +++ b/src/Composer/Repository/Pear/ChannelReader.php @@ -12,7 +12,7 @@ namespace Composer\Repository\Pear; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; /** * PEAR Channel package reader. @@ -26,12 +26,12 @@ class ChannelReader extends BaseChannelReader /** @var array of ('xpath test' => 'rest implementation') */ private $readerMap; - public function __construct(RemoteFilesystem $rfs) + public function __construct(HttpDownloader $httpDownloader) { - parent::__construct($rfs); + parent::__construct($httpDownloader); - $rest10reader = new ChannelRest10Reader($rfs); - $rest11reader = new ChannelRest11Reader($rfs); + $rest10reader = new ChannelRest10Reader($httpDownloader); + $rest11reader = new ChannelRest11Reader($httpDownloader); $this->readerMap = array( 'REST1.3' => $rest11reader, diff --git a/src/Composer/Repository/Pear/ChannelRest10Reader.php b/src/Composer/Repository/Pear/ChannelRest10Reader.php index 489914d5d..93969043a 100644 --- a/src/Composer/Repository/Pear/ChannelRest10Reader.php +++ b/src/Composer/Repository/Pear/ChannelRest10Reader.php @@ -13,6 +13,7 @@ namespace Composer\Repository\Pear; use Composer\Downloader\TransportException; +use Composer\Util\HttpDownloader; /** * Read PEAR packages using REST 1.0 interface @@ -29,9 +30,9 @@ class ChannelRest10Reader extends BaseChannelReader { private $dependencyReader; - public function __construct($rfs) + public function __construct(HttpDownloader $httpDownloader) { - parent::__construct($rfs); + parent::__construct($httpDownloader); $this->dependencyReader = new PackageDependencyParser(); } diff --git a/src/Composer/Repository/Pear/ChannelRest11Reader.php b/src/Composer/Repository/Pear/ChannelRest11Reader.php index f9e05f5be..18b1b10f3 100644 --- a/src/Composer/Repository/Pear/ChannelRest11Reader.php +++ b/src/Composer/Repository/Pear/ChannelRest11Reader.php @@ -12,6 +12,8 @@ namespace Composer\Repository\Pear; +use Composer\Util\HttpDownloader; + /** * Read PEAR packages using REST 1.1 interface * @@ -25,9 +27,9 @@ class ChannelRest11Reader extends BaseChannelReader { private $dependencyReader; - public function __construct($rfs) + public function __construct(HttpDownloader $httpDownloader) { - parent::__construct($rfs); + parent::__construct($httpDownloader); $this->dependencyReader = new PackageDependencyParser(); } diff --git a/src/Composer/Repository/PearRepository.php b/src/Composer/Repository/PearRepository.php index c4f0b83e7..aef5c0381 100644 --- a/src/Composer/Repository/PearRepository.php +++ b/src/Composer/Repository/PearRepository.php @@ -21,7 +21,7 @@ use Composer\Repository\Pear\ChannelInfo; use Composer\EventDispatcher\EventDispatcher; use Composer\Package\Link; use Composer\Semver\Constraint\Constraint; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Config; use Composer\Factory; @@ -38,7 +38,7 @@ class PearRepository extends ArrayRepository implements ConfigurableRepositoryIn { private $url; private $io; - private $rfs; + private $httpDownloader; private $versionParser; private $repoConfig; @@ -47,7 +47,7 @@ class PearRepository extends ArrayRepository implements ConfigurableRepositoryIn */ private $vendorAlias; - public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $dispatcher = null, RemoteFilesystem $rfs = null) + public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $dispatcher, HttpDownloader $httpDownloader) { parent::__construct(); if (!preg_match('{^https?://}', $repoConfig['url'])) { @@ -61,7 +61,7 @@ class PearRepository extends ArrayRepository implements ConfigurableRepositoryIn $this->url = rtrim($repoConfig['url'], '/'); $this->io = $io; - $this->rfs = $rfs ?: Factory::createRemoteFilesystem($this->io, $config); + $this->httpDownloader = $httpDownloader; $this->vendorAlias = isset($repoConfig['vendor-alias']) ? $repoConfig['vendor-alias'] : null; $this->versionParser = new VersionParser(); $this->repoConfig = $repoConfig; @@ -78,7 +78,7 @@ class PearRepository extends ArrayRepository implements ConfigurableRepositoryIn $this->io->writeError('Initializing PEAR repository '.$this->url); - $reader = new ChannelReader($this->rfs); + $reader = new ChannelReader($this->httpDownloader); try { $channelInfo = $reader->read($this->url); } catch (\Exception $e) { diff --git a/src/Composer/Repository/RepositoryFactory.php b/src/Composer/Repository/RepositoryFactory.php index 5737a5359..515908f64 100644 --- a/src/Composer/Repository/RepositoryFactory.php +++ b/src/Composer/Repository/RepositoryFactory.php @@ -108,12 +108,12 @@ class RepositoryFactory * @param IOInterface $io * @param Config $config * @param EventDispatcher $eventDispatcher - * @param HttpDownloader $rfs + * @param HttpDownloader $httpDownloader * @return RepositoryManager */ - public static function manager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null) + public static function manager(IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $eventDispatcher = null) { - $rm = new RepositoryManager($io, $config, $eventDispatcher, $rfs); + $rm = new RepositoryManager($io, $config, $eventDispatcher, $httpDownloader); $rm->setRepositoryClass('composer', 'Composer\Repository\ComposerRepository'); $rm->setRepositoryClass('vcs', 'Composer\Repository\VcsRepository'); $rm->setRepositoryClass('package', 'Composer\Repository\PackageRepository'); diff --git a/src/Composer/Repository/RepositoryManager.php b/src/Composer/Repository/RepositoryManager.php index 64568514c..8fc01cb08 100644 --- a/src/Composer/Repository/RepositoryManager.php +++ b/src/Composer/Repository/RepositoryManager.php @@ -33,14 +33,14 @@ class RepositoryManager private $io; private $config; private $eventDispatcher; - private $rfs; + private $httpDownloader; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher, HttpDownloader $httpDownloader) { $this->io = $io; $this->config = $config; $this->eventDispatcher = $eventDispatcher; - $this->rfs = $rfs; + $this->httpDownloader = $httpDownloader; } /** @@ -128,7 +128,7 @@ class RepositoryManager $reflMethod = new \ReflectionMethod($class, '__construct'); $params = $reflMethod->getParameters(); if (isset($params[4]) && $params[4]->getClass() && $params[4]->getClass()->getName() === 'Composer\Util\HttpDownloader') { - return new $class($config, $this->io, $this->config, $this->eventDispatcher, $this->rfs); + return new $class($config, $this->io, $this->config, $this->eventDispatcher, $this->httpDownloader); } return new $class($config, $this->io, $this->config, $this->eventDispatcher); diff --git a/src/Composer/Repository/Vcs/BitbucketDriver.php b/src/Composer/Repository/Vcs/BitbucketDriver.php index 24a4af4dd..bde4fc1b7 100644 --- a/src/Composer/Repository/Vcs/BitbucketDriver.php +++ b/src/Composer/Repository/Vcs/BitbucketDriver.php @@ -16,6 +16,7 @@ use Composer\Cache; use Composer\Downloader\TransportException; use Composer\Json\JsonFile; use Composer\Util\Bitbucket; +use Composer\Util\Http\Response; abstract class BitbucketDriver extends VcsDriver { @@ -92,7 +93,7 @@ abstract class BitbucketDriver extends VcsDriver ) ); - $repoData = JsonFile::parseJson($this->getContentsWithOAuthCredentials($resource, true), $resource); + $repoData = $this->fetchWithOAuthCredentials($resource, true)->decodeJson(); if ($this->fallbackDriver) { return false; } @@ -204,7 +205,7 @@ abstract class BitbucketDriver extends VcsDriver $file ); - return $this->getContentsWithOAuthCredentials($resource); + return $this->fetchWithOAuthCredentials($resource)->getBody(); } /** @@ -222,7 +223,7 @@ abstract class BitbucketDriver extends VcsDriver $this->repository, $identifier ); - $commit = JsonFile::parseJson($this->getContentsWithOAuthCredentials($resource), $resource); + $commit = $this->fetchWithOAuthCredentials($resource)->decodeJson(); return new \DateTime($commit['date']); } @@ -284,7 +285,7 @@ abstract class BitbucketDriver extends VcsDriver ); $hasNext = true; while ($hasNext) { - $tagsData = JsonFile::parseJson($this->getContentsWithOAuthCredentials($resource), $resource); + $tagsData = $this->fetchWithOAuthCredentials($resource)->decodeJson(); foreach ($tagsData['values'] as $data) { $this->tags[$data['name']] = $data['target']['hash']; } @@ -328,7 +329,7 @@ abstract class BitbucketDriver extends VcsDriver ); $hasNext = true; while ($hasNext) { - $branchData = JsonFile::parseJson($this->getContentsWithOAuthCredentials($resource), $resource); + $branchData = $this->fetchWithOAuthCredentials($resource)->decodeJson(); foreach ($branchData['values'] as $data) { // skip headless branches which seem to be deleted branches that bitbucket nevertheless returns in the API if ($this->vcsType === 'hg' && empty($data['heads'])) { @@ -354,14 +355,14 @@ abstract class BitbucketDriver extends VcsDriver * @param string $url The URL of content * @param bool $fetchingRepoData * - * @return mixed The result + * @return Response The result */ - protected function getContentsWithOAuthCredentials($url, $fetchingRepoData = false) + protected function fetchWithOAuthCredentials($url, $fetchingRepoData = false) { try { return parent::getContents($url); } catch (TransportException $e) { - $bitbucketUtil = new Bitbucket($this->io, $this->config, $this->process, $this->remoteFilesystem); + $bitbucketUtil = new Bitbucket($this->io, $this->config, $this->process, $this->httpDownloader); if (403 === $e->getCode() || (401 === $e->getCode() && strpos($e->getMessage(), 'Could not authenticate against') === 0)) { if (!$this->io->hasAuthentication($this->originUrl) @@ -371,7 +372,9 @@ abstract class BitbucketDriver extends VcsDriver } if (!$this->io->isInteractive() && $fetchingRepoData) { - return $this->attemptCloneFallback(); + if ($this->attemptCloneFallback()) { + return new Response(array('url' => 'dummy'), 200, array(), 'null'); + } } } @@ -390,6 +393,8 @@ abstract class BitbucketDriver extends VcsDriver { try { $this->setupFallbackDriver($this->generateSshUrl()); + + return true; } catch (\RuntimeException $e) { $this->fallbackDriver = null; @@ -433,7 +438,7 @@ abstract class BitbucketDriver extends VcsDriver $this->repository ); - $data = JsonFile::parseJson($this->getContentsWithOAuthCredentials($resource), $resource); + $data = $this->fetchWithOAuthCredentials($resource)->decodeJson(); if (isset($data['mainbranch'])) { return $data['mainbranch']; } diff --git a/src/Composer/Repository/Vcs/GitBitbucketDriver.php b/src/Composer/Repository/Vcs/GitBitbucketDriver.php index c8a5c9905..dd69e753a 100644 --- a/src/Composer/Repository/Vcs/GitBitbucketDriver.php +++ b/src/Composer/Repository/Vcs/GitBitbucketDriver.php @@ -76,7 +76,7 @@ class GitBitbucketDriver extends BitbucketDriver $this->io, $this->config, $this->process, - $this->remoteFilesystem + $this->httpDownloader ); $this->fallbackDriver->initialize(); } diff --git a/src/Composer/Repository/Vcs/GitHubDriver.php b/src/Composer/Repository/Vcs/GitHubDriver.php index d0b721af9..f1ad253d8 100644 --- a/src/Composer/Repository/Vcs/GitHubDriver.php +++ b/src/Composer/Repository/Vcs/GitHubDriver.php @@ -18,6 +18,8 @@ use Composer\Json\JsonFile; use Composer\Cache; use Composer\IO\IOInterface; use Composer\Util\GitHub; +use Composer\Util\Http\Response; +use Composer\Util\RemoteFilesystem; /** * @author Jordi Boggiano @@ -184,7 +186,7 @@ class GitHubDriver extends VcsDriver } $resource = $this->getApiUrl() . '/repos/'.$this->owner.'/'.$this->repository.'/contents/' . $file . '?ref='.urlencode($identifier); - $resource = JsonFile::parseJson($this->getContents($resource)); + $resource = $this->getContents($resource)->decodeJson(); if (empty($resource['content']) || $resource['encoding'] !== 'base64' || !($content = base64_decode($resource['content']))) { throw new \RuntimeException('Could not retrieve ' . $file . ' for '.$identifier); } @@ -202,7 +204,7 @@ class GitHubDriver extends VcsDriver } $resource = $this->getApiUrl() . '/repos/'.$this->owner.'/'.$this->repository.'/commits/'.urlencode($identifier); - $commit = JsonFile::parseJson($this->getContents($resource), $resource); + $commit = $this->getContents($resource)->decodeJson(); return new \DateTime($commit['commit']['committer']['date']); } @@ -220,12 +222,13 @@ class GitHubDriver extends VcsDriver $resource = $this->getApiUrl() . '/repos/'.$this->owner.'/'.$this->repository.'/tags?per_page=100'; do { - $tagsData = JsonFile::parseJson($this->getContents($resource), $resource); + $response = $this->getContents($resource); + $tagsData = $response->decodeJson(); foreach ($tagsData as $tag) { $this->tags[$tag['name']] = $tag['commit']['sha']; } - $resource = $this->getNextPage(); + $resource = $this->getNextPage($response); } while ($resource); } @@ -247,7 +250,8 @@ class GitHubDriver extends VcsDriver $branchBlacklist = array('gh-pages'); do { - $branchData = JsonFile::parseJson($this->getContents($resource), $resource); + $response = $this->getContents($resource); + $branchData = $response->decodeJson(); foreach ($branchData as $branch) { $name = substr($branch['ref'], 11); if (!in_array($name, $branchBlacklist)) { @@ -255,7 +259,7 @@ class GitHubDriver extends VcsDriver } } - $resource = $this->getNextPage(); + $resource = $this->getNextPage($response); } while ($resource); } @@ -315,7 +319,7 @@ class GitHubDriver extends VcsDriver try { return parent::getContents($url); } catch (TransportException $e) { - $gitHubUtil = new GitHub($this->io, $this->config, $this->process, $this->remoteFilesystem); + $gitHubUtil = new GitHub($this->io, $this->config, $this->process, $this->httpDownloader); switch ($e->getCode()) { case 401: @@ -330,16 +334,18 @@ class GitHubDriver extends VcsDriver } if (!$this->io->isInteractive()) { - return $this->attemptCloneFallback(); + if ($this->attemptCloneFallback()) { + return new Response(array('url' => 'dummy'), 200, array(), 'null'); + } } $scopesIssued = array(); $scopesNeeded = array(); if ($headers = $e->getHeaders()) { - if ($scopes = $this->remoteFilesystem->findHeaderValue($headers, 'X-OAuth-Scopes')) { + if ($scopes = RemoteFilesystem::findHeaderValue($headers, 'X-OAuth-Scopes')) { $scopesIssued = explode(' ', $scopes); } - if ($scopes = $this->remoteFilesystem->findHeaderValue($headers, 'X-Accepted-OAuth-Scopes')) { + if ($scopes = RemoteFilesystem::findHeaderValue($headers, 'X-Accepted-OAuth-Scopes')) { $scopesNeeded = explode(' ', $scopes); } } @@ -358,7 +364,9 @@ class GitHubDriver extends VcsDriver } if (!$this->io->isInteractive() && $fetchingRepoData) { - return $this->attemptCloneFallback(); + if ($this->attemptCloneFallback()) { + return new Response(array('url' => 'dummy'), 200, array(), 'null'); + } } $rateLimited = $gitHubUtil->isRateLimited($e->getHeaders()); @@ -404,7 +412,7 @@ class GitHubDriver extends VcsDriver $repoDataUrl = $this->getApiUrl() . '/repos/'.$this->owner.'/'.$this->repository; - $this->repoData = JsonFile::parseJson($this->getContents($repoDataUrl, true), $repoDataUrl); + $this->repoData = $this->getContents($repoDataUrl, true)->decodeJson(); if (null === $this->repoData && null !== $this->gitDriver) { return; } @@ -434,7 +442,7 @@ class GitHubDriver extends VcsDriver // are not interactive) then we fallback to GitDriver. $this->setupGitDriver($this->generateSshUrl()); - return; + return true; } catch (\RuntimeException $e) { $this->gitDriver = null; @@ -450,22 +458,19 @@ class GitHubDriver extends VcsDriver $this->io, $this->config, $this->process, - $this->remoteFilesystem + $this->httpDownloader ); $this->gitDriver->initialize(); } - protected function getNextPage() + protected function getNextPage(Response $response) { - $headers = $this->remoteFilesystem->getLastHeaders(); - foreach ($headers as $header) { - if (preg_match('{^link:\s*(.+?)\s*$}i', $header, $match)) { - $links = explode(',', $match[1]); - foreach ($links as $link) { - if (preg_match('{<(.+?)>; *rel="next"}', $link, $match)) { - return $match[1]; - } - } + $header = $response->getHeader('link'); + + $links = explode(',', $header); + foreach ($links as $link) { + if (preg_match('{<(.+?)>; *rel="next"}', $link, $match)) { + return $match[1]; } } } diff --git a/src/Composer/Repository/Vcs/GitLabDriver.php b/src/Composer/Repository/Vcs/GitLabDriver.php index 2044ff702..6a1aa8ac2 100644 --- a/src/Composer/Repository/Vcs/GitLabDriver.php +++ b/src/Composer/Repository/Vcs/GitLabDriver.php @@ -17,8 +17,9 @@ use Composer\Cache; use Composer\IO\IOInterface; use Composer\Json\JsonFile; use Composer\Downloader\TransportException; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Util\GitLab; +use Composer\Util\Http\Response; /** * Driver for GitLab API, use the Git driver for local checkouts. @@ -110,14 +111,14 @@ class GitLabDriver extends VcsDriver } /** - * Updates the RemoteFilesystem instance. + * Updates the HttpDownloader instance. * Mainly useful for tests. * * @internal */ - public function setRemoteFilesystem(RemoteFilesystem $remoteFilesystem) + public function setHttpDownloader(HttpDownloader $httpDownloader) { - $this->remoteFilesystem = $remoteFilesystem; + $this->httpDownloader = $httpDownloader; } /** @@ -140,7 +141,7 @@ class GitLabDriver extends VcsDriver $resource = $this->getApiUrl().'/repository/files/'.$this->urlEncodeAll($file).'/raw?ref='.$identifier; try { - $content = $this->getContents($resource); + $content = $this->getContents($resource)->getBody(); } catch (TransportException $e) { if ($e->getCode() !== 404) { throw $e; @@ -297,7 +298,8 @@ class GitLabDriver extends VcsDriver $references = array(); do { - $data = JsonFile::parseJson($this->getContents($resource), $resource); + $response = $this->getContents($resource); + $data = $response->decodeJson(); foreach ($data as $datum) { $references[$datum['name']] = $datum['commit']['id']; @@ -308,7 +310,7 @@ class GitLabDriver extends VcsDriver } if (count($data) >= $perPage) { - $resource = $this->getNextPage(); + $resource = $this->getNextPage($response); } else { $resource = false; } @@ -321,7 +323,7 @@ class GitLabDriver extends VcsDriver { // we need to fetch the default branch from the api $resource = $this->getApiUrl(); - $this->project = JsonFile::parseJson($this->getContents($resource, true), $resource); + $this->project = $this->getContents($resource, true)->decodeJson(); if (isset($this->project['visibility'])) { $this->isPrivate = $this->project['visibility'] !== 'public'; } else { @@ -344,7 +346,7 @@ class GitLabDriver extends VcsDriver // are not interactive) then we fallback to GitDriver. $this->setupGitDriver($url); - return; + return true; } catch (\RuntimeException $e) { $this->gitDriver = null; @@ -375,7 +377,7 @@ class GitLabDriver extends VcsDriver $this->io, $this->config, $this->process, - $this->remoteFilesystem + $this->httpDownloader ); $this->gitDriver->initialize(); } @@ -386,10 +388,10 @@ class GitLabDriver extends VcsDriver protected function getContents($url, $fetchingRepoData = false) { try { - $res = parent::getContents($url); + $response = parent::getContents($url); if ($fetchingRepoData) { - $json = JsonFile::parseJson($res, $url); + $json = $response->decodeJson(); // force auth as the unauthenticated version of the API is broken if (!isset($json['default_branch'])) { @@ -401,9 +403,9 @@ class GitLabDriver extends VcsDriver } } - return $res; + return $response; } catch (TransportException $e) { - $gitLabUtil = new GitLab($this->io, $this->config, $this->process, $this->remoteFilesystem); + $gitLabUtil = new GitLab($this->io, $this->config, $this->process, $this->httpDownloader); switch ($e->getCode()) { case 401: @@ -418,7 +420,9 @@ class GitLabDriver extends VcsDriver } if (!$this->io->isInteractive()) { - return $this->attemptCloneFallback(); + if ($this->attemptCloneFallback()) { + return new Response(array('url' => 'dummy'), 200, array(), 'null'); + } } $this->io->writeError('Failed to download ' . $this->namespace . '/' . $this->repository . ':' . $e->getMessage() . ''); $gitLabUtil->authorizeOAuthInteractively($this->scheme, $this->originUrl, 'Your credentials are required to fetch private repository metadata ('.$this->url.')'); @@ -431,7 +435,9 @@ class GitLabDriver extends VcsDriver } if (!$this->io->isInteractive() && $fetchingRepoData) { - return $this->attemptCloneFallback(); + if ($this->attemptCloneFallback()) { + return new Response(array('url' => 'dummy'), 200, array(), 'null'); + } } throw $e; @@ -471,17 +477,14 @@ class GitLabDriver extends VcsDriver return true; } - private function getNextPage() + protected function getNextPage(Response $response) { - $headers = $this->remoteFilesystem->getLastHeaders(); - foreach ($headers as $header) { - if (preg_match('{^link:\s*(.+?)\s*$}i', $header, $match)) { - $links = explode(',', $match[1]); - foreach ($links as $link) { - if (preg_match('{<(.+?)>; *rel="next"}', $link, $match)) { - return $match[1]; - } - } + $header = $response->getHeader('link'); + + $links = explode(',', $header); + foreach ($links as $link) { + if (preg_match('{<(.+?)>; *rel="next"}', $link, $match)) { + return $match[1]; } } } diff --git a/src/Composer/Repository/Vcs/HgBitbucketDriver.php b/src/Composer/Repository/Vcs/HgBitbucketDriver.php index 8324f22ac..4a00f2da0 100644 --- a/src/Composer/Repository/Vcs/HgBitbucketDriver.php +++ b/src/Composer/Repository/Vcs/HgBitbucketDriver.php @@ -76,7 +76,7 @@ class HgBitbucketDriver extends BitbucketDriver $this->io, $this->config, $this->process, - $this->remoteFilesystem + $this->httpDownloader ); $this->fallbackDriver->initialize(); } diff --git a/src/Composer/Repository/Vcs/VcsDriver.php b/src/Composer/Repository/Vcs/VcsDriver.php index 5227630f6..17ed706d2 100644 --- a/src/Composer/Repository/Vcs/VcsDriver.php +++ b/src/Composer/Repository/Vcs/VcsDriver.php @@ -19,8 +19,9 @@ use Composer\Factory; use Composer\IO\IOInterface; use Composer\Json\JsonFile; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Util\Filesystem; +use Composer\Util\Http\Response; /** * A driver implementation for driver with authentication interaction. @@ -41,8 +42,8 @@ abstract class VcsDriver implements VcsDriverInterface protected $config; /** @var ProcessExecutor */ protected $process; - /** @var RemoteFilesystem */ - protected $remoteFilesystem; + /** @var HttpDownloader */ + protected $httpDownloader; /** @var array */ protected $infoCache = array(); /** @var Cache */ @@ -55,9 +56,9 @@ abstract class VcsDriver implements VcsDriverInterface * @param IOInterface $io The IO instance * @param Config $config The composer configuration * @param ProcessExecutor $process Process instance, injectable for mocking - * @param RemoteFilesystem $remoteFilesystem Remote Filesystem, injectable for mocking + * @param HttpDownloader $httpDownloader Remote Filesystem, injectable for mocking */ - final public function __construct(array $repoConfig, IOInterface $io, Config $config, ProcessExecutor $process = null, RemoteFilesystem $remoteFilesystem = null) + final public function __construct(array $repoConfig, IOInterface $io, Config $config, ProcessExecutor $process = null, HttpDownloader $httpDownloader = null) { if (Filesystem::isLocalPath($repoConfig['url'])) { $repoConfig['url'] = Filesystem::getPlatformPath($repoConfig['url']); @@ -69,7 +70,7 @@ abstract class VcsDriver implements VcsDriverInterface $this->io = $io; $this->config = $config; $this->process = $process ?: new ProcessExecutor($io); - $this->remoteFilesystem = $remoteFilesystem ?: Factory::createRemoteFilesystem($this->io, $config); + $this->httpDownloader = $httpDownloader ?: Factory::createHttpDownloader($this->io, $config); } /** @@ -156,13 +157,13 @@ abstract class VcsDriver implements VcsDriverInterface * * @param string $url The URL of content * - * @return mixed The result + * @return Response */ protected function getContents($url) { $options = isset($this->repoConfig['options']) ? $this->repoConfig['options'] : array(); - return $this->remoteFilesystem->getContents($this->originUrl, $url, false, $options); + return $this->httpDownloader->get($url, $options); } /** diff --git a/src/Composer/Util/Bitbucket.php b/src/Composer/Util/Bitbucket.php index 1fc286ac4..d9f569b1b 100644 --- a/src/Composer/Util/Bitbucket.php +++ b/src/Composer/Util/Bitbucket.php @@ -25,7 +25,7 @@ class Bitbucket private $io; private $config; private $process; - private $remoteFilesystem; + private $httpDownloader; private $token = array(); private $time; @@ -37,15 +37,15 @@ class Bitbucket * @param IOInterface $io The IO instance * @param Config $config The composer configuration * @param ProcessExecutor $process Process instance, injectable for mocking - * @param RemoteFilesystem $remoteFilesystem Remote Filesystem, injectable for mocking + * @param HttpDownloader $httpDownloader Remote Filesystem, injectable for mocking * @param int $time Timestamp, injectable for mocking */ - public function __construct(IOInterface $io, Config $config, ProcessExecutor $process = null, RemoteFilesystem $remoteFilesystem = null, $time = null) + public function __construct(IOInterface $io, Config $config, ProcessExecutor $process = null, HttpDownloader $httpDownloader = null, $time = null) { $this->io = $io; $this->config = $config; $this->process = $process ?: new ProcessExecutor($io); - $this->remoteFilesystem = $remoteFilesystem ?: Factory::createRemoteFilesystem($this->io, $config); + $this->httpDownloader = $httpDownloader ?: Factory::createHttpDownloader($this->io, $config); $this->time = $time; } @@ -90,7 +90,7 @@ class Bitbucket private function requestAccessToken($originUrl) { try { - $json = $this->remoteFilesystem->getContents($originUrl, self::OAUTH2_ACCESS_TOKEN_URL, false, array( + $response = $this->httpDownloader->get(self::OAUTH2_ACCESS_TOKEN_URL, array( 'retry-auth-failure' => false, 'http' => array( 'method' => 'POST', @@ -98,7 +98,7 @@ class Bitbucket ), )); - $this->token = json_decode($json, true); + $this->token = $response->decodeJson(); } catch (TransportException $e) { if ($e->getCode() === 400) { $this->io->writeError('Invalid OAuth consumer provided.'); diff --git a/src/Composer/Util/GitHub.php b/src/Composer/Util/GitHub.php index 1eca1a9bb..c3046cb77 100644 --- a/src/Composer/Util/GitHub.php +++ b/src/Composer/Util/GitHub.php @@ -25,7 +25,7 @@ class GitHub protected $io; protected $config; protected $process; - protected $remoteFilesystem; + protected $httpDownloader; /** * Constructor. @@ -33,14 +33,14 @@ class GitHub * @param IOInterface $io The IO instance * @param Config $config The composer configuration * @param ProcessExecutor $process Process instance, injectable for mocking - * @param RemoteFilesystem $remoteFilesystem Remote Filesystem, injectable for mocking + * @param HttpDownloader $httpDownloader Remote Filesystem, injectable for mocking */ - public function __construct(IOInterface $io, Config $config, ProcessExecutor $process = null, RemoteFilesystem $remoteFilesystem = null) + public function __construct(IOInterface $io, Config $config, ProcessExecutor $process = null, HttpDownloader $httpDownloader = null) { $this->io = $io; $this->config = $config; $this->process = $process ?: new ProcessExecutor($io); - $this->remoteFilesystem = $remoteFilesystem ?: Factory::createRemoteFilesystem($this->io, $config); + $this->httpDownloader = $httpDownloader ?: Factory::createHttpDownloader($this->io, $config); } /** @@ -104,7 +104,7 @@ class GitHub try { $apiUrl = ('github.com' === $originUrl) ? 'api.github.com/' : $originUrl . '/api/v3/'; - $this->remoteFilesystem->getContents($originUrl, 'https://'. $apiUrl, false, array( + $this->httpDownloader->get('https://'. $apiUrl, array( 'retry-auth-failure' => false, )); } catch (TransportException $e) { diff --git a/src/Composer/Util/GitLab.php b/src/Composer/Util/GitLab.php index 475c5e7ee..2a4867954 100644 --- a/src/Composer/Util/GitLab.php +++ b/src/Composer/Util/GitLab.php @@ -26,7 +26,7 @@ class GitLab protected $io; protected $config; protected $process; - protected $remoteFilesystem; + protected $httpDownloader; /** * Constructor. @@ -34,14 +34,14 @@ class GitLab * @param IOInterface $io The IO instance * @param Config $config The composer configuration * @param ProcessExecutor $process Process instance, injectable for mocking - * @param RemoteFilesystem $remoteFilesystem Remote Filesystem, injectable for mocking + * @param HttpDownloader $httpDownloader Remote Filesystem, injectable for mocking */ - public function __construct(IOInterface $io, Config $config, ProcessExecutor $process = null, RemoteFilesystem $remoteFilesystem = null) + public function __construct(IOInterface $io, Config $config, ProcessExecutor $process = null, HttpDownloader $httpDownloader = null) { $this->io = $io; $this->config = $config; $this->process = $process ?: new ProcessExecutor($io); - $this->remoteFilesystem = $remoteFilesystem ?: Factory::createRemoteFilesystem($this->io, $config); + $this->httpDownloader = $httpDownloader ?: Factory::createHttpDownloader($this->io, $config); } /** @@ -154,10 +154,10 @@ class GitLab ), ); - $json = $this->remoteFilesystem->getContents($originUrl, $scheme.'://'.$apiUrl.'/oauth/token', false, $options); + $token = $this->httpDownloader->get($scheme.'://'.$apiUrl.'/oauth/token', $options)->decodeJson(); $this->io->writeError('Token successfully created'); - return JsonFile::parseJson($json); + return $token; } } diff --git a/src/Composer/Util/Http/Response.php b/src/Composer/Util/Http/Response.php index ff48fdb40..f76057f3e 100644 --- a/src/Composer/Util/Http/Response.php +++ b/src/Composer/Util/Http/Response.php @@ -23,6 +23,9 @@ class Response public function __construct(array $request, $code, array $headers, $body) { + if (!isset($request['url'])) { + throw new \LogicException('url key missing from request array'); + } $this->request = $request; $this->code = $code; $this->headers = $headers; diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php index 31c615e0c..631c0df7d 100644 --- a/src/Composer/Util/HttpDownloader.php +++ b/src/Composer/Util/HttpDownloader.php @@ -117,8 +117,8 @@ class HttpDownloader $origin = $this->getOrigin($job['request']['url']); - // TODO only send http/https through curl - if ($curl) { + // TODO experiment with allowing file:// through curl too + if ($curl && preg_match('{^https?://}i', $job['request']['url'])) { $resolver = function ($resolve, $reject) use (&$job, $curl, $origin) { // start job $url = $job['request']['url']; @@ -141,11 +141,7 @@ class HttpDownloader $job['status'] = HttpDownloader::STATUS_STARTED; if ($job['request']['copyTo']) { - if ($curl) { - $result = $curl->download($origin, $url, $options, $job['request']['copyTo']); - } else { - $result = $rfs->copy($origin, $url, $job['request']['copyTo'], false /* TODO progress */, $options); - } + $result = $rfs->copy($origin, $url, $job['request']['copyTo'], false /* TODO progress */, $options); $resolve($result); } else { diff --git a/src/Composer/Util/RemoteFilesystem.php b/src/Composer/Util/RemoteFilesystem.php index f4a211acb..00fe35294 100644 --- a/src/Composer/Util/RemoteFilesystem.php +++ b/src/Composer/Util/RemoteFilesystem.php @@ -147,7 +147,7 @@ class RemoteFilesystem * @param string $name header name (case insensitive) * @return string|null */ - public function findHeaderValue(array $headers, $name) + public static function findHeaderValue(array $headers, $name) { $value = null; foreach ($headers as $header) { @@ -167,7 +167,7 @@ class RemoteFilesystem * @param array $headers array of returned headers like from getLastHeaders() * @return int|null */ - public function findStatusCode(array $headers) + public static function findStatusCode(array $headers) { $value = null; foreach ($headers as $header) { diff --git a/tests/Composer/Test/Downloader/ArchiveDownloaderTest.php b/tests/Composer/Test/Downloader/ArchiveDownloaderTest.php index 68852d8e0..ddf21c64b 100644 --- a/tests/Composer/Test/Downloader/ArchiveDownloaderTest.php +++ b/tests/Composer/Test/Downloader/ArchiveDownloaderTest.php @@ -156,7 +156,11 @@ class ArchiveDownloaderTest extends TestCase { return $this->getMockForAbstractClass( 'Composer\Downloader\ArchiveDownloader', - array($this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getMockBuilder('Composer\Config')->getMock()) + array( + $io = $this->getMockBuilder('Composer\IO\IOInterface')->getMock(), + $config = $this->getMockBuilder('Composer\Config')->getMock(), + new \Composer\Util\HttpDownloader($io, $config), + ) ); } } diff --git a/tests/Composer/Test/Downloader/FileDownloaderTest.php b/tests/Composer/Test/Downloader/FileDownloaderTest.php index 476b9a8f7..12edfe19d 100644 --- a/tests/Composer/Test/Downloader/FileDownloaderTest.php +++ b/tests/Composer/Test/Downloader/FileDownloaderTest.php @@ -18,13 +18,13 @@ use Composer\Util\Filesystem; class FileDownloaderTest extends TestCase { - protected function getDownloader($io = null, $config = null, $eventDispatcher = null, $cache = null, $rfs = null, $filesystem = null) + protected function getDownloader($io = null, $config = null, $eventDispatcher = null, $cache = null, $httpDownloader = null, $filesystem = null) { $io = $io ?: $this->getMockBuilder('Composer\IO\IOInterface')->getMock(); $config = $config ?: $this->getMockBuilder('Composer\Config')->getMock(); - $rfs = $rfs ?: $this->getMockBuilder('Composer\Util\RemoteFilesystem')->disableOriginalConstructor()->getMock(); + $httpDownloader = $httpDownloader ?: $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock(); - return new FileDownloader($io, $config, $eventDispatcher, $cache, $rfs, $filesystem); + return new FileDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $filesystem); } /** diff --git a/tests/Composer/Test/Downloader/XzDownloaderTest.php b/tests/Composer/Test/Downloader/XzDownloaderTest.php index 6df782ddb..451592d37 100644 --- a/tests/Composer/Test/Downloader/XzDownloaderTest.php +++ b/tests/Composer/Test/Downloader/XzDownloaderTest.php @@ -16,7 +16,7 @@ use Composer\Downloader\XzDownloader; use Composer\Test\TestCase; use Composer\Util\Filesystem; use Composer\Util\Platform; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; class XzDownloaderTest extends TestCase { @@ -66,7 +66,7 @@ class XzDownloaderTest extends TestCase ->method('get') ->with('vendor-dir') ->will($this->returnValue($this->testDir)); - $downloader = new XzDownloader($io, $config, null, null, null, new RemoteFilesystem($io)); + $downloader = new XzDownloader($io, $config, new HttpDownloader($io, $this->getMockBuilder('Composer\Config')->getMock()), null, null, null); try { $downloader->download($packageMock, $this->getUniqueTmpDirectory()); diff --git a/tests/Composer/Test/Downloader/ZipDownloaderTest.php b/tests/Composer/Test/Downloader/ZipDownloaderTest.php index 466fd35c7..0c1311427 100644 --- a/tests/Composer/Test/Downloader/ZipDownloaderTest.php +++ b/tests/Composer/Test/Downloader/ZipDownloaderTest.php @@ -16,6 +16,7 @@ use Composer\Downloader\ZipDownloader; use Composer\Package\PackageInterface; use Composer\Test\TestCase; use Composer\Util\Filesystem; +use Composer\Util\HttpDownloader; class ZipDownloaderTest extends TestCase { @@ -32,6 +33,8 @@ class ZipDownloaderTest extends TestCase $this->testDir = $this->getUniqueTmpDirectory(); $this->io = $this->getMockBuilder('Composer\IO\IOInterface')->getMock(); $this->config = $this->getMockBuilder('Composer\Config')->getMock(); + $dlConfig = $this->getMockBuilder('Composer\Config')->getMock(); + $this->httpDownloader = new HttpDownloader($this->io, $dlConfig); } public function tearDown() @@ -64,18 +67,6 @@ class ZipDownloaderTest extends TestCase } $this->config->expects($this->at(0)) - ->method('get') - ->with('disable-tls') - ->will($this->returnValue(false)); - $this->config->expects($this->at(1)) - ->method('get') - ->with('cafile') - ->will($this->returnValue(null)); - $this->config->expects($this->at(2)) - ->method('get') - ->with('capath') - ->will($this->returnValue(null)); - $this->config->expects($this->at(3)) ->method('get') ->with('vendor-dir') ->will($this->returnValue($this->testDir)); @@ -94,7 +85,7 @@ class ZipDownloaderTest extends TestCase ->will($this->returnValue(array())) ; - $downloader = new ZipDownloader($this->io, $this->config); + $downloader = new ZipDownloader($this->io, $this->config, $this->httpDownloader); $this->setPrivateProperty('hasSystemUnzip', false); @@ -118,8 +109,7 @@ class ZipDownloaderTest extends TestCase $this->setPrivateProperty('hasSystemUnzip', false); $this->setPrivateProperty('hasZipArchive', true); - $downloader = new MockedZipDownloader($this->io, $this->config); - + $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader); $zipArchive = $this->getMockBuilder('ZipArchive')->getMock(); $zipArchive->expects($this->at(0)) ->method('open') @@ -144,8 +134,7 @@ class ZipDownloaderTest extends TestCase $this->setPrivateProperty('hasSystemUnzip', false); $this->setPrivateProperty('hasZipArchive', true); - $downloader = new MockedZipDownloader($this->io, $this->config); - + $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader); $zipArchive = $this->getMockBuilder('ZipArchive')->getMock(); $zipArchive->expects($this->at(0)) ->method('open') @@ -169,8 +158,7 @@ class ZipDownloaderTest extends TestCase $this->setPrivateProperty('hasSystemUnzip', false); $this->setPrivateProperty('hasZipArchive', true); - $downloader = new MockedZipDownloader($this->io, $this->config); - + $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader); $zipArchive = $this->getMockBuilder('ZipArchive')->getMock(); $zipArchive->expects($this->at(0)) ->method('open') @@ -200,7 +188,7 @@ class ZipDownloaderTest extends TestCase ->method('execute') ->will($this->returnValue(1)); - $downloader = new MockedZipDownloader($this->io, $this->config, null, null, $processExecutor); + $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); $downloader->extract('testfile.zip', 'vendor/dir'); } @@ -217,7 +205,7 @@ class ZipDownloaderTest extends TestCase ->method('execute') ->will($this->returnValue(0)); - $downloader = new MockedZipDownloader($this->io, $this->config, null, null, $processExecutor); + $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); $downloader->extract('testfile.zip', 'vendor/dir'); } @@ -244,7 +232,7 @@ class ZipDownloaderTest extends TestCase ->method('extractTo') ->will($this->returnValue(true)); - $downloader = new MockedZipDownloader($this->io, $this->config, null, null, $processExecutor); + $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); $downloader->extract('testfile.zip', 'vendor/dir'); } @@ -276,7 +264,7 @@ class ZipDownloaderTest extends TestCase ->method('extractTo') ->will($this->returnValue(false)); - $downloader = new MockedZipDownloader($this->io, $this->config, null, null, $processExecutor); + $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); $downloader->extract('testfile.zip', 'vendor/dir'); } @@ -304,7 +292,7 @@ class ZipDownloaderTest extends TestCase ->method('extractTo') ->will($this->returnValue(false)); - $downloader = new MockedZipDownloader($this->io, $this->config, null, null, $processExecutor); + $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); $downloader->extract('testfile.zip', 'vendor/dir'); } @@ -336,7 +324,7 @@ class ZipDownloaderTest extends TestCase ->method('extractTo') ->will($this->returnValue(false)); - $downloader = new MockedZipDownloader($this->io, $this->config, null, null, $processExecutor); + $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); $downloader->extract('testfile.zip', 'vendor/dir'); } diff --git a/tests/Composer/Test/InstallerTest.php b/tests/Composer/Test/InstallerTest.php index f21007281..2e7d70639 100644 --- a/tests/Composer/Test/InstallerTest.php +++ b/tests/Composer/Test/InstallerTest.php @@ -63,7 +63,9 @@ class InstallerTest extends TestCase ->getMock(); $config = $this->getMockBuilder('Composer\Config')->getMock(); - $repositoryManager = new RepositoryManager($io, $config); + $eventDispatcher = $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock(); + $httpDownloader = $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock(); + $repositoryManager = new RepositoryManager($io, $config, $eventDispatcher, $httpDownloader); $repositoryManager->setLocalRepository(new InstalledArrayRepository()); if (!is_array($repositories)) { @@ -76,7 +78,6 @@ class InstallerTest extends TestCase $locker = $this->getMockBuilder('Composer\Package\Locker')->disableOriginalConstructor()->getMock(); $installationManager = new InstallationManagerMock(); - $eventDispatcher = $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock(); $autoloadGenerator = $this->getMockBuilder('Composer\Autoload\AutoloadGenerator')->disableOriginalConstructor()->getMock(); $installer = new Installer($io, $config, clone $rootPackage, $downloadManager, $repositoryManager, $locker, $installationManager, $eventDispatcher, $autoloadGenerator); diff --git a/tests/Composer/Test/Mock/RemoteFilesystemMock.php b/tests/Composer/Test/Mock/HttpDownloaderMock.php similarity index 73% rename from tests/Composer/Test/Mock/RemoteFilesystemMock.php rename to tests/Composer/Test/Mock/HttpDownloaderMock.php index 5d4f52e54..1e2774af0 100644 --- a/tests/Composer/Test/Mock/RemoteFilesystemMock.php +++ b/tests/Composer/Test/Mock/HttpDownloaderMock.php @@ -12,13 +12,11 @@ namespace Composer\Test\Mock; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; +use Composer\Util\Http\Response; use Composer\Downloader\TransportException; -/** - * Remote filesystem mock - */ -class RemoteFilesystemMock extends RemoteFilesystem +class HttpDownloaderMock extends HttpDownloader { protected $contentMap; @@ -30,10 +28,10 @@ class RemoteFilesystemMock extends RemoteFilesystem $this->contentMap = $contentMap; } - public function getContents($originUrl, $fileUrl, $progress = true, $options = array()) + public function get($fileUrl, $options = array()) { if (!empty($this->contentMap[$fileUrl])) { - return $this->contentMap[$fileUrl]; + return new Response(array('url' => $fileUrl), 200, array(), $this->contentMap[$fileUrl]); } throw new TransportException('The "'.$fileUrl.'" file could not be downloaded (NOT FOUND)', 404); diff --git a/tests/Composer/Test/Package/Archiver/ArchiveManagerTest.php b/tests/Composer/Test/Package/Archiver/ArchiveManagerTest.php index f9fe308fa..b9f08e693 100644 --- a/tests/Composer/Test/Package/Archiver/ArchiveManagerTest.php +++ b/tests/Composer/Test/Package/Archiver/ArchiveManagerTest.php @@ -12,9 +12,11 @@ namespace Composer\Test\Package\Archiver; +use Composer\IO\NullIO; use Composer\Factory; use Composer\Package\Archiver\ArchiveManager; use Composer\Package\PackageInterface; +use Composer\Test\Mock\FactoryMock; class ArchiveManagerTest extends ArchiverTest { @@ -30,7 +32,12 @@ class ArchiveManagerTest extends ArchiverTest parent::setUp(); $factory = new Factory(); - $this->manager = $factory->createArchiveManager($factory->createConfig()); + $dm = $factory->createDownloadManager( + $io = new NullIO, + $config = FactoryMock::createConfig(), + $factory->createHttpDownloader($io, $config) + ); + $this->manager = $factory->createArchiveManager($factory->createConfig(), $dm); $this->targetDir = $this->testDir.'/composer_archiver_tests'; } diff --git a/tests/Composer/Test/Repository/ComposerRepositoryTest.php b/tests/Composer/Test/Repository/ComposerRepositoryTest.php index 05e1afada..1fe60f589 100644 --- a/tests/Composer/Test/Repository/ComposerRepositoryTest.php +++ b/tests/Composer/Test/Repository/ComposerRepositoryTest.php @@ -18,7 +18,7 @@ use Composer\Repository\RepositoryInterface; use Composer\Test\Mock\FactoryMock; use Composer\Test\TestCase; use Composer\Package\Loader\ArrayLoader; -use Composer\Semver\VersionParser; +use Composer\Package\Version\VersionParser; class ComposerRepositoryTest extends TestCase { @@ -37,6 +37,8 @@ class ComposerRepositoryTest extends TestCase $repoConfig, new NullIO, FactoryMock::createConfig(), + $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock(), + $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock() )) ->getMock(); @@ -179,21 +181,29 @@ class ComposerRepositoryTest extends TestCase ), ); - $rfs = $this->getMockBuilder('Composer\Util\RemoteFilesystem') + $httpDownloader = $this->getMockBuilder('Composer\Util\HttpDownloader') + ->disableOriginalConstructor() + ->getMock(); + $eventDispatcher = $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher') ->disableOriginalConstructor() ->getMock(); - $rfs->expects($this->at(0)) - ->method('getContents') - ->with('example.org', 'http://example.org/packages.json', false) - ->willReturn(json_encode(array('search' => '/search.json?q=%query%&type=%type%'))); + $httpDownloader->expects($this->at(0)) + ->method('get') + ->with($url = 'http://example.org/packages.json') + ->willReturn(new \Composer\Util\Http\Response(array('url' => $url), 200, array(), json_encode(array('search' => '/search.json?q=%query%&type=%type%')))); - $rfs->expects($this->at(1)) - ->method('getContents') - ->with('example.org', 'http://example.org/search.json?q=foo&type=composer-plugin', false) - ->willReturn(json_encode($result)); + $httpDownloader->expects($this->at(1)) + ->method('get') + ->with($url = 'http://example.org/search.json?q=foo&type=composer-plugin') + ->willReturn(new \Composer\Util\Http\Response(array('url' => $url), 200, array(), json_encode($result))); - $repository = new ComposerRepository($repoConfig, new NullIO, FactoryMock::createConfig(), null, $rfs); + $httpDownloader->expects($this->at(2)) + ->method('get') + ->with($url = 'http://example.org/search.json?q=foo&type=library') + ->willReturn(new \Composer\Util\Http\Response(array('url' => $url), 200, array(), json_encode(array()))); + + $repository = new ComposerRepository($repoConfig, new NullIO, FactoryMock::createConfig(), $eventDispatcher, $httpDownloader); $this->assertSame( array(array('name' => 'foo', 'description' => null)), diff --git a/tests/Composer/Test/Repository/PathRepositoryTest.php b/tests/Composer/Test/Repository/PathRepositoryTest.php index a9594257c..abe6063f4 100644 --- a/tests/Composer/Test/Repository/PathRepositoryTest.php +++ b/tests/Composer/Test/Repository/PathRepositoryTest.php @@ -14,8 +14,8 @@ namespace Composer\Test\Repository; use Composer\Package\Loader\ArrayLoader; use Composer\Repository\PathRepository; -use Composer\Semver\VersionParser; use Composer\Test\TestCase; +use Composer\Package\Version\VersionParser; class PathRepositoryTest extends TestCase { diff --git a/tests/Composer/Test/Repository/Pear/ChannelReaderTest.php b/tests/Composer/Test/Repository/Pear/ChannelReaderTest.php index e766065a7..7ad30825d 100644 --- a/tests/Composer/Test/Repository/Pear/ChannelReaderTest.php +++ b/tests/Composer/Test/Repository/Pear/ChannelReaderTest.php @@ -22,13 +22,13 @@ use Composer\Semver\VersionParser; use Composer\Semver\Constraint\Constraint; use Composer\Package\Link; use Composer\Package\CompletePackage; -use Composer\Test\Mock\RemoteFilesystemMock; +use Composer\Test\Mock\HttpDownloaderMock; class ChannelReaderTest extends TestCase { public function testShouldBuildPackagesFromPearSchema() { - $rfs = new RemoteFilesystemMock(array( + $rfs = new HttpDownloaderMock(array( 'http://pear.net/channel.xml' => file_get_contents(__DIR__ . '/Fixtures/channel.1.1.xml'), 'http://test.loc/rest11/c/categories.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/categories.xml'), 'http://test.loc/rest11/c/Default/packagesinfo.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/packagesinfo.xml'), @@ -50,11 +50,15 @@ class ChannelReaderTest extends TestCase public function testShouldSelectCorrectReader() { - $rfs = new RemoteFilesystemMock(array( + $rfs = new HttpDownloaderMock(array( 'http://pear.1.0.net/channel.xml' => file_get_contents(__DIR__ . '/Fixtures/channel.1.0.xml'), 'http://test.loc/rest10/p/packages.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/packages.xml'), 'http://test.loc/rest10/p/http_client/info.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_client_info.xml'), + 'http://test.loc/rest10/r/http_client/allreleases.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_client_allreleases.xml'), + 'http://test.loc/rest10/r/http_client/deps.1.2.1.txt' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_client_deps.1.2.1.txt'), 'http://test.loc/rest10/p/http_request/info.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_request_info.xml'), + 'http://test.loc/rest10/r/http_request/allreleases.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_request_allreleases.xml'), + 'http://test.loc/rest10/r/http_request/deps.1.4.0.txt' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_request_deps.1.4.0.txt'), 'http://pear.1.1.net/channel.xml' => file_get_contents(__DIR__ . '/Fixtures/channel.1.1.xml'), 'http://test.loc/rest11/c/categories.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/categories.xml'), 'http://test.loc/rest11/c/Default/packagesinfo.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/packagesinfo.xml'), diff --git a/tests/Composer/Test/Repository/Pear/ChannelRest10ReaderTest.php b/tests/Composer/Test/Repository/Pear/ChannelRest10ReaderTest.php index 4aa7bbba2..5a40915e1 100644 --- a/tests/Composer/Test/Repository/Pear/ChannelRest10ReaderTest.php +++ b/tests/Composer/Test/Repository/Pear/ChannelRest10ReaderTest.php @@ -13,13 +13,13 @@ namespace Composer\Test\Repository\Pear; use Composer\Test\TestCase; -use Composer\Test\Mock\RemoteFilesystemMock; +use Composer\Test\Mock\HttpDownloaderMock; class ChannelRest10ReaderTest extends TestCase { public function testShouldBuildPackagesFromPearSchema() { - $rfs = new RemoteFilesystemMock(array( + $rfs = new HttpDownloaderMock(array( 'http://test.loc/rest10/p/packages.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/packages.xml'), 'http://test.loc/rest10/p/http_client/info.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_client_info.xml'), 'http://test.loc/rest10/r/http_client/allreleases.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_client_allreleases.xml'), diff --git a/tests/Composer/Test/Repository/Pear/ChannelRest11ReaderTest.php b/tests/Composer/Test/Repository/Pear/ChannelRest11ReaderTest.php index 04e48426e..08c3a2998 100644 --- a/tests/Composer/Test/Repository/Pear/ChannelRest11ReaderTest.php +++ b/tests/Composer/Test/Repository/Pear/ChannelRest11ReaderTest.php @@ -13,13 +13,13 @@ namespace Composer\Test\Repository\Pear; use Composer\Test\TestCase; -use Composer\Test\Mock\RemoteFilesystemMock; +use Composer\Test\Mock\HttpDownloaderMock; class ChannelRest11ReaderTest extends TestCase { public function testShouldBuildPackagesFromPearSchema() { - $rfs = new RemoteFilesystemMock(array( + $rfs = new HttpDownloaderMock(array( 'http://pear.1.1.net/channel.xml' => file_get_contents(__DIR__ . '/Fixtures/channel.1.1.xml'), 'http://test.loc/rest11/c/categories.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/categories.xml'), 'http://test.loc/rest11/c/Default/packagesinfo.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/packagesinfo.xml'), diff --git a/tests/Composer/Test/Repository/PearRepositoryTest.php b/tests/Composer/Test/Repository/PearRepositoryTest.php index b1a3c0b5e..6046fefb4 100644 --- a/tests/Composer/Test/Repository/PearRepositoryTest.php +++ b/tests/Composer/Test/Repository/PearRepositoryTest.php @@ -133,7 +133,7 @@ class PearRepositoryTest extends TestCase $config = new \Composer\Config(); - $this->remoteFilesystem = $this->getMockBuilder('Composer\Util\RemoteFilesystem') + $this->httpDownloader = $this->getMockBuilder('Composer\Util\HttpDownloader') ->disableOriginalConstructor() ->getMock(); @@ -143,6 +143,6 @@ class PearRepositoryTest extends TestCase protected function tearDown() { $this->repository = null; - $this->remoteFilesystem = null; + $this->httpDownloader = null; } } diff --git a/tests/Composer/Test/Repository/RepositoryFactoryTest.php b/tests/Composer/Test/Repository/RepositoryFactoryTest.php index e54624415..e0a854d46 100644 --- a/tests/Composer/Test/Repository/RepositoryFactoryTest.php +++ b/tests/Composer/Test/Repository/RepositoryFactoryTest.php @@ -21,7 +21,9 @@ class RepositoryFactoryTest extends TestCase { $manager = RepositoryFactory::manager( $this->getMockBuilder('Composer\IO\IOInterface')->getMock(), - $this->getMockBuilder('Composer\Config')->getMock() + $this->getMockBuilder('Composer\Config')->getMock(), + $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock(), + $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock() ); $ref = new \ReflectionProperty($manager, 'repositoryClasses'); diff --git a/tests/Composer/Test/Repository/RepositoryManagerTest.php b/tests/Composer/Test/Repository/RepositoryManagerTest.php index 3774dd268..c4f09de87 100644 --- a/tests/Composer/Test/Repository/RepositoryManagerTest.php +++ b/tests/Composer/Test/Repository/RepositoryManagerTest.php @@ -38,7 +38,8 @@ class RepositoryManagerTest extends TestCase $rm = new RepositoryManager( $this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getMockBuilder('Composer\Config')->getMock(), - $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock() + $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock(), + $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock() ); $repository1 = $this->getMockBuilder('Composer\Repository\RepositoryInterface')->getMock(); @@ -61,7 +62,8 @@ class RepositoryManagerTest extends TestCase $rm = new RepositoryManager( $this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $config = $this->getMockBuilder('Composer\Config')->setMethods(array('get'))->getMock(), - $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock() + $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock(), + $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock() ); $tmpdir = $this->tmpdir; diff --git a/tests/Composer/Test/Repository/Vcs/GitBitbucketDriverTest.php b/tests/Composer/Test/Repository/Vcs/GitBitbucketDriverTest.php index 8d711e8f0..a3a9219d9 100644 --- a/tests/Composer/Test/Repository/Vcs/GitBitbucketDriverTest.php +++ b/tests/Composer/Test/Repository/Vcs/GitBitbucketDriverTest.php @@ -16,6 +16,7 @@ use Composer\Config; use Composer\Repository\Vcs\GitBitbucketDriver; use Composer\Test\TestCase; use Composer\Util\Filesystem; +use Composer\Util\Http\Response; /** * @group bitbucket @@ -26,8 +27,8 @@ class GitBitbucketDriverTest extends TestCase private $io; /** @type \Composer\Config */ private $config; - /** @type \Composer\Util\RemoteFilesystem|\PHPUnit_Framework_MockObject_MockObject */ - private $rfs; + /** @type \Composer\Util\HttpDownloader|\PHPUnit_Framework_MockObject_MockObject */ + private $httpDownloader; /** @type string */ private $home; /** @type string */ @@ -46,7 +47,7 @@ class GitBitbucketDriverTest extends TestCase ), )); - $this->rfs = $this->getMockBuilder('Composer\Util\RemoteFilesystem') + $this->httpDownloader = $this->getMockBuilder('Composer\Util\HttpDownloader') ->disableOriginalConstructor() ->getMock(); } @@ -68,7 +69,7 @@ class GitBitbucketDriverTest extends TestCase $this->io, $this->config, null, - $this->rfs + $this->httpDownloader ); $driver->initialize(); @@ -83,15 +84,14 @@ class GitBitbucketDriverTest extends TestCase 'https://bitbucket.org/user/repo.git does not appear to be a git repository, use https://bitbucket.org/user/repo if this is a mercurial bitbucket repository' ); - $this->rfs->expects($this->once()) - ->method('getContents') + $this->httpDownloader->expects($this->once()) + ->method('get') ->with( - $this->originUrl, - 'https://api.bitbucket.org/2.0/repositories/user/repo?fields=-project%2C-owner', - false + $url = 'https://api.bitbucket.org/2.0/repositories/user/repo?fields=-project%2C-owner', + array() ) ->willReturn( - '{"scm":"hg","website":"","has_wiki":false,"name":"repo","links":{"branches":{"href":"https:\/\/api.bitbucket.org\/2.0\/repositories\/user\/repo\/refs\/branches"},"tags":{"href":"https:\/\/api.bitbucket.org\/2.0\/repositories\/user\/repo\/refs\/tags"},"clone":[{"href":"https:\/\/user@bitbucket.org\/user\/repo","name":"https"},{"href":"ssh:\/\/hg@bitbucket.org\/user\/repo","name":"ssh"}],"html":{"href":"https:\/\/bitbucket.org\/user\/repo"}},"language":"php","created_on":"2015-02-18T16:22:24.688+00:00","updated_on":"2016-05-17T13:20:21.993+00:00","is_private":true,"has_issues":false}' + new Response(array('url' => $url), 200, array(), '{"scm":"hg","website":"","has_wiki":false,"name":"repo","links":{"branches":{"href":"https:\/\/api.bitbucket.org\/2.0\/repositories\/user\/repo\/refs\/branches"},"tags":{"href":"https:\/\/api.bitbucket.org\/2.0\/repositories\/user\/repo\/refs\/tags"},"clone":[{"href":"https:\/\/user@bitbucket.org\/user\/repo","name":"https"},{"href":"ssh:\/\/hg@bitbucket.org\/user\/repo","name":"ssh"}],"html":{"href":"https:\/\/bitbucket.org\/user\/repo"}},"language":"php","created_on":"2015-02-18T16:22:24.688+00:00","updated_on":"2016-05-17T13:20:21.993+00:00","is_private":true,"has_issues":false}') ); $driver = $this->getDriver(array('url' => 'https://bitbucket.org/user/repo.git')); @@ -103,47 +103,43 @@ class GitBitbucketDriverTest extends TestCase { $driver = $this->getDriver(array('url' => 'https://bitbucket.org/user/repo.git')); - $this->rfs->expects($this->any()) - ->method('getContents') + $urls = array( + 'https://api.bitbucket.org/2.0/repositories/user/repo?fields=-project%2C-owner', + 'https://api.bitbucket.org/2.0/repositories/user/repo?fields=mainbranch', + 'https://api.bitbucket.org/2.0/repositories/user/repo/refs/tags?pagelen=100&fields=values.name%2Cvalues.target.hash%2Cnext&sort=-target.date', + 'https://api.bitbucket.org/2.0/repositories/user/repo/refs/branches?pagelen=100&fields=values.name%2Cvalues.target.hash%2Cvalues.heads%2Cnext&sort=-target.date', + 'https://api.bitbucket.org/2.0/repositories/user/repo/src/master/composer.json', + 'https://api.bitbucket.org/2.0/repositories/user/repo/commit/master?fields=date', + ); + $this->httpDownloader->expects($this->any()) + ->method('get') ->withConsecutive( array( - $this->originUrl, - 'https://api.bitbucket.org/2.0/repositories/user/repo?fields=-project%2C-owner', - false, + $urls[0], array() ), array( - $this->originUrl, - 'https://api.bitbucket.org/2.0/repositories/user/repo?fields=mainbranch', - false, + $urls[1], array() ), array( - $this->originUrl, - 'https://api.bitbucket.org/2.0/repositories/user/repo/refs/tags?pagelen=100&fields=values.name%2Cvalues.target.hash%2Cnext&sort=-target.date', - false, + $urls[2], array() ), array( - $this->originUrl, - 'https://api.bitbucket.org/2.0/repositories/user/repo/refs/branches?pagelen=100&fields=values.name%2Cvalues.target.hash%2Cvalues.heads%2Cnext&sort=-target.date', - false, + $urls[3], array() ), array( - $this->originUrl, - 'https://api.bitbucket.org/2.0/repositories/user/repo/src/master/composer.json', - false, + $urls[4], array() ), array( - $this->originUrl, - 'https://api.bitbucket.org/2.0/repositories/user/repo/commit/master?fields=date', - false, + $urls[5], array() ) ) ->willReturnOnConsecutiveCalls( - '{"scm":"git","website":"","has_wiki":false,"name":"repo","links":{"branches":{"href":"https:\/\/api.bitbucket.org\/2.0\/repositories\/user\/repo\/refs\/branches"},"tags":{"href":"https:\/\/api.bitbucket.org\/2.0\/repositories\/user\/repo\/refs\/tags"},"clone":[{"href":"https:\/\/user@bitbucket.org\/user\/repo.git","name":"https"},{"href":"ssh:\/\/git@bitbucket.org\/user\/repo.git","name":"ssh"}],"html":{"href":"https:\/\/bitbucket.org\/user\/repo"}},"language":"php","created_on":"2015-02-18T16:22:24.688+00:00","updated_on":"2016-05-17T13:20:21.993+00:00","is_private":true,"has_issues":false}', - '{"mainbranch": {"name": "master"}}', - '{"values":[{"name":"1.0.1","target":{"hash":"9b78a3932143497c519e49b8241083838c8ff8a1"}},{"name":"1.0.0","target":{"hash":"d3393d514318a9267d2f8ebbf463a9aaa389f8eb"}}]}', - '{"values":[{"name":"master","target":{"hash":"937992d19d72b5116c3e8c4a04f960e5fa270b22"}}]}', - '{"name": "user/repo","description": "test repo","license": "GPL","authors": [{"name": "Name","email": "local@domain.tld"}],"require": {"creator/package": "^1.0"},"require-dev": {"phpunit/phpunit": "~4.8"}}', - '{"date": "2016-05-17T13:19:52+00:00"}' + new Response(array('url' => $urls[0]), 200, array(), '{"scm":"git","website":"","has_wiki":false,"name":"repo","links":{"branches":{"href":"https:\/\/api.bitbucket.org\/2.0\/repositories\/user\/repo\/refs\/branches"},"tags":{"href":"https:\/\/api.bitbucket.org\/2.0\/repositories\/user\/repo\/refs\/tags"},"clone":[{"href":"https:\/\/user@bitbucket.org\/user\/repo.git","name":"https"},{"href":"ssh:\/\/git@bitbucket.org\/user\/repo.git","name":"ssh"}],"html":{"href":"https:\/\/bitbucket.org\/user\/repo"}},"language":"php","created_on":"2015-02-18T16:22:24.688+00:00","updated_on":"2016-05-17T13:20:21.993+00:00","is_private":true,"has_issues":false}'), + new Response(array('url' => $urls[1]), 200, array(), '{"mainbranch": {"name": "master"}}'), + new Response(array('url' => $urls[2]), 200, array(), '{"values":[{"name":"1.0.1","target":{"hash":"9b78a3932143497c519e49b8241083838c8ff8a1"}},{"name":"1.0.0","target":{"hash":"d3393d514318a9267d2f8ebbf463a9aaa389f8eb"}}]}'), + new Response(array('url' => $urls[3]), 200, array(), '{"values":[{"name":"master","target":{"hash":"937992d19d72b5116c3e8c4a04f960e5fa270b22"}}]}'), + new Response(array('url' => $urls[4]), 200, array(), '{"name": "user/repo","description": "test repo","license": "GPL","authors": [{"name": "Name","email": "local@domain.tld"}],"require": {"creator/package": "^1.0"},"require-dev": {"phpunit/phpunit": "~4.8"}}'), + new Response(array('url' => $urls[5]), 200, array(), '{"date": "2016-05-17T13:19:52+00:00"}') ); $this->assertEquals( diff --git a/tests/Composer/Test/Repository/Vcs/GitHubDriverTest.php b/tests/Composer/Test/Repository/Vcs/GitHubDriverTest.php index ba9c6d4f7..1f721a4c7 100644 --- a/tests/Composer/Test/Repository/Vcs/GitHubDriverTest.php +++ b/tests/Composer/Test/Repository/Vcs/GitHubDriverTest.php @@ -16,6 +16,7 @@ use Composer\Downloader\TransportException; use Composer\Repository\Vcs\GitHubDriver; use Composer\Test\TestCase; use Composer\Util\Filesystem; +use Composer\Util\Http\Response; use Composer\Config; class GitHubDriverTest extends TestCase @@ -53,8 +54,8 @@ class GitHubDriverTest extends TestCase ->method('isInteractive') ->will($this->returnValue(true)); - $remoteFilesystem = $this->getMockBuilder('Composer\Util\RemoteFilesystem') - ->setConstructorArgs(array($io)) + $httpDownloader = $this->getMockBuilder('Composer\Util\HttpDownloader') + ->setConstructorArgs(array($io, $this->config)) ->getMock(); $process = $this->getMockBuilder('Composer\Util\ProcessExecutor')->getMock(); @@ -62,9 +63,9 @@ class GitHubDriverTest extends TestCase ->method('execute') ->will($this->returnValue(1)); - $remoteFilesystem->expects($this->at(0)) - ->method('getContents') - ->with($this->equalTo('github.com'), $this->equalTo($repoApiUrl), $this->equalTo(false)) + $httpDownloader->expects($this->at(0)) + ->method('get') + ->with($this->equalTo($repoApiUrl)) ->will($this->throwException(new TransportException('HTTP/1.1 404 Not Found', 404))); $io->expects($this->once()) @@ -76,15 +77,15 @@ class GitHubDriverTest extends TestCase ->method('setAuthentication') ->with($this->equalTo('github.com'), $this->matchesRegularExpression('{sometoken}'), $this->matchesRegularExpression('{x-oauth-basic}')); - $remoteFilesystem->expects($this->at(1)) - ->method('getContents') - ->with($this->equalTo('github.com'), $this->equalTo('https://api.github.com/'), $this->equalTo(false)) - ->will($this->returnValue('{}')); + $httpDownloader->expects($this->at(1)) + ->method('get') + ->with($this->equalTo($url = 'https://api.github.com/')) + ->will($this->returnValue(new Response(array('url' => $url), 200, array(), '{}'))); - $remoteFilesystem->expects($this->at(2)) - ->method('getContents') - ->with($this->equalTo('github.com'), $this->equalTo($repoApiUrl), $this->equalTo(false)) - ->will($this->returnValue('{"master_branch": "test_master", "private": true, "owner": {"login": "composer"}, "name": "packagist"}')); + $httpDownloader->expects($this->at(2)) + ->method('get') + ->with($this->equalTo($url = $repoApiUrl)) + ->will($this->returnValue(new Response(array('url' => $url), 200, array(), '{"master_branch": "test_master", "private": true, "owner": {"login": "composer"}, "name": "packagist"}'))); $configSource = $this->getMockBuilder('Composer\Config\ConfigSourceInterface')->getMock(); $authConfigSource = $this->getMockBuilder('Composer\Config\ConfigSourceInterface')->getMock(); @@ -95,7 +96,7 @@ class GitHubDriverTest extends TestCase 'url' => $repoUrl, ); - $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, $process, $remoteFilesystem); + $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, $process, $httpDownloader); $gitHubDriver->initialize(); $this->setAttribute($gitHubDriver, 'tags', array($identifier => $sha)); @@ -124,21 +125,21 @@ class GitHubDriverTest extends TestCase ->method('isInteractive') ->will($this->returnValue(true)); - $remoteFilesystem = $this->getMockBuilder('Composer\Util\RemoteFilesystem') - ->setConstructorArgs(array($io)) + $httpDownloader = $this->getMockBuilder('Composer\Util\HttpDownloader') + ->setConstructorArgs(array($io, $this->config)) ->getMock(); - $remoteFilesystem->expects($this->at(0)) - ->method('getContents') - ->with($this->equalTo('github.com'), $this->equalTo($repoApiUrl), $this->equalTo(false)) - ->will($this->returnValue('{"master_branch": "test_master", "owner": {"login": "composer"}, "name": "packagist"}')); + $httpDownloader->expects($this->at(0)) + ->method('get') + ->with($this->equalTo($repoApiUrl)) + ->will($this->returnValue(new Response(array('url' => $repoApiUrl), 200, array(), '{"master_branch": "test_master", "owner": {"login": "composer"}, "name": "packagist"}'))); $repoConfig = array( 'url' => $repoUrl, ); $repoUrl = 'https://github.com/composer/packagist.git'; - $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, null, $remoteFilesystem); + $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, null, $httpDownloader); $gitHubDriver->initialize(); $this->setAttribute($gitHubDriver, 'tags', array($identifier => $sha)); @@ -167,31 +168,31 @@ class GitHubDriverTest extends TestCase ->method('isInteractive') ->will($this->returnValue(true)); - $remoteFilesystem = $this->getMockBuilder('Composer\Util\RemoteFilesystem') - ->setConstructorArgs(array($io)) + $httpDownloader = $this->getMockBuilder('Composer\Util\HttpDownloader') + ->setConstructorArgs(array($io, $this->config)) ->getMock(); - $remoteFilesystem->expects($this->at(0)) - ->method('getContents') - ->with($this->equalTo('github.com'), $this->equalTo($repoApiUrl), $this->equalTo(false)) - ->will($this->returnValue('{"master_branch": "test_master", "owner": {"login": "composer"}, "name": "packagist"}')); + $httpDownloader->expects($this->at(0)) + ->method('get') + ->with($this->equalTo($url = $repoApiUrl)) + ->will($this->returnValue(new Response(array('url' => $url), 200, array(), '{"master_branch": "test_master", "owner": {"login": "composer"}, "name": "packagist"}'))); - $remoteFilesystem->expects($this->at(1)) - ->method('getContents') - ->with($this->equalTo('github.com'), $this->equalTo('https://api.github.com/repos/composer/packagist/contents/composer.json?ref=feature%2F3.2-foo'), $this->equalTo(false)) - ->will($this->returnValue('{"encoding":"base64","content":"'.base64_encode('{"support": {"source": "'.$repoUrl.'" }}').'"}')); + $httpDownloader->expects($this->at(1)) + ->method('get') + ->with($this->equalTo($url = 'https://api.github.com/repos/composer/packagist/contents/composer.json?ref=feature%2F3.2-foo')) + ->will($this->returnValue(new Response(array('url' => $url), 200, array(), '{"encoding":"base64","content":"'.base64_encode('{"support": {"source": "'.$repoUrl.'" }}').'"}'))); - $remoteFilesystem->expects($this->at(2)) - ->method('getContents') - ->with($this->equalTo('github.com'), $this->equalTo('https://api.github.com/repos/composer/packagist/commits/feature%2F3.2-foo'), $this->equalTo(false)) - ->will($this->returnValue('{"commit": {"committer":{ "date": "2012-09-10"}}}')); + $httpDownloader->expects($this->at(2)) + ->method('get') + ->with($this->equalTo($url = 'https://api.github.com/repos/composer/packagist/commits/feature%2F3.2-foo')) + ->will($this->returnValue(new Response(array('url' => $url), 200, array(), '{"commit": {"committer":{ "date": "2012-09-10"}}}'))); $repoConfig = array( 'url' => $repoUrl, ); $repoUrl = 'https://github.com/composer/packagist.git'; - $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, null, $remoteFilesystem); + $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, null, $httpDownloader); $gitHubDriver->initialize(); $this->setAttribute($gitHubDriver, 'tags', array($identifier => $sha)); @@ -227,13 +228,13 @@ class GitHubDriverTest extends TestCase ->method('isInteractive') ->will($this->returnValue(false)); - $remoteFilesystem = $this->getMockBuilder('Composer\Util\RemoteFilesystem') - ->setConstructorArgs(array($io)) + $httpDownloader = $this->getMockBuilder('Composer\Util\HttpDownloader') + ->setConstructorArgs(array($io, $this->config)) ->getMock(); - $remoteFilesystem->expects($this->at(0)) - ->method('getContents') - ->with($this->equalTo('github.com'), $this->equalTo($repoApiUrl), $this->equalTo(false)) + $httpDownloader->expects($this->at(0)) + ->method('get') + ->with($this->equalTo($repoApiUrl)) ->will($this->throwException(new TransportException('HTTP/1.1 404 Not Found', 404))); // clean local clone if present @@ -278,7 +279,7 @@ class GitHubDriverTest extends TestCase 'url' => $repoUrl, ); - $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, $process, $remoteFilesystem); + $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, $process, $httpDownloader); $gitHubDriver->initialize(); $this->assertEquals('test_master', $gitHubDriver->getRootIdentifier()); diff --git a/tests/Composer/Test/Repository/Vcs/GitLabDriverTest.php b/tests/Composer/Test/Repository/Vcs/GitLabDriverTest.php index a5eb799f2..f940733ae 100644 --- a/tests/Composer/Test/Repository/Vcs/GitLabDriverTest.php +++ b/tests/Composer/Test/Repository/Vcs/GitLabDriverTest.php @@ -17,6 +17,7 @@ use Composer\Config; use Composer\Test\TestCase; use Composer\Util\Filesystem; use Prophecy\Argument; +use Composer\Util\Http\Response; /** * @author Jérôme Tamarelle @@ -27,7 +28,7 @@ class GitLabDriverTest extends TestCase private $config; private $io; private $process; - private $remoteFilesystem; + private $httpDownloader; public function setUp() { @@ -47,7 +48,7 @@ class GitLabDriverTest extends TestCase $this->io = $this->prophesize('Composer\IO\IOInterface'); $this->process = $this->prophesize('Composer\Util\ProcessExecutor'); - $this->remoteFilesystem = $this->prophesize('Composer\Util\RemoteFilesystem'); + $this->httpDownloader = $this->prophesize('Composer\Util\HttpDownloader'); } public function tearDown() @@ -87,13 +88,11 @@ class GitLabDriverTest extends TestCase } JSON; - $this->remoteFilesystem - ->getContents('gitlab.com', $apiUrl, false, array()) - ->willReturn($projectData) + $this->mockResponse($apiUrl, array(), $projectData) ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->remoteFilesystem->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -126,13 +125,11 @@ JSON; } JSON; - $this->remoteFilesystem - ->getContents('gitlab.com', $apiUrl, false, array()) - ->willReturn($projectData) + $this->mockResponse($apiUrl, array(), $projectData) ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->remoteFilesystem->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -164,13 +161,11 @@ JSON; } JSON; - $this->remoteFilesystem - ->getContents('gitlab.com', $apiUrl, false, array()) - ->willReturn($projectData) + $this->mockResponse($apiUrl, array(), $projectData) ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->remoteFilesystem->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -206,12 +201,10 @@ JSON; } JSON; - $this->remoteFilesystem - ->getContents($domain, $apiUrl, false, array()) - ->willReturn(sprintf($projectData, $domain, $port, $namespace)) + $this->mockResponse($apiUrl, array(), sprintf($projectData, $domain, $port, $namespace)) ->shouldBeCalledTimes(1); - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->remoteFilesystem->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -289,15 +282,11 @@ JSON; ] JSON; - $this->remoteFilesystem - ->getContents('gitlab.com', $apiUrl, false, array()) - ->willReturn($tagData) + $this->mockResponse($apiUrl, array(), $tagData) ->shouldBeCalledTimes(1) ; - $this->remoteFilesystem->getLastHeaders() - ->willReturn(array()); - $driver->setRemoteFilesystem($this->remoteFilesystem->reveal()); + $driver->setHttpDownloader($this->httpDownloader->reveal()); $expected = array( 'v1.0.0' => '092ed2c762bbae331e3f51d4a17f67310bf99a81', @@ -344,26 +333,20 @@ JSON; $branchData = json_encode($branchData); - $this->remoteFilesystem - ->getContents('gitlab.com', $apiUrl, false, array()) - ->willReturn($branchData) - ->shouldBeCalledTimes(1) - ; + $headers = array('Link: ; rel="next", ; rel="first", ; rel="last"'); + $this->httpDownloader + ->get($apiUrl, array()) + ->willReturn(new Response(array('url' => $apiUrl), 200, $headers, $branchData)) + ->shouldBeCalledTimes(1); - $this->remoteFilesystem - ->getContents('gitlab.com', "http://gitlab.com/api/v4/projects/mygroup%2Fmyproject/repository/tags?id=mygroup%2Fmyproject&page=2&per_page=20", false, array()) - ->willReturn($branchData) - ->shouldBeCalledTimes(1) - ; + $apiUrl = "http://gitlab.com/api/v4/projects/mygroup%2Fmyproject/repository/tags?id=mygroup%2Fmyproject&page=2&per_page=20"; + $headers = array('Link: ; rel="prev", ; rel="first", ; rel="last"'); + $this->httpDownloader + ->get($apiUrl, array()) + ->willReturn(new Response(array('url' => $apiUrl), 200, $headers, $branchData)) + ->shouldBeCalledTimes(1); - $this->remoteFilesystem->getLastHeaders() - ->willReturn( - array('Link: ; rel="next", ; rel="first", ; rel="last"'), - array('Link: ; rel="prev", ; rel="first", ; rel="last"') - ) - ->shouldBeCalledTimes(2); - - $driver->setRemoteFilesystem($this->remoteFilesystem->reveal()); + $driver->setHttpDownloader($this->httpDownloader->reveal()); $expected = array( 'mymaster' => '97eda36b5c1dd953a3792865c222d4e85e5f302e', @@ -401,15 +384,11 @@ JSON; ] JSON; - $this->remoteFilesystem - ->getContents('gitlab.com', $apiUrl, false, array()) - ->willReturn($branchData) + $this->mockResponse($apiUrl, array(), $branchData) ->shouldBeCalledTimes(1) ; - $this->remoteFilesystem->getLastHeaders() - ->willReturn(array()); - $driver->setRemoteFilesystem($this->remoteFilesystem->reveal()); + $driver->setHttpDownloader($this->httpDownloader->reveal()); $expected = array( 'mymaster' => '97eda36b5c1dd953a3792865c222d4e85e5f302e', @@ -474,13 +453,11 @@ JSON; } JSON; - $this->remoteFilesystem - ->getContents('mycompany.com/gitlab', $apiUrl, false, array()) - ->willReturn($projectData) + $this->mockResponse($apiUrl, array(), $projectData) ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->remoteFilesystem->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -507,13 +484,11 @@ JSON; } JSON; - $this->remoteFilesystem - ->getContents('gitlab.com', $apiUrl, false, array()) - ->willReturn($projectData) + $this->mockResponse($apiUrl, array(), $projectData) ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->remoteFilesystem->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -540,13 +515,11 @@ JSON; } JSON; - $this->remoteFilesystem - ->getContents('mycompany.com/gitlab', $apiUrl, false, array()) - ->willReturn($projectData) + $this->mockResponse($apiUrl, array(), $projectData) ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->remoteFilesystem->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -575,9 +548,7 @@ JSON; } JSON; - $this->remoteFilesystem - ->getContents(Argument::cetera(), $options) - ->willReturn($projectData) + $this->mockResponse(Argument::cetera(), $options, $projectData) ->shouldBeCalled(); $driver = new GitLabDriver( @@ -585,8 +556,15 @@ JSON; $this->io->reveal(), $this->config, $this->process->reveal(), - $this->remoteFilesystem->reveal() + $this->httpDownloader->reveal() ); $driver->initialize(); } + + private function mockResponse($url, $options, $return) + { + return $this->httpDownloader + ->get($url, $options) + ->willReturn(new Response(array('url' => $url), 200, array(), $return)); + } } diff --git a/tests/Composer/Test/Repository/Vcs/PerforceDriverTest.php b/tests/Composer/Test/Repository/Vcs/PerforceDriverTest.php index a5e5d4b4c..8225a4a54 100644 --- a/tests/Composer/Test/Repository/Vcs/PerforceDriverTest.php +++ b/tests/Composer/Test/Repository/Vcs/PerforceDriverTest.php @@ -26,7 +26,7 @@ class PerforceDriverTest extends TestCase protected $config; protected $io; protected $process; - protected $remoteFileSystem; + protected $httpDownloader; protected $testPath; protected $driver; protected $repoConfig; @@ -43,9 +43,9 @@ class PerforceDriverTest extends TestCase $this->repoConfig = $this->getTestRepoConfig(); $this->io = $this->getMockIOInterface(); $this->process = $this->getMockProcessExecutor(); - $this->remoteFileSystem = $this->getMockRemoteFilesystem(); + $this->httpDownloader = $this->getMockHttpDownloader(); $this->perforce = $this->getMockPerforce(); - $this->driver = new PerforceDriver($this->repoConfig, $this->io, $this->config, $this->process, $this->remoteFileSystem); + $this->driver = new PerforceDriver($this->repoConfig, $this->io, $this->config, $this->process, $this->httpDownloader); $this->overrideDriverInternalPerforce($this->perforce); } @@ -56,7 +56,7 @@ class PerforceDriverTest extends TestCase $fs->removeDirectory($this->testPath); $this->driver = null; $this->perforce = null; - $this->remoteFileSystem = null; + $this->httpDownloader = null; $this->process = null; $this->io = null; $this->repoConfig = null; @@ -99,9 +99,9 @@ class PerforceDriverTest extends TestCase return $this->getMockBuilder('Composer\Util\ProcessExecutor')->getMock(); } - protected function getMockRemoteFilesystem() + protected function getMockHttpDownloader() { - return $this->getMockBuilder('Composer\Util\RemoteFilesystem')->disableOriginalConstructor()->getMock(); + return $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock(); } protected function getMockPerforce() @@ -113,7 +113,7 @@ class PerforceDriverTest extends TestCase public function testInitializeCapturesVariablesFromRepoConfig() { - $driver = new PerforceDriver($this->repoConfig, $this->io, $this->config, $this->process, $this->remoteFileSystem); + $driver = new PerforceDriver($this->repoConfig, $this->io, $this->config, $this->process, $this->httpDownloader); $driver->initialize(); $this->assertEquals(self::TEST_URL, $driver->getUrl()); $this->assertEquals(self::TEST_DEPOT, $driver->getDepot()); diff --git a/tests/Composer/Test/Util/BitbucketTest.php b/tests/Composer/Test/Util/BitbucketTest.php index 4323a15f5..f50bdd818 100644 --- a/tests/Composer/Test/Util/BitbucketTest.php +++ b/tests/Composer/Test/Util/BitbucketTest.php @@ -13,6 +13,7 @@ namespace Composer\Test\Util; use Composer\Util\Bitbucket; +use Composer\Util\Http\Response; use PHPUnit\Framework\TestCase; /** @@ -30,8 +31,8 @@ class BitbucketTest extends TestCase /** @type \Composer\IO\ConsoleIO|\PHPUnit_Framework_MockObject_MockObject */ private $io; - /** @type \Composer\Util\RemoteFilesystem|\PHPUnit_Framework_MockObject_MockObject */ - private $rfs; + /** @type \Composer\Util\HttpDownloader|\PHPUnit_Framework_MockObject_MockObject */ + private $httpDownloader; /** @type \Composer\Config|\PHPUnit_Framework_MockObject_MockObject */ private $config; /** @type Bitbucket */ @@ -47,8 +48,8 @@ class BitbucketTest extends TestCase ->getMock() ; - $this->rfs = $this - ->getMockBuilder('Composer\Util\RemoteFilesystem') + $this->httpDownloader = $this + ->getMockBuilder('Composer\Util\HttpDownloader') ->disableOriginalConstructor() ->getMock() ; @@ -57,7 +58,7 @@ class BitbucketTest extends TestCase $this->time = time(); - $this->bitbucket = new Bitbucket($this->io, $this->config, null, $this->rfs, $this->time); + $this->bitbucket = new Bitbucket($this->io, $this->config, null, $this->httpDownloader, $this->time); } public function testRequestAccessTokenWithValidOAuthConsumer() @@ -66,12 +67,10 @@ class BitbucketTest extends TestCase ->method('setAuthentication') ->with($this->origin, $this->consumer_key, $this->consumer_secret); - $this->rfs->expects($this->once()) - ->method('getContents') + $this->httpDownloader->expects($this->once()) + ->method('get') ->with( - $this->origin, Bitbucket::OAUTH2_ACCESS_TOKEN_URL, - false, array( 'retry-auth-failure' => false, 'http' => array( @@ -81,9 +80,14 @@ class BitbucketTest extends TestCase ) ) ->willReturn( - sprintf( - '{"access_token": "%s", "scopes": "repository", "expires_in": 3600, "refresh_token": "refreshtoken", "token_type": "bearer"}', - $this->token + new Response( + array('url' => Bitbucket::OAUTH2_ACCESS_TOKEN_URL), + 200, + array(), + sprintf( + '{"access_token": "%s", "scopes": "repository", "expires_in": 3600, "refresh_token": "refreshtoken", "token_type": "bearer"}', + $this->token + ) ) ); @@ -142,12 +146,10 @@ class BitbucketTest extends TestCase ->method('setAuthentication') ->with($this->origin, $this->consumer_key, $this->consumer_secret); - $this->rfs->expects($this->once()) - ->method('getContents') + $this->httpDownloader->expects($this->once()) + ->method('get') ->with( - $this->origin, Bitbucket::OAUTH2_ACCESS_TOKEN_URL, - false, array( 'retry-auth-failure' => false, 'http' => array( @@ -157,9 +159,14 @@ class BitbucketTest extends TestCase ) ) ->willReturn( - sprintf( - '{"access_token": "%s", "scopes": "repository", "expires_in": 3600, "refresh_token": "refreshtoken", "token_type": "bearer"}', - $this->token + new Response( + array('url' => Bitbucket::OAUTH2_ACCESS_TOKEN_URL), + 200, + array(), + sprintf( + '{"access_token": "%s", "scopes": "repository", "expires_in": 3600, "refresh_token": "refreshtoken", "token_type": "bearer"}', + $this->token + ) ) ); @@ -186,12 +193,10 @@ class BitbucketTest extends TestCase array('2. You are using an OAuth consumer, but didn\'t configure a (dummy) callback url') ); - $this->rfs->expects($this->once()) - ->method('getContents') + $this->httpDownloader->expects($this->once()) + ->method('get') ->with( - $this->origin, Bitbucket::OAUTH2_ACCESS_TOKEN_URL, - false, array( 'retry-auth-failure' => false, 'http' => array( @@ -234,21 +239,24 @@ class BitbucketTest extends TestCase ) ->willReturnOnConsecutiveCalls($this->consumer_key, $this->consumer_secret); - $this->rfs + $this->httpDownloader ->expects($this->once()) - ->method('getContents') + ->method('get') ->with( - $this->equalTo($this->origin), - $this->equalTo(sprintf('https://%s/site/oauth2/access_token', $this->origin)), - $this->isFalse(), + $this->equalTo($url = sprintf('https://%s/site/oauth2/access_token', $this->origin)), $this->anything() ) ->willReturn( - sprintf( - '{"access_token": "%s", "scopes": "repository", "expires_in": 3600, "refresh_token": "refresh_token", "token_type": "bearer"}', - $this->token + new Response( + array('url' => $url), + 200, + array(), + sprintf( + '{"access_token": "%s", "scopes": "repository", "expires_in": 3600, "refresh_token": "refresh_token", "token_type": "bearer"}', + $this->token + ) ) - ) + ); ; $this->setExpectationsForStoringAccessToken(true); diff --git a/tests/Composer/Test/Util/GitHubTest.php b/tests/Composer/Test/Util/GitHubTest.php index 28d00ce69..1893486e0 100644 --- a/tests/Composer/Test/Util/GitHubTest.php +++ b/tests/Composer/Test/Util/GitHubTest.php @@ -14,6 +14,7 @@ namespace Composer\Test\Util; use Composer\Downloader\TransportException; use Composer\Util\GitHub; +use Composer\Util\Http\Response; use PHPUnit\Framework\TestCase; use RecursiveArrayIterator; use RecursiveIteratorIterator; @@ -45,17 +46,15 @@ class GitHubTest extends TestCase ->willReturn($this->password) ; - $rfs = $this->getRemoteFilesystemMock(); - $rfs + $httpDownloader = $this->getHttpDownloaderMock(); + $httpDownloader ->expects($this->once()) - ->method('getContents') + ->method('get') ->with( - $this->equalTo($this->origin), - $this->equalTo(sprintf('https://api.%s/', $this->origin)), - $this->isFalse(), + $this->equalTo($url = sprintf('https://api.%s/', $this->origin)), $this->anything() ) - ->willReturn('{}') + ->willReturn(new Response(array('url' => $url), 200, array(), '{}')); ; $config = $this->getConfigMock(); @@ -70,7 +69,7 @@ class GitHubTest extends TestCase ->willReturn($this->getConfJsonMock()) ; - $github = new GitHub($io, $config, null, $rfs); + $github = new GitHub($io, $config, null, $httpDownloader); $this->assertTrue($github->authorizeOAuthInteractively($this->origin, $this->message)); } @@ -85,10 +84,10 @@ class GitHubTest extends TestCase ->willReturn($this->password) ; - $rfs = $this->getRemoteFilesystemMock(); - $rfs + $httpDownloader = $this->getHttpDownloaderMock(); + $httpDownloader ->expects($this->exactly(1)) - ->method('getContents') + ->method('get') ->will($this->throwException(new TransportException('', 401))) ; @@ -99,7 +98,7 @@ class GitHubTest extends TestCase ->willReturn($this->getAuthJsonMock()) ; - $github = new GitHub($io, $config, null, $rfs); + $github = new GitHub($io, $config, null, $httpDownloader); $this->assertFalse($github->authorizeOAuthInteractively($this->origin)); } @@ -120,15 +119,15 @@ class GitHubTest extends TestCase return $this->getMockBuilder('Composer\Config')->getMock(); } - private function getRemoteFilesystemMock() + private function getHttpDownloaderMock() { - $rfs = $this - ->getMockBuilder('Composer\Util\RemoteFilesystem') + $httpDownloader = $this + ->getMockBuilder('Composer\Util\HttpDownloader') ->disableOriginalConstructor() ->getMock() ; - return $rfs; + return $httpDownloader; } private function getAuthJsonMock() diff --git a/tests/Composer/Test/Util/GitLabTest.php b/tests/Composer/Test/Util/GitLabTest.php index 27f46b4ad..611b25256 100644 --- a/tests/Composer/Test/Util/GitLabTest.php +++ b/tests/Composer/Test/Util/GitLabTest.php @@ -14,6 +14,7 @@ namespace Composer\Test\Util; use Composer\Downloader\TransportException; use Composer\Util\GitLab; +use Composer\Util\Http\Response; use PHPUnit\Framework\TestCase; /** @@ -49,17 +50,15 @@ class GitLabTest extends TestCase ->willReturn($this->password) ; - $rfs = $this->getRemoteFilesystemMock(); - $rfs + $httpDownloader = $this->getHttpDownloaderMock(); + $httpDownloader ->expects($this->once()) - ->method('getContents') + ->method('get') ->with( - $this->equalTo($this->origin), - $this->equalTo(sprintf('http://%s/oauth/token', $this->origin)), - $this->isFalse(), + $this->equalTo($url = sprintf('http://%s/oauth/token', $this->origin)), $this->anything() ) - ->willReturn(sprintf('{"access_token": "%s", "token_type": "bearer", "expires_in": 7200}', $this->token)) + ->willReturn(new Response(array('url' => $url), 200, array(), sprintf('{"access_token": "%s", "token_type": "bearer", "expires_in": 7200}', $this->token))); ; $config = $this->getConfigMock(); @@ -69,7 +68,7 @@ class GitLabTest extends TestCase ->willReturn($this->getAuthJsonMock()) ; - $gitLab = new GitLab($io, $config, null, $rfs); + $gitLab = new GitLab($io, $config, null, $httpDownloader); $this->assertTrue($gitLab->authorizeOAuthInteractively('http', $this->origin, $this->message)); } @@ -94,10 +93,10 @@ class GitLabTest extends TestCase ->willReturn($this->password) ; - $rfs = $this->getRemoteFilesystemMock(); - $rfs + $httpDownloader = $this->getHttpDownloaderMock(); + $httpDownloader ->expects($this->exactly(5)) - ->method('getContents') + ->method('get') ->will($this->throwException(new TransportException('', 401))) ; @@ -108,7 +107,7 @@ class GitLabTest extends TestCase ->willReturn($this->getAuthJsonMock()) ; - $gitLab = new GitLab($io, $config, null, $rfs); + $gitLab = new GitLab($io, $config, null, $httpDownloader); $gitLab->authorizeOAuthInteractively('https', $this->origin); } @@ -129,15 +128,15 @@ class GitLabTest extends TestCase return $this->getMockBuilder('Composer\Config')->getMock(); } - private function getRemoteFilesystemMock() + private function getHttpDownloaderMock() { - $rfs = $this - ->getMockBuilder('Composer\Util\RemoteFilesystem') + $httpDownloader = $this + ->getMockBuilder('Composer\Util\HttpDownloader') ->disableOriginalConstructor() ->getMock() ; - return $rfs; + return $httpDownloader; } private function getAuthJsonMock() From f946d8eb5a0104d3aefbd5b5fb8803355f482abf Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Mon, 12 Nov 2018 15:34:54 +0100 Subject: [PATCH 03/34] More RemoteFilesystem usage removals and some repository/vcs driver refactorings --- doc/articles/plugins.md | 2 +- doc/articles/scripts.md | 2 +- src/Composer/Command/DiagnoseCommand.php | 29 ++++++++-------- src/Composer/Command/SelfUpdateCommand.php | 8 ++--- src/Composer/Compiler.php | 1 + src/Composer/Factory.php | 12 +++---- src/Composer/Json/JsonFile.php | 20 +++++------ .../Package/Version/VersionGuesser.php | 3 +- .../Repository/ComposerRepository.php | 33 ++++++++++++------- src/Composer/Repository/PearRepository.php | 2 +- src/Composer/Repository/RepositoryFactory.php | 8 ++--- src/Composer/Repository/RepositoryManager.php | 8 ++--- .../Repository/Vcs/GitBitbucketDriver.php | 4 +-- src/Composer/Repository/Vcs/GitHubDriver.php | 4 +-- src/Composer/Repository/Vcs/GitLabDriver.php | 4 +-- .../Repository/Vcs/HgBitbucketDriver.php | 4 +-- src/Composer/Repository/Vcs/VcsDriver.php | 8 ++--- src/Composer/Repository/VcsRepository.php | 14 +++++--- src/Composer/SelfUpdate/Versions.php | 10 +++--- src/Composer/Util/HttpDownloader.php | 20 +++++++++++ .../Downloader/PerforceDownloaderTest.php | 3 +- tests/Composer/Test/InstallerTest.php | 2 +- .../Repository/ComposerRepositoryTest.php | 6 ++-- .../Test/Repository/PearRepositoryTest.php | 2 +- .../Test/Repository/RepositoryManagerTest.php | 8 ++--- .../Repository/Vcs/GitBitbucketDriverTest.php | 5 +-- .../Test/Repository/Vcs/GitHubDriverTest.php | 16 ++++++--- .../Test/Repository/Vcs/GitLabDriverTest.php | 18 +++++----- .../Repository/Vcs/PerforceDriverTest.php | 4 +-- .../Test/Repository/Vcs/SvnDriverTest.php | 3 +- 30 files changed, 156 insertions(+), 107 deletions(-) diff --git a/doc/articles/plugins.md b/doc/articles/plugins.md index 59e2a2f15..da20193f6 100644 --- a/doc/articles/plugins.md +++ b/doc/articles/plugins.md @@ -176,7 +176,7 @@ class AwsPlugin implements PluginInterface, EventSubscriberInterface if ($protocol === 's3') { $awsClient = new AwsClient($this->io, $this->composer->getConfig()); - $s3Downloader = new S3Downloader($this->io, $event->getRemoteFilesystem()->getOptions(), $awsClient); + $s3Downloader = new S3Downloader($this->io, $event->getHttpDownloader()->getOptions(), $awsClient); $event->setHttpdownloader($s3Downloader); } } diff --git a/doc/articles/scripts.md b/doc/articles/scripts.md index e0c27b10f..17c83c373 100644 --- a/doc/articles/scripts.md +++ b/doc/articles/scripts.md @@ -61,7 +61,7 @@ Composer fires the following named events during its execution process: - **command**: occurs before any Composer Command is executed on the CLI. It provides you with access to the input and output objects of the program. - **pre-file-download**: occurs before files are downloaded and allows - you to manipulate the `RemoteFilesystem` object prior to downloading files + you to manipulate the `HttpDownloader` object prior to downloading files based on the URL to be downloaded. - **pre-command-run**: occurs before a command is executed and allows you to manipulate the `InputInterface` object's options and arguments to tweak diff --git a/src/Composer/Command/DiagnoseCommand.php b/src/Composer/Command/DiagnoseCommand.php index 3c4c3bb32..481d58060 100644 --- a/src/Composer/Command/DiagnoseCommand.php +++ b/src/Composer/Command/DiagnoseCommand.php @@ -22,7 +22,7 @@ use Composer\Plugin\PluginEvents; use Composer\Util\ConfigValidator; use Composer\Util\IniHelper; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Util\StreamContextFactory; use Composer\SelfUpdate\Keys; use Composer\SelfUpdate\Versions; @@ -35,8 +35,8 @@ use Symfony\Component\Console\Output\OutputInterface; */ class DiagnoseCommand extends BaseCommand { - /** @var RemoteFilesystem */ - protected $rfs; + /** @var HttpDownloader */ + protected $httpDownloader; /** @var ProcessExecutor */ protected $process; @@ -85,7 +85,7 @@ EOT $config->merge(array('config' => array('secure-http' => false))); $config->prohibitUrlByConfig('http://repo.packagist.org', new NullIO); - $this->rfs = Factory::createRemoteFilesystem($io, $config); + $this->httpDownloader = Factory::createHttpDownloader($io, $config); $this->process = new ProcessExecutor($io); $io->write('Checking platform settings: ', false); @@ -226,7 +226,7 @@ EOT } try { - $this->rfs->getContents('packagist.org', $proto . '://repo.packagist.org/packages.json', false); + $this->httpDownloader->get($proto . '://repo.packagist.org/packages.json'); } catch (TransportException $e) { if (false !== strpos($e->getMessage(), 'cafile')) { $result[] = '[' . get_class($e) . '] ' . $e->getMessage() . ''; @@ -253,11 +253,11 @@ EOT $protocol = extension_loaded('openssl') ? 'https' : 'http'; try { - $json = json_decode($this->rfs->getContents('packagist.org', $protocol . '://repo.packagist.org/packages.json', false), true); + $json = $this->httpDownloader->get($protocol . '://repo.packagist.org/packages.json')->parseJson(); $hash = reset($json['provider-includes']); $hash = $hash['sha256']; $path = str_replace('%hash%', $hash, key($json['provider-includes'])); - $provider = $this->rfs->getContents('packagist.org', $protocol . '://repo.packagist.org/'.$path, false); + $provider = $this->httpDownloader->get($protocol . '://repo.packagist.org/'.$path)->getBody(); if (hash('sha256', $provider) !== $hash) { return 'It seems that your proxy is modifying http traffic on the fly'; @@ -285,10 +285,10 @@ EOT $url = 'http://repo.packagist.org/packages.json'; try { - $this->rfs->getContents('packagist.org', $url, false); + $this->httpDownloader->get($url); } catch (TransportException $e) { try { - $this->rfs->getContents('packagist.org', $url, false, array('http' => array('request_fulluri' => false))); + $this->httpDownloader->get($url, array('http' => array('request_fulluri' => false))); } catch (TransportException $e) { return 'Unable to assess the situation, maybe packagist.org is down ('.$e->getMessage().')'; } @@ -319,10 +319,10 @@ EOT $url = 'https://api.github.com/repos/Seldaek/jsonlint/zipball/1.0.0'; try { - $this->rfs->getContents('github.com', $url, false); + $this->httpDownloader->get($url); } catch (TransportException $e) { try { - $this->rfs->getContents('github.com', $url, false, array('http' => array('request_fulluri' => false))); + $this->httpDownloader->get($url, array('http' => array('request_fulluri' => false))); } catch (TransportException $e) { return 'Unable to assess the situation, maybe github is down ('.$e->getMessage().')'; } @@ -344,7 +344,7 @@ EOT try { $url = $domain === 'github.com' ? 'https://api.'.$domain.'/' : 'https://'.$domain.'/api/v3/'; - return $this->rfs->getContents($domain, $url, false, array( + return $this->httpDownloader->get($url, array( 'retry-auth-failure' => false, )) ? true : 'Unexpected error'; } catch (\Exception $e) { @@ -374,8 +374,7 @@ EOT } $url = $domain === 'github.com' ? 'https://api.'.$domain.'/rate_limit' : 'https://'.$domain.'/api/rate_limit'; - $json = $this->rfs->getContents($domain, $url, false, array('retry-auth-failure' => false)); - $data = json_decode($json, true); + $data = $this->httpDownloader->get($url, array('retry-auth-failure' => false))->parseJson(); return $data['resources']['core']; } @@ -428,7 +427,7 @@ EOT return $result; } - $versionsUtil = new Versions($config, $this->rfs); + $versionsUtil = new Versions($config, $this->httpDownloader); $latest = $versionsUtil->getLatest(); if (Composer::VERSION !== $latest['version'] && Composer::VERSION !== '@package_version@') { diff --git a/src/Composer/Command/SelfUpdateCommand.php b/src/Composer/Command/SelfUpdateCommand.php index 243755963..903b49d94 100644 --- a/src/Composer/Command/SelfUpdateCommand.php +++ b/src/Composer/Command/SelfUpdateCommand.php @@ -76,9 +76,9 @@ EOT } $io = $this->getIO(); - $remoteFilesystem = Factory::createRemoteFilesystem($io, $config); + $httpDownloader = Factory::createHttpDownloader($io, $config); - $versionsUtil = new Versions($config, $remoteFilesystem); + $versionsUtil = new Versions($config, $httpDownloader); // switch channel if requested foreach (array('stable', 'preview', 'snapshot') as $channel) { @@ -155,9 +155,9 @@ EOT $io->write(sprintf("Updating to version %s (%s channel).", $updateVersion, $versionsUtil->getChannel())); $remoteFilename = $baseUrl . ($updatingToTag ? "/download/{$updateVersion}/composer.phar" : '/composer.phar'); - $signature = $remoteFilesystem->getContents(self::HOMEPAGE, $remoteFilename.'.sig', false); + $signature = $httpDownloader->get($remoteFilename.'.sig')->getBody(); $io->writeError(' ', false); - $remoteFilesystem->copy(self::HOMEPAGE, $remoteFilename, $tempFilename, !$input->getOption('no-progress')); + $httpDownloader->copy($remoteFilename, $tempFilename); $io->writeError(''); if (!file_exists($tempFilename) || !$signature) { diff --git a/src/Composer/Compiler.php b/src/Composer/Compiler.php index 27b1f4816..86be2d7db 100644 --- a/src/Composer/Compiler.php +++ b/src/Composer/Compiler.php @@ -123,6 +123,7 @@ class Compiler ->in(__DIR__.'/../../vendor/composer/ca-bundle/') ->in(__DIR__.'/../../vendor/composer/xdebug-handler/') ->in(__DIR__.'/../../vendor/psr/') + ->in(__DIR__.'/../../vendor/react/') ->sort($finderSort) ; diff --git a/src/Composer/Factory.php b/src/Composer/Factory.php index 00aa499d0..c96fd2188 100644 --- a/src/Composer/Factory.php +++ b/src/Composer/Factory.php @@ -590,18 +590,18 @@ class Factory throw new Exception\NoSslException('The openssl extension is required for SSL/TLS protection but is not available. ' . 'If you can not enable the openssl extension, you can disable this error, at your own risk, by setting the \'disable-tls\' option to true.'); } - $remoteFilesystemOptions = array(); + $httpDownloaderOptions = array(); if ($disableTls === false) { if ($config && $config->get('cafile')) { - $remoteFilesystemOptions['ssl']['cafile'] = $config->get('cafile'); + $httpDownloaderOptions['ssl']['cafile'] = $config->get('cafile'); } if ($config && $config->get('capath')) { - $remoteFilesystemOptions['ssl']['capath'] = $config->get('capath'); + $httpDownloaderOptions['ssl']['capath'] = $config->get('capath'); } - $remoteFilesystemOptions = array_replace_recursive($remoteFilesystemOptions, $options); + $httpDownloaderOptions = array_replace_recursive($httpDownloaderOptions, $options); } try { - $remoteFilesystem = new HttpDownloader($io, $config, $remoteFilesystemOptions, $disableTls); + $httpDownloader = new HttpDownloader($io, $config, $httpDownloaderOptions, $disableTls); } catch (TransportException $e) { if (false !== strpos($e->getMessage(), 'cafile')) { $io->write('Unable to locate a valid CA certificate file. You must set a valid \'cafile\' option.'); @@ -614,7 +614,7 @@ class Factory throw $e; } - return $remoteFilesystem; + return $httpDownloader; } /** diff --git a/src/Composer/Json/JsonFile.php b/src/Composer/Json/JsonFile.php index b84791420..a61a75c34 100644 --- a/src/Composer/Json/JsonFile.php +++ b/src/Composer/Json/JsonFile.php @@ -15,7 +15,7 @@ namespace Composer\Json; use JsonSchema\Validator; use Seld\JsonLint\JsonParser; use Seld\JsonLint\ParsingException; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; use Composer\Downloader\TransportException; @@ -35,25 +35,25 @@ class JsonFile const JSON_UNESCAPED_UNICODE = 256; private $path; - private $rfs; + private $httpDownloader; private $io; /** * Initializes json file reader/parser. * - * @param string $path path to a lockfile - * @param RemoteFilesystem $rfs required for loading http/https json files + * @param string $path path to a lockfile + * @param HttpDownloader $httpDownloader required for loading http/https json files * @param IOInterface $io * @throws \InvalidArgumentException */ - public function __construct($path, RemoteFilesystem $rfs = null, IOInterface $io = null) + public function __construct($path, HttpDownloader $httpDownloader = null, IOInterface $io = null) { $this->path = $path; - if (null === $rfs && preg_match('{^https?://}i', $path)) { - throw new \InvalidArgumentException('http urls require a RemoteFilesystem instance to be passed'); + if (null === $httpDownloader && preg_match('{^https?://}i', $path)) { + throw new \InvalidArgumentException('http urls require a HttpDownloader instance to be passed'); } - $this->rfs = $rfs; + $this->httpDownloader = $httpDownloader; $this->io = $io; } @@ -84,8 +84,8 @@ class JsonFile public function read() { try { - if ($this->rfs) { - $json = $this->rfs->getContents($this->path, $this->path, false); + if ($this->httpDownloader) { + $json = $this->httpDownloader->get($this->path)->getBody(); } else { if ($this->io && $this->io->isDebug()) { $this->io->writeError('Reading ' . $this->path); diff --git a/src/Composer/Package/Version/VersionGuesser.php b/src/Composer/Package/Version/VersionGuesser.php index e6ff84965..1c2fdf986 100644 --- a/src/Composer/Package/Version/VersionGuesser.php +++ b/src/Composer/Package/Version/VersionGuesser.php @@ -192,7 +192,8 @@ class VersionGuesser } // re-use the HgDriver to fetch branches (this properly includes bookmarks) - $driver = new HgDriver(array('url' => $path), new NullIO(), $this->config, $this->process); + $io = new NullIO(); + $driver = new HgDriver(array('url' => $path), $io, $this->config, new HttpDownloader($io, $this->config), $this->process); $branches = array_keys($driver->getBranches()); // try to find the best (nearest) version branch to assume this feature's version diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index a0036e4fd..b337ea106 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -62,7 +62,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito private $partialPackagesByName; private $versionParser; - public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $eventDispatcher, HttpDownloader $httpDownloader) + public function __construct(array $repoConfig, IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $eventDispatcher = null) { parent::__construct(); if (!preg_match('{^[\w.]+\??://}', $repoConfig['url'])) { @@ -101,7 +101,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->cache = new Cache($io, $config->get('cache-repo-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', $this->url), 'a-z0-9.$'); $this->versionParser = new VersionParser(); $this->loader = new ArrayLoader($this->versionParser); - if ($httpDownloader && $this->options) { + if ($this->options) { // TODO solve this somehow - should be sent at request time not on the instance $httpDownloader = clone $httpDownloader; $httpDownloader->setOptions($this->options); @@ -781,10 +781,13 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $retries = 3; while ($retries--) { try { - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); - $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); + $httpDownloader = $this->httpDownloader; - $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); + if ($this->eventDispatcher) { + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); + $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); + } $response = $httpDownloader->get($filename); $json = $response->getBody(); @@ -869,10 +872,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $retries = 3; while ($retries--) { try { - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); - $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); + $httpDownloader = $this->httpDownloader; + + if ($this->eventDispatcher) { + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); + $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); + } - $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); $options = array('http' => array('header' => array('If-Modified-Since: '.$lastModifiedTime))); $response = $httpDownloader->get($filename, $options); $json = $response->getBody(); @@ -925,10 +932,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito protected function asyncFetchFile($filename, $cacheKey, $lastModifiedTime = null) { $retries = 3; - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); - $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); + $httpDownloader = $this->httpDownloader; + + if ($this->eventDispatcher) { + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); + $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); + } - $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); $options = $lastModifiedTime ? array('http' => array('header' => array('If-Modified-Since: '.$lastModifiedTime))) : array(); $io = $this->io; diff --git a/src/Composer/Repository/PearRepository.php b/src/Composer/Repository/PearRepository.php index aef5c0381..1bb22c0ed 100644 --- a/src/Composer/Repository/PearRepository.php +++ b/src/Composer/Repository/PearRepository.php @@ -47,7 +47,7 @@ class PearRepository extends ArrayRepository implements ConfigurableRepositoryIn */ private $vendorAlias; - public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $dispatcher, HttpDownloader $httpDownloader) + public function __construct(array $repoConfig, IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $dispatcher = null) { parent::__construct(); if (!preg_match('{^https?://}', $repoConfig['url'])) { diff --git a/src/Composer/Repository/RepositoryFactory.php b/src/Composer/Repository/RepositoryFactory.php index 515908f64..9508f5886 100644 --- a/src/Composer/Repository/RepositoryFactory.php +++ b/src/Composer/Repository/RepositoryFactory.php @@ -36,7 +36,7 @@ class RepositoryFactory if (0 === strpos($repository, 'http')) { $repoConfig = array('type' => 'composer', 'url' => $repository); } elseif ("json" === pathinfo($repository, PATHINFO_EXTENSION)) { - $json = new JsonFile($repository, Factory::createRemoteFilesystem($io, $config)); + $json = new JsonFile($repository, Factory::createHttpDownloader($io, $config)); $data = $json->read(); if (!empty($data['packages']) || !empty($data['includes']) || !empty($data['provider-includes'])) { $repoConfig = array('type' => 'composer', 'url' => 'file://' . strtr(realpath($repository), '\\', '/')); @@ -77,7 +77,7 @@ class RepositoryFactory */ public static function createRepo(IOInterface $io, Config $config, array $repoConfig) { - $rm = static::manager($io, $config, null, Factory::createRemoteFilesystem($io, $config)); + $rm = static::manager($io, $config, Factory::createHttpDownloader($io, $config)); $repos = static::createRepos($rm, array($repoConfig)); return reset($repos); @@ -98,7 +98,7 @@ class RepositoryFactory if (!$io) { throw new \InvalidArgumentException('This function requires either an IOInterface or a RepositoryManager'); } - $rm = static::manager($io, $config, null, Factory::createRemoteFilesystem($io, $config)); + $rm = static::manager($io, $config, Factory::createHttpDownloader($io, $config)); } return static::createRepos($rm, $config->getRepositories()); @@ -113,7 +113,7 @@ class RepositoryFactory */ public static function manager(IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $eventDispatcher = null) { - $rm = new RepositoryManager($io, $config, $eventDispatcher, $httpDownloader); + $rm = new RepositoryManager($io, $config, $httpDownloader, $eventDispatcher); $rm->setRepositoryClass('composer', 'Composer\Repository\ComposerRepository'); $rm->setRepositoryClass('vcs', 'Composer\Repository\VcsRepository'); $rm->setRepositoryClass('package', 'Composer\Repository\PackageRepository'); diff --git a/src/Composer/Repository/RepositoryManager.php b/src/Composer/Repository/RepositoryManager.php index 8fc01cb08..c3ce0c24a 100644 --- a/src/Composer/Repository/RepositoryManager.php +++ b/src/Composer/Repository/RepositoryManager.php @@ -35,12 +35,12 @@ class RepositoryManager private $eventDispatcher; private $httpDownloader; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher, HttpDownloader $httpDownloader) + public function __construct(IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $eventDispatcher = null) { $this->io = $io; $this->config = $config; - $this->eventDispatcher = $eventDispatcher; $this->httpDownloader = $httpDownloader; + $this->eventDispatcher = $eventDispatcher; } /** @@ -127,8 +127,8 @@ class RepositoryManager $reflMethod = new \ReflectionMethod($class, '__construct'); $params = $reflMethod->getParameters(); - if (isset($params[4]) && $params[4]->getClass() && $params[4]->getClass()->getName() === 'Composer\Util\HttpDownloader') { - return new $class($config, $this->io, $this->config, $this->eventDispatcher, $this->httpDownloader); + if (isset($params[3]) && $params[3]->getClass() && $params[3]->getClass()->getName() === 'Composer\Util\HttpDownloader') { + return new $class($config, $this->io, $this->config, $this->httpDownloader, $this->eventDispatcher); } return new $class($config, $this->io, $this->config, $this->eventDispatcher); diff --git a/src/Composer/Repository/Vcs/GitBitbucketDriver.php b/src/Composer/Repository/Vcs/GitBitbucketDriver.php index dd69e753a..5770a8326 100644 --- a/src/Composer/Repository/Vcs/GitBitbucketDriver.php +++ b/src/Composer/Repository/Vcs/GitBitbucketDriver.php @@ -75,8 +75,8 @@ class GitBitbucketDriver extends BitbucketDriver array('url' => $url), $this->io, $this->config, - $this->process, - $this->httpDownloader + $this->httpDownloader, + $this->process ); $this->fallbackDriver->initialize(); } diff --git a/src/Composer/Repository/Vcs/GitHubDriver.php b/src/Composer/Repository/Vcs/GitHubDriver.php index f1ad253d8..69eef200d 100644 --- a/src/Composer/Repository/Vcs/GitHubDriver.php +++ b/src/Composer/Repository/Vcs/GitHubDriver.php @@ -457,8 +457,8 @@ class GitHubDriver extends VcsDriver array('url' => $url), $this->io, $this->config, - $this->process, - $this->httpDownloader + $this->httpDownloader, + $this->process ); $this->gitDriver->initialize(); } diff --git a/src/Composer/Repository/Vcs/GitLabDriver.php b/src/Composer/Repository/Vcs/GitLabDriver.php index 6a1aa8ac2..1e2775ff7 100644 --- a/src/Composer/Repository/Vcs/GitLabDriver.php +++ b/src/Composer/Repository/Vcs/GitLabDriver.php @@ -376,8 +376,8 @@ class GitLabDriver extends VcsDriver array('url' => $url), $this->io, $this->config, - $this->process, - $this->httpDownloader + $this->httpDownloader, + $this->process ); $this->gitDriver->initialize(); } diff --git a/src/Composer/Repository/Vcs/HgBitbucketDriver.php b/src/Composer/Repository/Vcs/HgBitbucketDriver.php index 4a00f2da0..a919e7860 100644 --- a/src/Composer/Repository/Vcs/HgBitbucketDriver.php +++ b/src/Composer/Repository/Vcs/HgBitbucketDriver.php @@ -75,8 +75,8 @@ class HgBitbucketDriver extends BitbucketDriver array('url' => $url), $this->io, $this->config, - $this->process, - $this->httpDownloader + $this->httpDownloader, + $this->process ); $this->fallbackDriver->initialize(); } diff --git a/src/Composer/Repository/Vcs/VcsDriver.php b/src/Composer/Repository/Vcs/VcsDriver.php index 17ed706d2..37946da23 100644 --- a/src/Composer/Repository/Vcs/VcsDriver.php +++ b/src/Composer/Repository/Vcs/VcsDriver.php @@ -55,10 +55,10 @@ abstract class VcsDriver implements VcsDriverInterface * @param array $repoConfig The repository configuration * @param IOInterface $io The IO instance * @param Config $config The composer configuration + * @param HttpDownloader $httpDownloader Remote Filesystem, injectable for mocking * @param ProcessExecutor $process Process instance, injectable for mocking - * @param HttpDownloader $httpDownloader Remote Filesystem, injectable for mocking */ - final public function __construct(array $repoConfig, IOInterface $io, Config $config, ProcessExecutor $process = null, HttpDownloader $httpDownloader = null) + final public function __construct(array $repoConfig, IOInterface $io, Config $config, HttpDownloader $httpDownloader, ProcessExecutor $process) { if (Filesystem::isLocalPath($repoConfig['url'])) { $repoConfig['url'] = Filesystem::getPlatformPath($repoConfig['url']); @@ -69,8 +69,8 @@ abstract class VcsDriver implements VcsDriverInterface $this->repoConfig = $repoConfig; $this->io = $io; $this->config = $config; - $this->process = $process ?: new ProcessExecutor($io); - $this->httpDownloader = $httpDownloader ?: Factory::createHttpDownloader($this->io, $config); + $this->httpDownloader = $httpDownloader; + $this->process = $process; } /** diff --git a/src/Composer/Repository/VcsRepository.php b/src/Composer/Repository/VcsRepository.php index d6fb1bbee..d8e4b1501 100644 --- a/src/Composer/Repository/VcsRepository.php +++ b/src/Composer/Repository/VcsRepository.php @@ -20,6 +20,8 @@ use Composer\Package\Loader\ValidatingArrayLoader; use Composer\Package\Loader\InvalidPackageException; use Composer\Package\Loader\LoaderInterface; use Composer\EventDispatcher\EventDispatcher; +use Composer\Util\ProcessExecutor; +use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; use Composer\Config; @@ -37,6 +39,8 @@ class VcsRepository extends ArrayRepository implements ConfigurableRepositoryInt protected $type; protected $loader; protected $repoConfig; + protected $httpDownloader; + protected $processExecutor; protected $branchErrorOccurred = false; private $drivers; /** @var VcsDriverInterface */ @@ -44,7 +48,7 @@ class VcsRepository extends ArrayRepository implements ConfigurableRepositoryInt /** @var VersionCacheInterface */ private $versionCache; - public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $dispatcher = null, array $drivers = null, VersionCacheInterface $versionCache = null) + public function __construct(array $repoConfig, IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $dispatcher = null, array $drivers = null, VersionCacheInterface $versionCache = null) { parent::__construct(); $this->drivers = $drivers ?: array( @@ -67,6 +71,8 @@ class VcsRepository extends ArrayRepository implements ConfigurableRepositoryInt $this->config = $config; $this->repoConfig = $repoConfig; $this->versionCache = $versionCache; + $this->httpDownloader = $httpDownloader; + $this->processExecutor = new ProcessExecutor($io); } public function getRepoConfig() @@ -87,7 +93,7 @@ class VcsRepository extends ArrayRepository implements ConfigurableRepositoryInt if (isset($this->drivers[$this->type])) { $class = $this->drivers[$this->type]; - $this->driver = new $class($this->repoConfig, $this->io, $this->config); + $this->driver = new $class($this->repoConfig, $this->io, $this->config, $this->httpDownloader, $this->processExecutor); $this->driver->initialize(); return $this->driver; @@ -95,7 +101,7 @@ class VcsRepository extends ArrayRepository implements ConfigurableRepositoryInt foreach ($this->drivers as $driver) { if ($driver::supports($this->io, $this->config, $this->url)) { - $this->driver = new $driver($this->repoConfig, $this->io, $this->config); + $this->driver = new $driver($this->repoConfig, $this->io, $this->config, $this->httpDownloader, $this->processExecutor); $this->driver->initialize(); return $this->driver; @@ -104,7 +110,7 @@ class VcsRepository extends ArrayRepository implements ConfigurableRepositoryInt foreach ($this->drivers as $driver) { if ($driver::supports($this->io, $this->config, $this->url, true)) { - $this->driver = new $driver($this->repoConfig, $this->io, $this->config); + $this->driver = new $driver($this->repoConfig, $this->io, $this->config, $this->httpDownloader, $this->processExecutor); $this->driver->initialize(); return $this->driver; diff --git a/src/Composer/SelfUpdate/Versions.php b/src/Composer/SelfUpdate/Versions.php index b619bda16..431abecb5 100644 --- a/src/Composer/SelfUpdate/Versions.php +++ b/src/Composer/SelfUpdate/Versions.php @@ -12,7 +12,7 @@ namespace Composer\SelfUpdate; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Config; use Composer\Json\JsonFile; @@ -21,13 +21,13 @@ use Composer\Json\JsonFile; */ class Versions { - private $rfs; + private $httpDownloader; private $config; private $channel; - public function __construct(Config $config, RemoteFilesystem $rfs) + public function __construct(Config $config, HttpDownloader $httpDownloader) { - $this->rfs = $rfs; + $this->httpDownloader = $httpDownloader; $this->config = $config; } @@ -62,7 +62,7 @@ class Versions public function getLatest() { $protocol = extension_loaded('openssl') ? 'https' : 'http'; - $versions = JsonFile::parseJson($this->rfs->getContents('getcomposer.org', $protocol . '://getcomposer.org/versions', false)); + $versions = $this->httpDownloader->get($protocol . '://getcomposer.org/versions')->decodeJson(); foreach ($versions[$this->getChannel()] as $version) { if ($version['min-php'] <= PHP_VERSION_ID) { diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php index 631c0df7d..920c5f75a 100644 --- a/src/Composer/Util/HttpDownloader.php +++ b/src/Composer/Util/HttpDownloader.php @@ -102,6 +102,26 @@ class HttpDownloader return $promise; } + /** + * Retrieve the options set in the constructor + * + * @return array Options + */ + public function getOptions() + { + return $this->options; + } + + /** + * Merges new options + * + * @return array $options + */ + public function setOptions(array $options) + { + $this->options = array_replace_recursive($this->options, $options); + } + private function addJob($request, $sync = false) { $job = array( diff --git a/tests/Composer/Test/Downloader/PerforceDownloaderTest.php b/tests/Composer/Test/Downloader/PerforceDownloaderTest.php index ebb1f0456..1b5041d9f 100644 --- a/tests/Composer/Test/Downloader/PerforceDownloaderTest.php +++ b/tests/Composer/Test/Downloader/PerforceDownloaderTest.php @@ -17,6 +17,7 @@ use Composer\Config; use Composer\Repository\VcsRepository; use Composer\IO\IOInterface; use Composer\Test\TestCase; +use Composer\Factory; use Composer\Util\Filesystem; /** @@ -96,7 +97,7 @@ class PerforceDownloaderTest extends TestCase { $repository = $this->getMockBuilder('Composer\Repository\VcsRepository') ->setMethods(array('getRepoConfig')) - ->setConstructorArgs(array($repoConfig, $io, $config)) + ->setConstructorArgs(array($repoConfig, $io, $config, Factory::createHttpDownloader($io, $config))) ->getMock(); $repository->expects($this->any())->method('getRepoConfig')->will($this->returnValue($repoConfig)); diff --git a/tests/Composer/Test/InstallerTest.php b/tests/Composer/Test/InstallerTest.php index 2e7d70639..acaf8f1ff 100644 --- a/tests/Composer/Test/InstallerTest.php +++ b/tests/Composer/Test/InstallerTest.php @@ -65,7 +65,7 @@ class InstallerTest extends TestCase $eventDispatcher = $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock(); $httpDownloader = $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock(); - $repositoryManager = new RepositoryManager($io, $config, $eventDispatcher, $httpDownloader); + $repositoryManager = new RepositoryManager($io, $config, $httpDownloader, $eventDispatcher); $repositoryManager->setLocalRepository(new InstalledArrayRepository()); if (!is_array($repositories)) { diff --git a/tests/Composer/Test/Repository/ComposerRepositoryTest.php b/tests/Composer/Test/Repository/ComposerRepositoryTest.php index 1fe60f589..0ffd70751 100644 --- a/tests/Composer/Test/Repository/ComposerRepositoryTest.php +++ b/tests/Composer/Test/Repository/ComposerRepositoryTest.php @@ -37,8 +37,8 @@ class ComposerRepositoryTest extends TestCase $repoConfig, new NullIO, FactoryMock::createConfig(), - $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock(), - $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock() + $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock(), + $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock() )) ->getMock(); @@ -203,7 +203,7 @@ class ComposerRepositoryTest extends TestCase ->with($url = 'http://example.org/search.json?q=foo&type=library') ->willReturn(new \Composer\Util\Http\Response(array('url' => $url), 200, array(), json_encode(array()))); - $repository = new ComposerRepository($repoConfig, new NullIO, FactoryMock::createConfig(), $eventDispatcher, $httpDownloader); + $repository = new ComposerRepository($repoConfig, new NullIO, FactoryMock::createConfig(), $httpDownloader, $eventDispatcher); $this->assertSame( array(array('name' => 'foo', 'description' => null)), diff --git a/tests/Composer/Test/Repository/PearRepositoryTest.php b/tests/Composer/Test/Repository/PearRepositoryTest.php index 6046fefb4..867d4978d 100644 --- a/tests/Composer/Test/Repository/PearRepositoryTest.php +++ b/tests/Composer/Test/Repository/PearRepositoryTest.php @@ -28,7 +28,7 @@ class PearRepositoryTest extends TestCase /** * @var \PHPUnit_Framework_MockObject_MockObject */ - private $remoteFilesystem; + private $httpDownloader; public function testComposerShouldSetIncludePath() { diff --git a/tests/Composer/Test/Repository/RepositoryManagerTest.php b/tests/Composer/Test/Repository/RepositoryManagerTest.php index c4f09de87..35afd91e2 100644 --- a/tests/Composer/Test/Repository/RepositoryManagerTest.php +++ b/tests/Composer/Test/Repository/RepositoryManagerTest.php @@ -38,8 +38,8 @@ class RepositoryManagerTest extends TestCase $rm = new RepositoryManager( $this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getMockBuilder('Composer\Config')->getMock(), - $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock(), - $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock() + $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock(), + $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock() ); $repository1 = $this->getMockBuilder('Composer\Repository\RepositoryInterface')->getMock(); @@ -62,8 +62,8 @@ class RepositoryManagerTest extends TestCase $rm = new RepositoryManager( $this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $config = $this->getMockBuilder('Composer\Config')->setMethods(array('get'))->getMock(), - $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock(), - $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock() + $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock(), + $this->getMockBuilder('Composer\EventDispatcher\EventDispatcher')->disableOriginalConstructor()->getMock() ); $tmpdir = $this->tmpdir; diff --git a/tests/Composer/Test/Repository/Vcs/GitBitbucketDriverTest.php b/tests/Composer/Test/Repository/Vcs/GitBitbucketDriverTest.php index a3a9219d9..f0139970b 100644 --- a/tests/Composer/Test/Repository/Vcs/GitBitbucketDriverTest.php +++ b/tests/Composer/Test/Repository/Vcs/GitBitbucketDriverTest.php @@ -16,6 +16,7 @@ use Composer\Config; use Composer\Repository\Vcs\GitBitbucketDriver; use Composer\Test\TestCase; use Composer\Util\Filesystem; +use Composer\Util\ProcessExecutor; use Composer\Util\Http\Response; /** @@ -68,8 +69,8 @@ class GitBitbucketDriverTest extends TestCase $repoConfig, $this->io, $this->config, - null, - $this->httpDownloader + $this->httpDownloader, + new ProcessExecutor($this->io) ); $driver->initialize(); diff --git a/tests/Composer/Test/Repository/Vcs/GitHubDriverTest.php b/tests/Composer/Test/Repository/Vcs/GitHubDriverTest.php index 1f721a4c7..977a4a7aa 100644 --- a/tests/Composer/Test/Repository/Vcs/GitHubDriverTest.php +++ b/tests/Composer/Test/Repository/Vcs/GitHubDriverTest.php @@ -96,7 +96,7 @@ class GitHubDriverTest extends TestCase 'url' => $repoUrl, ); - $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, $process, $httpDownloader); + $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, $httpDownloader, $process); $gitHubDriver->initialize(); $this->setAttribute($gitHubDriver, 'tags', array($identifier => $sha)); @@ -139,7 +139,11 @@ class GitHubDriverTest extends TestCase ); $repoUrl = 'https://github.com/composer/packagist.git'; - $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, null, $httpDownloader); + $process = $this->getMockBuilder('Composer\Util\ProcessExecutor') + ->disableOriginalConstructor() + ->getMock(); + + $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, $httpDownloader, $process); $gitHubDriver->initialize(); $this->setAttribute($gitHubDriver, 'tags', array($identifier => $sha)); @@ -192,7 +196,11 @@ class GitHubDriverTest extends TestCase ); $repoUrl = 'https://github.com/composer/packagist.git'; - $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, null, $httpDownloader); + $process = $this->getMockBuilder('Composer\Util\ProcessExecutor') + ->disableOriginalConstructor() + ->getMock(); + + $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config,$httpDownloader, $process); $gitHubDriver->initialize(); $this->setAttribute($gitHubDriver, 'tags', array($identifier => $sha)); @@ -279,7 +287,7 @@ class GitHubDriverTest extends TestCase 'url' => $repoUrl, ); - $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, $process, $httpDownloader); + $gitHubDriver = new GitHubDriver($repoConfig, $io, $this->config, $httpDownloader, $process); $gitHubDriver->initialize(); $this->assertEquals('test_master', $gitHubDriver->getRootIdentifier()); diff --git a/tests/Composer/Test/Repository/Vcs/GitLabDriverTest.php b/tests/Composer/Test/Repository/Vcs/GitLabDriverTest.php index f940733ae..0fd2fa956 100644 --- a/tests/Composer/Test/Repository/Vcs/GitLabDriverTest.php +++ b/tests/Composer/Test/Repository/Vcs/GitLabDriverTest.php @@ -92,7 +92,7 @@ JSON; ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->httpDownloader->reveal(), $this->process->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -129,7 +129,7 @@ JSON; ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->httpDownloader->reveal(), $this->process->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -165,7 +165,7 @@ JSON; ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->httpDownloader->reveal(), $this->process->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -204,7 +204,7 @@ JSON; $this->mockResponse($apiUrl, array(), sprintf($projectData, $domain, $port, $namespace)) ->shouldBeCalledTimes(1); - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->httpDownloader->reveal(), $this->process->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -457,7 +457,7 @@ JSON; ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->httpDownloader->reveal(), $this->process->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -488,7 +488,7 @@ JSON; ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->httpDownloader->reveal(), $this->process->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -519,7 +519,7 @@ JSON; ->shouldBeCalledTimes(1) ; - $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->process->reveal(), $this->httpDownloader->reveal()); + $driver = new GitLabDriver(array('url' => $url), $this->io->reveal(), $this->config, $this->httpDownloader->reveal(), $this->process->reveal()); $driver->initialize(); $this->assertEquals($apiUrl, $driver->getApiUrl(), 'API URL is derived from the repository URL'); @@ -555,8 +555,8 @@ JSON; array('url' => 'https://gitlab.mycompany.local/mygroup/myproject', 'options' => $options), $this->io->reveal(), $this->config, - $this->process->reveal(), - $this->httpDownloader->reveal() + $this->httpDownloader->reveal(), + $this->process->reveal() ); $driver->initialize(); } diff --git a/tests/Composer/Test/Repository/Vcs/PerforceDriverTest.php b/tests/Composer/Test/Repository/Vcs/PerforceDriverTest.php index 8225a4a54..ff4e19121 100644 --- a/tests/Composer/Test/Repository/Vcs/PerforceDriverTest.php +++ b/tests/Composer/Test/Repository/Vcs/PerforceDriverTest.php @@ -45,7 +45,7 @@ class PerforceDriverTest extends TestCase $this->process = $this->getMockProcessExecutor(); $this->httpDownloader = $this->getMockHttpDownloader(); $this->perforce = $this->getMockPerforce(); - $this->driver = new PerforceDriver($this->repoConfig, $this->io, $this->config, $this->process, $this->httpDownloader); + $this->driver = new PerforceDriver($this->repoConfig, $this->io, $this->config, $this->httpDownloader, $this->process); $this->overrideDriverInternalPerforce($this->perforce); } @@ -113,7 +113,7 @@ class PerforceDriverTest extends TestCase public function testInitializeCapturesVariablesFromRepoConfig() { - $driver = new PerforceDriver($this->repoConfig, $this->io, $this->config, $this->process, $this->httpDownloader); + $driver = new PerforceDriver($this->repoConfig, $this->io, $this->config, $this->httpDownloader, $this->process); $driver->initialize(); $this->assertEquals(self::TEST_URL, $driver->getUrl()); $this->assertEquals(self::TEST_DEPOT, $driver->getDepot()); diff --git a/tests/Composer/Test/Repository/Vcs/SvnDriverTest.php b/tests/Composer/Test/Repository/Vcs/SvnDriverTest.php index 029d20160..946c198f2 100644 --- a/tests/Composer/Test/Repository/Vcs/SvnDriverTest.php +++ b/tests/Composer/Test/Repository/Vcs/SvnDriverTest.php @@ -46,6 +46,7 @@ class SvnDriverTest extends TestCase public function testWrongCredentialsInUrl() { $console = $this->getMockBuilder('Composer\IO\IOInterface')->getMock(); + $httpDownloader = $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock(); $output = "svn: OPTIONS of 'https://corp.svn.local/repo':"; $output .= " authorization failed: Could not authenticate to server:"; @@ -66,7 +67,7 @@ class SvnDriverTest extends TestCase 'url' => 'https://till:secret@corp.svn.local/repo', ); - $svn = new SvnDriver($repoConfig, $console, $this->config, $process); + $svn = new SvnDriver($repoConfig, $console, $this->config, $httpDownloader, $process); $svn->initialize(); } From 1cd9f4f9dbd5a2830e116b427076ce6c3d7f4b16 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Mon, 12 Nov 2018 15:35:46 +0100 Subject: [PATCH 04/34] Disable request_fulluri by default for HTTPS connections --- src/Composer/Util/StreamContextFactory.php | 6 +++--- tests/Composer/Test/Util/StreamContextFactoryTest.php | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Composer/Util/StreamContextFactory.php b/src/Composer/Util/StreamContextFactory.php index 72d12115d..b25b307a1 100644 --- a/src/Composer/Util/StreamContextFactory.php +++ b/src/Composer/Util/StreamContextFactory.php @@ -87,15 +87,15 @@ final class StreamContextFactory // enabled request_fulluri unless it is explicitly disabled switch (parse_url($url, PHP_URL_SCHEME)) { - case 'http': // default request_fulluri to true + case 'http': // default request_fulluri to true for HTTP $reqFullUriEnv = getenv('HTTP_PROXY_REQUEST_FULLURI'); if ($reqFullUriEnv === false || $reqFullUriEnv === '' || (strtolower($reqFullUriEnv) !== 'false' && (bool) $reqFullUriEnv)) { $options['http']['request_fulluri'] = true; } break; - case 'https': // default request_fulluri to true + case 'https': // default request_fulluri to false for HTTPS $reqFullUriEnv = getenv('HTTPS_PROXY_REQUEST_FULLURI'); - if ($reqFullUriEnv === false || $reqFullUriEnv === '' || (strtolower($reqFullUriEnv) !== 'false' && (bool) $reqFullUriEnv)) { + if (strtolower($reqFullUriEnv) !== 'false' && (bool) $reqFullUriEnv) { $options['http']['request_fulluri'] = true; } break; diff --git a/tests/Composer/Test/Util/StreamContextFactoryTest.php b/tests/Composer/Test/Util/StreamContextFactoryTest.php index 9bb04aaa1..3d60a9a38 100644 --- a/tests/Composer/Test/Util/StreamContextFactoryTest.php +++ b/tests/Composer/Test/Util/StreamContextFactoryTest.php @@ -142,7 +142,6 @@ class StreamContextFactoryTest extends TestCase $expected = array( 'http' => array( 'proxy' => 'tcp://proxyserver.net:80', - 'request_fulluri' => true, 'method' => 'GET', 'header' => array('User-Agent: foo', "Proxy-Authorization: Basic " . base64_encode('username:password')), 'max_redirects' => 20, @@ -173,7 +172,6 @@ class StreamContextFactoryTest extends TestCase $expected = array( 'http' => array( 'proxy' => 'ssl://woopproxy.net:443', - 'request_fulluri' => true, 'method' => 'GET', 'max_redirects' => 20, 'follow_location' => 1, From 09fd239f243e53e989568802a18d5643f7ff2091 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Mon, 12 Nov 2018 16:19:06 +0100 Subject: [PATCH 05/34] Fix factory test --- src/Composer/Util/HttpDownloader.php | 1 + tests/Composer/Test/FactoryTest.php | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php index 920c5f75a..d07823ec0 100644 --- a/src/Composer/Util/HttpDownloader.php +++ b/src/Composer/Util/HttpDownloader.php @@ -32,6 +32,7 @@ class HttpDownloader private $io; private $config; private $jobs = array(); + private $options = array(); private $index; private $progress; private $lastProgress; diff --git a/tests/Composer/Test/FactoryTest.php b/tests/Composer/Test/FactoryTest.php index 6704e5b15..96b0e95d5 100644 --- a/tests/Composer/Test/FactoryTest.php +++ b/tests/Composer/Test/FactoryTest.php @@ -35,6 +35,6 @@ class FactoryTest extends TestCase ->with($this->equalTo('disable-tls')) ->will($this->returnValue(true)); - Factory::createRemoteFilesystem($ioMock, $config); + Factory::createHttpDownloader($ioMock, $config); } } From 346de47af23cbc8441bd1683491d5c25592b8a42 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Mon, 12 Nov 2018 17:21:23 +0100 Subject: [PATCH 06/34] Small fixes --- src/Composer/Command/CreateProjectCommand.php | 2 +- .../Repository/ComposerRepository.php | 25 +++++++++++++++++-- .../Repository/RepositoryInterface.php | 11 ++++---- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/Composer/Command/CreateProjectCommand.php b/src/Composer/Command/CreateProjectCommand.php index 1b58d59c5..a1c364539 100644 --- a/src/Composer/Command/CreateProjectCommand.php +++ b/src/Composer/Command/CreateProjectCommand.php @@ -374,7 +374,7 @@ EOT { $factory = new Factory(); - return $factory->createDownloadManager($io, $config); + return $factory->createDownloadManager($io, $config, $factory->createHttpDownloader($io, $config)); } protected function createInstallationManager() diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index b337ea106..7b916d95a 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -28,6 +28,7 @@ use Composer\EventDispatcher\EventDispatcher; use Composer\Downloader\TransportException; use Composer\Semver\Constraint\ConstraintInterface; use Composer\Semver\Constraint\Constraint; +use Composer\Semver\Constraint\EmptyConstraint; /** * @author Jordi Boggiano @@ -156,9 +157,19 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito */ public function findPackages($name, $constraint = null) { - if (!$this->hasProviders()) { + // this call initializes loadRootServerFile which is needed for the rest below to work + $hasProviders = $this->hasProviders(); + + // TODO we need a new way for the repo to report this v2 protocol somehow + if ($this->lazyProvidersUrl) { + return $this->loadAsyncPackages(array($name => new EmptyConstraint()), function ($name, $stability) { + return true; + }); + } + if (!$hasProviders) { return parent::findPackages($name, $constraint); } + // normalize name $name = strtolower($name); @@ -197,10 +208,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito public function loadPackages(array $packageNameMap, $isPackageAcceptableCallable) { + // this call initializes loadRootServerFile which is needed for the rest below to work + $hasProviders = $this->hasProviders(); + + // TODO we need a new way for the repo to report this v2 protocol somehow if ($this->lazyProvidersUrl) { return $this->loadAsyncPackages($packageNameMap, $isPackageAcceptableCallable); } - if (!$this->hasProviders()) { + if (!$hasProviders) { return parent::loadPackages($packageNameMap, $isPackageAcceptableCallable); } @@ -225,6 +240,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } $packages = array_merge($packages, $matches); } + return $packages; } @@ -528,6 +544,11 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito if ($this->lazyProvidersUrl) { foreach ($packageNames as $name => $constraint) { + // skip platform packages, root package and composer-plugin-api + if (preg_match(PlatformRepository::PLATFORM_PACKAGE_REGEX, $name) || '__root__' === $name || 'composer-plugin-api' === $name) { + continue; + } + $url = str_replace('%package%', $name, $this->lazyProvidersUrl); $cacheKey = 'provider-'.strtr($name, '/', '$').'.json'; diff --git a/src/Composer/Repository/RepositoryInterface.php b/src/Composer/Repository/RepositoryInterface.php index 55b76d33d..567455163 100644 --- a/src/Composer/Repository/RepositoryInterface.php +++ b/src/Composer/Repository/RepositoryInterface.php @@ -13,6 +13,7 @@ namespace Composer\Repository; use Composer\Package\PackageInterface; +use Composer\Semver\Constraint\ConstraintInterface; /** * Repository interface. @@ -38,8 +39,8 @@ interface RepositoryInterface extends \Countable /** * Searches for the first match of a package by name and version. * - * @param string $name package name - * @param string|\Composer\Semver\Constraint\ConstraintInterface $constraint package version or version constraint to match against + * @param string $name package name + * @param string|ConstraintInterface $constraint package version or version constraint to match against * * @return PackageInterface|null */ @@ -48,8 +49,8 @@ interface RepositoryInterface extends \Countable /** * Searches for all packages matching a name and optionally a version. * - * @param string $name package name - * @param string|\Composer\Semver\Constraint\ConstraintInterface $constraint package version or version constraint to match against + * @param string $name package name + * @param string|ConstraintInterface $constraint package version or version constraint to match against * * @return PackageInterface[] */ @@ -66,7 +67,7 @@ interface RepositoryInterface extends \Countable /** * Returns list of registered packages with the supplied name * - * @param bool[] $packageNameMap + * @param ConstraintInterface[] $packageNameMap package names pointing to constraints * @param $isPackageAcceptableCallable * @return PackageInterface[] */ From 655a784fac39230dcc5065aa37888ff614e6f5d4 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Mon, 12 Nov 2018 17:44:18 +0100 Subject: [PATCH 07/34] Fix findPackage(s) implementation --- src/Composer/Repository/ComposerRepository.php | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 7b916d95a..dc9f5a79c 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -127,15 +127,24 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito */ public function findPackage($name, $constraint) { - if (!$this->hasProviders()) { - return parent::findPackage($name, $constraint); - } + // this call initializes loadRootServerFile which is needed for the rest below to work + $hasProviders = $this->hasProviders(); $name = strtolower($name); if (!$constraint instanceof ConstraintInterface) { $constraint = $this->versionParser->parseConstraints($constraint); } + // TODO we need a new way for the repo to report this v2 protocol somehow + if ($this->lazyProvidersUrl) { + return $this->loadAsyncPackages(array($name => $constraint), function ($name, $stability) { + return true; + }); + } + if (!$hasProviders) { + return parent::findPackage($name, $constraint); + } + foreach ($this->getProviderNames() as $providerName) { if ($name === $providerName) { $packages = $this->whatProvides($providerName); @@ -162,7 +171,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito // TODO we need a new way for the repo to report this v2 protocol somehow if ($this->lazyProvidersUrl) { - return $this->loadAsyncPackages(array($name => new EmptyConstraint()), function ($name, $stability) { + return $this->loadAsyncPackages(array($name => $constraint ?: new EmptyConstraint()), function ($name, $stability) { return true; }); } From 4a8a1cb0c9d462c715b01a765df3bbc3b9ae69de Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Mon, 12 Nov 2018 17:58:28 +0100 Subject: [PATCH 08/34] Fix PHP 5.3 support --- .../Repository/ComposerRepository.php | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index dc9f5a79c..8c92d3e4d 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -529,28 +529,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $packages = array(); $repo = $this; - $createPackageIfAcceptable = function ($version, $constraint) use (&$packages, $isPackageAcceptableCallable, $repo) { - if (!call_user_func($isPackageAcceptableCallable, strtolower($version['name']), VersionParser::parseStability($version['version']))) { - return; - } - - if (isset($version['version_normalized']) && $constraint && !$constraint->matches(new Constraint('==', $version['version_normalized']))) { - return; - } - - // load acceptable packages in the providers - $package = $this->createPackage($version, 'Composer\Package\CompletePackage'); - $package->setRepository($repo); - - // if there was no version_normalized, then we need to check now for the constraint - if (!$constraint || isset($version['version_normalized']) || $constraint->matches(new Constraint('==', $package->getVersion()))) { - $packages[spl_object_hash($package)] = $package; - if ($package instanceof AliasPackage && !isset($packages[spl_object_hash($package->getAliasOf())])) { - $packages[spl_object_hash($package->getAliasOf())] = $package->getAliasOf(); - } - } - }; - + // TODO what if not, then throw? if ($this->lazyProvidersUrl) { foreach ($packageNames as $name => $constraint) { // skip platform packages, root package and composer-plugin-api @@ -568,7 +547,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } $this->asyncFetchFile($url, $cacheKey, $lastModified) - ->then(function ($response) use (&$packages, $contents, $name, $constraint, $createPackageIfAcceptable) { + ->then(function ($response) use (&$packages, $contents, $name, $constraint, $repo, $isPackageAcceptableCallable) { if (true === $response) { $response = $contents; } @@ -591,10 +570,10 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $unpackedVersion[$key] = $version[$key.'s'][$index]; } - $createPackageIfAcceptable($unpackedVersion, $constraint); + $repo->createPackageIfAcceptable($packages, $isPackageAcceptableCallable, $unpackedVersion, $constraint); } } else { - $createPackageIfAcceptable($version, $constraint); + $repo->createPackageIfAcceptable($packages, $isPackageAcceptableCallable, $version, $constraint); } } }, function ($e) { @@ -611,6 +590,34 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito // RepositorySet should call loadMetadata, getMetadata when all promises resolved, then metadataComplete when done so we can GC the loaded json and whatnot then as needed } + /** + * TODO v3 should make this private once we can drop PHP 5.3 support + * + * @private + */ + public function createPackageIfAcceptable(&$packages, $isPackageAcceptableCallable, $version, $constraint) + { + if (!call_user_func($isPackageAcceptableCallable, strtolower($version['name']), VersionParser::parseStability($version['version']))) { + return; + } + + if (isset($version['version_normalized']) && $constraint && !$constraint->matches(new Constraint('==', $version['version_normalized']))) { + return; + } + + // load acceptable packages in the providers + $package = $this->createPackage($version, 'Composer\Package\CompletePackage'); + $package->setRepository($this); + + // if there was no version_normalized, then we need to check now for the constraint + if (!$constraint || isset($version['version_normalized']) || $constraint->matches(new Constraint('==', $package->getVersion()))) { + $packages[spl_object_hash($package)] = $package; + if ($package instanceof AliasPackage && !isset($packages[spl_object_hash($package->getAliasOf())])) { + $packages[spl_object_hash($package->getAliasOf())] = $package->getAliasOf(); + } + } + } + protected function loadRootServerFile() { if (null !== $this->rootData) { From fd11cf3618d960e4fd00e891b9e260a42ed92e0b Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Wed, 14 Nov 2018 17:54:19 +0100 Subject: [PATCH 09/34] Port/extract most behavior of RemoteFilesystem to CurlDownloader --- src/Composer/Composer.php | 1 + src/Composer/Util/AuthHelper.php | 169 ++++++++++ src/Composer/Util/Http/CurlDownloader.php | 317 +++++++++++++----- src/Composer/Util/Http/Response.php | 22 +- src/Composer/Util/HttpDownloader.php | 35 +- src/Composer/Util/RemoteFilesystem.php | 197 +---------- src/Composer/Util/StreamContextFactory.php | 56 +++- .../Test/Util/RemoteFilesystemTest.php | 40 ++- 8 files changed, 518 insertions(+), 319 deletions(-) diff --git a/src/Composer/Composer.php b/src/Composer/Composer.php index a3972f44f..c1a61545c 100644 --- a/src/Composer/Composer.php +++ b/src/Composer/Composer.php @@ -32,6 +32,7 @@ class Composer const VERSION = '@package_version@'; const BRANCH_ALIAS_VERSION = '@package_branch_alias_version@'; const RELEASE_DATE = '@release_date@'; + const SOURCE_VERSION = '2.0-source'; /** * @var Package\RootPackageInterface diff --git a/src/Composer/Util/AuthHelper.php b/src/Composer/Util/AuthHelper.php index 72b23ba22..e80a5b0c3 100644 --- a/src/Composer/Util/AuthHelper.php +++ b/src/Composer/Util/AuthHelper.php @@ -14,6 +14,7 @@ namespace Composer\Util; use Composer\Config; use Composer\IO\IOInterface; +use Composer\Downloader\TransportException; /** * @author Jordi Boggiano @@ -60,4 +61,172 @@ class AuthHelper ); } } + + + public function promptAuthIfNeeded($url, $origin, $httpStatus, $reason = null, $warning = null, $headers = array()) + { + $storeAuth = false; + $retry = false; + + if (in_array($origin, $this->config->get('github-domains'), true)) { + $gitHubUtil = new GitHub($this->io, $this->config, null); + $message = "\n"; + + $rateLimited = $gitHubUtil->isRateLimited($headers); + if ($rateLimited) { + $rateLimit = $gitHubUtil->getRateLimit($headers); + if ($this->io->hasAuthentication($origin)) { + $message = 'Review your configured GitHub OAuth token or enter a new one to go over the API rate limit.'; + } else { + $message = 'Create a GitHub OAuth token to go over the API rate limit.'; + } + + $message = sprintf( + 'GitHub API limit (%d calls/hr) is exhausted, could not fetch '.$url.'. '.$message.' You can also wait until %s for the rate limit to reset.', + $rateLimit['limit'], + $rateLimit['reset'] + )."\n"; + } else { + $message .= 'Could not fetch '.$url.', please '; + if ($this->io->hasAuthentication($origin)) { + $message .= 'review your configured GitHub OAuth token or enter a new one to access private repos'; + } else { + $message .= 'create a GitHub OAuth token to access private repos'; + } + } + + if (!$gitHubUtil->authorizeOAuth($origin) + && (!$this->io->isInteractive() || !$gitHubUtil->authorizeOAuthInteractively($origin, $message)) + ) { + throw new TransportException('Could not authenticate against '.$origin, 401); + } + } elseif (in_array($origin, $this->config->get('gitlab-domains'), true)) { + $message = "\n".'Could not fetch '.$url.', enter your ' . $origin . ' credentials ' .($httpStatus === 401 ? 'to access private repos' : 'to go over the API rate limit'); + $gitLabUtil = new GitLab($this->io, $this->config, null); + + if ($this->io->hasAuthentication($origin) && ($auth = $this->io->getAuthentication($origin)) && $auth['password'] === 'private-token') { + throw new TransportException("Invalid credentials for '" . $url . "', aborting.", $httpStatus); + } + + if (!$gitLabUtil->authorizeOAuth($origin) + && (!$this->io->isInteractive() || !$gitLabUtil->authorizeOAuthInteractively(parse_url($url, PHP_URL_SCHEME), $origin, $message)) + ) { + throw new TransportException('Could not authenticate against '.$origin, 401); + } + } elseif ($origin === 'bitbucket.org') { + $askForOAuthToken = true; + if ($this->io->hasAuthentication($origin)) { + $auth = $this->io->getAuthentication($origin); + if ($auth['username'] !== 'x-token-auth') { + $bitbucketUtil = new Bitbucket($this->io, $this->config); + $accessToken = $bitbucketUtil->requestToken($origin, $auth['username'], $auth['password']); + if (!empty($accessToken)) { + $this->io->setAuthentication($origin, 'x-token-auth', $accessToken); + $askForOAuthToken = false; + } + } else { + throw new TransportException('Could not authenticate against ' . $origin, 401); + } + } + + if ($askForOAuthToken) { + $message = "\n".'Could not fetch ' . $url . ', please create a bitbucket OAuth token to ' . (($httpStatus === 401 || $httpStatus === 403) ? 'access private repos' : 'go over the API rate limit'); + $bitBucketUtil = new Bitbucket($this->io, $this->config); + if (! $bitBucketUtil->authorizeOAuth($origin) + && (! $this->io->isInteractive() || !$bitBucketUtil->authorizeOAuthInteractively($origin, $message)) + ) { + throw new TransportException('Could not authenticate against ' . $origin, 401); + } + } + } else { + // 404s are only handled for github + if ($httpStatus === 404) { + return; + } + + // fail if the console is not interactive + if (!$this->io->isInteractive()) { + if ($httpStatus === 401) { + $message = "The '" . $url . "' URL required authentication.\nYou must be using the interactive console to authenticate"; + } + if ($httpStatus === 403) { + $message = "The '" . $url . "' URL could not be accessed: " . $reason; + } + + throw new TransportException($message, $httpStatus); + } + // fail if we already have auth + if ($this->io->hasAuthentication($origin)) { + throw new TransportException("Invalid credentials for '" . $url . "', aborting.", $httpStatus); + } + + $this->io->overwriteError(''); + if ($warning) { + $this->io->writeError(' '.$warning.''); + } + $this->io->writeError(' Authentication required ('.parse_url($url, PHP_URL_HOST).'):'); + $username = $this->io->ask(' Username: '); + $password = $this->io->askAndHideAnswer(' Password: '); + $this->io->setAuthentication($origin, $username, $password); + $storeAuth = $this->config->get('store-auths'); + } + + $retry = true; + + return array('retry' => $retry, 'storeAuth' => $storeAuth); + } + + public function addAuthenticationHeader(array $headers, $origin, $url) + { + if ($this->io->hasAuthentication($origin)) { + $auth = $this->io->getAuthentication($origin); + if ('github.com' === $origin && 'x-oauth-basic' === $auth['password']) { + $headers[] = 'Authorization: token '.$auth['username']; + } elseif (in_array($origin, $this->config->get('gitlab-domains'), true)) { + if ($auth['password'] === 'oauth2') { + $headers[] = 'Authorization: Bearer '.$auth['username']; + } elseif ($auth['password'] === 'private-token') { + $headers[] = 'PRIVATE-TOKEN: '.$auth['username']; + } + } elseif ( + 'bitbucket.org' === $origin + && $url !== Bitbucket::OAUTH2_ACCESS_TOKEN_URL + && 'x-token-auth' === $auth['username'] + ) { + if (!$this->isPublicBitBucketDownload($url)) { + $headers[] = 'Authorization: Bearer ' . $auth['password']; + } + } else { + $authStr = base64_encode($auth['username'] . ':' . $auth['password']); + $headers[] = 'Authorization: Basic '.$authStr; + } + } + + return $headers; + } + + /** + * @link https://github.com/composer/composer/issues/5584 + * + * @param string $urlToBitBucketFile URL to a file at bitbucket.org. + * + * @return bool Whether the given URL is a public BitBucket download which requires no authentication. + */ + public function isPublicBitBucketDownload($urlToBitBucketFile) + { + $domain = parse_url($urlToBitBucketFile, PHP_URL_HOST); + if (strpos($domain, 'bitbucket.org') === false) { + // Bitbucket downloads are hosted on amazonaws. + // We do not need to authenticate there at all + return true; + } + + $path = parse_url($urlToBitBucketFile, PHP_URL_PATH); + + // Path for a public download follows this pattern /{user}/{repo}/downloads/{whatever} + // {@link https://blog.bitbucket.org/2009/04/12/new-feature-downloads/} + $pathParts = explode('/', $path); + + return count($pathParts) >= 4 && $pathParts[3] == 'downloads'; + } } diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php index 846c41883..2accb7a0c 100644 --- a/src/Composer/Util/Http/CurlDownloader.php +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -16,6 +16,9 @@ use Composer\Config; use Composer\IO\IOInterface; use Composer\Downloader\TransportException; use Composer\CaBundle\CaBundle; +use Composer\Util\RemoteFilesystem; +use Composer\Util\StreamContextFactory; +use Composer\Util\AuthHelper; use Psr\Log\LoggerInterface; use React\Promise\Promise; @@ -28,8 +31,14 @@ class CurlDownloader private $multiHandle; private $shareHandle; private $jobs = array(); + /** @var IOInterface */ private $io; + /** @var Config */ + private $config; + /** @var AuthHelper */ + private $authHelper; private $selectTimeout = 5.0; + private $maxRedirects = 20; protected $multiErrors = array( CURLM_BAD_HANDLE => array('CURLM_BAD_HANDLE', 'The passed-in handle is not a valid CURLM handle.'), CURLM_BAD_EASY_HANDLE => array('CURLM_BAD_EASY_HANDLE', "An easy handle was not good/valid. It could mean that it isn't an easy handle at all, or possibly that the handle already is in used by this or another multi handle."), @@ -42,6 +51,7 @@ class CurlDownloader 'method' => CURLOPT_CUSTOMREQUEST, 'content' => CURLOPT_POSTFIELDS, 'proxy' => CURLOPT_PROXY, + 'header' => CURLOPT_HTTPHEADER, ), 'ssl' => array( 'ciphers' => CURLOPT_SSL_CIPHER_LIST, @@ -62,6 +72,7 @@ class CurlDownloader public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false) { $this->io = $io; + $this->config = $config; $this->multiHandle = $mh = curl_multi_init(); if (function_exists('curl_multi_setopt')) { @@ -77,79 +88,112 @@ class CurlDownloader curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); } + + $this->authHelper = new AuthHelper($io, $config); } public function download($resolve, $reject, $origin, $url, $options, $copyTo = null) { - $ch = curl_init(); - $hd = fopen('php://temp/maxmemory:32768', 'w+b'); + return $this->initDownload($resolve, $reject, $origin, $url, $options, $copyTo); + } - // TODO auth & other context - // TODO cleanup + private function initDownload($resolve, $reject, $origin, $url, $options, $copyTo = null, array $attributes = array()) + { + // TODO allow setting attributes somehow + $attributes = array_merge(array( + 'retryAuthFailure' => true, + 'redirects' => 1, + 'storeAuth' => false, + ), $attributes); - if ($copyTo && !$fd = @fopen($copyTo.'~', 'w+b')) { - // TODO throw here probably? - $copyTo = null; + $originalOptions = $options; + + // check URL can be accessed (i.e. is not insecure) + $this->config->prohibitUrlByConfig($url, $this->io); + + $curlHandle = curl_init(); + $headerHandle = fopen('php://temp/maxmemory:32768', 'w+b'); + + if ($copyTo) { + $errorMessage = ''; + set_error_handler(function ($code, $msg) use (&$errorMessage) { + if ($errorMessage) { + $errorMessage .= "\n"; + } + $errorMessage .= preg_replace('{^fopen\(.*?\): }', '', $msg); + }); + $bodyHandle = fopen($copyTo.'~', 'w+b'); + restore_error_handler(); + if (!$bodyHandle) { + throw new TransportException('The "'.$url.'" file could not be written to '.$copyTo.': '.$errorMessage); + } + } else { + $bodyHandle = @fopen('php://temp/maxmemory:524288', 'w+b'); } - if (!$copyTo) { - $fd = @fopen('php://temp/maxmemory:524288', 'w+b'); + + curl_setopt($curlHandle, CURLOPT_URL, $url); + curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($curlHandle, CURLOPT_MAXREDIRS, 20); + //curl_setopt($curlHandle, CURLOPT_DNS_USE_GLOBAL_CACHE, false); + curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 10); + curl_setopt($curlHandle, CURLOPT_TIMEOUT, 10); // TODO increase + curl_setopt($curlHandle, CURLOPT_WRITEHEADER, $headerHandle); + curl_setopt($curlHandle, CURLOPT_FILE, $bodyHandle); + curl_setopt($curlHandle, CURLOPT_ENCODING, "gzip"); + curl_setopt($curlHandle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS); + if (defined('CURLOPT_SSL_FALSESTART')) { + curl_setopt($curlHandle, CURLOPT_SSL_FALSESTART, true); + } + if (function_exists('curl_share_init')) { + curl_setopt($curlHandle, CURLOPT_SHARE, $this->shareHandle); } if (!isset($options['http']['header'])) { $options['http']['header'] = array(); } - $headers = array_diff($options['http']['header'], array('Connection: close')); + $options['http']['header'] = array_diff($options['http']['header'], array('Connection: close')); + $options['http']['header'][] = 'Connection: keep-alive'; - // TODO - $degradedMode = false; - if ($degradedMode) { - curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); - } else { - $headers[] = 'Connection: keep-alive'; - $version = curl_version(); - $features = $version['features']; - if (0 === strpos($url, 'https://') && \defined('CURL_VERSION_HTTP2') && \defined('CURL_HTTP_VERSION_2_0') && (CURL_VERSION_HTTP2 & $features)) { - curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); - } + $version = curl_version(); + $features = $version['features']; + if (0 === strpos($url, 'https://') && \defined('CURL_VERSION_HTTP2') && \defined('CURL_HTTP_VERSION_2_0') && (CURL_VERSION_HTTP2 & $features)) { + curl_setopt($curlHandle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); } - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - //curl_setopt($ch, CURLOPT_DNS_USE_GLOBAL_CACHE, false); - curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); - curl_setopt($ch, CURLOPT_TIMEOUT, 10); // TODO increase - curl_setopt($ch, CURLOPT_WRITEHEADER, $hd); - curl_setopt($ch, CURLOPT_FILE, $fd); - if (function_exists('curl_share_init')) { - curl_setopt($ch, CURLOPT_SHARE, $this->shareHandle); - } + $options['http']['header'] = $this->authHelper->addAuthenticationHeader($options['http']['header'], $origin, $url); + $options = StreamContextFactory::initOptions($url, $options); foreach (self::$options as $type => $curlOptions) { foreach ($curlOptions as $name => $curlOption) { if (isset($options[$type][$name])) { - curl_setopt($ch, $curlOption, $options[$type][$name]); + curl_setopt($curlHandle, $curlOption, $options[$type][$name]); } } } - $progress = array_diff_key(curl_getinfo($ch), self::$timeInfo); + $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); - $this->jobs[(int) $ch] = array( + $this->jobs[(int) $curlHandle] = array( + 'url' => $url, + 'origin' => $origin, + 'attributes' => $attributes, + 'options' => $originalOptions, 'progress' => $progress, - 'ch' => $ch, + 'curlHandle' => $curlHandle, //'callback' => $params['notification'], - 'file' => $copyTo, - 'hd' => $hd, - 'fd' => $fd, + 'filename' => $copyTo, + 'headerHandle' => $headerHandle, + 'bodyHandle' => $bodyHandle, 'resolve' => $resolve, 'reject' => $reject, ); - $this->io->write('Downloading '.$url, true, IOInterface::DEBUG); + $usingProxy = !empty($options['http']['proxy']) ? ' using proxy ' . $options['http']['proxy'] : ''; + $ifModified = false !== strpos(strtolower(implode(',', $options['http']['header'])), 'if-modified-since:') ? ' if modified' : ''; + $this->io->writeError('Downloading ' . $url . $usingProxy . $ifModified, true, IOInterface::DEBUG); - $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $ch)); + $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $curlHandle)); //$params['notification'](STREAM_NOTIFY_RESOLVE, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, 0, false); } @@ -169,74 +213,114 @@ class CurlDownloader } while ($progress = curl_multi_info_read($this->multiHandle)) { - $h = $progress['handle']; - $i = (int) $h; + $curlHandle = $progress['handle']; + $i = (int) $curlHandle; if (!isset($this->jobs[$i])) { continue; } - $progress = array_diff_key(curl_getinfo($h), self::$timeInfo); + $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); $job = $this->jobs[$i]; unset($this->jobs[$i]); - curl_multi_remove_handle($this->multiHandle, $h); - $error = curl_error($h); - $errno = curl_errno($h); - curl_close($h); + curl_multi_remove_handle($this->multiHandle, $curlHandle); + $error = curl_error($curlHandle); + $errno = curl_errno($curlHandle); + curl_close($curlHandle); + $headers = null; + $statusCode = null; + $response = null; try { - //$this->onProgress($h, $job['callback'], $progress, $job['progress']); - if ('' !== $error) { - throw new TransportException(curl_error($h)); + //$this->onProgress($curlHandle, $job['callback'], $progress, $job['progress']); + if (CURLE_OK !== $errno) { + throw new TransportException($error); } - if ($job['file']) { - if (CURLE_OK === $errno) { - fclose($job['fd']); - rename($job['file'].'~', $job['file']); - call_user_func($job['resolve'], true); - } - // TODO otherwise show error? + $statusCode = $progress['http_code']; + rewind($job['headerHandle']); + $headers = explode("\r\n", rtrim(stream_get_contents($job['headerHandle']))); + fclose($job['headerHandle']); + + // prepare response object + if ($job['filename']) { + fclose($job['bodyHandle']); + $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $job['filename'].'~'); } else { - rewind($job['hd']); - $headers = explode("\r\n", rtrim(stream_get_contents($job['hd']))); - fclose($job['hd']); - rewind($job['fd']); - $contents = stream_get_contents($job['fd']); - fclose($job['fd']); - $this->io->writeError('['.$progress['http_code'].'] '.$progress['url'], true, IOInterface::DEBUG); - call_user_func($job['resolve'], new Response(array('url' => $progress['url']), $progress['http_code'], $headers, $contents)); + rewind($job['bodyHandle']); + $contents = stream_get_contents($job['bodyHandle']); + fclose($job['bodyHandle']); + $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $contents); + $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG); } - } catch (TransportException $e) { - fclose($job['hd']); - fclose($job['fd']); - if ($job['file']) { - @unlink($job['file'].'~'); + + $response = $this->retryIfAuthNeeded($job, $response); + + // handle 3xx redirects, 304 Not Modified is excluded + if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['redirects'] < $this->maxRedirects) { + // TODO + $response = $this->handleRedirect($job, $response); + } + + // fail 4xx and 5xx responses and capture the response + if ($statusCode >= 400 && $statusCode <= 599) { + throw $this->failResponse($job, $response, $response->getStatusMessage()); +// $this->io->overwriteError("Downloading (failed)", false); + } + + if ($job['attributes']['storeAuth']) { + $this->authHelper->storeAuth($job['origin'], $job['attributes']['storeAuth']); + } + + // resolve promise + if ($job['filename']) { + rename($job['filename'].'~', $job['filename']); + call_user_func($job['resolve'], true); + } else { + call_user_func($job['resolve'], $response); + } + } catch (\Exception $e) { + if ($e instanceof TransportException && $headers) { + $e->setHeaders($headers); + $e->setStatusCode($statusCode); + } + if ($e instanceof TransportException && $response) { + $e->setResponse($response->getBody()); + } + + if (is_resource($job['headerHandle'])) { + fclose($job['headerHandle']); + } + if (is_resource($job['bodyHandle'])) { + fclose($job['bodyHandle']); + } + if ($job['filename']) { + @unlink($job['filename'].'~'); } call_user_func($job['reject'], $e); } } - foreach ($this->jobs as $i => $h) { + foreach ($this->jobs as $i => $curlHandle) { if (!isset($this->jobs[$i])) { continue; } - $h = $this->jobs[$i]['ch']; - $progress = array_diff_key(curl_getinfo($h), self::$timeInfo); + $curlHandle = $this->jobs[$i]['curlHandle']; + $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); if ($this->jobs[$i]['progress'] !== $progress) { $previousProgress = $this->jobs[$i]['progress']; $this->jobs[$i]['progress'] = $progress; try { - //$this->onProgress($h, $this->jobs[$i]['callback'], $progress, $previousProgress); + //$this->onProgress($curlHandle, $this->jobs[$i]['callback'], $progress, $previousProgress); } catch (TransportException $e) { var_dump('Caught '.$e->getMessage());die; unset($this->jobs[$i]); - curl_multi_remove_handle($this->multiHandle, $h); - curl_close($h); + curl_multi_remove_handle($this->multiHandle, $curlHandle); + curl_close($curlHandle); - fclose($job['hd']); - fclose($job['fd']); - if ($job['file']) { - @unlink($job['file'].'~'); + fclose($job['headerHandle']); + fclose($job['bodyHandle']); + if ($job['filename']) { + @unlink($job['filename'].'~'); } call_user_func($job['reject'], $e); } @@ -245,22 +329,77 @@ class CurlDownloader } catch (\Exception $e) { var_dump('Caught2', get_class($e), $e->getMessage(), $e);die; } - -// TODO finalize / resolve -// if ($copyTo && !isset($this->exceptions[(int) $ch])) { -// $fd = fopen($copyTo, 'rb'); -// } -// } - private function onProgress($ch, callable $notify, array $progress, array $previousProgress) + private function retryIfAuthNeeded(array $job, Response $response) + { + if (in_array($response->getStatusCode(), array(401, 403)) && $job['attributes']['retryAuthFailure']) { + $warning = null; + if ($response->getHeader('content-type') === 'application/json') { + $data = json_decode($response->getBody(), true); + if (!empty($data['warning'])) { + $warning = $data['warning']; + } + } + + $result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], $response->getStatusCode(), $response->getStatusMessage(), $warning, $response->getHeaders()); + + if ($result['retry']) { + // TODO retry somehow using $result['storeAuth'] in the attributes + } + } + + $locationHeader = $response->getHeader('location'); + $needsAuthRetry = false; + + // check for bitbucket login page asking to authenticate + if ( + $job['origin'] === 'bitbucket.org' + && !$this->authHelper->isPublicBitBucketDownload($job['url']) + && substr($job['url'], -4) === '.zip' + && (!$locationHeader || substr($locationHeader, -4) !== '.zip') + && preg_match('{^text/html\b}i', $response->getHeader('content-type')) + ) { + $needsAuthRetry = 'Bitbucket requires authentication and it was not provided'; + } + + // check for gitlab 404 when downloading archives + if ( + $response->getStatusCode() === 404 + && $this->config && in_array($job['origin'], $this->config->get('gitlab-domains'), true) + && false !== strpos($job['url'], 'archive.zip') + ) { + $needsAuthRetry = 'GitLab requires authentication and it was not provided'; + } + + if ($needsAuthRetry) { + if ($job['attributes']['retryAuthFailure']) { + $result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], 401); + if ($result['retry']) { + // TODO ... + // TODO return early here to abort failResponse + } + } + + throw $this->failResponse($job, $response, $needsAuthRetry); + } + + return $response; + } + + private function failResponse(array $job, Response $response, $errorMessage) + { + return new TransportException('The "'.$job['url'].'" file could not be downloaded ('.$errorMessage.')', $response->getStatusCode()); + } + + private function onProgress($curlHandle, callable $notify, array $progress, array $previousProgress) { if (300 <= $progress['http_code'] && $progress['http_code'] < 400) { return; } if (!$previousProgress['http_code'] && $progress['http_code'] && $progress['http_code'] < 200 || 400 <= $progress['http_code']) { $code = 403 === $progress['http_code'] ? STREAM_NOTIFY_AUTH_RESULT : STREAM_NOTIFY_FAILURE; - $notify($code, STREAM_NOTIFY_SEVERITY_ERR, curl_error($ch), $progress['http_code'], 0, 0, false); + $notify($code, STREAM_NOTIFY_SEVERITY_ERR, curl_error($curlHandle), $progress['http_code'], 0, 0, false); } if ($previousProgress['download_content_length'] < $progress['download_content_length']) { $notify(STREAM_NOTIFY_FILE_SIZE_IS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, (int) $progress['download_content_length'], false); diff --git a/src/Composer/Util/Http/Response.php b/src/Composer/Util/Http/Response.php index f76057f3e..d2774c938 100644 --- a/src/Composer/Util/Http/Response.php +++ b/src/Composer/Util/Http/Response.php @@ -27,7 +27,7 @@ class Response throw new \LogicException('url key missing from request array'); } $this->request = $request; - $this->code = $code; + $this->code = (int) $code; $this->headers = $headers; $this->body = $body; } @@ -37,6 +37,23 @@ class Response return $this->code; } + /** + * @return string|null + */ + public function getStatusMessage() + { + $value = null; + foreach ($this->headers as $header) { + if (preg_match('{^HTTP/\S+ \d+}i', $header)) { + // In case of redirects, headers contain the headers of all responses + // so we can not return directly and need to keep iterating + $value = $header; + } + } + + return $value; + } + public function getHeaders() { return $this->headers; @@ -51,7 +68,7 @@ class Response } elseif (preg_match('{^HTTP/}i', $header)) { // TODO ideally redirects would be handled in CurlDownloader/RemoteFilesystem and this becomes unnecessary // - // In case of redirects, http_response_headers contains the headers of all responses + // In case of redirects, headers contains the headers of all responses // so we reset the flag when a new response is being parsed as we are only interested in the last response $value = null; } @@ -60,7 +77,6 @@ class Response return $value; } - public function getBody() { return $this->body; diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php index d07823ec0..9cdc0c919 100644 --- a/src/Composer/Util/HttpDownloader.php +++ b/src/Composer/Util/HttpDownloader.php @@ -66,6 +66,7 @@ class HttpDownloader $this->options = array_replace_recursive($this->options, $options); $this->config = $config; + // TODO enable curl only on 5.6+ if older versions cause any problem if (extension_loaded('curl')) { $this->curl = new Http\CurlDownloader($io, $config, $options, $disableTls); } @@ -125,6 +126,11 @@ class HttpDownloader private function addJob($request, $sync = false) { + // capture username/password from URL if there is one + if (preg_match('{^https?://([^:/]+):([^@/]+)@([^/]+)}i', $request['url'], $match)) { + $this->io->setAuthentication($originUrl, rawurldecode($match[1]), rawurldecode($match[2])); + } + $job = array( 'id' => $this->idGen++, 'status' => self::STATUS_QUEUED, @@ -138,7 +144,6 @@ class HttpDownloader $origin = $this->getOrigin($job['request']['url']); - // TODO experiment with allowing file:// through curl too if ($curl && preg_match('{^https?://}i', $job['request']['url'])) { $resolver = function ($resolve, $reject) use (&$job, $curl, $origin) { // start job @@ -183,10 +188,10 @@ class HttpDownloader $job['response'] = $response; // TODO look for more jobs to start once we throttle to max X jobs }, function ($e) use ($io, &$job) { - var_dump(__CLASS__ . __LINE__); - var_dump(gettype($e)); - var_dump($e->getMessage()); - die; + // var_dump(__CLASS__ . __LINE__); + // var_dump(get_class($e)); + // var_dump($e->getMessage()); + // die; $job['status'] = HttpDownloader::STATUS_FAILED; $job['exception'] = $e; }); @@ -248,9 +253,13 @@ class HttpDownloader private function getOrigin($url) { + if (0 === strpos($url, 'file://')) { + return $url; + } + $origin = parse_url($url, PHP_URL_HOST); - if ($origin === 'api.github.com') { + if (strpos($origin, '.github.com') === (strlen($origin) - 11)) { return 'github.com'; } @@ -258,6 +267,20 @@ class HttpDownloader return 'packagist.org'; } + // Gitlab can be installed in a non-root context (i.e. gitlab.com/foo). When downloading archives the originUrl + // is the host without the path, so we look for the registered gitlab-domains matching the host here + if ( + is_array($this->config->get('gitlab-domains')) + && false === strpos($origin, '/') + && !in_array($origin, $this->config->get('gitlab-domains')) + ) { + foreach ($this->config->get('gitlab-domains') as $gitlabDomain) { + if (0 === strpos($gitlabDomain, $origin)) { + return $gitlabDomain; + } + } + } + return $origin ?: $url; } } diff --git a/src/Composer/Util/RemoteFilesystem.php b/src/Composer/Util/RemoteFilesystem.php index 00fe35294..2709f7006 100644 --- a/src/Composer/Util/RemoteFilesystem.php +++ b/src/Composer/Util/RemoteFilesystem.php @@ -41,6 +41,7 @@ class RemoteFilesystem private $retryAuthFailure; private $lastHeaders; private $storeAuth; + private $authHelper; private $degradedMode = false; private $redirects; private $maxRedirects = 20; @@ -53,7 +54,7 @@ class RemoteFilesystem * @param array $options The options * @param bool $disableTls */ - public function __construct(IOInterface $io, Config $config = null, array $options = array(), $disableTls = false) + public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false) { $this->io = $io; @@ -69,6 +70,7 @@ class RemoteFilesystem // handle the other externally set options normally. $this->options = array_replace_recursive($this->options, $options); $this->config = $config; + $this->authHelper = new AuthHelper($io, $config); } /** @@ -215,27 +217,6 @@ class RemoteFilesystem */ protected function get($originUrl, $fileUrl, $additionalOptions = array(), $fileName = null, $progress = true) { - if (strpos($originUrl, '.github.com') === (strlen($originUrl) - 11)) { - $originUrl = 'github.com'; - } - - // Gitlab can be installed in a non-root context (i.e. gitlab.com/foo). When downloading archives the originUrl - // is the host without the path, so we look for the registered gitlab-domains matching the host here - if ( - $this->config - && is_array($this->config->get('gitlab-domains')) - && false === strpos($originUrl, '/') - && !in_array($originUrl, $this->config->get('gitlab-domains')) - ) { - foreach ($this->config->get('gitlab-domains') as $gitlabDomain) { - if (0 === strpos($gitlabDomain, $originUrl)) { - $originUrl = $gitlabDomain; - break; - } - } - unset($gitlabDomain); - } - $this->scheme = parse_url($fileUrl, PHP_URL_SCHEME); $this->bytesMax = 0; $this->originUrl = $originUrl; @@ -247,11 +228,6 @@ class RemoteFilesystem $this->lastHeaders = array(); $this->redirects = 1; // The first request counts. - // capture username/password from URL if there is one - if (preg_match('{^https?://([^:/]+):([^@/]+)@([^/]+)}i', $fileUrl, $match)) { - $this->io->setAuthentication($originUrl, rawurldecode($match[1]), rawurldecode($match[2])); - } - $tempAdditionalOptions = $additionalOptions; if (isset($tempAdditionalOptions['retry-auth-failure'])) { $this->retryAuthFailure = (bool) $tempAdditionalOptions['retry-auth-failure']; @@ -272,14 +248,6 @@ class RemoteFilesystem $origFileUrl = $fileUrl; - if (isset($options['github-token'])) { - // only add the access_token if it is actually a github URL (in case we were redirected to S3) - if (preg_match('{^https?://([a-z0-9-]+\.)*github\.com/}', $fileUrl)) { - $fileUrl .= (false === strpos($fileUrl, '?') ? '?' : '&') . 'access_token='.$options['github-token']; - } - unset($options['github-token']); - } - if (isset($options['gitlab-token'])) { $fileUrl .= (false === strpos($fileUrl, '?') ? '?' : '&') . 'access_token='.$options['gitlab-token']; unset($options['gitlab-token']); @@ -400,7 +368,7 @@ class RemoteFilesystem // check for bitbucket login page asking to authenticate if ($originUrl === 'bitbucket.org' - && !$this->isPublicBitBucketDownload($fileUrl) + && !$this->authHelper->isPublicBitBucketDownload($fileUrl) && substr($fileUrl, -4) === '.zip' && (!$locationHeader || substr($locationHeader, -4) !== '.zip') && $contentType && preg_match('{^text/html\b}i', $contentType) @@ -544,8 +512,7 @@ class RemoteFilesystem $result = $this->get($this->originUrl, $this->fileUrl, $additionalOptions, $this->fileName, $this->progress); if ($this->storeAuth && $this->config) { - $authHelper = new AuthHelper($this->io, $this->config); - $authHelper->storeAuth($this->originUrl, $this->storeAuth); + $this->authHelper->storeAuth($this->originUrl, $this->storeAuth); $this->storeAuth = false; } @@ -650,111 +617,14 @@ class RemoteFilesystem protected function promptAuthAndRetry($httpStatus, $reason = null, $warning = null, $headers = array()) { - if ($this->config && in_array($this->originUrl, $this->config->get('github-domains'), true)) { - $gitHubUtil = new GitHub($this->io, $this->config, null); - $message = "\n"; + $result = $this->authHelper->promptAuthIfNeeded($this->fileUrl, $this->originUrl, $httpStatus, $reason, $warning, $headers); - $rateLimited = $gitHubUtil->isRateLimited($headers); - if ($rateLimited) { - $rateLimit = $gitHubUtil->getRateLimit($headers); - if ($this->io->hasAuthentication($this->originUrl)) { - $message = 'Review your configured GitHub OAuth token or enter a new one to go over the API rate limit.'; - } else { - $message = 'Create a GitHub OAuth token to go over the API rate limit.'; - } + $this->storeAuth = $result['storeAuth']; + $this->retry = $result['retry']; - $message = sprintf( - 'GitHub API limit (%d calls/hr) is exhausted, could not fetch '.$this->fileUrl.'. '.$message.' You can also wait until %s for the rate limit to reset.', - $rateLimit['limit'], - $rateLimit['reset'] - )."\n"; - } else { - $message .= 'Could not fetch '.$this->fileUrl.', please '; - if ($this->io->hasAuthentication($this->originUrl)) { - $message .= 'review your configured GitHub OAuth token or enter a new one to access private repos'; - } else { - $message .= 'create a GitHub OAuth token to access private repos'; - } - } - - if (!$gitHubUtil->authorizeOAuth($this->originUrl) - && (!$this->io->isInteractive() || !$gitHubUtil->authorizeOAuthInteractively($this->originUrl, $message)) - ) { - throw new TransportException('Could not authenticate against '.$this->originUrl, 401); - } - } elseif ($this->config && in_array($this->originUrl, $this->config->get('gitlab-domains'), true)) { - $message = "\n".'Could not fetch '.$this->fileUrl.', enter your ' . $this->originUrl . ' credentials ' .($httpStatus === 401 ? 'to access private repos' : 'to go over the API rate limit'); - $gitLabUtil = new GitLab($this->io, $this->config, null); - - if ($this->io->hasAuthentication($this->originUrl) && ($auth = $this->io->getAuthentication($this->originUrl)) && $auth['password'] === 'private-token') { - throw new TransportException("Invalid credentials for '" . $this->fileUrl . "', aborting.", $httpStatus); - } - - if (!$gitLabUtil->authorizeOAuth($this->originUrl) - && (!$this->io->isInteractive() || !$gitLabUtil->authorizeOAuthInteractively($this->scheme, $this->originUrl, $message)) - ) { - throw new TransportException('Could not authenticate against '.$this->originUrl, 401); - } - } elseif ($this->config && $this->originUrl === 'bitbucket.org') { - $askForOAuthToken = true; - if ($this->io->hasAuthentication($this->originUrl)) { - $auth = $this->io->getAuthentication($this->originUrl); - if ($auth['username'] !== 'x-token-auth') { - $bitbucketUtil = new Bitbucket($this->io, $this->config); - $accessToken = $bitbucketUtil->requestToken($this->originUrl, $auth['username'], $auth['password']); - if (!empty($accessToken)) { - $this->io->setAuthentication($this->originUrl, 'x-token-auth', $accessToken); - $askForOAuthToken = false; - } - } else { - throw new TransportException('Could not authenticate against ' . $this->originUrl, 401); - } - } - - if ($askForOAuthToken) { - $message = "\n".'Could not fetch ' . $this->fileUrl . ', please create a bitbucket OAuth token to ' . (($httpStatus === 401 || $httpStatus === 403) ? 'access private repos' : 'go over the API rate limit'); - $bitBucketUtil = new Bitbucket($this->io, $this->config); - if (! $bitBucketUtil->authorizeOAuth($this->originUrl) - && (! $this->io->isInteractive() || !$bitBucketUtil->authorizeOAuthInteractively($this->originUrl, $message)) - ) { - throw new TransportException('Could not authenticate against ' . $this->originUrl, 401); - } - } - } else { - // 404s are only handled for github - if ($httpStatus === 404) { - return; - } - - // fail if the console is not interactive - if (!$this->io->isInteractive()) { - if ($httpStatus === 401) { - $message = "The '" . $this->fileUrl . "' URL required authentication.\nYou must be using the interactive console to authenticate"; - } - if ($httpStatus === 403) { - $message = "The '" . $this->fileUrl . "' URL could not be accessed: " . $reason; - } - - throw new TransportException($message, $httpStatus); - } - // fail if we already have auth - if ($this->io->hasAuthentication($this->originUrl)) { - throw new TransportException("Invalid credentials for '" . $this->fileUrl . "', aborting.", $httpStatus); - } - - $this->io->overwriteError(''); - if ($warning) { - $this->io->writeError(' '.$warning.''); - } - $this->io->writeError(' Authentication required ('.parse_url($this->fileUrl, PHP_URL_HOST).'):'); - $username = $this->io->ask(' Username: '); - $password = $this->io->askAndHideAnswer(' Password: '); - $this->io->setAuthentication($this->originUrl, $username, $password); - $this->storeAuth = $this->config->get('store-auths'); + if ($this->retry) { + throw new TransportException('RETRY'); } - - $this->retry = true; - throw new TransportException('RETRY'); } protected function getOptionsForUrl($originUrl, $additionalOptions) @@ -814,27 +684,7 @@ class RemoteFilesystem $headers[] = 'Connection: close'; } - if ($this->io->hasAuthentication($originUrl)) { - $auth = $this->io->getAuthentication($originUrl); - if ('github.com' === $originUrl && 'x-oauth-basic' === $auth['password']) { - $options['github-token'] = $auth['username']; - } elseif ($this->config && in_array($originUrl, $this->config->get('gitlab-domains'), true)) { - if ($auth['password'] === 'oauth2') { - $headers[] = 'Authorization: Bearer '.$auth['username']; - } elseif ($auth['password'] === 'private-token') { - $headers[] = 'PRIVATE-TOKEN: '.$auth['username']; - } - } elseif ('bitbucket.org' === $originUrl - && $this->fileUrl !== Bitbucket::OAUTH2_ACCESS_TOKEN_URL && 'x-token-auth' === $auth['username'] - ) { - if (!$this->isPublicBitBucketDownload($this->fileUrl)) { - $headers[] = 'Authorization: Bearer ' . $auth['password']; - } - } else { - $authStr = base64_encode($auth['username'] . ':' . $auth['password']); - $headers[] = 'Authorization: Basic '.$authStr; - } - } + $headers = $this->authHelper->addAuthenticationHeader($headers, $originUrl, $this->fileUrl); $options['http']['follow_location'] = 0; @@ -961,29 +811,4 @@ class RemoteFilesystem return parse_url($url, PHP_URL_HOST).':'.$port; } - - /** - * @link https://github.com/composer/composer/issues/5584 - * - * @param string $urlToBitBucketFile URL to a file at bitbucket.org. - * - * @return bool Whether the given URL is a public BitBucket download which requires no authentication. - */ - private function isPublicBitBucketDownload($urlToBitBucketFile) - { - $domain = parse_url($urlToBitBucketFile, PHP_URL_HOST); - if (strpos($domain, 'bitbucket.org') === false) { - // Bitbucket downloads are hosted on amazonaws. - // We do not need to authenticate there at all - return true; - } - - $path = parse_url($urlToBitBucketFile, PHP_URL_PATH); - - // Path for a public download follows this pattern /{user}/{repo}/downloads/{whatever} - // {@link https://blog.bitbucket.org/2009/04/12/new-feature-downloads/} - $pathParts = explode('/', $path); - - return count($pathParts) >= 4 && $pathParts[3] == 'downloads'; - } } diff --git a/src/Composer/Util/StreamContextFactory.php b/src/Composer/Util/StreamContextFactory.php index b25b307a1..a87bc6d8b 100644 --- a/src/Composer/Util/StreamContextFactory.php +++ b/src/Composer/Util/StreamContextFactory.php @@ -41,6 +41,32 @@ final class StreamContextFactory 'max_redirects' => 20, )); + $options = array_replace_recursive($options, self::initOptions($url, $defaultOptions)); + unset($defaultOptions['http']['header']); + $options = array_replace_recursive($options, $defaultOptions); + + if (isset($options['http']['header'])) { + $options['http']['header'] = self::fixHttpHeaderField($options['http']['header']); + } + + return stream_context_create($options, $defaultParams); + } + + /** + * @param string $url + * @param array $options + * @return array ['http' => ['header' => [...], 'proxy' => '..', 'request_fulluri' => bool]] formatted as a stream context array + */ + public static function initOptions($url, array $options) + { + // Make sure the headers are in an array form + if (!isset($options['http']['header'])) { + $options['http']['header'] = array(); + } + if (is_string($options['http']['header'])) { + $options['http']['header'] = explode("\r\n", $options['http']['header']); + } + // Handle HTTP_PROXY/http_proxy on CLI only for security reasons if ((PHP_SAPI === 'cli' || PHP_SAPI === 'phpdbg') && (!empty($_SERVER['HTTP_PROXY']) || !empty($_SERVER['http_proxy']))) { $proxy = parse_url(!empty($_SERVER['http_proxy']) ? $_SERVER['http_proxy'] : $_SERVER['HTTP_PROXY']); @@ -117,42 +143,36 @@ final class StreamContextFactory } $auth = base64_encode($auth); - // Preserve headers if already set in default options - if (isset($defaultOptions['http']['header'])) { - if (is_string($defaultOptions['http']['header'])) { - $defaultOptions['http']['header'] = array($defaultOptions['http']['header']); - } - $defaultOptions['http']['header'][] = "Proxy-Authorization: Basic {$auth}"; - } else { - $options['http']['header'] = array("Proxy-Authorization: Basic {$auth}"); - } + $options['http']['header'][] = "Proxy-Authorization: Basic {$auth}"; } } - $options = array_replace_recursive($options, $defaultOptions); - - if (isset($options['http']['header'])) { - $options['http']['header'] = self::fixHttpHeaderField($options['http']['header']); - } - if (defined('HHVM_VERSION')) { $phpVersion = 'HHVM ' . HHVM_VERSION; } else { $phpVersion = 'PHP ' . PHP_MAJOR_VERSION . '.' . PHP_MINOR_VERSION . '.' . PHP_RELEASE_VERSION; } + if (extension_loaded('curl')) { + $curl = curl_version(); + $httpVersion = 'curl '.$curl['version']; + } else { + $httpVersion = 'streams'; + } + if (!isset($options['http']['header']) || false === stripos(implode('', $options['http']['header']), 'user-agent')) { $options['http']['header'][] = sprintf( - 'User-Agent: Composer/%s (%s; %s; %s%s)', - Composer::VERSION === '@package_version@' ? 'source' : Composer::VERSION, + 'User-Agent: Composer/%s (%s; %s; %s; %s%s)', + Composer::VERSION === '@package_version@' ? Composer::SOURCE_VERSION : Composer::VERSION, function_exists('php_uname') ? php_uname('s') : 'Unknown', function_exists('php_uname') ? php_uname('r') : 'Unknown', $phpVersion, + $httpVersion, getenv('CI') ? '; CI' : '' ); } - return stream_context_create($options, $defaultParams); + return $options; } /** diff --git a/tests/Composer/Test/Util/RemoteFilesystemTest.php b/tests/Composer/Test/Util/RemoteFilesystemTest.php index 7da88bc8a..8d1bf3194 100644 --- a/tests/Composer/Test/Util/RemoteFilesystemTest.php +++ b/tests/Composer/Test/Util/RemoteFilesystemTest.php @@ -17,6 +17,20 @@ use PHPUnit\Framework\TestCase; class RemoteFilesystemTest extends TestCase { + private function getConfigMock() + { + $config = $this->getMockBuilder('Composer\Config')->getMock(); + $config->expects($this->any()) + ->method('get') + ->will($this->returnCallback(function ($key) { + if ($key === 'github-domains' || $key === 'gitlab-domains') { + return array(); + } + })); + + return $config; + } + public function testGetOptionsForUrl() { $io = $this->getMockBuilder('Composer\IO\IOInterface')->getMock(); @@ -101,7 +115,7 @@ class RemoteFilesystemTest extends TestCase public function testCallbackGetFileSize() { - $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock()); + $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getConfigMock()); $this->callCallbackGet($fs, STREAM_NOTIFY_FILE_SIZE_IS, 0, '', 0, 0, 20); $this->assertAttributeEquals(20, 'bytesMax', $fs); } @@ -114,7 +128,7 @@ class RemoteFilesystemTest extends TestCase ->method('overwriteError') ; - $fs = new RemoteFilesystem($io); + $fs = new RemoteFilesystem($io, $this->getConfigMock()); $this->setAttribute($fs, 'bytesMax', 20); $this->setAttribute($fs, 'progress', true); @@ -124,7 +138,7 @@ class RemoteFilesystemTest extends TestCase public function testCallbackGetPassesThrough404() { - $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock()); + $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getConfigMock()); $this->assertNull($this->callCallbackGet($fs, STREAM_NOTIFY_FAILURE, 0, 'HTTP/1.1 404 Not Found', 404, 0, 0)); } @@ -139,7 +153,7 @@ class RemoteFilesystemTest extends TestCase ->method('setAuthentication') ->with($this->equalTo('github.com'), $this->equalTo('user'), $this->equalTo('pass')); - $fs = new RemoteFilesystem($io); + $fs = new RemoteFilesystem($io, $this->getConfigMock()); try { $fs->getContents('github.com', 'https://user:pass@github.com/composer/composer/404'); } catch (\Exception $e) { @@ -150,14 +164,14 @@ class RemoteFilesystemTest extends TestCase public function testGetContents() { - $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock()); + $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getConfigMock()); $this->assertContains('testGetContents', $fs->getContents('http://example.org', 'file://'.__FILE__)); } public function testCopy() { - $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock()); + $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getConfigMock()); $file = tempnam(sys_get_temp_dir(), 'c'); $this->assertTrue($fs->copy('http://example.org', 'file://'.__FILE__, $file)); @@ -218,7 +232,7 @@ class RemoteFilesystemTest extends TestCase ->disableOriginalConstructor() ->getMock(); - $rfs = new RemoteFilesystem($io); + $rfs = new RemoteFilesystem($io, $this->getConfigMock()); $hostname = parse_url($url, PHP_URL_HOST); $result = $rfs->getContents($hostname, $url, false); @@ -240,14 +254,6 @@ class RemoteFilesystemTest extends TestCase ->disableOriginalConstructor() ->getMock(); - $config = $this - ->getMockBuilder('Composer\Config') - ->getMock(); - $config - ->method('get') - ->withAnyParameters() - ->willReturn(array()); - $domains = array(); $io ->expects($this->any()) @@ -267,7 +273,7 @@ class RemoteFilesystemTest extends TestCase 'password' => '1A0yeK5Po3ZEeiiRiMWLivS0jirLdoGuaSGq9NvESFx1Fsdn493wUDXC8rz_1iKVRTl1GINHEUCsDxGh5lZ=', )); - $rfs = new RemoteFilesystem($io, $config); + $rfs = new RemoteFilesystem($io, $this->getConfigMock()); $hostname = parse_url($url, PHP_URL_HOST); $result = $rfs->getContents($hostname, $url, false); @@ -278,7 +284,7 @@ class RemoteFilesystemTest extends TestCase protected function callGetOptionsForUrl($io, array $args = array(), array $options = array(), $fileUrl = '') { - $fs = new RemoteFilesystem($io, null, $options); + $fs = new RemoteFilesystem($io, $this->getConfigMock(), $options); $ref = new \ReflectionMethod($fs, 'getOptionsForUrl'); $prop = new \ReflectionProperty($fs, 'fileUrl'); $ref->setAccessible(true); From 9986b797fb1b9acca9add379337717fe16a27ea0 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 16 Nov 2018 14:28:00 +0100 Subject: [PATCH 10/34] Add support for redirects/retries in curl downloader --- src/Composer/Util/Http/CurlDownloader.php | 72 +++++++++++++++++++---- src/Composer/Util/HttpDownloader.php | 35 +---------- src/Composer/Util/Url.php | 34 +++++++++++ 3 files changed, 96 insertions(+), 45 deletions(-) diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php index 2accb7a0c..83f07c44a 100644 --- a/src/Composer/Util/Http/CurlDownloader.php +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -19,6 +19,7 @@ use Composer\CaBundle\CaBundle; use Composer\Util\RemoteFilesystem; use Composer\Util\StreamContextFactory; use Composer\Util\AuthHelper; +use Composer\Util\Url; use Psr\Log\LoggerInterface; use React\Promise\Promise; @@ -132,11 +133,10 @@ class CurlDownloader } curl_setopt($curlHandle, CURLOPT_URL, $url); - curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($curlHandle, CURLOPT_MAXREDIRS, 20); + curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, false); //curl_setopt($curlHandle, CURLOPT_DNS_USE_GLOBAL_CACHE, false); curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 10); - curl_setopt($curlHandle, CURLOPT_TIMEOUT, 10); // TODO increase + curl_setopt($curlHandle, CURLOPT_TIMEOUT, 60); curl_setopt($curlHandle, CURLOPT_WRITEHEADER, $headerHandle); curl_setopt($curlHandle, CURLOPT_FILE, $bodyHandle); curl_setopt($curlHandle, CURLOPT_ENCODING, "gzip"); @@ -181,7 +181,6 @@ class CurlDownloader 'options' => $originalOptions, 'progress' => $progress, 'curlHandle' => $curlHandle, - //'callback' => $params['notification'], 'filename' => $copyTo, 'headerHandle' => $headerHandle, 'bodyHandle' => $bodyHandle, @@ -252,12 +251,24 @@ class CurlDownloader $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG); } - $response = $this->retryIfAuthNeeded($job, $response); + $result = $this->isAuthenticatedRetryNeeded($job, $response); + if ($result['retry']) { + if ($job['filename']) { + @unlink($job['filename'].'~'); + } + + $this->restartJob($job, $job['url'], array('storeAuth' => $result['storeAuth'])); + continue; + } // handle 3xx redirects, 304 Not Modified is excluded if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['redirects'] < $this->maxRedirects) { // TODO - $response = $this->handleRedirect($job, $response); + $location = $this->handleRedirect($job, $response); + if ($location) { + $this->restartJob($job, $location, array('redirects' => $job['attributes']['redirects'] + 1)); + continue; + } } // fail 4xx and 5xx responses and capture the response @@ -331,7 +342,39 @@ class CurlDownloader } } - private function retryIfAuthNeeded(array $job, Response $response) + private function handleRedirect(array $job, Response $response) + { + if ($locationHeader = $response->getHeader('location')) { + if (parse_url($locationHeader, PHP_URL_SCHEME)) { + // Absolute URL; e.g. https://example.com/composer + $targetUrl = $locationHeader; + } elseif (parse_url($locationHeader, PHP_URL_HOST)) { + // Scheme relative; e.g. //example.com/foo + $targetUrl = parse_url($job['url'], PHP_URL_SCHEME).':'.$locationHeader; + } elseif ('/' === $locationHeader[0]) { + // Absolute path; e.g. /foo + $urlHost = parse_url($job['url'], PHP_URL_HOST); + + // Replace path using hostname as an anchor. + $targetUrl = preg_replace('{^(.+(?://|@)'.preg_quote($urlHost).'(?::\d+)?)(?:[/\?].*)?$}', '\1'.$locationHeader, $job['url']); + } else { + // Relative path; e.g. foo + // This actually differs from PHP which seems to add duplicate slashes. + $targetUrl = preg_replace('{^(.+/)[^/?]*(?:\?.*)?$}', '\1'.$locationHeader, $job['url']); + } + } + + if (!empty($targetUrl)) { + $this->io->writeError('', true, IOInterface::DEBUG); + $this->io->writeError(sprintf('Following redirect (%u) %s', $job['redirects'] + 1, $targetUrl), true, IOInterface::DEBUG); + + return $targetUrl; + } + + throw new TransportException('The "'.$job['url'].'" file could not be downloaded, got redirect without Location ('.$response->getStatusMessage().')'); + } + + private function isAuthenticatedRetryNeeded(array $job, Response $response) { if (in_array($response->getStatusCode(), array(401, 403)) && $job['attributes']['retryAuthFailure']) { $warning = null; @@ -345,7 +388,7 @@ class CurlDownloader $result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], $response->getStatusCode(), $response->getStatusMessage(), $warning, $response->getHeaders()); if ($result['retry']) { - // TODO retry somehow using $result['storeAuth'] in the attributes + return $result; } } @@ -376,15 +419,22 @@ class CurlDownloader if ($job['attributes']['retryAuthFailure']) { $result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], 401); if ($result['retry']) { - // TODO ... - // TODO return early here to abort failResponse + return $result; } } throw $this->failResponse($job, $response, $needsAuthRetry); } - return $response; + return array('retry' => false, 'storeAuth' => false); + } + + private function restartJob(array $job, $url, array $attributes = array()) + { + $attributes = array_merge($job['attributes'], $attributes); + $origin = Url::getOrigin($this->config, $url); + + $this->initDownload($job['resolve'], $job['reject'], $origin, $url, $job['originalOptions'], $job['filename'], $attributes); } private function failResponse(array $job, Response $response, $errorMessage) diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php index 9cdc0c919..da3906728 100644 --- a/src/Composer/Util/HttpDownloader.php +++ b/src/Composer/Util/HttpDownloader.php @@ -142,7 +142,7 @@ class HttpDownloader $rfs = $this->rfs; $io = $this->io; - $origin = $this->getOrigin($job['request']['url']); + $origin = Url::getOrigin($this->config, $job['request']['url']); if ($curl && preg_match('{^https?://}i', $job['request']['url'])) { $resolver = function ($resolve, $reject) use (&$job, $curl, $origin) { @@ -250,37 +250,4 @@ class HttpDownloader return $resp; } - - private function getOrigin($url) - { - if (0 === strpos($url, 'file://')) { - return $url; - } - - $origin = parse_url($url, PHP_URL_HOST); - - if (strpos($origin, '.github.com') === (strlen($origin) - 11)) { - return 'github.com'; - } - - if ($origin === 'repo.packagist.org') { - return 'packagist.org'; - } - - // Gitlab can be installed in a non-root context (i.e. gitlab.com/foo). When downloading archives the originUrl - // is the host without the path, so we look for the registered gitlab-domains matching the host here - if ( - is_array($this->config->get('gitlab-domains')) - && false === strpos($origin, '/') - && !in_array($origin, $this->config->get('gitlab-domains')) - ) { - foreach ($this->config->get('gitlab-domains') as $gitlabDomain) { - if (0 === strpos($gitlabDomain, $origin)) { - return $gitlabDomain; - } - } - } - - return $origin ?: $url; - } } diff --git a/src/Composer/Util/Url.php b/src/Composer/Util/Url.php index 4a5d5f90c..3266c4220 100644 --- a/src/Composer/Util/Url.php +++ b/src/Composer/Util/Url.php @@ -52,4 +52,38 @@ class Url return $url; } + + public static function getOrigin(Config $config, $url) + { + if (0 === strpos($url, 'file://')) { + return $url; + } + + $origin = parse_url($url, PHP_URL_HOST); + + if (strpos($origin, '.github.com') === (strlen($origin) - 11)) { + return 'github.com'; + } + + if ($origin === 'repo.packagist.org') { + return 'packagist.org'; + } + + // Gitlab can be installed in a non-root context (i.e. gitlab.com/foo). When downloading archives the originUrl + // is the host without the path, so we look for the registered gitlab-domains matching the host here + if ( + is_array($config->get('gitlab-domains')) + && false === strpos($origin, '/') + && !in_array($origin, $config->get('gitlab-domains')) + ) { + foreach ($config->get('gitlab-domains') as $gitlabDomain) { + if (0 === strpos($gitlabDomain, $origin)) { + return $gitlabDomain; + } + } + } + + return $origin ?: $url; + } + } From 64384f8b15314e618f1a6821dd71325e8b8178ab Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 16 Nov 2018 14:43:25 +0100 Subject: [PATCH 11/34] Fix tests --- src/Composer/Util/HttpDownloader.php | 14 ++--- src/Composer/Util/Url.php | 8 ++- .../Composer/Test/Util/HttpDownloaderTest.php | 51 +++++++++++++++++++ .../Test/Util/RemoteFilesystemTest.php | 19 ------- 4 files changed, 64 insertions(+), 28 deletions(-) create mode 100644 tests/Composer/Test/Util/HttpDownloaderTest.php diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php index da3906728..b259ef3e5 100644 --- a/src/Composer/Util/HttpDownloader.php +++ b/src/Composer/Util/HttpDownloader.php @@ -126,11 +126,6 @@ class HttpDownloader private function addJob($request, $sync = false) { - // capture username/password from URL if there is one - if (preg_match('{^https?://([^:/]+):([^@/]+)@([^/]+)}i', $request['url'], $match)) { - $this->io->setAuthentication($originUrl, rawurldecode($match[1]), rawurldecode($match[2])); - } - $job = array( 'id' => $this->idGen++, 'status' => self::STATUS_QUEUED, @@ -138,12 +133,17 @@ class HttpDownloader 'sync' => $sync, ); + $origin = Url::getOrigin($this->config, $job['request']['url']); + + // capture username/password from URL if there is one + if (preg_match('{^https?://([^:/]+):([^@/]+)@([^/]+)}i', $request['url'], $match)) { + $this->io->setAuthentication($origin, rawurldecode($match[1]), rawurldecode($match[2])); + } + $curl = $this->curl; $rfs = $this->rfs; $io = $this->io; - $origin = Url::getOrigin($this->config, $job['request']['url']); - if ($curl && preg_match('{^https?://}i', $job['request']['url'])) { $resolver = function ($resolve, $reject) use (&$job, $curl, $origin) { // start job diff --git a/src/Composer/Util/Url.php b/src/Composer/Util/Url.php index 3266c4220..2874524ed 100644 --- a/src/Composer/Util/Url.php +++ b/src/Composer/Util/Url.php @@ -59,7 +59,7 @@ class Url return $url; } - $origin = parse_url($url, PHP_URL_HOST); + $origin = (string) parse_url($url, PHP_URL_HOST); if (strpos($origin, '.github.com') === (strlen($origin) - 11)) { return 'github.com'; @@ -69,6 +69,10 @@ class Url return 'packagist.org'; } + if ($origin === '') { + $origin = $url; + } + // Gitlab can be installed in a non-root context (i.e. gitlab.com/foo). When downloading archives the originUrl // is the host without the path, so we look for the registered gitlab-domains matching the host here if ( @@ -83,7 +87,7 @@ class Url } } - return $origin ?: $url; + return $origin; } } diff --git a/tests/Composer/Test/Util/HttpDownloaderTest.php b/tests/Composer/Test/Util/HttpDownloaderTest.php new file mode 100644 index 000000000..b65aa760a --- /dev/null +++ b/tests/Composer/Test/Util/HttpDownloaderTest.php @@ -0,0 +1,51 @@ + + * Jordi Boggiano + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Composer\Test\Util; + +use Composer\Util\HttpDownloader; +use PHPUnit\Framework\TestCase; + +class HttpDownloaderTest extends TestCase +{ + private function getConfigMock() + { + $config = $this->getMockBuilder('Composer\Config')->getMock(); + $config->expects($this->any()) + ->method('get') + ->will($this->returnCallback(function ($key) { + if ($key === 'github-domains' || $key === 'gitlab-domains') { + return array(); + } + })); + + return $config; + } + + /** + * @group slow + */ + public function testCaptureAuthenticationParamsFromUrl() + { + $io = $this->getMockBuilder('Composer\IO\IOInterface')->getMock(); + $io->expects($this->once()) + ->method('setAuthentication') + ->with($this->equalTo('github.com'), $this->equalTo('user'), $this->equalTo('pass')); + + $fs = new HttpDownloader($io, $this->getConfigMock()); + try { + $fs->get('https://user:pass@github.com/composer/composer/404'); + } catch (\Composer\Downloader\TransportException $e) { + $this->assertNotEquals(200, $e->getCode()); + } + } +} diff --git a/tests/Composer/Test/Util/RemoteFilesystemTest.php b/tests/Composer/Test/Util/RemoteFilesystemTest.php index 8d1bf3194..2c7f3112a 100644 --- a/tests/Composer/Test/Util/RemoteFilesystemTest.php +++ b/tests/Composer/Test/Util/RemoteFilesystemTest.php @@ -143,25 +143,6 @@ class RemoteFilesystemTest extends TestCase $this->assertNull($this->callCallbackGet($fs, STREAM_NOTIFY_FAILURE, 0, 'HTTP/1.1 404 Not Found', 404, 0, 0)); } - /** - * @group slow - */ - public function testCaptureAuthenticationParamsFromUrl() - { - $io = $this->getMockBuilder('Composer\IO\IOInterface')->getMock(); - $io->expects($this->once()) - ->method('setAuthentication') - ->with($this->equalTo('github.com'), $this->equalTo('user'), $this->equalTo('pass')); - - $fs = new RemoteFilesystem($io, $this->getConfigMock()); - try { - $fs->getContents('github.com', 'https://user:pass@github.com/composer/composer/404'); - } catch (\Exception $e) { - $this->assertInstanceOf('Composer\Downloader\TransportException', $e); - $this->assertNotEquals(200, $e->getCode()); - } - } - public function testGetContents() { $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getConfigMock()); From 5d2b3276ebf6b98114eb73a5bd315e59f0f08c0a Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 16 Nov 2018 15:38:01 +0100 Subject: [PATCH 12/34] Avoid starting all jobs immediately --- src/Composer/Util/Http/CurlDownloader.php | 36 +++---- src/Composer/Util/HttpDownloader.php | 110 +++++++++++++++------- 2 files changed, 92 insertions(+), 54 deletions(-) diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php index 83f07c44a..4015b0bac 100644 --- a/src/Composer/Util/Http/CurlDownloader.php +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -95,12 +95,17 @@ class CurlDownloader public function download($resolve, $reject, $origin, $url, $options, $copyTo = null) { - return $this->initDownload($resolve, $reject, $origin, $url, $options, $copyTo); + $attributes = array(); + if (isset($options['retry-auth-failure'])) { + $attributes['retryAuthFailure'] = $options['retry-auth-failure']; + unset($options['retry-auth-failure']); + } + + return $this->initDownload($resolve, $reject, $origin, $url, $options, $copyTo, $attributes); } private function initDownload($resolve, $reject, $origin, $url, $options, $copyTo = null, array $attributes = array()) { - // TODO allow setting attributes somehow $attributes = array_merge(array( 'retryAuthFailure' => true, 'redirects' => 1, @@ -193,12 +198,12 @@ class CurlDownloader $this->io->writeError('Downloading ' . $url . $usingProxy . $ifModified, true, IOInterface::DEBUG); $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $curlHandle)); +// TODO progress //$params['notification'](STREAM_NOTIFY_RESOLVE, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, 0, false); } public function tick() { - // TODO check we have active handles before doing this if (!$this->jobs) { return; } @@ -229,6 +234,7 @@ class CurlDownloader $statusCode = null; $response = null; try { +// TODO progress //$this->onProgress($curlHandle, $job['callback'], $progress, $job['progress']); if (CURLE_OK !== $errno) { throw new TransportException($error); @@ -263,7 +269,6 @@ class CurlDownloader // handle 3xx redirects, 304 Not Modified is excluded if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['redirects'] < $this->maxRedirects) { - // TODO $location = $this->handleRedirect($job, $response); if ($location) { $this->restartJob($job, $location, array('redirects' => $job['attributes']['redirects'] + 1)); @@ -274,6 +279,7 @@ class CurlDownloader // fail 4xx and 5xx responses and capture the response if ($statusCode >= 400 && $statusCode <= 599) { throw $this->failResponse($job, $response, $response->getStatusMessage()); +// TODO progress // $this->io->overwriteError("Downloading (failed)", false); } @@ -320,24 +326,13 @@ class CurlDownloader if ($this->jobs[$i]['progress'] !== $progress) { $previousProgress = $this->jobs[$i]['progress']; $this->jobs[$i]['progress'] = $progress; - try { - //$this->onProgress($curlHandle, $this->jobs[$i]['callback'], $progress, $previousProgress); - } catch (TransportException $e) { - var_dump('Caught '.$e->getMessage());die; - unset($this->jobs[$i]); - curl_multi_remove_handle($this->multiHandle, $curlHandle); - curl_close($curlHandle); - fclose($job['headerHandle']); - fclose($job['bodyHandle']); - if ($job['filename']) { - @unlink($job['filename'].'~'); - } - call_user_func($job['reject'], $e); - } + // TODO + //$this->onProgress($curlHandle, $this->jobs[$i]['callback'], $progress, $previousProgress); } } } catch (\Exception $e) { + // TODO var_dump('Caught2', get_class($e), $e->getMessage(), $e);die; } } @@ -444,13 +439,10 @@ class CurlDownloader private function onProgress($curlHandle, callable $notify, array $progress, array $previousProgress) { + // TODO add support for progress if (300 <= $progress['http_code'] && $progress['http_code'] < 400) { return; } - if (!$previousProgress['http_code'] && $progress['http_code'] && $progress['http_code'] < 200 || 400 <= $progress['http_code']) { - $code = 403 === $progress['http_code'] ? STREAM_NOTIFY_AUTH_RESULT : STREAM_NOTIFY_FAILURE; - $notify($code, STREAM_NOTIFY_SEVERITY_ERR, curl_error($curlHandle), $progress['http_code'], 0, 0, false); - } if ($previousProgress['download_content_length'] < $progress['download_content_length']) { $notify(STREAM_NOTIFY_FILE_SIZE_IS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, (int) $progress['download_content_length'], false); } diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php index b259ef3e5..0e882c4a3 100644 --- a/src/Composer/Util/HttpDownloader.php +++ b/src/Composer/Util/HttpDownloader.php @@ -33,8 +33,8 @@ class HttpDownloader private $config; private $jobs = array(); private $options = array(); - private $index; - private $progress; + private $runningJobs = 0; + private $maxJobs = 10; private $lastProgress; private $disableTls = false; private $curl; @@ -42,8 +42,6 @@ class HttpDownloader private $idGen = 0; /** - * Constructor. - * * @param IOInterface $io The IO instance * @param Config $config The config * @param array $options The options @@ -131,35 +129,24 @@ class HttpDownloader 'status' => self::STATUS_QUEUED, 'request' => $request, 'sync' => $sync, + 'origin' => Url::getOrigin($this->config, $request['url']), ); - $origin = Url::getOrigin($this->config, $job['request']['url']); - // capture username/password from URL if there is one if (preg_match('{^https?://([^:/]+):([^@/]+)@([^/]+)}i', $request['url'], $match)) { - $this->io->setAuthentication($origin, rawurldecode($match[1]), rawurldecode($match[2])); + $this->io->setAuthentication($job['origin'], rawurldecode($match[1]), rawurldecode($match[2])); } - $curl = $this->curl; $rfs = $this->rfs; - $io = $this->io; - if ($curl && preg_match('{^https?://}i', $job['request']['url'])) { - $resolver = function ($resolve, $reject) use (&$job, $curl, $origin) { - // start job - $url = $job['request']['url']; - $options = $job['request']['options']; - - $job['status'] = HttpDownloader::STATUS_STARTED; - - if ($job['request']['copyTo']) { - $curl->download($resolve, $reject, $origin, $url, $options, $job['request']['copyTo']); - } else { - $curl->download($resolve, $reject, $origin, $url, $options); - } + if ($this->curl && preg_match('{^https?://}i', $job['request']['url'])) { + $resolver = function ($resolve, $reject) use (&$job) { + $job['status'] = HttpDownloader::STATUS_QUEUED; + $job['resolve'] = $resolve; + $job['reject'] = $reject; }; } else { - $resolver = function ($resolve, $reject) use (&$job, $rfs, $curl, $origin) { + $resolver = function ($resolve, $reject) use (&$job, $rfs) { // start job $url = $job['request']['url']; $options = $job['request']['options']; @@ -167,11 +154,11 @@ class HttpDownloader $job['status'] = HttpDownloader::STATUS_STARTED; if ($job['request']['copyTo']) { - $result = $rfs->copy($origin, $url, $job['request']['copyTo'], false /* TODO progress */, $options); + $result = $rfs->copy($job['origin'], $url, $job['request']['copyTo'], false /* TODO progress */, $options); $resolve($result); } else { - $body = $rfs->getContents($origin, $url, false /* TODO progress */, $options); + $body = $rfs->getContents($job['origin'], $url, false /* TODO progress */, $options); $headers = $rfs->getLastHeaders(); $response = new Http\Response($job['request'], $rfs->findStatusCode($headers), $headers, $body); @@ -180,26 +167,85 @@ class HttpDownloader }; } + $downloader = $this; + $io = $this->io; + $canceler = function () {}; $promise = new Promise($resolver, $canceler); - $promise->then(function ($response) use (&$job) { + $promise->then(function ($response) use (&$job, $downloader) { $job['status'] = HttpDownloader::STATUS_COMPLETED; $job['response'] = $response; - // TODO look for more jobs to start once we throttle to max X jobs - }, function ($e) use ($io, &$job) { - // var_dump(__CLASS__ . __LINE__); - // var_dump(get_class($e)); - // var_dump($e->getMessage()); - // die; + + // TODO 3.0 this should be done directly on $this when PHP 5.3 is dropped + $downloader->markJobDone(); + $downloader->scheduleNextJob(); + + return $response; + }, function ($e) use ($io, &$job, $downloader) { $job['status'] = HttpDownloader::STATUS_FAILED; $job['exception'] = $e; + + $downloader->markJobDone(); + + throw $e; }); $this->jobs[$job['id']] =& $job; + if ($this->runningJobs < $this->maxJobs) { + $this->startJob($job['id']); + } + return array($job, $promise); } + private function startJob($id) + { + $job =& $this->jobs[$id]; + if ($job['status'] !== self::STATUS_QUEUED) { + return; + } + + // start job + $job['status'] = self::STATUS_STARTED; + $this->runningJobs++; + + $resolve = $job['resolve']; + $reject = $job['reject']; + $url = $job['request']['url']; + $options = $job['request']['options']; + $origin = $job['origin']; + + if ($job['request']['copyTo']) { + $this->curl->download($resolve, $reject, $origin, $url, $options, $job['request']['copyTo']); + } else { + $this->curl->download($resolve, $reject, $origin, $url, $options); + } + } + + /** + * @private + */ + public function markJobDone() + { + $this->runningJobs--; + } + + /** + * @private + */ + public function scheduleNextJob() + { + foreach ($this->jobs as $job) { + if ($job['status'] === self::STATUS_QUEUED) { + $this->startJob($job['id']); + if ($this->runningJobs >= $this->maxJobs) { + return; + } + } + } + } + public function wait($index = null, $progress = false) { while (true) { From 788a822b240dfc51569ebe842079a8a05f274ff9 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 16 Nov 2018 17:35:06 +0100 Subject: [PATCH 13/34] Add some phpdocs --- .../Repository/PlatformRepository.php | 4 ++ src/Composer/Util/AuthHelper.php | 47 +++++++++++++------ src/Composer/Util/Url.php | 11 ++++- 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/Composer/Repository/PlatformRepository.php b/src/Composer/Repository/PlatformRepository.php index 6d6e04d2f..f4c38f3ea 100644 --- a/src/Composer/Repository/PlatformRepository.php +++ b/src/Composer/Repository/PlatformRepository.php @@ -308,6 +308,10 @@ class PlatformRepository extends ArrayRepository $this->addPackage($ext); } + /** + * @param string $name + * @return string + */ private function buildPackageName($name) { return 'ext-' . str_replace(' ', '-', $name); diff --git a/src/Composer/Util/AuthHelper.php b/src/Composer/Util/AuthHelper.php index e80a5b0c3..3679b93da 100644 --- a/src/Composer/Util/AuthHelper.php +++ b/src/Composer/Util/AuthHelper.php @@ -30,7 +30,11 @@ class AuthHelper $this->config = $config; } - public function storeAuth($originUrl, $storeAuth) + /** + * @param string $origin + * @param string|bool $storeAuth + */ + public function storeAuth($origin, $storeAuth) { $store = false; $configSource = $this->config->getAuthConfigSource(); @@ -38,7 +42,7 @@ class AuthHelper $store = $configSource; } elseif ($storeAuth === 'prompt') { $answer = $this->io->askAndValidate( - 'Do you want to store credentials for '.$originUrl.' in '.$configSource->getName().' ? [Yn] ', + 'Do you want to store credentials for '.$origin.' in '.$configSource->getName().' ? [Yn] ', function ($value) { $input = strtolower(substr(trim($value), 0, 1)); if (in_array($input, array('y','n'))) { @@ -56,14 +60,23 @@ class AuthHelper } if ($store) { $store->addConfigSetting( - 'http-basic.'.$originUrl, - $this->io->getAuthentication($originUrl) + 'http-basic.'.$origin, + $this->io->getAuthentication($origin) ); } } - - public function promptAuthIfNeeded($url, $origin, $httpStatus, $reason = null, $warning = null, $headers = array()) + /** + * @param string $url + * @param string $origin + * @param int $statusCode HTTP status code that triggered this call + * @param string|null $reason a message/description explaining why this was called + * @param string $warning an authentication warning returned by the server as {"warning": ".."}, if present + * @param string[] $headers + * @return array containing retry (bool) and storeAuth (string|bool) keys, if retry is true the request should be + * retried, if storeAuth is true then on a successful retry the authentication should be persisted to auth.json + */ + public function promptAuthIfNeeded($url, $origin, $statusCode, $reason = null, $warning = null, $headers = array()) { $storeAuth = false; $retry = false; @@ -101,11 +114,11 @@ class AuthHelper throw new TransportException('Could not authenticate against '.$origin, 401); } } elseif (in_array($origin, $this->config->get('gitlab-domains'), true)) { - $message = "\n".'Could not fetch '.$url.', enter your ' . $origin . ' credentials ' .($httpStatus === 401 ? 'to access private repos' : 'to go over the API rate limit'); + $message = "\n".'Could not fetch '.$url.', enter your ' . $origin . ' credentials ' .($statusCode === 401 ? 'to access private repos' : 'to go over the API rate limit'); $gitLabUtil = new GitLab($this->io, $this->config, null); if ($this->io->hasAuthentication($origin) && ($auth = $this->io->getAuthentication($origin)) && $auth['password'] === 'private-token') { - throw new TransportException("Invalid credentials for '" . $url . "', aborting.", $httpStatus); + throw new TransportException("Invalid credentials for '" . $url . "', aborting.", $statusCode); } if (!$gitLabUtil->authorizeOAuth($origin) @@ -130,7 +143,7 @@ class AuthHelper } if ($askForOAuthToken) { - $message = "\n".'Could not fetch ' . $url . ', please create a bitbucket OAuth token to ' . (($httpStatus === 401 || $httpStatus === 403) ? 'access private repos' : 'go over the API rate limit'); + $message = "\n".'Could not fetch ' . $url . ', please create a bitbucket OAuth token to ' . (($statusCode === 401 || $statusCode === 403) ? 'access private repos' : 'go over the API rate limit'); $bitBucketUtil = new Bitbucket($this->io, $this->config); if (! $bitBucketUtil->authorizeOAuth($origin) && (! $this->io->isInteractive() || !$bitBucketUtil->authorizeOAuthInteractively($origin, $message)) @@ -140,24 +153,24 @@ class AuthHelper } } else { // 404s are only handled for github - if ($httpStatus === 404) { + if ($statusCode === 404) { return; } // fail if the console is not interactive if (!$this->io->isInteractive()) { - if ($httpStatus === 401) { + if ($statusCode === 401) { $message = "The '" . $url . "' URL required authentication.\nYou must be using the interactive console to authenticate"; } - if ($httpStatus === 403) { + if ($statusCode === 403) { $message = "The '" . $url . "' URL could not be accessed: " . $reason; } - throw new TransportException($message, $httpStatus); + throw new TransportException($message, $statusCode); } // fail if we already have auth if ($this->io->hasAuthentication($origin)) { - throw new TransportException("Invalid credentials for '" . $url . "', aborting.", $httpStatus); + throw new TransportException("Invalid credentials for '" . $url . "', aborting.", $statusCode); } $this->io->overwriteError(''); @@ -176,6 +189,12 @@ class AuthHelper return array('retry' => $retry, 'storeAuth' => $storeAuth); } + /** + * @param array $headers + * @param string $origin + * @param string $url + * @return array updated headers array + */ public function addAuthenticationHeader(array $headers, $origin, $url) { if ($this->io->hasAuthentication($origin)) { diff --git a/src/Composer/Util/Url.php b/src/Composer/Util/Url.php index 2874524ed..c01677522 100644 --- a/src/Composer/Util/Url.php +++ b/src/Composer/Util/Url.php @@ -19,6 +19,12 @@ use Composer\Config; */ class Url { + /** + * @param Config $config + * @param string $url + * @param string $ref + * @return string the updated URL + */ public static function updateDistReference(Config $config, $url, $ref) { $host = parse_url($url, PHP_URL_HOST); @@ -53,6 +59,10 @@ class Url return $url; } + /** + * @param string $url + * @return string + */ public static function getOrigin(Config $config, $url) { if (0 === strpos($url, 'file://')) { @@ -89,5 +99,4 @@ class Url return $origin; } - } From ed65625126c42354f3b27a3ed922eb75de8f57d7 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Wed, 28 Nov 2018 08:59:45 +0100 Subject: [PATCH 14/34] Handle custom http options cleaner in ComposerRepo --- src/Composer/Repository/ComposerRepository.php | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 8c92d3e4d..939fb84ba 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -102,11 +102,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->cache = new Cache($io, $config->get('cache-repo-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', $this->url), 'a-z0-9.$'); $this->versionParser = new VersionParser(); $this->loader = new ArrayLoader($this->versionParser); - if ($this->options) { - // TODO solve this somehow - should be sent at request time not on the instance - $httpDownloader = clone $httpDownloader; - $httpDownloader->setOptions($this->options); - } $this->httpDownloader = $httpDownloader; $this->eventDispatcher = $eventDispatcher; $this->repoConfig = $repoConfig; @@ -263,7 +258,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito if ($this->searchUrl && $mode === self::SEARCH_FULLTEXT) { $url = str_replace(array('%query%', '%type%'), array($query, $type), $this->searchUrl); - $search = $this->httpDownloader->get($url)->decodeJson(); + $search = $this->httpDownloader->get($url, $this->options)->decodeJson(); if (empty($search['results'])) { return array(); @@ -826,7 +821,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); } - $response = $httpDownloader->get($filename); + $response = $httpDownloader->get($filename, $this->options); $json = $response->getBody(); if ($sha256 && $sha256 !== hash('sha256', $json)) { // undo downgrade before trying again if http seems to be hijacked or modifying content somehow @@ -917,7 +912,11 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); } - $options = array('http' => array('header' => array('If-Modified-Since: '.$lastModifiedTime))); + $options = $this->options; + if (isset($options['http']['header'])) { + $options['http']['header'] = (array) $options['http']['header']; + } + $options['http']['header'][] = array('If-Modified-Since: '.$lastModifiedTime); $response = $httpDownloader->get($filename, $options); $json = $response->getBody(); if ($json === '' && $response->getStatusCode() === 304) { From e67030076a135bb9e1c1f1201441128c581b307e Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Wed, 28 Nov 2018 09:36:05 +0100 Subject: [PATCH 15/34] Fix show command --- src/Composer/Command/ShowCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Composer/Command/ShowCommand.php b/src/Composer/Command/ShowCommand.php index 1dc5876a8..9604bdc71 100644 --- a/src/Composer/Command/ShowCommand.php +++ b/src/Composer/Command/ShowCommand.php @@ -553,7 +553,7 @@ EOT $matches[$index] = $package->getId(); } - $pool = $repositorySet->createPool(); + $pool = $repositorySet->createPoolForPackage($package->getName()); // select preferred package according to policy rules if (!$matchedPackage && $matches && $preferred = $policy->selectPreferredPackages($pool, array(), $matches)) { From 0961e16795dffb5a1ed10eac209e28e44dfcdb2e Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Thu, 29 Nov 2018 19:31:41 +0100 Subject: [PATCH 16/34] Add support for new metadata-url repo attribute --- .../Repository/ComposerRepository.php | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 939fb84ba..101c23bcb 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -667,6 +667,17 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->hasPartialPackages = !empty($data['packages']) && is_array($data['packages']); } + // metadata-url indiates V2 repo protocol so it takes over from all the V1 types + // V2 only has lazyProviders and no ability to process anything else, plus support for async loading + if (!empty($data['metadata-url'])) { + $this->lazyProvidersUrl = $this->canonicalizeUrl($data['metadata-url']); + $this->providersUrl = null; + $this->hasProviders = false; + $this->hasPartialPackages = false; + $this->allowSslDowngrade = false; + unset($data['providers-url'], $data['providers'], $data['providers-includes']); + } + if ($this->allowSslDowngrade) { $this->url = str_replace('https://', 'http://', $this->url); $this->baseUrl = str_replace('https://', 'http://', $this->baseUrl); @@ -681,22 +692,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->hasProviders = true; } - // TODO this is for testing only, remove once packagist reports v2 protocol support - if (preg_match('{^https?://repo\.packagist\.org/?$}i', $this->url)) { - $this->repoConfig['force-lazy-providers'] = true; - } - - // force values for packagist - if (preg_match('{^https?://repo\.packagist\.org/?$}i', $this->url) && !empty($this->repoConfig['force-lazy-providers'])) { - $this->url = 'https://repo.packagist.org'; - $this->baseUrl = 'https://repo.packagist.org'; - $this->lazyProvidersUrl = $this->canonicalizeUrl('https://repo.packagist.org/p/%package%.json'); - $this->providersUrl = null; - } elseif (!empty($this->repoConfig['force-lazy-providers'])) { - $this->lazyProvidersUrl = $this->canonicalizeUrl('/p/%package%.json'); - $this->providersUrl = null; - } - return $this->rootData = $data; } From e753bf08b1cd3c2438a7e9aca0b94f5957626704 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Tue, 4 Dec 2018 09:34:32 +0100 Subject: [PATCH 17/34] Minor tweaks --- composer.lock | 2 +- src/Composer/DependencyResolver/Solver.php | 1 + src/Composer/Repository/ComposerRepository.php | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 63b8033b9..a22e11a6b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d356b92e869790db1e9d2c0f4b10935e", + "content-hash": "3243ce6f26231df34d1bceab1a148803", "packages": [ { "name": "composer/ca-bundle", diff --git a/src/Composer/DependencyResolver/Solver.php b/src/Composer/DependencyResolver/Solver.php index aa5432188..c3128c6c4 100644 --- a/src/Composer/DependencyResolver/Solver.php +++ b/src/Composer/DependencyResolver/Solver.php @@ -217,6 +217,7 @@ class Solver $this->setupInstalledMap(); + $this->io->writeError('Generating rules', true, IOInterface::DEBUG); $this->ruleSetGenerator = new RuleSetGenerator($this->policy, $this->pool); $this->rules = $this->ruleSetGenerator->getRulesFor($this->jobs, $this->installedMap, $ignorePlatformReqs); $this->checkForRootRequireProblems($ignorePlatformReqs); diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 101c23bcb..a491bec72 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -527,6 +527,8 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito // TODO what if not, then throw? if ($this->lazyProvidersUrl) { foreach ($packageNames as $name => $constraint) { + $name = strtolower($name); + // skip platform packages, root package and composer-plugin-api if (preg_match(PlatformRepository::PLATFORM_PACKAGE_REGEX, $name) || '__root__' === $name || 'composer-plugin-api' === $name) { continue; From e8c694877047f331e042c17a4ac3b63b1a25962d Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Tue, 4 Dec 2018 11:20:35 +0100 Subject: [PATCH 18/34] Deduplicate link instances between versions of a given package --- src/Composer/Package/Loader/ArrayLoader.php | 155 ++++++++++++---- .../Repository/ComposerRepository.php | 170 ++++++++++-------- .../Repository/ComposerRepositoryTest.php | 17 +- 3 files changed, 230 insertions(+), 112 deletions(-) diff --git a/src/Composer/Package/Loader/ArrayLoader.php b/src/Composer/Package/Loader/ArrayLoader.php index 49ba45aa8..ac91b18fe 100644 --- a/src/Composer/Package/Loader/ArrayLoader.php +++ b/src/Composer/Package/Loader/ArrayLoader.php @@ -38,6 +38,70 @@ class ArrayLoader implements LoaderInterface } public function load(array $config, $class = 'Composer\Package\CompletePackage') + { + $package = $this->createObject($config, $class); + + foreach (Package\BasePackage::$supportedLinkTypes as $type => $opts) { + if (isset($config[$type])) { + $method = 'set'.ucfirst($opts['method']); + $package->{$method}( + $this->parseLinks( + $package->getName(), + $package->getPrettyVersion(), + $opts['description'], + $config[$type] + ) + ); + } + } + + $package = $this->configureObject($package, $config); + + return $package; + } + + public function loadPackages(array $versions, $class) + { + static $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); + + $packages = array(); + $linkCache = array(); + + foreach ($versions as $version) { + if (isset($version['versions'])) { + $baseVersion = $version; + foreach ($uniqKeys as $key) { + unset($baseVersion[$key.'s']); + } + + foreach ($version['versions'] as $index => $dummy) { + $unpackedVersion = $baseVersion; + foreach ($uniqKeys as $key) { + $unpackedVersion[$key] = $version[$key.'s'][$index]; + } + + $package = $this->createObject($unpackedVersion, $class); + + $this->configureCachedLinks($linkCache, $package, $unpackedVersion); + $package = $this->configureObject($package, $unpackedVersion); + + $packages[] = $package; + } + } else { + $package = $this->createObject($version, $class); + + $this->configureCachedLinks($linkCache, $package, $version); + $package = $this->configureObject($package, $version); + + $packages[] = $package; + } + + } + + return $packages; + } + + private function createObject(array $config, $class) { if (!isset($config['name'])) { throw new \UnexpectedValueException('Unknown package has no name defined ('.json_encode($config).').'); @@ -52,7 +116,12 @@ class ArrayLoader implements LoaderInterface } else { $version = $this->versionParser->normalize($config['version']); } - $package = new $class($config['name'], $version, $config['version']); + + return new $class($config['name'], $version, $config['version']); + } + + private function configureObject($package, array $config) + { $package->setType(isset($config['type']) ? strtolower($config['type']) : 'library'); if (isset($config['target-dir'])) { @@ -109,20 +178,6 @@ class ArrayLoader implements LoaderInterface } } - foreach (Package\BasePackage::$supportedLinkTypes as $type => $opts) { - if (isset($config[$type])) { - $method = 'set'.ucfirst($opts['method']); - $package->{$method}( - $this->parseLinks( - $package->getName(), - $package->getPrettyVersion(), - $opts['description'], - $config[$type] - ) - ); - } - } - if (isset($config['suggest']) && is_array($config['suggest'])) { foreach ($config['suggest'] as $target => $reason) { if ('self.version' === trim($reason)) { @@ -202,21 +257,50 @@ class ArrayLoader implements LoaderInterface } } - if ($aliasNormalized = $this->getBranchAlias($config)) { - if ($package instanceof RootPackageInterface) { - $package = new RootAliasPackage($package, $aliasNormalized, preg_replace('{(\.9{7})+}', '.x', $aliasNormalized)); - } else { - $package = new AliasPackage($package, $aliasNormalized, preg_replace('{(\.9{7})+}', '.x', $aliasNormalized)); - } - } - if ($this->loadOptions && isset($config['transport-options'])) { $package->setTransportOptions($config['transport-options']); } + if ($aliasNormalized = $this->getBranchAlias($config)) { + if ($package instanceof RootPackageInterface) { + return new RootAliasPackage($package, $aliasNormalized, preg_replace('{(\.9{7})+}', '.x', $aliasNormalized)); + } + + return new AliasPackage($package, $aliasNormalized, preg_replace('{(\.9{7})+}', '.x', $aliasNormalized)); + } + return $package; } + private function configureCachedLinks(&$linkCache, $package, array $config) + { + $name = $package->getName(); + $prettyVersion = $package->getPrettyVersion(); + + foreach (Package\BasePackage::$supportedLinkTypes as $type => $opts) { + if (isset($config[$type])) { + $method = 'set'.ucfirst($opts['method']); + + $links = array(); + foreach ($config[$type] as $prettyTarget => $constraint) { + $target = strtolower($prettyTarget); + if ($constraint === 'self.version') { + $links[$target] = $this->createLink($name, $prettyVersion, $opts['description'], $target, $constraint); + } else { + if (!isset($linkCache[$name][$type][$target][$constraint])) { + $linkCache[$name][$type][$target][$constraint] = array($target, $this->createLink($name, $prettyVersion, $opts['description'], $target, $constraint)); + } + + list($target, $link) = $linkCache[$name][$type][$target][$constraint]; + $links[$target] = $link; + } + } + + $package->{$method}($links); + } + } + } + /** * @param string $source source package name * @param string $sourceVersion source package version (pretty version ideally) @@ -228,21 +312,26 @@ class ArrayLoader implements LoaderInterface { $res = array(); foreach ($links as $target => $constraint) { - if (!is_string($constraint)) { - throw new \UnexpectedValueException('Link constraint in '.$source.' '.$description.' > '.$target.' should be a string, got '.gettype($constraint) . ' (' . var_export($constraint, true) . ')'); - } - if ('self.version' === $constraint) { - $parsedConstraint = $this->versionParser->parseConstraints($sourceVersion); - } else { - $parsedConstraint = $this->versionParser->parseConstraints($constraint); - } - - $res[strtolower($target)] = new Link($source, $target, $parsedConstraint, $description, $constraint); + $res[strtolower($target)] = $this->createLink($source, $sourceVersion, $description, $target, $constraint); } return $res; } + private function createLink($source, $sourceVersion, $description, $target, $prettyConstraint) + { + if (!is_string($prettyConstraint)) { + throw new \UnexpectedValueException('Link constraint in '.$source.' '.$description.' > '.$target.' should be a string, got '.gettype($prettyConstraint) . ' (' . var_export($prettyConstraint, true) . ')'); + } + if ('self.version' === $prettyConstraint) { + $parsedConstraint = $this->versionParser->parseConstraints($sourceVersion); + } else { + $parsedConstraint = $this->versionParser->parseConstraints($prettyConstraint); + } + + return new Link($source, $target, $parsedConstraint, $description, $prettyConstraint); + } + /** * Retrieves a branch alias (dev-master => 1.0.x-dev for example) if it exists * diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index a491bec72..06c984bd5 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -61,7 +61,11 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito private $rootData; private $hasPartialPackages; private $partialPackagesByName; - private $versionParser; + /** + * TODO v3 should make this private once we can drop PHP 5.3 support + * @private + */ + public $versionParser; public function __construct(array $repoConfig, IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $eventDispatcher = null) { @@ -414,6 +418,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->providers[$name] = array(); foreach ($packages['packages'] as $versions) { + $versionsToLoad = array(); foreach ($versions as $version) { if (!$loadingPartialPackage && $this->hasPartialPackages && isset($this->partialPackagesByName[$version['name']])) { continue; @@ -440,40 +445,44 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito continue; } - // load acceptable packages in the providers - $package = $this->createPackage($version, 'Composer\Package\CompletePackage'); - $package->setRepository($this); + $versionsToLoad[] = $version; + } + } - if ($package instanceof AliasPackage) { - $aliased = $package->getAliasOf(); - $aliased->setRepository($this); + // load acceptable packages in the providers + $loadedPackages = $this->createPackages($versionsToLoad, 'Composer\Package\CompletePackage'); + foreach ($loadedPackages as $package) { + $package->setRepository($this); - $this->providers[$name][$version['uid']] = $aliased; - $this->providers[$name][$version['uid'].'-alias'] = $package; + if ($package instanceof AliasPackage) { + $aliased = $package->getAliasOf(); + $aliased->setRepository($this); - // override provider with its alias so it can be expanded in the if block above - $this->providersByUid[$version['uid']] = $package; - } else { - $this->providers[$name][$version['uid']] = $package; - $this->providersByUid[$version['uid']] = $package; - } + $this->providers[$name][$version['uid']] = $aliased; + $this->providers[$name][$version['uid'].'-alias'] = $package; - // handle root package aliases - unset($rootAliasData); + // override provider with its alias so it can be expanded in the if block above + $this->providersByUid[$version['uid']] = $package; + } else { + $this->providers[$name][$version['uid']] = $package; + $this->providersByUid[$version['uid']] = $package; + } - if (isset($this->rootAliases[$package->getName()][$package->getVersion()])) { - $rootAliasData = $this->rootAliases[$package->getName()][$package->getVersion()]; - } elseif ($package instanceof AliasPackage && isset($this->rootAliases[$package->getName()][$package->getAliasOf()->getVersion()])) { - $rootAliasData = $this->rootAliases[$package->getName()][$package->getAliasOf()->getVersion()]; - } + // handle root package aliases + unset($rootAliasData); - if (isset($rootAliasData)) { - $alias = $this->createAliasPackage($package, $rootAliasData['alias_normalized'], $rootAliasData['alias']); - $alias->setRepository($this); + if (isset($this->rootAliases[$package->getName()][$package->getVersion()])) { + $rootAliasData = $this->rootAliases[$package->getName()][$package->getVersion()]; + } elseif ($package instanceof AliasPackage && isset($this->rootAliases[$package->getName()][$package->getAliasOf()->getVersion()])) { + $rootAliasData = $this->rootAliases[$package->getName()][$package->getAliasOf()->getVersion()]; + } - $this->providers[$name][$version['uid'].'-root'] = $alias; - $this->providersByUid[$version['uid'].'-root'] = $alias; - } + if (isset($rootAliasData)) { + $alias = $this->createAliasPackage($package, $rootAliasData['alias_normalized'], $rootAliasData['alias']); + $alias->setRepository($this); + + $this->providers[$name][$version['uid'].'-root'] = $alias; + $this->providersByUid[$version['uid'].'-root'] = $alias; } } } @@ -501,8 +510,8 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $repoData = $this->loadDataFromServer(); - foreach ($repoData as $package) { - $this->addPackage($this->createPackage($package, 'Composer\Package\CompletePackage')); + foreach ($this->createPackages($repoData, 'Composer\Package\CompletePackage') as $package) { + $this->addPackage($package); } } @@ -545,6 +554,8 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->asyncFetchFile($url, $cacheKey, $lastModified) ->then(function ($response) use (&$packages, $contents, $name, $constraint, $repo, $isPackageAcceptableCallable) { + static $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); + if (true === $response) { $response = $contents; } @@ -553,24 +564,37 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return; } - $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); + $versionsToLoad = array(); foreach ($response['packages'][$name] as $version) { - if (isset($version['versions'])) { - $baseVersion = $version; - foreach ($uniqKeys as $key) { - unset($baseVersion[$key.'s']); - } - - foreach ($version['versions'] as $index => $dummy) { - $unpackedVersion = $baseVersion; - foreach ($uniqKeys as $key) { - $unpackedVersion[$key] = $version[$key.'s'][$index]; + if (isset($version['version_normalizeds'])) { + foreach ($version['version_normalizeds'] as $index => $normalizedVersion) { + if (!$repo->isVersionAcceptable($isPackageAcceptableCallable, $constraint, $name, $normalizedVersion)) { + foreach ($uniqKeys as $key) { + unset($version[$key.'s'][$index]); + } } - - $repo->createPackageIfAcceptable($packages, $isPackageAcceptableCallable, $unpackedVersion, $constraint); + } + if (count($version['version_normalizeds'])) { + $versionsToLoad[] = $version; } } else { - $repo->createPackageIfAcceptable($packages, $isPackageAcceptableCallable, $version, $constraint); + if (!isset($version['version_normalized'])) { + $version['version_normalized'] = $repo->versionParser->normalize($version['version']); + } + + if ($repo->isVersionAcceptable($isPackageAcceptableCallable, $constraint, $name, $version['version_normalized'])) { + $versionsToLoad[] = $version; + } + } + } + + $loadedPackages = $this->createPackages($versionsToLoad, 'Composer\Package\CompletePackage'); + foreach ($loadedPackages as $package) { + $package->setRepository($this); + + $packages[spl_object_hash($package)] = $package; + if ($package instanceof AliasPackage && !isset($packages[spl_object_hash($package->getAliasOf())])) { + $packages[spl_object_hash($package->getAliasOf())] = $package->getAliasOf(); } } }, function ($e) { @@ -592,27 +616,17 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito * * @private */ - public function createPackageIfAcceptable(&$packages, $isPackageAcceptableCallable, $version, $constraint) + public function isVersionAcceptable($isPackageAcceptableCallable, $constraint, $name, $versionNormalized) { - if (!call_user_func($isPackageAcceptableCallable, strtolower($version['name']), VersionParser::parseStability($version['version']))) { - return; + if (!call_user_func($isPackageAcceptableCallable, strtolower($name), VersionParser::parseStability($versionNormalized))) { + return false; } - if (isset($version['version_normalized']) && $constraint && !$constraint->matches(new Constraint('==', $version['version_normalized']))) { - return; + if ($constraint && !$constraint->matches(new Constraint('==', $versionNormalized))) { + return false; } - // load acceptable packages in the providers - $package = $this->createPackage($version, 'Composer\Package\CompletePackage'); - $package->setRepository($this); - - // if there was no version_normalized, then we need to check now for the constraint - if (!$constraint || isset($version['version_normalized']) || $constraint->matches(new Constraint('==', $package->getVersion()))) { - $packages[spl_object_hash($package)] = $package; - if ($package instanceof AliasPackage && !isset($packages[spl_object_hash($package->getAliasOf())])) { - $packages[spl_object_hash($package->getAliasOf())] = $package->getAliasOf(); - } - } + return true; } protected function loadRootServerFile() @@ -775,23 +789,37 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $packages; } - protected function createPackage(array $data, $class = 'Composer\Package\CompletePackage') + /** + * TODO v3 should make this private once we can drop PHP 5.3 support + * + * @private + */ + public function createPackages(array $packages, $class = 'Composer\Package\CompletePackage') { + if (!$packages) { + return; + } + try { - if (!isset($data['notification-url'])) { - $data['notification-url'] = $this->notifyUrl; + foreach ($packages as &$data) { + if (!isset($data['notification-url'])) { + $data['notification-url'] = $this->notifyUrl; + } } - $package = $this->loader->load($data, $class); - if (isset($this->sourceMirrors[$package->getSourceType()])) { - $package->setSourceMirrors($this->sourceMirrors[$package->getSourceType()]); - } - $package->setDistMirrors($this->distMirrors); - $this->configurePackageTransportOptions($package); + $packages = $this->loader->loadPackages($packages, $class); - return $package; + foreach ($packages as $package) { + if (isset($this->sourceMirrors[$package->getSourceType()])) { + $package->setSourceMirrors($this->sourceMirrors[$package->getSourceType()]); + } + $package->setDistMirrors($this->distMirrors); + $this->configurePackageTransportOptions($package); + } + + return $packages; } catch (\Exception $e) { - throw new \RuntimeException('Could not load package '.(isset($data['name']) ? $data['name'] : json_encode($data)).' in '.$this->url.': ['.get_class($e).'] '.$e->getMessage(), 0, $e); + throw new \RuntimeException('Could not load packages '.(isset($packages[0]['name']) ? $packages[0]['name'] : json_encode($packages)).' in '.$this->url.': ['.get_class($e).'] '.$e->getMessage(), 0, $e); } } diff --git a/tests/Composer/Test/Repository/ComposerRepositoryTest.php b/tests/Composer/Test/Repository/ComposerRepositoryTest.php index 0ffd70751..3594de101 100644 --- a/tests/Composer/Test/Repository/ComposerRepositoryTest.php +++ b/tests/Composer/Test/Repository/ComposerRepositoryTest.php @@ -32,7 +32,7 @@ class ComposerRepositoryTest extends TestCase ); $repository = $this->getMockBuilder('Composer\Repository\ComposerRepository') - ->setMethods(array('loadRootServerFile', 'createPackage')) + ->setMethods(array('loadRootServerFile', 'createPackages')) ->setConstructorArgs(array( $repoConfig, new NullIO, @@ -47,16 +47,17 @@ class ComposerRepositoryTest extends TestCase ->method('loadRootServerFile') ->will($this->returnValue($repoPackages)); + $stubs = array(); foreach ($expected as $at => $arg) { - $stubPackage = $this->getPackage('stub/stub', '1.0.0'); - - $repository - ->expects($this->at($at + 2)) - ->method('createPackage') - ->with($this->identicalTo($arg), $this->equalTo('Composer\Package\CompletePackage')) - ->will($this->returnValue($stubPackage)); + $stubs[] = $this->getPackage('stub/stub', '1.0.0'); } + $repository + ->expects($this->at(2)) + ->method('createPackages') + ->with($this->identicalTo($expected), $this->equalTo('Composer\Package\CompletePackage')) + ->will($this->returnValue($stubs)); + // Triggers initialization $packages = $repository->getPackages(); From fc03ab9bbae520a38bdc71f2d5620edd90967893 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Tue, 4 Dec 2018 11:36:15 +0100 Subject: [PATCH 19/34] Add COMPOSER_DISABLE_NETWORK env var for debugging --- doc/03-cli.md | 5 +++++ src/Composer/Repository/ComposerRepository.php | 7 ++++++- src/Composer/Util/HttpDownloader.php | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/doc/03-cli.md b/doc/03-cli.md index 74374ec6b..f0b3dca35 100644 --- a/doc/03-cli.md +++ b/doc/03-cli.md @@ -928,4 +928,9 @@ repository options. Defaults to `1`. If set to `0`, Composer will not create `.htaccess` files in the composer home, cache, and data directories. +### COMPOSER_DISABLE_NETWORK + +If set to `1`, disables network access (best effort). This can be used for debugging or +to run Composer on a plane or a starship with poor connectivity. + ← [Libraries](02-libraries.md) | [Schema](04-schema.md) → diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 06c984bd5..624621d79 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -1044,7 +1044,12 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return false; } - if (--$retries) { + // special error code returned when network is being artificially disabled + if ($e instanceof TransportException && $e->getStatusCode() === 499) { + $retries = 0; + } + + if (--$retries > 0) { usleep(100000); return $httpDownloader->add($filename, $options)->then($accept, $reject); diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php index 0e882c4a3..f2308d75a 100644 --- a/src/Composer/Util/HttpDownloader.php +++ b/src/Composer/Util/HttpDownloader.php @@ -16,6 +16,7 @@ use Composer\Config; use Composer\IO\IOInterface; use Composer\Downloader\TransportException; use Composer\CaBundle\CaBundle; +use Composer\Util\Http\Response; use Psr\Log\LoggerInterface; use React\Promise\Promise; @@ -40,6 +41,7 @@ class HttpDownloader private $curl; private $rfs; private $idGen = 0; + private $disabled; /** * @param IOInterface $io The IO instance @@ -51,6 +53,8 @@ class HttpDownloader { $this->io = $io; + $this->disabled = (bool) getenv('COMPOSER_DISABLE_NETWORK'); + // Setup TLS options // The cafile option can be set via config.json if ($disableTls === false) { @@ -216,6 +220,17 @@ class HttpDownloader $options = $job['request']['options']; $origin = $job['origin']; + if ($this->disabled) { + if (isset($job['request']['options']['http']['header']) && false !== stripos(implode('', $job['request']['options']['http']['header']), 'if-modified-since')) { + $resolve(new Response(array('url' => $url), 304, array(), '')); + } else { + $e = new TransportException('Network disabled', 499); + $e->setStatusCode(499); + $reject($e); + } + return; + } + if ($job['request']['copyTo']) { $this->curl->download($resolve, $reject, $origin, $url, $options, $job['request']['copyTo']); } else { From 00de0f58541b09d863e025d0b881256ef1d63ae8 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Tue, 4 Dec 2018 11:38:26 +0100 Subject: [PATCH 20/34] Fix 5.3 support --- src/Composer/Repository/ComposerRepository.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 624621d79..89b7c78f6 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -588,9 +588,9 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } } - $loadedPackages = $this->createPackages($versionsToLoad, 'Composer\Package\CompletePackage'); + $loadedPackages = $repo->createPackages($versionsToLoad, 'Composer\Package\CompletePackage'); foreach ($loadedPackages as $package) { - $package->setRepository($this); + $package->setRepository($repo); $packages[spl_object_hash($package)] = $package; if ($package instanceof AliasPackage && !isset($packages[spl_object_hash($package->getAliasOf())])) { From 14d6bcedda172844cda2dcc2ccb15c86a653d61b Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Tue, 4 Dec 2018 15:17:52 +0100 Subject: [PATCH 21/34] Fix redirect handling and some output tweaks --- src/Composer/Util/Http/CurlDownloader.php | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php index 4015b0bac..2528b9e3f 100644 --- a/src/Composer/Util/Http/CurlDownloader.php +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -108,7 +108,7 @@ class CurlDownloader { $attributes = array_merge(array( 'retryAuthFailure' => true, - 'redirects' => 1, + 'redirects' => 0, 'storeAuth' => false, ), $attributes); @@ -195,7 +195,9 @@ class CurlDownloader $usingProxy = !empty($options['http']['proxy']) ? ' using proxy ' . $options['http']['proxy'] : ''; $ifModified = false !== strpos(strtolower(implode(',', $options['http']['header'])), 'if-modified-since:') ? ' if modified' : ''; - $this->io->writeError('Downloading ' . $url . $usingProxy . $ifModified, true, IOInterface::DEBUG); + if ($attributes['redirects'] === 0) { + $this->io->writeError('Downloading ' . $url . $usingProxy . $ifModified, true, IOInterface::DEBUG); + } $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $curlHandle)); // TODO progress @@ -249,6 +251,7 @@ class CurlDownloader if ($job['filename']) { fclose($job['bodyHandle']); $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $job['filename'].'~'); + $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG); } else { rewind($job['bodyHandle']); $contents = stream_get_contents($job['bodyHandle']); @@ -268,7 +271,7 @@ class CurlDownloader } // handle 3xx redirects, 304 Not Modified is excluded - if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['redirects'] < $this->maxRedirects) { + if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['attributes']['redirects'] < $this->maxRedirects) { $location = $this->handleRedirect($job, $response); if ($location) { $this->restartJob($job, $location, array('redirects' => $job['attributes']['redirects'] + 1)); @@ -360,8 +363,7 @@ class CurlDownloader } if (!empty($targetUrl)) { - $this->io->writeError('', true, IOInterface::DEBUG); - $this->io->writeError(sprintf('Following redirect (%u) %s', $job['redirects'] + 1, $targetUrl), true, IOInterface::DEBUG); + $this->io->writeError(sprintf('Following redirect (%u) %s', $job['attributes']['redirects'] + 1, $targetUrl), true, IOInterface::DEBUG); return $targetUrl; } @@ -429,7 +431,7 @@ class CurlDownloader $attributes = array_merge($job['attributes'], $attributes); $origin = Url::getOrigin($this->config, $url); - $this->initDownload($job['resolve'], $job['reject'], $origin, $url, $job['originalOptions'], $job['filename'], $attributes); + $this->initDownload($job['resolve'], $job['reject'], $origin, $url, $job['options'], $job['filename'], $attributes); } private function failResponse(array $job, Response $response, $errorMessage) From b47330adf10738e2533b774e39f75fcec40365c2 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Tue, 4 Dec 2018 17:03:56 +0100 Subject: [PATCH 22/34] Refactor ComposerRepository to work with combined repos having lazy providers and partial packages --- src/Composer/Command/ShowCommand.php | 4 +- .../Repository/ComposerRepository.php | 338 ++++++++++-------- src/Composer/Util/Http/CurlDownloader.php | 217 ++++++----- .../Repository/ComposerRepositoryTest.php | 4 +- 4 files changed, 296 insertions(+), 267 deletions(-) diff --git a/src/Composer/Command/ShowCommand.php b/src/Composer/Command/ShowCommand.php index 9604bdc71..5cb3fa860 100644 --- a/src/Composer/Command/ShowCommand.php +++ b/src/Composer/Command/ShowCommand.php @@ -317,8 +317,8 @@ EOT } else { $type = 'available'; } - if ($repo instanceof ComposerRepository && $repo->hasProviders()) { - foreach ($repo->getProviderNames() as $name) { + if ($repo instanceof ComposerRepository) { + foreach ($repo->getPackageNames() as $name) { if (!$packageFilter || preg_match($packageFilter, $name)) { $packages[$type][$name] = $name; } diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 89b7c78f6..ff2895564 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -35,13 +35,13 @@ use Composer\Semver\Constraint\EmptyConstraint; */ class ComposerRepository extends ArrayRepository implements ConfigurableRepositoryInterface { - protected $config; - protected $repoConfig; - protected $options; - protected $url; - protected $baseUrl; - protected $io; - protected $httpDownloader; + private $config; + private $repoConfig; + private $options; + private $url; + private $baseUrl; + private $io; + private $httpDownloader; protected $cache; protected $notifyUrl; protected $searchUrl; @@ -49,14 +49,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito protected $providersUrl; protected $lazyProvidersUrl; protected $providerListing; - protected $providers = array(); - protected $providersByUid = array(); + private $providers = array(); + private $providersByUid = array(); protected $loader; - protected $rootAliases; - protected $allowSslDowngrade = false; - protected $eventDispatcher; - protected $sourceMirrors; - protected $distMirrors; + private $rootAliases; + private $allowSslDowngrade = false; + private $eventDispatcher; + private $sourceMirrors; + private $distMirrors; private $degradedMode = false; private $rootData; private $hasPartialPackages; @@ -134,28 +134,23 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $constraint = $this->versionParser->parseConstraints($constraint); } - // TODO we need a new way for the repo to report this v2 protocol somehow if ($this->lazyProvidersUrl) { + if ($this->hasPartialPackages() && isset($this->partialPackagesByName[$name])) { + return $this->filterPackages($this->whatProvides($name), $name, $constraint, true); + } + return $this->loadAsyncPackages(array($name => $constraint), function ($name, $stability) { return true; }); } + if (!$hasProviders) { return parent::findPackage($name, $constraint); } foreach ($this->getProviderNames() as $providerName) { if ($name === $providerName) { - $packages = $this->whatProvides($providerName); - foreach ($packages as $package) { - if ($name === $package->getName()) { - $pkgConstraint = new Constraint('==', $package->getVersion()); - if ($constraint->matches($pkgConstraint)) { - return $package; - } - } - } - break; + return $this->filterPackages($this->whatProvides($providerName), $name, $constraint, true); } } } @@ -168,40 +163,58 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito // this call initializes loadRootServerFile which is needed for the rest below to work $hasProviders = $this->hasProviders(); - // TODO we need a new way for the repo to report this v2 protocol somehow - if ($this->lazyProvidersUrl) { - return $this->loadAsyncPackages(array($name => $constraint ?: new EmptyConstraint()), function ($name, $stability) { - return true; - }); - } - if (!$hasProviders) { - return parent::findPackages($name, $constraint); - } - - // normalize name $name = strtolower($name); - if (null !== $constraint && !$constraint instanceof ConstraintInterface) { $constraint = $this->versionParser->parseConstraints($constraint); } - $packages = array(); + if ($this->lazyProvidersUrl) { + if ($this->hasPartialPackages() && isset($this->partialPackagesByName[$name])) { + return $this->filterPackages($this->whatProvides($name), $name, $constraint); + } + + return $this->loadAsyncPackages(array($name => $constraint ?: new EmptyConstraint()), function ($name, $stability) { + return true; + }); + } + + if (!$hasProviders) { + return parent::findPackages($name, $constraint); + } foreach ($this->getProviderNames() as $providerName) { if ($name === $providerName) { - $candidates = $this->whatProvides($providerName); - foreach ($candidates as $package) { - if ($name === $package->getName()) { - $pkgConstraint = new Constraint('==', $package->getVersion()); - if (null === $constraint || $constraint->matches($pkgConstraint)) { - $packages[] = $package; - } - } - } - break; + return $this->filterPackages($this->whatProvides($providerName), $name, $constraint); } } + return array(); + } + + private function filterPackages(array $packages, $name, $constraint = null, $returnFirstMatch = false) + { + $packages = array(); + + foreach ($packages as $package) { + // TODO this check can be removed once providers are only what really has that name anyway + if ($name !== $package->getName()) { + continue; + } + + $pkgConstraint = new Constraint('==', $package->getVersion()); + if (null === $constraint || $constraint->matches($pkgConstraint)) { + if ($returnFirstMatch) { + return $package; + } + + $packages[] = $package; + } + } + + if ($returnFirstMatch) { + return null; + } + return $packages; } @@ -214,39 +227,63 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return parent::getPackages(); } + public function getPackageNames() + { + if ($this->hasProviders()) { + return $this->getProviderNames(); + } + + // TODO implement new list API endpoint for repos somehow? + // TODO add getPackageNames to the RepositoryInterface perhaps? With filtering capability embedded? + + return $this->getPackages(); + } + public function loadPackages(array $packageNameMap, $isPackageAcceptableCallable) { // this call initializes loadRootServerFile which is needed for the rest below to work $hasProviders = $this->hasProviders(); - // TODO we need a new way for the repo to report this v2 protocol somehow - if ($this->lazyProvidersUrl) { - return $this->loadAsyncPackages($packageNameMap, $isPackageAcceptableCallable); - } - if (!$hasProviders) { + if (!$hasProviders && !$this->hasPartialPackages() && !$this->lazyProvidersUrl) { return parent::loadPackages($packageNameMap, $isPackageAcceptableCallable); } $packages = array(); - foreach ($packageNameMap as $name => $constraint) { - $matches = array(); - $candidates = $this->whatProvides($name, false, $isPackageAcceptableCallable); - foreach ($candidates as $candidate) { - if ($candidate->getName() === $name && (!$constraint || $constraint->matches(new Constraint('==', $candidate->getVersion())))) { - $matches[spl_object_hash($candidate)] = $candidate; - if ($candidate instanceof AliasPackage && !isset($matches[spl_object_hash($candidate->getAliasOf())])) { - $matches[spl_object_hash($candidate->getAliasOf())] = $candidate->getAliasOf(); - } + + if ($hasProviders || $this->hasPartialPackages()) { + foreach ($packageNameMap as $name => $constraint) { + $matches = array(); + + if (!$hasProviders && !isset($this->partialPackagesByName[$name])) { + continue; } - } - foreach ($candidates as $candidate) { - if ($candidate instanceof AliasPackage) { - if (isset($result[spl_object_hash($candidate->getAliasOf())])) { + + $candidates = $this->whatProvides($name, $isPackageAcceptableCallable); + foreach ($candidates as $candidate) { + if ($candidate->getName() === $name && (!$constraint || $constraint->matches(new Constraint('==', $candidate->getVersion())))) { $matches[spl_object_hash($candidate)] = $candidate; + if ($candidate instanceof AliasPackage && !isset($matches[spl_object_hash($candidate->getAliasOf())])) { + $matches[spl_object_hash($candidate->getAliasOf())] = $candidate->getAliasOf(); + } } } + foreach ($candidates as $candidate) { + if ($candidate instanceof AliasPackage) { + if (isset($result[spl_object_hash($candidate->getAliasOf())])) { + $matches[spl_object_hash($candidate)] = $candidate; + } + } + } + $packages = array_merge($packages, $matches); + + unset($packageNameMap[$name]); } - $packages = array_merge($packages, $matches); + + return $packages; + } + + if ($this->lazyProvidersUrl) { + $packages = array_merge($packages, $this->loadAsyncPackages($packageNameMap, $isPackageAcceptableCallable)); } return $packages; @@ -295,7 +332,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return parent::search($query, $mode); } - public function getProviderNames() + private function getProviderNames() { $this->loadRootServerFile(); @@ -315,7 +352,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return array(); } - protected function configurePackageTransportOptions(PackageInterface $package) + private function configurePackageTransportOptions(PackageInterface $package) { foreach ($package->getDistUrls() as $url) { if (strpos($url, $this->baseUrl) === 0) { @@ -326,7 +363,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } } - public function hasProviders() + private function hasProviders() { $this->loadRootServerFile(); @@ -335,21 +372,16 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito /** * @param string $name package name - * @param bool $bypassFilters If set to true, this bypasses the stability filtering, and forces a recompute without cache * @param callable $isPackageAcceptableCallable * @return array|mixed */ - public function whatProvides($name, $bypassFilters = false, $isPackageAcceptableCallable = null) + private function whatProvides($name, $isPackageAcceptableCallable = null) { - if (isset($this->providers[$name]) && !$bypassFilters) { + if (isset($this->providers[$name])) { return $this->providers[$name]; } - if ($this->hasPartialPackages && null === $this->partialPackagesByName) { - $this->initializePartialPackages(); - } - - if (!$this->hasPartialPackages || !isset($this->partialPackagesByName[$name])) { + if (!$this->hasPartialPackages() || !isset($this->partialPackagesByName[$name])) { // skip platform packages, root package and composer-plugin-api if (preg_match(PlatformRepository::PLATFORM_PACKAGE_REGEX, $name) || '__root__' === $name || 'composer-plugin-api' === $name) { return array(); @@ -420,7 +452,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito foreach ($packages['packages'] as $versions) { $versionsToLoad = array(); foreach ($versions as $version) { - if (!$loadingPartialPackage && $this->hasPartialPackages && isset($this->partialPackagesByName[$version['name']])) { + if (!$loadingPartialPackage && $this->hasPartialPackages() && isset($this->partialPackagesByName[$version['name']])) { continue; } @@ -441,7 +473,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } } } else { - if (!$bypassFilters && $isPackageAcceptableCallable && !call_user_func($isPackageAcceptableCallable, strtolower($version['name']), VersionParser::parseStability($version['version']))) { + if ($isPackageAcceptableCallable && !call_user_func($isPackageAcceptableCallable, strtolower($version['name']), VersionParser::parseStability($version['version']))) { continue; } @@ -489,15 +521,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $result = $this->providers[$name]; - // clean up the cache because otherwise using this puts the repo in an inconsistent state with a polluted unfiltered cache - // which is likely not an issue but might cause hard to track behaviors depending on how the repo is used - if ($bypassFilters) { - foreach ($this->providers[$name] as $uid => $provider) { - unset($this->providersByUid[$uid]); - } - unset($this->providers[$name]); - } - return $result; } @@ -533,76 +556,76 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $packages = array(); $repo = $this; - // TODO what if not, then throw? - if ($this->lazyProvidersUrl) { - foreach ($packageNames as $name => $constraint) { - $name = strtolower($name); + if (!$this->lazyProvidersUrl) { + throw new \LogicException('loadAsyncPackages only supports v2 protocol composer repos with a metadata-url'); + } - // skip platform packages, root package and composer-plugin-api - if (preg_match(PlatformRepository::PLATFORM_PACKAGE_REGEX, $name) || '__root__' === $name || 'composer-plugin-api' === $name) { - continue; - } + foreach ($packageNames as $name => $constraint) { + $name = strtolower($name); - $url = str_replace('%package%', $name, $this->lazyProvidersUrl); - $cacheKey = 'provider-'.strtr($name, '/', '$').'.json'; + // skip platform packages, root package and composer-plugin-api + if (preg_match(PlatformRepository::PLATFORM_PACKAGE_REGEX, $name) || '__root__' === $name || 'composer-plugin-api' === $name) { + continue; + } - $lastModified = null; - if ($contents = $this->cache->read($cacheKey)) { - $contents = json_decode($contents, true); - $lastModified = isset($contents['last-modified']) ? $contents['last-modified'] : null; - } + $url = str_replace('%package%', $name, $this->lazyProvidersUrl); + $cacheKey = 'provider-'.strtr($name, '/', '$').'.json'; - $this->asyncFetchFile($url, $cacheKey, $lastModified) - ->then(function ($response) use (&$packages, $contents, $name, $constraint, $repo, $isPackageAcceptableCallable) { - static $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); + $lastModified = null; + if ($contents = $this->cache->read($cacheKey)) { + $contents = json_decode($contents, true); + $lastModified = isset($contents['last-modified']) ? $contents['last-modified'] : null; + } - if (true === $response) { - $response = $contents; - } + $this->asyncFetchFile($url, $cacheKey, $lastModified) + ->then(function ($response) use (&$packages, $contents, $name, $constraint, $repo, $isPackageAcceptableCallable) { + static $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); - if (!isset($response['packages'][$name])) { - return; - } + if (true === $response) { + $response = $contents; + } - $versionsToLoad = array(); - foreach ($response['packages'][$name] as $version) { - if (isset($version['version_normalizeds'])) { - foreach ($version['version_normalizeds'] as $index => $normalizedVersion) { - if (!$repo->isVersionAcceptable($isPackageAcceptableCallable, $constraint, $name, $normalizedVersion)) { - foreach ($uniqKeys as $key) { - unset($version[$key.'s'][$index]); - } + if (!isset($response['packages'][$name])) { + return; + } + + $versionsToLoad = array(); + foreach ($response['packages'][$name] as $version) { + if (isset($version['version_normalizeds'])) { + foreach ($version['version_normalizeds'] as $index => $normalizedVersion) { + if (!$repo->isVersionAcceptable($isPackageAcceptableCallable, $constraint, $name, $normalizedVersion)) { + foreach ($uniqKeys as $key) { + unset($version[$key.'s'][$index]); } } - if (count($version['version_normalizeds'])) { - $versionsToLoad[] = $version; - } - } else { - if (!isset($version['version_normalized'])) { - $version['version_normalized'] = $repo->versionParser->normalize($version['version']); - } + } + if (count($version['version_normalizeds'])) { + $versionsToLoad[] = $version; + } + } else { + if (!isset($version['version_normalized'])) { + $version['version_normalized'] = $repo->versionParser->normalize($version['version']); + } - if ($repo->isVersionAcceptable($isPackageAcceptableCallable, $constraint, $name, $version['version_normalized'])) { - $versionsToLoad[] = $version; - } + if ($repo->isVersionAcceptable($isPackageAcceptableCallable, $constraint, $name, $version['version_normalized'])) { + $versionsToLoad[] = $version; } } + } - $loadedPackages = $repo->createPackages($versionsToLoad, 'Composer\Package\CompletePackage'); - foreach ($loadedPackages as $package) { - $package->setRepository($repo); + $loadedPackages = $repo->createPackages($versionsToLoad, 'Composer\Package\CompletePackage'); + foreach ($loadedPackages as $package) { + $package->setRepository($repo); - $packages[spl_object_hash($package)] = $package; - if ($package instanceof AliasPackage && !isset($packages[spl_object_hash($package->getAliasOf())])) { - $packages[spl_object_hash($package->getAliasOf())] = $package->getAliasOf(); - } + $packages[spl_object_hash($package)] = $package; + if ($package instanceof AliasPackage && !isset($packages[spl_object_hash($package->getAliasOf())])) { + $packages[spl_object_hash($package->getAliasOf())] = $package->getAliasOf(); } - }, function ($e) { - // TODO use ->done() above instead with react/promise 2.0 - var_dump('Uncaught Ex', $e->getMessage()); - throw $e; - }); - } + } + }, function ($e) { + // TODO use ->done() above instead with react/promise 2.0 + throw $e; + }); } $this->httpDownloader->wait(); @@ -684,12 +707,13 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } // metadata-url indiates V2 repo protocol so it takes over from all the V1 types - // V2 only has lazyProviders and no ability to process anything else, plus support for async loading + // V2 only has lazyProviders and possibly partial packages, but no ability to process anything else, + // V2 also supports async loading if (!empty($data['metadata-url'])) { $this->lazyProvidersUrl = $this->canonicalizeUrl($data['metadata-url']); $this->providersUrl = null; $this->hasProviders = false; - $this->hasPartialPackages = false; + $this->hasPartialPackages = !empty($data['packages']) && is_array($data['packages']); $this->allowSslDowngrade = false; unset($data['providers-url'], $data['providers'], $data['providers-includes']); } @@ -711,7 +735,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $this->rootData = $data; } - protected function canonicalizeUrl($url) + private function canonicalizeUrl($url) { if ('/' === $url[0]) { return preg_replace('{(https?://[^/]+).*}i', '$1' . $url, $this->url); @@ -720,14 +744,23 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $url; } - protected function loadDataFromServer() + private function loadDataFromServer() { $data = $this->loadRootServerFile(); return $this->loadIncludes($data); } - protected function loadProviderListings($data) + private function hasPartialPackages() + { + if ($this->hasPartialPackages && null === $this->partialPackagesByName) { + $this->initializePartialPackages(); + } + + return $this->hasPartialPackages; + } + + private function loadProviderListings($data) { if (isset($data['providers'])) { if (!is_array($this->providerListing)) { @@ -752,7 +785,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } } - protected function loadIncludes($data) + private function loadIncludes($data) { $packages = array(); @@ -924,7 +957,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $data; } - protected function fetchFileIfLastModified($filename, $cacheKey, $lastModifiedTime) + private function fetchFileIfLastModified($filename, $cacheKey, $lastModifiedTime) { $retries = 3; while ($retries--) { @@ -990,7 +1023,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } } - protected function asyncFetchFile($filename, $cacheKey, $lastModifiedTime = null) + private function asyncFetchFile($filename, $cacheKey, $lastModifiedTime = null) { $retries = 3; $httpDownloader = $this->httpDownloader; @@ -1039,7 +1072,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito }; $reject = function ($e) use (&$retries, $httpDownloader, $filename, $options, &$reject, $accept, $io, $url, $cache, &$degradedMode) { - var_dump('Caught8', $e->getMessage()); if ($e instanceof TransportException && $e->getStatusCode() === 404) { return false; } diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php index 2528b9e3f..1ff8393c9 100644 --- a/src/Composer/Util/Http/CurlDownloader.php +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -211,132 +211,127 @@ class CurlDownloader } $active = true; - try { - $this->checkCurlResult(curl_multi_exec($this->multiHandle, $active)); - if (-1 === curl_multi_select($this->multiHandle, $this->selectTimeout)) { - // sleep in case select returns -1 as it can happen on old php versions or some platforms where curl does not manage to do the select - usleep(150); + $this->checkCurlResult(curl_multi_exec($this->multiHandle, $active)); + if (-1 === curl_multi_select($this->multiHandle, $this->selectTimeout)) { + // sleep in case select returns -1 as it can happen on old php versions or some platforms where curl does not manage to do the select + usleep(150); + } + + while ($progress = curl_multi_info_read($this->multiHandle)) { + $curlHandle = $progress['handle']; + $i = (int) $curlHandle; + if (!isset($this->jobs[$i])) { + continue; } + $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); + $job = $this->jobs[$i]; + unset($this->jobs[$i]); + curl_multi_remove_handle($this->multiHandle, $curlHandle); + $error = curl_error($curlHandle); + $errno = curl_errno($curlHandle); + curl_close($curlHandle); - while ($progress = curl_multi_info_read($this->multiHandle)) { - $curlHandle = $progress['handle']; - $i = (int) $curlHandle; - if (!isset($this->jobs[$i])) { - continue; + $headers = null; + $statusCode = null; + $response = null; + try { +// TODO progress + //$this->onProgress($curlHandle, $job['callback'], $progress, $job['progress']); + if (CURLE_OK !== $errno) { + throw new TransportException($error); } - $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); - $job = $this->jobs[$i]; - unset($this->jobs[$i]); - curl_multi_remove_handle($this->multiHandle, $curlHandle); - $error = curl_error($curlHandle); - $errno = curl_errno($curlHandle); - curl_close($curlHandle); - $headers = null; - $statusCode = null; - $response = null; - try { -// TODO progress - //$this->onProgress($curlHandle, $job['callback'], $progress, $job['progress']); - if (CURLE_OK !== $errno) { - throw new TransportException($error); - } + $statusCode = $progress['http_code']; + rewind($job['headerHandle']); + $headers = explode("\r\n", rtrim(stream_get_contents($job['headerHandle']))); + fclose($job['headerHandle']); - $statusCode = $progress['http_code']; - rewind($job['headerHandle']); - $headers = explode("\r\n", rtrim(stream_get_contents($job['headerHandle']))); - fclose($job['headerHandle']); + // prepare response object + if ($job['filename']) { + fclose($job['bodyHandle']); + $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $job['filename'].'~'); + $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG); + } else { + rewind($job['bodyHandle']); + $contents = stream_get_contents($job['bodyHandle']); + fclose($job['bodyHandle']); + $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $contents); + $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG); + } - // prepare response object - if ($job['filename']) { - fclose($job['bodyHandle']); - $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $job['filename'].'~'); - $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG); - } else { - rewind($job['bodyHandle']); - $contents = stream_get_contents($job['bodyHandle']); - fclose($job['bodyHandle']); - $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $contents); - $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG); - } - - $result = $this->isAuthenticatedRetryNeeded($job, $response); - if ($result['retry']) { - if ($job['filename']) { - @unlink($job['filename'].'~'); - } - - $this->restartJob($job, $job['url'], array('storeAuth' => $result['storeAuth'])); - continue; - } - - // handle 3xx redirects, 304 Not Modified is excluded - if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['attributes']['redirects'] < $this->maxRedirects) { - $location = $this->handleRedirect($job, $response); - if ($location) { - $this->restartJob($job, $location, array('redirects' => $job['attributes']['redirects'] + 1)); - continue; - } - } - - // fail 4xx and 5xx responses and capture the response - if ($statusCode >= 400 && $statusCode <= 599) { - throw $this->failResponse($job, $response, $response->getStatusMessage()); -// TODO progress -// $this->io->overwriteError("Downloading (failed)", false); - } - - if ($job['attributes']['storeAuth']) { - $this->authHelper->storeAuth($job['origin'], $job['attributes']['storeAuth']); - } - - // resolve promise - if ($job['filename']) { - rename($job['filename'].'~', $job['filename']); - call_user_func($job['resolve'], true); - } else { - call_user_func($job['resolve'], $response); - } - } catch (\Exception $e) { - if ($e instanceof TransportException && $headers) { - $e->setHeaders($headers); - $e->setStatusCode($statusCode); - } - if ($e instanceof TransportException && $response) { - $e->setResponse($response->getBody()); - } - - if (is_resource($job['headerHandle'])) { - fclose($job['headerHandle']); - } - if (is_resource($job['bodyHandle'])) { - fclose($job['bodyHandle']); - } + $result = $this->isAuthenticatedRetryNeeded($job, $response); + if ($result['retry']) { if ($job['filename']) { @unlink($job['filename'].'~'); } - call_user_func($job['reject'], $e); - } - } - foreach ($this->jobs as $i => $curlHandle) { - if (!isset($this->jobs[$i])) { + $this->restartJob($job, $job['url'], array('storeAuth' => $result['storeAuth'])); continue; } - $curlHandle = $this->jobs[$i]['curlHandle']; - $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); - if ($this->jobs[$i]['progress'] !== $progress) { - $previousProgress = $this->jobs[$i]['progress']; - $this->jobs[$i]['progress'] = $progress; - - // TODO - //$this->onProgress($curlHandle, $this->jobs[$i]['callback'], $progress, $previousProgress); + // handle 3xx redirects, 304 Not Modified is excluded + if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['attributes']['redirects'] < $this->maxRedirects) { + $location = $this->handleRedirect($job, $response); + if ($location) { + $this->restartJob($job, $location, array('redirects' => $job['attributes']['redirects'] + 1)); + continue; + } } + + // fail 4xx and 5xx responses and capture the response + if ($statusCode >= 400 && $statusCode <= 599) { + throw $this->failResponse($job, $response, $response->getStatusMessage()); +// TODO progress +// $this->io->overwriteError("Downloading (failed)", false); + } + + if ($job['attributes']['storeAuth']) { + $this->authHelper->storeAuth($job['origin'], $job['attributes']['storeAuth']); + } + + // resolve promise + if ($job['filename']) { + rename($job['filename'].'~', $job['filename']); + call_user_func($job['resolve'], true); + } else { + call_user_func($job['resolve'], $response); + } + } catch (\Exception $e) { + if ($e instanceof TransportException && $headers) { + $e->setHeaders($headers); + $e->setStatusCode($statusCode); + } + if ($e instanceof TransportException && $response) { + $e->setResponse($response->getBody()); + } + + if (is_resource($job['headerHandle'])) { + fclose($job['headerHandle']); + } + if (is_resource($job['bodyHandle'])) { + fclose($job['bodyHandle']); + } + if ($job['filename']) { + @unlink($job['filename'].'~'); + } + call_user_func($job['reject'], $e); + } + } + + foreach ($this->jobs as $i => $curlHandle) { + if (!isset($this->jobs[$i])) { + continue; + } + $curlHandle = $this->jobs[$i]['curlHandle']; + $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); + + if ($this->jobs[$i]['progress'] !== $progress) { + $previousProgress = $this->jobs[$i]['progress']; + $this->jobs[$i]['progress'] = $progress; + + // TODO + //$this->onProgress($curlHandle, $this->jobs[$i]['callback'], $progress, $previousProgress); } - } catch (\Exception $e) { - // TODO - var_dump('Caught2', get_class($e), $e->getMessage(), $e);die; } } diff --git a/tests/Composer/Test/Repository/ComposerRepositoryTest.php b/tests/Composer/Test/Repository/ComposerRepositoryTest.php index 3594de101..552c767a5 100644 --- a/tests/Composer/Test/Repository/ComposerRepositoryTest.php +++ b/tests/Composer/Test/Repository/ComposerRepositoryTest.php @@ -153,7 +153,9 @@ class ComposerRepositoryTest extends TestCase ), )); - $packages = $repo->whatProvides('a', false, array($this, 'isPackageAcceptableReturnTrue')); + $reflMethod = new \ReflectionMethod($repo, 'whatProvides'); + $reflMethod->setAccessible(true); + $packages = $reflMethod->invoke($repo, 'a', array($this, 'isPackageAcceptableReturnTrue')); $this->assertCount(7, $packages); $this->assertEquals(array('1', '1-alias', '2', '2-alias', '2-root', '3', '3-root'), array_keys($packages)); From 137c32e72e45ec2c9cbdfb3dc7226897fe1ea92f Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Tue, 4 Dec 2018 17:27:23 +0100 Subject: [PATCH 23/34] Do not prohibit http for old provider URLs on .org in case they are used --- src/Composer/Util/Http/CurlDownloader.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php index 1ff8393c9..45b1d21de 100644 --- a/src/Composer/Util/Http/CurlDownloader.php +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -114,8 +114,10 @@ class CurlDownloader $originalOptions = $options; - // check URL can be accessed (i.e. is not insecure) - $this->config->prohibitUrlByConfig($url, $this->io); + // check URL can be accessed (i.e. is not insecure), but allow insecure Packagist calls to $hashed providers as file integrity is verified with sha256 + if (!preg_match('{^http://(repo\.)?packagist\.org/p/}', $url) || (false === strpos($url, '$') && false === strpos($url, '%24'))) { + $this->config->prohibitUrlByConfig($url, $this->io); + } $curlHandle = curl_init(); $headerHandle = fopen('php://temp/maxmemory:32768', 'w+b'); From a5d5270a7e8db39baf141bccdfc4c4875ed0f1db Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Tue, 4 Dec 2018 17:37:39 +0100 Subject: [PATCH 24/34] Make sure other providers of a name are never loaded --- .../Repository/ComposerRepository.php | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index ff2895564..420bde535 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -136,7 +136,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito if ($this->lazyProvidersUrl) { if ($this->hasPartialPackages() && isset($this->partialPackagesByName[$name])) { - return $this->filterPackages($this->whatProvides($name), $name, $constraint, true); + return $this->filterPackages($this->whatProvides($name), $constraint, true); } return $this->loadAsyncPackages(array($name => $constraint), function ($name, $stability) { @@ -150,7 +150,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito foreach ($this->getProviderNames() as $providerName) { if ($name === $providerName) { - return $this->filterPackages($this->whatProvides($providerName), $name, $constraint, true); + return $this->filterPackages($this->whatProvides($providerName), $constraint, true); } } } @@ -170,7 +170,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito if ($this->lazyProvidersUrl) { if ($this->hasPartialPackages() && isset($this->partialPackagesByName[$name])) { - return $this->filterPackages($this->whatProvides($name), $name, $constraint); + return $this->filterPackages($this->whatProvides($name), $constraint); } return $this->loadAsyncPackages(array($name => $constraint ?: new EmptyConstraint()), function ($name, $stability) { @@ -184,22 +184,18 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito foreach ($this->getProviderNames() as $providerName) { if ($name === $providerName) { - return $this->filterPackages($this->whatProvides($providerName), $name, $constraint); + return $this->filterPackages($this->whatProvides($providerName), $constraint); } } return array(); } - private function filterPackages(array $packages, $name, $constraint = null, $returnFirstMatch = false) + private function filterPackages(array $packages, $constraint = null, $returnFirstMatch = false) { $packages = array(); foreach ($packages as $package) { - // TODO this check can be removed once providers are only what really has that name anyway - if ($name !== $package->getName()) { - continue; - } $pkgConstraint = new Constraint('==', $package->getVersion()); if (null === $constraint || $constraint->matches($pkgConstraint)) { @@ -452,7 +448,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito foreach ($packages['packages'] as $versions) { $versionsToLoad = array(); foreach ($versions as $version) { - if (!$loadingPartialPackage && $this->hasPartialPackages() && isset($this->partialPackagesByName[$version['name']])) { + $normalizedName = strtolower($version['name']); + + // only load the actual named package, not other packages that might find themselves in the same file + if ($normalizedName !== $name) { + continue; + } + + if (!$loadingPartialPackage && $this->hasPartialPackages() && isset($this->partialPackagesByName[$normalizedName])) { continue; } @@ -473,7 +476,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } } } else { - if ($isPackageAcceptableCallable && !call_user_func($isPackageAcceptableCallable, strtolower($version['name']), VersionParser::parseStability($version['version']))) { + if ($isPackageAcceptableCallable && !call_user_func($isPackageAcceptableCallable, $normalizedName, VersionParser::parseStability($version['version']))) { continue; } @@ -830,7 +833,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito public function createPackages(array $packages, $class = 'Composer\Package\CompletePackage') { if (!$packages) { - return; + return array(); } try { @@ -1110,19 +1113,8 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->partialPackagesByName = array(); foreach ($rootData['packages'] as $package => $versions) { - $package = strtolower($package); foreach ($versions as $version) { - $this->partialPackagesByName[$package][] = $version; - if (!empty($version['provide']) && is_array($version['provide'])) { - foreach ($version['provide'] as $provided => $providedVersion) { - $this->partialPackagesByName[strtolower($provided)][] = $version; - } - } - if (!empty($version['replace']) && is_array($version['replace'])) { - foreach ($version['replace'] as $provided => $providedVersion) { - $this->partialPackagesByName[strtolower($provided)][] = $version; - } - } + $this->partialPackagesByName[strtolower($version['name'])][] = $version; } } From 4b7658a2a86ee9f834dd022de1e25487e4a140c4 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Wed, 5 Dec 2018 13:28:37 +0100 Subject: [PATCH 25/34] Small tweaks and make sure composer fails properly in a plane --- src/Composer/Repository/ComposerRepository.php | 9 +++++++-- src/Composer/Util/Http/CurlDownloader.php | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 420bde535..43087ee7f 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -250,13 +250,18 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito foreach ($packageNameMap as $name => $constraint) { $matches = array(); + // if a repo has no providers but only partial packages and the partial packages are missing + // then we don't want to call whatProvides as it would try to load from the providers and fail if (!$hasProviders && !isset($this->partialPackagesByName[$name])) { continue; } $candidates = $this->whatProvides($name, $isPackageAcceptableCallable); foreach ($candidates as $candidate) { - if ($candidate->getName() === $name && (!$constraint || $constraint->matches(new Constraint('==', $candidate->getVersion())))) { + if ($candidate->getName() !== $name) { + throw new \LogicException('whatProvides should never return a package with a different name than the requested one'); + } + if (!$constraint || $constraint->matches(new Constraint('==', $candidate->getVersion()))) { $matches[spl_object_hash($candidate)] = $candidate; if ($candidate instanceof AliasPackage && !isset($matches[spl_object_hash($candidate->getAliasOf())])) { $matches[spl_object_hash($candidate->getAliasOf())] = $candidate->getAliasOf(); @@ -278,7 +283,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $packages; } - if ($this->lazyProvidersUrl) { + if ($this->lazyProvidersUrl && count($packageNameMap)) { $packages = array_merge($packages, $this->loadAsyncPackages($packageNameMap, $isPackageAcceptableCallable)); } diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php index 45b1d21de..3dc6710b6 100644 --- a/src/Composer/Util/Http/CurlDownloader.php +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -225,6 +225,7 @@ class CurlDownloader if (!isset($this->jobs[$i])) { continue; } + $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); $job = $this->jobs[$i]; unset($this->jobs[$i]); @@ -239,7 +240,7 @@ class CurlDownloader try { // TODO progress //$this->onProgress($curlHandle, $job['callback'], $progress, $job['progress']); - if (CURLE_OK !== $errno) { + if (CURLE_OK !== $errno || $error) { throw new TransportException($error); } From fd5c5ff6bc4b7662d342efebace527295f5f4160 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Mon, 14 Jan 2019 15:30:55 +0100 Subject: [PATCH 26/34] Fix implementation of whatProvides for older provider-only repos --- src/Composer/Package/Loader/ArrayLoader.php | 1 - .../Repository/ComposerRepository.php | 92 +++++-------------- .../Repository/ComposerRepositoryTest.php | 13 +-- 3 files changed, 24 insertions(+), 82 deletions(-) diff --git a/src/Composer/Package/Loader/ArrayLoader.php b/src/Composer/Package/Loader/ArrayLoader.php index ac91b18fe..9fed111e0 100644 --- a/src/Composer/Package/Loader/ArrayLoader.php +++ b/src/Composer/Package/Loader/ArrayLoader.php @@ -95,7 +95,6 @@ class ArrayLoader implements LoaderInterface $packages[] = $package; } - } return $packages; diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 43087ee7f..829883b58 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -49,10 +49,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito protected $providersUrl; protected $lazyProvidersUrl; protected $providerListing; - private $providers = array(); - private $providersByUid = array(); protected $loader; - private $rootAliases; private $allowSslDowngrade = false; private $eventDispatcher; private $sourceMirrors; @@ -116,11 +113,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $this->repoConfig; } - public function setRootAliases(array $rootAliases) - { - $this->rootAliases = $rootAliases; - } - /** * {@inheritDoc} */ @@ -378,10 +370,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito */ private function whatProvides($name, $isPackageAcceptableCallable = null) { - if (isset($this->providers[$name])) { - return $this->providers[$name]; - } - if (!$this->hasPartialPackages() || !isset($this->partialPackagesByName[$name])) { // skip platform packages, root package and composer-plugin-api if (preg_match(PlatformRepository::PLATFORM_PACKAGE_REGEX, $name) || '__root__' === $name || 'composer-plugin-api' === $name) { @@ -449,9 +437,9 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $loadingPartialPackage = true; } - $this->providers[$name] = array(); + $result = array(); + $versionsToLoad = array(); foreach ($packages['packages'] as $versions) { - $versionsToLoad = array(); foreach ($versions as $version) { $normalizedName = strtolower($version['name']); @@ -464,70 +452,34 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito continue; } - // avoid loading the same objects twice - if (isset($this->providersByUid[$version['uid']])) { - // skip if already assigned - if (!isset($this->providers[$name][$version['uid']])) { - // expand alias in two packages - if ($this->providersByUid[$version['uid']] instanceof AliasPackage) { - $this->providers[$name][$version['uid']] = $this->providersByUid[$version['uid']]->getAliasOf(); - $this->providers[$name][$version['uid'].'-alias'] = $this->providersByUid[$version['uid']]; - } else { - $this->providers[$name][$version['uid']] = $this->providersByUid[$version['uid']]; - } - // check for root aliases - if (isset($this->providersByUid[$version['uid'].'-root'])) { - $this->providers[$name][$version['uid'].'-root'] = $this->providersByUid[$version['uid'].'-root']; - } - } - } else { + if (!isset($versionsToLoad[$version['uid']])) { if ($isPackageAcceptableCallable && !call_user_func($isPackageAcceptableCallable, $normalizedName, VersionParser::parseStability($version['version']))) { continue; } - $versionsToLoad[] = $version; - } - } - - // load acceptable packages in the providers - $loadedPackages = $this->createPackages($versionsToLoad, 'Composer\Package\CompletePackage'); - foreach ($loadedPackages as $package) { - $package->setRepository($this); - - if ($package instanceof AliasPackage) { - $aliased = $package->getAliasOf(); - $aliased->setRepository($this); - - $this->providers[$name][$version['uid']] = $aliased; - $this->providers[$name][$version['uid'].'-alias'] = $package; - - // override provider with its alias so it can be expanded in the if block above - $this->providersByUid[$version['uid']] = $package; - } else { - $this->providers[$name][$version['uid']] = $package; - $this->providersByUid[$version['uid']] = $package; - } - - // handle root package aliases - unset($rootAliasData); - - if (isset($this->rootAliases[$package->getName()][$package->getVersion()])) { - $rootAliasData = $this->rootAliases[$package->getName()][$package->getVersion()]; - } elseif ($package instanceof AliasPackage && isset($this->rootAliases[$package->getName()][$package->getAliasOf()->getVersion()])) { - $rootAliasData = $this->rootAliases[$package->getName()][$package->getAliasOf()->getVersion()]; - } - - if (isset($rootAliasData)) { - $alias = $this->createAliasPackage($package, $rootAliasData['alias_normalized'], $rootAliasData['alias']); - $alias->setRepository($this); - - $this->providers[$name][$version['uid'].'-root'] = $alias; - $this->providersByUid[$version['uid'].'-root'] = $alias; + $versionsToLoad[$version['uid']] = $version; } } } - $result = $this->providers[$name]; + // load acceptable packages in the providers + $loadedPackages = $this->createPackages($versionsToLoad, 'Composer\Package\CompletePackage'); + $uids = array_keys($versionsToLoad); + + foreach ($loadedPackages as $index => $package) { + $package->setRepository($this); + $uid = $uids[$index]; + + if ($package instanceof AliasPackage) { + $aliased = $package->getAliasOf(); + $aliased->setRepository($this); + + $result[$uid] = $aliased; + $result[$uid.'-alias'] = $package; + } else { + $result[$uid] = $package; + } + } return $result; } diff --git a/tests/Composer/Test/Repository/ComposerRepositoryTest.php b/tests/Composer/Test/Repository/ComposerRepositoryTest.php index 552c767a5..55ca6bf09 100644 --- a/tests/Composer/Test/Repository/ComposerRepositoryTest.php +++ b/tests/Composer/Test/Repository/ComposerRepositoryTest.php @@ -146,21 +146,12 @@ class ComposerRepositoryTest extends TestCase ))); $versionParser = new VersionParser(); - $repo->setRootAliases(array( - 'a' => array( - $versionParser->normalize('0.6') => array('alias' => 'dev-feature', 'alias_normalized' => $versionParser->normalize('dev-feature')), - $versionParser->normalize('1.1.x-dev') => array('alias' => '1.0', 'alias_normalized' => $versionParser->normalize('1.0')), - ), - )); - $reflMethod = new \ReflectionMethod($repo, 'whatProvides'); $reflMethod->setAccessible(true); $packages = $reflMethod->invoke($repo, 'a', array($this, 'isPackageAcceptableReturnTrue')); - $this->assertCount(7, $packages); - $this->assertEquals(array('1', '1-alias', '2', '2-alias', '2-root', '3', '3-root'), array_keys($packages)); - $this->assertInstanceOf('Composer\Package\AliasPackage', $packages['2-root']); - $this->assertSame($packages['2'], $packages['2-root']->getAliasOf()); + $this->assertCount(5, $packages); + $this->assertEquals(array('1', '1-alias', '2', '2-alias', '3'), array_keys($packages)); $this->assertSame($packages['2'], $packages['2-alias']->getAliasOf()); } From c97b7a9be5701ea606ef3eecb01016903f9d25ac Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Mon, 14 Jan 2019 16:37:33 +0100 Subject: [PATCH 27/34] Fix implementation of filterPackages --- src/Composer/Repository/ComposerRepository.php | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 829883b58..6e3994380 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -185,17 +185,25 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito private function filterPackages(array $packages, $constraint = null, $returnFirstMatch = false) { - $packages = array(); + if (null === $constraint) { + if ($returnFirstMatch) { + return reset($packages); + } + + return $packages; + } + + $filteredPackages = array(); foreach ($packages as $package) { - $pkgConstraint = new Constraint('==', $package->getVersion()); - if (null === $constraint || $constraint->matches($pkgConstraint)) { + + if ($constraint->matches($pkgConstraint)) { if ($returnFirstMatch) { return $package; } - $packages[] = $package; + $filteredPackages[] = $package; } } @@ -203,7 +211,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return null; } - return $packages; + return $filteredPackages; } public function getPackages() From 0f2f950cb6aa583b60c7b63845734dce185874a7 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Tue, 15 Jan 2019 11:40:49 +0100 Subject: [PATCH 28/34] Add available-packages key for new repo format, and many consistency tweaks/fixes across various repo formats --- .../Repository/ComposerRepository.php | 108 ++++++++++++++---- 1 file changed, 83 insertions(+), 25 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 6e3994380..8fe6a04ed 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -47,6 +47,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito protected $searchUrl; protected $hasProviders = false; protected $providersUrl; + protected $availablePackages; protected $lazyProvidersUrl; protected $providerListing; protected $loader; @@ -131,20 +132,28 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $this->filterPackages($this->whatProvides($name), $constraint, true); } - return $this->loadAsyncPackages(array($name => $constraint), function ($name, $stability) { + if (is_array($this->availablePackages) && !isset($this->availablePackages[$name])) { + return; + } + + $packages = $this->loadAsyncPackages(array($name => $constraint), function ($name, $stability) { return true; }); + + return reset($packages); } - if (!$hasProviders) { - return parent::findPackage($name, $constraint); - } - - foreach ($this->getProviderNames() as $providerName) { - if ($name === $providerName) { - return $this->filterPackages($this->whatProvides($providerName), $constraint, true); + if ($hasProviders) { + foreach ($this->getProviderNames() as $providerName) { + if ($name === $providerName) { + return $this->filterPackages($this->whatProvides($providerName), $constraint, true); + } } + + return; } + + return parent::findPackage($name, $constraint); } /** @@ -165,22 +174,26 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $this->filterPackages($this->whatProvides($name), $constraint); } + if (is_array($this->availablePackages) && !isset($this->availablePackages[$name])) { + return array(); + } + return $this->loadAsyncPackages(array($name => $constraint ?: new EmptyConstraint()), function ($name, $stability) { return true; }); } - if (!$hasProviders) { - return parent::findPackages($name, $constraint); - } - - foreach ($this->getProviderNames() as $providerName) { - if ($name === $providerName) { - return $this->filterPackages($this->whatProvides($providerName), $constraint); + if ($hasProviders) { + foreach ($this->getProviderNames() as $providerName) { + if ($name === $providerName) { + return $this->filterPackages($this->whatProvides($providerName), $constraint); + } } + + return array(); } - return array(); + return parent::findPackages($name, $constraint); } private function filterPackages(array $packages, $constraint = null, $returnFirstMatch = false) @@ -216,7 +229,22 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito public function getPackages() { - if ($this->hasProviders()) { + $hasProviders = $this->hasProviders(); + + if ($this->lazyProvidersUrl) { + if (is_array($this->availablePackages)) { + $packageMap = array(); + foreach ($this->availablePackages as $name) { + $packageMap[$name] = new EmptyConstraint(); + } + + return array_values($this->loadAsyncPackages($packageMap, function ($name, $stability) { return true; })); + } + + throw new \LogicException('Composer repositories that have lazy providers and no available-packages list can not load the complete list of packages, use getProviderNames instead.'); + } + + if ($hasProviders) { throw new \LogicException('Composer repositories that have providers can not load the complete list of packages, use getProviderNames instead.'); } @@ -225,14 +253,28 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito public function getPackageNames() { - if ($this->hasProviders()) { + // TODO add getPackageNames to the RepositoryInterface perhaps? With filtering capability embedded? + $hasProviders = $this->hasProviders(); + + if ($this->lazyProvidersUrl) { + if (is_array($this->availablePackages)) { + return array_keys($this->availablePackages); + } + + // TODO implement new list API endpoint for those repos somehow? + return array(); + } + + if ($hasProviders) { return $this->getProviderNames(); } - // TODO implement new list API endpoint for repos somehow? - // TODO add getPackageNames to the RepositoryInterface perhaps? With filtering capability embedded? + $names = array(); + foreach ($this->getPackages() as $package) { + $names[] = $package->getPrettyName(); + } - return $this->getPackages(); + return $names; } public function loadPackages(array $packageNameMap, $isPackageAcceptableCallable) @@ -279,11 +321,16 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito unset($packageNameMap[$name]); } - - return $packages; } if ($this->lazyProvidersUrl && count($packageNameMap)) { + if (is_array($this->availablePackages)) { + $availPackages = $this->availablePackages; + $packageNameMap = array_filter($packageNameMap, function ($name) use ($availPackages) { + return isset($availPackages[strtolower($name)]); + }, ARRAY_FILTER_USE_KEY); + } + $packages = array_merge($packages, $this->loadAsyncPackages($packageNameMap, $isPackageAcceptableCallable)); } @@ -317,11 +364,11 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $results; } - if ($this->hasProviders()) { + if ($this->hasProviders() || $this->lazyProvidersUrl) { $results = array(); $regex = '{(?:'.implode('|', preg_split('{\s+}', $query)).')}i'; - foreach ($this->getProviderNames() as $name) { + foreach ($this->getPackageNames() as $name) { if (preg_match($regex, $name)) { $results[] = array('name' => $name); } @@ -683,6 +730,17 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->hasProviders = false; $this->hasPartialPackages = !empty($data['packages']) && is_array($data['packages']); $this->allowSslDowngrade = false; + + // provides a list of package names that are available in this repo + // this disables lazy-provider behavior in the sense that if a list is available we assume it is finite and won't search for other packages in that repo + // while if no list is there lazyProvidersUrl is used when looking for any package name to see if the repo knows it + if (!empty($data['available-packages'])) { + $availPackages = array_map('strtolower', $data['available-packages']); + $this->availablePackages = array_combine($availPackages, $availPackages); + } + + // Remove legacy keys as most repos need to be compatible with Composer v1 + // as well but we are not interested in the old format anymore at this point unset($data['providers-url'], $data['providers'], $data['providers-includes']); } From 3dfcae99a901c8743272e93a0698b4838439041c Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Thu, 17 Jan 2019 17:12:33 +0100 Subject: [PATCH 29/34] Add parallel download capability to FileDownloader and derivatives --- src/Composer/Command/ArchiveCommand.php | 6 +- src/Composer/Command/CreateProjectCommand.php | 22 +- src/Composer/Command/StatusCommand.php | 2 +- src/Composer/Downloader/ArchiveDownloader.php | 106 +++---- src/Composer/Downloader/DownloadManager.php | 224 ++++++++++----- .../Downloader/DownloaderInterface.php | 10 +- src/Composer/Downloader/FileDownloader.php | 221 ++++++++------ src/Composer/Downloader/FossilDownloader.php | 2 +- src/Composer/Downloader/GitDownloader.php | 2 +- src/Composer/Downloader/GzipDownloader.php | 13 +- src/Composer/Downloader/HgDownloader.php | 2 +- src/Composer/Downloader/PathDownloader.php | 9 + .../Downloader/PerforceDownloader.php | 2 +- src/Composer/Downloader/PharDownloader.php | 4 +- src/Composer/Downloader/RarDownloader.php | 3 +- src/Composer/Downloader/SvnDownloader.php | 2 +- src/Composer/Downloader/TarDownloader.php | 4 +- src/Composer/Downloader/VcsDownloader.php | 12 +- src/Composer/Downloader/XzDownloader.php | 10 +- src/Composer/Downloader/ZipDownloader.php | 2 +- src/Composer/Factory.php | 14 +- .../Installer/InstallationManager.php | 27 +- src/Composer/Installer/InstallerInterface.php | 10 + src/Composer/Installer/LibraryInstaller.php | 10 +- .../Installer/MetapackageInstaller.php | 8 + src/Composer/Installer/NoopInstaller.php | 7 + src/Composer/Installer/PluginInstaller.php | 10 +- src/Composer/Installer/ProjectInstaller.php | 13 +- .../Package/Archiver/ArchiveManager.php | 9 +- .../Repository/ComposerRepository.php | 13 +- src/Composer/Util/Http/CurlDownloader.php | 2 +- src/Composer/Util/HttpDownloader.php | 6 +- src/Composer/Util/Loop.php | 47 +++ tests/Composer/Test/ComposerTest.php | 2 +- .../Test/Downloader/ArchiveDownloaderTest.php | 2 +- .../Test/Downloader/DownloadManagerTest.php | 271 ++++++++---------- .../Test/Downloader/FileDownloaderTest.php | 39 ++- .../Test/Downloader/FossilDownloaderTest.php | 4 +- .../Test/Downloader/GitDownloaderTest.php | 12 +- .../Test/Downloader/HgDownloaderTest.php | 4 +- .../Downloader/PerforceDownloaderTest.php | 4 +- .../Test/Downloader/XzDownloaderTest.php | 9 +- .../Test/Downloader/ZipDownloaderTest.php | 43 +-- .../EventDispatcher/EventDispatcherTest.php | 2 +- .../Installer/InstallationManagerTest.php | 41 ++- .../Test/Installer/LibraryInstallerTest.php | 2 +- tests/Composer/Test/Mock/FactoryMock.php | 5 +- .../Test/Mock/InstallationManagerMock.php | 13 + .../Package/Archiver/ArchiveManagerTest.php | 6 +- .../Test/Plugin/PluginInstallerTest.php | 2 +- 50 files changed, 803 insertions(+), 492 deletions(-) create mode 100644 src/Composer/Util/Loop.php diff --git a/src/Composer/Command/ArchiveCommand.php b/src/Composer/Command/ArchiveCommand.php index f893ed679..e26aa59a1 100644 --- a/src/Composer/Command/ArchiveCommand.php +++ b/src/Composer/Command/ArchiveCommand.php @@ -22,6 +22,7 @@ use Composer\Script\ScriptEvents; use Composer\Plugin\CommandEvent; use Composer\Plugin\PluginEvents; use Composer\Util\Filesystem; +use Composer\Util\Loop; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; @@ -104,8 +105,9 @@ EOT $archiveManager = $composer->getArchiveManager(); } else { $factory = new Factory; - $downloadManager = $factory->createDownloadManager($io, $config, $factory->createHttpDownloader($io, $config)); - $archiveManager = $factory->createArchiveManager($config, $downloadManager); + $httpDownloader = $factory->createHttpDownloader($io, $config); + $downloadManager = $factory->createDownloadManager($io, $config, $httpDownloader); + $archiveManager = $factory->createArchiveManager($config, $downloadManager, new Loop($httpDownloader)); } if ($packageName) { diff --git a/src/Composer/Command/CreateProjectCommand.php b/src/Composer/Command/CreateProjectCommand.php index a1c364539..e165649a6 100644 --- a/src/Composer/Command/CreateProjectCommand.php +++ b/src/Composer/Command/CreateProjectCommand.php @@ -38,6 +38,7 @@ use Symfony\Component\Finder\Finder; use Composer\Json\JsonFile; use Composer\Config\JsonConfigSource; use Composer\Util\Filesystem; +use Composer\Util\Loop; use Composer\Package\Version\VersionParser; /** @@ -345,15 +346,18 @@ EOT $package = $package->getAliasOf(); } - $dm = $this->createDownloadManager($io, $config); + $factory = new Factory(); + + $httpDownloader = $factory->createHttpDownloader($io, $config); + $dm = $factory->createDownloadManager($io, $config, $httpDownloader); $dm->setPreferSource($preferSource) ->setPreferDist($preferDist) ->setOutputProgress(!$noProgress); $projectInstaller = new ProjectInstaller($directory, $dm); - $im = $this->createInstallationManager(); + $im = $factory->createInstallationManager(new Loop($httpDownloader)); $im->addInstaller($projectInstaller); - $im->install(new InstalledFilesystemRepository(new JsonFile('php://memory')), new InstallOperation($package)); + $im->execute(new InstalledFilesystemRepository(new JsonFile('php://memory')), new InstallOperation($package)); $im->notifyInstalls($io); // collect suggestions @@ -369,16 +373,4 @@ EOT return $installedFromVcs; } - - protected function createDownloadManager(IOInterface $io, Config $config) - { - $factory = new Factory(); - - return $factory->createDownloadManager($io, $config, $factory->createHttpDownloader($io, $config)); - } - - protected function createInstallationManager() - { - return new InstallationManager(); - } } diff --git a/src/Composer/Command/StatusCommand.php b/src/Composer/Command/StatusCommand.php index 3e46b7fa0..06fc7638b 100644 --- a/src/Composer/Command/StatusCommand.php +++ b/src/Composer/Command/StatusCommand.php @@ -89,7 +89,7 @@ EOT // list packages foreach ($installedRepo->getCanonicalPackages() as $package) { - $downloader = $dm->getDownloaderForInstalledPackage($package); + $downloader = $dm->getDownloaderForPackage($package); $targetDir = $im->getInstallPath($package); if ($downloader instanceof ChangeReportInterface) { diff --git a/src/Composer/Downloader/ArchiveDownloader.php b/src/Composer/Downloader/ArchiveDownloader.php index d041a7f88..3c53a086e 100644 --- a/src/Composer/Downloader/ArchiveDownloader.php +++ b/src/Composer/Downloader/ArchiveDownloader.php @@ -30,33 +30,50 @@ abstract class ArchiveDownloader extends FileDownloader * @throws \RuntimeException * @throws \UnexpectedValueException */ - public function download(PackageInterface $package, $path, $output = true) + public function install(PackageInterface $package, $path, $output = true) { - $temporaryDir = $this->config->get('vendor-dir').'/composer/'.substr(md5(uniqid('', true)), 0, 8); - $retries = 3; - while ($retries--) { - $fileName = parent::download($package, $path, $output); + if ($output) { + $this->io->writeError(" - Installing " . $package->getName() . " (" . $package->getFullPrettyVersion() . ")"); + } - if ($output) { - $this->io->writeError(' Extracting archive', false, IOInterface::VERBOSE); + $temporaryDir = $this->config->get('vendor-dir').'/composer/'.substr(md5(uniqid('', true)), 0, 8); + $fileName = $this->getFileName($package, $path); + + if ($output) { + $this->io->writeError(' Extracting archive', true, IOInterface::VERBOSE); + } + + try { + $this->filesystem->ensureDirectoryExists($temporaryDir); + try { + $this->extract($package, $fileName, $temporaryDir); + } catch (\Exception $e) { + // remove cache if the file was corrupted + parent::clearLastCacheWrite($package); + throw $e; } - try { - $this->filesystem->ensureDirectoryExists($temporaryDir); - try { - $this->extract($fileName, $temporaryDir); - } catch (\Exception $e) { - // remove cache if the file was corrupted - parent::clearLastCacheWrite($package); - throw $e; + $this->filesystem->unlink($fileName); + + $renameAsOne = false; + if (!file_exists($path) || ($this->filesystem->isDirEmpty($path) && $this->filesystem->removeDirectory($path))) { + $renameAsOne = true; + } + + $contentDir = $this->getFolderContent($temporaryDir); + $singleDirAtTopLevel = 1 === count($contentDir) && is_dir(reset($contentDir)); + + if ($renameAsOne) { + // if the target $path is clear, we can rename the whole package in one go instead of looping over the contents + if ($singleDirAtTopLevel) { + $extractedDir = (string) reset($contentDir); + } else { + $extractedDir = $temporaryDir; } - - $this->filesystem->unlink($fileName); - - $contentDir = $this->getFolderContent($temporaryDir); - + $this->filesystem->rename($extractedDir, $path); + } else { // only one dir in the archive, extract its contents out of it - if (1 === count($contentDir) && is_dir(reset($contentDir))) { + if ($singleDirAtTopLevel) { $contentDir = $this->getFolderContent((string) reset($contentDir)); } @@ -65,35 +82,24 @@ abstract class ArchiveDownloader extends FileDownloader $file = (string) $file; $this->filesystem->rename($file, $path . '/' . basename($file)); } - - $this->filesystem->removeDirectory($temporaryDir); - if ($this->filesystem->isDirEmpty($this->config->get('vendor-dir').'/composer/')) { - $this->filesystem->removeDirectory($this->config->get('vendor-dir').'/composer/'); - } - if ($this->filesystem->isDirEmpty($this->config->get('vendor-dir'))) { - $this->filesystem->removeDirectory($this->config->get('vendor-dir')); - } - } catch (\Exception $e) { - // clean up - $this->filesystem->removeDirectory($path); - $this->filesystem->removeDirectory($temporaryDir); - - // retry downloading if we have an invalid zip file - if ($retries && $e instanceof \UnexpectedValueException && class_exists('ZipArchive') && $e->getCode() === \ZipArchive::ER_NOZIP) { - $this->io->writeError(''); - if ($this->io->isDebug()) { - $this->io->writeError(' Invalid zip file ('.$e->getMessage().'), retrying...'); - } else { - $this->io->writeError(' Invalid zip file, retrying...'); - } - usleep(500000); - continue; - } - - throw $e; } - break; + $this->filesystem->removeDirectory($temporaryDir); + if ($this->filesystem->isDirEmpty($this->config->get('vendor-dir').'/composer/')) { + $this->filesystem->removeDirectory($this->config->get('vendor-dir').'/composer/'); + } + if ($this->filesystem->isDirEmpty($this->config->get('vendor-dir'))) { + $this->filesystem->removeDirectory($this->config->get('vendor-dir')); + } + } catch (\Exception $e) { + // clean up + $this->filesystem->removeDirectory($path); + $this->filesystem->removeDirectory($temporaryDir); + if (file_exists($fileName)) { + $this->filesystem->unlink($fileName); + } + + throw $e; } } @@ -102,7 +108,7 @@ abstract class ArchiveDownloader extends FileDownloader */ protected function getFileName(PackageInterface $package, $path) { - return rtrim($path.'/'.md5($path.spl_object_hash($package)).'.'.pathinfo(parse_url($package->getDistUrl(), PHP_URL_PATH), PATHINFO_EXTENSION), '.'); + return rtrim($path.'_'.md5($path.spl_object_hash($package)).'.'.pathinfo(parse_url($package->getDistUrl(), PHP_URL_PATH), PATHINFO_EXTENSION), '.'); } /** @@ -113,7 +119,7 @@ abstract class ArchiveDownloader extends FileDownloader * * @throws \UnexpectedValueException If can not extract downloaded file to path */ - abstract protected function extract($file, $path); + abstract protected function extract(PackageInterface $package, $file, $path); /** * Returns the folder content, excluding dotfiles diff --git a/src/Composer/Downloader/DownloadManager.php b/src/Composer/Downloader/DownloadManager.php index 15c00a6e6..4bc865827 100644 --- a/src/Composer/Downloader/DownloadManager.php +++ b/src/Composer/Downloader/DownloadManager.php @@ -15,6 +15,7 @@ namespace Composer\Downloader; use Composer\Package\PackageInterface; use Composer\IO\IOInterface; use Composer\Util\Filesystem; +use React\Promise\PromiseInterface; /** * Downloaders manager. @@ -24,6 +25,7 @@ use Composer\Util\Filesystem; class DownloadManager { private $io; + private $httpDownloader; private $preferDist = false; private $preferSource = false; private $packagePreferences = array(); @@ -33,9 +35,9 @@ class DownloadManager /** * Initializes download manager. * - * @param IOInterface $io The Input Output Interface - * @param bool $preferSource prefer downloading from source - * @param Filesystem|null $filesystem custom Filesystem object + * @param IOInterface $io The Input Output Interface + * @param bool $preferSource prefer downloading from source + * @param Filesystem|null $filesystem custom Filesystem object */ public function __construct(IOInterface $io, $preferSource = false, Filesystem $filesystem = null) { @@ -140,7 +142,7 @@ class DownloadManager * wrong type * @return DownloaderInterface|null */ - public function getDownloaderForInstalledPackage(PackageInterface $package) + public function getDownloaderForPackage(PackageInterface $package) { $installationSource = $package->getInstallationSource(); @@ -154,7 +156,7 @@ class DownloadManager $downloader = $this->getDownloader($package->getSourceType()); } else { throw new \InvalidArgumentException( - 'Package '.$package.' seems not been installed properly' + 'Package '.$package.' does not have an installation source set' ); } @@ -171,63 +173,95 @@ class DownloadManager return $downloader; } + public function getDownloaderType(DownloaderInterface $downloader) + { + return array_search($downloader, $this->downloaders); + } + /** * Downloads package into target dir. * * @param PackageInterface $package package instance * @param string $targetDir target dir - * @param bool $preferSource prefer installation from source + * @param PackageInterface $prevPackage previous package instance in case of updates + * + * @return PromiseInterface + * @throws \InvalidArgumentException if package have no urls to download from + * @throws \RuntimeException + */ + public function download(PackageInterface $package, $targetDir, PackageInterface $prevPackage = null) + { + $this->filesystem->ensureDirectoryExists(dirname($targetDir)); + + $sources = $this->getAvailableSources($package, $prevPackage); + + $io = $this->io; + $self = $this; + + $download = function ($retry = false) use (&$sources, $io, $package, $self, $targetDir, &$download) { + $source = array_shift($sources); + if ($retry) { + $io->writeError(' Now trying to download from ' . $source . ''); + } + $package->setInstallationSource($source); + + $downloader = $self->getDownloaderForPackage($package); + if (!$downloader) { + return \React\Promise\resolve(); + } + + $handleError = function ($e) use ($sources, $source, $package, $io, $download) { + if ($e instanceof \RuntimeException) { + if (!$sources) { + throw $e; + } + + $io->writeError( + ' Failed to download '. + $package->getPrettyName(). + ' from ' . $source . ': '. + $e->getMessage().'' + ); + + return $download(true); + } + + throw $e; + }; + + try { + $result = $downloader->download($package, $targetDir); + } catch (\Exception $e) { + return $handleError($e); + } + if (!$result instanceof PromiseInterface) { + return \React\Promise\resolve($result); + } + + $res = $result->then(function ($res) { + return $res; + }, $handleError); + + return $res; + }; + + return $download(); + } + + /** + * Installs package into target dir. + * + * @param PackageInterface $package package instance + * @param string $targetDir target dir * * @throws \InvalidArgumentException if package have no urls to download from * @throws \RuntimeException */ - public function download(PackageInterface $package, $targetDir, $preferSource = null) + public function install(PackageInterface $package, $targetDir) { - $preferSource = null !== $preferSource ? $preferSource : $this->preferSource; - $sourceType = $package->getSourceType(); - $distType = $package->getDistType(); - - $sources = array(); - if ($sourceType) { - $sources[] = 'source'; - } - if ($distType) { - $sources[] = 'dist'; - } - - if (empty($sources)) { - throw new \InvalidArgumentException('Package '.$package.' must have a source or dist specified'); - } - - if (!$preferSource && ($this->preferDist || 'dist' === $this->resolvePackageInstallPreference($package))) { - $sources = array_reverse($sources); - } - - $this->filesystem->ensureDirectoryExists($targetDir); - - foreach ($sources as $i => $source) { - if (isset($e)) { - $this->io->writeError(' Now trying to download from ' . $source . ''); - } - $package->setInstallationSource($source); - try { - $downloader = $this->getDownloaderForInstalledPackage($package); - if ($downloader) { - $downloader->download($package, $targetDir); - } - break; - } catch (\RuntimeException $e) { - if ($i === count($sources) - 1) { - throw $e; - } - - $this->io->writeError( - ' Failed to download '. - $package->getPrettyName(). - ' from ' . $source . ': '. - $e->getMessage().'' - ); - } + $downloader = $this->getDownloaderForPackage($package); + if ($downloader) { + $downloader->install($package, $targetDir); } } @@ -242,31 +276,23 @@ class DownloadManager */ public function update(PackageInterface $initial, PackageInterface $target, $targetDir) { - $downloader = $this->getDownloaderForInstalledPackage($initial); + $downloader = $this->getDownloaderForPackage($target); + $initialDownloader = $this->getDownloaderForPackage($initial); + + // no downloaders present means update from metapackage to metapackage, nothing to do + if (!$initialDownloader && !$downloader) { + return; + } + + // if we have a downloader present before, but not after, the package became a metapackage and its files should be removed if (!$downloader) { + $initialDownloader->remove($initial, $targetDir); return; } - $installationSource = $initial->getInstallationSource(); - - if ('dist' === $installationSource) { - $initialType = $initial->getDistType(); - $targetType = $target->getDistType(); - } else { - $initialType = $initial->getSourceType(); - $targetType = $target->getSourceType(); - } - - // upgrading from a dist stable package to a dev package, force source reinstall - if ($target->isDev() && 'dist' === $installationSource) { - $downloader->remove($initial, $targetDir); - $this->download($target, $targetDir); - - return; - } - + $initialType = $this->getDownloaderType($initialDownloader); + $targetType = $this->getDownloaderType($downloader); if ($initialType === $targetType) { - $target->setInstallationSource($installationSource); try { $downloader->update($initial, $target, $targetDir); @@ -282,8 +308,12 @@ class DownloadManager } } - $downloader->remove($initial, $targetDir); - $this->download($target, $targetDir, 'source' === $installationSource); + // if downloader type changed, or update failed and user asks for reinstall, + // we wipe the dir and do a new install instead of updating it + if ($initialDownloader) { + $initialDownloader->remove($initial, $targetDir); + } + $this->install($target, $targetDir); } /** @@ -294,7 +324,7 @@ class DownloadManager */ public function remove(PackageInterface $package, $targetDir) { - $downloader = $this->getDownloaderForInstalledPackage($package); + $downloader = $this->getDownloaderForPackage($package); if ($downloader) { $downloader->remove($package, $targetDir); } @@ -322,4 +352,48 @@ class DownloadManager return $package->isDev() ? 'source' : 'dist'; } + + /** + * @return string[] + */ + private function getAvailableSources(PackageInterface $package, PackageInterface $prevPackage = null) + { + $sourceType = $package->getSourceType(); + $distType = $package->getDistType(); + + // add source before dist by default + $sources = array(); + if ($sourceType) { + $sources[] = 'source'; + } + if ($distType) { + $sources[] = 'dist'; + } + + if (empty($sources)) { + throw new \InvalidArgumentException('Package '.$package.' must have a source or dist specified'); + } + + if ( + $prevPackage + // if we are updating, we want to keep the same source as the previously installed package (if available in the new one) + && in_array($prevPackage->getInstallationSource(), $sources, true) + // unless the previous package was stable dist (by default) and the new package is dev, then we allow the new default to take over + && !(!$prevPackage->isDev() && $prevPackage->getInstallationSource() === 'dist' && $package->isDev()) + ) { + $prevSource = $prevPackage->getInstallationSource(); + usort($sources, function ($a, $b) use ($prevSource) { + return $a === $prevSource ? -1 : 1; + }); + + return $sources; + } + + // reverse sources in case dist is the preferred source for this package + if (!$this->preferSource && ($this->preferDist || 'dist' === $this->resolvePackageInstallPreference($package))) { + $sources = array_reverse($sources); + } + + return $sources; + } } diff --git a/src/Composer/Downloader/DownloaderInterface.php b/src/Composer/Downloader/DownloaderInterface.php index 713bf36dc..ac56583c4 100644 --- a/src/Composer/Downloader/DownloaderInterface.php +++ b/src/Composer/Downloader/DownloaderInterface.php @@ -13,6 +13,7 @@ namespace Composer\Downloader; use Composer\Package\PackageInterface; +use React\Promise\PromiseInterface; /** * Downloader interface. @@ -29,13 +30,20 @@ interface DownloaderInterface */ public function getInstallationSource(); + /** + * This should do any network-related tasks to prepare for install/update + * + * @return PromiseInterface|null + */ + public function download(PackageInterface $package, $path); + /** * Downloads specific package into specific folder. * * @param PackageInterface $package package instance * @param string $path download path */ - public function download(PackageInterface $package, $path); + public function install(PackageInterface $package, $path); /** * Updates specific package in specific folder from initial to target version. diff --git a/src/Composer/Downloader/FileDownloader.php b/src/Composer/Downloader/FileDownloader.php index 8b196e60c..3418eef84 100644 --- a/src/Composer/Downloader/FileDownloader.php +++ b/src/Composer/Downloader/FileDownloader.php @@ -26,6 +26,7 @@ use Composer\EventDispatcher\EventDispatcher; use Composer\Util\Filesystem; use Composer\Util\HttpDownloader; use Composer\Util\Url as UrlUtil; +use Composer\Downloader\TransportException; /** * Base downloader for files @@ -43,7 +44,10 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface protected $filesystem; protected $cache; protected $outputProgress = true; - private $lastCacheWrites = array(); + /** + * @private this is only public for php 5.3 support in closures + */ + public $lastCacheWrites = array(); private $eventDispatcher; /** @@ -87,108 +91,149 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface throw new \InvalidArgumentException('The given package is missing url information'); } - if ($output) { - $this->io->writeError(" - Installing " . $package->getName() . " (" . $package->getFullPrettyVersion() . "): ", false); - } - + $retries = 3; $urls = $package->getDistUrls(); - while ($url = array_shift($urls)) { - try { - $fileName = $this->doDownload($package, $path, $url); - break; - } catch (\Exception $e) { - if ($this->io->isDebug()) { - $this->io->writeError(''); - $this->io->writeError('Failed: ['.get_class($e).'] '.$e->getCode().': '.$e->getMessage()); - } elseif (count($urls)) { - $this->io->writeError(''); - $this->io->writeError(' Failed, trying the next URL ('.$e->getCode().': '.$e->getMessage().')', false); - } - - if (!count($urls)) { - throw $e; - } - } + foreach ($urls as $index => $url) { + $processedUrl = $this->processUrl($package, $url); + $urls[$index] = array( + 'base' => $url, + 'processed' => $processedUrl, + 'cacheKey' => $this->getCacheKey($package, $processedUrl) + ); } - if ($output) { - $this->io->writeError(''); - } - - return $fileName; - } - - protected function doDownload(PackageInterface $package, $path, $url) - { $this->filesystem->emptyDirectory($path); - $fileName = $this->getFileName($package, $path); - $processedUrl = $this->processUrl($package, $url); + $io = $this->io; + $cache = $this->cache; + $originalHttpDownloader = $this->httpDownloader; + $eventDispatcher = $this->eventDispatcher; + $filesystem = $this->filesystem; + $self = $this; - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $processedUrl); - if ($this->eventDispatcher) { - $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - } - $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); + $accept = null; + $reject = null; + $download = function () use ($io, $output, $originalHttpDownloader, $cache, $eventDispatcher, $package, $fileName, $path, &$urls, &$accept, &$reject) { + $url = reset($urls); + + $httpDownloader = $originalHttpDownloader; + if ($eventDispatcher) { + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $httpDownloader, $url['processed']); + $eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); + } - try { $checksum = $package->getDistSha1Checksum(); - $cacheKey = $this->getCacheKey($package, $processedUrl); + $cacheKey = $url['cacheKey']; // use from cache if it is present and has a valid checksum or we have no checksum to check against - if ($this->cache && (!$checksum || $checksum === $this->cache->sha1($cacheKey)) && $this->cache->copyTo($cacheKey, $fileName)) { - $this->io->writeError('Loading from cache', false); + if ($cache && (!$checksum || $checksum === $cache->sha1($cacheKey)) && $cache->copyTo($cacheKey, $fileName)) { + if ($output) { + $io->writeError(" - Loading " . $package->getName() . " (" . $package->getFullPrettyVersion() . ") from cache"); + } + $result = \React\Promise\resolve($fileName); } else { - // download if cache restore failed - if (!$this->outputProgress) { - $this->io->writeError('Downloading', false); + if ($output) { + $io->writeError(" - Downloading " . $package->getName() . " (" . $package->getFullPrettyVersion() . ")"); } - // try to download 3 times then fail hard - $retries = 3; - while ($retries--) { - try { - // TODO handle this->outputProgress - $httpDownloader->copy($processedUrl, $fileName, $package->getTransportOptions()); - break; - } catch (TransportException $e) { - // if we got an http response with a proper code, then requesting again will probably not help, abort - if ((0 !== $e->getCode() && !in_array($e->getCode(), array(500, 502, 503, 504))) || !$retries) { - throw $e; - } - $this->io->writeError(''); - $this->io->writeError(' Download failed, retrying...', true, IOInterface::VERBOSE); - usleep(500000); - } - } - - if (!$this->outputProgress) { - $this->io->writeError(' (100%)', false); - } - - if ($this->cache) { - $this->lastCacheWrites[$package->getName()] = $cacheKey; - $this->cache->copyFrom($cacheKey, $fileName); - } + $result = $httpDownloader->addCopy($url['processed'], $fileName, $package->getTransportOptions()) + ->then($accept, $reject); } - if (!file_exists($fileName)) { - throw new \UnexpectedValueException($url.' could not be saved to '.$fileName.', make sure the' - .' directory is writable and you have internet connectivity'); + return $result->then(function ($result) use ($fileName, $checksum, $url) { + // in case of retry, the first call's Promise chain finally calls this twice at the end, + // once with $result being the returned $fileName from $accept, and then once for every + // failed request with a null result, which can be skipped. + if (null === $result) { + return $fileName; + } + + if (!file_exists($fileName)) { + throw new \UnexpectedValueException($url['base'].' could not be saved to '.$fileName.', make sure the' + .' directory is writable and you have internet connectivity'); + } + + if ($checksum && hash_file('sha1', $fileName) !== $checksum) { + throw new \UnexpectedValueException('The checksum verification of the file failed (downloaded from '.$url['base'].')'); + } + + return $fileName; + }); + }; + + $accept = function ($response) use ($io, $cache, $package, $fileName, $path, $self, &$urls) { + $url = reset($urls); + $cacheKey = $url['cacheKey']; + + if ($cache) { + $self->lastCacheWrites[$package->getName()] = $cacheKey; + $cache->copyFrom($cacheKey, $fileName); } - if ($checksum && hash_file('sha1', $fileName) !== $checksum) { - throw new \UnexpectedValueException('The checksum verification of the file failed (downloaded from '.$url.')'); - } - } catch (\Exception $e) { + $response->collect(); + + return $fileName; + }; + + $reject = function ($e) use ($io, &$urls, $download, $fileName, $path, $package, &$retries, $filesystem, $self) { // clean up - $this->filesystem->removeDirectory($path); - $this->clearLastCacheWrite($package); + $filesystem->removeDirectory($path); + $self->clearLastCacheWrite($package); + + if ($e instanceof TransportException) { + // if we got an http response with a proper code, then requesting again will probably not help, abort + if ((0 !== $e->getCode() && !in_array($e->getCode(), array(500, 502, 503, 504))) || !$retries) { + $retries = 0; + } + } + + // special error code returned when network is being artificially disabled + if ($e instanceof TransportException && $e->getStatusCode() === 499) { + $retries = 0; + $urls = array(); + } + + if ($retries) { + usleep(500000); + $retries--; + + return $download(); + } + + array_shift($urls); + if ($urls) { + if ($io->isDebug()) { + $io->writeError(' Failed downloading '.$package->getName().': ['.get_class($e).'] '.$e->getCode().': '.$e->getMessage()); + $io->writeError(' Trying the next URL for '.$package->getName()); + } elseif (count($urls)) { + $io->writeError(' Failed downloading '.$package->getName().', trying the next URL ('.$e->getCode().': '.$e->getMessage().')'); + } + + $retries = 3; + usleep(100000); + + return $download(); + } + throw $e; + }; + + return $download(); + } + + /** + * {@inheritDoc} + */ + public function install(PackageInterface $package, $path, $output = true) + { + if ($output) { + $this->io->writeError(" - Installing " . $package->getName() . " (" . $package->getFullPrettyVersion() . ")"); } - return $fileName; + $this->filesystem->ensureDirectoryExists($path); + $this->filesystem->rename($this->getFileName($package, $path), $path . pathinfo(parse_url($package->getDistUrl(), PHP_URL_PATH), PATHINFO_BASENAME)); } /** @@ -201,7 +246,11 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface return $this; } - protected function clearLastCacheWrite(PackageInterface $package) + /** + * TODO mark private in v3 + * @protected This is public due to PHP 5.3 + */ + public function clearLastCacheWrite(PackageInterface $package) { if ($this->cache && isset($this->lastCacheWrites[$package->getName()])) { $this->cache->remove($this->lastCacheWrites[$package->getName()]); @@ -222,7 +271,7 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface $this->io->writeError(" - " . $actionName . " " . $name . " (" . $from . " => " . $to . "): ", false); $this->remove($initial, $path, false); - $this->download($target, $path, false); + $this->install($target, $path, false); $this->io->writeError(''); } @@ -249,7 +298,7 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface */ protected function getFileName(PackageInterface $package, $path) { - return $path.'/'.pathinfo(parse_url($package->getDistUrl(), PHP_URL_PATH), PATHINFO_BASENAME); + return $path.'_'.pathinfo(parse_url($package->getDistUrl(), PHP_URL_PATH), PATHINFO_BASENAME); } /** @@ -299,7 +348,9 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface $e = null; try { - $this->download($package, $targetDir.'_compare', false); + $res = $this->download($package, $targetDir.'_compare', false); + $this->httpDownloader->wait(); + $res = $this->install($package, $targetDir.'_compare', false); $comparer = new Comparer(); $comparer->setSource($targetDir.'_compare'); diff --git a/src/Composer/Downloader/FossilDownloader.php b/src/Composer/Downloader/FossilDownloader.php index 135e973e0..a814f89b7 100644 --- a/src/Composer/Downloader/FossilDownloader.php +++ b/src/Composer/Downloader/FossilDownloader.php @@ -23,7 +23,7 @@ class FossilDownloader extends VcsDownloader /** * {@inheritDoc} */ - public function doDownload(PackageInterface $package, $path, $url) + public function doInstall(PackageInterface $package, $path, $url) { // Ensure we are allowed to use this URL by config $this->config->prohibitUrlByConfig($url, $this->io); diff --git a/src/Composer/Downloader/GitDownloader.php b/src/Composer/Downloader/GitDownloader.php index 869d5330b..ff398f300 100644 --- a/src/Composer/Downloader/GitDownloader.php +++ b/src/Composer/Downloader/GitDownloader.php @@ -38,7 +38,7 @@ class GitDownloader extends VcsDownloader implements DvcsDownloaderInterface /** * {@inheritDoc} */ - public function doDownload(PackageInterface $package, $path, $url) + public function doInstall(PackageInterface $package, $path, $url) { GitUtil::cleanEnv(); $path = $this->normalizePath($path); diff --git a/src/Composer/Downloader/GzipDownloader.php b/src/Composer/Downloader/GzipDownloader.php index f65fcf27d..9748b91ac 100644 --- a/src/Composer/Downloader/GzipDownloader.php +++ b/src/Composer/Downloader/GzipDownloader.php @@ -36,9 +36,10 @@ class GzipDownloader extends ArchiveDownloader parent::__construct($io, $config, $downloader, $eventDispatcher, $cache); } - protected function extract($file, $path) + protected function extract(PackageInterface $package, $file, $path) { - $targetFilepath = $path . DIRECTORY_SEPARATOR . basename(substr($file, 0, -3)); + $filename = pathinfo(parse_url($package->getDistUrl(), PHP_URL_PATH), PATHINFO_FILENAME); + $targetFilepath = $path . DIRECTORY_SEPARATOR . $filename; // Try to use gunzip on *nix if (!Platform::isWindows()) { @@ -63,14 +64,6 @@ class GzipDownloader extends ArchiveDownloader $this->extractUsingExt($file, $targetFilepath); } - /** - * {@inheritdoc} - */ - protected function getFileName(PackageInterface $package, $path) - { - return $path.'/'.pathinfo(parse_url($package->getDistUrl(), PHP_URL_PATH), PATHINFO_BASENAME); - } - private function extractUsingExt($file, $targetFilepath) { $archiveFile = gzopen($file, 'rb'); diff --git a/src/Composer/Downloader/HgDownloader.php b/src/Composer/Downloader/HgDownloader.php index 2921cc4b7..add381a75 100644 --- a/src/Composer/Downloader/HgDownloader.php +++ b/src/Composer/Downloader/HgDownloader.php @@ -24,7 +24,7 @@ class HgDownloader extends VcsDownloader /** * {@inheritDoc} */ - public function doDownload(PackageInterface $package, $path, $url) + public function doInstall(PackageInterface $package, $path, $url) { $hgUtils = new HgUtils($this->io, $this->config, $this->process); diff --git a/src/Composer/Downloader/PathDownloader.php b/src/Composer/Downloader/PathDownloader.php index e7084bd97..ea583cfc0 100644 --- a/src/Composer/Downloader/PathDownloader.php +++ b/src/Composer/Downloader/PathDownloader.php @@ -61,6 +61,15 @@ class PathDownloader extends FileDownloader implements VcsCapableDownloaderInter $realUrl )); } + } + + /** + * {@inheritdoc} + */ + public function install(PackageInterface $package, $path, $output = true) + { + $url = $package->getDistUrl(); + $realUrl = realpath($url); // Get the transport options with default values $transportOptions = $package->getTransportOptions() + array('symlink' => null); diff --git a/src/Composer/Downloader/PerforceDownloader.php b/src/Composer/Downloader/PerforceDownloader.php index a472b84c6..df270417f 100644 --- a/src/Composer/Downloader/PerforceDownloader.php +++ b/src/Composer/Downloader/PerforceDownloader.php @@ -27,7 +27,7 @@ class PerforceDownloader extends VcsDownloader /** * {@inheritDoc} */ - public function doDownload(PackageInterface $package, $path, $url) + public function doInstall(PackageInterface $package, $path, $url) { $ref = $package->getSourceReference(); $label = $this->getLabelFromSourceReference($ref); diff --git a/src/Composer/Downloader/PharDownloader.php b/src/Composer/Downloader/PharDownloader.php index 13fec244b..62741ee0e 100644 --- a/src/Composer/Downloader/PharDownloader.php +++ b/src/Composer/Downloader/PharDownloader.php @@ -12,6 +12,8 @@ namespace Composer\Downloader; +use Composer\Package\PackageInterface; + /** * Downloader for phar files * @@ -22,7 +24,7 @@ class PharDownloader extends ArchiveDownloader /** * {@inheritDoc} */ - protected function extract($file, $path) + protected function extract(PackageInterface $package, $file, $path) { // Can throw an UnexpectedValueException $archive = new \Phar($file); diff --git a/src/Composer/Downloader/RarDownloader.php b/src/Composer/Downloader/RarDownloader.php index 6fe4cf27c..2ebc3bf18 100644 --- a/src/Composer/Downloader/RarDownloader.php +++ b/src/Composer/Downloader/RarDownloader.php @@ -20,6 +20,7 @@ use Composer\Util\Platform; use Composer\Util\ProcessExecutor; use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; +use Composer\Package\PackageInterface; use RarArchive; /** @@ -39,7 +40,7 @@ class RarDownloader extends ArchiveDownloader parent::__construct($io, $config, $downloader, $eventDispatcher, $cache); } - protected function extract($file, $path) + protected function extract(PackageInterface $package, $file, $path) { $processError = null; diff --git a/src/Composer/Downloader/SvnDownloader.php b/src/Composer/Downloader/SvnDownloader.php index e23958164..0aae163c6 100644 --- a/src/Composer/Downloader/SvnDownloader.php +++ b/src/Composer/Downloader/SvnDownloader.php @@ -28,7 +28,7 @@ class SvnDownloader extends VcsDownloader /** * {@inheritDoc} */ - public function doDownload(PackageInterface $package, $path, $url) + public function doInstall(PackageInterface $package, $path, $url) { SvnUtil::cleanEnv(); $ref = $package->getSourceReference(); diff --git a/src/Composer/Downloader/TarDownloader.php b/src/Composer/Downloader/TarDownloader.php index 34c43da5f..e48407230 100644 --- a/src/Composer/Downloader/TarDownloader.php +++ b/src/Composer/Downloader/TarDownloader.php @@ -12,6 +12,8 @@ namespace Composer\Downloader; +use Composer\Package\PackageInterface; + /** * Downloader for tar files: tar, tar.gz or tar.bz2 * @@ -22,7 +24,7 @@ class TarDownloader extends ArchiveDownloader /** * {@inheritDoc} */ - protected function extract($file, $path) + protected function extract(PackageInterface $package, $file, $path) { // Can throw an UnexpectedValueException $archive = new \PharData($file); diff --git a/src/Composer/Downloader/VcsDownloader.php b/src/Composer/Downloader/VcsDownloader.php index aa666058e..237d7e49d 100644 --- a/src/Composer/Downloader/VcsDownloader.php +++ b/src/Composer/Downloader/VcsDownloader.php @@ -55,6 +55,14 @@ abstract class VcsDownloader implements DownloaderInterface, ChangeReportInterfa * {@inheritDoc} */ public function download(PackageInterface $package, $path) + { + // noop for now, ideally we would do a git fetch already here, or make sure the cached git repo is synced, etc. + } + + /** + * {@inheritDoc} + */ + public function install(PackageInterface $package, $path) { if (!$package->getSourceReference()) { throw new \InvalidArgumentException('Package '.$package->getPrettyName().' is missing reference information'); @@ -87,7 +95,7 @@ abstract class VcsDownloader implements DownloaderInterface, ChangeReportInterfa $url = $needle . $url; } } - $this->doDownload($package, $path, $url); + $this->doInstall($package, $path, $url); break; } catch (\Exception $e) { // rethrow phpunit exceptions to avoid hard to debug bug failures @@ -260,7 +268,7 @@ abstract class VcsDownloader implements DownloaderInterface, ChangeReportInterfa * @param string $path download path * @param string $url package url */ - abstract protected function doDownload(PackageInterface $package, $path, $url); + abstract protected function doInstall(PackageInterface $package, $path, $url); /** * Updates specific package in specific folder from initial to target version. diff --git a/src/Composer/Downloader/XzDownloader.php b/src/Composer/Downloader/XzDownloader.php index bd7b028e2..19e51c321 100644 --- a/src/Composer/Downloader/XzDownloader.php +++ b/src/Composer/Downloader/XzDownloader.php @@ -37,7 +37,7 @@ class XzDownloader extends ArchiveDownloader parent::__construct($io, $config, $downloader, $eventDispatcher, $cache); } - protected function extract($file, $path) + protected function extract(PackageInterface $package, $file, $path) { $command = 'tar -xJf ' . ProcessExecutor::escape($file) . ' -C ' . ProcessExecutor::escape($path); @@ -49,12 +49,4 @@ class XzDownloader extends ArchiveDownloader throw new \RuntimeException($processError); } - - /** - * {@inheritdoc} - */ - protected function getFileName(PackageInterface $package, $path) - { - return $path.'/'.pathinfo(parse_url($package->getDistUrl(), PHP_URL_PATH), PATHINFO_BASENAME); - } } diff --git a/src/Composer/Downloader/ZipDownloader.php b/src/Composer/Downloader/ZipDownloader.php index 9eceab250..efa9fc994 100644 --- a/src/Composer/Downloader/ZipDownloader.php +++ b/src/Composer/Downloader/ZipDownloader.php @@ -185,7 +185,7 @@ class ZipDownloader extends ArchiveDownloader * @param string $file File to extract * @param string $path Path where to extract file */ - public function extract($file, $path) + public function extract(PackageInterface $package, $file, $path) { // Each extract calls its alternative if not available or fails if (self::$isWindows) { diff --git a/src/Composer/Factory.php b/src/Composer/Factory.php index c96fd2188..e66e64ed1 100644 --- a/src/Composer/Factory.php +++ b/src/Composer/Factory.php @@ -24,6 +24,7 @@ use Composer\Util\Filesystem; use Composer\Util\Platform; use Composer\Util\ProcessExecutor; use Composer\Util\HttpDownloader; +use Composer\Util\Loop; use Composer\Util\Silencer; use Composer\Plugin\PluginEvents; use Composer\EventDispatcher\Event; @@ -326,6 +327,7 @@ class Factory } $httpDownloader = self::createHttpDownloader($io, $config); + $loop = new Loop($httpDownloader); // initialize event dispatcher $dispatcher = new EventDispatcher($composer, $io); @@ -352,7 +354,7 @@ class Factory $composer->setPackage($package); // initialize installation manager - $im = $this->createInstallationManager(); + $im = $this->createInstallationManager($loop); $composer->setInstallationManager($im); if ($fullLoad) { @@ -365,7 +367,7 @@ class Factory $composer->setAutoloadGenerator($generator); // initialize archive manager - $am = $this->createArchiveManager($config, $dm); + $am = $this->createArchiveManager($config, $dm, $loop); $composer->setArchiveManager($am); } @@ -501,9 +503,9 @@ class Factory * @param Downloader\DownloadManager $dm Manager use to download sources * @return Archiver\ArchiveManager */ - public function createArchiveManager(Config $config, Downloader\DownloadManager $dm) + public function createArchiveManager(Config $config, Downloader\DownloadManager $dm, Loop $loop) { - $am = new Archiver\ArchiveManager($dm); + $am = new Archiver\ArchiveManager($dm, $loop); $am->addArchiver(new Archiver\ZipArchiver); $am->addArchiver(new Archiver\PharArchiver); @@ -525,9 +527,9 @@ class Factory /** * @return Installer\InstallationManager */ - protected function createInstallationManager() + public function createInstallationManager(Loop $loop) { - return new Installer\InstallationManager(); + return new Installer\InstallationManager($loop); } /** diff --git a/src/Composer/Installer/InstallationManager.php b/src/Composer/Installer/InstallationManager.php index 9f50b5980..ce10dc4da 100644 --- a/src/Composer/Installer/InstallationManager.php +++ b/src/Composer/Installer/InstallationManager.php @@ -24,6 +24,7 @@ use Composer\DependencyResolver\Operation\UninstallOperation; use Composer\DependencyResolver\Operation\MarkAliasInstalledOperation; use Composer\DependencyResolver\Operation\MarkAliasUninstalledOperation; use Composer\Util\StreamContextFactory; +use Composer\Util\Loop; /** * Package operation manager. @@ -37,6 +38,12 @@ class InstallationManager private $installers = array(); private $cache = array(); private $notifiablePackages = array(); + private $loop; + + public function __construct(Loop $loop) + { + $this->loop = $loop; + } public function reset() { @@ -156,7 +163,24 @@ class InstallationManager */ public function execute(RepositoryInterface $repo, OperationInterface $operation) { + // TODO this should take all operations in one go $method = $operation->getJobType(); + + if ($method === 'install') { + $package = $operation->getPackage(); + $installer = $this->getInstaller($package->getType()); + $promise = $installer->download($package); + } elseif ($method === 'update') { + $target = $operation->getTargetPackage(); + $targetType = $target->getType(); + $installer = $this->getInstaller($targetType); + $promise = $installer->download($target, $operation->getInitialPackage()); + } + + if (isset($promise)) { + $this->loop->wait(array($promise)); + } + $this->$method($repo, $operation); } @@ -194,7 +218,8 @@ class InstallationManager $this->markForNotification($target); } else { $this->getInstaller($initialType)->uninstall($repo, $initial); - $this->getInstaller($targetType)->install($repo, $target); + $installer = $this->getInstaller($targetType); + $installer->install($repo, $target); } } diff --git a/src/Composer/Installer/InstallerInterface.php b/src/Composer/Installer/InstallerInterface.php index e64dfadd2..e00877ed9 100644 --- a/src/Composer/Installer/InstallerInterface.php +++ b/src/Composer/Installer/InstallerInterface.php @@ -15,6 +15,7 @@ namespace Composer\Installer; use Composer\Package\PackageInterface; use Composer\Repository\InstalledRepositoryInterface; use InvalidArgumentException; +use React\Promise\PromiseInterface; /** * Interface for the package installation manager. @@ -42,6 +43,15 @@ interface InstallerInterface */ public function isInstalled(InstalledRepositoryInterface $repo, PackageInterface $package); + /** + * Downloads the files needed to later install the given package. + * + * @param PackageInterface $package package instance + * @param PackageInterface $prevPackage previous package instance in case of an update + * @return PromiseInterface + */ + public function download(PackageInterface $package, PackageInterface $prevPackage = null); + /** * Installs specific package. * diff --git a/src/Composer/Installer/LibraryInstaller.php b/src/Composer/Installer/LibraryInstaller.php index 34fbbbee4..4c2f45601 100644 --- a/src/Composer/Installer/LibraryInstaller.php +++ b/src/Composer/Installer/LibraryInstaller.php @@ -85,6 +85,14 @@ class LibraryInstaller implements InstallerInterface, BinaryPresenceInterface return (Platform::isWindows() && $this->filesystem->isJunction($installPath)) || is_link($installPath); } + public function download(PackageInterface $package, PackageInterface $prevPackage = null) + { + $this->initializeVendorDir(); + $downloadPath = $this->getInstallPath($package); + + return $this->downloadManager->download($package, $downloadPath, $prevPackage); + } + /** * {@inheritDoc} */ @@ -194,7 +202,7 @@ class LibraryInstaller implements InstallerInterface, BinaryPresenceInterface protected function installCode(PackageInterface $package) { $downloadPath = $this->getInstallPath($package); - $this->downloadManager->download($package, $downloadPath); + $this->downloadManager->install($package, $downloadPath); } protected function updateCode(PackageInterface $initial, PackageInterface $target) diff --git a/src/Composer/Installer/MetapackageInstaller.php b/src/Composer/Installer/MetapackageInstaller.php index 3f99ec03c..7dbf4af67 100644 --- a/src/Composer/Installer/MetapackageInstaller.php +++ b/src/Composer/Installer/MetapackageInstaller.php @@ -38,6 +38,14 @@ class MetapackageInstaller implements InstallerInterface return $repo->hasPackage($package); } + /** + * {@inheritDoc} + */ + public function download(PackageInterface $package, PackageInterface $prevPackage = null) + { + // noop + } + /** * {@inheritDoc} */ diff --git a/src/Composer/Installer/NoopInstaller.php b/src/Composer/Installer/NoopInstaller.php index 72cf17d22..51df3c305 100644 --- a/src/Composer/Installer/NoopInstaller.php +++ b/src/Composer/Installer/NoopInstaller.php @@ -40,6 +40,13 @@ class NoopInstaller implements InstallerInterface return $repo->hasPackage($package); } + /** + * {@inheritDoc} + */ + public function download(PackageInterface $package, PackageInterface $prevPackage = null) + { + } + /** * {@inheritDoc} */ diff --git a/src/Composer/Installer/PluginInstaller.php b/src/Composer/Installer/PluginInstaller.php index c400ca4a6..62a16fc62 100644 --- a/src/Composer/Installer/PluginInstaller.php +++ b/src/Composer/Installer/PluginInstaller.php @@ -50,13 +50,21 @@ class PluginInstaller extends LibraryInstaller /** * {@inheritDoc} */ - public function install(InstalledRepositoryInterface $repo, PackageInterface $package) + public function download(PackageInterface $package, PackageInterface $prevPackage = null) { $extra = $package->getExtra(); if (empty($extra['class'])) { throw new \UnexpectedValueException('Error while installing '.$package->getPrettyName().', composer-plugin packages should have a class defined in their extra key to be usable.'); } + return parent::download($package, $prevPackage); + } + + /** + * {@inheritDoc} + */ + public function install(InstalledRepositoryInterface $repo, PackageInterface $package) + { parent::install($repo, $package); try { $this->composer->getPluginManager()->registerPackage($package, true); diff --git a/src/Composer/Installer/ProjectInstaller.php b/src/Composer/Installer/ProjectInstaller.php index c79238b36..350b220f5 100644 --- a/src/Composer/Installer/ProjectInstaller.php +++ b/src/Composer/Installer/ProjectInstaller.php @@ -58,7 +58,7 @@ class ProjectInstaller implements InstallerInterface /** * {@inheritDoc} */ - public function install(InstalledRepositoryInterface $repo, PackageInterface $package) + public function download(PackageInterface $package, PackageInterface $prevPackage = null) { $installPath = $this->installPath; if (file_exists($installPath) && !$this->filesystem->isDirEmpty($installPath)) { @@ -67,7 +67,16 @@ class ProjectInstaller implements InstallerInterface if (!is_dir($installPath)) { mkdir($installPath, 0777, true); } - $this->downloadManager->download($package, $installPath); + + return $this->downloadManager->download($package, $installPath, $prevPackage); + } + + /** + * {@inheritDoc} + */ + public function install(InstalledRepositoryInterface $repo, PackageInterface $package) + { + $this->downloadManager->install($package, $this->installPath); } /** diff --git a/src/Composer/Package/Archiver/ArchiveManager.php b/src/Composer/Package/Archiver/ArchiveManager.php index 22f8eeafe..359d6b053 100644 --- a/src/Composer/Package/Archiver/ArchiveManager.php +++ b/src/Composer/Package/Archiver/ArchiveManager.php @@ -16,6 +16,7 @@ use Composer\Downloader\DownloadManager; use Composer\Package\PackageInterface; use Composer\Package\RootPackageInterface; use Composer\Util\Filesystem; +use Composer\Util\Loop; use Composer\Json\JsonFile; /** @@ -25,6 +26,7 @@ use Composer\Json\JsonFile; class ArchiveManager { protected $downloadManager; + protected $loop; protected $archivers = array(); @@ -36,9 +38,10 @@ class ArchiveManager /** * @param DownloadManager $downloadManager A manager used to download package sources */ - public function __construct(DownloadManager $downloadManager) + public function __construct(DownloadManager $downloadManager, Loop $loop) { $this->downloadManager = $downloadManager; + $this->loop = $loop; } /** @@ -148,7 +151,9 @@ class ArchiveManager $filesystem->ensureDirectoryExists($sourcePath); // Download sources - $this->downloadManager->download($package, $sourcePath); + $promise = $this->downloadManager->download($package, $sourcePath); + $this->loop->wait(array($promise)); + $this->downloadManager->install($package, $sourcePath); // Check exclude from downloaded composer.json if (file_exists($composerJsonPath = $sourcePath.'/composer.json')) { diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 8fe6a04ed..af627df73 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -22,6 +22,7 @@ use Composer\Config; use Composer\Factory; use Composer\IO\IOInterface; use Composer\Util\HttpDownloader; +use Composer\Util\Loop; use Composer\Plugin\PluginEvents; use Composer\Plugin\PreFileDownloadEvent; use Composer\EventDispatcher\EventDispatcher; @@ -42,6 +43,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito private $baseUrl; private $io; private $httpDownloader; + private $loop; protected $cache; protected $notifyUrl; protected $searchUrl; @@ -107,6 +109,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->httpDownloader = $httpDownloader; $this->eventDispatcher = $eventDispatcher; $this->repoConfig = $repoConfig; + $this->loop = new Loop($this->httpDownloader); } public function getRepoConfig() @@ -569,6 +572,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->loadRootServerFile(); $packages = array(); + $promises = array(); $repo = $this; if (!$this->lazyProvidersUrl) { @@ -592,7 +596,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $lastModified = isset($contents['last-modified']) ? $contents['last-modified'] : null; } - $this->asyncFetchFile($url, $cacheKey, $lastModified) + $promises[] = $this->asyncFetchFile($url, $cacheKey, $lastModified) ->then(function ($response) use (&$packages, $contents, $name, $constraint, $repo, $isPackageAcceptableCallable) { static $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); @@ -637,13 +641,10 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $packages[spl_object_hash($package->getAliasOf())] = $package->getAliasOf(); } } - }, function ($e) { - // TODO use ->done() above instead with react/promise 2.0 - throw $e; }); } - $this->httpDownloader->wait(); + $this->loop->wait($promises); return $packages; // RepositorySet should call loadMetadata, getMetadata when all promises resolved, then metadataComplete when done so we can GC the loaded json and whatnot then as needed @@ -1119,7 +1120,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } $degradedMode = true; - return true; + throw $e; }; return $httpDownloader->add($filename, $options)->then($accept, $reject); diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php index 3dc6710b6..ff31bf695 100644 --- a/src/Composer/Util/Http/CurlDownloader.php +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -295,7 +295,7 @@ class CurlDownloader // resolve promise if ($job['filename']) { rename($job['filename'].'~', $job['filename']); - call_user_func($job['resolve'], true); + call_user_func($job['resolve'], $response); } else { call_user_func($job['resolve'], $response); } diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php index f2308d75a..172ea875a 100644 --- a/src/Composer/Util/HttpDownloader.php +++ b/src/Composer/Util/HttpDownloader.php @@ -160,7 +160,10 @@ class HttpDownloader if ($job['request']['copyTo']) { $result = $rfs->copy($job['origin'], $url, $job['request']['copyTo'], false /* TODO progress */, $options); - $resolve($result); + $headers = $rfs->getLastHeaders(); + $response = new Http\Response($job['request'], $rfs->findStatusCode($headers), $headers, $job['request']['copyTo'].'~'); + + $resolve($response); } else { $body = $rfs->getContents($job['origin'], $url, false /* TODO progress */, $options); $headers = $rfs->getLastHeaders(); @@ -191,6 +194,7 @@ class HttpDownloader $job['exception'] = $e; $downloader->markJobDone(); + $downloader->scheduleNextJob(); throw $e; }); diff --git a/src/Composer/Util/Loop.php b/src/Composer/Util/Loop.php new file mode 100644 index 000000000..1be7d478b --- /dev/null +++ b/src/Composer/Util/Loop.php @@ -0,0 +1,47 @@ + + * Jordi Boggiano + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Composer\Util; + +use Composer\Util\HttpDownloader; +use React\Promise\Promise; + +/** + * @author Jordi Boggiano + */ +class Loop +{ + private $io; + + public function __construct(HttpDownloader $httpDownloader) + { + $this->httpDownloader = $httpDownloader; + } + + public function wait(array $promises) + { + $uncaught = null; + + \React\Promise\all($promises)->then( + function () { }, + function ($e) use (&$uncaught) { + $uncaught = $e; + } + ); + + $this->httpDownloader->wait(); + + if ($uncaught) { + throw $uncaught; + } + } +} diff --git a/tests/Composer/Test/ComposerTest.php b/tests/Composer/Test/ComposerTest.php index c2c425e76..87270baae 100644 --- a/tests/Composer/Test/ComposerTest.php +++ b/tests/Composer/Test/ComposerTest.php @@ -57,7 +57,7 @@ class ComposerTest extends TestCase public function testSetGetInstallationManager() { $composer = new Composer(); - $manager = $this->getMockBuilder('Composer\Installer\InstallationManager')->getMock(); + $manager = $this->getMockBuilder('Composer\Installer\InstallationManager')->disableOriginalConstructor()->getMock(); $composer->setInstallationManager($manager); $this->assertSame($manager, $composer->getInstallationManager()); diff --git a/tests/Composer/Test/Downloader/ArchiveDownloaderTest.php b/tests/Composer/Test/Downloader/ArchiveDownloaderTest.php index ddf21c64b..d887ba103 100644 --- a/tests/Composer/Test/Downloader/ArchiveDownloaderTest.php +++ b/tests/Composer/Test/Downloader/ArchiveDownloaderTest.php @@ -29,7 +29,7 @@ class ArchiveDownloaderTest extends TestCase $method->setAccessible(true); $first = $method->invoke($downloader, $packageMock, '/path'); - $this->assertRegExp('#/path/[a-z0-9]+\.js#', $first); + $this->assertRegExp('#/path_[a-z0-9]+\.js#', $first); $this->assertSame($first, $method->invoke($downloader, $packageMock, '/path')); } diff --git a/tests/Composer/Test/Downloader/DownloadManagerTest.php b/tests/Composer/Test/Downloader/DownloadManagerTest.php index 222e541d7..307b2294f 100644 --- a/tests/Composer/Test/Downloader/DownloadManagerTest.php +++ b/tests/Composer/Test/Downloader/DownloadManagerTest.php @@ -50,7 +50,7 @@ class DownloadManagerTest extends TestCase $this->setExpectedException('InvalidArgumentException'); - $manager->getDownloaderForInstalledPackage($package); + $manager->getDownloaderForPackage($package); } public function testGetDownloaderForCorrectlyInstalledDistPackage() @@ -82,7 +82,7 @@ class DownloadManagerTest extends TestCase ->with('pear') ->will($this->returnValue($downloader)); - $this->assertSame($downloader, $manager->getDownloaderForInstalledPackage($package)); + $this->assertSame($downloader, $manager->getDownloaderForPackage($package)); } public function testGetDownloaderForIncorrectlyInstalledDistPackage() @@ -116,7 +116,7 @@ class DownloadManagerTest extends TestCase $this->setExpectedException('LogicException'); - $manager->getDownloaderForInstalledPackage($package); + $manager->getDownloaderForPackage($package); } public function testGetDownloaderForCorrectlyInstalledSourcePackage() @@ -148,7 +148,7 @@ class DownloadManagerTest extends TestCase ->with('git') ->will($this->returnValue($downloader)); - $this->assertSame($downloader, $manager->getDownloaderForInstalledPackage($package)); + $this->assertSame($downloader, $manager->getDownloaderForPackage($package)); } public function testGetDownloaderForIncorrectlyInstalledSourcePackage() @@ -182,7 +182,7 @@ class DownloadManagerTest extends TestCase $this->setExpectedException('LogicException'); - $manager->getDownloaderForInstalledPackage($package); + $manager->getDownloaderForPackage($package); } public function testGetDownloaderForMetapackage() @@ -195,7 +195,7 @@ class DownloadManagerTest extends TestCase $manager = new DownloadManager($this->io, false, $this->filesystem); - $this->assertNull($manager->getDownloaderForInstalledPackage($package)); + $this->assertNull($manager->getDownloaderForPackage($package)); } public function testFullPackageDownload() @@ -223,11 +223,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); @@ -274,16 +274,16 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->at(0)) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloaderFail)); $manager ->expects($this->at(1)) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloaderSuccess)); @@ -333,11 +333,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); @@ -369,11 +369,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); @@ -399,11 +399,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue(null)); // There is no downloader for Metapackages. @@ -435,11 +435,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); @@ -472,11 +472,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); @@ -509,11 +509,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); @@ -550,33 +550,30 @@ class DownloadManagerTest extends TestCase $initial ->expects($this->once()) ->method('getDistType') - ->will($this->returnValue('pear')); + ->will($this->returnValue('zip')); $target = $this->createPackageMock(); $target ->expects($this->once()) - ->method('getDistType') - ->will($this->returnValue('pear')); + ->method('getInstallationSource') + ->will($this->returnValue('dist')); $target ->expects($this->once()) - ->method('setInstallationSource') - ->with('dist'); + ->method('getDistType') + ->will($this->returnValue('zip')); - $pearDownloader = $this->createDownloaderMock(); - $pearDownloader + $zipDownloader = $this->createDownloaderMock(); + $zipDownloader ->expects($this->once()) ->method('update') ->with($initial, $target, 'vendor/bundles/FOS/UserBundle'); + $zipDownloader + ->expects($this->any()) + ->method('getInstallationSource') + ->will($this->returnValue('dist')); - $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') - ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) - ->getMock(); - $manager - ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') - ->with($initial) - ->will($this->returnValue($pearDownloader)); + $manager = new DownloadManager($this->io, false, $this->filesystem); + $manager->setDownloader('zip', $zipDownloader); $manager->update($initial, $target, 'vendor/bundles/FOS/UserBundle'); } @@ -591,113 +588,89 @@ class DownloadManagerTest extends TestCase $initial ->expects($this->once()) ->method('getDistType') - ->will($this->returnValue('pear')); + ->will($this->returnValue('xz')); $target = $this->createPackageMock(); $target - ->expects($this->once()) + ->expects($this->any()) + ->method('getInstallationSource') + ->will($this->returnValue('dist')); + $target + ->expects($this->any()) ->method('getDistType') - ->will($this->returnValue('composer')); + ->will($this->returnValue('zip')); - $pearDownloader = $this->createDownloaderMock(); - $pearDownloader + $xzDownloader = $this->createDownloaderMock(); + $xzDownloader ->expects($this->once()) ->method('remove') ->with($initial, 'vendor/bundles/FOS/UserBundle'); + $xzDownloader + ->expects($this->any()) + ->method('getInstallationSource') + ->will($this->returnValue('dist')); - $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') - ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage', 'download')) - ->getMock(); - $manager + $zipDownloader = $this->createDownloaderMock(); + $zipDownloader ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') - ->with($initial) - ->will($this->returnValue($pearDownloader)); - $manager - ->expects($this->once()) - ->method('download') - ->with($target, 'vendor/bundles/FOS/UserBundle', false); + ->method('install') + ->with($target, 'vendor/bundles/FOS/UserBundle'); + $zipDownloader + ->expects($this->any()) + ->method('getInstallationSource') + ->will($this->returnValue('dist')); + + $manager = new DownloadManager($this->io, false, $this->filesystem); + $manager->setDownloader('xz', $xzDownloader); + $manager->setDownloader('zip', $zipDownloader); $manager->update($initial, $target, 'vendor/bundles/FOS/UserBundle'); } - public function testUpdateSourceWithEqualTypes() + /** + * @dataProvider updatesProvider + */ + public function testGetAvailableSourcesUpdateSticksToSameSource($prevPkgSource, $prevPkgIsDev, $targetAvailable, $targetIsDev, $expected) { - $initial = $this->createPackageMock(); - $initial - ->expects($this->once()) - ->method('getInstallationSource') - ->will($this->returnValue('source')); - $initial - ->expects($this->once()) - ->method('getSourceType') - ->will($this->returnValue('svn')); + $initial = null; + if ($prevPkgSource) { + $initial = $this->prophesize('Composer\Package\PackageInterface'); + $initial->getInstallationSource()->willReturn($prevPkgSource); + $initial->isDev()->willReturn($prevPkgIsDev); + } - $target = $this->createPackageMock(); - $target - ->expects($this->once()) - ->method('getSourceType') - ->will($this->returnValue('svn')); + $target = $this->prophesize('Composer\Package\PackageInterface'); + $target->getSourceType()->willReturn(in_array('source', $targetAvailable, true) ? 'git' : null); + $target->getDistType()->willReturn(in_array('dist', $targetAvailable, true) ? 'zip' : null); + $target->isDev()->willReturn($targetIsDev); - $svnDownloader = $this->createDownloaderMock(); - $svnDownloader - ->expects($this->once()) - ->method('update') - ->with($initial, $target, 'vendor/pkg'); - - $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') - ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage', 'download')) - ->getMock(); - $manager - ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') - ->with($initial) - ->will($this->returnValue($svnDownloader)); - - $manager->update($initial, $target, 'vendor/pkg'); + $manager = new DownloadManager($this->io, false, $this->filesystem); + $method = new \ReflectionMethod($manager, 'getAvailableSources'); + $method->setAccessible(true); + $this->assertEquals($expected, $method->invoke($manager, $target->reveal(), $initial ? $initial->reveal() : null)); } - public function testUpdateSourceWithNotEqualTypes() + public static function updatesProvider() { - $initial = $this->createPackageMock(); - $initial - ->expects($this->once()) - ->method('getInstallationSource') - ->will($this->returnValue('source')); - $initial - ->expects($this->once()) - ->method('getSourceType') - ->will($this->returnValue('svn')); - - $target = $this->createPackageMock(); - $target - ->expects($this->once()) - ->method('getSourceType') - ->will($this->returnValue('git')); - - $svnDownloader = $this->createDownloaderMock(); - $svnDownloader - ->expects($this->once()) - ->method('remove') - ->with($initial, 'vendor/pkg'); - - $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') - ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage', 'download')) - ->getMock(); - $manager - ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') - ->with($initial) - ->will($this->returnValue($svnDownloader)); - $manager - ->expects($this->once()) - ->method('download') - ->with($target, 'vendor/pkg', true); - - $manager->update($initial, $target, 'vendor/pkg'); + return array( + // prevPkg source, prevPkg isDev, pkg available, pkg isDev, expected + // updates keep previous source as preference + array('source', false, array('source', 'dist'), false, array('source', 'dist')), + array('dist', false, array('source', 'dist'), false, array('dist', 'source')), + // updates do not keep previous source if target package does not have it + array('source', false, array('dist'), false, array('dist')), + array('dist', false, array('source'), false, array('source')), + // updates do not keep previous source if target is dev and prev wasn't dev and installed from dist + array('source', false, array('source', 'dist'), true, array('source', 'dist')), + array('dist', false, array('source', 'dist'), true, array('source', 'dist')), + // install picks the right default + array(null, null, array('source', 'dist'), true, array('source', 'dist')), + array(null, null, array('dist'), true, array('dist')), + array(null, null, array('source'), true, array('source')), + array(null, null, array('source', 'dist'), false, array('dist', 'source')), + array(null, null, array('dist'), false, array('dist')), + array(null, null, array('source'), false, array('source')), + ); } public function testUpdateMetapackage() @@ -707,11 +680,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager - ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->expects($this->exactly(2)) + ->method('getDownloaderForPackage') ->with($initial) ->will($this->returnValue(null)); // There is no downloader for metapackages. @@ -730,11 +703,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($pearDownloader)); @@ -747,11 +720,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue(null)); // There is no downloader for metapackages. @@ -790,11 +763,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); @@ -833,11 +806,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); @@ -879,11 +852,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); $manager->setPreferences(array('foo/*' => 'source')); @@ -926,11 +899,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); $manager->setPreferences(array('foo/*' => 'source')); @@ -973,11 +946,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); $manager->setPreferences(array('foo/*' => 'auto')); @@ -1020,11 +993,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); $manager->setPreferences(array('foo/*' => 'auto')); @@ -1063,11 +1036,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); $manager->setPreferences(array('foo/*' => 'source')); @@ -1106,11 +1079,11 @@ class DownloadManagerTest extends TestCase $manager = $this->getMockBuilder('Composer\Downloader\DownloadManager') ->setConstructorArgs(array($this->io, false, $this->filesystem)) - ->setMethods(array('getDownloaderForInstalledPackage')) + ->setMethods(array('getDownloaderForPackage')) ->getMock(); $manager ->expects($this->once()) - ->method('getDownloaderForInstalledPackage') + ->method('getDownloaderForPackage') ->with($package) ->will($this->returnValue($downloader)); $manager->setPreferences(array('foo/*' => 'dist')); diff --git a/tests/Composer/Test/Downloader/FileDownloaderTest.php b/tests/Composer/Test/Downloader/FileDownloaderTest.php index 12edfe19d..9ce536474 100644 --- a/tests/Composer/Test/Downloader/FileDownloaderTest.php +++ b/tests/Composer/Test/Downloader/FileDownloaderTest.php @@ -15,6 +15,8 @@ namespace Composer\Test\Downloader; use Composer\Downloader\FileDownloader; use Composer\Test\TestCase; use Composer\Util\Filesystem; +use Composer\Util\Http\Response; +use Composer\Util\Loop; class FileDownloaderTest extends TestCase { @@ -23,6 +25,11 @@ class FileDownloaderTest extends TestCase $io = $io ?: $this->getMockBuilder('Composer\IO\IOInterface')->getMock(); $config = $config ?: $this->getMockBuilder('Composer\Config')->getMock(); $httpDownloader = $httpDownloader ?: $this->getMockBuilder('Composer\Util\HttpDownloader')->disableOriginalConstructor()->getMock(); + $httpDownloader + ->expects($this->any()) + ->method('addCopy') + ->will($this->returnValue(\React\Promise\resolve(new Response(array('url' => 'http://example.org/'), 200, array(), 'file~')))); + $this->httpDownloader = $httpDownloader; return new FileDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $filesystem); } @@ -84,7 +91,7 @@ class FileDownloaderTest extends TestCase $method = new \ReflectionMethod($downloader, 'getFileName'); $method->setAccessible(true); - $this->assertEquals('/path/script.js', $method->invoke($downloader, $packageMock, '/path')); + $this->assertEquals('/path_script.js', $method->invoke($downloader, $packageMock, '/path')); } public function testDownloadButFileIsUnsaved() @@ -118,8 +125,11 @@ class FileDownloaderTest extends TestCase $downloader = $this->getDownloader($ioMock); try { - $downloader->download($packageMock, $path); - $this->fail(); + $promise = $downloader->download($packageMock, $path); + $loop = new Loop($this->httpDownloader); + $loop->wait(array($promise)); + + $this->fail('Download was expected to throw'); } catch (\Exception $e) { if (is_dir($path)) { $fs = new Filesystem(); @@ -128,7 +138,7 @@ class FileDownloaderTest extends TestCase unlink($path); } - $this->assertInstanceOf('UnexpectedValueException', $e); + $this->assertInstanceOf('UnexpectedValueException', $e, $e->getMessage()); $this->assertContains('could not be saved to', $e->getMessage()); } } @@ -188,11 +198,14 @@ class FileDownloaderTest extends TestCase $path = $this->getUniqueTmpDirectory(); $downloader = $this->getDownloader(null, null, null, null, null, $filesystem); // make sure the file expected to be downloaded is on disk already - touch($path.'/script.js'); + touch($path.'_script.js'); try { - $downloader->download($packageMock, $path); - $this->fail(); + $promise = $downloader->download($packageMock, $path); + $loop = new Loop($this->httpDownloader); + $loop->wait(array($promise)); + + $this->fail('Download was expected to throw'); } catch (\Exception $e) { if (is_dir($path)) { $fs = new Filesystem(); @@ -201,7 +214,7 @@ class FileDownloaderTest extends TestCase unlink($path); } - $this->assertInstanceOf('UnexpectedValueException', $e); + $this->assertInstanceOf('UnexpectedValueException', $e, $e->getMessage()); $this->assertContains('checksum verification', $e->getMessage()); } } @@ -232,17 +245,25 @@ class FileDownloaderTest extends TestCase $ioMock = $this->getMock('Composer\IO\IOInterface'); $ioMock->expects($this->at(0)) + ->method('writeError') + ->with($this->stringContains('Downloading')); + + $ioMock->expects($this->at(1)) ->method('writeError') ->with($this->stringContains('Downgrading')); $path = $this->getUniqueTmpDirectory(); - touch($path.'/script.js'); + touch($path.'_script.js'); $filesystem = $this->getMock('Composer\Util\Filesystem'); $filesystem->expects($this->once()) ->method('removeDirectory') ->will($this->returnValue(true)); $downloader = $this->getDownloader($ioMock, null, null, null, null, $filesystem); + $promise = $downloader->download($newPackage, $path, $oldPackage); + $loop = new Loop($this->httpDownloader); + $loop->wait(array($promise)); + $downloader->update($oldPackage, $newPackage, $path); } } diff --git a/tests/Composer/Test/Downloader/FossilDownloaderTest.php b/tests/Composer/Test/Downloader/FossilDownloaderTest.php index 623f7dec2..9ab7b6b84 100644 --- a/tests/Composer/Test/Downloader/FossilDownloaderTest.php +++ b/tests/Composer/Test/Downloader/FossilDownloaderTest.php @@ -56,7 +56,7 @@ class FossilDownloaderTest extends TestCase ->will($this->returnValue(null)); $downloader = $this->getDownloaderMock(); - $downloader->download($packageMock, '/path'); + $downloader->install($packageMock, '/path'); } public function testDownload() @@ -89,7 +89,7 @@ class FossilDownloaderTest extends TestCase ->will($this->returnValue(0)); $downloader = $this->getDownloaderMock(null, null, $processExecutor); - $downloader->download($packageMock, 'repo'); + $downloader->install($packageMock, 'repo'); } /** diff --git a/tests/Composer/Test/Downloader/GitDownloaderTest.php b/tests/Composer/Test/Downloader/GitDownloaderTest.php index c3cd31a4a..b5d0054de 100644 --- a/tests/Composer/Test/Downloader/GitDownloaderTest.php +++ b/tests/Composer/Test/Downloader/GitDownloaderTest.php @@ -79,7 +79,7 @@ class GitDownloaderTest extends TestCase ->will($this->returnValue(null)); $downloader = $this->getDownloaderMock(); - $downloader->download($packageMock, '/path'); + $downloader->install($packageMock, '/path'); } public function testDownload() @@ -130,7 +130,7 @@ class GitDownloaderTest extends TestCase ->will($this->returnValue(0)); $downloader = $this->getDownloaderMock(null, null, $processExecutor); - $downloader->download($packageMock, 'composerPath'); + $downloader->install($packageMock, 'composerPath'); } public function testDownloadWithCache() @@ -195,7 +195,7 @@ class GitDownloaderTest extends TestCase ->will($this->returnValue(0)); $downloader = $this->getDownloaderMock(null, $config, $processExecutor); - $downloader->download($packageMock, 'composerPath'); + $downloader->install($packageMock, 'composerPath'); @rmdir($cachePath); } @@ -265,7 +265,7 @@ class GitDownloaderTest extends TestCase ->will($this->returnValue(0)); $downloader = $this->getDownloaderMock(null, new Config(), $processExecutor); - $downloader->download($packageMock, 'composerPath'); + $downloader->install($packageMock, 'composerPath'); } public function pushUrlProvider() @@ -329,7 +329,7 @@ class GitDownloaderTest extends TestCase $config->merge(array('config' => array('github-protocols' => $protocols))); $downloader = $this->getDownloaderMock(null, $config, $processExecutor); - $downloader->download($packageMock, 'composerPath'); + $downloader->install($packageMock, 'composerPath'); } /** @@ -360,7 +360,7 @@ class GitDownloaderTest extends TestCase ->will($this->returnValue(1)); $downloader = $this->getDownloaderMock(null, null, $processExecutor); - $downloader->download($packageMock, 'composerPath'); + $downloader->install($packageMock, 'composerPath'); } /** diff --git a/tests/Composer/Test/Downloader/HgDownloaderTest.php b/tests/Composer/Test/Downloader/HgDownloaderTest.php index c71d463cb..a4219d143 100644 --- a/tests/Composer/Test/Downloader/HgDownloaderTest.php +++ b/tests/Composer/Test/Downloader/HgDownloaderTest.php @@ -56,7 +56,7 @@ class HgDownloaderTest extends TestCase ->will($this->returnValue(null)); $downloader = $this->getDownloaderMock(); - $downloader->download($packageMock, '/path'); + $downloader->install($packageMock, '/path'); } public function testDownload() @@ -83,7 +83,7 @@ class HgDownloaderTest extends TestCase ->will($this->returnValue(0)); $downloader = $this->getDownloaderMock(null, null, $processExecutor); - $downloader->download($packageMock, 'composerPath'); + $downloader->install($packageMock, 'composerPath'); } /** diff --git a/tests/Composer/Test/Downloader/PerforceDownloaderTest.php b/tests/Composer/Test/Downloader/PerforceDownloaderTest.php index 1b5041d9f..d2b8fb753 100644 --- a/tests/Composer/Test/Downloader/PerforceDownloaderTest.php +++ b/tests/Composer/Test/Downloader/PerforceDownloaderTest.php @@ -138,7 +138,7 @@ class PerforceDownloaderTest extends TestCase $perforce->expects($this->at(5))->method('syncCodeBase')->with($label); $perforce->expects($this->at(6))->method('cleanupClientSpec'); $this->downloader->setPerforce($perforce); - $this->downloader->doDownload($this->package, $this->testPath, 'url'); + $this->downloader->doInstall($this->package, $this->testPath, 'url'); } /** @@ -161,6 +161,6 @@ class PerforceDownloaderTest extends TestCase $perforce->expects($this->at(5))->method('syncCodeBase')->with($label); $perforce->expects($this->at(6))->method('cleanupClientSpec'); $this->downloader->setPerforce($perforce); - $this->downloader->doDownload($this->package, $this->testPath, 'url'); + $this->downloader->doInstall($this->package, $this->testPath, 'url'); } } diff --git a/tests/Composer/Test/Downloader/XzDownloaderTest.php b/tests/Composer/Test/Downloader/XzDownloaderTest.php index 451592d37..4c2fdb2af 100644 --- a/tests/Composer/Test/Downloader/XzDownloaderTest.php +++ b/tests/Composer/Test/Downloader/XzDownloaderTest.php @@ -16,6 +16,7 @@ use Composer\Downloader\XzDownloader; use Composer\Test\TestCase; use Composer\Util\Filesystem; use Composer\Util\Platform; +use Composer\Util\Loop; use Composer\Util\HttpDownloader; class XzDownloaderTest extends TestCase @@ -66,10 +67,14 @@ class XzDownloaderTest extends TestCase ->method('get') ->with('vendor-dir') ->will($this->returnValue($this->testDir)); - $downloader = new XzDownloader($io, $config, new HttpDownloader($io, $this->getMockBuilder('Composer\Config')->getMock()), null, null, null); + $downloader = new XzDownloader($io, $config, $httpDownloader = new HttpDownloader($io, $this->getMockBuilder('Composer\Config')->getMock()), null, null, null); try { - $downloader->download($packageMock, $this->getUniqueTmpDirectory()); + $promise = $downloader->download($packageMock, $this->testDir); + $loop = new Loop($httpDownloader); + $loop->wait(array($promise)); + $downloader->install($packageMock, $this->testDir); + $this->fail('Download of invalid tarball should throw an exception'); } catch (\RuntimeException $e) { $this->assertRegexp('/(File format not recognized|Unrecognized archive format)/i', $e->getMessage()); diff --git a/tests/Composer/Test/Downloader/ZipDownloaderTest.php b/tests/Composer/Test/Downloader/ZipDownloaderTest.php index 0c1311427..b754af607 100644 --- a/tests/Composer/Test/Downloader/ZipDownloaderTest.php +++ b/tests/Composer/Test/Downloader/ZipDownloaderTest.php @@ -17,6 +17,7 @@ use Composer\Package\PackageInterface; use Composer\Test\TestCase; use Composer\Util\Filesystem; use Composer\Util\HttpDownloader; +use Composer\Util\Loop; class ZipDownloaderTest extends TestCase { @@ -27,6 +28,7 @@ class ZipDownloaderTest extends TestCase private $prophet; private $io; private $config; + private $package; public function setUp() { @@ -35,6 +37,7 @@ class ZipDownloaderTest extends TestCase $this->config = $this->getMockBuilder('Composer\Config')->getMock(); $dlConfig = $this->getMockBuilder('Composer\Config')->getMock(); $this->httpDownloader = new HttpDownloader($this->io, $dlConfig); + $this->package = $this->getMockBuilder('Composer\Package\PackageInterface')->getMock(); } public function tearDown() @@ -71,16 +74,15 @@ class ZipDownloaderTest extends TestCase ->with('vendor-dir') ->will($this->returnValue($this->testDir)); - $packageMock = $this->getMockBuilder('Composer\Package\PackageInterface')->getMock(); - $packageMock->expects($this->any()) + $this->package->expects($this->any()) ->method('getDistUrl') ->will($this->returnValue($distUrl = 'file://'.__FILE__)) ; - $packageMock->expects($this->any()) + $this->package->expects($this->any()) ->method('getDistUrls') ->will($this->returnValue(array($distUrl))) ; - $packageMock->expects($this->atLeastOnce()) + $this->package->expects($this->atLeastOnce()) ->method('getTransportOptions') ->will($this->returnValue(array())) ; @@ -90,7 +92,11 @@ class ZipDownloaderTest extends TestCase $this->setPrivateProperty('hasSystemUnzip', false); try { - $downloader->download($packageMock, sys_get_temp_dir().'/composer-zip-test'); + $promise = $downloader->download($this->package, $path = sys_get_temp_dir().'/composer-zip-test'); + $loop = new Loop($this->httpDownloader); + $loop->wait(array($promise)); + $downloader->install($this->package, $path); + $this->fail('Download of invalid zip files should throw an exception'); } catch (\Exception $e) { $this->assertContains('is not a zip archive', $e->getMessage()); @@ -119,7 +125,7 @@ class ZipDownloaderTest extends TestCase ->will($this->returnValue(false)); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); - $downloader->extract('testfile.zip', 'vendor/dir'); + $downloader->extract($this->package, 'testfile.zip', 'vendor/dir'); } /** @@ -144,7 +150,7 @@ class ZipDownloaderTest extends TestCase ->will($this->throwException(new \ErrorException('Not a directory'))); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); - $downloader->extract('testfile.zip', 'vendor/dir'); + $downloader->extract($this->package, 'testfile.zip', 'vendor/dir'); } /** @@ -168,7 +174,7 @@ class ZipDownloaderTest extends TestCase ->will($this->returnValue(true)); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); - $downloader->extract('testfile.zip', 'vendor/dir'); + $downloader->extract($this->package, 'testfile.zip', 'vendor/dir'); } /** @@ -189,7 +195,7 @@ class ZipDownloaderTest extends TestCase ->will($this->returnValue(1)); $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); - $downloader->extract('testfile.zip', 'vendor/dir'); + $downloader->extract($this->package, 'testfile.zip', 'vendor/dir'); } public function testSystemUnzipOnlyGood() @@ -206,7 +212,7 @@ class ZipDownloaderTest extends TestCase ->will($this->returnValue(0)); $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); - $downloader->extract('testfile.zip', 'vendor/dir'); + $downloader->extract($this->package, 'testfile.zip', 'vendor/dir'); } public function testNonWindowsFallbackGood() @@ -234,7 +240,7 @@ class ZipDownloaderTest extends TestCase $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); - $downloader->extract('testfile.zip', 'vendor/dir'); + $downloader->extract($this->package, 'testfile.zip', 'vendor/dir'); } /** @@ -266,7 +272,7 @@ class ZipDownloaderTest extends TestCase $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); - $downloader->extract('testfile.zip', 'vendor/dir'); + $downloader->extract($this->package, 'testfile.zip', 'vendor/dir'); } public function testWindowsFallbackGood() @@ -294,7 +300,7 @@ class ZipDownloaderTest extends TestCase $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); - $downloader->extract('testfile.zip', 'vendor/dir'); + $downloader->extract($this->package, 'testfile.zip', 'vendor/dir'); } /** @@ -326,7 +332,7 @@ class ZipDownloaderTest extends TestCase $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, $processExecutor); $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader); - $downloader->extract('testfile.zip', 'vendor/dir'); + $downloader->extract($this->package, 'testfile.zip', 'vendor/dir'); } } @@ -337,8 +343,13 @@ class MockedZipDownloader extends ZipDownloader return; } - public function extract($file, $path) + public function install(PackageInterface $package, $path, $output = true) { - parent::extract($file, $path); + return; + } + + public function extract(PackageInterface $package, $file, $path) + { + parent::extract($package, $file, $path); } } diff --git a/tests/Composer/Test/EventDispatcher/EventDispatcherTest.php b/tests/Composer/Test/EventDispatcher/EventDispatcherTest.php index 7786e7807..6d812e20a 100644 --- a/tests/Composer/Test/EventDispatcher/EventDispatcherTest.php +++ b/tests/Composer/Test/EventDispatcher/EventDispatcherTest.php @@ -101,7 +101,7 @@ class EventDispatcherTest extends TestCase $composer->setPackage($package); $composer->setRepositoryManager($this->getRepositoryManagerMockForDevModePassingTest()); - $composer->setInstallationManager($this->getMockBuilder('Composer\Installer\InstallationManager')->getMock()); + $composer->setInstallationManager($this->getMockBuilder('Composer\Installer\InstallationManager')->disableOriginalConstructor()->getMock()); $dispatcher = new EventDispatcher( $composer, diff --git a/tests/Composer/Test/Installer/InstallationManagerTest.php b/tests/Composer/Test/Installer/InstallationManagerTest.php index 86e860bc2..407a5b10f 100644 --- a/tests/Composer/Test/Installer/InstallationManagerTest.php +++ b/tests/Composer/Test/Installer/InstallationManagerTest.php @@ -13,6 +13,7 @@ namespace Composer\Test\Installer; use Composer\Installer\InstallationManager; +use Composer\Installer\NoopInstaller; use Composer\DependencyResolver\Operation\InstallOperation; use Composer\DependencyResolver\Operation\UpdateOperation; use Composer\DependencyResolver\Operation\UninstallOperation; @@ -21,9 +22,11 @@ use PHPUnit\Framework\TestCase; class InstallationManagerTest extends TestCase { protected $repository; + protected $loop; public function setUp() { + $this->loop = $this->getMockBuilder('Composer\Util\Loop')->disableOriginalConstructor()->getMock(); $this->repository = $this->getMockBuilder('Composer\Repository\InstalledRepositoryInterface')->getMock(); } @@ -38,7 +41,7 @@ class InstallationManagerTest extends TestCase return $arg === 'vendor'; })); - $manager = new InstallationManager(); + $manager = new InstallationManager($this->loop); $manager->addInstaller($installer); $this->assertSame($installer, $manager->getInstaller('vendor')); @@ -67,7 +70,7 @@ class InstallationManagerTest extends TestCase return $arg === 'vendor'; })); - $manager = new InstallationManager(); + $manager = new InstallationManager($this->loop); $manager->addInstaller($installer); $this->assertSame($installer, $manager->getInstaller('vendor')); @@ -80,16 +83,21 @@ class InstallationManagerTest extends TestCase public function testExecute() { $manager = $this->getMockBuilder('Composer\Installer\InstallationManager') + ->setConstructorArgs(array($this->loop)) ->setMethods(array('install', 'update', 'uninstall')) ->getMock(); - $installOperation = new InstallOperation($this->createPackageMock()); - $removeOperation = new UninstallOperation($this->createPackageMock()); + $installOperation = new InstallOperation($package = $this->createPackageMock()); + $removeOperation = new UninstallOperation($package); $updateOperation = new UpdateOperation( - $this->createPackageMock(), - $this->createPackageMock() + $package, + $package ); + $package->expects($this->any()) + ->method('getType') + ->will($this->returnValue('library')); + $manager ->expects($this->once()) ->method('install') @@ -103,6 +111,7 @@ class InstallationManagerTest extends TestCase ->method('update') ->with($this->repository, $updateOperation); + $manager->addInstaller(new NoopInstaller()); $manager->execute($this->repository, $installOperation); $manager->execute($this->repository, $removeOperation); $manager->execute($this->repository, $updateOperation); @@ -111,7 +120,7 @@ class InstallationManagerTest extends TestCase public function testInstall() { $installer = $this->createInstallerMock(); - $manager = new InstallationManager(); + $manager = new InstallationManager($this->loop); $manager->addInstaller($installer); $package = $this->createPackageMock(); @@ -139,7 +148,7 @@ class InstallationManagerTest extends TestCase public function testUpdateWithEqualTypes() { $installer = $this->createInstallerMock(); - $manager = new InstallationManager(); + $manager = new InstallationManager($this->loop); $manager->addInstaller($installer); $initial = $this->createPackageMock(); @@ -173,18 +182,17 @@ class InstallationManagerTest extends TestCase { $libInstaller = $this->createInstallerMock(); $bundleInstaller = $this->createInstallerMock(); - $manager = new InstallationManager(); + $manager = new InstallationManager($this->loop); $manager->addInstaller($libInstaller); $manager->addInstaller($bundleInstaller); $initial = $this->createPackageMock(); - $target = $this->createPackageMock(); - $operation = new UpdateOperation($initial, $target, 'test'); - $initial ->expects($this->once()) ->method('getType') ->will($this->returnValue('library')); + + $target = $this->createPackageMock(); $target ->expects($this->once()) ->method('getType') @@ -213,13 +221,14 @@ class InstallationManagerTest extends TestCase ->method('install') ->with($this->repository, $target); + $operation = new UpdateOperation($initial, $target, 'test'); $manager->update($this->repository, $operation); } public function testUninstall() { $installer = $this->createInstallerMock(); - $manager = new InstallationManager(); + $manager = new InstallationManager($this->loop); $manager->addInstaller($installer); $package = $this->createPackageMock(); @@ -249,7 +258,7 @@ class InstallationManagerTest extends TestCase $installer = $this->getMockBuilder('Composer\Installer\LibraryInstaller') ->disableOriginalConstructor() ->getMock(); - $manager = new InstallationManager(); + $manager = new InstallationManager($this->loop); $manager->addInstaller($installer); $package = $this->createPackageMock(); @@ -281,7 +290,9 @@ class InstallationManagerTest extends TestCase private function createPackageMock() { - return $this->getMockBuilder('Composer\Package\PackageInterface') + $mock = $this->getMockBuilder('Composer\Package\PackageInterface') ->getMock(); + + return $mock; } } diff --git a/tests/Composer/Test/Installer/LibraryInstallerTest.php b/tests/Composer/Test/Installer/LibraryInstallerTest.php index 772bb05c8..672f8eb0a 100644 --- a/tests/Composer/Test/Installer/LibraryInstallerTest.php +++ b/tests/Composer/Test/Installer/LibraryInstallerTest.php @@ -113,7 +113,7 @@ class LibraryInstallerTest extends TestCase $this->dm ->expects($this->once()) - ->method('download') + ->method('install') ->with($package, $this->vendorDir.'/some/package'); $this->repository diff --git a/tests/Composer/Test/Mock/FactoryMock.php b/tests/Composer/Test/Mock/FactoryMock.php index 47683afcd..fcb93d2cc 100644 --- a/tests/Composer/Test/Mock/FactoryMock.php +++ b/tests/Composer/Test/Mock/FactoryMock.php @@ -20,6 +20,7 @@ use Composer\Repository\WritableRepositoryInterface; use Composer\Installer; use Composer\IO\IOInterface; use Composer\Test\TestCase; +use Composer\Util\Loop; class FactoryMock extends Factory { @@ -39,9 +40,9 @@ class FactoryMock extends Factory { } - protected function createInstallationManager() + public function createInstallationManager(Loop $loop) { - return new InstallationManagerMock; + return new InstallationManagerMock(); } protected function createDefaultInstallers(Installer\InstallationManager $im, Composer $composer, IOInterface $io) diff --git a/tests/Composer/Test/Mock/InstallationManagerMock.php b/tests/Composer/Test/Mock/InstallationManagerMock.php index de1de514b..21e717224 100644 --- a/tests/Composer/Test/Mock/InstallationManagerMock.php +++ b/tests/Composer/Test/Mock/InstallationManagerMock.php @@ -17,6 +17,7 @@ use Composer\Repository\RepositoryInterface; use Composer\Repository\InstalledRepositoryInterface; use Composer\Package\PackageInterface; use Composer\DependencyResolver\Operation\InstallOperation; +use Composer\DependencyResolver\Operation\OperationInterface; use Composer\DependencyResolver\Operation\UpdateOperation; use Composer\DependencyResolver\Operation\UninstallOperation; use Composer\DependencyResolver\Operation\MarkAliasInstalledOperation; @@ -29,6 +30,18 @@ class InstallationManagerMock extends InstallationManager private $uninstalled = array(); private $trace = array(); + public function __construct() + { + + } + + public function execute(RepositoryInterface $repo, OperationInterface $operation) + { + $method = $operation->getJobType(); + // skipping download() step here for tests + $this->$method($repo, $operation); + } + public function getInstallPath(PackageInterface $package) { return ''; diff --git a/tests/Composer/Test/Package/Archiver/ArchiveManagerTest.php b/tests/Composer/Test/Package/Archiver/ArchiveManagerTest.php index b9f08e693..714c9b923 100644 --- a/tests/Composer/Test/Package/Archiver/ArchiveManagerTest.php +++ b/tests/Composer/Test/Package/Archiver/ArchiveManagerTest.php @@ -16,6 +16,7 @@ use Composer\IO\NullIO; use Composer\Factory; use Composer\Package\Archiver\ArchiveManager; use Composer\Package\PackageInterface; +use Composer\Util\Loop; use Composer\Test\Mock\FactoryMock; class ArchiveManagerTest extends ArchiverTest @@ -35,9 +36,10 @@ class ArchiveManagerTest extends ArchiverTest $dm = $factory->createDownloadManager( $io = new NullIO, $config = FactoryMock::createConfig(), - $factory->createHttpDownloader($io, $config) + $httpDownloader = $factory->createHttpDownloader($io, $config) ); - $this->manager = $factory->createArchiveManager($factory->createConfig(), $dm); + $loop = new Loop($httpDownloader); + $this->manager = $factory->createArchiveManager($factory->createConfig(), $dm, $loop); $this->targetDir = $this->testDir.'/composer_archiver_tests'; } diff --git a/tests/Composer/Test/Plugin/PluginInstallerTest.php b/tests/Composer/Test/Plugin/PluginInstallerTest.php index 01832f94d..633c5ab18 100644 --- a/tests/Composer/Test/Plugin/PluginInstallerTest.php +++ b/tests/Composer/Test/Plugin/PluginInstallerTest.php @@ -89,7 +89,7 @@ class PluginInstallerTest extends TestCase ->method('getLocalRepository') ->will($this->returnValue($this->repository)); - $im = $this->getMockBuilder('Composer\Installer\InstallationManager')->getMock(); + $im = $this->getMockBuilder('Composer\Installer\InstallationManager')->disableOriginalConstructor()->getMock(); $im->expects($this->any()) ->method('getInstallPath') ->will($this->returnCallback(function ($package) { From ea978ee7f8e4897667c4b490c839145582a33cd1 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 18 Jan 2019 08:46:46 +0100 Subject: [PATCH 30/34] Update react/promise requirement --- composer.json | 2 +- composer.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 1b75131bc..03839772e 100644 --- a/composer.json +++ b/composer.json @@ -35,7 +35,7 @@ "symfony/filesystem": "^2.7 || ^3.0 || ^4.0", "symfony/finder": "^2.7 || ^3.0 || ^4.0", "symfony/process": "^2.7 || ^3.0 || ^4.0", - "react/promise": "^1.2" + "react/promise": "^1.2 || ^2.7" }, "conflict": { "symfony/console": "2.8.38" diff --git a/composer.lock b/composer.lock index a22e11a6b..d2a448608 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3243ce6f26231df34d1bceab1a148803", + "content-hash": "b078b12b2912d599e0c6904f64def484", "packages": [ { "name": "composer/ca-bundle", From 549ccd8f794c0c1c51b957e864cc914482e82a85 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 18 Jan 2019 11:48:51 +0100 Subject: [PATCH 31/34] Remote outputProgress concept from downloaders as it does not make sense when things happen in parallel, refs #7901 --- src/Composer/Command/CreateProjectCommand.php | 4 +--- src/Composer/Command/InstallCommand.php | 1 - src/Composer/Command/RemoveCommand.php | 1 - src/Composer/Command/RequireCommand.php | 1 - src/Composer/Command/UpdateCommand.php | 2 -- src/Composer/Downloader/DownloadManager.php | 16 ---------------- src/Composer/Downloader/DownloaderInterface.php | 8 -------- src/Composer/Downloader/FileDownloader.php | 14 -------------- src/Composer/Downloader/VcsDownloader.php | 9 --------- 9 files changed, 1 insertion(+), 55 deletions(-) diff --git a/src/Composer/Command/CreateProjectCommand.php b/src/Composer/Command/CreateProjectCommand.php index e165649a6..c11a0595e 100644 --- a/src/Composer/Command/CreateProjectCommand.php +++ b/src/Composer/Command/CreateProjectCommand.php @@ -162,7 +162,6 @@ EOT } $composer = Factory::create($io, null, $disablePlugins); - $composer->getDownloadManager()->setOutputProgress(!$noProgress); $fs = new Filesystem(); @@ -351,8 +350,7 @@ EOT $httpDownloader = $factory->createHttpDownloader($io, $config); $dm = $factory->createDownloadManager($io, $config, $httpDownloader); $dm->setPreferSource($preferSource) - ->setPreferDist($preferDist) - ->setOutputProgress(!$noProgress); + ->setPreferDist($preferDist); $projectInstaller = new ProjectInstaller($directory, $dm); $im = $factory->createInstallationManager(new Loop($httpDownloader)); diff --git a/src/Composer/Command/InstallCommand.php b/src/Composer/Command/InstallCommand.php index cc590d8c9..951d20289 100644 --- a/src/Composer/Command/InstallCommand.php +++ b/src/Composer/Command/InstallCommand.php @@ -85,7 +85,6 @@ EOT } $composer = $this->getComposer(true, $input->getOption('no-plugins')); - $composer->getDownloadManager()->setOutputProgress(!$input->getOption('no-progress')); $commandEvent = new CommandEvent(PluginEvents::COMMAND, 'install', $input, $output); $composer->getEventDispatcher()->dispatch($commandEvent->getName(), $commandEvent); diff --git a/src/Composer/Command/RemoveCommand.php b/src/Composer/Command/RemoveCommand.php index 27be1a0ca..ea412ec66 100644 --- a/src/Composer/Command/RemoveCommand.php +++ b/src/Composer/Command/RemoveCommand.php @@ -126,7 +126,6 @@ EOT // Update packages $this->resetComposer(); $composer = $this->getComposer(true, $input->getOption('no-plugins')); - $composer->getDownloadManager()->setOutputProgress(!$input->getOption('no-progress')); $commandEvent = new CommandEvent(PluginEvents::COMMAND, 'remove', $input, $output); $composer->getEventDispatcher()->dispatch($commandEvent->getName(), $commandEvent); diff --git a/src/Composer/Command/RequireCommand.php b/src/Composer/Command/RequireCommand.php index 1f29751b9..f15308bad 100644 --- a/src/Composer/Command/RequireCommand.php +++ b/src/Composer/Command/RequireCommand.php @@ -167,7 +167,6 @@ EOT // Update packages $this->resetComposer(); $composer = $this->getComposer(true, $input->getOption('no-plugins')); - $composer->getDownloadManager()->setOutputProgress(!$input->getOption('no-progress')); $commandEvent = new CommandEvent(PluginEvents::COMMAND, 'require', $input, $output); $composer->getEventDispatcher()->dispatch($commandEvent->getName(), $commandEvent); diff --git a/src/Composer/Command/UpdateCommand.php b/src/Composer/Command/UpdateCommand.php index 34420b747..06f998d63 100644 --- a/src/Composer/Command/UpdateCommand.php +++ b/src/Composer/Command/UpdateCommand.php @@ -120,8 +120,6 @@ EOT } } - $composer->getDownloadManager()->setOutputProgress(!$input->getOption('no-progress')); - $commandEvent = new CommandEvent(PluginEvents::COMMAND, 'update', $input, $output); $composer->getEventDispatcher()->dispatch($commandEvent->getName(), $commandEvent); diff --git a/src/Composer/Downloader/DownloadManager.php b/src/Composer/Downloader/DownloadManager.php index 4bc865827..0b1ddb5a6 100644 --- a/src/Composer/Downloader/DownloadManager.php +++ b/src/Composer/Downloader/DownloadManager.php @@ -85,22 +85,6 @@ class DownloadManager return $this; } - /** - * Sets whether to output download progress information for all registered - * downloaders - * - * @param bool $outputProgress - * @return DownloadManager - */ - public function setOutputProgress($outputProgress) - { - foreach ($this->downloaders as $downloader) { - $downloader->setOutputProgress($outputProgress); - } - - return $this; - } - /** * Sets installer downloader for a specific installation type. * diff --git a/src/Composer/Downloader/DownloaderInterface.php b/src/Composer/Downloader/DownloaderInterface.php index ac56583c4..2074b16da 100644 --- a/src/Composer/Downloader/DownloaderInterface.php +++ b/src/Composer/Downloader/DownloaderInterface.php @@ -61,12 +61,4 @@ interface DownloaderInterface * @param string $path download path */ public function remove(PackageInterface $package, $path); - - /** - * Sets whether to output download progress information or not - * - * @param bool $outputProgress - * @return DownloaderInterface - */ - public function setOutputProgress($outputProgress); } diff --git a/src/Composer/Downloader/FileDownloader.php b/src/Composer/Downloader/FileDownloader.php index 3418eef84..54acae710 100644 --- a/src/Composer/Downloader/FileDownloader.php +++ b/src/Composer/Downloader/FileDownloader.php @@ -43,7 +43,6 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface protected $httpDownloader; protected $filesystem; protected $cache; - protected $outputProgress = true; /** * @private this is only public for php 5.3 support in closures */ @@ -236,16 +235,6 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface $this->filesystem->rename($this->getFileName($package, $path), $path . pathinfo(parse_url($package->getDistUrl(), PHP_URL_PATH), PATHINFO_BASENAME)); } - /** - * {@inheritDoc} - */ - public function setOutputProgress($outputProgress) - { - $this->outputProgress = $outputProgress; - - return $this; - } - /** * TODO mark private in v3 * @protected This is public due to PHP 5.3 @@ -340,11 +329,9 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface public function getLocalChanges(PackageInterface $package, $targetDir) { $prevIO = $this->io; - $prevProgress = $this->outputProgress; $this->io = new NullIO; $this->io->loadConfiguration($this->config); - $this->outputProgress = false; $e = null; try { @@ -362,7 +349,6 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface } $this->io = $prevIO; - $this->outputProgress = $prevProgress; if ($e) { throw $e; diff --git a/src/Composer/Downloader/VcsDownloader.php b/src/Composer/Downloader/VcsDownloader.php index 237d7e49d..b87f6433a 100644 --- a/src/Composer/Downloader/VcsDownloader.php +++ b/src/Composer/Downloader/VcsDownloader.php @@ -210,15 +210,6 @@ abstract class VcsDownloader implements DownloaderInterface, ChangeReportInterfa } } - /** - * Download progress information is not available for all VCS downloaders. - * {@inheritDoc} - */ - public function setOutputProgress($outputProgress) - { - return $this; - } - /** * {@inheritDoc} */ From bb2f64c7bc37bbc01de6eb4efa49719c5ff80a77 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 18 Jan 2019 12:14:37 +0100 Subject: [PATCH 32/34] Remove ability to override the entire HttpDownloader instance in PRE_FILE_DOWNLOAD events --- src/Composer/Downloader/FileDownloader.php | 6 ++---- src/Composer/Plugin/PreFileDownloadEvent.php | 18 +++++------------- src/Composer/Repository/ComposerRepository.php | 13 +++---------- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/src/Composer/Downloader/FileDownloader.php b/src/Composer/Downloader/FileDownloader.php index 54acae710..4f64a9501 100644 --- a/src/Composer/Downloader/FileDownloader.php +++ b/src/Composer/Downloader/FileDownloader.php @@ -106,21 +106,19 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface $io = $this->io; $cache = $this->cache; - $originalHttpDownloader = $this->httpDownloader; + $httpDownloader = $this->httpDownloader; $eventDispatcher = $this->eventDispatcher; $filesystem = $this->filesystem; $self = $this; $accept = null; $reject = null; - $download = function () use ($io, $output, $originalHttpDownloader, $cache, $eventDispatcher, $package, $fileName, $path, &$urls, &$accept, &$reject) { + $download = function () use ($io, $output, $httpDownloader, $cache, $eventDispatcher, $package, $fileName, $path, &$urls, &$accept, &$reject) { $url = reset($urls); - $httpDownloader = $originalHttpDownloader; if ($eventDispatcher) { $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $httpDownloader, $url['processed']); $eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); } $checksum = $package->getDistSha1Checksum(); diff --git a/src/Composer/Plugin/PreFileDownloadEvent.php b/src/Composer/Plugin/PreFileDownloadEvent.php index 076449484..c2751da02 100644 --- a/src/Composer/Plugin/PreFileDownloadEvent.php +++ b/src/Composer/Plugin/PreFileDownloadEvent.php @@ -25,7 +25,7 @@ class PreFileDownloadEvent extends Event /** * @var HttpDownloader */ - private $rfs; + private $httpDownloader; /** * @var string @@ -36,13 +36,13 @@ class PreFileDownloadEvent extends Event * Constructor. * * @param string $name The event name - * @param HttpDownloader $rfs + * @param HttpDownloader $httpDownloader * @param string $processedUrl */ - public function __construct($name, HttpDownloader $rfs, $processedUrl) + public function __construct($name, HttpDownloader $httpDownloader, $processedUrl) { parent::__construct($name); - $this->rfs = $rfs; + $this->httpDownloader = $httpDownloader; $this->processedUrl = $processedUrl; } @@ -51,15 +51,7 @@ class PreFileDownloadEvent extends Event */ public function getHttpDownloader() { - return $this->rfs; - } - - /** - * @param HttpDownloader $rfs - */ - public function setHttpDownloader(HttpDownloader $rfs) - { - $this->rfs = $rfs; + return $this->httpDownloader; } /** diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index af627df73..e905f2b22 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -898,15 +898,12 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $retries = 3; while ($retries--) { try { - $httpDownloader = $this->httpDownloader; - if ($this->eventDispatcher) { $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); } - $response = $httpDownloader->get($filename, $this->options); + $response = $this->httpDownloader->get($filename, $this->options); $json = $response->getBody(); if ($sha256 && $sha256 !== hash('sha256', $json)) { // undo downgrade before trying again if http seems to be hijacked or modifying content somehow @@ -989,12 +986,9 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $retries = 3; while ($retries--) { try { - $httpDownloader = $this->httpDownloader; - if ($this->eventDispatcher) { $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); } $options = $this->options; @@ -1002,7 +996,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $options['http']['header'] = (array) $options['http']['header']; } $options['http']['header'][] = array('If-Modified-Since: '.$lastModifiedTime); - $response = $httpDownloader->get($filename, $options); + $response = $this->httpDownloader->get($filename, $options); $json = $response->getBody(); if ($json === '' && $response->getStatusCode() === 304) { return true; @@ -1053,12 +1047,11 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito private function asyncFetchFile($filename, $cacheKey, $lastModifiedTime = null) { $retries = 3; - $httpDownloader = $this->httpDownloader; + $httpDownloader = $this->httpDownloader; if ($this->eventDispatcher) { $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); } $options = $lastModifiedTime ? array('http' => array('header' => array('If-Modified-Since: '.$lastModifiedTime))) : array(); From f54237159da2437aea09a0b70a8214c7766527a0 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 18 Jan 2019 12:14:47 +0100 Subject: [PATCH 33/34] Cleanups --- tests/Composer/Test/Repository/Pear/ChannelReaderTest.php | 8 ++++---- .../Test/Repository/Pear/ChannelRest10ReaderTest.php | 4 ++-- .../Test/Repository/Pear/ChannelRest11ReaderTest.php | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/Composer/Test/Repository/Pear/ChannelReaderTest.php b/tests/Composer/Test/Repository/Pear/ChannelReaderTest.php index 7ad30825d..95e59e906 100644 --- a/tests/Composer/Test/Repository/Pear/ChannelReaderTest.php +++ b/tests/Composer/Test/Repository/Pear/ChannelReaderTest.php @@ -28,13 +28,13 @@ class ChannelReaderTest extends TestCase { public function testShouldBuildPackagesFromPearSchema() { - $rfs = new HttpDownloaderMock(array( + $httpDownloader = new HttpDownloaderMock(array( 'http://pear.net/channel.xml' => file_get_contents(__DIR__ . '/Fixtures/channel.1.1.xml'), 'http://test.loc/rest11/c/categories.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/categories.xml'), 'http://test.loc/rest11/c/Default/packagesinfo.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/packagesinfo.xml'), )); - $reader = new \Composer\Repository\Pear\ChannelReader($rfs); + $reader = new \Composer\Repository\Pear\ChannelReader($httpDownloader); $channelInfo = $reader->read('http://pear.net/'); $packages = $channelInfo->getPackages(); @@ -50,7 +50,7 @@ class ChannelReaderTest extends TestCase public function testShouldSelectCorrectReader() { - $rfs = new HttpDownloaderMock(array( + $httpDownloader = new HttpDownloaderMock(array( 'http://pear.1.0.net/channel.xml' => file_get_contents(__DIR__ . '/Fixtures/channel.1.0.xml'), 'http://test.loc/rest10/p/packages.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/packages.xml'), 'http://test.loc/rest10/p/http_client/info.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_client_info.xml'), @@ -64,7 +64,7 @@ class ChannelReaderTest extends TestCase 'http://test.loc/rest11/c/Default/packagesinfo.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/packagesinfo.xml'), )); - $reader = new \Composer\Repository\Pear\ChannelReader($rfs); + $reader = new \Composer\Repository\Pear\ChannelReader($httpDownloader); $reader->read('http://pear.1.0.net/'); $reader->read('http://pear.1.1.net/'); diff --git a/tests/Composer/Test/Repository/Pear/ChannelRest10ReaderTest.php b/tests/Composer/Test/Repository/Pear/ChannelRest10ReaderTest.php index 5a40915e1..3960c7858 100644 --- a/tests/Composer/Test/Repository/Pear/ChannelRest10ReaderTest.php +++ b/tests/Composer/Test/Repository/Pear/ChannelRest10ReaderTest.php @@ -19,7 +19,7 @@ class ChannelRest10ReaderTest extends TestCase { public function testShouldBuildPackagesFromPearSchema() { - $rfs = new HttpDownloaderMock(array( + $httpDownloader = new HttpDownloaderMock(array( 'http://test.loc/rest10/p/packages.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/packages.xml'), 'http://test.loc/rest10/p/http_client/info.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_client_info.xml'), 'http://test.loc/rest10/r/http_client/allreleases.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_client_allreleases.xml'), @@ -29,7 +29,7 @@ class ChannelRest10ReaderTest extends TestCase 'http://test.loc/rest10/r/http_request/deps.1.4.0.txt' => file_get_contents(__DIR__ . '/Fixtures/Rest1.0/http_request_deps.1.4.0.txt'), )); - $reader = new \Composer\Repository\Pear\ChannelRest10Reader($rfs); + $reader = new \Composer\Repository\Pear\ChannelRest10Reader($httpDownloader); /** @var \Composer\Package\PackageInterface[] $packages */ $packages = $reader->read('http://test.loc/rest10'); diff --git a/tests/Composer/Test/Repository/Pear/ChannelRest11ReaderTest.php b/tests/Composer/Test/Repository/Pear/ChannelRest11ReaderTest.php index 08c3a2998..684c59155 100644 --- a/tests/Composer/Test/Repository/Pear/ChannelRest11ReaderTest.php +++ b/tests/Composer/Test/Repository/Pear/ChannelRest11ReaderTest.php @@ -19,13 +19,13 @@ class ChannelRest11ReaderTest extends TestCase { public function testShouldBuildPackagesFromPearSchema() { - $rfs = new HttpDownloaderMock(array( + $httpDownloader = new HttpDownloaderMock(array( 'http://pear.1.1.net/channel.xml' => file_get_contents(__DIR__ . '/Fixtures/channel.1.1.xml'), 'http://test.loc/rest11/c/categories.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/categories.xml'), 'http://test.loc/rest11/c/Default/packagesinfo.xml' => file_get_contents(__DIR__ . '/Fixtures/Rest1.1/packagesinfo.xml'), )); - $reader = new \Composer\Repository\Pear\ChannelRest11Reader($rfs); + $reader = new \Composer\Repository\Pear\ChannelRest11Reader($httpDownloader); /** @var \Composer\Package\PackageInterface[] $packages */ $packages = $reader->read('http://test.loc/rest11'); From 37550ce44b5c7d8f42c3528389a354af4b975d59 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 18 Jan 2019 18:49:45 +0100 Subject: [PATCH 34/34] Add support for new minified format --- .../Repository/ComposerRepository.php | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index e905f2b22..bb613497f 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -598,8 +598,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $promises[] = $this->asyncFetchFile($url, $cacheKey, $lastModified) ->then(function ($response) use (&$packages, $contents, $name, $constraint, $repo, $isPackageAcceptableCallable) { - static $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); - if (true === $response) { $response = $contents; } @@ -608,8 +606,38 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return; } + $versions = $response['packages'][$name]; + + if (isset($response['minified']) && $response['minified'] === 'composer/2.0') { + // TODO extract in other method + $expanded = array(); + $expandedVersion = null; + foreach ($versions as $versionData) { + if (!$expandedVersion) { + $expandedVersion = $versionData; + $expanded[] = $expandedVersion; + continue; + } + + // add any changes from the previous version to the expanded one + foreach ($versionData as $key => $val) { + if ($val === '__unset') { + unset($expandedVersion[$key]); + } else { + $expandedVersion[$key] = $val; + } + } + + $expanded[] = $expandedVersion; + } + + $versions = $expanded; + unset($expanded, $expandedVersion, $versionData); + } + + static $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); $versionsToLoad = array(); - foreach ($response['packages'][$name] as $version) { + foreach ($versions as $version) { if (isset($version['version_normalizeds'])) { foreach ($version['version_normalizeds'] as $index => $normalizedVersion) { if (!$repo->isVersionAcceptable($isPackageAcceptableCallable, $constraint, $name, $normalizedVersion)) {