From 9986b797fb1b9acca9add379337717fe16a27ea0 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 16 Nov 2018 14:28:00 +0100 Subject: [PATCH] Add support for redirects/retries in curl downloader --- src/Composer/Util/Http/CurlDownloader.php | 72 +++++++++++++++++++---- src/Composer/Util/HttpDownloader.php | 35 +---------- src/Composer/Util/Url.php | 34 +++++++++++ 3 files changed, 96 insertions(+), 45 deletions(-) diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php index 2accb7a0c..83f07c44a 100644 --- a/src/Composer/Util/Http/CurlDownloader.php +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -19,6 +19,7 @@ use Composer\CaBundle\CaBundle; use Composer\Util\RemoteFilesystem; use Composer\Util\StreamContextFactory; use Composer\Util\AuthHelper; +use Composer\Util\Url; use Psr\Log\LoggerInterface; use React\Promise\Promise; @@ -132,11 +133,10 @@ class CurlDownloader } curl_setopt($curlHandle, CURLOPT_URL, $url); - curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($curlHandle, CURLOPT_MAXREDIRS, 20); + curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, false); //curl_setopt($curlHandle, CURLOPT_DNS_USE_GLOBAL_CACHE, false); curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 10); - curl_setopt($curlHandle, CURLOPT_TIMEOUT, 10); // TODO increase + curl_setopt($curlHandle, CURLOPT_TIMEOUT, 60); curl_setopt($curlHandle, CURLOPT_WRITEHEADER, $headerHandle); curl_setopt($curlHandle, CURLOPT_FILE, $bodyHandle); curl_setopt($curlHandle, CURLOPT_ENCODING, "gzip"); @@ -181,7 +181,6 @@ class CurlDownloader 'options' => $originalOptions, 'progress' => $progress, 'curlHandle' => $curlHandle, - //'callback' => $params['notification'], 'filename' => $copyTo, 'headerHandle' => $headerHandle, 'bodyHandle' => $bodyHandle, @@ -252,12 +251,24 @@ class CurlDownloader $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG); } - $response = $this->retryIfAuthNeeded($job, $response); + $result = $this->isAuthenticatedRetryNeeded($job, $response); + if ($result['retry']) { + if ($job['filename']) { + @unlink($job['filename'].'~'); + } + + $this->restartJob($job, $job['url'], array('storeAuth' => $result['storeAuth'])); + continue; + } // handle 3xx redirects, 304 Not Modified is excluded if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['redirects'] < $this->maxRedirects) { // TODO - $response = $this->handleRedirect($job, $response); + $location = $this->handleRedirect($job, $response); + if ($location) { + $this->restartJob($job, $location, array('redirects' => $job['attributes']['redirects'] + 1)); + continue; + } } // fail 4xx and 5xx responses and capture the response @@ -331,7 +342,39 @@ class CurlDownloader } } - private function retryIfAuthNeeded(array $job, Response $response) + private function handleRedirect(array $job, Response $response) + { + if ($locationHeader = $response->getHeader('location')) { + if (parse_url($locationHeader, PHP_URL_SCHEME)) { + // Absolute URL; e.g. https://example.com/composer + $targetUrl = $locationHeader; + } elseif (parse_url($locationHeader, PHP_URL_HOST)) { + // Scheme relative; e.g. //example.com/foo + $targetUrl = parse_url($job['url'], PHP_URL_SCHEME).':'.$locationHeader; + } elseif ('/' === $locationHeader[0]) { + // Absolute path; e.g. /foo + $urlHost = parse_url($job['url'], PHP_URL_HOST); + + // Replace path using hostname as an anchor. + $targetUrl = preg_replace('{^(.+(?://|@)'.preg_quote($urlHost).'(?::\d+)?)(?:[/\?].*)?$}', '\1'.$locationHeader, $job['url']); + } else { + // Relative path; e.g. foo + // This actually differs from PHP which seems to add duplicate slashes. + $targetUrl = preg_replace('{^(.+/)[^/?]*(?:\?.*)?$}', '\1'.$locationHeader, $job['url']); + } + } + + if (!empty($targetUrl)) { + $this->io->writeError('', true, IOInterface::DEBUG); + $this->io->writeError(sprintf('Following redirect (%u) %s', $job['redirects'] + 1, $targetUrl), true, IOInterface::DEBUG); + + return $targetUrl; + } + + throw new TransportException('The "'.$job['url'].'" file could not be downloaded, got redirect without Location ('.$response->getStatusMessage().')'); + } + + private function isAuthenticatedRetryNeeded(array $job, Response $response) { if (in_array($response->getStatusCode(), array(401, 403)) && $job['attributes']['retryAuthFailure']) { $warning = null; @@ -345,7 +388,7 @@ class CurlDownloader $result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], $response->getStatusCode(), $response->getStatusMessage(), $warning, $response->getHeaders()); if ($result['retry']) { - // TODO retry somehow using $result['storeAuth'] in the attributes + return $result; } } @@ -376,15 +419,22 @@ class CurlDownloader if ($job['attributes']['retryAuthFailure']) { $result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], 401); if ($result['retry']) { - // TODO ... - // TODO return early here to abort failResponse + return $result; } } throw $this->failResponse($job, $response, $needsAuthRetry); } - return $response; + return array('retry' => false, 'storeAuth' => false); + } + + private function restartJob(array $job, $url, array $attributes = array()) + { + $attributes = array_merge($job['attributes'], $attributes); + $origin = Url::getOrigin($this->config, $url); + + $this->initDownload($job['resolve'], $job['reject'], $origin, $url, $job['originalOptions'], $job['filename'], $attributes); } private function failResponse(array $job, Response $response, $errorMessage) diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php index 9cdc0c919..da3906728 100644 --- a/src/Composer/Util/HttpDownloader.php +++ b/src/Composer/Util/HttpDownloader.php @@ -142,7 +142,7 @@ class HttpDownloader $rfs = $this->rfs; $io = $this->io; - $origin = $this->getOrigin($job['request']['url']); + $origin = Url::getOrigin($this->config, $job['request']['url']); if ($curl && preg_match('{^https?://}i', $job['request']['url'])) { $resolver = function ($resolve, $reject) use (&$job, $curl, $origin) { @@ -250,37 +250,4 @@ class HttpDownloader return $resp; } - - private function getOrigin($url) - { - if (0 === strpos($url, 'file://')) { - return $url; - } - - $origin = parse_url($url, PHP_URL_HOST); - - if (strpos($origin, '.github.com') === (strlen($origin) - 11)) { - return 'github.com'; - } - - if ($origin === 'repo.packagist.org') { - return 'packagist.org'; - } - - // Gitlab can be installed in a non-root context (i.e. gitlab.com/foo). When downloading archives the originUrl - // is the host without the path, so we look for the registered gitlab-domains matching the host here - if ( - is_array($this->config->get('gitlab-domains')) - && false === strpos($origin, '/') - && !in_array($origin, $this->config->get('gitlab-domains')) - ) { - foreach ($this->config->get('gitlab-domains') as $gitlabDomain) { - if (0 === strpos($gitlabDomain, $origin)) { - return $gitlabDomain; - } - } - } - - return $origin ?: $url; - } } diff --git a/src/Composer/Util/Url.php b/src/Composer/Util/Url.php index 4a5d5f90c..3266c4220 100644 --- a/src/Composer/Util/Url.php +++ b/src/Composer/Util/Url.php @@ -52,4 +52,38 @@ class Url return $url; } + + public static function getOrigin(Config $config, $url) + { + if (0 === strpos($url, 'file://')) { + return $url; + } + + $origin = parse_url($url, PHP_URL_HOST); + + if (strpos($origin, '.github.com') === (strlen($origin) - 11)) { + return 'github.com'; + } + + if ($origin === 'repo.packagist.org') { + return 'packagist.org'; + } + + // Gitlab can be installed in a non-root context (i.e. gitlab.com/foo). When downloading archives the originUrl + // is the host without the path, so we look for the registered gitlab-domains matching the host here + if ( + is_array($config->get('gitlab-domains')) + && false === strpos($origin, '/') + && !in_array($origin, $config->get('gitlab-domains')) + ) { + foreach ($config->get('gitlab-domains') as $gitlabDomain) { + if (0 === strpos($gitlabDomain, $origin)) { + return $gitlabDomain; + } + } + } + + return $origin ?: $url; + } + }