From ba1814f306a85cf55c4c153d6efa93ff12f65942 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Wed, 10 Nov 2021 22:12:42 +0100 Subject: [PATCH] Added retry behavior for certain http status and curl error codes (#10162) --- .../Repository/ComposerRepository.php | 148 ++++++------------ src/Composer/Util/Http/CurlDownloader.php | 29 +++- 2 files changed, 78 insertions(+), 99 deletions(-) diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 113c2074b..0aa86b7f8 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -1267,19 +1267,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito throw $e; } - // try to detect offline state (if dns resolution fails it is pretty likely to keep failing) and avoid retrying in that case - if ($e instanceof TransportException && $e->getStatusCode() === null) { - $responseInfo = $e->getResponseInfo(); - if (isset($responseInfo['namelookup_time']) && $responseInfo['namelookup_time'] == 0) { - $retries = 0; - } - } - - if ($retries) { - usleep(100000); - continue; - } - if ($e instanceof RepositorySecurityException) { throw $e; } @@ -1314,71 +1301,61 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito */ private function fetchFileIfLastModified($filename, $cacheKey, $lastModifiedTime) { - $retries = 3; - while ($retries--) { - try { - $options = $this->options; - if ($this->eventDispatcher) { - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename, 'metadata', array('repository' => $this)); - $preFileDownloadEvent->setTransportOptions($this->options); - $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - $filename = $preFileDownloadEvent->getProcessedUrl(); - $options = $preFileDownloadEvent->getTransportOptions(); - } - - if (isset($options['http']['header'])) { - $options['http']['header'] = (array) $options['http']['header']; - } - $options['http']['header'][] = 'If-Modified-Since: '.$lastModifiedTime; - $response = $this->httpDownloader->get($filename, $options); - $json = (string) $response->getBody(); - if ($json === '' && $response->getStatusCode() === 304) { - return true; - } - - if ($this->eventDispatcher) { - $postFileDownloadEvent = new PostFileDownloadEvent(PluginEvents::POST_FILE_DOWNLOAD, null, null, $filename, 'metadata', array('response' => $response, 'repository' => $this)); - $this->eventDispatcher->dispatch($postFileDownloadEvent->getName(), $postFileDownloadEvent); - } - - $data = $response->decodeJson(); - HttpDownloader::outputWarnings($this->io, $this->url, $data); - - $lastModifiedDate = $response->getHeader('last-modified'); - $response->collect(); - if ($lastModifiedDate) { - $data['last-modified'] = $lastModifiedDate; - $json = JsonFile::encode($data, 0); - } - if (!$this->cache->isReadOnly()) { - $this->cache->write($cacheKey, $json); - } - - return $data; - } catch (\Exception $e) { - if ($e instanceof \LogicException) { - throw $e; - } - - if ($e instanceof TransportException && $e->getStatusCode() === 404) { - throw $e; - } - - if ($retries) { - usleep(100000); - continue; - } - - if (!$this->degradedMode) { - $this->io->writeError(''.$this->url.' could not be fully loaded ('.$e->getMessage().'), package information was loaded from the local cache and may be out of date'); - } - $this->degradedMode = true; + try { + $options = $this->options; + if ($this->eventDispatcher) { + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename, 'metadata', array('repository' => $this)); + $preFileDownloadEvent->setTransportOptions($this->options); + $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); + $filename = $preFileDownloadEvent->getProcessedUrl(); + $options = $preFileDownloadEvent->getTransportOptions(); + } + if (isset($options['http']['header'])) { + $options['http']['header'] = (array) $options['http']['header']; + } + $options['http']['header'][] = 'If-Modified-Since: '.$lastModifiedTime; + $response = $this->httpDownloader->get($filename, $options); + $json = (string) $response->getBody(); + if ($json === '' && $response->getStatusCode() === 304) { return true; } - } - throw new \LogicException('Should not happen'); + if ($this->eventDispatcher) { + $postFileDownloadEvent = new PostFileDownloadEvent(PluginEvents::POST_FILE_DOWNLOAD, null, null, $filename, 'metadata', array('response' => $response, 'repository' => $this)); + $this->eventDispatcher->dispatch($postFileDownloadEvent->getName(), $postFileDownloadEvent); + } + + $data = $response->decodeJson(); + HttpDownloader::outputWarnings($this->io, $this->url, $data); + + $lastModifiedDate = $response->getHeader('last-modified'); + $response->collect(); + if ($lastModifiedDate) { + $data['last-modified'] = $lastModifiedDate; + $json = JsonFile::encode($data, 0); + } + if (!$this->cache->isReadOnly()) { + $this->cache->write($cacheKey, $json); + } + + return $data; + } catch (\Exception $e) { + if ($e instanceof \LogicException) { + throw $e; + } + + if ($e instanceof TransportException && $e->getStatusCode() === 404) { + throw $e; + } + + if (!$this->degradedMode) { + $this->io->writeError(''.$this->url.' could not be fully loaded ('.$e->getMessage().'), package information was loaded from the local cache and may be out of date'); + } + $this->degradedMode = true; + + return true; + } } /** @@ -1390,8 +1367,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito */ private function asyncFetchFile($filename, $cacheKey, $lastModifiedTime = null) { - $retries = 3; - if (isset($this->packagesNotFoundCache[$filename])) { return \React\Promise\resolve(array('packages' => array())); } @@ -1462,32 +1437,13 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $data; }; - $reject = function ($e) use (&$retries, $httpDownloader, $filename, $options, &$reject, $accept, $io, $url, &$degradedMode, $repo, $lastModifiedTime) { + $reject = function ($e) use ($filename, $accept, $io, $url, &$degradedMode, $repo, $lastModifiedTime) { if ($e instanceof TransportException && $e->getStatusCode() === 404) { $repo->packagesNotFoundCache[$filename] = true; return false; } - // special error code returned when network is being artificially disabled - if ($e instanceof TransportException && $e->getStatusCode() === 499) { - $retries = 0; - } - - // try to detect offline state (if dns resolution fails it is pretty likely to keep failing) and avoid retrying in that case - if ($e instanceof TransportException && $e->getStatusCode() === null) { - $responseInfo = $e->getResponseInfo(); - if (isset($responseInfo['namelookup_time']) && $responseInfo['namelookup_time'] == 0) { - $retries = 0; - } - } - - if (--$retries > 0) { - usleep(100000); - - return $httpDownloader->add($filename, $options)->then($accept, $reject); - } - if (!$degradedMode) { $io->writeError(''.$url.' could not be fully loaded ('.$e->getMessage().'), package information was loaded from the local cache and may be out of date'); } diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php index 2ecc6e9b6..6b5d3a061 100644 --- a/src/Composer/Util/Http/CurlDownloader.php +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -26,7 +26,7 @@ use React\Promise\Promise; * @internal * @author Jordi Boggiano * @author Nicolas Grekas - * @phpstan-type Attributes array{retryAuthFailure: bool, redirects: int, storeAuth: bool} + * @phpstan-type Attributes array{retryAuthFailure: bool, redirects: int, retries: int, storeAuth: bool} * @phpstan-type Job array{url: string, origin: string, attributes: Attributes, options: mixed[], progress: mixed[], curlHandle: resource, filename: string|false, headerHandle: resource, bodyHandle: resource, resolve: callable, reject: callable} */ class CurlDownloader @@ -47,6 +47,8 @@ class CurlDownloader private $selectTimeout = 5.0; /** @var int */ private $maxRedirects = 20; + /** @var int */ + private $maxRetries = 3; /** @var ProxyManager */ private $proxyManager; /** @var bool */ @@ -149,7 +151,7 @@ class CurlDownloader * @param mixed[] $options * @param ?string $copyTo * - * @param array{retryAuthFailure?: bool, redirects?: int, storeAuth?: bool} $attributes + * @param array{retryAuthFailure?: bool, redirects?: int, retries?: int, storeAuth?: bool} $attributes * * @return int internal job id */ @@ -158,6 +160,7 @@ class CurlDownloader $attributes = array_merge(array( 'retryAuthFailure' => true, 'redirects' => 0, + 'retries' => 0, 'storeAuth' => false, ), $attributes); @@ -267,7 +270,7 @@ class CurlDownloader $usingProxy = $proxy->getFormattedUrl(' using proxy (%s)'); $ifModified = false !== stripos(implode(',', $options['http']['header']), 'if-modified-since:') ? ' if modified' : ''; - if ($attributes['redirects'] === 0) { + if ($attributes['redirects'] === 0 && $attributes['retries'] === 0) { $this->io->writeError('Downloading ' . Url::sanitize($url) . $usingProxy . $ifModified, true, IOInterface::DEBUG); } @@ -346,6 +349,16 @@ class CurlDownloader } $progress['error_code'] = $errno; + if ( + (!isset($job['options']['http']['method']) || $job['options']['http']['method'] === 'GET') + && in_array($errno, array(7 /* CURLE_COULDNT_CONNECT */, 16 /* CURLE_HTTP2 */), true) + && $job['attributes']['retries'] < $this->maxRetries + ) { + $this->io->writeError('Retrying ('.($job['attributes']['retries'] + 1).') ' . Url::sanitize($job['url']) . ' due to curl error '. $errno, true, IOInterface::DEBUG); + $this->restartJob($job, $job['url'], array('retries' => $job['attributes']['retries'] + 1)); + continue; + } + if ($errno === 28 /* CURLE_OPERATION_TIMEDOUT */ && isset($progress['namelookup_time']) && $progress['namelookup_time'] == 0 && !$timeoutWarning) { $timeoutWarning = true; $this->io->writeError('A connection timeout was encountered. If you intend to run Composer without connecting to the internet, run the command again prefixed with COMPOSER_DISABLE_NETWORK=1 to make Composer run in offline mode.'); @@ -400,6 +413,16 @@ class CurlDownloader // fail 4xx and 5xx responses and capture the response if ($statusCode >= 400 && $statusCode <= 599) { + if ( + (!isset($job['options']['http']['method']) || $job['options']['http']['method'] === 'GET') + && in_array($statusCode, array(423, 425, 500, 502, 503, 504, 507, 510), true) + && $job['attributes']['retries'] < $this->maxRetries + ) { + $this->io->writeError('Retrying ('.($job['attributes']['retries'] + 1).') ' . Url::sanitize($job['url']) . ' due to status code '. $statusCode, true, IOInterface::DEBUG); + $this->restartJob($job, $job['url'], array('retries' => $job['attributes']['retries'] + 1)); + continue; + } + throw $this->failResponse($job, $response, $response->getStatusMessage()); }