public-mirrors
/
composer
mirror of https://github.com/composer/composer
1
0
Fork 0
Code

Added retry behavior for certain http status and curl error codes (#10162)

pull/10275/head
Jordi Boggiano 2021-11-10 22:12:42 +01:00 committed by GitHub
parent 6aa8a466b7
commit ba1814f306
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 78 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

148
src/Composer/Repository/ComposerRepository.php
View File

@ -1267,19 +1267,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
throw $e;
}
// try to detect offline state (if dns resolution fails it is pretty likely to keep failing) and avoid retrying in that case
if ($e instanceof TransportException && $e->getStatusCode() === null) {
$responseInfo = $e->getResponseInfo();
if (isset($responseInfo['namelookup_time']) && $responseInfo['namelookup_time'] == 0) {
$retries = 0;
}
}
if ($retries) {
usleep(100000);
continue;
}
if ($e instanceof RepositorySecurityException) {
throw $e;
}
@ -1314,71 +1301,61 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
*/
private function fetchFileIfLastModified($filename, $cacheKey, $lastModifiedTime)
{
$retries = 3;
while ($retries--) {
try {
$options = $this->options;
if ($this->eventDispatcher) {
$preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename, 'metadata', array('repository' => $this));
$preFileDownloadEvent->setTransportOptions($this->options);
$this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent);
$filename = $preFileDownloadEvent->getProcessedUrl();
$options = $preFileDownloadEvent->getTransportOptions();
}
if (isset($options['http']['header'])) {
$options['http']['header'] = (array) $options['http']['header'];
}
$options['http']['header'][] = 'If-Modified-Since: '.$lastModifiedTime;
$response = $this->httpDownloader->get($filename, $options);
$json = (string) $response->getBody();
if ($json === '' && $response->getStatusCode() === 304) {
return true;
}
if ($this->eventDispatcher) {
$postFileDownloadEvent = new PostFileDownloadEvent(PluginEvents::POST_FILE_DOWNLOAD, null, null, $filename, 'metadata', array('response' => $response, 'repository' => $this));
$this->eventDispatcher->dispatch($postFileDownloadEvent->getName(), $postFileDownloadEvent);
}
$data = $response->decodeJson();
HttpDownloader::outputWarnings($this->io, $this->url, $data);
$lastModifiedDate = $response->getHeader('last-modified');
$response->collect();
if ($lastModifiedDate) {
$data['last-modified'] = $lastModifiedDate;
$json = JsonFile::encode($data, 0);
}
if (!$this->cache->isReadOnly()) {
$this->cache->write($cacheKey, $json);
}
return $data;
} catch (\Exception $e) {
if ($e instanceof \LogicException) {
throw $e;
}
if ($e instanceof TransportException && $e->getStatusCode() === 404) {
throw $e;
}
if ($retries) {
usleep(100000);
continue;
}
if (!$this->degradedMode) {
$this->io->writeError('<warning>'.$this->url.' could not be fully loaded ('.$e->getMessage().'), package information was loaded from the local cache and may be out of date</warning>');
}
$this->degradedMode = true;
try {
$options = $this->options;
if ($this->eventDispatcher) {
$preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename, 'metadata', array('repository' => $this));
$preFileDownloadEvent->setTransportOptions($this->options);
$this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent);
$filename = $preFileDownloadEvent->getProcessedUrl();
$options = $preFileDownloadEvent->getTransportOptions();
}
if (isset($options['http']['header'])) {
$options['http']['header'] = (array) $options['http']['header'];
}
$options['http']['header'][] = 'If-Modified-Since: '.$lastModifiedTime;
$response = $this->httpDownloader->get($filename, $options);
$json = (string) $response->getBody();
if ($json === '' && $response->getStatusCode() === 304) {
return true;
}
}
throw new \LogicException('Should not happen');
if ($this->eventDispatcher) {
$postFileDownloadEvent = new PostFileDownloadEvent(PluginEvents::POST_FILE_DOWNLOAD, null, null, $filename, 'metadata', array('response' => $response, 'repository' => $this));
$this->eventDispatcher->dispatch($postFileDownloadEvent->getName(), $postFileDownloadEvent);
}
$data = $response->decodeJson();
HttpDownloader::outputWarnings($this->io, $this->url, $data);
$lastModifiedDate = $response->getHeader('last-modified');
$response->collect();
if ($lastModifiedDate) {
$data['last-modified'] = $lastModifiedDate;
$json = JsonFile::encode($data, 0);
}
if (!$this->cache->isReadOnly()) {
$this->cache->write($cacheKey, $json);
}
return $data;
} catch (\Exception $e) {
if ($e instanceof \LogicException) {
throw $e;
}
if ($e instanceof TransportException && $e->getStatusCode() === 404) {
throw $e;
}
if (!$this->degradedMode) {
$this->io->writeError('<warning>'.$this->url.' could not be fully loaded ('.$e->getMessage().'), package information was loaded from the local cache and may be out of date</warning>');
}
$this->degradedMode = true;
return true;
}
}
/**
@ -1390,8 +1367,6 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
*/
private function asyncFetchFile($filename, $cacheKey, $lastModifiedTime = null)
{
$retries = 3;
if (isset($this->packagesNotFoundCache[$filename])) {
return \React\Promise\resolve(array('packages' => array()));
}
@ -1462,32 +1437,13 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
return $data;
};
$reject = function ($e) use (&$retries, $httpDownloader, $filename, $options, &$reject, $accept, $io, $url, &$degradedMode, $repo, $lastModifiedTime) {
$reject = function ($e) use ($filename, $accept, $io, $url, &$degradedMode, $repo, $lastModifiedTime) {
if ($e instanceof TransportException && $e->getStatusCode() === 404) {
$repo->packagesNotFoundCache[$filename] = true;
return false;
}
// special error code returned when network is being artificially disabled
if ($e instanceof TransportException && $e->getStatusCode() === 499) {
$retries = 0;
}
// try to detect offline state (if dns resolution fails it is pretty likely to keep failing) and avoid retrying in that case
if ($e instanceof TransportException && $e->getStatusCode() === null) {
$responseInfo = $e->getResponseInfo();
if (isset($responseInfo['namelookup_time']) && $responseInfo['namelookup_time'] == 0) {
$retries = 0;
}
}
if (--$retries > 0) {
usleep(100000);
return $httpDownloader->add($filename, $options)->then($accept, $reject);
}
if (!$degradedMode) {
$io->writeError('<warning>'.$url.' could not be fully loaded ('.$e->getMessage().'), package information was loaded from the local cache and may be out of date</warning>');
}

29
src/Composer/Util/Http/CurlDownloader.php
View File

@ -26,7 +26,7 @@ use React\Promise\Promise;
* @internal
* @author Jordi Boggiano <j.boggiano@seld.be>
* @author Nicolas Grekas <p@tchwork.com>
* @phpstan-type Attributes array{retryAuthFailure: bool, redirects: int, storeAuth: bool}
* @phpstan-type Attributes array{retryAuthFailure: bool, redirects: int, retries: int, storeAuth: bool}
* @phpstan-type Job array{url: string, origin: string, attributes: Attributes, options: mixed[], progress: mixed[], curlHandle: resource, filename: string|false, headerHandle: resource, bodyHandle: resource, resolve: callable, reject: callable}
*/
class CurlDownloader
@ -47,6 +47,8 @@ class CurlDownloader
private $selectTimeout = 5.0;
/** @var int */
private $maxRedirects = 20;
/** @var int */
private $maxRetries = 3;
/** @var ProxyManager */
private $proxyManager;
/** @var bool */
@ -149,7 +151,7 @@ class CurlDownloader
* @param mixed[] $options
* @param ?string $copyTo
*
* @param array{retryAuthFailure?: bool, redirects?: int, storeAuth?: bool} $attributes
* @param array{retryAuthFailure?: bool, redirects?: int, retries?: int, storeAuth?: bool} $attributes
*
* @return int internal job id
*/
@ -158,6 +160,7 @@ class CurlDownloader
$attributes = array_merge(array(
'retryAuthFailure' => true,
'redirects' => 0,
'retries' => 0,
'storeAuth' => false,
), $attributes);
@ -267,7 +270,7 @@ class CurlDownloader
$usingProxy = $proxy->getFormattedUrl(' using proxy (%s)');
$ifModified = false !== stripos(implode(',', $options['http']['header']), 'if-modified-since:') ? ' if modified' : '';
if ($attributes['redirects'] === 0) {
if ($attributes['redirects'] === 0 && $attributes['retries'] === 0) {
$this->io->writeError('Downloading ' . Url::sanitize($url) . $usingProxy . $ifModified, true, IOInterface::DEBUG);
}
@ -346,6 +349,16 @@ class CurlDownloader
}
$progress['error_code'] = $errno;
if (
(!isset($job['options']['http']['method']) || $job['options']['http']['method'] === 'GET')
&& in_array($errno, array(7 /* CURLE_COULDNT_CONNECT */, 16 /* CURLE_HTTP2 */), true)
&& $job['attributes']['retries'] < $this->maxRetries
) {
$this->io->writeError('Retrying ('.($job['attributes']['retries'] + 1).') ' . Url::sanitize($job['url']) . ' due to curl error '. $errno, true, IOInterface::DEBUG);
$this->restartJob($job, $job['url'], array('retries' => $job['attributes']['retries'] + 1));
continue;
}
if ($errno === 28 /* CURLE_OPERATION_TIMEDOUT */ && isset($progress['namelookup_time']) && $progress['namelookup_time'] == 0 && !$timeoutWarning) {
$timeoutWarning = true;
$this->io->writeError('<warning>A connection timeout was encountered. If you intend to run Composer without connecting to the internet, run the command again prefixed with COMPOSER_DISABLE_NETWORK=1 to make Composer run in offline mode.</warning>');
@ -400,6 +413,16 @@ class CurlDownloader
// fail 4xx and 5xx responses and capture the response
if ($statusCode >= 400 && $statusCode <= 599) {
if (
(!isset($job['options']['http']['method']) || $job['options']['http']['method'] === 'GET')
&& in_array($statusCode, array(423, 425, 500, 502, 503, 504, 507, 510), true)
&& $job['attributes']['retries'] < $this->maxRetries
) {
$this->io->writeError('Retrying ('.($job['attributes']['retries'] + 1).') ' . Url::sanitize($job['url']) . ' due to status code '. $statusCode, true, IOInterface::DEBUG);
$this->restartJob($job, $job['url'], array('retries' => $job['attributes']['retries'] + 1));
continue;
}
throw $this->failResponse($job, $response, $response->getStatusMessage());
}