* Jordi Boggiano * * For the full copyright and license information, please view the LICENSE * file that was distributed with this source code. */ namespace Composer\Util\Http; use Composer\Config; use Composer\IO\IOInterface; use Composer\Downloader\TransportException; use Composer\CaBundle\CaBundle; use Composer\Util\RemoteFilesystem; use Composer\Util\StreamContextFactory; use Composer\Util\AuthHelper; use Composer\Util\Url; use Psr\Log\LoggerInterface; use React\Promise\Promise; /** * @author Jordi Boggiano * @author Nicolas Grekas */ class CurlDownloader { private $multiHandle; private $shareHandle; private $jobs = array(); /** @var IOInterface */ private $io; /** @var Config */ private $config; /** @var AuthHelper */ private $authHelper; private $selectTimeout = 5.0; private $maxRedirects = 20; protected $multiErrors = array( CURLM_BAD_HANDLE => array('CURLM_BAD_HANDLE', 'The passed-in handle is not a valid CURLM handle.'), CURLM_BAD_EASY_HANDLE => array('CURLM_BAD_EASY_HANDLE', "An easy handle was not good/valid. It could mean that it isn't an easy handle at all, or possibly that the handle already is in used by this or another multi handle."), CURLM_OUT_OF_MEMORY => array('CURLM_OUT_OF_MEMORY', 'You are doomed.'), CURLM_INTERNAL_ERROR => array('CURLM_INTERNAL_ERROR', 'This can only be returned if libcurl bugs. Please report it to us!') ); private static $options = array( 'http' => array( 'method' => CURLOPT_CUSTOMREQUEST, 'content' => CURLOPT_POSTFIELDS, 'proxy' => CURLOPT_PROXY, 'header' => CURLOPT_HTTPHEADER, ), 'ssl' => array( 'ciphers' => CURLOPT_SSL_CIPHER_LIST, 'cafile' => CURLOPT_CAINFO, 'capath' => CURLOPT_CAPATH, ), ); private static $timeInfo = array( 'total_time' => true, 'namelookup_time' => true, 'connect_time' => true, 'pretransfer_time' => true, 'starttransfer_time' => true, 'redirect_time' => true, ); public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false) { $this->io = $io; $this->config = $config; $this->multiHandle = $mh = curl_multi_init(); if (function_exists('curl_multi_setopt')) { curl_multi_setopt($mh, CURLMOPT_PIPELINING, /*CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX*/ 3); if (defined('CURLMOPT_MAX_HOST_CONNECTIONS')) { curl_multi_setopt($mh, CURLMOPT_MAX_HOST_CONNECTIONS, 8); } } if (function_exists('curl_share_init')) { $this->shareHandle = $sh = curl_share_init(); curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); } $this->authHelper = new AuthHelper($io, $config); } public function download($resolve, $reject, $origin, $url, $options, $copyTo = null) { $attributes = array(); if (isset($options['retry-auth-failure'])) { $attributes['retryAuthFailure'] = $options['retry-auth-failure']; unset($options['retry-auth-failure']); } return $this->initDownload($resolve, $reject, $origin, $url, $options, $copyTo, $attributes); } private function initDownload($resolve, $reject, $origin, $url, $options, $copyTo = null, array $attributes = array()) { $attributes = array_merge(array( 'retryAuthFailure' => true, 'redirects' => 0, 'storeAuth' => false, ), $attributes); $originalOptions = $options; // check URL can be accessed (i.e. is not insecure) $this->config->prohibitUrlByConfig($url, $this->io); $curlHandle = curl_init(); $headerHandle = fopen('php://temp/maxmemory:32768', 'w+b'); if ($copyTo) { $errorMessage = ''; set_error_handler(function ($code, $msg) use (&$errorMessage) { if ($errorMessage) { $errorMessage .= "\n"; } $errorMessage .= preg_replace('{^fopen\(.*?\): }', '', $msg); }); $bodyHandle = fopen($copyTo.'~', 'w+b'); restore_error_handler(); if (!$bodyHandle) { throw new TransportException('The "'.$url.'" file could not be written to '.$copyTo.': '.$errorMessage); } } else { $bodyHandle = @fopen('php://temp/maxmemory:524288', 'w+b'); } curl_setopt($curlHandle, CURLOPT_URL, $url); curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, false); //curl_setopt($curlHandle, CURLOPT_DNS_USE_GLOBAL_CACHE, false); curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 10); curl_setopt($curlHandle, CURLOPT_TIMEOUT, 60); curl_setopt($curlHandle, CURLOPT_WRITEHEADER, $headerHandle); curl_setopt($curlHandle, CURLOPT_FILE, $bodyHandle); curl_setopt($curlHandle, CURLOPT_ENCODING, "gzip"); curl_setopt($curlHandle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS); if (defined('CURLOPT_SSL_FALSESTART')) { curl_setopt($curlHandle, CURLOPT_SSL_FALSESTART, true); } if (function_exists('curl_share_init')) { curl_setopt($curlHandle, CURLOPT_SHARE, $this->shareHandle); } if (!isset($options['http']['header'])) { $options['http']['header'] = array(); } $options['http']['header'] = array_diff($options['http']['header'], array('Connection: close')); $options['http']['header'][] = 'Connection: keep-alive'; $version = curl_version(); $features = $version['features']; if (0 === strpos($url, 'https://') && \defined('CURL_VERSION_HTTP2') && \defined('CURL_HTTP_VERSION_2_0') && (CURL_VERSION_HTTP2 & $features)) { curl_setopt($curlHandle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); } $options['http']['header'] = $this->authHelper->addAuthenticationHeader($options['http']['header'], $origin, $url); $options = StreamContextFactory::initOptions($url, $options); foreach (self::$options as $type => $curlOptions) { foreach ($curlOptions as $name => $curlOption) { if (isset($options[$type][$name])) { curl_setopt($curlHandle, $curlOption, $options[$type][$name]); } } } $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); $this->jobs[(int) $curlHandle] = array( 'url' => $url, 'origin' => $origin, 'attributes' => $attributes, 'options' => $originalOptions, 'progress' => $progress, 'curlHandle' => $curlHandle, 'filename' => $copyTo, 'headerHandle' => $headerHandle, 'bodyHandle' => $bodyHandle, 'resolve' => $resolve, 'reject' => $reject, ); $usingProxy = !empty($options['http']['proxy']) ? ' using proxy ' . $options['http']['proxy'] : ''; $ifModified = false !== strpos(strtolower(implode(',', $options['http']['header'])), 'if-modified-since:') ? ' if modified' : ''; if ($attributes['redirects'] === 0) { $this->io->writeError('Downloading ' . $url . $usingProxy . $ifModified, true, IOInterface::DEBUG); } $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $curlHandle)); // TODO progress //$params['notification'](STREAM_NOTIFY_RESOLVE, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, 0, false); } public function tick() { if (!$this->jobs) { return; } $active = true; try { $this->checkCurlResult(curl_multi_exec($this->multiHandle, $active)); if (-1 === curl_multi_select($this->multiHandle, $this->selectTimeout)) { // sleep in case select returns -1 as it can happen on old php versions or some platforms where curl does not manage to do the select usleep(150); } while ($progress = curl_multi_info_read($this->multiHandle)) { $curlHandle = $progress['handle']; $i = (int) $curlHandle; if (!isset($this->jobs[$i])) { continue; } $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); $job = $this->jobs[$i]; unset($this->jobs[$i]); curl_multi_remove_handle($this->multiHandle, $curlHandle); $error = curl_error($curlHandle); $errno = curl_errno($curlHandle); curl_close($curlHandle); $headers = null; $statusCode = null; $response = null; try { // TODO progress //$this->onProgress($curlHandle, $job['callback'], $progress, $job['progress']); if (CURLE_OK !== $errno) { throw new TransportException($error); } $statusCode = $progress['http_code']; rewind($job['headerHandle']); $headers = explode("\r\n", rtrim(stream_get_contents($job['headerHandle']))); fclose($job['headerHandle']); // prepare response object if ($job['filename']) { fclose($job['bodyHandle']); $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $job['filename'].'~'); $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG); } else { rewind($job['bodyHandle']); $contents = stream_get_contents($job['bodyHandle']); fclose($job['bodyHandle']); $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $contents); $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG); } $result = $this->isAuthenticatedRetryNeeded($job, $response); if ($result['retry']) { if ($job['filename']) { @unlink($job['filename'].'~'); } $this->restartJob($job, $job['url'], array('storeAuth' => $result['storeAuth'])); continue; } // handle 3xx redirects, 304 Not Modified is excluded if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['attributes']['redirects'] < $this->maxRedirects) { $location = $this->handleRedirect($job, $response); if ($location) { $this->restartJob($job, $location, array('redirects' => $job['attributes']['redirects'] + 1)); continue; } } // fail 4xx and 5xx responses and capture the response if ($statusCode >= 400 && $statusCode <= 599) { throw $this->failResponse($job, $response, $response->getStatusMessage()); // TODO progress // $this->io->overwriteError("Downloading (failed)", false); } if ($job['attributes']['storeAuth']) { $this->authHelper->storeAuth($job['origin'], $job['attributes']['storeAuth']); } // resolve promise if ($job['filename']) { rename($job['filename'].'~', $job['filename']); call_user_func($job['resolve'], true); } else { call_user_func($job['resolve'], $response); } } catch (\Exception $e) { if ($e instanceof TransportException && $headers) { $e->setHeaders($headers); $e->setStatusCode($statusCode); } if ($e instanceof TransportException && $response) { $e->setResponse($response->getBody()); } if (is_resource($job['headerHandle'])) { fclose($job['headerHandle']); } if (is_resource($job['bodyHandle'])) { fclose($job['bodyHandle']); } if ($job['filename']) { @unlink($job['filename'].'~'); } call_user_func($job['reject'], $e); } } foreach ($this->jobs as $i => $curlHandle) { if (!isset($this->jobs[$i])) { continue; } $curlHandle = $this->jobs[$i]['curlHandle']; $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); if ($this->jobs[$i]['progress'] !== $progress) { $previousProgress = $this->jobs[$i]['progress']; $this->jobs[$i]['progress'] = $progress; // TODO //$this->onProgress($curlHandle, $this->jobs[$i]['callback'], $progress, $previousProgress); } } } catch (\Exception $e) { // TODO var_dump('Caught2', get_class($e), $e->getMessage(), $e);die; } } private function handleRedirect(array $job, Response $response) { if ($locationHeader = $response->getHeader('location')) { if (parse_url($locationHeader, PHP_URL_SCHEME)) { // Absolute URL; e.g. https://example.com/composer $targetUrl = $locationHeader; } elseif (parse_url($locationHeader, PHP_URL_HOST)) { // Scheme relative; e.g. //example.com/foo $targetUrl = parse_url($job['url'], PHP_URL_SCHEME).':'.$locationHeader; } elseif ('/' === $locationHeader[0]) { // Absolute path; e.g. /foo $urlHost = parse_url($job['url'], PHP_URL_HOST); // Replace path using hostname as an anchor. $targetUrl = preg_replace('{^(.+(?://|@)'.preg_quote($urlHost).'(?::\d+)?)(?:[/\?].*)?$}', '\1'.$locationHeader, $job['url']); } else { // Relative path; e.g. foo // This actually differs from PHP which seems to add duplicate slashes. $targetUrl = preg_replace('{^(.+/)[^/?]*(?:\?.*)?$}', '\1'.$locationHeader, $job['url']); } } if (!empty($targetUrl)) { $this->io->writeError(sprintf('Following redirect (%u) %s', $job['attributes']['redirects'] + 1, $targetUrl), true, IOInterface::DEBUG); return $targetUrl; } throw new TransportException('The "'.$job['url'].'" file could not be downloaded, got redirect without Location ('.$response->getStatusMessage().')'); } private function isAuthenticatedRetryNeeded(array $job, Response $response) { if (in_array($response->getStatusCode(), array(401, 403)) && $job['attributes']['retryAuthFailure']) { $warning = null; if ($response->getHeader('content-type') === 'application/json') { $data = json_decode($response->getBody(), true); if (!empty($data['warning'])) { $warning = $data['warning']; } } $result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], $response->getStatusCode(), $response->getStatusMessage(), $warning, $response->getHeaders()); if ($result['retry']) { return $result; } } $locationHeader = $response->getHeader('location'); $needsAuthRetry = false; // check for bitbucket login page asking to authenticate if ( $job['origin'] === 'bitbucket.org' && !$this->authHelper->isPublicBitBucketDownload($job['url']) && substr($job['url'], -4) === '.zip' && (!$locationHeader || substr($locationHeader, -4) !== '.zip') && preg_match('{^text/html\b}i', $response->getHeader('content-type')) ) { $needsAuthRetry = 'Bitbucket requires authentication and it was not provided'; } // check for gitlab 404 when downloading archives if ( $response->getStatusCode() === 404 && $this->config && in_array($job['origin'], $this->config->get('gitlab-domains'), true) && false !== strpos($job['url'], 'archive.zip') ) { $needsAuthRetry = 'GitLab requires authentication and it was not provided'; } if ($needsAuthRetry) { if ($job['attributes']['retryAuthFailure']) { $result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], 401); if ($result['retry']) { return $result; } } throw $this->failResponse($job, $response, $needsAuthRetry); } return array('retry' => false, 'storeAuth' => false); } private function restartJob(array $job, $url, array $attributes = array()) { $attributes = array_merge($job['attributes'], $attributes); $origin = Url::getOrigin($this->config, $url); $this->initDownload($job['resolve'], $job['reject'], $origin, $url, $job['options'], $job['filename'], $attributes); } private function failResponse(array $job, Response $response, $errorMessage) { return new TransportException('The "'.$job['url'].'" file could not be downloaded ('.$errorMessage.')', $response->getStatusCode()); } private function onProgress($curlHandle, callable $notify, array $progress, array $previousProgress) { // TODO add support for progress if (300 <= $progress['http_code'] && $progress['http_code'] < 400) { return; } if ($previousProgress['download_content_length'] < $progress['download_content_length']) { $notify(STREAM_NOTIFY_FILE_SIZE_IS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, (int) $progress['download_content_length'], false); } if ($previousProgress['size_download'] < $progress['size_download']) { $notify(STREAM_NOTIFY_PROGRESS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, (int) $progress['size_download'], (int) $progress['download_content_length'], false); } } private function checkCurlResult($code) { if ($code != CURLM_OK && $code != CURLM_CALL_MULTI_PERFORM) { throw new \RuntimeException(isset($this->multiErrors[$code]) ? "cURL error: {$code} ({$this->multiErrors[$code][0]}): cURL message: {$this->multiErrors[$code][1]}" : 'Unexpected cURL error: ' . $code ); } } }