diff --git a/src/Composer/Downloader/GitDownloader.php b/src/Composer/Downloader/GitDownloader.php index 3d3d947f4..ba026d2e2 100644 --- a/src/Composer/Downloader/GitDownloader.php +++ b/src/Composer/Downloader/GitDownloader.php @@ -74,10 +74,10 @@ class GitDownloader extends VcsDownloader implements DvcsDownloaderInterface $command = 'git clone --no-checkout %cachePath% %path% --dissociate --reference %cachePath% ' . '&& cd '.$flag.'%path% ' - . '&& git remote set-url origin %url% && git remote add composer %url%'; + . '&& git remote set-url origin %sanitizedUrl% && git remote add composer %sanitizedUrl%'; } else { $msg = "Cloning ".$this->getShortHash($ref); - $command = 'git clone --no-checkout %url% %path% && cd '.$flag.'%path% && git remote add composer %url% && git fetch composer'; + $command = 'git clone --no-checkout %url% %path% && cd '.$flag.'%path% && git remote add composer %url% && git fetch composer && git remote set-url origin %sanitizedUrl% && git remote set-url composer %sanitizedUrl%'; if (getenv('COMPOSER_DISABLE_NETWORK')) { throw new \RuntimeException('The required git reference for '.$package->getName().' is not in cache and network is disabled, aborting'); } @@ -87,11 +87,12 @@ class GitDownloader extends VcsDownloader implements DvcsDownloaderInterface $commandCallable = function ($url) use ($path, $command, $cachePath) { return str_replace( - array('%url%', '%path%', '%cachePath%'), + array('%url%', '%path%', '%cachePath%', '%sanitizedUrl%'), array( ProcessExecutor::escape($url), ProcessExecutor::escape($path), ProcessExecutor::escape($cachePath), + ProcessExecutor::escape(preg_replace('{://([^@]+?):(.+?)@}', '://', $url)), ), $command ); @@ -129,10 +130,10 @@ class GitDownloader extends VcsDownloader implements DvcsDownloaderInterface if (!empty($this->cachedPackages[$target->getId()][$ref])) { $msg = "Checking out ".$this->getShortHash($ref).' from cache'; - $command = 'git rev-parse --quiet --verify %ref% || (git remote set-url composer %cachePath% && git fetch composer && git fetch --tags composer); git remote set-url composer %url%'; + $command = 'git rev-parse --quiet --verify %ref% || (git remote set-url composer %cachePath% && git fetch composer && git fetch --tags composer); git remote set-url composer %sanitizedUrl%'; } else { $msg = "Checking out ".$this->getShortHash($ref); - $command = 'git remote set-url composer %url% && git rev-parse --quiet --verify %ref% || (git fetch composer && git fetch --tags composer)'; + $command = 'git remote set-url composer %url% && git rev-parse --quiet --verify %ref% || (git fetch composer && git fetch --tags composer); git remote set-url composer %sanitizedUrl%'; if (getenv('COMPOSER_DISABLE_NETWORK')) { throw new \RuntimeException('The required git reference for '.$target->getName().' is not in cache and network is disabled, aborting'); } @@ -142,11 +143,12 @@ class GitDownloader extends VcsDownloader implements DvcsDownloaderInterface $commandCallable = function ($url) use ($ref, $command, $cachePath) { return str_replace( - array('%url%', '%ref%', '%cachePath%'), + array('%url%', '%ref%', '%cachePath%', '%sanitizedUrl%'), array( ProcessExecutor::escape($url), ProcessExecutor::escape($ref.'^{commit}'), ProcessExecutor::escape($cachePath), + ProcessExecutor::escape(preg_replace('{://([^@]+?):(.+?)@}', '://', $url)), ), $command ); diff --git a/src/Composer/Util/AuthHelper.php b/src/Composer/Util/AuthHelper.php index 7e51730e2..7b8d33d68 100644 --- a/src/Composer/Util/AuthHelper.php +++ b/src/Composer/Util/AuthHelper.php @@ -117,7 +117,7 @@ class AuthHelper $message = "\n".'Could not fetch '.$url.', enter your ' . $origin . ' credentials ' .($statusCode === 401 ? 'to access private repos' : 'to go over the API rate limit'); $gitLabUtil = new GitLab($this->io, $this->config, null); - if ($this->io->hasAuthentication($origin) && ($auth = $this->io->getAuthentication($origin)) && $auth['password'] === 'private-token') { + if ($this->io->hasAuthentication($origin) && ($auth = $this->io->getAuthentication($origin)) && in_array($auth['password'], array('gitlab-ci-token', 'private-token'), true)) { throw new TransportException("Invalid credentials for '" . $url . "', aborting.", $statusCode); } @@ -203,7 +203,7 @@ class AuthHelper if ($auth['password'] === 'oauth2') { $headers[] = 'Authorization: Bearer '.$auth['username']; $authenticationDisplayMessage = 'Using GitLab OAuth token authentication'; - } elseif ($auth['password'] === 'private-token') { + } elseif ($auth['password'] === 'private-token' || $auth['password'] === 'gitlab-ci-token') { $headers[] = 'PRIVATE-TOKEN: '.$auth['username']; $authenticationDisplayMessage = 'Using GitLab private token authentication'; } diff --git a/src/Composer/Util/Git.php b/src/Composer/Util/Git.php index dd8a5ae79..39a66f898 100644 --- a/src/Composer/Util/Git.php +++ b/src/Composer/Util/Git.php @@ -54,9 +54,9 @@ class Git } if (!$initialClone) { - // capture username/password from URL if there is one + // capture username/password from URL if there is one and we have no auth configured yet $this->process->execute('git remote -v', $output, $cwd); - if (preg_match('{^(?:composer|origin)\s+https?://(.+):(.+)@([^/]+)}im', $output, $match)) { + if (preg_match('{^(?:composer|origin)\s+https?://(.+):(.+)@([^/]+)}im', $output, $match) && !$this->io->hasAuthentication($match[3])) { $this->io->setAuthentication($match[3], rawurldecode($match[1]), rawurldecode($match[2])); } } @@ -95,8 +95,10 @@ class Git $auth = null; if ($bypassSshForGitHub || 0 !== $this->process->execute($command, $ignoredOutput, $cwd)) { - // private github repository without git access, try https with auth - if (preg_match('{^git@' . self::getGitHubDomainsRegex($this->config) . ':(.+?)\.git$}i', $url, $match)) { + // private github repository without ssh key access, try https with auth + if (preg_match('{^git@' . self::getGitHubDomainsRegex($this->config) . ':(.+?)\.git$}i', $url, $match) + || preg_match('{^(https?)://' . self::getGitHubDomainsRegex($this->config) . '/(.*)}', $url, $match) + ) { if (!$this->io->hasAuthentication($match[1])) { $gitHubUtil = new GitHub($this->io, $this->config, $this->process); $message = 'Cloning failed using an ssh key for authentication, enter your GitHub credentials to access private repos'; @@ -153,7 +155,14 @@ class Git return; } } - } elseif (preg_match('{^(https?)://' . self::getGitLabDomainsRegex($this->config) . '/(.*)}', $url, $match)) { + } elseif ( + preg_match('{^(git)@' . self::getGitLabDomainsRegex($this->config) . ':(.+?)\.git$}i', $url, $match) + || preg_match('{^(https?)://' . self::getGitLabDomainsRegex($this->config) . '/(.*)}', $url, $match) + ) { + if ($match[1] === 'git') { + $match[1] = 'https'; + } + if (!$this->io->hasAuthentication($match[2])) { $gitLabUtil = new GitLab($this->io, $this->config, $this->process); $message = 'Cloning failed, enter your GitLab credentials to access private repos'; @@ -165,17 +174,18 @@ class Git if ($this->io->hasAuthentication($match[2])) { $auth = $this->io->getAuthentication($match[2]); - if($auth['password'] === 'private-token' || $auth['password'] === 'oauth2') { + if ($auth['password'] === 'private-token' || $auth['password'] === 'oauth2' || $auth['password'] === 'gitlab-ci-token') { $authUrl = $match[1] . '://' . rawurlencode($auth['password']) . ':' . rawurlencode($auth['username']) . '@' . $match[2] . '/' . $match[3]; // swap username and password } else { $authUrl = $match[1] . '://' . rawurlencode($auth['username']) . ':' . rawurlencode($auth['password']) . '@' . $match[2] . '/' . $match[3]; } + $command = call_user_func($commandCallable, $authUrl); if (0 === $this->process->execute($command, $ignoredOutput, $cwd)) { return; } } - } elseif ($this->isAuthenticationFailure($url, $match)) { // private non-github repo that failed to authenticate + } elseif ($this->isAuthenticationFailure($url, $match)) { // private non-github/gitlab/bitbucket repo that failed to authenticate if (strpos($match[2], '@')) { list($authParts, $match[2]) = explode('@', $match[2], 2); } @@ -215,10 +225,12 @@ class Git } } + $errorMsg = $this->process->getErrorOutput(); if ($initialClone) { $this->filesystem->removeDirectory($origCwd); } - $this->throwException('Failed to execute ' . $command . "\n\n" . $this->process->getErrorOutput(), $url); + + $this->throwException('Failed to execute ' . $command . "\n\n" . $errorMsg, $url); } } @@ -232,7 +244,9 @@ class Git if (is_dir($dir) && 0 === $this->process->execute('git rev-parse --git-dir', $output, $dir) && trim($output) === '.') { try { $commandCallable = function ($url) { - return sprintf('git remote set-url origin %s && git remote update --prune origin', ProcessExecutor::escape($url)); + $sanitizedUrl = preg_replace('{://([^@]+?):(.+?)@}', '://', $url); + + return sprintf('git remote set-url origin %s && git remote update --prune origin && git remote set-url origin %s', ProcessExecutor::escape($url), ProcessExecutor::escape($sanitizedUrl)); }; $this->runCommand($commandCallable, $url, $dir); } catch (\Exception $e) { @@ -255,16 +269,29 @@ class Git } public function fetchRefOrSyncMirror($url, $dir, $ref) + { + if ($this->checkRefIsInMirror($url, $dir, $ref)) { + return true; + } + + if ($this->syncMirror($url, $dir)) { + return $this->checkRefIsInMirror($url, $dir, $ref); + } + + return false; + } + + private function checkRefIsInMirror($url, $dir, $ref) { if (is_dir($dir) && 0 === $this->process->execute('git rev-parse --git-dir', $output, $dir) && trim($output) === '.') { $escapedRef = ProcessExecutor::escape($ref.'^{commit}'); - $exitCode = $this->process->execute(sprintf('git rev-parse --quiet --verify %s', $escapedRef), $output, $dir); + $exitCode = $this->process->execute(sprintf('git rev-parse --quiet --verify %s', $escapedRef), $ignoredOutput, $dir); if ($exitCode === 0) { return true; } } - return $this->syncMirror($url, $dir); + return false; } private function isAuthenticationFailure($url, &$match) diff --git a/tests/Composer/Test/Downloader/GitDownloaderTest.php b/tests/Composer/Test/Downloader/GitDownloaderTest.php index bf1402186..36ae3c364 100644 --- a/tests/Composer/Test/Downloader/GitDownloaderTest.php +++ b/tests/Composer/Test/Downloader/GitDownloaderTest.php @@ -112,7 +112,7 @@ class GitDownloaderTest extends TestCase return 0; })); - $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://example.com/composer/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://example.com/composer/composer' && git fetch composer"); + $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://example.com/composer/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://example.com/composer/composer' && git fetch composer && git remote set-url origin 'https://example.com/composer/composer' && git remote set-url composer 'https://example.com/composer/composer'"); $processExecutor->expects($this->at(1)) ->method('execute') ->with($this->equalTo($expectedGitCommand)) @@ -170,6 +170,9 @@ class GitDownloaderTest extends TestCase $this->setupConfig($config); $cachePath = $config->get('cache-vcs-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', 'https://example.com/composer/composer').'/'; + $filesystem = new \Composer\Util\Filesystem; + $filesystem->removeDirectory($cachePath); + $expectedGitCommand = $this->winCompat(sprintf("git clone --mirror 'https://example.com/composer/composer' '%s'", $cachePath)); $processExecutor->expects($this->at(1)) ->method('execute') @@ -179,24 +182,36 @@ class GitDownloaderTest extends TestCase return 0; })); + $processExecutor->expects($this->at(2)) + ->method('execute') + ->with($this->equalTo('git rev-parse --git-dir'), $this->anything(), $this->equalTo($this->winCompat($cachePath))) + ->will($this->returnCallback(function ($command, &$output = null) { + $output = '.'; + + return 0; + })); + $processExecutor->expects($this->at(3)) + ->method('execute') + ->with($this->equalTo($this->winCompat('git rev-parse --quiet --verify \'1234567890123456789012345678901234567890^{commit}\'')), $this->equalTo(null), $this->equalTo($this->winCompat($cachePath))) + ->will($this->returnValue(0)); $expectedGitCommand = $this->winCompat(sprintf("git clone --no-checkout '%1\$s' 'composerPath' --dissociate --reference '%1\$s' && cd 'composerPath' && git remote set-url origin 'https://example.com/composer/composer' && git remote add composer 'https://example.com/composer/composer'", $cachePath)); - $processExecutor->expects($this->at(2)) + $processExecutor->expects($this->at(4)) ->method('execute') ->with($this->equalTo($expectedGitCommand)) ->will($this->returnValue(0)); - $processExecutor->expects($this->at(3)) + $processExecutor->expects($this->at(5)) ->method('execute') ->with($this->equalTo($this->winCompat("git branch -r")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath'))) ->will($this->returnValue(0)); - $processExecutor->expects($this->at(4)) + $processExecutor->expects($this->at(6)) ->method('execute') ->with($this->equalTo($this->winCompat("git checkout 'master' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath'))) ->will($this->returnValue(0)); - $processExecutor->expects($this->at(5)) + $processExecutor->expects($this->at(7)) ->method('execute') ->with($this->equalTo($this->winCompat("git reset --hard '1234567890123456789012345678901234567890' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath'))) ->will($this->returnValue(0)); @@ -235,7 +250,7 @@ class GitDownloaderTest extends TestCase return 0; })); - $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://github.com/mirrors/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://github.com/mirrors/composer' && git fetch composer"); + $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://github.com/mirrors/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://github.com/mirrors/composer' && git fetch composer && git remote set-url origin 'https://github.com/mirrors/composer' && git remote set-url composer 'https://github.com/mirrors/composer'"); $processExecutor->expects($this->at(1)) ->method('execute') ->with($this->equalTo($expectedGitCommand)) @@ -246,7 +261,7 @@ class GitDownloaderTest extends TestCase ->with() ->will($this->returnValue('Error1')); - $expectedGitCommand = $this->winCompat("git clone --no-checkout 'git@github.com:mirrors/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'git@github.com:mirrors/composer' && git fetch composer"); + $expectedGitCommand = $this->winCompat("git clone --no-checkout 'git@github.com:mirrors/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'git@github.com:mirrors/composer' && git fetch composer && git remote set-url origin 'git@github.com:mirrors/composer' && git remote set-url composer 'git@github.com:mirrors/composer'"); $processExecutor->expects($this->at(3)) ->method('execute') ->with($this->equalTo($expectedGitCommand)) @@ -322,7 +337,7 @@ class GitDownloaderTest extends TestCase return 0; })); - $expectedGitCommand = $this->winCompat("git clone --no-checkout '{$url}' 'composerPath' && cd 'composerPath' && git remote add composer '{$url}' && git fetch composer"); + $expectedGitCommand = $this->winCompat("git clone --no-checkout '{$url}' 'composerPath' && cd 'composerPath' && git remote add composer '{$url}' && git fetch composer && git remote set-url origin '{$url}' && git remote set-url composer '{$url}'"); $processExecutor->expects($this->at(1)) ->method('execute') ->with($this->equalTo($expectedGitCommand)) @@ -350,7 +365,7 @@ class GitDownloaderTest extends TestCase public function testDownloadThrowsRuntimeExceptionIfGitCommandFails() { - $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://example.com/composer/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://example.com/composer/composer' && git fetch composer"); + $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://example.com/composer/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://example.com/composer/composer' && git fetch composer && git remote set-url origin 'https://example.com/composer/composer' && git remote set-url composer 'https://example.com/composer/composer'"); $packageMock = $this->getMockBuilder('Composer\Package\PackageInterface')->getMock(); $packageMock->expects($this->any()) ->method('getSourceReference') @@ -408,7 +423,7 @@ class GitDownloaderTest extends TestCase public function testUpdate() { - $expectedGitUpdateCommand = $this->winCompat("git remote set-url composer 'https://github.com/composer/composer' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer)"); + $expectedGitUpdateCommand = $this->winCompat("git remote set-url composer 'https://github.com/composer/composer' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer); git remote set-url composer 'https://github.com/composer/composer'"); $packageMock = $this->getMockBuilder('Composer\Package\PackageInterface')->getMock(); $packageMock->expects($this->any()) @@ -440,7 +455,7 @@ class GitDownloaderTest extends TestCase public function testUpdateWithNewRepoUrl() { - $expectedGitUpdateCommand = $this->winCompat("git remote set-url composer 'https://github.com/composer/composer' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer)"); + $expectedGitUpdateCommand = $this->winCompat("git remote set-url composer 'https://github.com/composer/composer' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer); git remote set-url composer 'https://github.com/composer/composer'"); $packageMock = $this->getMockBuilder('Composer\Package\PackageInterface')->getMock(); $packageMock->expects($this->any()) @@ -519,8 +534,8 @@ composer https://github.com/old/url (push) */ public function testUpdateThrowsRuntimeExceptionIfGitCommandFails() { - $expectedGitUpdateCommand = $this->winCompat("git remote set-url composer 'https://github.com/composer/composer' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer)"); - $expectedGitUpdateCommand2 = $this->winCompat("git remote set-url composer 'git@github.com:composer/composer' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer)"); + $expectedGitUpdateCommand = $this->winCompat("git remote set-url composer 'https://github.com/composer/composer' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer); git remote set-url composer 'https://github.com/composer/composer'"); + $expectedGitUpdateCommand2 = $this->winCompat("git remote set-url composer 'git@github.com:composer/composer' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer); git remote set-url composer 'git@github.com:composer/composer'"); $packageMock = $this->getMockBuilder('Composer\Package\PackageInterface')->getMock(); $packageMock->expects($this->any()) @@ -563,8 +578,8 @@ composer https://github.com/old/url (push) public function testUpdateDoesntThrowsRuntimeExceptionIfGitCommandFailsAtFirstButIsAbleToRecover() { - $expectedFirstGitUpdateCommand = $this->winCompat("git remote set-url composer '".(Platform::isWindows() ? 'C:\\' : '/')."' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer)"); - $expectedSecondGitUpdateCommand = $this->winCompat("git remote set-url composer 'https://github.com/composer/composer' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer)"); + $expectedFirstGitUpdateCommand = $this->winCompat("git remote set-url composer '".(Platform::isWindows() ? 'C:\\' : '/')."' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer); git remote set-url composer '/'"); + $expectedSecondGitUpdateCommand = $this->winCompat("git remote set-url composer 'https://github.com/composer/composer' && git rev-parse --quiet --verify 'ref^{commit}' || (git fetch composer && git fetch --tags composer); git remote set-url composer 'https://github.com/composer/composer'"); $packageMock = $this->getMockBuilder('Composer\Package\PackageInterface')->getMock(); $packageMock->expects($this->any())