Merge remote-tracking branch 'marvil07/faster-git-clones-using-cache'

2016-07-02 16:43:19 +01:00 · 2016-07-02 16:43:19 +01:00 · 55e90093a4
parent 59596ff012 869b3ef954
commit 55e90093a4
3 changed files with 151 additions and 22 deletions
--- a/src/Composer/Downloader/GitDownloader.php
+++ b/src/Composer/Downloader/GitDownloader.php
@ -42,10 +42,24 @@ class GitDownloader extends VcsDownloader implements DvcsDownloaderInterface
    {
        GitUtil::cleanEnv();
        $path = $this->normalizePath($path);
-
+        $cachePath = $this->config->get('cache-vcs-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', $url).'/';
+        $cacheOptions = '';
        $ref = $package->getSourceReference();
        $flag = Platform::isWindows() ? '/D ' : '';
-        $command = 'git clone --no-checkout %s %s && cd '.$flag.'%2$s && git remote add composer %1$s && git fetch composer';
+
+        // --dissociate option is only available since git 2.3.0-rc0
+        if (version_compare($this->gitUtil->getVersion(), '2.3.0-rc0', '>=')) {
+            if (!file_exists($cachePath)) {
+                $this->io->writeError(sprintf('    Cloning to cache at %s', ProcessExecutor::escape($cachePath)));
+                $mirrorCommand = 'git clone --mirror %s %s';
+                $mirrorCommandCallable = function ($url) use ($cachePath, $mirrorCommand) {
+                    return sprintf($mirrorCommand, ProcessExecutor::escape($url), ProcessExecutor::escape($cachePath));
+                };
+                $this->gitUtil->runCommand($mirrorCommandCallable, $url, $path, true);
+            }
+            $cacheOptions = sprintf('--dissociate --reference %s ', ProcessExecutor::escape($cachePath));
+        }
+        $command = 'git clone --no-checkout %s %s '.$cacheOptions.'&& cd '.$flag.'%2$s && git remote add composer %1$s && git fetch composer';
        $this->io->writeError("    Cloning ".$ref);

        $commandCallable = function ($url) use ($ref, $path, $command) {
--- a/src/Composer/Util/Git.php
+++ b/src/Composer/Util/Git.php
@ -273,4 +273,20 @@ class Git

        throw new \RuntimeException(self::sanitizeUrl($message));
    }
+
+    /**
+     * Retrieves the current git version.
+     *
+     * @return string
+     *   The git version number.
+     */
+    public function getVersion() {
+        if (0 !== $this->process->execute('git --version', $output)) {
+            throw new \RuntimeException(self::sanitizeUrl('Failed retrieve git version, git was not found, check that it is installed and in your PATH env.' . "\n\n" . $this->process->getErrorOutput()));
+        }
+        if (preg_match('/^git version (.*)/', $output, $matches) !== 1) {
+            throw new \RuntimeException('git --version output seems to have changed, expected "git version x.y.z".');
+        }
+        return $matches[1];
+    }
 }
--- a/tests/Composer/Test/Downloader/GitDownloaderTest.php
+++ b/tests/Composer/Test/Downloader/GitDownloaderTest.php
@ -38,14 +38,23 @@ class GitDownloaderTest extends TestCase
        }
    }

+    protected function setupConfig($config = null) {
+        if (!$config) {
+            $config = new Config();
+        }
+        if (!$config->has('home')) {
+            $tmpDir = realpath(sys_get_temp_dir()).DIRECTORY_SEPARATOR.'cmptest-'.md5(uniqid('', true));
+            $config->merge(array('config' => array('home' => $tmpDir)));
+        }
+        return $config;
+    }
+
    protected function getDownloaderMock($io = null, $config = null, $executor = null, $filesystem = null)
    {
        $io = $io ?: $this->getMock('Composer\IO\IOInterface');
        $executor = $executor ?: $this->getMock('Composer\Util\ProcessExecutor');
        $filesystem = $filesystem ?: $this->getMock('Composer\Util\Filesystem');
-        if (!$config) {
-            $config = new Config();
-        }
+        $config = $this->setupConfig($config);

        return new GitDownloader($io, $config, $executor, $filesystem);
    }
@ -81,23 +90,31 @@ class GitDownloaderTest extends TestCase
            ->will($this->returnValue('dev-master'));
        $processExecutor = $this->getMock('Composer\Util\ProcessExecutor');

-        $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://example.com/composer/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://example.com/composer/composer' && git fetch composer");
        $processExecutor->expects($this->at(0))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat('git --version')))
+            ->will($this->returnCallback(function($command, &$output = null) {
+                $output = 'git version 1.0.0';
+                return 0;
+            }));
+
+        $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://example.com/composer/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://example.com/composer/composer' && git fetch composer");
+        $processExecutor->expects($this->at(1))
            ->method('execute')
            ->with($this->equalTo($expectedGitCommand))
            ->will($this->returnValue(0));

-        $processExecutor->expects($this->at(1))
+        $processExecutor->expects($this->at(2))
            ->method('execute')
            ->with($this->equalTo($this->winCompat("git branch -r")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
            ->will($this->returnValue(0));

-        $processExecutor->expects($this->at(2))
+        $processExecutor->expects($this->at(3))
            ->method('execute')
            ->with($this->equalTo($this->winCompat("git checkout 'master' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
            ->will($this->returnValue(0));

-        $processExecutor->expects($this->at(3))
+        $processExecutor->expects($this->at(4))
            ->method('execute')
            ->with($this->equalTo($this->winCompat("git reset --hard '1234567890123456789012345678901234567890' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
            ->will($this->returnValue(0));
@ -106,6 +123,65 @@ class GitDownloaderTest extends TestCase
        $downloader->download($packageMock, 'composerPath');
    }

+    public function testDownloadWithCache()
+    {
+        $packageMock = $this->getMock('Composer\Package\PackageInterface');
+        $packageMock->expects($this->any())
+            ->method('getSourceReference')
+            ->will($this->returnValue('1234567890123456789012345678901234567890'));
+        $packageMock->expects($this->any())
+            ->method('getSourceUrls')
+            ->will($this->returnValue(array('https://example.com/composer/composer')));
+        $packageMock->expects($this->any())
+            ->method('getSourceUrl')
+            ->will($this->returnValue('https://example.com/composer/composer'));
+        $packageMock->expects($this->any())
+            ->method('getPrettyVersion')
+            ->will($this->returnValue('dev-master'));
+        $processExecutor = $this->getMock('Composer\Util\ProcessExecutor');
+
+        $processExecutor->expects($this->at(0))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat('git --version')))
+            ->will($this->returnCallback(function($command, &$output = null) {
+                $output = 'git version 2.3.1';
+                return 0;
+            }));
+
+        $config = new Config;
+        $this->setupConfig($config);
+        $cachePath = $config->get('cache-vcs-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', 'https://example.com/composer/composer').'/';
+        $expectedGitCommand = $this->winCompat(sprintf("git clone --mirror 'https://example.com/composer/composer' '%s'", $cachePath));
+        $processExecutor->expects($this->at(1))
+            ->method('execute')
+            ->with($this->equalTo($expectedGitCommand))
+            ->will($this->returnValue(0));
+
+        $expectedGitCommand = $this->winCompat(sprintf("git clone --no-checkout 'https://example.com/composer/composer' 'composerPath' --dissociate --reference '%s' && cd 'composerPath' && git remote add composer 'https://example.com/composer/composer' && git fetch composer", $cachePath));
+        $processExecutor->expects($this->at(2))
+            ->method('execute')
+            ->with($this->equalTo($expectedGitCommand))
+            ->will($this->returnValue(0));
+
+        $processExecutor->expects($this->at(3))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat("git branch -r")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
+            ->will($this->returnValue(0));
+
+        $processExecutor->expects($this->at(4))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat("git checkout 'master' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
+            ->will($this->returnValue(0));
+
+        $processExecutor->expects($this->at(5))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat("git reset --hard '1234567890123456789012345678901234567890' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
+            ->will($this->returnValue(0));
+
+        $downloader = $this->getDownloaderMock(null, $config, $processExecutor);
+        $downloader->download($packageMock, 'composerPath');
+    }
+
    public function testDownloadUsesVariousProtocolsAndSetsPushUrlForGithub()
    {
        $packageMock = $this->getMock('Composer\Package\PackageInterface');
@ -123,41 +199,49 @@ class GitDownloaderTest extends TestCase
            ->will($this->returnValue('1.0.0'));
        $processExecutor = $this->getMock('Composer\Util\ProcessExecutor');

-        $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://github.com/mirrors/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://github.com/mirrors/composer' && git fetch composer");
        $processExecutor->expects($this->at(0))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat('git --version')))
+            ->will($this->returnCallback(function($command, &$output = null) {
+                $output = 'git version 1.0.0';
+                return 0;
+            }));
+
+        $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://github.com/mirrors/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://github.com/mirrors/composer' && git fetch composer");
+        $processExecutor->expects($this->at(1))
            ->method('execute')
            ->with($this->equalTo($expectedGitCommand))
            ->will($this->returnValue(1));

-        $processExecutor->expects($this->at(1))
+        $processExecutor->expects($this->at(2))
            ->method('getErrorOutput')
            ->with()
            ->will($this->returnValue('Error1'));

        $expectedGitCommand = $this->winCompat("git clone --no-checkout 'git@github.com:mirrors/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'git@github.com:mirrors/composer' && git fetch composer");
-        $processExecutor->expects($this->at(2))
+        $processExecutor->expects($this->at(3))
            ->method('execute')
            ->with($this->equalTo($expectedGitCommand))
            ->will($this->returnValue(0));

        $expectedGitCommand = $this->winCompat("git remote set-url origin 'https://github.com/composer/composer'");
-        $processExecutor->expects($this->at(3))
-            ->method('execute')
-            ->with($this->equalTo($expectedGitCommand), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
-            ->will($this->returnValue(0));
-
-        $expectedGitCommand = $this->winCompat("git remote set-url --push origin 'git@github.com:composer/composer.git'");
        $processExecutor->expects($this->at(4))
            ->method('execute')
            ->with($this->equalTo($expectedGitCommand), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
            ->will($this->returnValue(0));

+        $expectedGitCommand = $this->winCompat("git remote set-url --push origin 'git@github.com:composer/composer.git'");
        $processExecutor->expects($this->at(5))
+            ->method('execute')
+            ->with($this->equalTo($expectedGitCommand), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
+            ->will($this->returnValue(0));
+
+        $processExecutor->expects($this->at(6))
            ->method('execute')
            ->with($this->equalTo('git branch -r'))
            ->will($this->returnValue(0));

-        $processExecutor->expects($this->at(6))
+        $processExecutor->expects($this->at(7))
            ->method('execute')
            ->with($this->equalTo($this->winCompat("git checkout 'ref' -- && git reset --hard 'ref' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
            ->will($this->returnValue(0));
@ -198,19 +282,27 @@ class GitDownloaderTest extends TestCase
            ->will($this->returnValue('1.0.0'));
        $processExecutor = $this->getMock('Composer\Util\ProcessExecutor');

-        $expectedGitCommand = $this->winCompat("git clone --no-checkout '{$url}' 'composerPath' && cd 'composerPath' && git remote add composer '{$url}' && git fetch composer");
        $processExecutor->expects($this->at(0))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat('git --version')))
+            ->will($this->returnCallback(function($command, &$output = null) {
+                $output = 'git version 1.0.0';
+                return 0;
+            }));
+
+        $expectedGitCommand = $this->winCompat("git clone --no-checkout '{$url}' 'composerPath' && cd 'composerPath' && git remote add composer '{$url}' && git fetch composer");
+        $processExecutor->expects($this->at(1))
            ->method('execute')
            ->with($this->equalTo($expectedGitCommand))
            ->will($this->returnValue(0));

        $expectedGitCommand = $this->winCompat("git remote set-url --push origin '{$pushUrl}'");
-        $processExecutor->expects($this->at(1))
+        $processExecutor->expects($this->at(2))
            ->method('execute')
            ->with($this->equalTo($expectedGitCommand), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
            ->will($this->returnValue(0));

-        $processExecutor->expects($this->exactly(4))
+        $processExecutor->expects($this->exactly(5))
            ->method('execute')
            ->will($this->returnValue(0));

@ -236,6 +328,13 @@ class GitDownloaderTest extends TestCase
            ->will($this->returnValue(array('https://example.com/composer/composer')));
        $processExecutor = $this->getMock('Composer\Util\ProcessExecutor');
        $processExecutor->expects($this->at(0))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat('git --version')))
+            ->will($this->returnCallback(function($command, &$output = null) {
+                $output = 'git version 1.0.0';
+                return 0;
+            }));
+        $processExecutor->expects($this->at(1))
            ->method('execute')
            ->with($this->equalTo($expectedGitCommand))
            ->will($this->returnValue(1));