Merge remote-tracking branch 'marvil07/faster-git-clones-using-cache'

src/Composer/Downloader/GitDownloader.php

@@ -42,10 +42,24 @@ class GitDownloader extends VcsDownloader implements DvcsDownloaderInterface
     {
         GitUtil::cleanEnv();
         $path = $this->normalizePath($path);
-
+        $cachePath = $this->config->get('cache-vcs-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', $url).'/';
+        $cacheOptions = '';
         $ref = $package->getSourceReference();
         $flag = Platform::isWindows() ? '/D ' : '';
-        $command = 'git clone --no-checkout %s %s && cd '.$flag.'%2$s && git remote add composer %1$s && git fetch composer';
+
+        // --dissociate option is only available since git 2.3.0-rc0
+        if (version_compare($this->gitUtil->getVersion(), '2.3.0-rc0', '>=')) {
+            if (!file_exists($cachePath)) {
+                $this->io->writeError(sprintf('    Cloning to cache at %s', ProcessExecutor::escape($cachePath)));
+                $mirrorCommand = 'git clone --mirror %s %s';
+                $mirrorCommandCallable = function ($url) use ($cachePath, $mirrorCommand) {
+                    return sprintf($mirrorCommand, ProcessExecutor::escape($url), ProcessExecutor::escape($cachePath));
+                };
+                $this->gitUtil->runCommand($mirrorCommandCallable, $url, $path, true);
+            }
+            $cacheOptions = sprintf('--dissociate --reference %s ', ProcessExecutor::escape($cachePath));
+        }
+        $command = 'git clone --no-checkout %s %s '.$cacheOptions.'&& cd '.$flag.'%2$s && git remote add composer %1$s && git fetch composer';
         $this->io->writeError("    Cloning ".$ref);
 
         $commandCallable = function ($url) use ($ref, $path, $command) {

src/Composer/Util/Git.php

@@ -273,4 +273,20 @@ class Git
 
         throw new \RuntimeException(self::sanitizeUrl($message));
     }
+
+    /**
+     * Retrieves the current git version.
+     *
+     * @return string
+     *   The git version number.
+     */
+    public function getVersion() {
+        if (0 !== $this->process->execute('git --version', $output)) {
+            throw new \RuntimeException(self::sanitizeUrl('Failed retrieve git version, git was not found, check that it is installed and in your PATH env.' . "\n\n" . $this->process->getErrorOutput()));
+        }
+        if (preg_match('/^git version (.*)/', $output, $matches) !== 1) {
+            throw new \RuntimeException('git --version output seems to have changed, expected "git version x.y.z".');
+        }
+        return $matches[1];
+    }
 }

tests/Composer/Test/Downloader/GitDownloaderTest.php

@@ -38,14 +38,23 @@ class GitDownloaderTest extends TestCase
         }
     }
 
+    protected function setupConfig($config = null) {
+        if (!$config) {
+            $config = new Config();
+        }
+        if (!$config->has('home')) {
+            $tmpDir = realpath(sys_get_temp_dir()).DIRECTORY_SEPARATOR.'cmptest-'.md5(uniqid('', true));
+            $config->merge(array('config' => array('home' => $tmpDir)));
+        }
+        return $config;
+    }
+
     protected function getDownloaderMock($io = null, $config = null, $executor = null, $filesystem = null)
     {
         $io = $io ?: $this->getMock('Composer\IO\IOInterface');
         $executor = $executor ?: $this->getMock('Composer\Util\ProcessExecutor');
         $filesystem = $filesystem ?: $this->getMock('Composer\Util\Filesystem');
-        if (!$config) {
+        $config = $this->setupConfig($config);
-            $config = new Config();
-        }
 
         return new GitDownloader($io, $config, $executor, $filesystem);
     }
@@ -81,23 +90,31 @@ class GitDownloaderTest extends TestCase
             ->will($this->returnValue('dev-master'));
         $processExecutor = $this->getMock('Composer\Util\ProcessExecutor');
 
-        $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://example.com/composer/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://example.com/composer/composer' && git fetch composer");
         $processExecutor->expects($this->at(0))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat('git --version')))
+            ->will($this->returnCallback(function($command, &$output = null) {
+                $output = 'git version 1.0.0';
+                return 0;
+            }));
+
+        $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://example.com/composer/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://example.com/composer/composer' && git fetch composer");
+        $processExecutor->expects($this->at(1))
             ->method('execute')
             ->with($this->equalTo($expectedGitCommand))
             ->will($this->returnValue(0));
 
-        $processExecutor->expects($this->at(1))
+        $processExecutor->expects($this->at(2))
             ->method('execute')
             ->with($this->equalTo($this->winCompat("git branch -r")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
             ->will($this->returnValue(0));
 
-        $processExecutor->expects($this->at(2))
+        $processExecutor->expects($this->at(3))
             ->method('execute')
             ->with($this->equalTo($this->winCompat("git checkout 'master' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
             ->will($this->returnValue(0));
 
-        $processExecutor->expects($this->at(3))
+        $processExecutor->expects($this->at(4))
             ->method('execute')
             ->with($this->equalTo($this->winCompat("git reset --hard '1234567890123456789012345678901234567890' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
             ->will($this->returnValue(0));
@@ -106,6 +123,65 @@ class GitDownloaderTest extends TestCase
         $downloader->download($packageMock, 'composerPath');
     }
 
+    public function testDownloadWithCache()
+    {
+        $packageMock = $this->getMock('Composer\Package\PackageInterface');
+        $packageMock->expects($this->any())
+            ->method('getSourceReference')
+            ->will($this->returnValue('1234567890123456789012345678901234567890'));
+        $packageMock->expects($this->any())
+            ->method('getSourceUrls')
+            ->will($this->returnValue(array('https://example.com/composer/composer')));
+        $packageMock->expects($this->any())
+            ->method('getSourceUrl')
+            ->will($this->returnValue('https://example.com/composer/composer'));
+        $packageMock->expects($this->any())
+            ->method('getPrettyVersion')
+            ->will($this->returnValue('dev-master'));
+        $processExecutor = $this->getMock('Composer\Util\ProcessExecutor');
+
+        $processExecutor->expects($this->at(0))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat('git --version')))
+            ->will($this->returnCallback(function($command, &$output = null) {
+                $output = 'git version 2.3.1';
+                return 0;
+            }));
+
+        $config = new Config;
+        $this->setupConfig($config);
+        $cachePath = $config->get('cache-vcs-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', 'https://example.com/composer/composer').'/';
+        $expectedGitCommand = $this->winCompat(sprintf("git clone --mirror 'https://example.com/composer/composer' '%s'", $cachePath));
+        $processExecutor->expects($this->at(1))
+            ->method('execute')
+            ->with($this->equalTo($expectedGitCommand))
+            ->will($this->returnValue(0));
+
+        $expectedGitCommand = $this->winCompat(sprintf("git clone --no-checkout 'https://example.com/composer/composer' 'composerPath' --dissociate --reference '%s' && cd 'composerPath' && git remote add composer 'https://example.com/composer/composer' && git fetch composer", $cachePath));
+        $processExecutor->expects($this->at(2))
+            ->method('execute')
+            ->with($this->equalTo($expectedGitCommand))
+            ->will($this->returnValue(0));
+
+        $processExecutor->expects($this->at(3))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat("git branch -r")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
+            ->will($this->returnValue(0));
+
+        $processExecutor->expects($this->at(4))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat("git checkout 'master' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
+            ->will($this->returnValue(0));
+
+        $processExecutor->expects($this->at(5))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat("git reset --hard '1234567890123456789012345678901234567890' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
+            ->will($this->returnValue(0));
+
+        $downloader = $this->getDownloaderMock(null, $config, $processExecutor);
+        $downloader->download($packageMock, 'composerPath');
+    }
+
     public function testDownloadUsesVariousProtocolsAndSetsPushUrlForGithub()
     {
         $packageMock = $this->getMock('Composer\Package\PackageInterface');
@@ -123,41 +199,49 @@ class GitDownloaderTest extends TestCase
             ->will($this->returnValue('1.0.0'));
         $processExecutor = $this->getMock('Composer\Util\ProcessExecutor');
 
-        $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://github.com/mirrors/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://github.com/mirrors/composer' && git fetch composer");
         $processExecutor->expects($this->at(0))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat('git --version')))
+            ->will($this->returnCallback(function($command, &$output = null) {
+                $output = 'git version 1.0.0';
+                return 0;
+            }));
+
+        $expectedGitCommand = $this->winCompat("git clone --no-checkout 'https://github.com/mirrors/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'https://github.com/mirrors/composer' && git fetch composer");
+        $processExecutor->expects($this->at(1))
             ->method('execute')
             ->with($this->equalTo($expectedGitCommand))
             ->will($this->returnValue(1));
 
-        $processExecutor->expects($this->at(1))
+        $processExecutor->expects($this->at(2))
             ->method('getErrorOutput')
             ->with()
             ->will($this->returnValue('Error1'));
 
         $expectedGitCommand = $this->winCompat("git clone --no-checkout 'git@github.com:mirrors/composer' 'composerPath' && cd 'composerPath' && git remote add composer 'git@github.com:mirrors/composer' && git fetch composer");
-        $processExecutor->expects($this->at(2))
+        $processExecutor->expects($this->at(3))
             ->method('execute')
             ->with($this->equalTo($expectedGitCommand))
             ->will($this->returnValue(0));
 
         $expectedGitCommand = $this->winCompat("git remote set-url origin 'https://github.com/composer/composer'");
-        $processExecutor->expects($this->at(3))
-            ->method('execute')
-            ->with($this->equalTo($expectedGitCommand), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
-            ->will($this->returnValue(0));
-
-        $expectedGitCommand = $this->winCompat("git remote set-url --push origin 'git@github.com:composer/composer.git'");
         $processExecutor->expects($this->at(4))
             ->method('execute')
             ->with($this->equalTo($expectedGitCommand), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
             ->will($this->returnValue(0));
 
+        $expectedGitCommand = $this->winCompat("git remote set-url --push origin 'git@github.com:composer/composer.git'");
         $processExecutor->expects($this->at(5))
+            ->method('execute')
+            ->with($this->equalTo($expectedGitCommand), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
+            ->will($this->returnValue(0));
+
+        $processExecutor->expects($this->at(6))
             ->method('execute')
             ->with($this->equalTo('git branch -r'))
             ->will($this->returnValue(0));
 
-        $processExecutor->expects($this->at(6))
+        $processExecutor->expects($this->at(7))
             ->method('execute')
             ->with($this->equalTo($this->winCompat("git checkout 'ref' -- && git reset --hard 'ref' --")), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
             ->will($this->returnValue(0));
@@ -198,19 +282,27 @@ class GitDownloaderTest extends TestCase
             ->will($this->returnValue('1.0.0'));
         $processExecutor = $this->getMock('Composer\Util\ProcessExecutor');
 
-        $expectedGitCommand = $this->winCompat("git clone --no-checkout '{$url}' 'composerPath' && cd 'composerPath' && git remote add composer '{$url}' && git fetch composer");
         $processExecutor->expects($this->at(0))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat('git --version')))
+            ->will($this->returnCallback(function($command, &$output = null) {
+                $output = 'git version 1.0.0';
+                return 0;
+            }));
+
+        $expectedGitCommand = $this->winCompat("git clone --no-checkout '{$url}' 'composerPath' && cd 'composerPath' && git remote add composer '{$url}' && git fetch composer");
+        $processExecutor->expects($this->at(1))
             ->method('execute')
             ->with($this->equalTo($expectedGitCommand))
             ->will($this->returnValue(0));
 
         $expectedGitCommand = $this->winCompat("git remote set-url --push origin '{$pushUrl}'");
-        $processExecutor->expects($this->at(1))
+        $processExecutor->expects($this->at(2))
             ->method('execute')
             ->with($this->equalTo($expectedGitCommand), $this->equalTo(null), $this->equalTo($this->winCompat('composerPath')))
             ->will($this->returnValue(0));
 
-        $processExecutor->expects($this->exactly(4))
+        $processExecutor->expects($this->exactly(5))
             ->method('execute')
             ->will($this->returnValue(0));
 
@@ -236,6 +328,13 @@ class GitDownloaderTest extends TestCase
             ->will($this->returnValue(array('https://example.com/composer/composer')));
         $processExecutor = $this->getMock('Composer\Util\ProcessExecutor');
         $processExecutor->expects($this->at(0))
+            ->method('execute')
+            ->with($this->equalTo($this->winCompat('git --version')))
+            ->will($this->returnCallback(function($command, &$output = null) {
+                $output = 'git version 1.0.0';
+                return 0;
+            }));
+        $processExecutor->expects($this->at(1))
             ->method('execute')
             ->with($this->equalTo($expectedGitCommand))
             ->will($this->returnValue(1));