diff --git a/src/Composer/Downloader/FileDownloader.php b/src/Composer/Downloader/FileDownloader.php
index da2955638..6a08bd67d 100644
--- a/src/Composer/Downloader/FileDownloader.php
+++ b/src/Composer/Downloader/FileDownloader.php
@@ -175,7 +175,9 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
 
         $reject = function ($e) use ($io, &$urls, $download, $fileName, $path, $package, &$retries, $filesystem, $self) {
             // clean up
-            $filesystem->removeDirectory($path);
+            if (file_exists($fileName)) {
+                $filesystem->unlink($fileName);
+            }
             $self->clearLastCacheWrite($package);
 
             if ($e instanceof TransportException) {