diff --git a/src/Configurator/CopyFromRecipeConfigurator.php b/src/Configurator/CopyFromRecipeConfigurator.php
index 8671d0e91..059ae3e5d 100644
--- a/src/Configurator/CopyFromRecipeConfigurator.php
+++ b/src/Configurator/CopyFromRecipeConfigurator.php
@@ -24,7 +24,32 @@ public function configure(Recipe $recipe, $config, Lock $lock, array $options =
         $this->write('Copying files from recipe');
         $options = array_merge($this->options->toArray(), $options);
 
-        $lock->add($recipe->getName(), ['files' => $this->copyFiles($config, $recipe->getFiles(), $options)]);
+        $files = $this->copyFiles($config, $recipe->getFiles(), $options);
+        $lock->add($recipe->getName(), ['files' => $files]);
+
+        // Remove extra file only on force
+        if (!($options['force'] ?? false)) {
+            return;
+        }
+
+        // Recipe not installed yet
+        if (null === $recipeLock = $options['beforeState']) {
+            return;
+        }
+
+        $lockFiles = $recipeLock['files'] ?? null;
+        if (null === $lockFiles) {
+            return;
+        }
+
+        $lockFiles = array_flip($lockFiles);
+        foreach ($files as $file) {
+            if (isset($lockFiles[$file])) {
+                unset($lockFiles[$file]);
+            }
+        }
+
+        $this->removeFiles($config, $lockFiles, $this->options->get('root-dir'));
     }
 
     public function unconfigure(Recipe $recipe, $config, Lock $lock)
diff --git a/src/Flex.php b/src/Flex.php
index 3ee15d28b..5575107ea 100644
--- a/src/Flex.php
+++ b/src/Flex.php
@@ -64,6 +64,7 @@ class Flex implements PluginInterface, EventSubscriberInterface
     private $postInstallOutput = [''];
     private $operations = [];
     private $lock;
+    private $frozenLock;
     private $cacheDirPopulated = false;
     private $displayThanksReminder = 0;
     private $rfs;
@@ -133,6 +134,7 @@ class_exists(__NAMESPACE__.str_replace('/', '\\', substr($file, \strlen(__DIR__)
         $composer->setRepositoryManager($manager);
         $this->configurator = new Configurator($composer, $io, $this->options);
         $this->lock = new Lock(getenv('SYMFONY_LOCKFILE') ?: str_replace('composer.json', 'symfony.lock', Factory::getComposerFile()));
+        $this->frozenLock = $this->lock->all();
 
         $disable = true;
         foreach (array_merge($composer->getPackage()->getRequires() ?? [], $composer->getPackage()->getDevRequires() ?? []) as $link) {
@@ -475,6 +477,7 @@ function ($value) {
                     $this->io->writeError(sprintf('  - Configuring %s', $this->formatOrigin($recipe->getOrigin())));
                     $this->configurator->install($recipe, $this->lock, [
                         'force' => $event instanceof UpdateEvent && $event->force(),
+                        'beforeState' => $this->frozenLock[$recipe->getName()] ?? null,
                     ]);
                     $manifest = $recipe->getManifest();
                     if (isset($manifest['post-install-output'])) {
diff --git a/tests/Configurator/CopyFromRecipeConfiguratorTest.php b/tests/Configurator/CopyFromRecipeConfiguratorTest.php
index c6b31d46d..6e0354bcd 100644
--- a/tests/Configurator/CopyFromRecipeConfiguratorTest.php
+++ b/tests/Configurator/CopyFromRecipeConfiguratorTest.php
@@ -74,7 +74,7 @@ public function testConfigureAndOverwriteFiles()
             $this->recipe,
             [$this->sourceFileRelativePath => $this->targetFileRelativePath],
             $lock,
-            ['force' => true]
+            ['force' => true, 'beforeState' => null]
         );
         $this->assertFileExists($this->targetFile);
         $this->assertSame('somecontent', file_get_contents($this->targetFile));