Skip to content

[FrameworkBundler] Fix cache:clear with buildDir #39360

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 15, 2020
Merged

[FrameworkBundler] Fix cache:clear with buildDir #39360

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 39 additions & 27 deletions src/Symfony/Bundle/FrameworkBundle/Command/CacheClearCommand.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,20 @@ protected function execute(InputInterface $input, OutputInterface $output): int
}

$useBuildDir = $realBuildDir !== $realCacheDir;
$oldBuildDir = substr($realBuildDir, 0, -1).('~' === substr($realBuildDir, -1) ? '+' : '~');
if ($useBuildDir) {
$oldBuildDir = substr($realBuildDir, 0, -1).('~' === substr($realBuildDir, -1) ? '+' : '~');
$fs->remove($oldBuildDir);

if (!is_writable($realBuildDir)) {
throw new RuntimeException(sprintf('Unable to write in the "%s" directory.', $realBuildDir));
}

if ($this->isNfs($realCacheDir)) {
$fs->remove($realCacheDir);
} else {
$fs->rename($realCacheDir, $oldCacheDir);
}
$fs->mkdir($realCacheDir);
}

$io->comment(sprintf('Clearing the cache for the <info>%s</info> environment with debug <info>%s</info>', $kernel->getEnvironment(), var_export($kernel->isDebug(), true)));
Expand All @@ -114,7 +121,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int

// the warmup cache dir name must have the same length as the real one
// to avoid the many problems in serialized resources files
$warmupDir = substr($realCacheDir, 0, -1).('_' === substr($realCacheDir, -1) ? '-' : '_');
$warmupDir = substr($realBuildDir, 0, -1).('_' === substr($realBuildDir, -1) ? '-' : '_');

if ($output->isVerbose() && $fs->exists($warmupDir)) {
$io->comment('Clearing outdated warmup directory...');
Expand Down Expand Up @@ -153,35 +160,15 @@ protected function execute(InputInterface $input, OutputInterface $output): int
touch($warmupDir.'/'.$containerDir.'.legacy');
}

if ('/' === \DIRECTORY_SEPARATOR && $mounts = @file('/proc/mounts')) {
foreach ($mounts as $mount) {
$mount = \array_slice(explode(' ', $mount), 1, -3);
if (!\in_array(array_pop($mount), ['vboxsf', 'nfs'])) {
continue;
}
$mount = implode(' ', $mount).'/';

if (0 === strpos($realCacheDir, $mount)) {
$io->note('For better performances, you should move the cache and log directories to a non-shared folder of the VM.');
$oldCacheDir = false;
break;
}
}
}

if ($oldCacheDir) {
$fs->rename($realCacheDir, $oldCacheDir);
if ($this->isNfs($realBuildDir)) {
$io->note('For better performances, you should move the cache and log directories to a non-shared folder of the VM.');
$fs->remove($realBuildDir);
} else {
$fs->remove($realCacheDir);
}
$fs->rename($warmupDir, $realCacheDir);

if ($useBuildDir) {
$fs->rename($realBuildDir, $oldBuildDir);
// Copy the content of the warmed cache in the build dir
$fs->mirror($realCacheDir, $realBuildDir);
}

$fs->rename($warmupDir, $realBuildDir);

if ($output->isVerbose()) {
$io->comment('Removing old build and cache directory...');
}
Expand Down Expand Up @@ -214,6 +201,31 @@ protected function execute(InputInterface $input, OutputInterface $output): int
return 0;
}

private function isNfs(string $dir): bool
{
static $mounts = null;

if (null === $mounts) {
$mounts = [];
if ('/' === \DIRECTORY_SEPARATOR && $mounts = @file('/proc/mounts')) {
foreach ($mounts as $mount) {
$mount = \array_slice(explode(' ', $mount), 1, -3);
if (!\in_array(array_pop($mount), ['vboxsf', 'nfs'])) {
continue;
}
$mounts[] = implode(' ', $mount).'/';
}
}
}
foreach ($mounts as $mount) {
if (0 === strpos($dir, $mount)) {
return true;
}
}

return false;
}

private function warmup(string $warmupDir, string $realBuildDir, bool $enableOptionalWarmers = true)
{
// create a temporary kernel
Expand Down
2 changes: 1 addition & 1 deletion src/Symfony/Bundle/FrameworkBundle/Console/Descriptor/JsonDescriptor.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ protected function describeContainerEnvVars(array $envs, array $options = [])

protected function describeContainerDeprecations(ContainerBuilder $builder, array $options = []): void
{
$containerDeprecationFilePath = sprintf('%s/%sDeprecations.log', $builder->getParameter('kernel.cache_dir'), $builder->getParameter('kernel.container_class'));
$containerDeprecationFilePath = sprintf('%s/%sDeprecations.log', $builder->getParameter('kernel.build_dir'), $builder->getParameter('kernel.container_class'));
if (!file_exists($containerDeprecationFilePath)) {
throw new RuntimeException('The deprecation file does not exist, please try warming the cache first.');
}
Expand Down
2 changes: 1 addition & 1 deletion src/Symfony/Bundle/FrameworkBundle/Console/Descriptor/MarkdownDescriptor.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ protected function describeContainerService($service, array $options = [], Conta

protected function describeContainerDeprecations(ContainerBuilder $builder, array $options = []): void
{
$containerDeprecationFilePath = sprintf('%s/%sDeprecations.log', $builder->getParameter('kernel.cache_dir'), $builder->getParameter('kernel.container_class'));
$containerDeprecationFilePath = sprintf('%s/%sDeprecations.log', $builder->getParameter('kernel.build_dir'), $builder->getParameter('kernel.container_class'));
if (!file_exists($containerDeprecationFilePath)) {
throw new RuntimeException('The deprecation file does not exist, please try warming the cache first.');
}
Expand Down
2 changes: 1 addition & 1 deletion src/Symfony/Bundle/FrameworkBundle/Console/Descriptor/TextDescriptor.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ protected function describeContainerDefinition(Definition $definition, array $op

protected function describeContainerDeprecations(ContainerBuilder $builder, array $options = []): void
{
$containerDeprecationFilePath = sprintf('%s/%sDeprecations.log', $builder->getParameter('kernel.cache_dir'), $builder->getParameter('kernel.container_class'));
$containerDeprecationFilePath = sprintf('%s/%sDeprecations.log', $builder->getParameter('kernel.build_dir'), $builder->getParameter('kernel.container_class'));
if (!file_exists($containerDeprecationFilePath)) {
$options['output']->warning('The deprecation file does not exist, please try warming the cache first.');

Expand Down
2 changes: 1 addition & 1 deletion src/Symfony/Bundle/FrameworkBundle/Console/Descriptor/XmlDescriptor.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ protected function describeContainerEnvVars(array $envs, array $options = [])

protected function describeContainerDeprecations(ContainerBuilder $builder, array $options = []): void
{
$containerDeprecationFilePath = sprintf('%s/%sDeprecations.log', $builder->getParameter('kernel.cache_dir'), $builder->getParameter('kernel.container_class'));
$containerDeprecationFilePath = sprintf('%s/%sDeprecations.log', $builder->getParameter('kernel.build_dir'), $builder->getParameter('kernel.container_class'));
if (!file_exists($containerDeprecationFilePath)) {
throw new RuntimeException('The deprecation file does not exist, please try warming the cache first.');
}
Expand Down
2 changes: 1 addition & 1 deletion src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -899,7 +899,7 @@ private function registerDebugConfiguration(array $config, ContainerBuilder $con
$debug = $container->getParameter('kernel.debug');

if ($debug) {
$container->setParameter('debug.container.dump', '%kernel.cache_dir%/%kernel.container_class%.xml');
$container->setParameter('debug.container.dump', '%kernel.build_dir%/%kernel.container_class%.xml');
}

if ($debug && class_exists(Stopwatch::class)) {
Expand Down
2 changes: 1 addition & 1 deletion src/Symfony/Bundle/FrameworkBundle/Resources/config/collectors.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
->set('data_collector.logger', LoggerDataCollector::class)
->args([
service('logger')->ignoreOnInvalid(),
sprintf('%s/%s', param('kernel.cache_dir'), param('kernel.container_class')),
sprintf('%s/%s', param('kernel.build_dir'), param('kernel.container_class')),
service('request_stack')->ignoreOnInvalid(),
])
->tag('monolog.logger', ['channel' => 'profiler'])
Expand Down
2 changes: 1 addition & 1 deletion src/Symfony/Bundle/FrameworkBundle/Resources/config/services.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ class_exists(WorkflowEvents::class) ? WorkflowEvents::ALIASES : []
->args([
tagged_iterator('kernel.cache_warmer'),
param('kernel.debug'),
sprintf('%s/%sDeprecations.log', param('kernel.cache_dir'), param('kernel.container_class')),
sprintf('%s/%sDeprecations.log', param('kernel.build_dir'), param('kernel.container_class')),
])
->tag('container.no_preload')

Expand Down
79 changes: 55 additions & 24 deletions src/Symfony/Bundle/FrameworkBundle/Tests/Command/CacheClearCommand/CacheClearCommandTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
use Symfony\Component\Config\Resource\ResourceInterface;
use Symfony\Component\Console\Input\ArrayInput;
use Symfony\Component\Console\Output\NullOutput;
use Symfony\Component\DependencyInjection\Container;
use Symfony\Component\Filesystem\Filesystem;
use Symfony\Component\Finder\Finder;
use Symfony\Component\HttpKernel\KernelInterface;

class CacheClearCommandTest extends TestCase
{
Expand All @@ -41,65 +41,96 @@ protected function tearDown(): void
$this->fs->remove($this->kernel->getProjectDir());
}

/** @dataProvider getKernel */
public function testCacheIsFreshAfterCacheClearedWithWarmup(KernelInterface $kernel)
public function testCacheIsFreshAfterCacheClearedWithWarmup()
{
$this->fs->mkdir($this->kernel->getProjectDir());

$input = new ArrayInput(['cache:clear']);
$application = new Application($kernel);
$application = new Application($this->kernel);
$application->setCatchExceptions(false);

$application->doRun($input, new NullOutput());

// Ensure that all *.meta files are fresh
$finder = new Finder();
$metaFiles = $finder->files()->in($kernel->getCacheDir())->name('*.php.meta');
$metaFiles = $finder->files()->in($this->kernel->getCacheDir())->name('*.php.meta');
// check that cache is warmed up
$this->assertNotEmpty($metaFiles);
$configCacheFactory = new ConfigCacheFactory(true);

foreach ($metaFiles as $file) {
$configCacheFactory->cache(substr($file, 0, -5), function () use ($file) {
$this->fail(sprintf('Meta file "%s" is not fresh', (string) $file));
});
$configCacheFactory->cache(
substr($file, 0, -5),
function () use ($file) {
$this->fail(sprintf('Meta file "%s" is not fresh', (string) $file));
}
);
}

// check that app kernel file present in meta file of container's cache
$containerClass = $kernel->getContainer()->getParameter('kernel.container_class');
$containerClass = $this->kernel->getContainer()->getParameter('kernel.container_class');
$containerRef = new \ReflectionClass($containerClass);
$containerFile = \dirname($containerRef->getFileName(), 2).'/'.$containerClass.'.php';
$containerMetaFile = $containerFile.'.meta';
$kernelRef = new \ReflectionObject($kernel);
$kernelFile = $kernelRef->getFileName();
$this->kernelRef = new \ReflectionObject($this->kernel);
$this->kernelFile = $this->kernelRef->getFileName();
/** @var ResourceInterface[] $meta */
$meta = unserialize(file_get_contents($containerMetaFile));
$found = false;
foreach ($meta as $resource) {
if ((string) $resource === $kernelFile) {
if ((string) $resource === $this->kernelFile) {
$found = true;
break;
}
}
$this->assertTrue($found, 'Kernel file should present as resource');

$containerRef = new \ReflectionClass(require $containerFile);
$containerFile = str_replace('tes_'.\DIRECTORY_SEPARATOR, 'test'.\DIRECTORY_SEPARATOR, $containerRef->getFileName());
$this->assertMatchesRegularExpression(sprintf('/\'kernel.container_class\'\s*=>\s*\'%s\'/', $containerClass), file_get_contents($containerFile), 'kernel.container_class is properly set on the dumped container');
$containerFile = str_replace(
'tes_'.\DIRECTORY_SEPARATOR,
'test'.\DIRECTORY_SEPARATOR,
$containerRef->getFileName()
);
$this->assertMatchesRegularExpression(
sprintf('/\'kernel.container_class\'\s*=>\s*\'%s\'/', $containerClass),
file_get_contents($containerFile),
'kernel.container_class is properly set on the dumped container'
);
}

public function getKernel()
public function testCacheIsWarmedWhenCalledTwice()
{
yield [new TestAppKernel('test', true)];
yield [new NoBuildDirKernel('test', true)];
$input = new ArrayInput(['cache:clear']);
$application = new Application(clone $this->kernel);
$application->setCatchExceptions(false);
$application->doRun($input, new NullOutput());

$_SERVER['REQUEST_TIME'] = time() + 1;
$application = new Application(clone $this->kernel);
$application->setCatchExceptions(false);
$application->doRun($input, new NullOutput());

$this->assertTrue(is_file($this->kernel->getCacheDir().'/annotations.php'));
}
}

class NoBuildDirKernel extends TestAppKernel
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you have a look @chalasr ? Previous code can't work, because kernel.build_dir is now needed to build the container (and now needed to initialize the cache:clear command).
I refactored the code t build a container and then remove the parameter from the dumped container.

{
protected function getKernelParameters()
public function testCacheIsWarmedWithOldContainer()
{
$parameters = parent::getKernelParameters();
unset($parameters['kernel.build_dir']);
$kernel = clone $this->kernel;

// Hack to get a dumped working container,
// BUT without "kernel.build_dir" parameter (like an old dumped container)
$kernel->boot();
$container = $kernel->getContainer();
\Closure::bind(function (Container $class) {
unset($class->loadedDynamicParameters['kernel.build_dir']);
unset($class->parameters['kernel.build_dir']);
}, null, \get_class($container))($container);

$input = new ArrayInput(['cache:clear']);
$application = new Application($kernel);
$application->setCatchExceptions(false);
$application->doRun($input, new NullOutput());

return $parameters;
$this->expectNotToPerformAssertions();
}
}
2 changes: 2 additions & 0 deletions src/Symfony/Bundle/FrameworkBundle/Tests/Console/Descriptor/ObjectsProvider.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,12 @@ public static function getContainerDeprecations()
{
$builderWithDeprecations = new ContainerBuilder();
$builderWithDeprecations->setParameter('kernel.cache_dir', __DIR__.'/../../Fixtures/Descriptor/cache');
$builderWithDeprecations->setParameter('kernel.build_dir', __DIR__.'/../../Fixtures/Descriptor/cache');
$builderWithDeprecations->setParameter('kernel.container_class', 'KernelContainerWith');

$builderWithoutDeprecations = new ContainerBuilder();
$builderWithoutDeprecations->setParameter('kernel.cache_dir', __DIR__.'/../../Fixtures/Descriptor/cache');
$builderWithoutDeprecations->setParameter('kernel.build_dir', __DIR__.'/../../Fixtures/Descriptor/cache');
$builderWithoutDeprecations->setParameter('kernel.container_class', 'KernelContainerWithout');

return [
Expand Down
6 changes: 3 additions & 3 deletions src/Symfony/Bundle/FrameworkBundle/Tests/Functional/ContainerDebugCommandTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public function testDescribeEnvVar()
public function testGetDeprecation()
{
static::bootKernel(['test_case' => 'ContainerDebug', 'root_config' => 'config.yml', 'debug' => true]);
$path = sprintf('%s/%sDeprecations.log', static::$kernel->getContainer()->getParameter('kernel.cache_dir'), static::$kernel->getContainer()->getParameter('kernel.container_class'));
$path = sprintf('%s/%sDeprecations.log', static::$kernel->getContainer()->getParameter('kernel.build_dir'), static::$kernel->getContainer()->getParameter('kernel.container_class'));
touch($path);
file_put_contents($path, serialize([[
'type' => 16384,
Expand Down Expand Up @@ -169,7 +169,7 @@ public function testGetDeprecation()
public function testGetDeprecationNone()
{
static::bootKernel(['test_case' => 'ContainerDebug', 'root_config' => 'config.yml', 'debug' => true]);
$path = sprintf('%s/%sDeprecations.log', static::$kernel->getContainer()->getParameter('kernel.cache_dir'), static::$kernel->getContainer()->getParameter('kernel.container_class'));
$path = sprintf('%s/%sDeprecations.log', static::$kernel->getContainer()->getParameter('kernel.build_dir'), static::$kernel->getContainer()->getParameter('kernel.container_class'));
touch($path);
file_put_contents($path, serialize([]));

Expand All @@ -188,7 +188,7 @@ public function testGetDeprecationNone()
public function testGetDeprecationNoFile()
{
static::bootKernel(['test_case' => 'ContainerDebug', 'root_config' => 'config.yml', 'debug' => true]);
$path = sprintf('%s/%sDeprecations.log', static::$kernel->getContainer()->getParameter('kernel.cache_dir'), static::$kernel->getContainer()->getParameter('kernel.container_class'));
$path = sprintf('%s/%sDeprecations.log', static::$kernel->getContainer()->getParameter('kernel.build_dir'), static::$kernel->getContainer()->getParameter('kernel.container_class'));
@unlink($path);

$application = new Application(static::$kernel);
Expand Down
2 changes: 1 addition & 1 deletion src/Symfony/Bundle/FrameworkBundle/composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
"symfony/event-dispatcher": "^5.1",
"symfony/error-handler": "^4.4.1|^5.0.1",
"symfony/http-foundation": "^5.2.1",
"symfony/http-kernel": "^5.2",
"symfony/http-kernel": "^5.2.1",
"symfony/polyfill-mbstring": "~1.0",
"symfony/polyfill-php80": "^1.15",
"symfony/filesystem": "^4.4|^5.0",
Expand Down
2 changes: 1 addition & 1 deletion src/Symfony/Component/HttpKernel/Kernel.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -609,7 +609,7 @@ protected function getKernelParameters()
'kernel.runtime_environment' => '%env(default:kernel.environment:APP_RUNTIME_ENV)%',
'kernel.debug' => $this->debug,
'kernel.build_dir' => realpath($buildDir = $this->warmupDir ?: $this->getBuildDir()) ?: $buildDir,
'kernel.cache_dir' => realpath($this->getCacheDir()) ?: $this->getCacheDir(),
'kernel.cache_dir' => realpath($cacheDir = ($this->getCacheDir() === $this->getBuildDir() ? ($this->warmupDir ?: $this->getCacheDir()) : $this->getCacheDir())) ?: $cacheDir,
'kernel.logs_dir' => realpath($this->getLogDir()) ?: $this->getLogDir(),
'kernel.bundles' => $bundles,
'kernel.bundles_metadata' => $bundlesMetadata,
Expand Down