diff --git a/.github/workflows/intl-data-tests.yml b/.github/workflows/intl-data-tests.yml index 58e1aa80ae0bd..c0e56036fff6c 100644 --- a/.github/workflows/intl-data-tests.yml +++ b/.github/workflows/intl-data-tests.yml @@ -79,3 +79,12 @@ jobs: - name: Run intl-data tests run: ./phpunit --group intl-data -v + + - name: Test with compressed data + run: | + [ -f src/Symfony/Component/Intl/Resources/data/locales/en.php ] + [ ! -f src/Symfony/Component/Intl/Resources/data/locales/en.php.gz ] + src/Symfony/Component/Intl/Resources/bin/compress + [ ! -f src/Symfony/Component/Intl/Resources/data/locales/en.php ] + [ -f src/Symfony/Component/Intl/Resources/data/locales/en.php.gz ] + ./phpunit src/Symfony/Component/Intl diff --git a/src/Symfony/Component/Intl/CHANGELOG.md b/src/Symfony/Component/Intl/CHANGELOG.md index 3a84dd3d42d56..1af3e900b6fb7 100644 --- a/src/Symfony/Component/Intl/CHANGELOG.md +++ b/src/Symfony/Component/Intl/CHANGELOG.md @@ -5,6 +5,7 @@ CHANGELOG --- * Add the special `strip` locale to `EmojiTransliterator` to strip all emojis from a string + * Add `compress` script to compress the `Resources/data` directory when disk space matters 6.2 --- diff --git a/src/Symfony/Component/Intl/Data/Bundle/Reader/PhpBundleReader.php b/src/Symfony/Component/Intl/Data/Bundle/Reader/PhpBundleReader.php index 9f5bee4e7ade2..2d179be2d696c 100644 --- a/src/Symfony/Component/Intl/Data/Bundle/Reader/PhpBundleReader.php +++ b/src/Symfony/Component/Intl/Data/Bundle/Reader/PhpBundleReader.php @@ -12,6 +12,7 @@ namespace Symfony\Component\Intl\Data\Bundle\Reader; use Symfony\Component\Intl\Exception\ResourceBundleNotFoundException; +use Symfony\Component\Intl\Util\GzipStreamWrapper; /** * Reads .php resource bundles. @@ -31,6 +32,10 @@ public function read(string $path, string $locale): mixed throw new ResourceBundleNotFoundException(sprintf('The resource bundle "%s" does not exist.', $fileName)); } + if (is_file($fileName.'.gz')) { + return GzipStreamWrapper::require($fileName.'.gz'); + } + if (!is_file($fileName)) { throw new ResourceBundleNotFoundException(sprintf('The resource bundle "%s" does not exist.', $fileName)); } diff --git a/src/Symfony/Component/Intl/Resources/bin/compress b/src/Symfony/Component/Intl/Resources/bin/compress new file mode 100755 index 0000000000000..e786bd2a041c9 --- /dev/null +++ b/src/Symfony/Component/Intl/Resources/bin/compress @@ -0,0 +1,29 @@ +#!/usr/bin/env php + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +if ('cli' !== PHP_SAPI) { + throw new Exception('This script must be run from the command line.'); +} +if (!extension_loaded('zlib')) { + throw new Exception('This script requires the zlib extension.'); +} + +foreach (glob(dirname(__DIR__).'/data/*/*.php') as $file) { + if ('meta.php' === basename($file)) { + continue; + } + + $data = file_get_contents($file); + file_put_contents('compress.zlib://'.$file.'.gz', $data); + + unlink($file.(filesize($file.'.gz') >= strlen($data) ? '.gz' : '')); +} diff --git a/src/Symfony/Component/Intl/Resources/emoji/build.php b/src/Symfony/Component/Intl/Resources/emoji/build.php index f69eaa5f17bb3..3647326e034d0 100755 --- a/src/Symfony/Component/Intl/Resources/emoji/build.php +++ b/src/Symfony/Component/Intl/Resources/emoji/build.php @@ -19,9 +19,9 @@ Builder::cleanTarget(); $emojisCodePoints = Builder::getEmojisCodePoints(); Builder::saveRules(Builder::buildRules($emojisCodePoints)); +Builder::saveRules(Builder::buildStripRules($emojisCodePoints)); Builder::saveRules(Builder::buildGitHubRules($emojisCodePoints)); Builder::saveRules(Builder::buildSlackRules($emojisCodePoints)); -Builder::saveRules(Builder::buildStripRules($emojisCodePoints)); final class Builder { @@ -178,7 +178,7 @@ public static function buildStripRules(array $emojisCodePoints): iterable $maps[$codePointsCount][$emoji] = ''; } - return ['strip' => self::createRules($maps)]; + return ['emoji-strip' => self::createRules($maps)]; } public static function cleanTarget(): void diff --git a/src/Symfony/Component/Intl/Transliterator/EmojiTransliterator.php b/src/Symfony/Component/Intl/Transliterator/EmojiTransliterator.php index bb52943ecabe2..0702fa43b2fca 100644 --- a/src/Symfony/Component/Intl/Transliterator/EmojiTransliterator.php +++ b/src/Symfony/Component/Intl/Transliterator/EmojiTransliterator.php @@ -11,6 +11,8 @@ namespace Symfony\Component\Intl\Transliterator; +use Symfony\Component\Intl\Util\GzipStreamWrapper; + if (!class_exists(\Transliterator::class)) { throw new \LogicException(sprintf('You cannot use the "%s\EmojiTransliterator" class as the "intl" extension is not installed. See https://php.net/intl.', __NAMESPACE__)); } else { @@ -40,7 +42,8 @@ public static function create(string $id, int $direction = self::FORWARD): self $id = self::REVERSEABLE_IDS[$id]; } - if (!preg_match('/^[a-z0-9@_\\.\\-]*$/', $id) || !is_file(\dirname(__DIR__)."/Resources/data/transliterator/emoji/{$id}.php")) { + $file = \dirname(__DIR__)."/Resources/data/transliterator/emoji/{$id}.php"; + if (!preg_match('/^[a-z0-9@_\\.\\-]*$/', $id) || !is_file($file) && !is_file($file .= '.gz')) { \Transliterator::create($id); // Populate intl_get_error_*() throw new \IntlException(intl_get_error_message(), intl_get_error_code()); @@ -57,7 +60,7 @@ public static function create(string $id, int $direction = self::FORWARD): self $instance = unserialize(sprintf('O:%d:"%s":1:{s:2:"id";s:%d:"%s";}', \strlen(self::class), self::class, \strlen($id), $id)); } - $instance->map = $maps[$id] ??= require \dirname(__DIR__)."/Resources/data/transliterator/emoji/{$id}.php"; + $instance->map = $maps[$id] ??= str_ends_with($file, '.gz') ? GzipStreamWrapper::require($file) : require $file; return $instance; } @@ -86,7 +89,9 @@ public static function listIDs(): array } foreach (scandir(\dirname(__DIR__).'/Resources/data/transliterator/emoji/') as $file) { - if (str_ends_with($file, '.php')) { + if (str_ends_with($file, '.php.gz')) { + $ids[] = substr($file, 0, -7); + } elseif (str_ends_with($file, '.php')) { $ids[] = substr($file, 0, -4); } } diff --git a/src/Symfony/Component/Intl/Util/GzipStreamWrapper.php b/src/Symfony/Component/Intl/Util/GzipStreamWrapper.php new file mode 100644 index 0000000000000..e5e30bbadbf3b --- /dev/null +++ b/src/Symfony/Component/Intl/Util/GzipStreamWrapper.php @@ -0,0 +1,83 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Intl\Util; + +/** + * @internal + */ +class GzipStreamWrapper +{ + /** @var resource|null */ + public $context; + + /** @var resource */ + private $handle; + private string $path; + + public static function require(string $path): array + { + if (!\extension_loaded('zlib')) { + throw new \LogicException(sprintf('The "zlib" extension is required to load the "%s/%s" map, please enable it in your php.ini file.', basename(\dirname($path)), basename($path))); + } + + if (!\function_exists('opcache_is_script_cached') || !@opcache_is_script_cached($path)) { + stream_wrapper_unregister('file'); + stream_wrapper_register('file', self::class); + } + + return require $path; + } + + public function stream_open(string $path, string $mode): bool + { + stream_wrapper_restore('file'); + $this->path = $path; + + return false !== $this->handle = fopen('compress.zlib://'.$path, $mode); + } + + public function stream_read(int $count): string|false + { + return fread($this->handle, $count); + } + + public function stream_eof(): bool + { + return feof($this->handle); + } + + public function stream_set_option(int $option, int $arg1, int $arg2): bool + { + return match ($option) { + \STREAM_OPTION_BLOCKING => stream_set_blocking($this->handle, $arg1), + \STREAM_OPTION_READ_TIMEOUT => stream_set_timeout($this->handle, $arg1, $arg2), + \STREAM_OPTION_WRITE_BUFFER => 0 === stream_set_write_buffer($this->handle, $arg2), + default => false, + }; + } + + public function stream_stat(): array|false + { + if (!$stat = stat($this->path)) { + return false; + } + + $h = fopen($this->path, 'r'); + fseek($h, -4, \SEEK_END); + $size = unpack('V', fread($h, 4)); + fclose($h); + + $stat[7] = $stat['size'] = end($size); + + return $stat; + } +}