Skip to content

Commit ff336fa

Browse files
nicolas-grekasnicolas-grekas
committed
feature #50055 [Intl] Allow compressing emoji and data maps (nicolas-grekas)
This PR was merged into the 6.3 branch. Discussion ---------- [Intl] Allow compressing emoji and data maps | Q | A | ------------- | --- | Branch? | 6.3 | Bug fix? | no | New feature? | yes | Deprecations? | no | Tickets | Fix #49943 | License | MIT | Doc PR | - Resuming #49970, which was closed by mistake :) Commits ------- de6a74c [Intl] Allow compressing emoji and data maps
2 parents d1d70c4 + de6a74c commit ff336fa

File tree

7 files changed

+137
-5
lines changed

7 files changed

+137
-5
lines changed

‎.github/workflows/intl-data-tests.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,3 +79,12 @@ jobs:
7979

8080
- name: Run intl-data tests
8181
run: ./phpunit --group intl-data -v
82+
83+
- name: Test with compressed data
84+
run: |
85+
[ -f src/Symfony/Component/Intl/Resources/data/locales/en.php ]
86+
[ ! -f src/Symfony/Component/Intl/Resources/data/locales/en.php.gz ]
87+
src/Symfony/Component/Intl/Resources/bin/compress
88+
[ ! -f src/Symfony/Component/Intl/Resources/data/locales/en.php ]
89+
[ -f src/Symfony/Component/Intl/Resources/data/locales/en.php.gz ]
90+
./phpunit src/Symfony/Component/Intl

‎src/Symfony/Component/Intl/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ CHANGELOG
55
---
66

77
* Add the special `strip` locale to `EmojiTransliterator` to strip all emojis from a string
8+
* Add `compress` script to compress the `Resources/data` directory when disk space matters
89

910
6.2
1011
---

‎src/Symfony/Component/Intl/Data/Bundle/Reader/PhpBundleReader.php

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
namespace Symfony\Component\Intl\Data\Bundle\Reader;
1313

1414
use Symfony\Component\Intl\Exception\ResourceBundleNotFoundException;
15+
use Symfony\Component\Intl\Util\GzipStreamWrapper;
1516

1617
/**
1718
* Reads .php resource bundles.
@@ -31,6 +32,10 @@ public function read(string $path, string $locale): mixed
3132
throw new ResourceBundleNotFoundException(sprintf('The resource bundle "%s" does not exist.', $fileName));
3233
}
3334

35+
if (is_file($fileName.'.gz')) {
36+
return GzipStreamWrapper::require($fileName.'.gz');
37+
}
38+
3439
if (!is_file($fileName)) {
3540
throw new ResourceBundleNotFoundException(sprintf('The resource bundle "%s" does not exist.', $fileName));
3641
}

‎src/Symfony/Component/Intl/Resources/bin/compress

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/usr/bin/env php
2+
<?php
3+
4+
/*
5+
* This file is part of the Symfony package.
6+
*
7+
* (c) Fabien Potencier <fabien@symfony.com>
8+
*
9+
* For the full copyright and license information, please view the LICENSE
10+
* file that was distributed with this source code.
11+
*/
12+
13+
if ('cli' !== PHP_SAPI) {
14+
throw new Exception('This script must be run from the command line.');
15+
}
16+
if (!extension_loaded('zlib')) {
17+
throw new Exception('This script requires the zlib extension.');
18+
}
19+
20+
foreach (glob(dirname(__DIR__).'/data/*/*.php') as $file) {
21+
if ('meta.php' === basename($file)) {
22+
continue;
23+
}
24+
25+
$data = file_get_contents($file);
26+
file_put_contents('compress.zlib://'.$file.'.gz', $data);
27+
28+
unlink($file.(filesize($file.'.gz') >= strlen($data) ? '.gz' : ''));
29+
}

‎src/Symfony/Component/Intl/Resources/emoji/build.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
Builder::cleanTarget();
2020
$emojisCodePoints = Builder::getEmojisCodePoints();
2121
Builder::saveRules(Builder::buildRules($emojisCodePoints));
22+
Builder::saveRules(Builder::buildStripRules($emojisCodePoints));
2223
Builder::saveRules(Builder::buildGitHubRules($emojisCodePoints));
2324
Builder::saveRules(Builder::buildSlackRules($emojisCodePoints));
24-
Builder::saveRules(Builder::buildStripRules($emojisCodePoints));
2525

2626
final class Builder
2727
{
@@ -178,7 +178,7 @@ public static function buildStripRules(array $emojisCodePoints): iterable
178178
$maps[$codePointsCount][$emoji] = '';
179179
}
180180

181-
return ['strip' => self::createRules($maps)];
181+
return ['emoji-strip' => self::createRules($maps)];
182182
}
183183

184184
public static function cleanTarget(): void

‎src/Symfony/Component/Intl/Transliterator/EmojiTransliterator.php

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111

1212
namespace Symfony\Component\Intl\Transliterator;
1313

14+
use Symfony\Component\Intl\Util\GzipStreamWrapper;
15+
1416
if (!class_exists(\Transliterator::class)) {
1517
throw new \LogicException(sprintf('You cannot use the "%s\EmojiTransliterator" class as the "intl" extension is not installed. See https://php.net/intl.', __NAMESPACE__));
1618
} else {
@@ -40,7 +42,8 @@ public static function create(string $id, int $direction = self::FORWARD): self
4042
$id = self::REVERSEABLE_IDS[$id];
4143
}
4244

43-
if (!preg_match('/^[a-z0-9@_\\.\\-]*$/', $id) || !is_file(\dirname(__DIR__)."/Resources/data/transliterator/emoji/{$id}.php")) {
45+
$file = \dirname(__DIR__)."/Resources/data/transliterator/emoji/{$id}.php";
46+
if (!preg_match('/^[a-z0-9@_\\.\\-]*$/', $id) || !is_file($file) && !is_file($file .= '.gz')) {
4447
\Transliterator::create($id); // Populate intl_get_error_*()
4548

4649
throw new \IntlException(intl_get_error_message(), intl_get_error_code());
@@ -57,7 +60,7 @@ public static function create(string $id, int $direction = self::FORWARD): self
5760
$instance = unserialize(sprintf('O:%d:"%s":1:{s:2:"id";s:%d:"%s";}', \strlen(self::class), self::class, \strlen($id), $id));
5861
}
5962

60-
$instance->map = $maps[$id] ??= require \dirname(__DIR__)."/Resources/data/transliterator/emoji/{$id}.php";
63+
$instance->map = $maps[$id] ??= str_ends_with($file, '.gz') ? GzipStreamWrapper::require($file) : require $file;
6164

6265
return $instance;
6366
}
@@ -86,7 +89,9 @@ public static function listIDs(): array
8689
}
8790

8891
foreach (scandir(\dirname(__DIR__).'/Resources/data/transliterator/emoji/') as $file) {
89-
if (str_ends_with($file, '.php')) {
92+
if (str_ends_with($file, '.php.gz')) {
93+
$ids[] = substr($file, 0, -7);
94+
} elseif (str_ends_with($file, '.php')) {
9095
$ids[] = substr($file, 0, -4);
9196
}
9297
}

‎src/Symfony/Component/Intl/Util/GzipStreamWrapper.php

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\Component\Intl\Util;
13+
14+
/**
15+
* @internal
16+
*/
17+
class GzipStreamWrapper
18+
{
19+
/** @var resource|null */
20+
public $context;
21+
22+
/** @var resource */
23+
private $handle;
24+
private string $path;
25+
26+
public static function require(string $path): array
27+
{
28+
if (!\extension_loaded('zlib')) {
29+
throw new \LogicException(sprintf('The "zlib" extension is required to load the "%s/%s" map, please enable it in your php.ini file.', basename(\dirname($path)), basename($path)));
30+
}
31+
32+
if (!\function_exists('opcache_is_script_cached') || !@opcache_is_script_cached($path)) {
33+
stream_wrapper_unregister('file');
34+
stream_wrapper_register('file', self::class);
35+
}
36+
37+
return require $path;
38+
}
39+
40+
public function stream_open(string $path, string $mode): bool
41+
{
42+
stream_wrapper_restore('file');
43+
$this->path = $path;
44+
45+
return false !== $this->handle = fopen('compress.zlib://'.$path, $mode);
46+
}
47+
48+
public function stream_read(int $count): string|false
49+
{
50+
return fread($this->handle, $count);
51+
}
52+
53+
public function stream_eof(): bool
54+
{
55+
return feof($this->handle);
56+
}
57+
58+
public function stream_set_option(int $option, int $arg1, int $arg2): bool
59+
{
60+
return match ($option) {
61+
\STREAM_OPTION_BLOCKING => stream_set_blocking($this->handle, $arg1),
62+
\STREAM_OPTION_READ_TIMEOUT => stream_set_timeout($this->handle, $arg1, $arg2),
63+
\STREAM_OPTION_WRITE_BUFFER => 0 === stream_set_write_buffer($this->handle, $arg2),
64+
default => false,
65+
};
66+
}
67+
68+
public function stream_stat(): array|false
69+
{
70+
if (!$stat = stat($this->path)) {
71+
return false;
72+
}
73+
74+
$h = fopen($this->path, 'r');
75+
fseek($h, -4, \SEEK_END);
76+
$size = unpack('V', fread($h, 4));
77+
fclose($h);
78+
79+
$stat[7] = $stat['size'] = end($size);
80+
81+
return $stat;
82+
}
83+
}

0 commit comments

Comments
 (0)