Skip to content

Commit a8a40fc

Browse files
mtibbenfabpot
mtibben
authored and
fabpot
committed
[FrameworkBundle] PhpExtractor bugfix and improvements
1 parent 55d17fa commit a8a40fc

File tree

4 files changed

+265
-19
lines changed

4 files changed

+265
-19
lines changed

‎src/Symfony/Bundle/FrameworkBundle/Tests/Fixtures/Resources/views/translation.html.php

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,33 @@
11
This template is used for translation message extraction tests
22
<?php echo $view['translator']->trans('single-quoted key') ?>
33
<?php echo $view['translator']->trans("double-quoted key") ?>
4+
<?php echo $view['translator']->trans(<<<EOF
5+
heredoc key
6+
EOF
7+
) ?>
8+
<?php echo $view['translator']->trans(<<<'EOF'
9+
nowdoc key
10+
EOF
11+
) ?>
12+
<?php echo $view['translator']->trans(
13+
"double-quoted key with whitespace and escaped \$\n\" sequences"
14+
) ?>
15+
<?php echo $view['translator']->trans(
16+
'single-quoted key with whitespace and nonescaped \$\n\' sequences'
17+
) ?>
18+
<?php echo $view['translator']->trans( <<<EOF
19+
heredoc key with whitespace and escaped \$\n sequences
20+
EOF
21+
) ?>
22+
<?php echo $view['translator']->trans( <<<'EOF'
23+
nowdoc key with whitespace and nonescaped \$\n sequences
24+
EOF
25+
) ?>
26+
27+
<?php echo $view['translator']->trans('single-quoted key with "quote mark at the end"') ?>
28+
29+
<?php echo $view['translator']->transChoice(
30+
'{0} There is no apples|{1} There is one apple|]1,Inf[ There are %count% apples',
31+
10,
32+
array('%count%' => 10)
33+
) ?>

‎src/Symfony/Bundle/FrameworkBundle/Tests/Translation/PhpExtractorTest.php

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,27 @@ public function testExtraction()
2727
// Act
2828
$extractor->extract(__DIR__.'/../Fixtures/Resources/views/', $catalogue);
2929

30+
$expectedHeredoc = <<<EOF
31+
heredoc key with whitespace and escaped \$\n sequences
32+
EOF;
33+
$expectedNowdoc = <<<'EOF'
34+
nowdoc key with whitespace and nonescaped \$\n sequences
35+
EOF;
3036
// Assert
31-
$this->assertCount(2, $catalogue->all('messages'), '->extract() should find 1 translation');
32-
$this->assertTrue($catalogue->has('single-quoted key'), '->extract() should find the "single-quoted key" message');
33-
$this->assertTrue($catalogue->has('double-quoted key'), '->extract() should find the "double-quoted key" message');
34-
$this->assertEquals('prefixsingle-quoted key', $catalogue->get('single-quoted key'), '->extract() should apply "prefix" as prefix');
37+
$expectedCatalogue = array('messages' => array(
38+
'single-quoted key' => 'prefixsingle-quoted key',
39+
'double-quoted key' => 'prefixdouble-quoted key',
40+
'heredoc key' => 'prefixheredoc key',
41+
'nowdoc key' => 'prefixnowdoc key',
42+
"double-quoted key with whitespace and escaped \$\n\" sequences" => "prefixdouble-quoted key with whitespace and escaped \$\n\" sequences",
43+
'single-quoted key with whitespace and nonescaped \$\n\' sequences' => 'prefixsingle-quoted key with whitespace and nonescaped \$\n\' sequences',
44+
'single-quoted key with "quote mark at the end"' => 'prefixsingle-quoted key with "quote mark at the end"',
45+
$expectedHeredoc => "prefix".$expectedHeredoc,
46+
$expectedNowdoc => "prefix".$expectedNowdoc,
47+
'{0} There is no apples|{1} There is one apple|]1,Inf[ There are %count% apples' => 'prefix{0} There is no apples|{1} There is one apple|]1,Inf[ There are %count% apples',
48+
));
49+
$actualCatalogue = $catalogue->all();
50+
51+
$this->assertEquals($expectedCatalogue, $actualCatalogue);
3552
}
3653
}

‎src/Symfony/Bundle/FrameworkBundle/Translation/PhpExtractor.php

Lines changed: 72 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
class PhpExtractor implements ExtractorInterface
2424
{
2525
const MESSAGE_TOKEN = 300;
26-
const IGNORE_TOKEN = 400;
2726

2827
/**
2928
* Prefix for new found message.
@@ -39,15 +38,16 @@ class PhpExtractor implements ExtractorInterface
3938
*/
4039
protected $sequences = array(
4140
array(
42-
'$view',
43-
'[',
44-
'\'translator\'',
45-
']',
4641
'->',
4742
'trans',
4843
'(',
4944
self::MESSAGE_TOKEN,
50-
')',
45+
),
46+
array(
47+
'->',
48+
'transChoice',
49+
'(',
50+
self::MESSAGE_TOKEN,
5151
),
5252
);
5353

@@ -75,7 +75,7 @@ public function setPrefix($prefix)
7575
/**
7676
* Normalizes a token.
7777
*
78-
* @param mixed $token
78+
* @param mixed $token
7979
* @return string
8080
*/
8181
protected function normalizeToken($token)
@@ -87,6 +87,60 @@ protected function normalizeToken($token)
8787
return $token;
8888
}
8989

90+
/**
91+
* Seeks to a non-whitespace token
92+
*
93+
* @param \ArrayIterator $tokenIterator
94+
*/
95+
protected function seekToNextRelaventToken($tokenIterator)
96+
{
97+
for ( ; $tokenIterator->valid(); $tokenIterator->next()) {
98+
$t = $tokenIterator->current();
99+
if (!is_array($t) || ($t[0] !== T_WHITESPACE)) {
100+
break;
101+
}
102+
}
103+
}
104+
105+
/**
106+
* Extracts the message from the iterator while the tokens
107+
* match allowed message tokens
108+
*
109+
* @param \ArrayIterator $tokenIterator
110+
*/
111+
protected function getMessage($tokenIterator)
112+
{
113+
$message = '';
114+
$docToken = '';
115+
116+
for ( ; $tokenIterator->valid(); $tokenIterator->next()) {
117+
$t = $tokenIterator->current();
118+
if (!is_array($t)) {
119+
break;
120+
}
121+
122+
switch ($t[0]) {
123+
case T_START_HEREDOC:
124+
$docToken = $t[1];
125+
break;
126+
case T_ENCAPSED_AND_WHITESPACE:
127+
case T_CONSTANT_ENCAPSED_STRING:
128+
$message .= $t[1];
129+
break;
130+
case T_END_HEREDOC:
131+
return PhpStringTokenParser::parseDocString($docToken, $message);
132+
default:
133+
break 2;
134+
}
135+
}
136+
137+
if ($message) {
138+
$message = PhpStringTokenParser::parse($message);
139+
}
140+
141+
return $message;
142+
}
143+
90144
/**
91145
* Extracts trans message from PHP tokens.
92146
*
@@ -95,24 +149,27 @@ protected function normalizeToken($token)
95149
*/
96150
protected function parseTokens($tokens, MessageCatalogue $catalog)
97151
{
98-
foreach ($tokens as $key => $token) {
152+
$tokenIterator = new \ArrayIterator($tokens);
153+
154+
for ($key = 0; $key < $tokenIterator->count(); $key++) {
99155
foreach ($this->sequences as $sequence) {
100156
$message = '';
157+
$tokenIterator->seek($key);
101158

102-
foreach ($sequence as $id => $item) {
103-
if ($this->normalizeToken($tokens[$key + $id]) == $item) {
159+
foreach ($sequence as $item) {
160+
$this->seekToNextRelaventToken($tokenIterator);
161+
162+
if ($this->normalizeToken($tokenIterator->current()) == $item) {
163+
$tokenIterator->next();
104164
continue;
105165
} elseif (self::MESSAGE_TOKEN == $item) {
106-
$message = $this->normalizeToken($tokens[$key + $id]);
107-
} elseif (self::IGNORE_TOKEN == $item) {
108-
continue;
166+
$message = $this->getMessage($tokenIterator);
167+
break;
109168
} else {
110169
break;
111170
}
112171
}
113172

114-
$message = trim($message, '\'"');
115-
116173
if ($message) {
117174
$catalog->set($message, $this->prefix.$message);
118175
break;

‎src/Symfony/Bundle/FrameworkBundle/Translation/PhpStringTokenParser.php

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\Bundle\FrameworkBundle\Translation;
13+
14+
/*
15+
* The following is derived from code at http://github.com/nikic/PHP-Parser
16+
*
17+
* Copyright (c) 2011 by Nikita Popov
18+
*
19+
* Some rights reserved.
20+
*
21+
* Redistribution and use in source and binary forms, with or without
22+
* modification, are permitted provided that the following conditions are
23+
* met:
24+
*
25+
* * Redistributions of source code must retain the above copyright
26+
* notice, this list of conditions and the following disclaimer.
27+
*
28+
* * Redistributions in binary form must reproduce the above
29+
* copyright notice, this list of conditions and the following
30+
* disclaimer in the documentation and/or other materials provided
31+
* with the distribution.
32+
*
33+
* * The names of the contributors may not be used to endorse or
34+
* promote products derived from this software without specific
35+
* prior written permission.
36+
*
37+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
38+
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39+
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40+
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41+
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42+
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43+
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44+
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45+
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46+
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47+
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48+
*/
49+
50+
class PhpStringTokenParser
51+
{
52+
protected static $replacements = array(
53+
'\\' => '\\',
54+
'$' => '$',
55+
'n' => "\n",
56+
'r' => "\r",
57+
't' => "\t",
58+
'f' => "\f",
59+
'v' => "\v",
60+
'e' => "\x1B",
61+
);
62+
63+
/**
64+
* Parses a string token.
65+
*
66+
* @param string $str String token content
67+
*
68+
* @return string The parsed string
69+
*/
70+
public static function parse($str)
71+
{
72+
$bLength = 0;
73+
if ('b' === $str[0]) {
74+
$bLength = 1;
75+
}
76+
77+
if ('\'' === $str[$bLength]) {
78+
return str_replace(
79+
array('\\\\', '\\\''),
80+
array( '\\', '\''),
81+
substr($str, $bLength + 1, -1)
82+
);
83+
} else {
84+
return self::parseEscapeSequences(substr($str, $bLength + 1, -1), '"');
85+
}
86+
}
87+
88+
/**
89+
* Parses escape sequences in strings (all string types apart from single quoted).
90+
*
91+
* @param string $str String without quotes
92+
* @param null|string $quote Quote type
93+
*
94+
* @return string String with escape sequences parsed
95+
*/
96+
public static function parseEscapeSequences($str, $quote)
97+
{
98+
if (null !== $quote) {
99+
$str = str_replace('\\' . $quote, $quote, $str);
100+
}
101+
102+
return preg_replace_callback(
103+
'~\\\\([\\\\$nrtfve]|[xX][0-9a-fA-F]{1,2}|[0-7]{1,3})~',
104+
array(__CLASS__, 'parseCallback'),
105+
$str
106+
);
107+
}
108+
109+
public static function parseCallback($matches)
110+
{
111+
$str = $matches[1];
112+
113+
if (isset(self::$replacements[$str])) {
114+
return self::$replacements[$str];
115+
} elseif ('x' === $str[0] || 'X' === $str[0]) {
116+
return chr(hexdec($str));
117+
} else {
118+
return chr(octdec($str));
119+
}
120+
}
121+
122+
/**
123+
* Parses a constant doc string.
124+
*
125+
* @param string $startToken Doc string start token content (<<<SMTHG)
126+
* @param string $str String token content
127+
*
128+
* @return string Parsed string
129+
*/
130+
public static function parseDocString($startToken, $str)
131+
{
132+
// strip last newline (thanks tokenizer for sticking it into the string!)
133+
$str = preg_replace('~(\r\n|\n|\r)$~', '', $str);
134+
135+
// nowdoc string
136+
if (false !== strpos($startToken, '\'')) {
137+
return $str;
138+
}
139+
140+
return self::parseEscapeSequences($str, null);
141+
}
142+
}

0 commit comments

Comments
 (0)