From 38de9c4e05bc871496c232c1ccbfc4be06a580e2 Mon Sep 17 00:00:00 2001 From: Liam Miller-Cushon Date: Thu, 21 Dec 2023 13:40:10 -0800 Subject: [PATCH 1/3] Improve support for string templates The initial implementation passed through the entire string unmodified, this allows formatting the Java expressions inside the `\{...}`. See https://github.com/google/google-java-format/pull/1010 Co-authored-by: butterunderflow PiperOrigin-RevId: 592940163 --- .../googlejavaformat/java/JavacTokens.java | 75 ++++++++++++------- .../java/java21/Java21InputAstVisitor.java | 11 ++- .../java/FormatterIntegrationTest.java | 3 +- .../googlejavaformat/java/FormatterTest.java | 24 ++++++ .../java/testdata/StringTemplate.input | 10 +++ .../java/testdata/StringTemplate.output | 9 +++ 6 files changed, 103 insertions(+), 29 deletions(-) create mode 100644 core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.input create mode 100644 core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.output diff --git a/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java b/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java index dd8760b2..da77cf82 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java +++ b/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java @@ -28,6 +28,11 @@ import com.sun.tools.javac.parser.Tokens.TokenKind; import com.sun.tools.javac.parser.UnicodeReader; import com.sun.tools.javac.util.Context; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; import java.util.Objects; import java.util.Set; @@ -83,9 +88,8 @@ static boolean isStringFragment(TokenKind kind) { return STRINGFRAGMENT != null && Objects.equals(kind, STRINGFRAGMENT); } - /** Lex the input and return a list of {@link RawTok}s. */ - public static ImmutableList getTokens( - String source, Context context, Set stopTokens) { + private static ImmutableList readAllTokens( + String source, Context context, Set nonTerminalStringFragments) { if (source == null) { return ImmutableList.of(); } @@ -93,12 +97,44 @@ public static ImmutableList getTokens( char[] buffer = (source + EOF_COMMENT).toCharArray(); Scanner scanner = new AccessibleScanner(fac, new CommentSavingTokenizer(fac, buffer, buffer.length)); + List tokens = new ArrayList<>(); + do { + scanner.nextToken(); + tokens.add(scanner.token()); + } while (scanner.token().kind != TokenKind.EOF); + for (int i = 0; i < tokens.size(); i++) { + if (isStringFragment(tokens.get(i).kind)) { + int start = i; + while (isStringFragment(tokens.get(i).kind)) { + i++; + } + for (int j = start; j < i - 1; j++) { + nonTerminalStringFragments.add(tokens.get(j).pos); + } + } + } + // A string template is tokenized as a series of STRINGFRAGMENT tokens containing the string + // literal values, followed by the tokens for the template arguments. For the formatter, we + // want the stream of tokens to appear in order by their start position. + if (Runtime.version().feature() >= 21) { + Collections.sort(tokens, Comparator.comparingInt(t -> t.pos)); + } + return ImmutableList.copyOf(tokens); + } + + /** Lex the input and return a list of {@link RawTok}s. */ + public static ImmutableList getTokens( + String source, Context context, Set stopTokens) { + if (source == null) { + return ImmutableList.of(); + } + Set nonTerminalStringFragments = new HashSet<>(); + ImmutableList javacTokens = readAllTokens(source, context, nonTerminalStringFragments); + ImmutableList.Builder tokens = ImmutableList.builder(); int end = source.length(); int last = 0; - do { - scanner.nextToken(); - Token t = scanner.token(); + for (Token t : javacTokens) { if (t.comments != null) { for (Comment c : Lists.reverse(t.comments)) { if (last < c.getSourcePos(0)) { @@ -118,27 +154,12 @@ public static ImmutableList getTokens( if (last < t.pos) { tokens.add(new RawTok(null, null, last, t.pos)); } - int pos = t.pos; - int endPos = t.endPos; if (isStringFragment(t.kind)) { - // A string template is tokenized as a series of STRINGFRAGMENT tokens containing the string - // literal values, followed by the tokens for the template arguments. For the formatter, we - // want the stream of tokens to appear in order by their start position, and also to have - // all the content from the original source text (including leading and trailing ", and the - // \ escapes from template arguments). This logic processes the token stream from javac to - // meet those requirements. - while (isStringFragment(t.kind)) { - endPos = t.endPos; - scanner.nextToken(); - t = scanner.token(); - } - // Read tokens for the string template arguments, until we read the end of the string - // template. The last token in a string template is always a trailing string fragment. Use - // lookahead to defer reading the token after the template until the next iteration of the - // outer loop. - while (scanner.token(/* lookahead= */ 1).endPos < endPos) { - scanner.nextToken(); - t = scanner.token(); + int endPos = t.endPos; + int pos = t.pos; + if (nonTerminalStringFragments.contains(t.pos)) { + // Include the \ escape from \{...} in the preceding string fragment + endPos++; } tokens.add(new RawTok(source.substring(pos, endPos), t.kind, pos, endPos)); last = endPos; @@ -151,7 +172,7 @@ public static ImmutableList getTokens( t.endPos)); last = t.endPos; } - } while (scanner.token().kind != TokenKind.EOF); + } if (last < end) { tokens.add(new RawTok(null, null, last, end)); } diff --git a/core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java b/core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java index 897d6ffc..5ef81fad 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java +++ b/core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java @@ -82,11 +82,20 @@ public Void visitDeconstructionPattern(DeconstructionPatternTree node, Void unus @SuppressWarnings("preview") @Override - public Void visitStringTemplate(StringTemplateTree node, Void aVoid) { + public Void visitStringTemplate(StringTemplateTree node, Void unused) { sync(node); + builder.open(plusFour); scan(node.getProcessor(), null); token("."); token(builder.peekToken().get()); + for (int i = 0; i < node.getFragments().size() - 1; i++) { + token("{"); + builder.breakOp(); + scan(node.getExpressions().get(i), null); + token("}"); + token(builder.peekToken().get()); + } + builder.close(); return null; } diff --git a/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java b/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java index cf15ecbc..b31e7bb1 100644 --- a/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java +++ b/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java @@ -60,7 +60,8 @@ public class FormatterIntegrationTest { "SwitchUnderscore", "I880", "Unnamed", - "I981") + "I981", + "StringTemplate") .build(); @Parameters(name = "{index}: {0}") diff --git a/core/src/test/java/com/google/googlejavaformat/java/FormatterTest.java b/core/src/test/java/com/google/googlejavaformat/java/FormatterTest.java index 1653e564..9bbca496 100644 --- a/core/src/test/java/com/google/googlejavaformat/java/FormatterTest.java +++ b/core/src/test/java/com/google/googlejavaformat/java/FormatterTest.java @@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertWithMessage; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; import com.google.common.base.Joiner; import com.google.common.io.CharStreams; @@ -492,4 +493,27 @@ public void removeTrailingTabsInComments() throws Exception { + " }\n" + "}\n"); } + + @Test + public void stringTemplateTests() throws Exception { + assumeTrue(Runtime.version().feature() >= 21); + assertThat( + new Formatter() + .formatSource( + "public class Foo {\n" + + " String test(){\n" + + " var simple = STR.\"mytemplate1XXXX \\{exampleXXXX.foo()}yyy\";\n" + + " var nested = STR.\"template \\{example. foo()+" + + " STR.\"templateInner\\{ example}\"}xxx }\";\n" + + " }\n" + + "}\n")) + .isEqualTo( + "public class Foo {\n" + + " String test() {\n" + + " var simple = STR.\"mytemplate1XXXX \\{exampleXXXX.foo()}yyy\";\n" + + " var nested = STR.\"template \\{example.foo() +" + + " STR.\"templateInner\\{example}\"}xxx }\";\n" + + " }\n" + + "}\n"); + } } diff --git a/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.input b/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.input new file mode 100644 index 00000000..98a19bba --- /dev/null +++ b/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.input @@ -0,0 +1,10 @@ +public class StringTemplates { + void test(){ + var m = STR."template \{example}xxx"; + var nested = STR."template \{example.foo()+ STR."templateInner\{example}"}xxx }"; + var nestNested = STR."template \{example0. + foo() + + STR."templateInner\{example1.test(STR."\{example2 + }")}"}xxx }"; + } +} diff --git a/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.output b/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.output new file mode 100644 index 00000000..e60bab70 --- /dev/null +++ b/core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.output @@ -0,0 +1,9 @@ +public class StringTemplates { + void test() { + var m = STR."template \{example}xxx"; + var nested = STR."template \{example.foo() + STR."templateInner\{example}"}xxx }"; + var nestNested = + STR."template \{ + example0.foo() + STR."templateInner\{example1.test(STR."\{example2}")}"}xxx }"; + } +} From d157ec879b43c4837f71eaafb45a6e6d58071a9a Mon Sep 17 00:00:00 2001 From: Liam Miller-Cushon Date: Wed, 3 Jan 2024 16:49:38 -0800 Subject: [PATCH 2/3] Handle `var` in record patterns https://github.com/google/google-java-format/issues/1020 PiperOrigin-RevId: 595537430 --- .../java/JavaInputAstVisitor.java | 5 ++-- .../java/java17/Java17InputAstVisitor.java | 25 +++++++++---------- .../java/FormatterIntegrationTest.java | 3 ++- .../java/testdata/I1020.input | 15 +++++++++++ .../java/testdata/I1020.output | 16 ++++++++++++ 5 files changed, 48 insertions(+), 16 deletions(-) create mode 100644 core/src/test/resources/com/google/googlejavaformat/java/testdata/I1020.input create mode 100644 core/src/test/resources/com/google/googlejavaformat/java/testdata/I1020.output diff --git a/core/src/main/java/com/google/googlejavaformat/java/JavaInputAstVisitor.java b/core/src/main/java/com/google/googlejavaformat/java/JavaInputAstVisitor.java index ea967b3d..8914c8ea 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/JavaInputAstVisitor.java +++ b/core/src/main/java/com/google/googlejavaformat/java/JavaInputAstVisitor.java @@ -3475,14 +3475,15 @@ private static boolean expressionsAreParallel( // General helper functions. - enum DeclarationKind { + /** Kind of declaration. */ + protected enum DeclarationKind { NONE, FIELD, PARAMETER } /** Declare one variable or variable-like thing. */ - int declareOne( + protected int declareOne( DeclarationKind kind, Direction annotationsDirection, Optional modifiers, diff --git a/core/src/main/java/com/google/googlejavaformat/java/java17/Java17InputAstVisitor.java b/core/src/main/java/com/google/googlejavaformat/java/java17/Java17InputAstVisitor.java index 6818f4a0..a7b5dbc4 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/java17/Java17InputAstVisitor.java +++ b/core/src/main/java/com/google/googlejavaformat/java/java17/Java17InputAstVisitor.java @@ -22,7 +22,6 @@ import com.google.googlejavaformat.OpsBuilder; import com.google.googlejavaformat.OpsBuilder.BlankLineWanted; import com.google.googlejavaformat.java.JavaInputAstVisitor; -import com.sun.source.tree.AnnotationTree; import com.sun.source.tree.BindingPatternTree; import com.sun.source.tree.BlockTree; import com.sun.source.tree.CaseLabelTree; @@ -84,18 +83,18 @@ public Void visitBindingPattern(BindingPatternTree node, Void unused) { private void visitBindingPattern(ModifiersTree modifiers, Tree type, Name name) { builder.open(plusFour); - if (modifiers != null) { - List annotations = - visitModifiers(modifiers, Direction.HORIZONTAL, Optional.empty()); - visitAnnotations(annotations, BreakOrNot.NO, BreakOrNot.YES); - } - scan(type, null); - builder.breakOp(" "); - if (name.isEmpty()) { - token("_"); - } else { - visit(name); - } + declareOne( + DeclarationKind.PARAMETER, + Direction.HORIZONTAL, + Optional.of(modifiers), + type, + name, + /* op= */ "", + /* equals= */ "", + /* initializer= */ Optional.empty(), + /* trailing= */ Optional.empty(), + /* receiverExpression= */ Optional.empty(), + /* typeWithDims= */ Optional.empty()); builder.close(); } diff --git a/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java b/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java index b31e7bb1..5fb35675 100644 --- a/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java +++ b/core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java @@ -61,7 +61,8 @@ public class FormatterIntegrationTest { "I880", "Unnamed", "I981", - "StringTemplate") + "StringTemplate", + "I1020") .build(); @Parameters(name = "{index}: {0}") diff --git a/core/src/test/resources/com/google/googlejavaformat/java/testdata/I1020.input b/core/src/test/resources/com/google/googlejavaformat/java/testdata/I1020.input new file mode 100644 index 00000000..8ff08426 --- /dev/null +++ b/core/src/test/resources/com/google/googlejavaformat/java/testdata/I1020.input @@ -0,0 +1,15 @@ +public sealed interface A { + record AA(Long a) implements A {} + record AAA(String b) implements A {} + static Long mySwitch(A a) { + switch (a) { + case AA(var aa) -> { + return aa; + } + case AAA(var b) -> { + return Long.parseLong(b); + } + } + } +} + diff --git a/core/src/test/resources/com/google/googlejavaformat/java/testdata/I1020.output b/core/src/test/resources/com/google/googlejavaformat/java/testdata/I1020.output new file mode 100644 index 00000000..21e4a515 --- /dev/null +++ b/core/src/test/resources/com/google/googlejavaformat/java/testdata/I1020.output @@ -0,0 +1,16 @@ +public sealed interface A { + record AA(Long a) implements A {} + + record AAA(String b) implements A {} + + static Long mySwitch(A a) { + switch (a) { + case AA(var aa) -> { + return aa; + } + case AAA(var b) -> { + return Long.parseLong(b); + } + } + } +} From ffb9d5bc094cc8782524ebbc386951b1ad334362 Mon Sep 17 00:00:00 2001 From: cushon Date: Fri, 5 Jan 2024 19:48:58 +0000 Subject: [PATCH 3/3] Release google-java-format 1.19.2 --- core/pom.xml | 2 +- eclipse_plugin/META-INF/MANIFEST.MF | 2 +- eclipse_plugin/pom.xml | 2 +- pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index d1363fed..3e1719f9 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ com.google.googlejavaformat google-java-format-parent - HEAD-SNAPSHOT + 1.19.2 google-java-format diff --git a/eclipse_plugin/META-INF/MANIFEST.MF b/eclipse_plugin/META-INF/MANIFEST.MF index 91324539..2f4a8734 100644 --- a/eclipse_plugin/META-INF/MANIFEST.MF +++ b/eclipse_plugin/META-INF/MANIFEST.MF @@ -3,7 +3,7 @@ Bundle-ManifestVersion: 2 Bundle-Name: google-java-format Bundle-SymbolicName: google-java-format-eclipse-plugin;singleton:=true Bundle-Vendor: Google -Bundle-Version: 1.13.0 +Bundle-Version: 1.19.2 Bundle-RequiredExecutionEnvironment: JavaSE-11 Require-Bundle: org.eclipse.jdt.core;bundle-version="3.10.0", org.eclipse.jface, diff --git a/eclipse_plugin/pom.xml b/eclipse_plugin/pom.xml index 9e6acdac..3bd7a63d 100644 --- a/eclipse_plugin/pom.xml +++ b/eclipse_plugin/pom.xml @@ -22,7 +22,7 @@ com.google.googlejavaformat google-java-format-eclipse-plugin eclipse-plugin - 1.13.0 + 1.19.2 Google Java Format Plugin for Eclipse 4.5+ diff --git a/pom.xml b/pom.xml index 8b3ca51c..94074c94 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.google.googlejavaformat google-java-format-parent pom - HEAD-SNAPSHOT + 1.19.2 core