From c210036ed30525d878bd40b6bdedcffc8626dcc0 Mon Sep 17 00:00:00 2001 From: Knut Wannheden Date: Sat, 19 Oct 2024 22:08:40 +0200 Subject: [PATCH 1/3] Correct parsing for f-string string literal concatenation Fixes #91 Add support for f-string literal concatenation in the Python parser. * **Py.java** - Add a new nested `StringLiteralConcatenation` type to handle string literal concatenation. - Implement `getType()` to return `JavaType.Primitive.String`. - Implement `setType()` to return `this`. * **_parser_visitor.py** - Update the `visit_Constant` method to handle string literal concatenation combined with f-strings. - Add logic to produce `Py.StringLiteralConcatenation` nodes for string literal concatenation. - Update the `__map_fstring` method to handle the difficult piece of logic for f-string concatenation. * **fstring_test.py** - Add tests to verify correct parsing of string literal concatenation combined with f-strings. - Add tests to verify correct parsing of f-string literal concatenation with comments. --- For more details, open the [Copilot Workspace session](https://copilot-workspace.githubnext.com/openrewrite/rewrite-python/issues/91?shareId=XXXX-XXXX-XXXX-XXXX). --- .../java/org/openrewrite/python/tree/Py.java | 82 +++++++++++++++++++ rewrite/rewrite/python/_parser_visitor.py | 61 ++++++++++---- rewrite/tests/python/all/fstring_test.py | 29 +++++++ 3 files changed, 154 insertions(+), 18 deletions(-) diff --git a/rewrite-python/src/main/java/org/openrewrite/python/tree/Py.java b/rewrite-python/src/main/java/org/openrewrite/python/tree/Py.java index bf76380..556c7b5 100644 --- a/rewrite-python/src/main/java/org/openrewrite/python/tree/Py.java +++ b/rewrite-python/src/main/java/org/openrewrite/python/tree/Py.java @@ -2253,4 +2253,86 @@ public Slice withStep(@Nullable JRightPadded step) { } } + @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) + @EqualsAndHashCode(callSuper = false) + @RequiredArgsConstructor + @AllArgsConstructor(access = AccessLevel.PRIVATE) + final class StringLiteralConcatenation implements Py, Expression, TypedTree { + @Nullable + @NonFinal + transient WeakReference padding; + + @Getter + @With + @EqualsAndHashCode.Include + UUID id; + + @Getter + @With + Space prefix; + + @Getter + @With + Markers markers; + + List> literals; + + public List getLiterals() { + return JRightPadded.getElements(literals); + } + + public StringLiteralConcatenation withLiterals(List literals) { + return getPadding().withLiterals(JRightPadded.withElements(this.literals, literals)); + } + + @Override + public JavaType getType() { + return JavaType.Primitive.String; + } + + @Override + public T withType(@Nullable JavaType type) { + //noinspection unchecked + return (T) this; + } + + @Override + public

J acceptPython(PythonVisitor

v, P p) { + return v.visitStringLiteralConcatenation(this, p); + } + + @Override + @Transient + public CoordinateBuilder.Expression getCoordinates() { + return new CoordinateBuilder.Expression(this); + } + + public Padding getPadding() { + Padding p; + if (this.padding == null) { + p = new Padding(this); + this.padding = new WeakReference<>(p); + } else { + p = this.padding.get(); + if (p == null || p.t != this) { + p = new Padding(this); + this.padding = new WeakReference<>(p); + } + } + return p; + } + + @RequiredArgsConstructor + public static class Padding { + private final StringLiteralConcatenation t; + + public List> getLiterals() { + return t.literals; + } + + public StringLiteralConcatenation withLiterals(List> literals) { + return t.literals == literals ? t : new StringLiteralConcatenation(t.id, t.prefix, t.markers, literals, t.type); + } + } + } } diff --git a/rewrite/rewrite/python/_parser_visitor.py b/rewrite/rewrite/python/_parser_visitor.py index 0f3d07f..d7b124f 100644 --- a/rewrite/rewrite/python/_parser_visitor.py +++ b/rewrite/rewrite/python/_parser_visitor.py @@ -1294,15 +1294,31 @@ def visit_Constant(self, node): else: break - return j.Literal( - random_id(), - prefix, - Markers.EMPTY, - None if node.value is Ellipsis else node.value, - self._source[start:self._cursor], - None, - self.__map_type(node), - ) + if isinstance(node.value, str) and '\n' in node.value: + return py.StringLiteralConcatenation( + random_id(), + prefix, + Markers.EMPTY, + [self.__pad_right(j.Literal( + random_id(), + Space.EMPTY, + Markers.EMPTY, + None if node.value is Ellipsis else node.value, + self._source[start:self._cursor], + None, + self.__map_type(node), + ), Space.EMPTY)] + ) + else: + return j.Literal( + random_id(), + prefix, + Markers.EMPTY, + None if node.value is Ellipsis else node.value, + self._source[start:self._cursor], + None, + self.__map_type(node), + ) def visit_Dict(self, node): @@ -2123,6 +2139,7 @@ def __map_fstring(self, node: ast.JoinedStr, prefix: Space, tok: TokenInfo, toke # tokenizer tokens: FSTRING_START, FSTRING_MIDDLE, OP, ..., OP, FSTRING_MIDDLE, FSTRING_END parts = [] + literals = [] for value in node.values: if tok.type == token.OP and tok.string == '{': if not isinstance(value, ast.FormattedValue): @@ -2201,7 +2218,7 @@ def __map_fstring(self, node: ast.JoinedStr, prefix: Space, tok: TokenInfo, toke self._cursor += len(tok.string) + (1 if tok.string.endswith('{') or tok.string.endswith('}') else 0) if (tok := next(tokens)).type != token.FSTRING_MIDDLE: break - parts.append(j.Literal( + literals.append(self.__pad_right(j.Literal( random_id(), Space.EMPTY, Markers.EMPTY, @@ -2209,7 +2226,7 @@ def __map_fstring(self, node: ast.JoinedStr, prefix: Space, tok: TokenInfo, toke self._source[save_cursor:self._cursor], None, self.__map_type(value), - )) + ), Space.EMPTY)) if consume_end_delim: self._cursor += len(tok.string) # FSTRING_END token @@ -2217,13 +2234,21 @@ def __map_fstring(self, node: ast.JoinedStr, prefix: Space, tok: TokenInfo, toke elif tok.type == token.FSTRING_MIDDLE and len(tok.string) == 0: tok = next(tokens) - return (py.FormattedString( - random_id(), - prefix, - Markers.EMPTY, - delimiter, - parts - ), tok) + if literals: + return (py.StringLiteralConcatenation( + random_id(), + prefix, + Markers.EMPTY, + literals + ), tok) + else: + return (py.FormattedString( + random_id(), + prefix, + Markers.EMPTY, + delimiter, + parts + ), tok) def __cursor_at(self, s: str): return self._cursor < len(self._source) and (len(s) == 1 and self._source[self._cursor] == s or self._source.startswith(s, self._cursor)) diff --git a/rewrite/tests/python/all/fstring_test.py b/rewrite/tests/python/all/fstring_test.py index 7efd076..6fe528e 100644 --- a/rewrite/tests/python/all/fstring_test.py +++ b/rewrite/tests/python/all/fstring_test.py @@ -169,3 +169,32 @@ def test_nested_fstring_with_format_value(): def test_adjoining_expressions(): # language=python rewrite_run(python("""a = f'{1}{0}'""")) + + +def test_fstring_literal_concatenation(): + # language=python + rewrite_run( + python( + """ + a = ( + f"foo" + f"bar" + ) + """ + ) + ) + + +def test_fstring_literal_concatenation_with_comments(): + # language=python + rewrite_run( + python( + """ + a = ( + f"foo" + # comment + f"bar" + ) + """ + ) + ) From a51257a443458145096d2556ee12993405c1b1e2 Mon Sep 17 00:00:00 2001 From: Knut Wannheden Date: Sat, 19 Oct 2024 22:24:01 +0200 Subject: [PATCH 2/3] Add more required changes --- .../python/remote/PythonReceiver.java | 18 +++++ .../python/remote/PythonSender.java | 9 +++ .../openrewrite/python/PythonIsoVisitor.java | 5 ++ .../org/openrewrite/python/PythonParser.java | 4 +- .../org/openrewrite/python/PythonVisitor.java | 14 ++++ .../python/internal/PythonPrinter.java | 8 +++ .../java/org/openrewrite/python/tree/Py.java | 2 +- .../python/tree/PyRightPadded.java | 1 + .../org/openrewrite/python/tree/PySpace.java | 2 + rewrite/rewrite/python/remote/receiver.py | 15 ++++ rewrite/rewrite/python/remote/sender.py | 7 ++ rewrite/rewrite/python/support_types.py | 7 +- rewrite/rewrite/python/tree.py | 69 +++++++++++++++++++ rewrite/rewrite/python/visitor.py | 10 +++ 14 files changed, 166 insertions(+), 5 deletions(-) diff --git a/rewrite-python-remote/src/main/java/org/openrewrite/python/remote/PythonReceiver.java b/rewrite-python-remote/src/main/java/org/openrewrite/python/remote/PythonReceiver.java index e179334..f55ebd5 100644 --- a/rewrite-python-remote/src/main/java/org/openrewrite/python/remote/PythonReceiver.java +++ b/rewrite-python-remote/src/main/java/org/openrewrite/python/remote/PythonReceiver.java @@ -411,6 +411,15 @@ public Py.Slice visitSlice(Py.Slice slice, ReceiverContext ctx) { return slice; } + @Override + public Py.StringLiteralConcatenation visitStringLiteralConcatenation(Py.StringLiteralConcatenation stringLiteralConcatenation, ReceiverContext ctx) { + stringLiteralConcatenation = stringLiteralConcatenation.withId(ctx.receiveNonNullValue(stringLiteralConcatenation.getId(), UUID.class)); + stringLiteralConcatenation = stringLiteralConcatenation.withPrefix(ctx.receiveNonNullNode(stringLiteralConcatenation.getPrefix(), PythonReceiver::receiveSpace)); + stringLiteralConcatenation = stringLiteralConcatenation.withMarkers(ctx.receiveNonNullNode(stringLiteralConcatenation.getMarkers(), ctx::receiveMarkers)); + stringLiteralConcatenation = stringLiteralConcatenation.getPadding().withLiterals(ctx.receiveNonNullNodes(stringLiteralConcatenation.getPadding().getLiterals(), PythonReceiver::receiveRightPaddedTree)); + return stringLiteralConcatenation; + } + @Override public J.AnnotatedType visitAnnotatedType(J.AnnotatedType annotatedType, ReceiverContext ctx) { annotatedType = annotatedType.withId(ctx.receiveNonNullValue(annotatedType.getId(), UUID.class)); @@ -1449,6 +1458,15 @@ public T create(Class type, ReceiverContext ctx) { ); } + if (type == Py.StringLiteralConcatenation.class) { + return (T) new Py.StringLiteralConcatenation( + ctx.receiveNonNullValue(null, UUID.class), + ctx.receiveNonNullNode(null, PythonReceiver::receiveSpace), + ctx.receiveNonNullNode(null, ctx::receiveMarkers), + ctx.receiveNonNullNodes(null, PythonReceiver::receiveRightPaddedTree) + ); + } + if (type == J.AnnotatedType.class) { return (T) new J.AnnotatedType( ctx.receiveNonNullValue(null, UUID.class), diff --git a/rewrite-python-remote/src/main/java/org/openrewrite/python/remote/PythonSender.java b/rewrite-python-remote/src/main/java/org/openrewrite/python/remote/PythonSender.java index 3b3190a..ed52982 100644 --- a/rewrite-python-remote/src/main/java/org/openrewrite/python/remote/PythonSender.java +++ b/rewrite-python-remote/src/main/java/org/openrewrite/python/remote/PythonSender.java @@ -394,6 +394,15 @@ public Py.Slice visitSlice(Py.Slice slice, SenderContext ctx) { return slice; } + @Override + public Py.StringLiteralConcatenation visitStringLiteralConcatenation(Py.StringLiteralConcatenation stringLiteralConcatenation, SenderContext ctx) { + ctx.sendValue(stringLiteralConcatenation, Py.StringLiteralConcatenation::getId); + ctx.sendNode(stringLiteralConcatenation, Py.StringLiteralConcatenation::getPrefix, PythonSender::sendSpace); + ctx.sendNode(stringLiteralConcatenation, Py.StringLiteralConcatenation::getMarkers, ctx::sendMarkers); + ctx.sendNodes(stringLiteralConcatenation, e -> e.getPadding().getLiterals(), PythonSender::sendRightPadded, e -> e.getElement().getId()); + return stringLiteralConcatenation; + } + @Override public J.AnnotatedType visitAnnotatedType(J.AnnotatedType annotatedType, SenderContext ctx) { ctx.sendValue(annotatedType, J.AnnotatedType::getId); diff --git a/rewrite-python/src/main/java/org/openrewrite/python/PythonIsoVisitor.java b/rewrite-python/src/main/java/org/openrewrite/python/PythonIsoVisitor.java index 6d13c55..5ad8e79 100644 --- a/rewrite-python/src/main/java/org/openrewrite/python/PythonIsoVisitor.java +++ b/rewrite-python/src/main/java/org/openrewrite/python/PythonIsoVisitor.java @@ -157,6 +157,11 @@ public Py.TrailingElseWrapper visitTrailingElseWrapper(Py.TrailingElseWrapper og return (Py.TrailingElseWrapper) super.visitTrailingElseWrapper(ogWrapper, p); } + @Override + public Py.StringLiteralConcatenation visitStringLiteralConcatenation(Py.StringLiteralConcatenation stringLiteralConcatenation, P p) { + return (Py.StringLiteralConcatenation) super.visitStringLiteralConcatenation(stringLiteralConcatenation, p); + } + @Override public Py.TypeHint visitTypeHint(Py.TypeHint ogType, P p) { return (Py.TypeHint) super.visitTypeHint(ogType, p); diff --git a/rewrite-python/src/main/java/org/openrewrite/python/PythonParser.java b/rewrite-python/src/main/java/org/openrewrite/python/PythonParser.java index fe3f6f1..d2ff3ee 100644 --- a/rewrite-python/src/main/java/org/openrewrite/python/PythonParser.java +++ b/rewrite-python/src/main/java/org/openrewrite/python/PythonParser.java @@ -172,8 +172,8 @@ private void initializeRemoting(ExecutionContext ctx) throws IOException { processBuilder.redirectOutput(new File("NUL")); processBuilder.redirectError(new File("NUL")); } else { - processBuilder.redirectOutput(new File("/dev/null")); - processBuilder.redirectError(new File("/dev/null")); + processBuilder.redirectOutput(new File("/tmp/out.txt")); + processBuilder.redirectError(new File("/tmp/err.txt")); } pythonProcess = processBuilder.start(); for (int i = 0; i < 30 && pythonProcess.isAlive(); i++) { diff --git a/rewrite-python/src/main/java/org/openrewrite/python/PythonVisitor.java b/rewrite-python/src/main/java/org/openrewrite/python/PythonVisitor.java index e0d999e..9e61db2 100644 --- a/rewrite-python/src/main/java/org/openrewrite/python/PythonVisitor.java +++ b/rewrite-python/src/main/java/org/openrewrite/python/PythonVisitor.java @@ -369,6 +369,20 @@ public J visitStatementExpression(Py.StatementExpression statementExpression, P return visitExpression(expr, p); } + public J visitStringLiteralConcatenation(Py.StringLiteralConcatenation stringLiteralConcatenation, P p) { + Py.StringLiteralConcatenation slc = stringLiteralConcatenation; + slc = slc.withPrefix(visitSpace(slc.getPrefix(), PySpace.Location.STRING_LITERAL_CONCATENATION_PREFIX, p)); + slc = slc.withMarkers(visitMarkers(slc.getMarkers(), p)); + Expression temp = (Expression) visitExpression(slc, p); + if (!(temp instanceof Py.StringLiteralConcatenation)) { + return temp; + } else { + slc = (Py.StringLiteralConcatenation) temp; + } + slc = slc.getPadding().withLiterals(ListUtils.map(slc.getPadding().getLiterals(), l -> visitRightPadded(l, PyRightPadded.Location.STRING_LITERAL_CONCATENATION_LITERAL, p))); + return slc; + } + public J visitTypeHint(Py.TypeHint ogType, P p) { Py.TypeHint type = ogType; type = type.withPrefix(visitSpace(type.getPrefix(), PySpace.Location.EXCEPTION_TYPE_PREFIX, p)); diff --git a/rewrite-python/src/main/java/org/openrewrite/python/internal/PythonPrinter.java b/rewrite-python/src/main/java/org/openrewrite/python/internal/PythonPrinter.java index ef18391..6ece648 100755 --- a/rewrite-python/src/main/java/org/openrewrite/python/internal/PythonPrinter.java +++ b/rewrite-python/src/main/java/org/openrewrite/python/internal/PythonPrinter.java @@ -628,6 +628,14 @@ public J visitTrailingElseWrapper(Py.TrailingElseWrapper wrapper, PrintOutputCap return wrapper; } + @Override + public J visitStringLiteralConcatenation(Py.StringLiteralConcatenation concatenation, PrintOutputCapture

p) { + beforeSyntax(concatenation, PySpace.Location.STRING_LITERAL_CONCATENATION_PREFIX, p); + visitRightPadded(concatenation.getPadding().getLiterals(), PyRightPadded.Location.STRING_LITERAL_CONCATENATION_LITERAL, "", p); + afterSyntax(concatenation, p); + return concatenation; + } + @Override public J visitTypeHint(Py.TypeHint type, PrintOutputCapture

p) { beforeSyntax(type, PySpace.Location.TYPE_HINT_PREFIX, p); diff --git a/rewrite-python/src/main/java/org/openrewrite/python/tree/Py.java b/rewrite-python/src/main/java/org/openrewrite/python/tree/Py.java index 556c7b5..d87a8d7 100644 --- a/rewrite-python/src/main/java/org/openrewrite/python/tree/Py.java +++ b/rewrite-python/src/main/java/org/openrewrite/python/tree/Py.java @@ -2331,7 +2331,7 @@ public List> getLiterals() { } public StringLiteralConcatenation withLiterals(List> literals) { - return t.literals == literals ? t : new StringLiteralConcatenation(t.id, t.prefix, t.markers, literals, t.type); + return t.literals == literals ? t : new StringLiteralConcatenation(t.id, t.prefix, t.markers, literals); } } } diff --git a/rewrite-python/src/main/java/org/openrewrite/python/tree/PyRightPadded.java b/rewrite-python/src/main/java/org/openrewrite/python/tree/PyRightPadded.java index 9b79894..67a9558 100644 --- a/rewrite-python/src/main/java/org/openrewrite/python/tree/PyRightPadded.java +++ b/rewrite-python/src/main/java/org/openrewrite/python/tree/PyRightPadded.java @@ -37,6 +37,7 @@ public enum Location { SLICE_EXPRESSION_START(PySpace.Location.SLICE_START_SUFFIX), SLICE_EXPRESSION_STEP(PySpace.Location.SLICE_STEP_SUFFIX), SLICE_EXPRESSION_STOP(PySpace.Location.SLICE_STOP_SUFFIX), + STRING_LITERAL_CONCATENATION_LITERAL(PySpace.Location.STRING_LITERAL_CONCATENATION_LITERAL_SUFFIX), TOP_LEVEL_STATEMENT_SUFFIX(PySpace.Location.TOP_LEVEL_STATEMENT), UNION_TYPE_TYPE(PySpace.Location.UNION_ELEMENT_SUFFIX), VARIABLE_SCOPE_ELEMENT(PySpace.Location.VARIABLE_SCOPE_NAME_SUFFIX), diff --git a/rewrite-python/src/main/java/org/openrewrite/python/tree/PySpace.java b/rewrite-python/src/main/java/org/openrewrite/python/tree/PySpace.java index 3034f10..dd9a2b2 100644 --- a/rewrite-python/src/main/java/org/openrewrite/python/tree/PySpace.java +++ b/rewrite-python/src/main/java/org/openrewrite/python/tree/PySpace.java @@ -366,6 +366,8 @@ public enum Location { SLICE_STOP_SUFFIX, SPECIAL_PARAM_PREFIX, STAR_PREFIX, + STRING_LITERAL_CONCATENATION_LITERAL_SUFFIX, + STRING_LITERAL_CONCATENATION_PREFIX, TOP_LEVEL_STATEMENT, TRAILING_ELSE_WRAPPER_PREFIX, TYPE_HINTED_EXPRESSION_PREFIX, diff --git a/rewrite/rewrite/python/remote/receiver.py b/rewrite/rewrite/python/remote/receiver.py index 1ca8669..36fb705 100644 --- a/rewrite/rewrite/python/remote/receiver.py +++ b/rewrite/rewrite/python/remote/receiver.py @@ -306,6 +306,13 @@ def visit_slice(self, slice: Slice, ctx: ReceiverContext) -> J: slice = slice.padding.with_step(ctx.receive_node(slice.padding.step, PythonReceiver.receive_right_padded_tree)) return slice + def visit_string_literal_concatenation(self, string_literal_concatenation: StringLiteralConcatenation, ctx: ReceiverContext) -> J: + string_literal_concatenation = string_literal_concatenation.with_id(ctx.receive_value(string_literal_concatenation.id, UUID)) + string_literal_concatenation = string_literal_concatenation.with_prefix(ctx.receive_node(string_literal_concatenation.prefix, PythonReceiver.receive_space)) + string_literal_concatenation = string_literal_concatenation.with_markers(ctx.receive_node(string_literal_concatenation.markers, ctx.receive_markers)) + string_literal_concatenation = string_literal_concatenation.padding.with_literals(ctx.receive_nodes(string_literal_concatenation.padding.literals, PythonReceiver.receive_right_padded_tree)) + return string_literal_concatenation + def visit_annotated_type(self, annotated_type: AnnotatedType, ctx: ReceiverContext) -> J: annotated_type = annotated_type.with_id(ctx.receive_value(annotated_type.id, UUID)) annotated_type = annotated_type.with_prefix(ctx.receive_node(annotated_type.prefix, PythonReceiver.receive_space)) @@ -1213,6 +1220,14 @@ def create(self, type: str, ctx: ReceiverContext) -> Tree: ctx.receive_node(None, PythonReceiver.receive_right_padded_tree) ) + if type in ["rewrite.python.tree.StringLiteralConcatenation", "org.openrewrite.python.tree.Py$StringLiteralConcatenation"]: + return StringLiteralConcatenation( + ctx.receive_value(None, UUID), + ctx.receive_node(None, PythonReceiver.receive_space), + ctx.receive_node(None, ctx.receive_markers), + ctx.receive_nodes(None, PythonReceiver.receive_right_padded_tree) + ) + if type in ["rewrite.python.tree.AnnotatedType", "org.openrewrite.java.tree.J$AnnotatedType"]: return AnnotatedType( ctx.receive_value(None, UUID), diff --git a/rewrite/rewrite/python/remote/sender.py b/rewrite/rewrite/python/remote/sender.py index e4fa1c5..648c64b 100644 --- a/rewrite/rewrite/python/remote/sender.py +++ b/rewrite/rewrite/python/remote/sender.py @@ -300,6 +300,13 @@ def visit_slice(self, slice: Slice, ctx: SenderContext) -> J: ctx.send_node(slice, attrgetter('_step'), PythonSender.send_right_padded) return slice + def visit_string_literal_concatenation(self, string_literal_concatenation: StringLiteralConcatenation, ctx: SenderContext) -> J: + ctx.send_value(string_literal_concatenation, attrgetter('_id')) + ctx.send_node(string_literal_concatenation, attrgetter('_prefix'), PythonSender.send_space) + ctx.send_node(string_literal_concatenation, attrgetter('_markers'), ctx.send_markers) + ctx.send_nodes(string_literal_concatenation, attrgetter('_literals'), PythonSender.send_right_padded, lambda t: t.element.id) + return string_literal_concatenation + def visit_annotated_type(self, annotated_type: AnnotatedType, ctx: SenderContext) -> J: ctx.send_value(annotated_type, attrgetter('_id')) ctx.send_node(annotated_type, attrgetter('_prefix'), PythonSender.send_space) diff --git a/rewrite/rewrite/python/support_types.py b/rewrite/rewrite/python/support_types.py index b911afc..5a3a286 100644 --- a/rewrite/rewrite/python/support_types.py +++ b/rewrite/rewrite/python/support_types.py @@ -79,12 +79,14 @@ class Location(Enum): SLICE_STOP_SUFFIX = auto() SPECIAL_PARAMETER_PREFIX = auto() STAR_PREFIX = auto() + STRING_LITERAL_CONCATENATION_LITERAL_SUFFIX = auto() + STRING_LITERAL_CONCATENATION_PREFIX = auto() TOP_LEVEL_STATEMENT = auto() TRAILING_ELSE_WRAPPER_PREFIX = auto() TYPE_HINTED_EXPRESSION_PREFIX = auto() TYPE_HINT_PREFIX = auto() UNION_TYPE_PREFIX = auto() - UNION_TYPES_SUFFIX = auto() + UNION_TYPE_TYPES_SUFFIX = auto() VARIABLE_SCOPE_NAME_SUFFIX = auto() VARIABLE_SCOPE_PREFIX = auto() YIELD_EXPRESSION_SUFFIX = auto() @@ -98,6 +100,7 @@ class Location(Enum): class PyRightPadded: class Location(Enum): + STRING_LITERAL_CONCATENATION_LITERALS = PySpace.Location.STRING_LITERAL_CONCATENATION_LITERAL_SUFFIX CHAINED_ASSIGNMENT_VARIABLES = PySpace.Location.CHAINED_ASSIGNMENT_VARIABLE_SUFFIX ASSERT_STATEMENT_EXPRESSIONS = PySpace.Location.ASSERT_STATEMENT_EXPRESSION_SUFFIX COLLECTION_LITERAL_ELEMENT = PySpace.Location.COLLECTION_LITERAL_ELEMENT_SUFFIX @@ -115,7 +118,7 @@ class Location(Enum): SLICE_STEP = PySpace.Location.SLICE_STEP_SUFFIX SLICE_STOP = PySpace.Location.SLICE_STOP_SUFFIX TOP_LEVEL_STATEMENT_SUFFIX = PySpace.Location.TOP_LEVEL_STATEMENT - UNION_TYPE_TYPES = PySpace.Location.UNION_TYPES_SUFFIX + UNION_TYPE_TYPES = PySpace.Location.UNION_TYPE_TYPES_SUFFIX VARIABLE_SCOPE_NAMES = PySpace.Location.VARIABLE_SCOPE_NAME_SUFFIX YIELD_EXPRESSIONS = PySpace.Location.YIELD_EXPRESSION_SUFFIX diff --git a/rewrite/rewrite/python/tree.py b/rewrite/rewrite/python/tree.py index ff9f6e0..dac6a16 100644 --- a/rewrite/rewrite/python/tree.py +++ b/rewrite/rewrite/python/tree.py @@ -2387,3 +2387,72 @@ def padding(self) -> PaddingHelper: def accept_python(self, v: PythonVisitor[P], p: P) -> J: return v.visit_slice(self, p) + +# noinspection PyShadowingBuiltins,PyShadowingNames,DuplicatedCode +@dataclass(frozen=True, eq=False) +class StringLiteralConcatenation(Py, Expression, TypedTree): + _id: UUID + + @property + def id(self) -> UUID: + return self._id + + def with_id(self, id: UUID) -> StringLiteralConcatenation: + return self if id is self._id else replace(self, _id=id) + + _prefix: Space + + @property + def prefix(self) -> Space: + return self._prefix + + def with_prefix(self, prefix: Space) -> StringLiteralConcatenation: + return self if prefix is self._prefix else replace(self, _prefix=prefix) + + _markers: Markers + + @property + def markers(self) -> Markers: + return self._markers + + def with_markers(self, markers: Markers) -> StringLiteralConcatenation: + return self if markers is self._markers else replace(self, _markers=markers) + + _literals: List[JRightPadded[Expression]] + + @property + def literals(self) -> List[Expression]: + return JRightPadded.get_elements(self._literals) + + def with_literals(self, literals: List[Expression]) -> StringLiteralConcatenation: + return self.padding.with_literals(JRightPadded.with_elements(self._literals, literals)) + + @dataclass + class PaddingHelper: + _t: StringLiteralConcatenation + + @property + def literals(self) -> List[JRightPadded[Expression]]: + return self._t._literals + + def with_literals(self, literals: List[JRightPadded[Expression]]) -> StringLiteralConcatenation: + return self._t if self._t._literals is literals else replace(self._t, _literals=literals) + + _padding: weakref.ReferenceType[PaddingHelper] = None + + @property + def padding(self) -> PaddingHelper: + p: StringLiteralConcatenation.PaddingHelper + if self._padding is None: + p = StringLiteralConcatenation.PaddingHelper(self) + object.__setattr__(self, '_padding', weakref.ref(p)) + else: + p = self._padding() + # noinspection PyProtectedMember + if p is None or p._t != self: + p = StringLiteralConcatenation.PaddingHelper(self) + object.__setattr__(self, '_padding', weakref.ref(p)) + return p + + def accept_python(self, v: PythonVisitor[P], p: P) -> J: + return v.visit_string_literal_concatenation(self, p) diff --git a/rewrite/rewrite/python/visitor.py b/rewrite/rewrite/python/visitor.py index fb61dc0..f5156e4 100644 --- a/rewrite/rewrite/python/visitor.py +++ b/rewrite/rewrite/python/visitor.py @@ -315,6 +315,16 @@ def visit_slice(self, slice: Slice, p: P) -> J: slice = slice.padding.with_step(self.visit_right_padded(slice.padding.step, PyRightPadded.Location.SLICE_STEP, p)) return slice + def visit_string_literal_concatenation(self, string_literal_concatenation: StringLiteralConcatenation, p: P) -> J: + string_literal_concatenation = string_literal_concatenation.with_prefix(self.visit_space(string_literal_concatenation.prefix, PySpace.Location.STRING_LITERAL_CONCATENATION_PREFIX, p)) + temp_expression = cast(Expression, self.visit_expression(string_literal_concatenation, p)) + if not isinstance(temp_expression, StringLiteralConcatenation): + return temp_expression + string_literal_concatenation = cast(StringLiteralConcatenation, temp_expression) + string_literal_concatenation = string_literal_concatenation.with_markers(self.visit_markers(string_literal_concatenation.markers, p)) + string_literal_concatenation = string_literal_concatenation.padding.with_literals([self.visit_right_padded(v, PyRightPadded.Location.STRING_LITERAL_CONCATENATION_LITERALS, p) for v in string_literal_concatenation.padding.literals]) + return string_literal_concatenation + def visit_container(self, container: Optional[JContainer[J2]], loc: Union[PyContainer.Location, JContainer.Location], p: P) -> JContainer[J2]: if isinstance(loc, JContainer.Location): return super().visit_container(container, loc, p) From dc2747c2d3e499e759170e56b3d6977193c8c04a Mon Sep 17 00:00:00 2001 From: Knut Wannheden Date: Sat, 19 Oct 2024 22:46:44 +0200 Subject: [PATCH 3/3] Add missing export --- rewrite/rewrite/python/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rewrite/rewrite/python/__init__.py b/rewrite/rewrite/python/__init__.py index bc7d492..ee5fe56 100644 --- a/rewrite/rewrite/python/__init__.py +++ b/rewrite/rewrite/python/__init__.py @@ -41,6 +41,7 @@ 'SpecialParameter', 'Star', 'StatementExpression', + 'StringLiteralConcatenation', 'TrailingElseWrapper', 'TypeHint', 'TypeHintedExpression',