diff --git a/freemarker-core/src/main/java/freemarker/core/LegacyConstructorParserConfiguration.java b/freemarker-core/src/main/java/freemarker/core/LegacyConstructorParserConfiguration.java index e2f51b5a6..5b4bde671 100644 --- a/freemarker-core/src/main/java/freemarker/core/LegacyConstructorParserConfiguration.java +++ b/freemarker-core/src/main/java/freemarker/core/LegacyConstructorParserConfiguration.java @@ -37,6 +37,8 @@ class LegacyConstructorParserConfiguration implements ParserConfiguration { private OutputFormat outputFormat; private Boolean recognizeStandardFileExtensions; private Integer tabSize; + private String normalizedEol; + private boolean normalizedEolSet; private final Version incompatibleImprovements; LegacyConstructorParserConfiguration(boolean strictSyntaxMode, boolean whitespaceStripping, @@ -150,6 +152,21 @@ public int getTabSize() { } return tabSize.intValue(); } + + @Override + public String getNormalizedEol() { + if (!normalizedEolSet) { + throw new IllegalStateException(); + } + return normalizedEol; + } + + void setNormalizedEolIfNotSet(String normalizedEol) { + if (!normalizedEolSet) { + this.normalizedEol = normalizedEol; + this.normalizedEolSet = true; + } + } void setTabSizeIfNotSet(int tabSize) { if (this.tabSize == null) { diff --git a/freemarker-core/src/main/java/freemarker/core/ParserConfiguration.java b/freemarker-core/src/main/java/freemarker/core/ParserConfiguration.java index 8c125dadb..e44793f21 100644 --- a/freemarker-core/src/main/java/freemarker/core/ParserConfiguration.java +++ b/freemarker-core/src/main/java/freemarker/core/ParserConfiguration.java @@ -89,4 +89,12 @@ public interface ParserConfiguration { */ int getTabSize(); + /** + * See {@link Configuration#setNormalizedEol(String)}; {@code null} if the output shouldn't be made to use any + * particular line break. + * + * @since 2.3.36 + */ + String getNormalizedEol(); + } diff --git a/freemarker-core/src/main/java/freemarker/core/TemplateConfiguration.java b/freemarker-core/src/main/java/freemarker/core/TemplateConfiguration.java index 121c18b6b..3fa54b78a 100644 --- a/freemarker-core/src/main/java/freemarker/core/TemplateConfiguration.java +++ b/freemarker-core/src/main/java/freemarker/core/TemplateConfiguration.java @@ -88,6 +88,8 @@ public final class TemplateConfiguration extends Configurable implements ParserC private OutputFormat outputFormat; private String encoding; private Integer tabSize; + private String normalizedEol; + private boolean normalizedEolSet; /** * Creates a new instance. The parent will be {@link Configuration#getDefaultConfiguration()} initially, but it will @@ -224,6 +226,9 @@ public void merge(TemplateConfiguration tc) { if (tc.isOutputEncodingSet()) { setOutputEncoding(tc.getOutputEncoding()); } + if (tc.isNormalizedEolSet()) { + setNormalizedEol(tc.getNormalizedEol()); + } if (tc.isOutputFormatSet()) { setOutputFormat(tc.getOutputFormat()); } @@ -266,6 +271,9 @@ public void merge(TemplateConfiguration tc) { if (tc.isTabSizeSet()) { setTabSize(tc.getTabSize()); } + if (tc.isNormalizedEolSet()) { + setNormalizedEol(tc.getNormalizedEol()); + } if (tc.isLazyImportsSet()) { setLazyImports(tc.getLazyImports()); } @@ -645,6 +653,35 @@ public int getTabSize() { return tabSize != null ? tabSize.intValue() : getNonNullParentConfiguration().getTabSize(); } + + /** + * See {@link Configuration#setNormalizedEol(String)}. + * + * @since 2.3.36 + */ + public void setNormalizedEol(String normalizedEol) { + this.normalizedEol = normalizedEol; + this.normalizedEolSet = true; + } + + /** + * Getter pair of {@link #setNormalizedEol(String)}. + * + * @since 2.3.36 + */ + @Override + public String getNormalizedEol() { + return normalizedEolSet ? normalizedEol : getNonNullParentConfiguration().getNormalizedEol(); + } + + /** + * Tells if this setting is set in this object, as opposed to being inherited. + * + * @since 2.3.36 + */ + public boolean isNormalizedEolSet() { + return normalizedEolSet; + } /** * Tells if this setting is set directly in this object or its value is coming from the {@link #getParent() parent}. diff --git a/freemarker-core/src/main/java/freemarker/core/_ParserConfigurationWithInheritedFormat.java b/freemarker-core/src/main/java/freemarker/core/_ParserConfigurationWithInheritedFormat.java index 82d20a030..f43715c5d 100644 --- a/freemarker-core/src/main/java/freemarker/core/_ParserConfigurationWithInheritedFormat.java +++ b/freemarker-core/src/main/java/freemarker/core/_ParserConfigurationWithInheritedFormat.java @@ -90,5 +90,10 @@ public ArithmeticEngine getArithmeticEngine() { public int getTabSize() { return wrappedPCfg.getTabSize(); } - + + @Override + public String getNormalizedEol() { + return wrappedPCfg.getNormalizedEol(); + } + } \ No newline at end of file diff --git a/freemarker-core/src/main/java/freemarker/template/Configuration.java b/freemarker-core/src/main/java/freemarker/template/Configuration.java index 4cf6ce613..3b9796d4d 100644 --- a/freemarker-core/src/main/java/freemarker/template/Configuration.java +++ b/freemarker-core/src/main/java/freemarker/template/Configuration.java @@ -573,6 +573,7 @@ public class Configuration extends Configurable implements Cloneable, ParserConf private int interpolationSyntax = LEGACY_INTERPOLATION_SYNTAX; private int namingConvention = AUTO_DETECT_NAMING_CONVENTION; private int tabSize = 8; // Default from JavaCC 3.x + private String normalizedEol; private boolean fallbackOnNullLoopVariable = true; // Default for backward compatibility private boolean preventStrippings; @@ -2808,6 +2809,52 @@ public int getTabSize() { return tabSize; } + /** + * Sets the end-of-line (line break) string that the output of the templates should use, or {@code null} (the + * default) to not prescribe any. This is a parser-level setting, as it's applied when the template is parsed; + * it therefore can't be changed with the {@code setting} directive from inside a template, but it can differ + * per template, via {@link TemplateConfiguration}. + * + *

It affects two things: + * + *

+ * + *

Note that the values inserted by ${...} are never affected; this setting is about the + * template, not about the data. + * + *

Usually you set this to {@code "\n"} or {@code "\r\n"}. Prefer specifying the line break explicitly + * rather than deriving it from the machine the template is executed on, as then the output doesn't depend on + * that, which matters if it's stored in a version control system, for example. + * + * @param normalizedEol + * The line break to use, or {@code null} to not prescribe any. Should be {@code "\n"}, + * {@code "\r\n"} or {@code "\r"}, though other values aren't rejected. + * + * @since 2.3.36 + */ + public void setNormalizedEol(String normalizedEol) { + this.normalizedEol = normalizedEol; + } + + /** + * The getter pair of {@link #setNormalizedEol(String)}. + * + * @since 2.3.36 + */ + @Override + public String getNormalizedEol() { + return normalizedEol; + } + /** * The getter pair of {@link #setFallbackOnNullLoopVariable(boolean)}. * diff --git a/freemarker-core/src/main/java/freemarker/template/utility/StringUtil.java b/freemarker-core/src/main/java/freemarker/template/utility/StringUtil.java index c4664d48c..3ed076797 100644 --- a/freemarker-core/src/main/java/freemarker/template/utility/StringUtil.java +++ b/freemarker-core/src/main/java/freemarker/template/utility/StringUtil.java @@ -39,7 +39,7 @@ * Some text related utilities. */ public class StringUtil { - + /** * Used to look up if the chars with low code needs to be escaped, but note that it gives bad result for '=', as * there the it matters if it's after '['. @@ -557,6 +557,18 @@ private static boolean isInterpolationStart(char c) { * @throws ParseException if there string contains illegal escapes */ public static String FTLStringLiteralDec(String s) throws ParseException { + return FTLStringLiteralDec(s, null); + } + + /** + * Same as {@link #FTLStringLiteralDec(String)}, but you can say what the \R escape stands for. + * + * @param normalizedEol + * The value of the {@code normalized_eol} setting, or {@code null} for a line feed. + * + * @since 2.3.36 + */ + public static String FTLStringLiteralDec(String s, String normalizedEol) throws ParseException { int idx = s.indexOf('\\'); if (idx == -1) { @@ -589,6 +601,10 @@ public static String FTLStringLiteralDec(String s) throws ParseException { buf.append('\n'); bidx = idx + 2; break; + case 'R': + buf.append(normalizedEol != null ? normalizedEol : "\n"); + bidx = idx + 2; + break; case 'r': buf.append('\r'); bidx = idx + 2; diff --git a/freemarker-core/src/main/javacc/freemarker/core/FTL.jj b/freemarker-core/src/main/javacc/freemarker/core/FTL.jj index 871956a2b..259b0d38a 100644 --- a/freemarker-core/src/main/javacc/freemarker/core/FTL.jj +++ b/freemarker-core/src/main/javacc/freemarker/core/FTL.jj @@ -220,6 +220,8 @@ public class FMParser { template.getParserConfiguration().getRecognizeStandardFileExtensions()); lpCfg.setTabSizeIfNotSet( template.getParserConfiguration().getTabSize()); + lpCfg.setNormalizedEolIfNotSet( + template.getConfiguration().getNormalizedEol()); } int incompatibleImprovements = pCfg.getIncompatibleImprovements().intValue(); @@ -394,6 +396,48 @@ public class FMParser { return token_source.namingConvention; } + /** + * Replaces the line breaks of static template text with the {@code normalized_eol} setting value, if that + * setting is set. Done here, when the template is parsed, so that executing it costs nothing. + * + * @return {@code s} itself if there's nothing to replace. + */ + private String normalizeEol(String s) { + String normalizedEol = pCfg.getNormalizedEol(); + if (normalizedEol == null) { + return s; + } + + int len = s.length(); + boolean needsReplacement = false; + for (int i = 0; i < len; i++) { + char c = s.charAt(i); + if (c == '\r' || (c == '\n' && !normalizedEol.equals("\n"))) { + needsReplacement = true; + break; + } + } + if (!needsReplacement) { + return s; + } + + StringBuilder sb = new StringBuilder(len + 16); + for (int i = 0; i < len; i++) { + char c = s.charAt(i); + if (c == '\r') { + sb.append(normalizedEol); + if (i + 1 < len && s.charAt(i + 1) == '\n') { + i++; // CRLF is one line break + } + } else if (c == '\n') { + sb.append(normalizedEol); + } else { + sb.append(c); + } + } + return sb.toString(); + } + /** * Throw an exception if the expression passed in is a String Literal */ @@ -1224,7 +1268,7 @@ TOKEN: <#ESCAPED_CHAR : "\\" ( - ("n" | "t" | "r" | "f" | "b" | "g" | "l" | "a" | "\\" | "'" | "\"" | "{" | "=") + ("n" | "t" | "r" | "f" | "b" | "g" | "l" | "a" | "R" | "\\" | "'" | "\"" | "{" | "=") | ("x" ["0"-"9", "A"-"F", "a"-"f"]) ) @@ -2510,7 +2554,7 @@ StringLiteral StringLiteral(boolean interpolate) : s = t.image.substring(2, t.image.length() -1); } else { try { - s = StringUtil.FTLStringLiteralDec(t.image.substring(1, t.image.length() -1)); + s = StringUtil.FTLStringLiteralDec(t.image.substring(1, t.image.length() -1), pCfg.getNormalizedEol()); } catch (ParseException pe) { pe.lineNumber = t.beginLine; pe.columnNumber = t.beginColumn; @@ -3847,7 +3891,7 @@ TextBlock NoParse() : start = end = UnparsedContent(start, buf) { - TextBlock result = new TextBlock(buf.toString(), true); + TextBlock result = new TextBlock(normalizeEol(buf.toString()), true); result.setLocation(template, start, end); return result; } @@ -4374,7 +4418,7 @@ TextBlock PCData() : { if (stripText && mixedContentNesting == 1 && !preventStrippings) return null; - TextBlock result = new TextBlock(buf.toString(), false); + TextBlock result = new TextBlock(normalizeEol(buf.toString()), false); result.setLocation(template, start, t); return result; } @@ -4389,7 +4433,7 @@ TextBlock WhitespaceText() : { if (stripText && mixedContentNesting == 1 && !preventStrippings) return null; - TextBlock result = new TextBlock(t.image, false); + TextBlock result = new TextBlock(normalizeEol(t.image), false); result.setLocation(template, t, t); return result; } diff --git a/freemarker-core/src/test/java/freemarker/core/NormalizedEolTest.java b/freemarker-core/src/test/java/freemarker/core/NormalizedEolTest.java new file mode 100644 index 000000000..d7dc0ed12 --- /dev/null +++ b/freemarker-core/src/test/java/freemarker/core/NormalizedEolTest.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package freemarker.core; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; + +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import freemarker.template.Configuration; +import freemarker.template.Template; +import freemarker.template.TemplateException; +import freemarker.test.TemplateTest; + +@RunWith(Enclosed.class) +public class NormalizedEolTest { + + /** + * Tests that the {@code normalized_eol} setting normalizes the line breaks of the static text of the template, + * whatever line breaks the template file itself uses. + */ + @RunWith(Parameterized.class) + public static class StaticTextTest extends TemplateTest { + + @Parameters(name = "templateEol={0}") + public static Collection parameters() { + return Arrays.asList(new Object[][] { { "LF", "\n" }, { "CRLF", "\r\n" }, { "CR", "\r" } }); + } + + private final String templateEol; + + public StaticTextTest(String templateEolName, String templateEol) { + this.templateEol = templateEol; + } + + /** The same template content, but with the line breaks of the template file we are simulating. */ + private String tpl() { + return ("line 1\nline 2\n").replace("\n", templateEol); + } + + @Test + public void testUnsetKeepsTemplateEol() throws Exception { + // This is how FreeMarker behaved before 2.3.36, and still does if the setting isn't set. + assertEquals(tpl(), processToString(tpl())); + } + + @Test + public void testNormalizesToLf() throws Exception { + getConfiguration().setNormalizedEol("\n"); + assertEquals("line 1\nline 2\n", processToString(tpl())); + } + + @Test + public void testNormalizesToCrLf() throws Exception { + getConfiguration().setNormalizedEol("\r\n"); + assertEquals("line 1\r\nline 2\r\n", processToString(tpl())); + } + + @Test + public void testNormalizesToCr() throws Exception { + getConfiguration().setNormalizedEol("\r"); + assertEquals("line 1\rline 2\r", processToString(tpl())); + } + + private String processToString(String template) throws IOException, TemplateException { + return getOutput(template); + } + } + + /** + * Tests of the \R escape, and of what the setting does and doesn't affect. + */ + public static class EscapeAndScopeTest extends TemplateTest { + + @Test + public void testEscapeDefaultsToLf() throws Exception { + assertOutput("${'a\\Rb'}", "a\nb"); + } + + @Test + public void testEscapeFollowsSetting() throws Exception { + getConfiguration().setNormalizedEol("\r\n"); + assertOutput("${'a\\Rb'}", "a\r\nb"); + getConfiguration().setNormalizedEol("\r"); + assertOutput("${'a\\Rb'}", "a\rb"); + } + + @Test + public void testEscapeInInterpolatedLiteral() throws Exception { + getConfiguration().setNormalizedEol("\r\n"); + addToDataModel("x", "X"); + assertOutput("${'a\\R${x}\\Rb'}", "a\r\nX\r\nb"); + } + + @Test + public void testLiteralBackslashNIsNotAffected() throws Exception { + // \n always gives a line feed; that's the whole difference between it and \R. + getConfiguration().setNormalizedEol("\r\n"); + assertOutput("${'a\\nb'}", "a\nb"); + } + + @Test + public void testLiteralBackslashRIsNotAffected() throws Exception { + // \R is case-sensitively distinct from \r, which still means carriage return only. + getConfiguration().setNormalizedEol("\n"); + assertOutput("${'a\\rb'}", "a\rb"); + } + + @Test + public void testInterpolatedValuesAreNotAffected() throws Exception { + // The setting is about the template, not about the data. + getConfiguration().setNormalizedEol("\r\n"); + addToDataModel("x", "p\nq"); + assertOutput("${x}", "p\nq"); + } + + @Test + public void testStringConcatenationOfEscape() throws Exception { + getConfiguration().setNormalizedEol("\r\n"); + assertOutput("${'a' + '\\R' + 'b'}", "a\r\nb"); + } + + @Test + public void testUnknownEscapeIsStillRejected() throws Exception { + // Sanity check that adding "R" to the accepted escapes didn't make all letters pass: + assertErrorContains("${'a\\qb'}", "Lexical error"); + } + } + + /** + * Tests of the setting itself. It's a parser-level setting, so it's applied when the template is parsed: it + * can't be changed from inside a template, but it can differ per template. + */ + public static class SettingTest extends TemplateTest { + + @Test + public void testDefaultIsNull() throws Exception { + assertNull(getConfiguration().getNormalizedEol()); + } + + @Test + public void testNotSettableFromInsideATemplate() throws Exception { + // A parser setting can't be changed while the template runs. + assertErrorContains("<#setting normalized_eol='\\r\\n'>", "normalized_eol"); + } + + @Test + public void testAppliedWhenTheTemplateIsParsed() throws Exception { + Configuration cfg = getConfiguration(); + cfg.setNormalizedEol("\r\n"); + assertEquals("a\r\nb", getOutput("a\nb")); + assertEquals("a\r\nb", getOutput("${'a\\Rb'}")); + } + + @Test + public void testChangingItAfterParsingHasNoEffect() throws Exception { + // The template was already parsed with the old value, so it keeps it. + Configuration cfg = getConfiguration(); + cfg.setNormalizedEol("\r\n"); + Template t = new Template("t", new StringReader("a\nb"), cfg); + cfg.setNormalizedEol("\n"); + StringWriter sw = new StringWriter(); + t.process(new HashMap(), sw); + assertEquals("a\r\nb", sw.toString()); + } + + @Test + public void testCanDifferPerTemplate() throws Exception { + Configuration cfg = getConfiguration(); + TemplateConfiguration tc = new TemplateConfiguration(); + tc.setParentConfiguration(cfg); + tc.setNormalizedEol("\r\n"); + + StringWriter sw = new StringWriter(); + new Template("crlf", null, new StringReader("a\nb"), cfg, tc, null) + .process(new HashMap(), sw); + assertEquals("a\r\nb", sw.toString()); + + // The same source, without that TemplateConfiguration, is unaffected. + assertEquals("a\nb", getOutput("a\nb")); + } + } +} diff --git a/freemarker-core/src/test/java/freemarker/core/TemplateConfigurationTest.java b/freemarker-core/src/test/java/freemarker/core/TemplateConfigurationTest.java index be3ed16bb..03b103a7e 100644 --- a/freemarker-core/src/test/java/freemarker/core/TemplateConfigurationTest.java +++ b/freemarker-core/src/test/java/freemarker/core/TemplateConfigurationTest.java @@ -192,6 +192,7 @@ public Number toNumber(String s) { SETTING_ASSIGNMENTS.put("outputFormat", HTMLOutputFormat.INSTANCE); SETTING_ASSIGNMENTS.put("recognizeStandardFileExtensions", true); SETTING_ASSIGNMENTS.put("tabSize", 1); + SETTING_ASSIGNMENTS.put("normalizedEol", "\r\n"); SETTING_ASSIGNMENTS.put("lazyImports", Boolean.TRUE); SETTING_ASSIGNMENTS.put("lazyAutoImports", Boolean.FALSE); SETTING_ASSIGNMENTS.put("autoImports", ImmutableMap.of("a", "/lib/a.ftl")); @@ -694,7 +695,17 @@ public void testConfigureParser() throws Exception { "13", "8"); testedProps.add(Configuration.TAB_SIZE_KEY_CAMEL_CASE); } - + + { + TemplateConfiguration tc = new TemplateConfiguration(); + tc.setLogTemplateExceptions(false); + tc.setParentConfiguration(new Configuration(new Version(2, 3, 22))); + tc.setNormalizedEol("\r\n"); + // The line breaks of the static text are replaced when the template is parsed: + assertOutputWithoutAndWithTC(tc, "a\nb", "a\nb", "a\r\nb"); + testedProps.add("normalizedEol"); + } + assertEquals("Check that you have tested all parser settings; ", PARSER_PROP_NAMES, testedProps); } diff --git a/freemarker-manual/src/main/docgen/en_US/book.xml b/freemarker-manual/src/main/docgen/en_US/book.xml index cfbfba9c9..df661a62d 100644 --- a/freemarker-manual/src/main/docgen/en_US/book.xml +++ b/freemarker-manual/src/main/docgen/en_US/book.xml @@ -2315,6 +2315,20 @@ this is a backslash: \ Carriage return (u000D) + + \R + + The end-of-line (line break) that the output should use; + that is, the value of the normalized_eol + setting (see + Configuration.setNormalizedEol(String)), + or a line feed (u000A) if that setting isn't set. + Unlike \n and \r\n, + which always give the same characters, this adapts to how the + output is supposed to look. (Supported since FreeMarker + 2.3.36.) + + \t @@ -30996,6 +31010,43 @@ TemplateModel x = env.getVariable("x"); // get variable x Version history +

+ 2.3.36 + + Release date: Not released yet + +
+ Changes on the FTL side + + + + Added the normalized_eol setting, which + prescribes the end-of-line (line break) that the output should + use. When it's set, the line breaks of the static text of the + template are all replaced with it, and so the line breaks that + the template file happens to use don't leak into the output. + Values inserted by + ${...} are never + affected. It defaults to not being set, in which case FreeMarker + behaves like before. It's a parser-level setting, applied when + the template is parsed, so it can't be changed with the + setting directive, but it can differ per + template via TemplateConfiguration. See + Configuration.setNormalizedEol(String). + + + + Added the \R escape sequence of string + literals, which gives the value of the + normalized_eol setting (or a line feed, if + that setting isn't set). Unlike \n, it + adapts to how the output is supposed to look. See more here... + + +
+
+
2.3.35