From b860d595d50dac753a676348962a8f65b48534fa Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Tue, 23 Jun 2020 15:31:43 +0800 Subject: [PATCH 01/27] Start writing up the serializer --- fluent-syntax/src/lib.rs | 1 + fluent-syntax/src/serializer.rs | 343 ++++++++++++++++++++++++++++++++ 2 files changed, 344 insertions(+) create mode 100644 fluent-syntax/src/serializer.rs diff --git a/fluent-syntax/src/lib.rs b/fluent-syntax/src/lib.rs index 3c1b8715..d22003e2 100644 --- a/fluent-syntax/src/lib.rs +++ b/fluent-syntax/src/lib.rs @@ -1,6 +1,7 @@ pub mod ast; pub mod parser; pub mod unicode; +pub mod serializer; #[cfg(feature = "json")] pub mod json; diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs new file mode 100644 index 00000000..0df4f1cd --- /dev/null +++ b/fluent-syntax/src/serializer.rs @@ -0,0 +1,343 @@ +use crate::ast::*; +use std::io::{Error, Write}; + +pub fn serialize(resource: &Resource<'_>) -> String { + let mut buffer = Vec::new(); + let options = Options::default(); + + Serializer::new(&mut buffer, options) + .serialize_resource(resource) + .expect("Writing to an in-memory buffer never fails"); + + String::from_utf8(buffer).expect("The serializer only ever emits valid UTF-8") +} + +#[derive(Debug)] +pub struct Serializer { + writer: W, + options: Options, + state: State, +} + +impl Serializer { + pub fn new(writer: W, options: Options) -> Self { + Serializer { + writer, + options, + state: State::default(), + } + } + + pub fn serialize_resource(&mut self, res: &Resource<'_>) -> Result<(), Error> { + for entry in &res.body { + match entry { + ResourceEntry::Entry(entry) => self.serialize_entry(entry)?, + ResourceEntry::Junk(junk) if self.options.with_junk => self.serialize_junk(junk)?, + ResourceEntry::Junk(_) => continue, + } + + self.state.has_entries = true; + } + + Ok(()) + } + + fn serialize_entry(&mut self, entry: &Entry<'_>) -> Result<(), Error> { + match entry { + Entry::Message(msg) => self.serialize_message(msg), + Entry::Comment(comment) => self.serialize_comment(comment), + Entry::Term(term) => self.serialize_term(term), + } + } + + fn serialize_junk(&mut self, junk: &str) -> Result<(), Error> { + write!(self.writer, "{}", junk) + } + + fn serialize_comment(&mut self, comment: &Comment<'_>) -> Result<(), Error> { + let (prefix, lines) = match comment { + Comment::Comment { content } => ("#", content), + Comment::GroupComment { content } => ("##", content), + Comment::ResourceComment { content } => ("###", content), + }; + + if self.state.has_entries { + writeln!(self.writer)?; + } + + for line in lines { + writeln!(self.writer, "{} {}", prefix, line)?; + } + + Ok(()) + } + + fn serialize_message(&mut self, msg: &Message<'_>) -> Result<(), Error> { + if let Some(comment) = msg.comment.as_ref() { + self.serialize_comment(comment)?; + } + + write!(self.writer, "{} =", msg.id.name)?; + + if let Some(value) = msg.value.as_ref() { + self.serialize_pattern(value)?; + } + + for attr in &msg.attributes { + self.serialize_attribute(attr)?; + } + + writeln!(self.writer)?; + Ok(()) + } + + fn serialize_term(&mut self, term: &Term<'_>) -> Result<(), Error> { + if let Some(comment) = term.comment.as_ref() { + self.serialize_comment(comment)?; + } + + write!(self.writer, "{} =", term.id.name)?; + self.serialize_pattern(&term.value)?; + + for attr in &term.attributes { + self.serialize_attribute(attr)?; + } + + writeln!(self.writer)?; + + Ok(()) + } + + fn serialize_pattern(&mut self, pattern: &Pattern<'_>) -> Result<(), Error> { + let start_on_newline = pattern.elements.iter().any(|elem| match elem { + PatternElement::TextElement(text) => text.contains("\n"), + PatternElement::Placeable(expr) => is_select_expr(expr), + }); + + if start_on_newline { + unimplemented!("Write a new line then indent everything afterwards"); + } + + write!(self.writer, " ")?; + + for element in &pattern.elements { + self.serialize_element(element)?; + } + + Ok(()) + } + + fn serialize_attribute(&mut self, attr: &Attribute<'_>) -> Result<(), Error> { + writeln!(self.writer)?; + write!(self.writer, " .{} =", attr.id.name)?; + self.serialize_pattern(&attr.value)?; + + Ok(()) + } + + fn serialize_element(&mut self, elem: &PatternElement<'_>) -> Result<(), Error> { + match elem { + PatternElement::TextElement(text) => write!(self.writer, "{}", text), + PatternElement::Placeable(expr) => { + write!(self.writer, "{{ ")?; + self.serialize_expression(expr)?; + write!(self.writer, " }}")?; + Ok(()) + } + } + } + + fn serialize_expression(&mut self, expr: &Expression<'_>) -> Result<(), Error> { + match expr { + Expression::InlineExpression(inline) => self.serialize_inline_expression(inline), + Expression::SelectExpression { selector, variants } => { + self.serialize_select_expression(selector, variants) + } + } + } + + fn serialize_inline_expression(&mut self, expr: &InlineExpression<'_>) -> Result<(), Error> { + match expr { + InlineExpression::StringLiteral { value } => write!(self.writer, "\"{}\"", value), + InlineExpression::NumberLiteral { value } => write!(self.writer, "{}", value), + InlineExpression::VariableReference { + id: Identifier { name: value }, + } => write!(self.writer, "${}", value), + InlineExpression::FunctionReference { id, arguments } => { + write!(self.writer, "{}", id.name)?; + if let Some(args) = arguments.as_ref() { + self.serialize_call_arguments(args)?; + } + Ok(()) + } + InlineExpression::MessageReference { id, attribute } => { + write!(self.writer, "{}", id.name)?; + + if let Some(attr) = attribute.as_ref() { + write!(self.writer, ".{}", attr.name)?; + } + + Ok(()) + } + InlineExpression::TermReference { + id, + attribute, + arguments, + } => { + write!(self.writer, "-{}", id.name)?; + + if let Some(attr) = attribute.as_ref() { + write!(self.writer, ".{}", attr.name)?; + } + if let Some(args) = arguments.as_ref() { + self.serialize_call_arguments(args)?; + } + + Ok(()) + } + InlineExpression::Placeable { expression } => self.serialize_expression(expression), + } + } + + fn serialize_select_expression( + &mut self, + _selector: &InlineExpression<'_>, + _variants: &[Variant<'_>], + ) -> Result<(), Error> { + unimplemented!() + } + + fn serialize_call_arguments(&mut self, args: &CallArguments<'_>) -> Result<(), Error> { + let mut argument_written = false; + + write!(self.writer, "(")?; + + for positional in &args.positional { + if !argument_written { + write!(self.writer, ", ")?; + argument_written = true; + } + + self.serialize_inline_expression(positional)?; + } + + for named in &args.named { + if !argument_written { + write!(self.writer, ", ")?; + argument_written = true; + } + + write!(self.writer, "{}: ", named.name.name)?; + self.serialize_inline_expression(&named.value)?; + } + + write!(self.writer, ")")?; + Ok(()) + } +} + +fn is_select_expr(expr: &Expression) -> bool { + match expr { + Expression::SelectExpression { .. } => true, + Expression::InlineExpression(InlineExpression::Placeable { expression }) => { + is_select_expr(&*expression) + } + Expression::InlineExpression(_) => false, + } +} + +#[derive(Debug, Default, Copy, Clone, PartialEq)] +pub struct Options { + pub with_junk: bool, +} + +#[derive(Debug, Default, PartialEq)] +struct State { + has_entries: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! round_trip_test { + ($name:ident, $text:expr) => { + round_trip_test!($name, $text, $text); + }; + ($name:ident, $text:expr, $should_be:expr) => { + #[test] + fn $name() { + let resource = crate::parser::parse($text).unwrap(); + let got = serialize(&resource); + + assert_eq!(got, $should_be); + } + }; + } + + round_trip_test!(simple_message_without_eol, "foo = Foo", "foo = Foo\n"); + round_trip_test!(simple_message, "foo = Foo\n"); + round_trip_test!(two_simple_messages, "foo = Foo\nbar = Bar\n"); + round_trip_test!(block_multiline_message, "foo =\n Foo\n Bar\n"); + round_trip_test!( + inline_multiline_message, + "foo = Foo\n Bar\n", + "foo =\n Foo\n Bar\n" + ); + round_trip_test!(message_reference, "foo = Foo { bar }\n"); + round_trip_test!(term_reference, "foo = Foo { -bar }\n"); + round_trip_test!(external_reference, "foo = Foo { $bar }\n"); + round_trip_test!(number_element, "foo = Foo { 1 }\n"); + round_trip_test!(string_element, "foo = Foo { \"bar\" }\n"); + round_trip_test!(attribute_expression, "foo = Foo { bar.baz }\n"); + round_trip_test!( + resource_comment, + "### A multiline\n### resource comment.\n\nfoo = Foo\n" + ); + round_trip_test!( + message_comment, + "# A multiline\n# message comment.\nfoo = Foo\n" + ); + round_trip_test!( + group_comment, + "## Comment Header\n##\n## A multiline\n# group comment.\n\nfoo = Foo\n" + ); + round_trip_test!( + standalone_comment, + "foo = Foo\n\n# A Standalone Comment\n\nbar = Bar\n" + ); + round_trip_test!( + multiline_with_placeable, + "foo =\n Foo { bar }\n Baz\n" + ); + round_trip_test!(attribute, "foo =\n .attr = Foo Attr\n"); + round_trip_test!( + multiline_attribute, + "foo =\n .attr =\n Foo Attr\n Continued\n" + ); + round_trip_test!( + two_attributes, + "foo =\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n" + ); + round_trip_test!( + value_and_attributes, + "foo = Foo Value\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n" + ); + round_trip_test!( + multiline_value_and_attributes, + "foo = Foo Value\n Continued\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n" + ); + round_trip_test!( + select_expression, + "foo =\n { $sel ->\n *[a] A\n [b] B\n }\n" + ); + round_trip_test!( + multiline_variant, + "foo =\n { $sel ->\n *[a]\n AAA\n BBBB\n }\n" + ); + round_trip_test!( + multiline_variant_with_first_line_inline, + "foo =\n { $sel ->\n *[a] AAA\n BBB\n }\n", + "foo =\n { $sel ->\n *[a]\n AAA\n BBB\n }\n" + ); +} From 2eee000c3f56b4cdc942b3a1f6f0911e46ed6fdf Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Tue, 23 Jun 2020 17:08:53 +0800 Subject: [PATCH 02/27] Made sure indentation is handled properly --- fluent-syntax/src/serializer.rs | 237 ++++++++++++++++++++++++-------- 1 file changed, 177 insertions(+), 60 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 0df4f1cd..db081483 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -1,28 +1,27 @@ use crate::ast::*; -use std::io::{Error, Write}; +use std::fmt::{self, Display, Error, Write}; pub fn serialize(resource: &Resource<'_>) -> String { - let mut buffer = Vec::new(); let options = Options::default(); + let mut ser = Serializer::new(options); - Serializer::new(&mut buffer, options) - .serialize_resource(resource) + ser.serialize_resource(resource) .expect("Writing to an in-memory buffer never fails"); - String::from_utf8(buffer).expect("The serializer only ever emits valid UTF-8") + ser.into_serialized_text() } #[derive(Debug)] -pub struct Serializer { - writer: W, +pub struct Serializer { + writer: TextWriter, options: Options, state: State, } -impl Serializer { - pub fn new(writer: W, options: Options) -> Self { +impl Serializer { + pub fn new(options: Options) -> Self { Serializer { - writer, + writer: TextWriter::default(), options, state: State::default(), } @@ -42,16 +41,24 @@ impl Serializer { Ok(()) } + pub fn into_serialized_text(self) -> String { + self.writer.buffer + } + fn serialize_entry(&mut self, entry: &Entry<'_>) -> Result<(), Error> { match entry { Entry::Message(msg) => self.serialize_message(msg), - Entry::Comment(comment) => self.serialize_comment(comment), + Entry::Comment(comment) => { + self.serialize_comment(comment)?; + self.writer.newline(); + Ok(()) + } Entry::Term(term) => self.serialize_term(term), } } fn serialize_junk(&mut self, junk: &str) -> Result<(), Error> { - write!(self.writer, "{}", junk) + self.writer.write_literal(junk) } fn serialize_comment(&mut self, comment: &Comment<'_>) -> Result<(), Error> { @@ -62,11 +69,18 @@ impl Serializer { }; if self.state.has_entries { - writeln!(self.writer)?; + self.writer.newline(); } for line in lines { - writeln!(self.writer, "{} {}", prefix, line)?; + self.writer.write_literal(prefix)?; + + if !line.trim().is_empty() { + self.writer.write_literal(" ")?; + self.writer.write_literal(line)?; + } + + self.writer.newline(); } Ok(()) @@ -77,17 +91,16 @@ impl Serializer { self.serialize_comment(comment)?; } - write!(self.writer, "{} =", msg.id.name)?; + self.writer.write_literal(&msg.id.name)?; + self.writer.write_literal(" =")?; if let Some(value) = msg.value.as_ref() { self.serialize_pattern(value)?; } - for attr in &msg.attributes { - self.serialize_attribute(attr)?; - } + self.serialize_attributes(&msg.attributes)?; - writeln!(self.writer)?; + self.writer.newline(); Ok(()) } @@ -96,14 +109,14 @@ impl Serializer { self.serialize_comment(comment)?; } - write!(self.writer, "{} =", term.id.name)?; + self.writer.write_literal("-")?; + self.writer.write_literal(&term.id.name)?; + self.writer.write_literal(" =")?; self.serialize_pattern(&term.value)?; - for attr in &term.attributes { - self.serialize_attribute(attr)?; - } + self.serialize_attributes(&term.attributes)?; - writeln!(self.writer)?; + self.writer.newline(); Ok(()) } @@ -115,21 +128,45 @@ impl Serializer { }); if start_on_newline { - unimplemented!("Write a new line then indent everything afterwards"); + self.writer.newline(); + self.writer.indent(); + } else { + self.writer.write_literal(" ")?; } - write!(self.writer, " ")?; - for element in &pattern.elements { self.serialize_element(element)?; } + if start_on_newline { + self.writer.dedent(); + } + + Ok(()) + } + + fn serialize_attributes(&mut self, attrs: &[Attribute<'_>]) -> Result<(), Error> { + if attrs.is_empty() { + return Ok(()); + } + + self.writer.indent(); + + for attr in attrs { + self.writer.newline(); + self.serialize_attribute(attr)?; + } + + self.writer.dedent(); + Ok(()) } fn serialize_attribute(&mut self, attr: &Attribute<'_>) -> Result<(), Error> { - writeln!(self.writer)?; - write!(self.writer, " .{} =", attr.id.name)?; + self.writer.write_literal(".")?; + self.writer.write_literal(&attr.id.name)?; + self.writer.write_literal(" =")?; + self.serialize_pattern(&attr.value)?; Ok(()) @@ -137,11 +174,11 @@ impl Serializer { fn serialize_element(&mut self, elem: &PatternElement<'_>) -> Result<(), Error> { match elem { - PatternElement::TextElement(text) => write!(self.writer, "{}", text), + PatternElement::TextElement(text) => self.writer.write_literal(text), PatternElement::Placeable(expr) => { - write!(self.writer, "{{ ")?; + self.writer.write_literal("{ ")?; self.serialize_expression(expr)?; - write!(self.writer, " }}")?; + self.writer.write_literal(" }")?; Ok(()) } } @@ -158,23 +195,34 @@ impl Serializer { fn serialize_inline_expression(&mut self, expr: &InlineExpression<'_>) -> Result<(), Error> { match expr { - InlineExpression::StringLiteral { value } => write!(self.writer, "\"{}\"", value), - InlineExpression::NumberLiteral { value } => write!(self.writer, "{}", value), + InlineExpression::StringLiteral { value } => { + self.writer.write_literal("\"")?; + self.writer.write_literal(value)?; + self.writer.write_literal("\"")?; + Ok(()) + } + InlineExpression::NumberLiteral { value } => self.writer.write_literal(value), InlineExpression::VariableReference { id: Identifier { name: value }, - } => write!(self.writer, "${}", value), + } => { + self.writer.write_literal("$")?; + self.writer.write_literal(value)?; + Ok(()) + } InlineExpression::FunctionReference { id, arguments } => { - write!(self.writer, "{}", id.name)?; + self.writer.write_literal(&id.name)?; + if let Some(args) = arguments.as_ref() { self.serialize_call_arguments(args)?; } Ok(()) } InlineExpression::MessageReference { id, attribute } => { - write!(self.writer, "{}", id.name)?; + self.writer.write_literal(&id.name)?; if let Some(attr) = attribute.as_ref() { - write!(self.writer, ".{}", attr.name)?; + self.writer.write_literal(".")?; + self.writer.write_literal(&attr.name)?; } Ok(()) @@ -184,10 +232,12 @@ impl Serializer { attribute, arguments, } => { - write!(self.writer, "-{}", id.name)?; + self.writer.write_literal("-")?; + self.writer.write_literal(&id.name)?; if let Some(attr) = attribute.as_ref() { - write!(self.writer, ".{}", attr.name)?; + self.writer.write_literal(".")?; + self.writer.write_literal(&attr.name)?; } if let Some(args) = arguments.as_ref() { self.serialize_call_arguments(args)?; @@ -210,11 +260,11 @@ impl Serializer { fn serialize_call_arguments(&mut self, args: &CallArguments<'_>) -> Result<(), Error> { let mut argument_written = false; - write!(self.writer, "(")?; + self.writer.write_literal("(")?; for positional in &args.positional { if !argument_written { - write!(self.writer, ", ")?; + self.writer.write_literal(", ")?; argument_written = true; } @@ -223,15 +273,16 @@ impl Serializer { for named in &args.named { if !argument_written { - write!(self.writer, ", ")?; + self.writer.write_literal(", ")?; argument_written = true; } - write!(self.writer, "{}: ", named.name.name)?; + self.writer.write_literal(&named.name.name)?; + self.writer.write_literal(": ")?; self.serialize_inline_expression(&named.value)?; } - write!(self.writer, ")")?; + self.writer.write_literal(")")?; Ok(()) } } @@ -256,15 +307,77 @@ struct State { has_entries: bool, } +#[derive(Debug, Clone, Default)] +struct TextWriter { + buffer: String, + indent_level: usize, +} + +impl TextWriter { + fn indent(&mut self) { + self.indent_level += 1; + } + + fn dedent(&mut self) { + self.indent_level = self + .indent_level + .checked_sub(1) + .expect("Dedenting without a corresponding indent"); + } + + fn write_indent(&mut self) { + for _ in 0..self.indent_level { + self.buffer.push_str(" "); + } + } + + fn newline(&mut self) { + self.buffer.push_str("\n"); + } + + fn write_literal(&mut self, item: D) -> fmt::Result { + if self.buffer.ends_with("\n") { + // we've just added a newline, make sure it's properly indented + self.write_indent(); + } + + write!(self.buffer, "{}", item) + } +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn write_something_then_indent() -> fmt::Result { + let mut writer = TextWriter::default(); + + writer.write_literal("foo =")?; + writer.newline(); + writer.indent(); + writer.write_literal("first line")?; + writer.newline(); + writer.write_literal("second line")?; + writer.newline(); + writer.dedent(); + writer.write_literal("not indented")?; + writer.newline(); + + let got = &writer.buffer; + assert_eq!( + got, + "foo =\n first line\n second line\nnot indented\n" + ); + + Ok(()) + } + macro_rules! round_trip_test { - ($name:ident, $text:expr) => { + ($name:ident, $text:expr $(,)?) => { round_trip_test!($name, $text, $text); }; - ($name:ident, $text:expr, $should_be:expr) => { + ($name:ident, $text:expr, $should_be:expr $(,)?) => { #[test] fn $name() { let resource = crate::parser::parse($text).unwrap(); @@ -282,7 +395,7 @@ mod tests { round_trip_test!( inline_multiline_message, "foo = Foo\n Bar\n", - "foo =\n Foo\n Bar\n" + "foo =\n Foo\n Bar\n", ); round_trip_test!(message_reference, "foo = Foo { bar }\n"); round_trip_test!(term_reference, "foo = Foo { -bar }\n"); @@ -292,52 +405,56 @@ mod tests { round_trip_test!(attribute_expression, "foo = Foo { bar.baz }\n"); round_trip_test!( resource_comment, - "### A multiline\n### resource comment.\n\nfoo = Foo\n" + "### A multiline\n### resource comment.\n\nfoo = Foo\n", ); round_trip_test!( message_comment, - "# A multiline\n# message comment.\nfoo = Foo\n" + "# A multiline\n# message comment.\nfoo = Foo\n", ); round_trip_test!( group_comment, - "## Comment Header\n##\n## A multiline\n# group comment.\n\nfoo = Foo\n" + "## Comment Header\n##\n## A multiline\n## group comment.\n\nfoo = Foo\n", ); round_trip_test!( standalone_comment, - "foo = Foo\n\n# A Standalone Comment\n\nbar = Bar\n" + "foo = Foo\n\n# A Standalone Comment\n\nbar = Bar\n", ); round_trip_test!( multiline_with_placeable, - "foo =\n Foo { bar }\n Baz\n" + "foo =\n Foo { bar }\n Baz\n", ); round_trip_test!(attribute, "foo =\n .attr = Foo Attr\n"); round_trip_test!( multiline_attribute, - "foo =\n .attr =\n Foo Attr\n Continued\n" + "foo =\n .attr =\n Foo Attr\n Continued\n", ); round_trip_test!( two_attributes, - "foo =\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n" + "foo =\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n", ); round_trip_test!( value_and_attributes, - "foo = Foo Value\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n" + "foo = Foo Value\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n", ); round_trip_test!( multiline_value_and_attributes, - "foo = Foo Value\n Continued\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n" + "foo =\n Foo Value\n Continued\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n", ); round_trip_test!( select_expression, - "foo =\n { $sel ->\n *[a] A\n [b] B\n }\n" + "foo =\n { $sel ->\n *[a] A\n [b] B\n }\n", ); round_trip_test!( multiline_variant, - "foo =\n { $sel ->\n *[a]\n AAA\n BBBB\n }\n" + "foo =\n { $sel ->\n *[a]\n AAA\n BBBB\n }\n", ); round_trip_test!( multiline_variant_with_first_line_inline, "foo =\n { $sel ->\n *[a] AAA\n BBB\n }\n", - "foo =\n { $sel ->\n *[a]\n AAA\n BBB\n }\n" + "foo =\n { $sel ->\n *[a]\n AAA\n BBB\n }\n", + ); + round_trip_test!( + variant_key_number, + "foo =\n { $sel ->\n *[a] A\n [b] B\n }\n", ); } From 2c2f006496a48141f87c6ce0e40b032c6e43a707 Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Tue, 23 Jun 2020 17:31:27 +0800 Subject: [PATCH 03/27] Started handling select expressions --- fluent-syntax/src/serializer.rs | 72 +++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index db081483..40f7250c 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -1,5 +1,5 @@ use crate::ast::*; -use std::fmt::{self, Display, Error, Write}; +use std::fmt::{self, Error, Write}; pub fn serialize(resource: &Resource<'_>) -> String { let options = Options::default(); @@ -178,7 +178,15 @@ impl Serializer { PatternElement::Placeable(expr) => { self.writer.write_literal("{ ")?; self.serialize_expression(expr)?; - self.writer.write_literal(" }")?; + + if matches!(expr, Expression::SelectExpression { .. }) { + // select adds its own newline and indent, emit the brace + // *without* a space so we don't get 5 spaces instead of 4 + self.writer.write_literal("}")?; + } else { + self.writer.write_literal(" }")?; + } + Ok(()) } } @@ -251,10 +259,43 @@ impl Serializer { fn serialize_select_expression( &mut self, - _selector: &InlineExpression<'_>, - _variants: &[Variant<'_>], + selector: &InlineExpression<'_>, + variants: &[Variant<'_>], ) -> Result<(), Error> { - unimplemented!() + self.serialize_inline_expression(selector)?; + self.writer.write_literal(" ->")?; + + self.writer.newline(); + self.writer.indent(); + + for variant in variants { + self.serialize_variant(variant)?; + self.writer.newline(); + } + + self.writer.dedent(); + Ok(()) + } + + fn serialize_variant(&mut self, variant: &Variant<'_>) -> Result<(), Error> { + if variant.default { + self.writer.write_literal("*")?; + } + + self.writer.write_literal("[")?; + self.serialize_variant_key(&variant.key)?; + self.writer.write_literal("]")?; + self.serialize_pattern(&variant.value)?; + + Ok(()) + } + + fn serialize_variant_key(&mut self, key: &VariantKey<'_>) -> Result<(), Error> { + match key { + VariantKey::NumberLiteral { value } | VariantKey::Identifier { name: value } => { + self.writer.write_literal(value) + } + } } fn serialize_call_arguments(&mut self, args: &CallArguments<'_>) -> Result<(), Error> { @@ -335,10 +376,14 @@ impl TextWriter { self.buffer.push_str("\n"); } - fn write_literal(&mut self, item: D) -> fmt::Result { + fn write_literal(&mut self, mut item: &str) -> fmt::Result { if self.buffer.ends_with("\n") { // we've just added a newline, make sure it's properly indented self.write_indent(); + + // we've just added indentation, so we don't care about leading + // spaces + item = item.trim_start(); } write!(self.buffer, "{}", item) @@ -446,7 +491,7 @@ mod tests { ); round_trip_test!( multiline_variant, - "foo =\n { $sel ->\n *[a]\n AAA\n BBBB\n }\n", + "foo =\n { $sel ->\n *[a]\n AAA\n BBBB\n }\n", ); round_trip_test!( multiline_variant_with_first_line_inline, @@ -457,4 +502,17 @@ mod tests { variant_key_number, "foo =\n { $sel ->\n *[a] A\n [b] B\n }\n", ); + round_trip_test!( + select_expression_in_block_value, + "foo =\n Foo { $sel ->\n *[a] A\n [b] B\n }\n", + ); + round_trip_test!( + select_expression_in_inline_value, + "foo = Foo { $sel ->\n *[a] A\n [b] B\n }\n", + "foo =\n Foo { $sel ->\n *[a] A\n [b] B\n }\n", + ); + round_trip_test!( + select_expression_in_multiline_value, + "foo =\n Foo\n Bar { $sel ->\n *[a] A\n [b] B\n }\n", + ); } From f8f3f722b082a743df3466c598f9a4d136bd1a93 Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Wed, 24 Jun 2020 09:31:38 +0800 Subject: [PATCH 04/27] Switched " " to "\t" to make recognising indents easier --- fluent-syntax/src/serializer.rs | 48 ++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 40f7250c..b39c18ff 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -2,7 +2,10 @@ use crate::ast::*; use std::fmt::{self, Error, Write}; pub fn serialize(resource: &Resource<'_>) -> String { - let options = Options::default(); + serialize_with_options(resource, Options::default()) +} + +pub fn serialize_with_options(resource: &Resource<'_>, options: Options) -> String { let mut ser = Serializer::new(options); ser.serialize_resource(resource) @@ -425,10 +428,15 @@ mod tests { ($name:ident, $text:expr, $should_be:expr $(,)?) => { #[test] fn $name() { - let resource = crate::parser::parse($text).unwrap(); + // Note: We add tabs to the input so it's easier to recognise + // indentation + let input_without_tabs = $text.replace("\t", " "); + let should_be_without_tabs = $should_be.replace("\t", " "); + + let resource = crate::parser::parse(&input_without_tabs).unwrap(); let got = serialize(&resource); - assert_eq!(got, $should_be); + assert_eq!(got, should_be_without_tabs); } }; } @@ -464,55 +472,53 @@ mod tests { standalone_comment, "foo = Foo\n\n# A Standalone Comment\n\nbar = Bar\n", ); - round_trip_test!( - multiline_with_placeable, - "foo =\n Foo { bar }\n Baz\n", - ); - round_trip_test!(attribute, "foo =\n .attr = Foo Attr\n"); + round_trip_test!(multiline_with_placeable, "foo =\n\tFoo { bar }\n\tBaz\n",); + round_trip_test!(attribute, "foo =\n\t.attr = Foo Attr\n"); round_trip_test!( multiline_attribute, - "foo =\n .attr =\n Foo Attr\n Continued\n", + "foo =\n\t.attr =\n\t\tFoo Attr\n\t\tContinued\n", ); round_trip_test!( two_attributes, - "foo =\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n", + "foo =\n\t.attr-a = Foo Attr A\n\t.attr-b = Foo Attr B\n", ); round_trip_test!( value_and_attributes, - "foo = Foo Value\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n", + "foo = Foo Value\n\t.attr-a = Foo Attr A\n\t.attr-b = Foo Attr B\n", ); round_trip_test!( multiline_value_and_attributes, - "foo =\n Foo Value\n Continued\n .attr-a = Foo Attr A\n .attr-b = Foo Attr B\n", + "foo =\n\tFoo Value\n\tContinued\n\t.attr-a = Foo Attr A\n\t.attr-b = Foo Attr B\n", ); round_trip_test!( select_expression, - "foo =\n { $sel ->\n *[a] A\n [b] B\n }\n", + "foo =\n\t{ $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", ); round_trip_test!( multiline_variant, - "foo =\n { $sel ->\n *[a]\n AAA\n BBBB\n }\n", + "foo =\n\t{ $sel ->\n\t\t*[a]\n\t\t\tAAA\n\t\t\tBBBB\n\t}\n", ); round_trip_test!( multiline_variant_with_first_line_inline, - "foo =\n { $sel ->\n *[a] AAA\n BBB\n }\n", - "foo =\n { $sel ->\n *[a]\n AAA\n BBB\n }\n", + "foo =\n\t{ $sel ->\n\t\t*[a] AAA\n\t\tBBB\n\t}\n", + "foo =\n\t{ $sel ->\n\t\t*[a]\n\t\t\tAAA\n\t\t\tBBB\n\t}\n", ); round_trip_test!( variant_key_number, - "foo =\n { $sel ->\n *[a] A\n [b] B\n }\n", + "foo =\n\t{ $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", ); round_trip_test!( select_expression_in_block_value, - "foo =\n Foo { $sel ->\n *[a] A\n [b] B\n }\n", + "foo =\n\tFoo { $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", ); round_trip_test!( select_expression_in_inline_value, - "foo = Foo { $sel ->\n *[a] A\n [b] B\n }\n", - "foo =\n Foo { $sel ->\n *[a] A\n [b] B\n }\n", + "foo = Foo { $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", + "foo =\n\tFoo { $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", ); round_trip_test!( select_expression_in_multiline_value, - "foo =\n Foo\n Bar { $sel ->\n *[a] A\n [b] B\n }\n", + "foo =\n\tFoo\n\tBar { $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", + ); ); } From a650135905a17a94f2ce099703b417c19630f506 Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Wed, 24 Jun 2020 09:48:42 +0800 Subject: [PATCH 05/27] Everything from TypeScript's "Serialize resource" test suite passes --- fluent-syntax/src/serializer.rs | 101 +++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 14 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index b39c18ff..76b4961e 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -178,18 +178,28 @@ impl Serializer { fn serialize_element(&mut self, elem: &PatternElement<'_>) -> Result<(), Error> { match elem { PatternElement::TextElement(text) => self.writer.write_literal(text), + PatternElement::Placeable(Expression::InlineExpression( + InlineExpression::Placeable { expression }, + )) => { + // A placeable inside a placeable is a special case because we + // don't want the braces to look silly (e.g. "{ { Foo() } }"). + self.writer.write_literal("{{ ")?; + self.serialize_expression(expression)?; + self.writer.write_literal(" }}")?; + Ok(()) + } + PatternElement::Placeable(expr @ Expression::SelectExpression { .. }) => { + // select adds its own newline and indent, emit the brace + // *without* a space so we don't get 5 spaces instead of 4 + self.writer.write_literal("{ ")?; + self.serialize_expression(expr)?; + self.writer.write_literal("}")?; + Ok(()) + } PatternElement::Placeable(expr) => { self.writer.write_literal("{ ")?; self.serialize_expression(expr)?; - - if matches!(expr, Expression::SelectExpression { .. }) { - // select adds its own newline and indent, emit the brace - // *without* a space so we don't get 5 spaces instead of 4 - self.writer.write_literal("}")?; - } else { - self.writer.write_literal(" }")?; - } - + self.writer.write_literal(" }")?; Ok(()) } } @@ -256,7 +266,13 @@ impl Serializer { Ok(()) } - InlineExpression::Placeable { expression } => self.serialize_expression(expression), + InlineExpression::Placeable { expression } => { + self.writer.write_literal("{")?; + self.serialize_expression(expression)?; + self.writer.write_literal("}")?; + + Ok(()) + } } } @@ -307,23 +323,23 @@ impl Serializer { self.writer.write_literal("(")?; for positional in &args.positional { - if !argument_written { + if argument_written { self.writer.write_literal(", ")?; - argument_written = true; } self.serialize_inline_expression(positional)?; + argument_written = true; } for named in &args.named { - if !argument_written { + if argument_written { self.writer.write_literal(", ")?; - argument_written = true; } self.writer.write_literal(&named.name.name)?; self.writer.write_literal(": ")?; self.serialize_inline_expression(&named.value)?; + argument_written = true; } self.writer.write_literal(")")?; @@ -520,5 +536,62 @@ mod tests { select_expression_in_multiline_value, "foo =\n\tFoo\n\tBar { $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", ); + round_trip_test!( + nested_select_expression, + "foo =\n\t{ $a ->\n\t\t*[a]\n\t\t\t{ $b ->\n\t\t\t\t*[b] Foo\n\t\t\t}\n\t}\n", + ); + round_trip_test!( + selector_external_argument, + "foo =\n\t{ $bar ->\n\t\t*[a] A\n\t}\n", + ); + round_trip_test!( + selector_number_expression, + "foo =\n\t{ 1 ->\n\t\t*[a] A\n\t}\n", + ); + round_trip_test!( + selector_string_expression, + "foo =\n\t{ \"bar\" ->\n\t\t*[a] A\n\t}\n", + ); + round_trip_test!( + selector_attribute_expression, + "foo =\n\t{ -bar.baz ->\n\t\t*[a] A\n\t}\n", + ); + round_trip_test!(call_expression, "foo = { FOO() }\n",); + round_trip_test!( + call_expression_with_string_expression, + "foo = { FOO(\"bar\") }\n", + ); + round_trip_test!(call_expression_with_number_expression, "foo = { FOO(1) }\n",); + round_trip_test!( + call_expression_with_message_reference, + "foo = { FOO(bar) }\n", + ); + round_trip_test!( + call_expression_with_external_argument, + "foo = { FOO($bar) }\n", + ); + round_trip_test!( + call_expression_with_number_named_argument, + "foo = { FOO(bar: 1) }\n", + ); + round_trip_test!( + call_expression_with_string_named_argument, + "foo = { FOO(bar: \"bar\") }\n", + ); + round_trip_test!( + call_expression_with_two_positional_arguments, + "foo = { FOO(bar, baz) }\n", + ); + round_trip_test!( + call_expression_with_positional_and_named_arguments, + "foo = { FOO(bar, 1, baz: \"baz\") }\n", + ); + round_trip_test!(macro_call, "foo = { -term() }\n",); + round_trip_test!(nested_placeables, "foo = {{ FOO() }}\n",); + round_trip_test!(backslash_in_text_element, "foo = \\{ placeable }\n",); + round_trip_test!( + excaped_special_char_in_string_literal, + "foo = { \"Escaped \\\" quote\" }\n", ); + round_trip_test!(unicode_escape_sequence, "foo = { \"\\u0065\" }\n",); } From f95efcebf1fb7313026f520a686866b3f09d3ca4 Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Wed, 24 Jun 2020 10:19:43 +0800 Subject: [PATCH 06/27] Added the rest of the tests --- fluent-syntax/src/serializer.rs | 119 +++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 76b4961e..7959ce00 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -410,7 +410,7 @@ impl TextWriter { } #[cfg(test)] -mod tests { +mod serialize_resource_tests { use super::*; #[test] @@ -594,4 +594,121 @@ mod tests { "foo = { \"Escaped \\\" quote\" }\n", ); round_trip_test!(unicode_escape_sequence, "foo = { \"\\u0065\" }\n",); + + // Serialize padding around comments + + round_trip_test!( + standalone_comment_has_not_padding_when_first, + "# Comment A\n\nfoo = Foo\n\n# Comment B\n\nbar = Bar\n" + ); + round_trip_test!( + group_comment_has_not_padding_when_first, + "## Group A\n\nfoo = Foo\n\n## Group B\n\nbar = Bar\n" + ); + round_trip_test!( + resource_comment_has_not_padding_when_first, + "### Resource Comment A\n\nfoo = Foo\n\n### Resource Comment B\n\nbar = Bar\n" + ); +} + +#[cfg(test)] +mod serialize_expression_tests { + use super::*; + + macro_rules! expression_test { + ($name:ident, $input:expr) => { + #[test] + fn $name() { + let input_without_tabs = $input.replace("\t", " "); + let src = format!("foo = {{ {} }}", input_without_tabs); + let resource = crate::parser::parse(&src).unwrap(); + + // extract the first expression from the value of the first + // message + assert_eq!(resource.body.len(), 1); + let first_item = &resource.body[0]; + let message = match first_item { + ResourceEntry::Entry(Entry::Message(msg)) => msg, + other => panic!("Expected a message but found {:#?}", other), + }; + let value = message.value.as_ref().expect("The message has a value"); + assert_eq!(value.elements.len(), 1); + let expr = match &value.elements[0] { + PatternElement::Placeable(expr) => expr, + other => panic!("Expected a single expression but found {:#?}", other), + }; + + // we've finally extracted the first expression, now we can + // actually serialize it and finish the test + let mut serializer = Serializer::new(Options::default()); + serializer.serialize_expression(expr).unwrap(); + let got = serializer.into_serialized_text(); + + assert_eq!(got, input_without_tabs); + } + }; + } + + expression_test!(string_expression, "\"str\""); + expression_test!(number_expression, "3"); + expression_test!(message_reference, "msg"); + expression_test!(external_arguemnt, "$ext"); + expression_test!(attribute_expression, "msg.attr"); + expression_test!(call_expression, "BUILTIN(3.14, kwarg: \"value\")"); + expression_test!(select_expression, "$num ->\n\t*[one] One\n"); +} + +#[cfg(test)] +mod serialize_variant_key_tests { + use super::*; + + macro_rules! variant_key_test { + ($name:ident, $input:expr => $( $keys:expr ),+ $(,)?) => { + #[test] + #[allow(unused_assignments)] + fn $name() { + let input_without_tabs = $input.replace("\t", " "); + let src = format!("foo = {{ {}\n }}", input_without_tabs); + let resource = crate::parser::parse(&src).unwrap(); + + // extract variant from the first expression from the value of + // the first message + assert_eq!(resource.body.len(), 1); + let first_item = &resource.body[0]; + let message = match first_item { + ResourceEntry::Entry(Entry::Message(msg)) => msg, + other => panic!("Expected a message but found {:#?}", other), + }; + let value = message.value.as_ref().expect("The message has a value"); + assert_eq!(value.elements.len(), 1); + let variants = match &value.elements[0] { + PatternElement::Placeable(Expression::SelectExpression { variants, .. }) => variants, + other => panic!("Expected a single select expression but found {:#?}", other), + }; + + let mut ix = 0; + + $( + let variant_key = &variants[ix].key; + + // we've finally extracted the variant key, now we can + // actually serialize it and finish the test + let mut serializer = Serializer::new(Options::default()); + serializer.serialize_variant_key(variant_key).unwrap(); + let got = serializer.into_serialized_text(); + + assert_eq!(got, $keys); + + ix += 1; + )* + } + }; + } + + variant_key_test!(identifiers, "$num ->\n\t[one] One\n\t*[other] Other" => "one", "other"); + variant_key_test!( + number_literals, + "$num ->\n\t[-123456789] Minus a lot\n\t[0] Zero\n\t*[3.14] Pi\n\t[007] James" + => "-123456789", "0", "3.14", "007", + ); } From 34f7a939801bcb1035b1683f3c10337b88bc6e44 Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Fri, 26 Jun 2020 10:07:43 +0800 Subject: [PATCH 07/27] Added a test to make sure subsequent entries with a comment aren't separated by newlines --- fluent-syntax/src/serializer.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 7959ce00..e86a95f2 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -52,6 +52,10 @@ impl Serializer { match entry { Entry::Message(msg) => self.serialize_message(msg), Entry::Comment(comment) => { + if self.state.has_entries { + self.writer.newline(); + } + self.serialize_comment(comment)?; self.writer.newline(); Ok(()) @@ -71,10 +75,6 @@ impl Serializer { Comment::ResourceComment { content } => ("###", content), }; - if self.state.has_entries { - self.writer.newline(); - } - for line in lines { self.writer.write_literal(prefix)?; @@ -480,6 +480,10 @@ mod serialize_resource_tests { message_comment, "# A multiline\n# message comment.\nfoo = Foo\n", ); + round_trip_test!( + dont_prefix_a_subsequent_entry_comment_with_a_newline, + "first = Firstsubsequent_ Comment\nfoo = Foo\n", + ); round_trip_test!( group_comment, "## Comment Header\n##\n## A multiline\n## group comment.\n\nfoo = Foo\n", From f3378e2194c4f4058bad285e9f4c11e02f342251 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Mon, 8 Nov 2021 11:17:14 +0100 Subject: [PATCH 08/27] Fix serializer for fluent-syntax 0.11.0 Botch with only `&str` supported. --- fluent-syntax/src/lib.rs | 2 +- fluent-syntax/src/serializer.rs | 170 ++++++++++++++++---------------- 2 files changed, 84 insertions(+), 88 deletions(-) diff --git a/fluent-syntax/src/lib.rs b/fluent-syntax/src/lib.rs index e1dfa456..ab3e43ac 100644 --- a/fluent-syntax/src/lib.rs +++ b/fluent-syntax/src/lib.rs @@ -48,5 +48,5 @@ //! ``` pub mod ast; pub mod parser; -pub mod unicode; pub mod serializer; +pub mod unicode; diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index e86a95f2..f2af5a96 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -1,11 +1,11 @@ use crate::ast::*; use std::fmt::{self, Error, Write}; -pub fn serialize(resource: &Resource<'_>) -> String { +pub fn serialize(resource: &Resource<&str>) -> String { serialize_with_options(resource, Options::default()) } -pub fn serialize_with_options(resource: &Resource<'_>, options: Options) -> String { +pub fn serialize_with_options(resource: &Resource<&str>, options: Options) -> String { let mut ser = Serializer::new(options); ser.serialize_resource(resource) @@ -30,12 +30,18 @@ impl Serializer { } } - pub fn serialize_resource(&mut self, res: &Resource<'_>) -> Result<(), Error> { + pub fn serialize_resource(&mut self, res: &Resource<&str>) -> Result<(), Error> { for entry in &res.body { match entry { - ResourceEntry::Entry(entry) => self.serialize_entry(entry)?, - ResourceEntry::Junk(junk) if self.options.with_junk => self.serialize_junk(junk)?, - ResourceEntry::Junk(_) => continue, + Entry::Message(msg) => self.serialize_message(msg)?, + Entry::Term(term) => self.serialize_term(term)?, + Entry::Comment(comment) => self.serialize_free_comment(comment, "#")?, + Entry::GroupComment(comment) => self.serialize_free_comment(comment, "##")?, + Entry::ResourceComment(comment) => self.serialize_free_comment(comment, "###")?, + Entry::Junk { content } if self.options.with_junk => { + self.serialize_junk(content)? + } + Entry::Junk { .. } => continue, } self.state.has_entries = true; @@ -48,34 +54,26 @@ impl Serializer { self.writer.buffer } - fn serialize_entry(&mut self, entry: &Entry<'_>) -> Result<(), Error> { - match entry { - Entry::Message(msg) => self.serialize_message(msg), - Entry::Comment(comment) => { - if self.state.has_entries { - self.writer.newline(); - } - - self.serialize_comment(comment)?; - self.writer.newline(); - Ok(()) - } - Entry::Term(term) => self.serialize_term(term), - } - } - fn serialize_junk(&mut self, junk: &str) -> Result<(), Error> { self.writer.write_literal(junk) } - fn serialize_comment(&mut self, comment: &Comment<'_>) -> Result<(), Error> { - let (prefix, lines) = match comment { - Comment::Comment { content } => ("#", content), - Comment::GroupComment { content } => ("##", content), - Comment::ResourceComment { content } => ("###", content), - }; + fn serialize_free_comment( + &mut self, + comment: &Comment<&str>, + prefix: &str, + ) -> Result<(), Error> { + if self.state.has_entries { + self.writer.newline(); + } + self.serialize_comment(comment, prefix)?; + self.writer.newline(); - for line in lines { + Ok(()) + } + + fn serialize_comment(&mut self, comment: &Comment<&str>, prefix: &str) -> Result<(), Error> { + for line in &comment.content { self.writer.write_literal(prefix)?; if !line.trim().is_empty() { @@ -89,9 +87,9 @@ impl Serializer { Ok(()) } - fn serialize_message(&mut self, msg: &Message<'_>) -> Result<(), Error> { + fn serialize_message(&mut self, msg: &Message<&str>) -> Result<(), Error> { if let Some(comment) = msg.comment.as_ref() { - self.serialize_comment(comment)?; + self.serialize_comment(comment, "#")?; } self.writer.write_literal(&msg.id.name)?; @@ -107,9 +105,9 @@ impl Serializer { Ok(()) } - fn serialize_term(&mut self, term: &Term<'_>) -> Result<(), Error> { + fn serialize_term(&mut self, term: &Term<&str>) -> Result<(), Error> { if let Some(comment) = term.comment.as_ref() { - self.serialize_comment(comment)?; + self.serialize_comment(comment, "#")?; } self.writer.write_literal("-")?; @@ -124,10 +122,10 @@ impl Serializer { Ok(()) } - fn serialize_pattern(&mut self, pattern: &Pattern<'_>) -> Result<(), Error> { + fn serialize_pattern(&mut self, pattern: &Pattern<&str>) -> Result<(), Error> { let start_on_newline = pattern.elements.iter().any(|elem| match elem { - PatternElement::TextElement(text) => text.contains("\n"), - PatternElement::Placeable(expr) => is_select_expr(expr), + PatternElement::TextElement { value } => value.contains("\n"), + PatternElement::Placeable { expression } => is_select_expr(expression), }); if start_on_newline { @@ -148,7 +146,7 @@ impl Serializer { Ok(()) } - fn serialize_attributes(&mut self, attrs: &[Attribute<'_>]) -> Result<(), Error> { + fn serialize_attributes(&mut self, attrs: &[Attribute<&str>]) -> Result<(), Error> { if attrs.is_empty() { return Ok(()); } @@ -165,7 +163,7 @@ impl Serializer { Ok(()) } - fn serialize_attribute(&mut self, attr: &Attribute<'_>) -> Result<(), Error> { + fn serialize_attribute(&mut self, attr: &Attribute<&str>) -> Result<(), Error> { self.writer.write_literal(".")?; self.writer.write_literal(&attr.id.name)?; self.writer.write_literal(" =")?; @@ -175,46 +173,46 @@ impl Serializer { Ok(()) } - fn serialize_element(&mut self, elem: &PatternElement<'_>) -> Result<(), Error> { + fn serialize_element(&mut self, elem: &PatternElement<&str>) -> Result<(), Error> { match elem { - PatternElement::TextElement(text) => self.writer.write_literal(text), - PatternElement::Placeable(Expression::InlineExpression( - InlineExpression::Placeable { expression }, - )) => { - // A placeable inside a placeable is a special case because we - // don't want the braces to look silly (e.g. "{ { Foo() } }"). - self.writer.write_literal("{{ ")?; - self.serialize_expression(expression)?; - self.writer.write_literal(" }}")?; - Ok(()) - } - PatternElement::Placeable(expr @ Expression::SelectExpression { .. }) => { - // select adds its own newline and indent, emit the brace - // *without* a space so we don't get 5 spaces instead of 4 - self.writer.write_literal("{ ")?; - self.serialize_expression(expr)?; - self.writer.write_literal("}")?; - Ok(()) - } - PatternElement::Placeable(expr) => { - self.writer.write_literal("{ ")?; - self.serialize_expression(expr)?; - self.writer.write_literal(" }")?; - Ok(()) - } + PatternElement::TextElement { value } => self.writer.write_literal(value), + PatternElement::Placeable { expression } => match expression { + Expression::Inline(InlineExpression::Placeable { expression }) => { + // A placeable inside a placeable is a special case because we + // don't want the braces to look silly (e.g. "{ { Foo() } }"). + self.writer.write_literal("{{ ")?; + self.serialize_expression(expression)?; + self.writer.write_literal(" }}")?; + Ok(()) + } + Expression::Select { .. } => { + // select adds its own newline and indent, emit the brace + // *without* a space so we don't get 5 spaces instead of 4 + self.writer.write_literal("{ ")?; + self.serialize_expression(expression)?; + self.writer.write_literal("}")?; + Ok(()) + } + Expression::Inline(_) => { + self.writer.write_literal("{ ")?; + self.serialize_expression(expression)?; + self.writer.write_literal(" }")?; + Ok(()) + } + }, } } - fn serialize_expression(&mut self, expr: &Expression<'_>) -> Result<(), Error> { + fn serialize_expression(&mut self, expr: &Expression<&str>) -> Result<(), Error> { match expr { - Expression::InlineExpression(inline) => self.serialize_inline_expression(inline), - Expression::SelectExpression { selector, variants } => { + Expression::Inline(inline) => self.serialize_inline_expression(inline), + Expression::Select { selector, variants } => { self.serialize_select_expression(selector, variants) } } } - fn serialize_inline_expression(&mut self, expr: &InlineExpression<'_>) -> Result<(), Error> { + fn serialize_inline_expression(&mut self, expr: &InlineExpression<&str>) -> Result<(), Error> { match expr { InlineExpression::StringLiteral { value } => { self.writer.write_literal("\"")?; @@ -232,10 +230,8 @@ impl Serializer { } InlineExpression::FunctionReference { id, arguments } => { self.writer.write_literal(&id.name)?; + self.serialize_call_arguments(arguments)?; - if let Some(args) = arguments.as_ref() { - self.serialize_call_arguments(args)?; - } Ok(()) } InlineExpression::MessageReference { id, attribute } => { @@ -278,8 +274,8 @@ impl Serializer { fn serialize_select_expression( &mut self, - selector: &InlineExpression<'_>, - variants: &[Variant<'_>], + selector: &InlineExpression<&str>, + variants: &[Variant<&str>], ) -> Result<(), Error> { self.serialize_inline_expression(selector)?; self.writer.write_literal(" ->")?; @@ -296,7 +292,7 @@ impl Serializer { Ok(()) } - fn serialize_variant(&mut self, variant: &Variant<'_>) -> Result<(), Error> { + fn serialize_variant(&mut self, variant: &Variant<&str>) -> Result<(), Error> { if variant.default { self.writer.write_literal("*")?; } @@ -309,7 +305,7 @@ impl Serializer { Ok(()) } - fn serialize_variant_key(&mut self, key: &VariantKey<'_>) -> Result<(), Error> { + fn serialize_variant_key(&mut self, key: &VariantKey<&str>) -> Result<(), Error> { match key { VariantKey::NumberLiteral { value } | VariantKey::Identifier { name: value } => { self.writer.write_literal(value) @@ -317,7 +313,7 @@ impl Serializer { } } - fn serialize_call_arguments(&mut self, args: &CallArguments<'_>) -> Result<(), Error> { + fn serialize_call_arguments(&mut self, args: &CallArguments<&str>) -> Result<(), Error> { let mut argument_written = false; self.writer.write_literal("(")?; @@ -347,13 +343,13 @@ impl Serializer { } } -fn is_select_expr(expr: &Expression) -> bool { +fn is_select_expr(expr: &Expression<&str>) -> bool { match expr { - Expression::SelectExpression { .. } => true, - Expression::InlineExpression(InlineExpression::Placeable { expression }) => { + Expression::Select { .. } => true, + Expression::Inline(InlineExpression::Placeable { expression }) => { is_select_expr(&*expression) } - Expression::InlineExpression(_) => false, + Expression::Inline(_) => false, } } @@ -449,7 +445,7 @@ mod serialize_resource_tests { let input_without_tabs = $text.replace("\t", " "); let should_be_without_tabs = $should_be.replace("\t", " "); - let resource = crate::parser::parse(&input_without_tabs).unwrap(); + let resource = crate::parser::parse(input_without_tabs.as_str()).unwrap(); let got = serialize(&resource); assert_eq!(got, should_be_without_tabs); @@ -625,20 +621,20 @@ mod serialize_expression_tests { fn $name() { let input_without_tabs = $input.replace("\t", " "); let src = format!("foo = {{ {} }}", input_without_tabs); - let resource = crate::parser::parse(&src).unwrap(); + let resource = crate::parser::parse(src.as_str()).unwrap(); // extract the first expression from the value of the first // message assert_eq!(resource.body.len(), 1); let first_item = &resource.body[0]; let message = match first_item { - ResourceEntry::Entry(Entry::Message(msg)) => msg, + Entry::Message(msg) => msg, other => panic!("Expected a message but found {:#?}", other), }; let value = message.value.as_ref().expect("The message has a value"); assert_eq!(value.elements.len(), 1); let expr = match &value.elements[0] { - PatternElement::Placeable(expr) => expr, + PatternElement::Placeable { expression } => expression, other => panic!("Expected a single expression but found {:#?}", other), }; @@ -673,20 +669,20 @@ mod serialize_variant_key_tests { fn $name() { let input_without_tabs = $input.replace("\t", " "); let src = format!("foo = {{ {}\n }}", input_without_tabs); - let resource = crate::parser::parse(&src).unwrap(); + let resource = crate::parser::parse(src.as_str()).unwrap(); // extract variant from the first expression from the value of // the first message assert_eq!(resource.body.len(), 1); let first_item = &resource.body[0]; let message = match first_item { - ResourceEntry::Entry(Entry::Message(msg)) => msg, + Entry::Message(msg) => msg, other => panic!("Expected a message but found {:#?}", other), }; let value = message.value.as_ref().expect("The message has a value"); assert_eq!(value.elements.len(), 1); let variants = match &value.elements[0] { - PatternElement::Placeable(Expression::SelectExpression { variants, .. }) => variants, + PatternElement::Placeable { expression: Expression::Select { variants, .. } } => variants, other => panic!("Expected a single select expression but found {:#?}", other), }; From b627c05e0ec03db5270bc02588be6626219416ee Mon Sep 17 00:00:00 2001 From: RumovZ Date: Tue, 9 Nov 2021 08:44:35 +0100 Subject: [PATCH 09/27] Fix empty lines in messages being swallowed --- fluent-syntax/src/serializer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index f2af5a96..a91cd07d 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -398,7 +398,7 @@ impl TextWriter { // we've just added indentation, so we don't care about leading // spaces - item = item.trim_start(); + item = item.trim_start_matches(' '); } write!(self.buffer, "{}", item) From ef6d1a8943fb7eb514a1cef7705a8cb0f3a009fa Mon Sep 17 00:00:00 2001 From: RumovZ Date: Tue, 9 Nov 2021 08:46:20 +0100 Subject: [PATCH 10/27] Fix indentation of default asterisk It seems like `*` is supposed to go _inside_ the indent now. --- fluent-syntax/src/serializer.rs | 44 +++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index a91cd07d..c8ec81d1 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -294,7 +294,7 @@ impl Serializer { fn serialize_variant(&mut self, variant: &Variant<&str>) -> Result<(), Error> { if variant.default { - self.writer.write_literal("*")?; + self.writer.write_char_into_indent('*'); } self.writer.write_literal("[")?; @@ -403,6 +403,14 @@ impl TextWriter { write!(self.buffer, "{}", item) } + + fn write_char_into_indent(&mut self, ch: char) { + if self.buffer.ends_with("\n") { + self.write_indent(); + } + self.buffer.pop(); + self.buffer.push(ch); + } } #[cfg(test)] @@ -508,53 +516,53 @@ mod serialize_resource_tests { ); round_trip_test!( select_expression, - "foo =\n\t{ $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", + "foo =\n\t{ $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", ); round_trip_test!( multiline_variant, - "foo =\n\t{ $sel ->\n\t\t*[a]\n\t\t\tAAA\n\t\t\tBBBB\n\t}\n", + "foo =\n\t{ $sel ->\n\t *[a]\n\t\t\tAAA\n\t\t\tBBBB\n\t}\n", ); round_trip_test!( multiline_variant_with_first_line_inline, - "foo =\n\t{ $sel ->\n\t\t*[a] AAA\n\t\tBBB\n\t}\n", - "foo =\n\t{ $sel ->\n\t\t*[a]\n\t\t\tAAA\n\t\t\tBBB\n\t}\n", + "foo =\n\t{ $sel ->\n\t *[a] AAA\n\t\tBBB\n\t}\n", + "foo =\n\t{ $sel ->\n\t *[a]\n\t\t\tAAA\n\t\t\tBBB\n\t}\n", ); round_trip_test!( variant_key_number, - "foo =\n\t{ $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", + "foo =\n\t{ $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", ); round_trip_test!( select_expression_in_block_value, - "foo =\n\tFoo { $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", + "foo =\n\tFoo { $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", ); round_trip_test!( select_expression_in_inline_value, - "foo = Foo { $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", - "foo =\n\tFoo { $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", + "foo = Foo { $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", + "foo =\n\tFoo { $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", ); round_trip_test!( select_expression_in_multiline_value, - "foo =\n\tFoo\n\tBar { $sel ->\n\t\t*[a] A\n\t\t[b] B\n\t}\n", + "foo =\n\tFoo\n\tBar { $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", ); round_trip_test!( nested_select_expression, - "foo =\n\t{ $a ->\n\t\t*[a]\n\t\t\t{ $b ->\n\t\t\t\t*[b] Foo\n\t\t\t}\n\t}\n", + "foo =\n\t{ $a ->\n\t *[a]\n\t\t\t{ $b ->\n\t\t\t *[b] Foo\n\t\t\t}\n\t}\n", ); round_trip_test!( selector_external_argument, - "foo =\n\t{ $bar ->\n\t\t*[a] A\n\t}\n", + "foo =\n\t{ $bar ->\n\t *[a] A\n\t}\n", ); round_trip_test!( selector_number_expression, - "foo =\n\t{ 1 ->\n\t\t*[a] A\n\t}\n", + "foo =\n\t{ 1 ->\n\t *[a] A\n\t}\n", ); round_trip_test!( selector_string_expression, - "foo =\n\t{ \"bar\" ->\n\t\t*[a] A\n\t}\n", + "foo =\n\t{ \"bar\" ->\n\t *[a] A\n\t}\n", ); round_trip_test!( selector_attribute_expression, - "foo =\n\t{ -bar.baz ->\n\t\t*[a] A\n\t}\n", + "foo =\n\t{ -bar.baz ->\n\t *[a] A\n\t}\n", ); round_trip_test!(call_expression, "foo = { FOO() }\n",); round_trip_test!( @@ -655,7 +663,7 @@ mod serialize_expression_tests { expression_test!(external_arguemnt, "$ext"); expression_test!(attribute_expression, "msg.attr"); expression_test!(call_expression, "BUILTIN(3.14, kwarg: \"value\")"); - expression_test!(select_expression, "$num ->\n\t*[one] One\n"); + expression_test!(select_expression, "$num ->\n *[one] One\n"); } #[cfg(test)] @@ -705,10 +713,10 @@ mod serialize_variant_key_tests { }; } - variant_key_test!(identifiers, "$num ->\n\t[one] One\n\t*[other] Other" => "one", "other"); + variant_key_test!(identifiers, "$num ->\n\t[one] One\n *[other] Other" => "one", "other"); variant_key_test!( number_literals, - "$num ->\n\t[-123456789] Minus a lot\n\t[0] Zero\n\t*[3.14] Pi\n\t[007] James" + "$num ->\n\t[-123456789] Minus a lot\n\t[0] Zero\n *[3.14] Pi\n\t[007] James" => "-123456789", "0", "3.14", "007", ); } From 18014395d11f39e379264a0e46d820dc67ba8708 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Wed, 10 Nov 2021 22:19:33 +0100 Subject: [PATCH 11/27] Fix lints --- fluent-syntax/src/serializer.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index c8ec81d1..4c593272 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -92,7 +92,7 @@ impl Serializer { self.serialize_comment(comment, "#")?; } - self.writer.write_literal(&msg.id.name)?; + self.writer.write_literal(msg.id.name)?; self.writer.write_literal(" =")?; if let Some(value) = msg.value.as_ref() { @@ -111,7 +111,7 @@ impl Serializer { } self.writer.write_literal("-")?; - self.writer.write_literal(&term.id.name)?; + self.writer.write_literal(term.id.name)?; self.writer.write_literal(" =")?; self.serialize_pattern(&term.value)?; @@ -124,7 +124,7 @@ impl Serializer { fn serialize_pattern(&mut self, pattern: &Pattern<&str>) -> Result<(), Error> { let start_on_newline = pattern.elements.iter().any(|elem| match elem { - PatternElement::TextElement { value } => value.contains("\n"), + PatternElement::TextElement { value } => value.contains('\n'), PatternElement::Placeable { expression } => is_select_expr(expression), }); @@ -165,7 +165,7 @@ impl Serializer { fn serialize_attribute(&mut self, attr: &Attribute<&str>) -> Result<(), Error> { self.writer.write_literal(".")?; - self.writer.write_literal(&attr.id.name)?; + self.writer.write_literal(attr.id.name)?; self.writer.write_literal(" =")?; self.serialize_pattern(&attr.value)?; @@ -229,17 +229,17 @@ impl Serializer { Ok(()) } InlineExpression::FunctionReference { id, arguments } => { - self.writer.write_literal(&id.name)?; + self.writer.write_literal(id.name)?; self.serialize_call_arguments(arguments)?; Ok(()) } InlineExpression::MessageReference { id, attribute } => { - self.writer.write_literal(&id.name)?; + self.writer.write_literal(id.name)?; if let Some(attr) = attribute.as_ref() { self.writer.write_literal(".")?; - self.writer.write_literal(&attr.name)?; + self.writer.write_literal(attr.name)?; } Ok(()) @@ -250,11 +250,11 @@ impl Serializer { arguments, } => { self.writer.write_literal("-")?; - self.writer.write_literal(&id.name)?; + self.writer.write_literal(id.name)?; if let Some(attr) = attribute.as_ref() { self.writer.write_literal(".")?; - self.writer.write_literal(&attr.name)?; + self.writer.write_literal(attr.name)?; } if let Some(args) = arguments.as_ref() { self.serialize_call_arguments(args)?; @@ -332,7 +332,7 @@ impl Serializer { self.writer.write_literal(", ")?; } - self.writer.write_literal(&named.name.name)?; + self.writer.write_literal(named.name.name)?; self.writer.write_literal(": ")?; self.serialize_inline_expression(&named.value)?; argument_written = true; @@ -388,11 +388,11 @@ impl TextWriter { } fn newline(&mut self) { - self.buffer.push_str("\n"); + self.buffer.push('\n'); } fn write_literal(&mut self, mut item: &str) -> fmt::Result { - if self.buffer.ends_with("\n") { + if self.buffer.ends_with('\n') { // we've just added a newline, make sure it's properly indented self.write_indent(); @@ -405,7 +405,7 @@ impl TextWriter { } fn write_char_into_indent(&mut self, ch: char) { - if self.buffer.ends_with("\n") { + if self.buffer.ends_with('\n') { self.write_indent(); } self.buffer.pop(); From df8c691dd56f1358972153c2e8223bb1ec7a5f2e Mon Sep 17 00:00:00 2001 From: RumovZ Date: Thu, 11 Nov 2021 09:59:43 +0100 Subject: [PATCH 12/27] Make `serialize()` generic over `Slice` --- fluent-syntax/src/serializer.rs | 105 ++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 40 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 4c593272..e9fd3e28 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -1,11 +1,14 @@ -use crate::ast::*; +use crate::{ast::*, parser::Slice}; use std::fmt::{self, Error, Write}; -pub fn serialize(resource: &Resource<&str>) -> String { +pub fn serialize<'s, S: Slice<'s>>(resource: &Resource) -> String { serialize_with_options(resource, Options::default()) } -pub fn serialize_with_options(resource: &Resource<&str>, options: Options) -> String { +pub fn serialize_with_options<'s, S: Slice<'s>>( + resource: &Resource, + options: Options, +) -> String { let mut ser = Serializer::new(options); ser.serialize_resource(resource) @@ -30,7 +33,7 @@ impl Serializer { } } - pub fn serialize_resource(&mut self, res: &Resource<&str>) -> Result<(), Error> { + pub fn serialize_resource<'s, S: Slice<'s>>(&mut self, res: &Resource) -> Result<(), Error> { for entry in &res.body { match entry { Entry::Message(msg) => self.serialize_message(msg)?, @@ -39,7 +42,7 @@ impl Serializer { Entry::GroupComment(comment) => self.serialize_free_comment(comment, "##")?, Entry::ResourceComment(comment) => self.serialize_free_comment(comment, "###")?, Entry::Junk { content } if self.options.with_junk => { - self.serialize_junk(content)? + self.serialize_junk(content.as_ref())? } Entry::Junk { .. } => continue, } @@ -58,9 +61,9 @@ impl Serializer { self.writer.write_literal(junk) } - fn serialize_free_comment( + fn serialize_free_comment<'s, S: Slice<'s>>( &mut self, - comment: &Comment<&str>, + comment: &Comment, prefix: &str, ) -> Result<(), Error> { if self.state.has_entries { @@ -72,13 +75,17 @@ impl Serializer { Ok(()) } - fn serialize_comment(&mut self, comment: &Comment<&str>, prefix: &str) -> Result<(), Error> { + fn serialize_comment<'s, S: Slice<'s>>( + &mut self, + comment: &Comment, + prefix: &str, + ) -> Result<(), Error> { for line in &comment.content { self.writer.write_literal(prefix)?; - if !line.trim().is_empty() { + if !line.as_ref().trim().is_empty() { self.writer.write_literal(" ")?; - self.writer.write_literal(line)?; + self.writer.write_literal(line.as_ref())?; } self.writer.newline(); @@ -87,12 +94,12 @@ impl Serializer { Ok(()) } - fn serialize_message(&mut self, msg: &Message<&str>) -> Result<(), Error> { + fn serialize_message<'s, S: Slice<'s>>(&mut self, msg: &Message) -> Result<(), Error> { if let Some(comment) = msg.comment.as_ref() { self.serialize_comment(comment, "#")?; } - self.writer.write_literal(msg.id.name)?; + self.writer.write_literal(msg.id.name.as_ref())?; self.writer.write_literal(" =")?; if let Some(value) = msg.value.as_ref() { @@ -105,13 +112,13 @@ impl Serializer { Ok(()) } - fn serialize_term(&mut self, term: &Term<&str>) -> Result<(), Error> { + fn serialize_term<'s, S: Slice<'s>>(&mut self, term: &Term) -> Result<(), Error> { if let Some(comment) = term.comment.as_ref() { self.serialize_comment(comment, "#")?; } self.writer.write_literal("-")?; - self.writer.write_literal(term.id.name)?; + self.writer.write_literal(term.id.name.as_ref())?; self.writer.write_literal(" =")?; self.serialize_pattern(&term.value)?; @@ -122,9 +129,9 @@ impl Serializer { Ok(()) } - fn serialize_pattern(&mut self, pattern: &Pattern<&str>) -> Result<(), Error> { + fn serialize_pattern<'s, S: Slice<'s>>(&mut self, pattern: &Pattern) -> Result<(), Error> { let start_on_newline = pattern.elements.iter().any(|elem| match elem { - PatternElement::TextElement { value } => value.contains('\n'), + PatternElement::TextElement { value } => value.as_ref().contains('\n'), PatternElement::Placeable { expression } => is_select_expr(expression), }); @@ -146,7 +153,10 @@ impl Serializer { Ok(()) } - fn serialize_attributes(&mut self, attrs: &[Attribute<&str>]) -> Result<(), Error> { + fn serialize_attributes<'s, S: Slice<'s>>( + &mut self, + attrs: &[Attribute], + ) -> Result<(), Error> { if attrs.is_empty() { return Ok(()); } @@ -163,9 +173,9 @@ impl Serializer { Ok(()) } - fn serialize_attribute(&mut self, attr: &Attribute<&str>) -> Result<(), Error> { + fn serialize_attribute<'s, S: Slice<'s>>(&mut self, attr: &Attribute) -> Result<(), Error> { self.writer.write_literal(".")?; - self.writer.write_literal(attr.id.name)?; + self.writer.write_literal(attr.id.name.as_ref())?; self.writer.write_literal(" =")?; self.serialize_pattern(&attr.value)?; @@ -173,9 +183,12 @@ impl Serializer { Ok(()) } - fn serialize_element(&mut self, elem: &PatternElement<&str>) -> Result<(), Error> { + fn serialize_element<'s, S: Slice<'s>>( + &mut self, + elem: &PatternElement, + ) -> Result<(), Error> { match elem { - PatternElement::TextElement { value } => self.writer.write_literal(value), + PatternElement::TextElement { value } => self.writer.write_literal(value.as_ref()), PatternElement::Placeable { expression } => match expression { Expression::Inline(InlineExpression::Placeable { expression }) => { // A placeable inside a placeable is a special case because we @@ -203,7 +216,10 @@ impl Serializer { } } - fn serialize_expression(&mut self, expr: &Expression<&str>) -> Result<(), Error> { + fn serialize_expression<'s, S: Slice<'s>>( + &mut self, + expr: &Expression, + ) -> Result<(), Error> { match expr { Expression::Inline(inline) => self.serialize_inline_expression(inline), Expression::Select { selector, variants } => { @@ -212,34 +228,37 @@ impl Serializer { } } - fn serialize_inline_expression(&mut self, expr: &InlineExpression<&str>) -> Result<(), Error> { + fn serialize_inline_expression<'s, S: Slice<'s>>( + &mut self, + expr: &InlineExpression, + ) -> Result<(), Error> { match expr { InlineExpression::StringLiteral { value } => { self.writer.write_literal("\"")?; - self.writer.write_literal(value)?; + self.writer.write_literal(value.as_ref())?; self.writer.write_literal("\"")?; Ok(()) } - InlineExpression::NumberLiteral { value } => self.writer.write_literal(value), + InlineExpression::NumberLiteral { value } => self.writer.write_literal(value.as_ref()), InlineExpression::VariableReference { id: Identifier { name: value }, } => { self.writer.write_literal("$")?; - self.writer.write_literal(value)?; + self.writer.write_literal(value.as_ref())?; Ok(()) } InlineExpression::FunctionReference { id, arguments } => { - self.writer.write_literal(id.name)?; + self.writer.write_literal(id.name.as_ref())?; self.serialize_call_arguments(arguments)?; Ok(()) } InlineExpression::MessageReference { id, attribute } => { - self.writer.write_literal(id.name)?; + self.writer.write_literal(id.name.as_ref())?; if let Some(attr) = attribute.as_ref() { self.writer.write_literal(".")?; - self.writer.write_literal(attr.name)?; + self.writer.write_literal(attr.name.as_ref())?; } Ok(()) @@ -250,11 +269,11 @@ impl Serializer { arguments, } => { self.writer.write_literal("-")?; - self.writer.write_literal(id.name)?; + self.writer.write_literal(id.name.as_ref())?; if let Some(attr) = attribute.as_ref() { self.writer.write_literal(".")?; - self.writer.write_literal(attr.name)?; + self.writer.write_literal(attr.name.as_ref())?; } if let Some(args) = arguments.as_ref() { self.serialize_call_arguments(args)?; @@ -272,10 +291,10 @@ impl Serializer { } } - fn serialize_select_expression( + fn serialize_select_expression<'s, S: Slice<'s>>( &mut self, - selector: &InlineExpression<&str>, - variants: &[Variant<&str>], + selector: &InlineExpression, + variants: &[Variant], ) -> Result<(), Error> { self.serialize_inline_expression(selector)?; self.writer.write_literal(" ->")?; @@ -292,7 +311,7 @@ impl Serializer { Ok(()) } - fn serialize_variant(&mut self, variant: &Variant<&str>) -> Result<(), Error> { + fn serialize_variant<'s, S: Slice<'s>>(&mut self, variant: &Variant) -> Result<(), Error> { if variant.default { self.writer.write_char_into_indent('*'); } @@ -305,15 +324,21 @@ impl Serializer { Ok(()) } - fn serialize_variant_key(&mut self, key: &VariantKey<&str>) -> Result<(), Error> { + fn serialize_variant_key<'s, S: Slice<'s>>( + &mut self, + key: &VariantKey, + ) -> Result<(), Error> { match key { VariantKey::NumberLiteral { value } | VariantKey::Identifier { name: value } => { - self.writer.write_literal(value) + self.writer.write_literal(value.as_ref()) } } } - fn serialize_call_arguments(&mut self, args: &CallArguments<&str>) -> Result<(), Error> { + fn serialize_call_arguments<'s, S: Slice<'s>>( + &mut self, + args: &CallArguments, + ) -> Result<(), Error> { let mut argument_written = false; self.writer.write_literal("(")?; @@ -332,7 +357,7 @@ impl Serializer { self.writer.write_literal(", ")?; } - self.writer.write_literal(named.name.name)?; + self.writer.write_literal(named.name.name.as_ref())?; self.writer.write_literal(": ")?; self.serialize_inline_expression(&named.value)?; argument_written = true; @@ -343,7 +368,7 @@ impl Serializer { } } -fn is_select_expr(expr: &Expression<&str>) -> bool { +fn is_select_expr<'s, S: Slice<'s>>(expr: &Expression) -> bool { match expr { Expression::Select { .. } => true, Expression::Inline(InlineExpression::Placeable { expression }) => { From 1b340e75277df2ca786645f7e9e06d3166703f1b Mon Sep 17 00:00:00 2001 From: RumovZ Date: Mon, 25 Apr 2022 21:50:27 +0200 Subject: [PATCH 13/27] Handle rare edge case of line terminating `\r` --- fluent-syntax/src/serializer.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index e9fd3e28..f419a0f2 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -413,6 +413,11 @@ impl TextWriter { } fn newline(&mut self) { + if self.buffer.ends_with('\r') { + // handle rare edge case, where the trailing `\r` would get confused + // as part of the line ending + self.buffer.push('\r'); + } self.buffer.push('\n'); } From 099995a498a99f4cdfbf82af27e0e1f71ad2ce3f Mon Sep 17 00:00:00 2001 From: RumovZ Date: Mon, 25 Apr 2022 21:51:05 +0200 Subject: [PATCH 14/27] Fix redundant line break after after junk --- fluent-syntax/src/serializer.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index f419a0f2..937f7c8a 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -41,13 +41,14 @@ impl Serializer { Entry::Comment(comment) => self.serialize_free_comment(comment, "#")?, Entry::GroupComment(comment) => self.serialize_free_comment(comment, "##")?, Entry::ResourceComment(comment) => self.serialize_free_comment(comment, "###")?, - Entry::Junk { content } if self.options.with_junk => { - self.serialize_junk(content.as_ref())? + Entry::Junk { content } => { + if self.options.with_junk { + self.serialize_junk(content.as_ref())? + } } - Entry::Junk { .. } => continue, - } + }; - self.state.has_entries = true; + self.state.wrote_non_junk_entry = !matches!(entry, Entry::Junk { .. }); } Ok(()) @@ -66,7 +67,7 @@ impl Serializer { comment: &Comment, prefix: &str, ) -> Result<(), Error> { - if self.state.has_entries { + if self.state.wrote_non_junk_entry { self.writer.newline(); } self.serialize_comment(comment, prefix)?; @@ -385,7 +386,7 @@ pub struct Options { #[derive(Debug, Default, PartialEq)] struct State { - has_entries: bool, + wrote_non_junk_entry: bool, } #[derive(Debug, Clone, Default)] From c0096744bf61727c4414121f60964a591b4b459a Mon Sep 17 00:00:00 2001 From: RumovZ Date: Mon, 25 Apr 2022 22:26:44 +0200 Subject: [PATCH 15/27] Align parsing of CRLF-terminated patterns `\r\n` was parsed as a separate TextElement of `\n`, whereas `\n` is parsed as part of the antecedent TextElement. --- fluent-syntax/src/parser/pattern.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fluent-syntax/src/parser/pattern.rs b/fluent-syntax/src/parser/pattern.rs index 85c8925e..9141d4f3 100644 --- a/fluent-syntax/src/parser/pattern.rs +++ b/fluent-syntax/src/parser/pattern.rs @@ -174,8 +174,8 @@ where b'\r' if self.is_byte_at(b'\n', self.ptr + 1) => { self.ptr += 1; return Ok(( - start_pos, - self.ptr - 1, + start_pos + 1, + self.ptr, text_element_type, TextElementTermination::Crlf, )); From c469cf1fafd030999133de9feca0111d473b7bec Mon Sep 17 00:00:00 2001 From: RumovZ Date: Mon, 25 Apr 2022 22:55:12 +0200 Subject: [PATCH 16/27] Don't break line before leading dot pattern --- fluent-syntax/src/serializer.rs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 937f7c8a..e138b062 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -131,10 +131,7 @@ impl Serializer { } fn serialize_pattern<'s, S: Slice<'s>>(&mut self, pattern: &Pattern) -> Result<(), Error> { - let start_on_newline = pattern.elements.iter().any(|elem| match elem { - PatternElement::TextElement { value } => value.as_ref().contains('\n'), - PatternElement::Placeable { expression } => is_select_expr(expression), - }); + let start_on_newline = pattern.starts_on_new_line(); if start_on_newline { self.writer.newline(); @@ -369,6 +366,27 @@ impl Serializer { } } +impl<'s, S: Slice<'s>> Pattern { + fn starts_on_new_line(&self) -> bool { + !self.has_leading_text_dot() && self.is_multiline() + } + + fn is_multiline(&self) -> bool { + self.elements.iter().any(|elem| match elem { + PatternElement::TextElement { value } => value.as_ref().contains('\n'), + PatternElement::Placeable { expression } => is_select_expr(expression), + }) + } + + fn has_leading_text_dot(&self) -> bool { + if let Some(PatternElement::TextElement { value }) = self.elements.get(0) { + value.as_ref().starts_with('.') + } else { + false + } + } +} + fn is_select_expr<'s, S: Slice<'s>>(expr: &Expression) -> bool { match expr { Expression::Select { .. } => true, From 6fbf7f9ff929b75004cf0cba0a6309c1307b6116 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Tue, 26 Apr 2022 08:46:21 +0200 Subject: [PATCH 17/27] Don't implicitly trim when writing literals --- fluent-syntax/src/serializer.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index e138b062..e3b61c8b 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -440,14 +440,10 @@ impl TextWriter { self.buffer.push('\n'); } - fn write_literal(&mut self, mut item: &str) -> fmt::Result { + fn write_literal(&mut self, item: &str) -> fmt::Result { if self.buffer.ends_with('\n') { // we've just added a newline, make sure it's properly indented self.write_indent(); - - // we've just added indentation, so we don't care about leading - // spaces - item = item.trim_start_matches(' '); } write!(self.buffer, "{}", item) From 24e6d49dfde169f8dbf6b18145d55d8223b00185 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Tue, 8 Nov 2022 16:41:57 +0100 Subject: [PATCH 18/27] Replace roundtrip tests with manipulation tests --- fluent-syntax/src/serializer.rs | 425 ++++++++++++-------------------- 1 file changed, 160 insertions(+), 265 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index e3b61c8b..1ef2644e 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -459,8 +459,9 @@ impl TextWriter { } #[cfg(test)] -mod serialize_resource_tests { +mod test { use super::*; + use crate::parser::parse; #[test] fn write_something_then_indent() -> fmt::Result { @@ -486,282 +487,176 @@ mod serialize_resource_tests { Ok(()) } - macro_rules! round_trip_test { - ($name:ident, $text:expr $(,)?) => { - round_trip_test!($name, $text, $text); + macro_rules! text_message { + ($name:expr, $value:expr) => { + Entry::Message(Message { + id: Identifier { name: $name }, + value: Some(Pattern { + elements: vec![PatternElement::TextElement { value: $value }], + }), + attributes: vec![], + comment: None, + }) }; - ($name:ident, $text:expr, $should_be:expr $(,)?) => { - #[test] - fn $name() { - // Note: We add tabs to the input so it's easier to recognise - // indentation - let input_without_tabs = $text.replace("\t", " "); - let should_be_without_tabs = $should_be.replace("\t", " "); - - let resource = crate::parser::parse(input_without_tabs.as_str()).unwrap(); - let got = serialize(&resource); - - assert_eq!(got, should_be_without_tabs); + } + + impl<'a> Entry<&'a str> { + fn as_message(&mut self) -> &mut Message<&'a str> { + match self { + Self::Message(msg) => msg, + _ => panic!("Expected Message"), } - }; + } } - round_trip_test!(simple_message_without_eol, "foo = Foo", "foo = Foo\n"); - round_trip_test!(simple_message, "foo = Foo\n"); - round_trip_test!(two_simple_messages, "foo = Foo\nbar = Bar\n"); - round_trip_test!(block_multiline_message, "foo =\n Foo\n Bar\n"); - round_trip_test!( - inline_multiline_message, - "foo = Foo\n Bar\n", - "foo =\n Foo\n Bar\n", - ); - round_trip_test!(message_reference, "foo = Foo { bar }\n"); - round_trip_test!(term_reference, "foo = Foo { -bar }\n"); - round_trip_test!(external_reference, "foo = Foo { $bar }\n"); - round_trip_test!(number_element, "foo = Foo { 1 }\n"); - round_trip_test!(string_element, "foo = Foo { \"bar\" }\n"); - round_trip_test!(attribute_expression, "foo = Foo { bar.baz }\n"); - round_trip_test!( - resource_comment, - "### A multiline\n### resource comment.\n\nfoo = Foo\n", - ); - round_trip_test!( - message_comment, - "# A multiline\n# message comment.\nfoo = Foo\n", - ); - round_trip_test!( - dont_prefix_a_subsequent_entry_comment_with_a_newline, - "first = Firstsubsequent_ Comment\nfoo = Foo\n", - ); - round_trip_test!( - group_comment, - "## Comment Header\n##\n## A multiline\n## group comment.\n\nfoo = Foo\n", - ); - round_trip_test!( - standalone_comment, - "foo = Foo\n\n# A Standalone Comment\n\nbar = Bar\n", - ); - round_trip_test!(multiline_with_placeable, "foo =\n\tFoo { bar }\n\tBaz\n",); - round_trip_test!(attribute, "foo =\n\t.attr = Foo Attr\n"); - round_trip_test!( - multiline_attribute, - "foo =\n\t.attr =\n\t\tFoo Attr\n\t\tContinued\n", - ); - round_trip_test!( - two_attributes, - "foo =\n\t.attr-a = Foo Attr A\n\t.attr-b = Foo Attr B\n", - ); - round_trip_test!( - value_and_attributes, - "foo = Foo Value\n\t.attr-a = Foo Attr A\n\t.attr-b = Foo Attr B\n", - ); - round_trip_test!( - multiline_value_and_attributes, - "foo =\n\tFoo Value\n\tContinued\n\t.attr-a = Foo Attr A\n\t.attr-b = Foo Attr B\n", - ); - round_trip_test!( - select_expression, - "foo =\n\t{ $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", - ); - round_trip_test!( - multiline_variant, - "foo =\n\t{ $sel ->\n\t *[a]\n\t\t\tAAA\n\t\t\tBBBB\n\t}\n", - ); - round_trip_test!( - multiline_variant_with_first_line_inline, - "foo =\n\t{ $sel ->\n\t *[a] AAA\n\t\tBBB\n\t}\n", - "foo =\n\t{ $sel ->\n\t *[a]\n\t\t\tAAA\n\t\t\tBBB\n\t}\n", - ); - round_trip_test!( - variant_key_number, - "foo =\n\t{ $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", - ); - round_trip_test!( - select_expression_in_block_value, - "foo =\n\tFoo { $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", - ); - round_trip_test!( - select_expression_in_inline_value, - "foo = Foo { $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", - "foo =\n\tFoo { $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", - ); - round_trip_test!( - select_expression_in_multiline_value, - "foo =\n\tFoo\n\tBar { $sel ->\n\t *[a] A\n\t\t[b] B\n\t}\n", - ); - round_trip_test!( - nested_select_expression, - "foo =\n\t{ $a ->\n\t *[a]\n\t\t\t{ $b ->\n\t\t\t *[b] Foo\n\t\t\t}\n\t}\n", - ); - round_trip_test!( - selector_external_argument, - "foo =\n\t{ $bar ->\n\t *[a] A\n\t}\n", - ); - round_trip_test!( - selector_number_expression, - "foo =\n\t{ 1 ->\n\t *[a] A\n\t}\n", - ); - round_trip_test!( - selector_string_expression, - "foo =\n\t{ \"bar\" ->\n\t *[a] A\n\t}\n", - ); - round_trip_test!( - selector_attribute_expression, - "foo =\n\t{ -bar.baz ->\n\t *[a] A\n\t}\n", - ); - round_trip_test!(call_expression, "foo = { FOO() }\n",); - round_trip_test!( - call_expression_with_string_expression, - "foo = { FOO(\"bar\") }\n", - ); - round_trip_test!(call_expression_with_number_expression, "foo = { FOO(1) }\n",); - round_trip_test!( - call_expression_with_message_reference, - "foo = { FOO(bar) }\n", - ); - round_trip_test!( - call_expression_with_external_argument, - "foo = { FOO($bar) }\n", - ); - round_trip_test!( - call_expression_with_number_named_argument, - "foo = { FOO(bar: 1) }\n", - ); - round_trip_test!( - call_expression_with_string_named_argument, - "foo = { FOO(bar: \"bar\") }\n", - ); - round_trip_test!( - call_expression_with_two_positional_arguments, - "foo = { FOO(bar, baz) }\n", - ); - round_trip_test!( - call_expression_with_positional_and_named_arguments, - "foo = { FOO(bar, 1, baz: \"baz\") }\n", - ); - round_trip_test!(macro_call, "foo = { -term() }\n",); - round_trip_test!(nested_placeables, "foo = {{ FOO() }}\n",); - round_trip_test!(backslash_in_text_element, "foo = \\{ placeable }\n",); - round_trip_test!( - excaped_special_char_in_string_literal, - "foo = { \"Escaped \\\" quote\" }\n", - ); - round_trip_test!(unicode_escape_sequence, "foo = { \"\\u0065\" }\n",); - - // Serialize padding around comments - - round_trip_test!( - standalone_comment_has_not_padding_when_first, - "# Comment A\n\nfoo = Foo\n\n# Comment B\n\nbar = Bar\n" - ); - round_trip_test!( - group_comment_has_not_padding_when_first, - "## Group A\n\nfoo = Foo\n\n## Group B\n\nbar = Bar\n" - ); - round_trip_test!( - resource_comment_has_not_padding_when_first, - "### Resource Comment A\n\nfoo = Foo\n\n### Resource Comment B\n\nbar = Bar\n" - ); -} + impl<'a> Message<&'a str> { + fn as_pattern(&mut self) -> &mut Pattern<&'a str> { + self.value.as_mut().expect("Expected Pattern") + } + } -#[cfg(test)] -mod serialize_expression_tests { - use super::*; + impl<'a> PatternElement<&'a str> { + fn as_text(&mut self) -> &mut &'a str { + match self { + Self::TextElement { value } => value, + _ => panic!("Expected TextElement"), + } + } - macro_rules! expression_test { - ($name:ident, $input:expr) => { - #[test] - fn $name() { - let input_without_tabs = $input.replace("\t", " "); - let src = format!("foo = {{ {} }}", input_without_tabs); - let resource = crate::parser::parse(src.as_str()).unwrap(); - - // extract the first expression from the value of the first - // message - assert_eq!(resource.body.len(), 1); - let first_item = &resource.body[0]; - let message = match first_item { - Entry::Message(msg) => msg, - other => panic!("Expected a message but found {:#?}", other), - }; - let value = message.value.as_ref().expect("The message has a value"); - assert_eq!(value.elements.len(), 1); - let expr = match &value.elements[0] { - PatternElement::Placeable { expression } => expression, - other => panic!("Expected a single expression but found {:#?}", other), - }; - - // we've finally extracted the first expression, now we can - // actually serialize it and finish the test - let mut serializer = Serializer::new(Options::default()); - serializer.serialize_expression(expr).unwrap(); - let got = serializer.into_serialized_text(); - - assert_eq!(got, input_without_tabs); + fn as_expression(&mut self) -> &mut Expression<&'a str> { + match self { + Self::Placeable { expression } => expression, + _ => panic!("Expected Placeable"), } - }; + } } - expression_test!(string_expression, "\"str\""); - expression_test!(number_expression, "3"); - expression_test!(message_reference, "msg"); - expression_test!(external_arguemnt, "$ext"); - expression_test!(attribute_expression, "msg.attr"); - expression_test!(call_expression, "BUILTIN(3.14, kwarg: \"value\")"); - expression_test!(select_expression, "$num ->\n *[one] One\n"); -} + impl<'a> Expression<&'a str> { + fn as_variants(&mut self) -> &mut Vec> { + match self { + Self::Select { variants, .. } => variants, + _ => panic!("Expected Select"), + } + } + fn as_inline_variable_id(&mut self) -> &mut Identifier<&'a str> { + match self { + Self::Inline(InlineExpression::VariableReference { id }) => id, + _ => panic!("Expected Inline"), + } + } + } -#[cfg(test)] -mod serialize_variant_key_tests { - use super::*; + #[test] + fn change_id() { + let mut ast = parse("foo = bar\n").unwrap(); + ast.body[0].as_message().id.name = "baz"; + assert_eq!(serialize(&ast), "baz = bar\n"); + } - macro_rules! variant_key_test { - ($name:ident, $input:expr => $( $keys:expr ),+ $(,)?) => { - #[test] - #[allow(unused_assignments)] - fn $name() { - let input_without_tabs = $input.replace("\t", " "); - let src = format!("foo = {{ {}\n }}", input_without_tabs); - let resource = crate::parser::parse(src.as_str()).unwrap(); - - // extract variant from the first expression from the value of - // the first message - assert_eq!(resource.body.len(), 1); - let first_item = &resource.body[0]; - let message = match first_item { - Entry::Message(msg) => msg, - other => panic!("Expected a message but found {:#?}", other), - }; - let value = message.value.as_ref().expect("The message has a value"); - assert_eq!(value.elements.len(), 1); - let variants = match &value.elements[0] { - PatternElement::Placeable { expression: Expression::Select { variants, .. } } => variants, - other => panic!("Expected a single select expression but found {:#?}", other), - }; - - let mut ix = 0; - - $( - let variant_key = &variants[ix].key; - - // we've finally extracted the variant key, now we can - // actually serialize it and finish the test - let mut serializer = Serializer::new(Options::default()); - serializer.serialize_variant_key(variant_key).unwrap(); - let got = serializer.into_serialized_text(); - - assert_eq!(got, $keys); - - ix += 1; - )* - } + #[test] + fn change_value() { + let mut ast = parse("foo = bar\n").unwrap(); + *ast.body[0].as_message().as_pattern().elements[0].as_text() = "baz"; + assert_eq!("foo = baz\n", serialize(&ast)); + } + + #[test] + fn add_expression_variant() { + let message = concat!( + "foo =\n", + " { $num ->\n", + " *[other] { $num } bars\n", + " }\n" + ); + let mut ast = parse(message).unwrap(); + + let one_variant = Variant { + key: VariantKey::Identifier { name: "one" }, + value: Pattern { + elements: vec![ + PatternElement::Placeable { + expression: Expression::Inline(InlineExpression::VariableReference { + id: Identifier { name: "num" }, + }), + }, + PatternElement::TextElement { value: " bar" }, + ], + }, + default: false, }; + ast.body[0].as_message().as_pattern().elements[0] + .as_expression() + .as_variants() + .insert(0, one_variant); + + let expected = concat!( + "foo =\n", + " { $num ->\n", + " [one] { $num } bar\n", + " *[other] { $num } bars\n", + " }\n" + ); + assert_eq!(serialize(&ast), expected); + } + + #[test] + fn change_variable_reference() { + let mut ast = parse("foo = { $bar }\n").unwrap(); + ast.body[0].as_message().as_pattern().elements[0] + .as_expression() + .as_inline_variable_id() + .name = "qux"; + assert_eq!("foo = { $qux }\n", serialize(&ast)); } - variant_key_test!(identifiers, "$num ->\n\t[one] One\n *[other] Other" => "one", "other"); - variant_key_test!( - number_literals, - "$num ->\n\t[-123456789] Minus a lot\n\t[0] Zero\n *[3.14] Pi\n\t[007] James" - => "-123456789", "0", "3.14", "007", - ); + #[test] + fn remove_message() { + let mut ast = parse("foo = bar\nbaz = qux\n").unwrap(); + ast.body.pop(); + assert_eq!("foo = bar\n", serialize(&ast)); + } + + #[test] + fn add_message_at_top() { + let mut ast = parse("foo = bar\n").unwrap(); + ast.body.insert(0, text_message!("baz", "qux")); + assert_eq!("baz = qux\nfoo = bar\n", serialize(&ast)); + } + + #[test] + fn add_message_at_end() { + let mut ast = parse("foo = bar\n").unwrap(); + ast.body.push(text_message!("baz", "qux")); + assert_eq!("foo = bar\nbaz = qux\n", serialize(&ast)); + } + + #[test] + fn add_message_in_between() { + let mut ast = parse("foo = bar\nbaz = qux\n").unwrap(); + ast.body.insert(1, text_message!("hello", "there")); + assert_eq!("foo = bar\nhello = there\nbaz = qux\n", serialize(&ast)); + } + + #[test] + fn add_message_comment() { + let mut ast = parse("foo = bar\n").unwrap(); + ast.body[0].as_message().comment.replace(Comment { + content: vec!["great message!"], + }); + assert_eq!("# great message!\nfoo = bar\n", serialize(&ast)); + } + + #[test] + fn remove_message_comment() { + let mut ast = parse("# great message!\nfoo = bar\n").unwrap(); + ast.body[0].as_message().comment.take(); + assert_eq!("foo = bar\n", serialize(&ast)); + } + + #[test] + fn edit_message_comment() { + let mut ast = parse("# great message!\nfoo = bar\n").unwrap(); + ast.body[0].as_message().comment.as_mut().unwrap().content[0] = "very original"; + assert_eq!("# very original\nfoo = bar\n", serialize(&ast)); + } } From 99cd884357216bdf9c4b4f5e65c8da386833d077 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Tue, 8 Nov 2022 16:46:48 +0100 Subject: [PATCH 19/27] Reintroduce roundtrip tests * Move old fixtures into resource files. * Test on unnormalized fixtures as well. --- .../tests/fixtures/normalized/attributes.ftl | 17 +++++ .../fixtures/normalized/call_expressions.ftl | 11 +++ .../tests/fixtures/normalized/comments.ftl | 13 ++++ .../tests/fixtures/normalized/escapes.ftl | 3 + .../normalized/inline_expressions.ftl | 6 ++ .../tests/fixtures/normalized/messages.ftl | 4 ++ .../fixtures/normalized/select_expression.ftl | 50 ++++++++++++++ fluent-syntax/tests/serializer_fixtures.rs | 68 +++++++++++++++++++ 8 files changed, 172 insertions(+) create mode 100644 fluent-syntax/tests/fixtures/normalized/attributes.ftl create mode 100644 fluent-syntax/tests/fixtures/normalized/call_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/normalized/comments.ftl create mode 100644 fluent-syntax/tests/fixtures/normalized/escapes.ftl create mode 100644 fluent-syntax/tests/fixtures/normalized/inline_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/normalized/messages.ftl create mode 100644 fluent-syntax/tests/fixtures/normalized/select_expression.ftl create mode 100644 fluent-syntax/tests/serializer_fixtures.rs diff --git a/fluent-syntax/tests/fixtures/normalized/attributes.ftl b/fluent-syntax/tests/fixtures/normalized/attributes.ftl new file mode 100644 index 00000000..2140da61 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/attributes.ftl @@ -0,0 +1,17 @@ +attribute = + .attr = attribute +multiline-attribute = + .attr = + multiline + attribute +two-attributes = + .attr-a = attribute A + .attr-b = attribute B +value-and-attributes = value + .attr-a = attribute A + .attr-b = attribute B +multiline-value-and_attributes = + value A + value B + .attr-a = attribute A + .attr-b = attribute B diff --git a/fluent-syntax/tests/fixtures/normalized/call_expressions.ftl b/fluent-syntax/tests/fixtures/normalized/call_expressions.ftl new file mode 100644 index 00000000..6785f8cf --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/call_expressions.ftl @@ -0,0 +1,11 @@ +call-expression = { FOO() } +call-expression-with-string-expression = { FOO("bar") } +call-expression-with-number-expression = { FOO(1) } +call-expression-with-message-reference = { FOO(bar) } +call-expression-with-external-argument = { FOO($bar) } +call-expression-with-number-named-argument = { FOO(bar: 1) } +call-expression-with-string-named-argument = { FOO(bar: "bar") } +call-expression-with-two-positional-arguments = { FOO(bar, baz) } +call-expression-with-positional-and-named-arguments = { FOO(bar, 1, baz: "baz") } +macro-call = { -term() } +nested-placeables = {{ FOO() }} diff --git a/fluent-syntax/tests/fixtures/normalized/comments.ftl b/fluent-syntax/tests/fixtures/normalized/comments.ftl new file mode 100644 index 00000000..a42e2793 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/comments.ftl @@ -0,0 +1,13 @@ +# standalone comment + + +### multiline +### resource comment + + +## multiline +## group comment + +# multiline +# message comment +foo = bar diff --git a/fluent-syntax/tests/fixtures/normalized/escapes.ftl b/fluent-syntax/tests/fixtures/normalized/escapes.ftl new file mode 100644 index 00000000..c5433786 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/escapes.ftl @@ -0,0 +1,3 @@ +backslash_in_text_element = \{ placeable } +excaped_special_char_in_string_literal = { "Escaped \" quote" } +unicode_escape_sequence = { "\u0065" } diff --git a/fluent-syntax/tests/fixtures/normalized/inline_expressions.ftl b/fluent-syntax/tests/fixtures/normalized/inline_expressions.ftl new file mode 100644 index 00000000..8901f1a4 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/inline_expressions.ftl @@ -0,0 +1,6 @@ +simple-reference = simple { reference } +term-reference = term { -reference } +external-reference = external { $reference } +number-element = number { 1 } +string-element = string { "element" } +attribute-expression = attribute { ex.pression } diff --git a/fluent-syntax/tests/fixtures/normalized/messages.ftl b/fluent-syntax/tests/fixtures/normalized/messages.ftl new file mode 100644 index 00000000..9aba5cb1 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/messages.ftl @@ -0,0 +1,4 @@ +simple = simple +multiline = + multi + line diff --git a/fluent-syntax/tests/fixtures/normalized/select_expression.ftl b/fluent-syntax/tests/fixtures/normalized/select_expression.ftl new file mode 100644 index 00000000..24546f96 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/select_expression.ftl @@ -0,0 +1,50 @@ +select-expression = + { $sel -> + *[a] A + [b] B + } +multiline-variant = + { $sel -> + *[a] + AAA + BBBB + } +variant-key-number = + { $sel -> + *[a] A + [b] B + } +select-expression-in-block-value = + Foo { $sel -> + *[a] A + [b] B + } +select-expression-in-multiline-value = + Foo + Bar { $sel -> + *[a] A + [b] B + } +nested-select-expression = + { $a -> + *[a] + { $b -> + *[b] Foo + } + } +selector-external-argument = + { $bar -> + *[a] A + } +selector-number-expression = + { 1 -> + *[a] A + } +selector-string-expression = + { "bar" -> + *[a] A + } +selector-attribute-expression = + { -bar.baz -> + *[a] A + } diff --git a/fluent-syntax/tests/serializer_fixtures.rs b/fluent-syntax/tests/serializer_fixtures.rs new file mode 100644 index 00000000..e381ef61 --- /dev/null +++ b/fluent-syntax/tests/serializer_fixtures.rs @@ -0,0 +1,68 @@ +use fluent_syntax::ast::{Entry, Resource}; +use glob::glob; +use std::ffi::OsStr; +use std::fs; +use std::path::Path; + +use fluent_syntax::parser::parse; +use fluent_syntax::serializer::{serialize, serialize_with_options, Options}; + +/// List of files that currently do not roundtrip correctly. +/// +/// - `multiline_values.ftl`: `key12` is parsed differently if indented. +const BLACKLIST: [&str; 1] = ["multiline_values.ftl"]; + +fn is_blacklisted(path: &Path) -> bool { + path.file_name() + .and_then(OsStr::to_str) + .map(|s| BLACKLIST.contains(&s)) + .unwrap_or_default() +} + +fn clone_without_junk<'a>(original: &Resource<&'a str>) -> Resource<&'a str> { + Resource { + body: original + .body + .iter() + .filter(|entry| !matches!(entry, Entry::Junk { .. })) + .cloned() + .collect(), + } +} + +#[test] +fn roundtrip_normalized_fixtures() { + for entry in glob("./tests/fixtures/normalized/*.ftl").expect("Failed to read glob pattern") { + let path = entry.expect("Error while getting an entry"); + let content = fs::read_to_string(&path).expect("Failed to read file"); + let parsed = parse(content.as_str()).unwrap_or_else(|(res, _)| res); + let reserialized = serialize(&parsed); + assert_eq!(content, reserialized); + } +} + +/// Compares a parsed AST with a parsed, serialized and reparsed AST, as these fixtures +/// contain unnormalized syntax that is not supposed to be preserved on a roundtrip. +/// Tests both parsing with and without junk. +#[test] +fn roundtrip_unnormalized_fixtures() { + for entry in glob("./tests/fixtures/*.ftl").expect("Failed to read glob pattern") { + let path = entry.expect("Error while getting an entry"); + if is_blacklisted(&path) { + continue; + } + + let content = fs::read_to_string(&path).expect("Failed to read file"); + let parsed = parse(content.as_str()).unwrap_or_else(|(res, _)| res); + let parsed_without_junk = clone_without_junk(&parsed); + let reserialized = serialize_with_options(&parsed, Options { with_junk: true }); + let reserialized_without_junk = + serialize_with_options(&parsed, Options { with_junk: false }); + let reparsed = parse(reserialized.as_str()).unwrap_or_else(|(res, _)| res); + let reparsed_without_junk = + parse(reserialized_without_junk.as_str()).unwrap_or_else(|(res, _)| res); + + assert_eq!(reparsed_without_junk, parsed_without_junk); + assert_eq!(reparsed, parsed); + } +} From 40e1a42f4d7400f07d4529e97d5dca468fb54d0a Mon Sep 17 00:00:00 2001 From: RumovZ Date: Tue, 8 Nov 2022 16:51:36 +0100 Subject: [PATCH 20/27] Fix clippy lints --- fluent-syntax/src/serializer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 1ef2644e..5352a4b7 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -391,13 +391,13 @@ fn is_select_expr<'s, S: Slice<'s>>(expr: &Expression) -> bool { match expr { Expression::Select { .. } => true, Expression::Inline(InlineExpression::Placeable { expression }) => { - is_select_expr(&*expression) + is_select_expr(expression) } Expression::Inline(_) => false, } } -#[derive(Debug, Default, Copy, Clone, PartialEq)] +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq)] pub struct Options { pub with_junk: bool, } From 5da5604a0dcd80ef8fa5be55f72ddc0d40981ce0 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Tue, 8 Nov 2022 17:22:49 +0100 Subject: [PATCH 21/27] Document serializer mod and pub functions Also make Serializer private. --- fluent-syntax/src/serializer.rs | 59 ++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 5352a4b7..4e908d77 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -1,10 +1,59 @@ +//! Fluent Translation List serialization utilities +//! +//! This modules provides a way to serialize an abstract syntax tree representing a +//! Fluent Translation List. Use cases include normalization and programmatical +//! manipulation of a Fluent Translation List. +//! +//! # Example +//! +//! ``` +//! use fluent_syntax::parser; +//! use fluent_syntax::serializer; +//! +//! let ftl = r#"# This is a message comment +//! hello-world = Hello World! +//! "#; +//! +//! let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); +//! +//! let serialzed = serializer::serialize(&resource); +//! +//! assert_eq!(ftl, serialzed); +//! ``` + use crate::{ast::*, parser::Slice}; use std::fmt::{self, Error, Write}; +/// Serializes an abstract syntax tree representing a Fluent Translation List into a +/// String. +/// +/// # Example +/// +/// ``` +/// use fluent_syntax::parser; +/// use fluent_syntax::serializer; +/// +/// let ftl = r#" +/// unnormalized-message=This message has +/// abnormal spacing and indentation"#; +/// +/// let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); +/// +/// let serialzed = serializer::serialize(&resource); +/// +/// let expected = r#"unnormalized-message = +/// This message has +/// abnormal spacing and indentation +/// "#; +/// +/// assert_eq!(expected, serialzed); +/// ``` pub fn serialize<'s, S: Slice<'s>>(resource: &Resource) -> String { serialize_with_options(resource, Options::default()) } +/// Serializes an abstract syntax tree representing a Fluent Translation List into a +/// String accepting custom options. pub fn serialize_with_options<'s, S: Slice<'s>>( resource: &Resource, options: Options, @@ -18,14 +67,14 @@ pub fn serialize_with_options<'s, S: Slice<'s>>( } #[derive(Debug)] -pub struct Serializer { +struct Serializer { writer: TextWriter, options: Options, state: State, } impl Serializer { - pub fn new(options: Options) -> Self { + fn new(options: Options) -> Self { Serializer { writer: TextWriter::default(), options, @@ -33,7 +82,7 @@ impl Serializer { } } - pub fn serialize_resource<'s, S: Slice<'s>>(&mut self, res: &Resource) -> Result<(), Error> { + fn serialize_resource<'s, S: Slice<'s>>(&mut self, res: &Resource) -> Result<(), Error> { for entry in &res.body { match entry { Entry::Message(msg) => self.serialize_message(msg)?, @@ -54,7 +103,7 @@ impl Serializer { Ok(()) } - pub fn into_serialized_text(self) -> String { + fn into_serialized_text(self) -> String { self.writer.buffer } @@ -397,8 +446,10 @@ fn is_select_expr<'s, S: Slice<'s>>(expr: &Expression) -> bool { } } +/// Options for serializing an abstract syntax tree. #[derive(Debug, Default, Copy, Clone, PartialEq, Eq)] pub struct Options { + /// Whether invalid text fragments should be serialized, too. pub with_junk: bool, } From a1ae7dd83fe8dda6290fb0efc6b7a9c89fcc4e83 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Tue, 8 Nov 2022 17:25:07 +0100 Subject: [PATCH 22/27] Replace unwrap with expect --- fluent-syntax/src/serializer.rs | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 4e908d77..50b4dc37 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -599,14 +599,14 @@ mod test { #[test] fn change_id() { - let mut ast = parse("foo = bar\n").unwrap(); + let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); ast.body[0].as_message().id.name = "baz"; assert_eq!(serialize(&ast), "baz = bar\n"); } #[test] fn change_value() { - let mut ast = parse("foo = bar\n").unwrap(); + let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); *ast.body[0].as_message().as_pattern().elements[0].as_text() = "baz"; assert_eq!("foo = baz\n", serialize(&ast)); } @@ -619,7 +619,7 @@ mod test { " *[other] { $num } bars\n", " }\n" ); - let mut ast = parse(message).unwrap(); + let mut ast = parse(message).expect("failed to parse ftl resource"); let one_variant = Variant { key: VariantKey::Identifier { name: "one" }, @@ -652,7 +652,7 @@ mod test { #[test] fn change_variable_reference() { - let mut ast = parse("foo = { $bar }\n").unwrap(); + let mut ast = parse("foo = { $bar }\n").expect("failed to parse ftl resource"); ast.body[0].as_message().as_pattern().elements[0] .as_expression() .as_inline_variable_id() @@ -662,35 +662,35 @@ mod test { #[test] fn remove_message() { - let mut ast = parse("foo = bar\nbaz = qux\n").unwrap(); + let mut ast = parse("foo = bar\nbaz = qux\n").expect("failed to parse ftl resource"); ast.body.pop(); assert_eq!("foo = bar\n", serialize(&ast)); } #[test] fn add_message_at_top() { - let mut ast = parse("foo = bar\n").unwrap(); + let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); ast.body.insert(0, text_message!("baz", "qux")); assert_eq!("baz = qux\nfoo = bar\n", serialize(&ast)); } #[test] fn add_message_at_end() { - let mut ast = parse("foo = bar\n").unwrap(); + let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); ast.body.push(text_message!("baz", "qux")); assert_eq!("foo = bar\nbaz = qux\n", serialize(&ast)); } #[test] fn add_message_in_between() { - let mut ast = parse("foo = bar\nbaz = qux\n").unwrap(); + let mut ast = parse("foo = bar\nbaz = qux\n").expect("failed to parse ftl resource"); ast.body.insert(1, text_message!("hello", "there")); assert_eq!("foo = bar\nhello = there\nbaz = qux\n", serialize(&ast)); } #[test] fn add_message_comment() { - let mut ast = parse("foo = bar\n").unwrap(); + let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); ast.body[0].as_message().comment.replace(Comment { content: vec!["great message!"], }); @@ -699,15 +699,20 @@ mod test { #[test] fn remove_message_comment() { - let mut ast = parse("# great message!\nfoo = bar\n").unwrap(); + let mut ast = parse("# great message!\nfoo = bar\n").expect("failed to parse ftl resource"); ast.body[0].as_message().comment.take(); assert_eq!("foo = bar\n", serialize(&ast)); } #[test] fn edit_message_comment() { - let mut ast = parse("# great message!\nfoo = bar\n").unwrap(); - ast.body[0].as_message().comment.as_mut().unwrap().content[0] = "very original"; + let mut ast = parse("# great message!\nfoo = bar\n").expect("failed to parse ftl resource"); + ast.body[0] + .as_message() + .comment + .as_mut() + .expect("comment is missing") + .content[0] = "very original"; assert_eq!("# very original\nfoo = bar\n", serialize(&ast)); } } From 2e9c63f7fd038b307e125cbcbf8c1242da875f10 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Tue, 8 Nov 2022 17:54:55 +0100 Subject: [PATCH 23/27] Remove redundant error propogation --- fluent-syntax/src/serializer.rs | 255 ++++++++++++-------------------- 1 file changed, 96 insertions(+), 159 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 50b4dc37..0d384464 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -22,7 +22,7 @@ //! ``` use crate::{ast::*, parser::Slice}; -use std::fmt::{self, Error, Write}; +use std::fmt::Write; /// Serializes an abstract syntax tree representing a Fluent Translation List into a /// String. @@ -59,10 +59,7 @@ pub fn serialize_with_options<'s, S: Slice<'s>>( options: Options, ) -> String { let mut ser = Serializer::new(options); - - ser.serialize_resource(resource) - .expect("Writing to an in-memory buffer never fails"); - + ser.serialize_resource(resource); ser.into_serialized_text() } @@ -82,191 +79,156 @@ impl Serializer { } } - fn serialize_resource<'s, S: Slice<'s>>(&mut self, res: &Resource) -> Result<(), Error> { + fn serialize_resource<'s, S: Slice<'s>>(&mut self, res: &Resource) { for entry in &res.body { match entry { - Entry::Message(msg) => self.serialize_message(msg)?, - Entry::Term(term) => self.serialize_term(term)?, - Entry::Comment(comment) => self.serialize_free_comment(comment, "#")?, - Entry::GroupComment(comment) => self.serialize_free_comment(comment, "##")?, - Entry::ResourceComment(comment) => self.serialize_free_comment(comment, "###")?, + Entry::Message(msg) => self.serialize_message(msg), + Entry::Term(term) => self.serialize_term(term), + Entry::Comment(comment) => self.serialize_free_comment(comment, "#"), + Entry::GroupComment(comment) => self.serialize_free_comment(comment, "##"), + Entry::ResourceComment(comment) => self.serialize_free_comment(comment, "###"), Entry::Junk { content } => { if self.options.with_junk { - self.serialize_junk(content.as_ref())? + self.serialize_junk(content.as_ref()) } } }; self.state.wrote_non_junk_entry = !matches!(entry, Entry::Junk { .. }); } - - Ok(()) } fn into_serialized_text(self) -> String { self.writer.buffer } - fn serialize_junk(&mut self, junk: &str) -> Result<(), Error> { + fn serialize_junk(&mut self, junk: &str) { self.writer.write_literal(junk) } - fn serialize_free_comment<'s, S: Slice<'s>>( - &mut self, - comment: &Comment, - prefix: &str, - ) -> Result<(), Error> { + fn serialize_free_comment<'s, S: Slice<'s>>(&mut self, comment: &Comment, prefix: &str) { if self.state.wrote_non_junk_entry { self.writer.newline(); } - self.serialize_comment(comment, prefix)?; + self.serialize_comment(comment, prefix); self.writer.newline(); - - Ok(()) } - fn serialize_comment<'s, S: Slice<'s>>( - &mut self, - comment: &Comment, - prefix: &str, - ) -> Result<(), Error> { + fn serialize_comment<'s, S: Slice<'s>>(&mut self, comment: &Comment, prefix: &str) { for line in &comment.content { - self.writer.write_literal(prefix)?; + self.writer.write_literal(prefix); if !line.as_ref().trim().is_empty() { - self.writer.write_literal(" ")?; - self.writer.write_literal(line.as_ref())?; + self.writer.write_literal(" "); + self.writer.write_literal(line.as_ref()); } self.writer.newline(); } - - Ok(()) } - fn serialize_message<'s, S: Slice<'s>>(&mut self, msg: &Message) -> Result<(), Error> { + fn serialize_message<'s, S: Slice<'s>>(&mut self, msg: &Message) { if let Some(comment) = msg.comment.as_ref() { - self.serialize_comment(comment, "#")?; + self.serialize_comment(comment, "#"); } - self.writer.write_literal(msg.id.name.as_ref())?; - self.writer.write_literal(" =")?; + self.writer.write_literal(msg.id.name.as_ref()); + self.writer.write_literal(" ="); if let Some(value) = msg.value.as_ref() { - self.serialize_pattern(value)?; + self.serialize_pattern(value); } - self.serialize_attributes(&msg.attributes)?; + self.serialize_attributes(&msg.attributes); self.writer.newline(); - Ok(()) } - fn serialize_term<'s, S: Slice<'s>>(&mut self, term: &Term) -> Result<(), Error> { + fn serialize_term<'s, S: Slice<'s>>(&mut self, term: &Term) { if let Some(comment) = term.comment.as_ref() { - self.serialize_comment(comment, "#")?; + self.serialize_comment(comment, "#"); } - self.writer.write_literal("-")?; - self.writer.write_literal(term.id.name.as_ref())?; - self.writer.write_literal(" =")?; - self.serialize_pattern(&term.value)?; + self.writer.write_literal("-"); + self.writer.write_literal(term.id.name.as_ref()); + self.writer.write_literal(" ="); + self.serialize_pattern(&term.value); - self.serialize_attributes(&term.attributes)?; + self.serialize_attributes(&term.attributes); self.writer.newline(); - - Ok(()) } - fn serialize_pattern<'s, S: Slice<'s>>(&mut self, pattern: &Pattern) -> Result<(), Error> { + fn serialize_pattern<'s, S: Slice<'s>>(&mut self, pattern: &Pattern) { let start_on_newline = pattern.starts_on_new_line(); if start_on_newline { self.writer.newline(); self.writer.indent(); } else { - self.writer.write_literal(" ")?; + self.writer.write_literal(" "); } for element in &pattern.elements { - self.serialize_element(element)?; + self.serialize_element(element); } if start_on_newline { self.writer.dedent(); } - - Ok(()) } - fn serialize_attributes<'s, S: Slice<'s>>( - &mut self, - attrs: &[Attribute], - ) -> Result<(), Error> { + fn serialize_attributes<'s, S: Slice<'s>>(&mut self, attrs: &[Attribute]) { if attrs.is_empty() { - return Ok(()); + return; } self.writer.indent(); for attr in attrs { self.writer.newline(); - self.serialize_attribute(attr)?; + self.serialize_attribute(attr); } self.writer.dedent(); - - Ok(()) } - fn serialize_attribute<'s, S: Slice<'s>>(&mut self, attr: &Attribute) -> Result<(), Error> { - self.writer.write_literal(".")?; - self.writer.write_literal(attr.id.name.as_ref())?; - self.writer.write_literal(" =")?; - - self.serialize_pattern(&attr.value)?; + fn serialize_attribute<'s, S: Slice<'s>>(&mut self, attr: &Attribute) { + self.writer.write_literal("."); + self.writer.write_literal(attr.id.name.as_ref()); + self.writer.write_literal(" ="); - Ok(()) + self.serialize_pattern(&attr.value); } - fn serialize_element<'s, S: Slice<'s>>( - &mut self, - elem: &PatternElement, - ) -> Result<(), Error> { + fn serialize_element<'s, S: Slice<'s>>(&mut self, elem: &PatternElement) { match elem { PatternElement::TextElement { value } => self.writer.write_literal(value.as_ref()), PatternElement::Placeable { expression } => match expression { Expression::Inline(InlineExpression::Placeable { expression }) => { // A placeable inside a placeable is a special case because we // don't want the braces to look silly (e.g. "{ { Foo() } }"). - self.writer.write_literal("{{ ")?; - self.serialize_expression(expression)?; - self.writer.write_literal(" }}")?; - Ok(()) + self.writer.write_literal("{{ "); + self.serialize_expression(expression); + self.writer.write_literal(" }}"); } Expression::Select { .. } => { // select adds its own newline and indent, emit the brace // *without* a space so we don't get 5 spaces instead of 4 - self.writer.write_literal("{ ")?; - self.serialize_expression(expression)?; - self.writer.write_literal("}")?; - Ok(()) + self.writer.write_literal("{ "); + self.serialize_expression(expression); + self.writer.write_literal("}"); } Expression::Inline(_) => { - self.writer.write_literal("{ ")?; - self.serialize_expression(expression)?; - self.writer.write_literal(" }")?; - Ok(()) + self.writer.write_literal("{ "); + self.serialize_expression(expression); + self.writer.write_literal(" }"); } }, } } - fn serialize_expression<'s, S: Slice<'s>>( - &mut self, - expr: &Expression, - ) -> Result<(), Error> { + fn serialize_expression<'s, S: Slice<'s>>(&mut self, expr: &Expression) { match expr { Expression::Inline(inline) => self.serialize_inline_expression(inline), Expression::Select { selector, variants } => { @@ -275,65 +237,52 @@ impl Serializer { } } - fn serialize_inline_expression<'s, S: Slice<'s>>( - &mut self, - expr: &InlineExpression, - ) -> Result<(), Error> { + fn serialize_inline_expression<'s, S: Slice<'s>>(&mut self, expr: &InlineExpression) { match expr { InlineExpression::StringLiteral { value } => { - self.writer.write_literal("\"")?; - self.writer.write_literal(value.as_ref())?; - self.writer.write_literal("\"")?; - Ok(()) + self.writer.write_literal("\""); + self.writer.write_literal(value.as_ref()); + self.writer.write_literal("\""); } InlineExpression::NumberLiteral { value } => self.writer.write_literal(value.as_ref()), InlineExpression::VariableReference { id: Identifier { name: value }, } => { - self.writer.write_literal("$")?; - self.writer.write_literal(value.as_ref())?; - Ok(()) + self.writer.write_literal("$"); + self.writer.write_literal(value.as_ref()); } InlineExpression::FunctionReference { id, arguments } => { - self.writer.write_literal(id.name.as_ref())?; - self.serialize_call_arguments(arguments)?; - - Ok(()) + self.writer.write_literal(id.name.as_ref()); + self.serialize_call_arguments(arguments); } InlineExpression::MessageReference { id, attribute } => { - self.writer.write_literal(id.name.as_ref())?; + self.writer.write_literal(id.name.as_ref()); if let Some(attr) = attribute.as_ref() { - self.writer.write_literal(".")?; - self.writer.write_literal(attr.name.as_ref())?; + self.writer.write_literal("."); + self.writer.write_literal(attr.name.as_ref()); } - - Ok(()) } InlineExpression::TermReference { id, attribute, arguments, } => { - self.writer.write_literal("-")?; - self.writer.write_literal(id.name.as_ref())?; + self.writer.write_literal("-"); + self.writer.write_literal(id.name.as_ref()); if let Some(attr) = attribute.as_ref() { - self.writer.write_literal(".")?; - self.writer.write_literal(attr.name.as_ref())?; + self.writer.write_literal("."); + self.writer.write_literal(attr.name.as_ref()); } if let Some(args) = arguments.as_ref() { - self.serialize_call_arguments(args)?; + self.serialize_call_arguments(args); } - - Ok(()) } InlineExpression::Placeable { expression } => { - self.writer.write_literal("{")?; - self.serialize_expression(expression)?; - self.writer.write_literal("}")?; - - Ok(()) + self.writer.write_literal("{"); + self.serialize_expression(expression); + self.writer.write_literal("}"); } } } @@ -342,39 +291,33 @@ impl Serializer { &mut self, selector: &InlineExpression, variants: &[Variant], - ) -> Result<(), Error> { - self.serialize_inline_expression(selector)?; - self.writer.write_literal(" ->")?; + ) { + self.serialize_inline_expression(selector); + self.writer.write_literal(" ->"); self.writer.newline(); self.writer.indent(); for variant in variants { - self.serialize_variant(variant)?; + self.serialize_variant(variant); self.writer.newline(); } self.writer.dedent(); - Ok(()) } - fn serialize_variant<'s, S: Slice<'s>>(&mut self, variant: &Variant) -> Result<(), Error> { + fn serialize_variant<'s, S: Slice<'s>>(&mut self, variant: &Variant) { if variant.default { self.writer.write_char_into_indent('*'); } - self.writer.write_literal("[")?; - self.serialize_variant_key(&variant.key)?; - self.writer.write_literal("]")?; - self.serialize_pattern(&variant.value)?; - - Ok(()) + self.writer.write_literal("["); + self.serialize_variant_key(&variant.key); + self.writer.write_literal("]"); + self.serialize_pattern(&variant.value); } - fn serialize_variant_key<'s, S: Slice<'s>>( - &mut self, - key: &VariantKey, - ) -> Result<(), Error> { + fn serialize_variant_key<'s, S: Slice<'s>>(&mut self, key: &VariantKey) { match key { VariantKey::NumberLiteral { value } | VariantKey::Identifier { name: value } => { self.writer.write_literal(value.as_ref()) @@ -382,36 +325,32 @@ impl Serializer { } } - fn serialize_call_arguments<'s, S: Slice<'s>>( - &mut self, - args: &CallArguments, - ) -> Result<(), Error> { + fn serialize_call_arguments<'s, S: Slice<'s>>(&mut self, args: &CallArguments) { let mut argument_written = false; - self.writer.write_literal("(")?; + self.writer.write_literal("("); for positional in &args.positional { if argument_written { - self.writer.write_literal(", ")?; + self.writer.write_literal(", "); } - self.serialize_inline_expression(positional)?; + self.serialize_inline_expression(positional); argument_written = true; } for named in &args.named { if argument_written { - self.writer.write_literal(", ")?; + self.writer.write_literal(", "); } - self.writer.write_literal(named.name.name.as_ref())?; - self.writer.write_literal(": ")?; - self.serialize_inline_expression(&named.value)?; + self.writer.write_literal(named.name.name.as_ref()); + self.writer.write_literal(": "); + self.serialize_inline_expression(&named.value); argument_written = true; } - self.writer.write_literal(")")?; - Ok(()) + self.writer.write_literal(")"); } } @@ -491,13 +430,13 @@ impl TextWriter { self.buffer.push('\n'); } - fn write_literal(&mut self, item: &str) -> fmt::Result { + fn write_literal(&mut self, item: &str) { if self.buffer.ends_with('\n') { // we've just added a newline, make sure it's properly indented self.write_indent(); } - write!(self.buffer, "{}", item) + write!(self.buffer, "{}", item).expect("Writing to an in-memory buffer never fails"); } fn write_char_into_indent(&mut self, ch: char) { @@ -515,18 +454,18 @@ mod test { use crate::parser::parse; #[test] - fn write_something_then_indent() -> fmt::Result { + fn write_something_then_indent() { let mut writer = TextWriter::default(); - writer.write_literal("foo =")?; + writer.write_literal("foo ="); writer.newline(); writer.indent(); - writer.write_literal("first line")?; + writer.write_literal("first line"); writer.newline(); - writer.write_literal("second line")?; + writer.write_literal("second line"); writer.newline(); writer.dedent(); - writer.write_literal("not indented")?; + writer.write_literal("not indented"); writer.newline(); let got = &writer.buffer; @@ -534,8 +473,6 @@ mod test { got, "foo =\n first line\n second line\nnot indented\n" ); - - Ok(()) } macro_rules! text_message { From 4c23b5468de59d027515de790935278bb7314c2b Mon Sep 17 00:00:00 2001 From: RumovZ Date: Wed, 9 Nov 2022 21:43:45 +0100 Subject: [PATCH 24/27] BLACKLIST -> IGNORE_LIST --- fluent-syntax/tests/serializer_fixtures.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fluent-syntax/tests/serializer_fixtures.rs b/fluent-syntax/tests/serializer_fixtures.rs index e381ef61..4e8f532c 100644 --- a/fluent-syntax/tests/serializer_fixtures.rs +++ b/fluent-syntax/tests/serializer_fixtures.rs @@ -10,12 +10,12 @@ use fluent_syntax::serializer::{serialize, serialize_with_options, Options}; /// List of files that currently do not roundtrip correctly. /// /// - `multiline_values.ftl`: `key12` is parsed differently if indented. -const BLACKLIST: [&str; 1] = ["multiline_values.ftl"]; +const IGNORE_LIST: [&str; 1] = ["multiline_values.ftl"]; -fn is_blacklisted(path: &Path) -> bool { +fn is_ignored(path: &Path) -> bool { path.file_name() .and_then(OsStr::to_str) - .map(|s| BLACKLIST.contains(&s)) + .map(|s| IGNORE_LIST.contains(&s)) .unwrap_or_default() } @@ -48,7 +48,7 @@ fn roundtrip_normalized_fixtures() { fn roundtrip_unnormalized_fixtures() { for entry in glob("./tests/fixtures/*.ftl").expect("Failed to read glob pattern") { let path = entry.expect("Error while getting an entry"); - if is_blacklisted(&path) { + if is_ignored(&path) { continue; } From 6fa7576f315a73df62e6c89ad08d35956a1ad982 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Wed, 9 Nov 2022 21:45:59 +0100 Subject: [PATCH 25/27] Fix typo --- fluent-syntax/src/serializer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs index 0d384464..60b33ec6 100644 --- a/fluent-syntax/src/serializer.rs +++ b/fluent-syntax/src/serializer.rs @@ -16,9 +16,9 @@ //! //! let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); //! -//! let serialzed = serializer::serialize(&resource); +//! let serialized = serializer::serialize(&resource); //! -//! assert_eq!(ftl, serialzed); +//! assert_eq!(ftl, serialized); //! ``` use crate::{ast::*, parser::Slice}; @@ -39,14 +39,14 @@ use std::fmt::Write; /// /// let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); /// -/// let serialzed = serializer::serialize(&resource); +/// let serialized = serializer::serialize(&resource); /// /// let expected = r#"unnormalized-message = /// This message has /// abnormal spacing and indentation /// "#; /// -/// assert_eq!(expected, serialzed); +/// assert_eq!(expected, serialized); /// ``` pub fn serialize<'s, S: Slice<'s>>(resource: &Resource) -> String { serialize_with_options(resource, Options::default()) From c0e172234a2fe25d867aca10a6a5bf571992cc7b Mon Sep 17 00:00:00 2001 From: RumovZ Date: Wed, 9 Nov 2022 21:58:49 +0100 Subject: [PATCH 26/27] Mention serializer on changelog --- fluent-syntax/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-syntax/CHANGELOG.md b/fluent-syntax/CHANGELOG.md index 3db55457..38f6e9b3 100644 --- a/fluent-syntax/CHANGELOG.md +++ b/fluent-syntax/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## Unreleased - + - Add module `serializer`. - … ## fluent-syntax 0.11.0 (February 9, 2021) From c534eb70b67cbe5b2029953bf3ff441c394bb077 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Wed, 9 Nov 2022 23:41:11 +0100 Subject: [PATCH 27/27] Revert c009674 and add crlf.ftl to ignore list --- fluent-syntax/src/parser/pattern.rs | 4 ++-- fluent-syntax/tests/serializer_fixtures.rs | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fluent-syntax/src/parser/pattern.rs b/fluent-syntax/src/parser/pattern.rs index 9141d4f3..85c8925e 100644 --- a/fluent-syntax/src/parser/pattern.rs +++ b/fluent-syntax/src/parser/pattern.rs @@ -174,8 +174,8 @@ where b'\r' if self.is_byte_at(b'\n', self.ptr + 1) => { self.ptr += 1; return Ok(( - start_pos + 1, - self.ptr, + start_pos, + self.ptr - 1, text_element_type, TextElementTermination::Crlf, )); diff --git a/fluent-syntax/tests/serializer_fixtures.rs b/fluent-syntax/tests/serializer_fixtures.rs index 4e8f532c..85af4dc5 100644 --- a/fluent-syntax/tests/serializer_fixtures.rs +++ b/fluent-syntax/tests/serializer_fixtures.rs @@ -9,8 +9,11 @@ use fluent_syntax::serializer::{serialize, serialize_with_options, Options}; /// List of files that currently do not roundtrip correctly. /// -/// - `multiline_values.ftl`: `key12` is parsed differently if indented. -const IGNORE_LIST: [&str; 1] = ["multiline_values.ftl"]; +/// - `multiline_values.ftl`: https://github.com/projectfluent/fluent-rs/issues/286 +/// - `crlf.ftl`: Parsing `foo =\r\n bar\r\n baz\r\n` results in a TextElement "bar" and a TextElement "\n", +/// whereas parsing `foo =\n bar\n baz\n` results in a single TextElement "bar\n". That means resources with +/// text separated by CRLF do not roundtrip correctly. +const IGNORE_LIST: [&str; 2] = ["crlf.ftl", "multiline_values.ftl"]; fn is_ignored(path: &Path) -> bool { path.file_name()