diff --git a/fluent-syntax/CHANGELOG.md b/fluent-syntax/CHANGELOG.md index 3db55457..38f6e9b3 100644 --- a/fluent-syntax/CHANGELOG.md +++ b/fluent-syntax/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## Unreleased - + - Add module `serializer`. - … ## fluent-syntax 0.11.0 (February 9, 2021) diff --git a/fluent-syntax/src/lib.rs b/fluent-syntax/src/lib.rs index 5b9cbbfe..ab3e43ac 100644 --- a/fluent-syntax/src/lib.rs +++ b/fluent-syntax/src/lib.rs @@ -48,4 +48,5 @@ //! ``` pub mod ast; pub mod parser; +pub mod serializer; pub mod unicode; diff --git a/fluent-syntax/src/serializer.rs b/fluent-syntax/src/serializer.rs new file mode 100644 index 00000000..60b33ec6 --- /dev/null +++ b/fluent-syntax/src/serializer.rs @@ -0,0 +1,655 @@ +//! Fluent Translation List serialization utilities +//! +//! This modules provides a way to serialize an abstract syntax tree representing a +//! Fluent Translation List. Use cases include normalization and programmatical +//! manipulation of a Fluent Translation List. +//! +//! # Example +//! +//! ``` +//! use fluent_syntax::parser; +//! use fluent_syntax::serializer; +//! +//! let ftl = r#"# This is a message comment +//! hello-world = Hello World! +//! "#; +//! +//! let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); +//! +//! let serialized = serializer::serialize(&resource); +//! +//! assert_eq!(ftl, serialized); +//! ``` + +use crate::{ast::*, parser::Slice}; +use std::fmt::Write; + +/// Serializes an abstract syntax tree representing a Fluent Translation List into a +/// String. +/// +/// # Example +/// +/// ``` +/// use fluent_syntax::parser; +/// use fluent_syntax::serializer; +/// +/// let ftl = r#" +/// unnormalized-message=This message has +/// abnormal spacing and indentation"#; +/// +/// let resource = parser::parse(ftl).expect("Failed to parse an FTL resource."); +/// +/// let serialized = serializer::serialize(&resource); +/// +/// let expected = r#"unnormalized-message = +/// This message has +/// abnormal spacing and indentation +/// "#; +/// +/// assert_eq!(expected, serialized); +/// ``` +pub fn serialize<'s, S: Slice<'s>>(resource: &Resource) -> String { + serialize_with_options(resource, Options::default()) +} + +/// Serializes an abstract syntax tree representing a Fluent Translation List into a +/// String accepting custom options. +pub fn serialize_with_options<'s, S: Slice<'s>>( + resource: &Resource, + options: Options, +) -> String { + let mut ser = Serializer::new(options); + ser.serialize_resource(resource); + ser.into_serialized_text() +} + +#[derive(Debug)] +struct Serializer { + writer: TextWriter, + options: Options, + state: State, +} + +impl Serializer { + fn new(options: Options) -> Self { + Serializer { + writer: TextWriter::default(), + options, + state: State::default(), + } + } + + fn serialize_resource<'s, S: Slice<'s>>(&mut self, res: &Resource) { + for entry in &res.body { + match entry { + Entry::Message(msg) => self.serialize_message(msg), + Entry::Term(term) => self.serialize_term(term), + Entry::Comment(comment) => self.serialize_free_comment(comment, "#"), + Entry::GroupComment(comment) => self.serialize_free_comment(comment, "##"), + Entry::ResourceComment(comment) => self.serialize_free_comment(comment, "###"), + Entry::Junk { content } => { + if self.options.with_junk { + self.serialize_junk(content.as_ref()) + } + } + }; + + self.state.wrote_non_junk_entry = !matches!(entry, Entry::Junk { .. }); + } + } + + fn into_serialized_text(self) -> String { + self.writer.buffer + } + + fn serialize_junk(&mut self, junk: &str) { + self.writer.write_literal(junk) + } + + fn serialize_free_comment<'s, S: Slice<'s>>(&mut self, comment: &Comment, prefix: &str) { + if self.state.wrote_non_junk_entry { + self.writer.newline(); + } + self.serialize_comment(comment, prefix); + self.writer.newline(); + } + + fn serialize_comment<'s, S: Slice<'s>>(&mut self, comment: &Comment, prefix: &str) { + for line in &comment.content { + self.writer.write_literal(prefix); + + if !line.as_ref().trim().is_empty() { + self.writer.write_literal(" "); + self.writer.write_literal(line.as_ref()); + } + + self.writer.newline(); + } + } + + fn serialize_message<'s, S: Slice<'s>>(&mut self, msg: &Message) { + if let Some(comment) = msg.comment.as_ref() { + self.serialize_comment(comment, "#"); + } + + self.writer.write_literal(msg.id.name.as_ref()); + self.writer.write_literal(" ="); + + if let Some(value) = msg.value.as_ref() { + self.serialize_pattern(value); + } + + self.serialize_attributes(&msg.attributes); + + self.writer.newline(); + } + + fn serialize_term<'s, S: Slice<'s>>(&mut self, term: &Term) { + if let Some(comment) = term.comment.as_ref() { + self.serialize_comment(comment, "#"); + } + + self.writer.write_literal("-"); + self.writer.write_literal(term.id.name.as_ref()); + self.writer.write_literal(" ="); + self.serialize_pattern(&term.value); + + self.serialize_attributes(&term.attributes); + + self.writer.newline(); + } + + fn serialize_pattern<'s, S: Slice<'s>>(&mut self, pattern: &Pattern) { + let start_on_newline = pattern.starts_on_new_line(); + + if start_on_newline { + self.writer.newline(); + self.writer.indent(); + } else { + self.writer.write_literal(" "); + } + + for element in &pattern.elements { + self.serialize_element(element); + } + + if start_on_newline { + self.writer.dedent(); + } + } + + fn serialize_attributes<'s, S: Slice<'s>>(&mut self, attrs: &[Attribute]) { + if attrs.is_empty() { + return; + } + + self.writer.indent(); + + for attr in attrs { + self.writer.newline(); + self.serialize_attribute(attr); + } + + self.writer.dedent(); + } + + fn serialize_attribute<'s, S: Slice<'s>>(&mut self, attr: &Attribute) { + self.writer.write_literal("."); + self.writer.write_literal(attr.id.name.as_ref()); + self.writer.write_literal(" ="); + + self.serialize_pattern(&attr.value); + } + + fn serialize_element<'s, S: Slice<'s>>(&mut self, elem: &PatternElement) { + match elem { + PatternElement::TextElement { value } => self.writer.write_literal(value.as_ref()), + PatternElement::Placeable { expression } => match expression { + Expression::Inline(InlineExpression::Placeable { expression }) => { + // A placeable inside a placeable is a special case because we + // don't want the braces to look silly (e.g. "{ { Foo() } }"). + self.writer.write_literal("{{ "); + self.serialize_expression(expression); + self.writer.write_literal(" }}"); + } + Expression::Select { .. } => { + // select adds its own newline and indent, emit the brace + // *without* a space so we don't get 5 spaces instead of 4 + self.writer.write_literal("{ "); + self.serialize_expression(expression); + self.writer.write_literal("}"); + } + Expression::Inline(_) => { + self.writer.write_literal("{ "); + self.serialize_expression(expression); + self.writer.write_literal(" }"); + } + }, + } + } + + fn serialize_expression<'s, S: Slice<'s>>(&mut self, expr: &Expression) { + match expr { + Expression::Inline(inline) => self.serialize_inline_expression(inline), + Expression::Select { selector, variants } => { + self.serialize_select_expression(selector, variants) + } + } + } + + fn serialize_inline_expression<'s, S: Slice<'s>>(&mut self, expr: &InlineExpression) { + match expr { + InlineExpression::StringLiteral { value } => { + self.writer.write_literal("\""); + self.writer.write_literal(value.as_ref()); + self.writer.write_literal("\""); + } + InlineExpression::NumberLiteral { value } => self.writer.write_literal(value.as_ref()), + InlineExpression::VariableReference { + id: Identifier { name: value }, + } => { + self.writer.write_literal("$"); + self.writer.write_literal(value.as_ref()); + } + InlineExpression::FunctionReference { id, arguments } => { + self.writer.write_literal(id.name.as_ref()); + self.serialize_call_arguments(arguments); + } + InlineExpression::MessageReference { id, attribute } => { + self.writer.write_literal(id.name.as_ref()); + + if let Some(attr) = attribute.as_ref() { + self.writer.write_literal("."); + self.writer.write_literal(attr.name.as_ref()); + } + } + InlineExpression::TermReference { + id, + attribute, + arguments, + } => { + self.writer.write_literal("-"); + self.writer.write_literal(id.name.as_ref()); + + if let Some(attr) = attribute.as_ref() { + self.writer.write_literal("."); + self.writer.write_literal(attr.name.as_ref()); + } + if let Some(args) = arguments.as_ref() { + self.serialize_call_arguments(args); + } + } + InlineExpression::Placeable { expression } => { + self.writer.write_literal("{"); + self.serialize_expression(expression); + self.writer.write_literal("}"); + } + } + } + + fn serialize_select_expression<'s, S: Slice<'s>>( + &mut self, + selector: &InlineExpression, + variants: &[Variant], + ) { + self.serialize_inline_expression(selector); + self.writer.write_literal(" ->"); + + self.writer.newline(); + self.writer.indent(); + + for variant in variants { + self.serialize_variant(variant); + self.writer.newline(); + } + + self.writer.dedent(); + } + + fn serialize_variant<'s, S: Slice<'s>>(&mut self, variant: &Variant) { + if variant.default { + self.writer.write_char_into_indent('*'); + } + + self.writer.write_literal("["); + self.serialize_variant_key(&variant.key); + self.writer.write_literal("]"); + self.serialize_pattern(&variant.value); + } + + fn serialize_variant_key<'s, S: Slice<'s>>(&mut self, key: &VariantKey) { + match key { + VariantKey::NumberLiteral { value } | VariantKey::Identifier { name: value } => { + self.writer.write_literal(value.as_ref()) + } + } + } + + fn serialize_call_arguments<'s, S: Slice<'s>>(&mut self, args: &CallArguments) { + let mut argument_written = false; + + self.writer.write_literal("("); + + for positional in &args.positional { + if argument_written { + self.writer.write_literal(", "); + } + + self.serialize_inline_expression(positional); + argument_written = true; + } + + for named in &args.named { + if argument_written { + self.writer.write_literal(", "); + } + + self.writer.write_literal(named.name.name.as_ref()); + self.writer.write_literal(": "); + self.serialize_inline_expression(&named.value); + argument_written = true; + } + + self.writer.write_literal(")"); + } +} + +impl<'s, S: Slice<'s>> Pattern { + fn starts_on_new_line(&self) -> bool { + !self.has_leading_text_dot() && self.is_multiline() + } + + fn is_multiline(&self) -> bool { + self.elements.iter().any(|elem| match elem { + PatternElement::TextElement { value } => value.as_ref().contains('\n'), + PatternElement::Placeable { expression } => is_select_expr(expression), + }) + } + + fn has_leading_text_dot(&self) -> bool { + if let Some(PatternElement::TextElement { value }) = self.elements.get(0) { + value.as_ref().starts_with('.') + } else { + false + } + } +} + +fn is_select_expr<'s, S: Slice<'s>>(expr: &Expression) -> bool { + match expr { + Expression::Select { .. } => true, + Expression::Inline(InlineExpression::Placeable { expression }) => { + is_select_expr(expression) + } + Expression::Inline(_) => false, + } +} + +/// Options for serializing an abstract syntax tree. +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq)] +pub struct Options { + /// Whether invalid text fragments should be serialized, too. + pub with_junk: bool, +} + +#[derive(Debug, Default, PartialEq)] +struct State { + wrote_non_junk_entry: bool, +} + +#[derive(Debug, Clone, Default)] +struct TextWriter { + buffer: String, + indent_level: usize, +} + +impl TextWriter { + fn indent(&mut self) { + self.indent_level += 1; + } + + fn dedent(&mut self) { + self.indent_level = self + .indent_level + .checked_sub(1) + .expect("Dedenting without a corresponding indent"); + } + + fn write_indent(&mut self) { + for _ in 0..self.indent_level { + self.buffer.push_str(" "); + } + } + + fn newline(&mut self) { + if self.buffer.ends_with('\r') { + // handle rare edge case, where the trailing `\r` would get confused + // as part of the line ending + self.buffer.push('\r'); + } + self.buffer.push('\n'); + } + + fn write_literal(&mut self, item: &str) { + if self.buffer.ends_with('\n') { + // we've just added a newline, make sure it's properly indented + self.write_indent(); + } + + write!(self.buffer, "{}", item).expect("Writing to an in-memory buffer never fails"); + } + + fn write_char_into_indent(&mut self, ch: char) { + if self.buffer.ends_with('\n') { + self.write_indent(); + } + self.buffer.pop(); + self.buffer.push(ch); + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::parser::parse; + + #[test] + fn write_something_then_indent() { + let mut writer = TextWriter::default(); + + writer.write_literal("foo ="); + writer.newline(); + writer.indent(); + writer.write_literal("first line"); + writer.newline(); + writer.write_literal("second line"); + writer.newline(); + writer.dedent(); + writer.write_literal("not indented"); + writer.newline(); + + let got = &writer.buffer; + assert_eq!( + got, + "foo =\n first line\n second line\nnot indented\n" + ); + } + + macro_rules! text_message { + ($name:expr, $value:expr) => { + Entry::Message(Message { + id: Identifier { name: $name }, + value: Some(Pattern { + elements: vec![PatternElement::TextElement { value: $value }], + }), + attributes: vec![], + comment: None, + }) + }; + } + + impl<'a> Entry<&'a str> { + fn as_message(&mut self) -> &mut Message<&'a str> { + match self { + Self::Message(msg) => msg, + _ => panic!("Expected Message"), + } + } + } + + impl<'a> Message<&'a str> { + fn as_pattern(&mut self) -> &mut Pattern<&'a str> { + self.value.as_mut().expect("Expected Pattern") + } + } + + impl<'a> PatternElement<&'a str> { + fn as_text(&mut self) -> &mut &'a str { + match self { + Self::TextElement { value } => value, + _ => panic!("Expected TextElement"), + } + } + + fn as_expression(&mut self) -> &mut Expression<&'a str> { + match self { + Self::Placeable { expression } => expression, + _ => panic!("Expected Placeable"), + } + } + } + + impl<'a> Expression<&'a str> { + fn as_variants(&mut self) -> &mut Vec> { + match self { + Self::Select { variants, .. } => variants, + _ => panic!("Expected Select"), + } + } + fn as_inline_variable_id(&mut self) -> &mut Identifier<&'a str> { + match self { + Self::Inline(InlineExpression::VariableReference { id }) => id, + _ => panic!("Expected Inline"), + } + } + } + + #[test] + fn change_id() { + let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); + ast.body[0].as_message().id.name = "baz"; + assert_eq!(serialize(&ast), "baz = bar\n"); + } + + #[test] + fn change_value() { + let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); + *ast.body[0].as_message().as_pattern().elements[0].as_text() = "baz"; + assert_eq!("foo = baz\n", serialize(&ast)); + } + + #[test] + fn add_expression_variant() { + let message = concat!( + "foo =\n", + " { $num ->\n", + " *[other] { $num } bars\n", + " }\n" + ); + let mut ast = parse(message).expect("failed to parse ftl resource"); + + let one_variant = Variant { + key: VariantKey::Identifier { name: "one" }, + value: Pattern { + elements: vec![ + PatternElement::Placeable { + expression: Expression::Inline(InlineExpression::VariableReference { + id: Identifier { name: "num" }, + }), + }, + PatternElement::TextElement { value: " bar" }, + ], + }, + default: false, + }; + ast.body[0].as_message().as_pattern().elements[0] + .as_expression() + .as_variants() + .insert(0, one_variant); + + let expected = concat!( + "foo =\n", + " { $num ->\n", + " [one] { $num } bar\n", + " *[other] { $num } bars\n", + " }\n" + ); + assert_eq!(serialize(&ast), expected); + } + + #[test] + fn change_variable_reference() { + let mut ast = parse("foo = { $bar }\n").expect("failed to parse ftl resource"); + ast.body[0].as_message().as_pattern().elements[0] + .as_expression() + .as_inline_variable_id() + .name = "qux"; + assert_eq!("foo = { $qux }\n", serialize(&ast)); + } + + #[test] + fn remove_message() { + let mut ast = parse("foo = bar\nbaz = qux\n").expect("failed to parse ftl resource"); + ast.body.pop(); + assert_eq!("foo = bar\n", serialize(&ast)); + } + + #[test] + fn add_message_at_top() { + let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); + ast.body.insert(0, text_message!("baz", "qux")); + assert_eq!("baz = qux\nfoo = bar\n", serialize(&ast)); + } + + #[test] + fn add_message_at_end() { + let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); + ast.body.push(text_message!("baz", "qux")); + assert_eq!("foo = bar\nbaz = qux\n", serialize(&ast)); + } + + #[test] + fn add_message_in_between() { + let mut ast = parse("foo = bar\nbaz = qux\n").expect("failed to parse ftl resource"); + ast.body.insert(1, text_message!("hello", "there")); + assert_eq!("foo = bar\nhello = there\nbaz = qux\n", serialize(&ast)); + } + + #[test] + fn add_message_comment() { + let mut ast = parse("foo = bar\n").expect("failed to parse ftl resource"); + ast.body[0].as_message().comment.replace(Comment { + content: vec!["great message!"], + }); + assert_eq!("# great message!\nfoo = bar\n", serialize(&ast)); + } + + #[test] + fn remove_message_comment() { + let mut ast = parse("# great message!\nfoo = bar\n").expect("failed to parse ftl resource"); + ast.body[0].as_message().comment.take(); + assert_eq!("foo = bar\n", serialize(&ast)); + } + + #[test] + fn edit_message_comment() { + let mut ast = parse("# great message!\nfoo = bar\n").expect("failed to parse ftl resource"); + ast.body[0] + .as_message() + .comment + .as_mut() + .expect("comment is missing") + .content[0] = "very original"; + assert_eq!("# very original\nfoo = bar\n", serialize(&ast)); + } +} diff --git a/fluent-syntax/tests/fixtures/normalized/attributes.ftl b/fluent-syntax/tests/fixtures/normalized/attributes.ftl new file mode 100644 index 00000000..2140da61 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/attributes.ftl @@ -0,0 +1,17 @@ +attribute = + .attr = attribute +multiline-attribute = + .attr = + multiline + attribute +two-attributes = + .attr-a = attribute A + .attr-b = attribute B +value-and-attributes = value + .attr-a = attribute A + .attr-b = attribute B +multiline-value-and_attributes = + value A + value B + .attr-a = attribute A + .attr-b = attribute B diff --git a/fluent-syntax/tests/fixtures/normalized/call_expressions.ftl b/fluent-syntax/tests/fixtures/normalized/call_expressions.ftl new file mode 100644 index 00000000..6785f8cf --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/call_expressions.ftl @@ -0,0 +1,11 @@ +call-expression = { FOO() } +call-expression-with-string-expression = { FOO("bar") } +call-expression-with-number-expression = { FOO(1) } +call-expression-with-message-reference = { FOO(bar) } +call-expression-with-external-argument = { FOO($bar) } +call-expression-with-number-named-argument = { FOO(bar: 1) } +call-expression-with-string-named-argument = { FOO(bar: "bar") } +call-expression-with-two-positional-arguments = { FOO(bar, baz) } +call-expression-with-positional-and-named-arguments = { FOO(bar, 1, baz: "baz") } +macro-call = { -term() } +nested-placeables = {{ FOO() }} diff --git a/fluent-syntax/tests/fixtures/normalized/comments.ftl b/fluent-syntax/tests/fixtures/normalized/comments.ftl new file mode 100644 index 00000000..a42e2793 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/comments.ftl @@ -0,0 +1,13 @@ +# standalone comment + + +### multiline +### resource comment + + +## multiline +## group comment + +# multiline +# message comment +foo = bar diff --git a/fluent-syntax/tests/fixtures/normalized/escapes.ftl b/fluent-syntax/tests/fixtures/normalized/escapes.ftl new file mode 100644 index 00000000..c5433786 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/escapes.ftl @@ -0,0 +1,3 @@ +backslash_in_text_element = \{ placeable } +excaped_special_char_in_string_literal = { "Escaped \" quote" } +unicode_escape_sequence = { "\u0065" } diff --git a/fluent-syntax/tests/fixtures/normalized/inline_expressions.ftl b/fluent-syntax/tests/fixtures/normalized/inline_expressions.ftl new file mode 100644 index 00000000..8901f1a4 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/inline_expressions.ftl @@ -0,0 +1,6 @@ +simple-reference = simple { reference } +term-reference = term { -reference } +external-reference = external { $reference } +number-element = number { 1 } +string-element = string { "element" } +attribute-expression = attribute { ex.pression } diff --git a/fluent-syntax/tests/fixtures/normalized/messages.ftl b/fluent-syntax/tests/fixtures/normalized/messages.ftl new file mode 100644 index 00000000..9aba5cb1 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/messages.ftl @@ -0,0 +1,4 @@ +simple = simple +multiline = + multi + line diff --git a/fluent-syntax/tests/fixtures/normalized/select_expression.ftl b/fluent-syntax/tests/fixtures/normalized/select_expression.ftl new file mode 100644 index 00000000..24546f96 --- /dev/null +++ b/fluent-syntax/tests/fixtures/normalized/select_expression.ftl @@ -0,0 +1,50 @@ +select-expression = + { $sel -> + *[a] A + [b] B + } +multiline-variant = + { $sel -> + *[a] + AAA + BBBB + } +variant-key-number = + { $sel -> + *[a] A + [b] B + } +select-expression-in-block-value = + Foo { $sel -> + *[a] A + [b] B + } +select-expression-in-multiline-value = + Foo + Bar { $sel -> + *[a] A + [b] B + } +nested-select-expression = + { $a -> + *[a] + { $b -> + *[b] Foo + } + } +selector-external-argument = + { $bar -> + *[a] A + } +selector-number-expression = + { 1 -> + *[a] A + } +selector-string-expression = + { "bar" -> + *[a] A + } +selector-attribute-expression = + { -bar.baz -> + *[a] A + } diff --git a/fluent-syntax/tests/serializer_fixtures.rs b/fluent-syntax/tests/serializer_fixtures.rs new file mode 100644 index 00000000..85af4dc5 --- /dev/null +++ b/fluent-syntax/tests/serializer_fixtures.rs @@ -0,0 +1,71 @@ +use fluent_syntax::ast::{Entry, Resource}; +use glob::glob; +use std::ffi::OsStr; +use std::fs; +use std::path::Path; + +use fluent_syntax::parser::parse; +use fluent_syntax::serializer::{serialize, serialize_with_options, Options}; + +/// List of files that currently do not roundtrip correctly. +/// +/// - `multiline_values.ftl`: https://github.com/projectfluent/fluent-rs/issues/286 +/// - `crlf.ftl`: Parsing `foo =\r\n bar\r\n baz\r\n` results in a TextElement "bar" and a TextElement "\n", +/// whereas parsing `foo =\n bar\n baz\n` results in a single TextElement "bar\n". That means resources with +/// text separated by CRLF do not roundtrip correctly. +const IGNORE_LIST: [&str; 2] = ["crlf.ftl", "multiline_values.ftl"]; + +fn is_ignored(path: &Path) -> bool { + path.file_name() + .and_then(OsStr::to_str) + .map(|s| IGNORE_LIST.contains(&s)) + .unwrap_or_default() +} + +fn clone_without_junk<'a>(original: &Resource<&'a str>) -> Resource<&'a str> { + Resource { + body: original + .body + .iter() + .filter(|entry| !matches!(entry, Entry::Junk { .. })) + .cloned() + .collect(), + } +} + +#[test] +fn roundtrip_normalized_fixtures() { + for entry in glob("./tests/fixtures/normalized/*.ftl").expect("Failed to read glob pattern") { + let path = entry.expect("Error while getting an entry"); + let content = fs::read_to_string(&path).expect("Failed to read file"); + let parsed = parse(content.as_str()).unwrap_or_else(|(res, _)| res); + let reserialized = serialize(&parsed); + assert_eq!(content, reserialized); + } +} + +/// Compares a parsed AST with a parsed, serialized and reparsed AST, as these fixtures +/// contain unnormalized syntax that is not supposed to be preserved on a roundtrip. +/// Tests both parsing with and without junk. +#[test] +fn roundtrip_unnormalized_fixtures() { + for entry in glob("./tests/fixtures/*.ftl").expect("Failed to read glob pattern") { + let path = entry.expect("Error while getting an entry"); + if is_ignored(&path) { + continue; + } + + let content = fs::read_to_string(&path).expect("Failed to read file"); + let parsed = parse(content.as_str()).unwrap_or_else(|(res, _)| res); + let parsed_without_junk = clone_without_junk(&parsed); + let reserialized = serialize_with_options(&parsed, Options { with_junk: true }); + let reserialized_without_junk = + serialize_with_options(&parsed, Options { with_junk: false }); + let reparsed = parse(reserialized.as_str()).unwrap_or_else(|(res, _)| res); + let reparsed_without_junk = + parse(reserialized_without_junk.as_str()).unwrap_or_else(|(res, _)| res); + + assert_eq!(reparsed_without_junk, parsed_without_junk); + assert_eq!(reparsed, parsed); + } +}