Use visitor for terminal extraction

Use visitor for terminal extraction instead of complicated and pmap / matrix construct.

Author: lorenzwalthert

R/transform.R

@@ -85,7 +85,7 @@ parse_transform_serialize <- function(text, transformers) {
   pd_nested <- compute_parse_data_nested(text)
   transformed_pd <- apply_transformers(pd_nested, transformers)
   # TODO verify_roundtrip
-  flattened_pd <- extract_terminals(transformed_pd) %>%
+  flattened_pd <- post_visit(transformed_pd, list(extract_terminals)) %>%
     enrich_terminals()
 
   serialized_transformed_text <- serialize_parse_data_flattened(flattened_pd)

R/visit.R

@@ -99,55 +99,17 @@ context_towards_terminals <- function(pd_nested,
 
 #' Extract terminal tokens
 #'
-#' Turns a nested parse table into a flat parse table. In particular it extracts
-#'   terminal tokens and the following attributes:
-#'
-#'  * lag_newlines
-#'  * indent
-#'  * token
-#'  * text
-#'  * spaces
-#'  * id
-#'  * parent
-#'  * line1
-#' @inheritParams extract_terminals_helper
+#' Turns a nested parse table into a flat parse table and extracts *all*
+#' attributes
+#' @param pd_nested A nested parse table.
 #' @importFrom readr type_convert col_integer cols
 extract_terminals <- function(pd_nested) {
-  flat_vec <- extract_terminals_helper(pd_nested) %>%
-    unlist()
-  nms <- list(
-    NULL,
-    c("lag_newlines", "indent", "token", "text", "spaces", "id", "parent", "line1")
-  )
-  flat_tbl <- matrix(flat_vec, ncol = length(nms[[2]]), byrow = TRUE, dimnames = nms) %>%
-    as_tibble() %>%
-    type_convert(
-      col_types = cols(
-        lag_newlines = col_integer(),
-        indent       = col_integer(),
-        spaces       = col_integer()
-      )
-    )
-}
-
-#' Helper to extract terminals
-#'
-#' @param pd_nested A nested parse table.
-extract_terminals_helper <- function(pd_nested) {
   if (is.null(pd_nested)) return(pd)
-  pmap(list(pd_nested$terminal, pd_nested$token, pd_nested$text,
-            pd_nested$lag_newlines, pd_nested$spaces, pd_nested$indent,
-            pd_nested$id, pd_nested$parent, pd_nested$line1, pd_nested$child),
-       function(terminal, token, text, lag_newlines, spaces, indent, id,
-                parent, line1, child) {
-         if (terminal) {
-           c(lag_newlines, indent, token, text, spaces, id, parent, line1)
-         } else {
-           extract_terminals_helper(child)
-         }
-       })
+  pd_splitted <- split(pd_nested, seq_len(nrow(pd_nested)))
+  bind_rows(ifelse(pd_nested$terminal, pd_splitted, pd_nested$child))
 }
 
+
 #' Enrich flattened parse table
 #'
 #' Enriches a flattened parse table with terminals only. In particular, it is