New PR for --print option without printing --actual results

README.md

@@ -87,34 +87,46 @@ There are also some alternate test formats you'll read about below.
 
 ## Options
 
+<!--
+Command to generate doc:
+shelltest --help | sed -e '/^shelltest file formats/,$d' -e 's/^/    /'
+-->
+
     $ shelltest --help
     shelltest 1.9
 
     shelltest [OPTIONS] [TESTFILES|TESTDIRS]
 
     Common flags:
-      -l --list             List all parsed tests and stop
-      -a --all              Don't truncate output, even if large
-      -c --color            Show colored output if your terminal supports it
-      -d --diff             Show expected output mismatches in diff format
-      -p --precise          Show expected/actual output precisely (eg whitespace)
-      -h --hide-successes   Show only test failures
-         --xmlout=FILE      Specify file to store test results in xml format.
-      -D --defmacro=D=DEF   Specify a macro that is evaluated by preprocessor
-                            before the test files are parsed. D stands for macro
-                            definition that is replaced with the value of DEF.
+      -l --list             List the names of all tests found
       -i --include=PAT      Include tests whose name contains this glob pattern
+                            (eg: -i1 -i{4,5,6})
       -x --exclude=STR      Exclude test files whose path contains STR
-         --execdir          Run tests from within the test file's directory
+      -a --all              Show all output without truncating, even if large
+      -c --color            Show colored output if your terminal supports it
+      -d --diff             Show differences between expected/actual output
+         --precise          Show expected/actual output precisely, with quoting
+         --hide-successes   Show only test failures
+         --xmlout=FILE      Save test results to FILE in XML format.
+      -D --defmacro=D=DEF   Define a macro D to be replaced by DEF while parsing
+                            test files.
+         --execdir          Run tests from within each test file's directory
          --extension=EXT    File suffix of test files (default: .test)
-      -w --with=EXECUTABLE  Replace the first word of (unindented) test commands
+      -w --with=EXE         Replace the first word of test commands with EXE
+                            (unindented commands only)
       -o --timeout=SECS     Number of seconds a test may run (default: no limit)
       -j --threads=N        Number of threads for running tests (default: 1)
-         --debug            Show debug info, for troubleshooting
-         --debug-parse      Show test file parsing info and stop
-      -? --help             Display help message
+         --shell=EXE        The shell program to use (must accept -c CMD;
+                            default: /bin/sh on POSIX, cmd.exe on Windows)
+         --debug            Show debug info while running
+         --debug-parse      Show test file parsing results and stop
+    Print test file:
+         --print[=FORMAT]   Print test files in specified format (default: v3).
+
+      -h --help             Display help message
       -V --version          Print version information
          --numeric-version  Print just the version number
+
 
 `shelltest` accepts one or more test file or directory arguments.
 A directory means all files below it named `*.test` (customisable with `--extension`).
@@ -338,6 +350,41 @@ Non-required `<` and `>` delimiters omitted:
 
 [shelltestrunner](https://github.com/simonmichael/shelltestrunner/tree/master/tests/format3)
 
+## Printing tests
+
+The `--print` option prints tests to stdout.
+This can be used to convert between test formats.
+Format 1, 2, and 3 are supported.
+
+Caveats:
+
+- Comments before the first test in v2/v3 are currently not printed because of
+  the shared input feature which is not yet parsed.
+- Missing newline at EOF will not be preserved.
+- Most tests in tests/, when printed, are not identical because
+  - redundant/empty results
+  - `>=1` instead of `>= 1`
+  - comments before first test
+  - shared input
+  - no newline at EOF
+  - no tests at all, just comments
+  - no input/output marker `<<<`/`>>>`
+
+Planned feature: `--print-with-results` or something like this:
+
+     --actual[=MODE]    Combined with --print, print test files with actual
+                        results (stdout, stderr, exit status). Mode 'all'
+                        prints all actual results (default). Mode 'update'
+                        prints actual results only for non-matching results,
+                        i.e. regular expressions in tests are retained.
+
+which would allow this workflow to update test results:
+
+```
+shelltest --print --actual=update example.test > tmp
+vim -d example.test tmp  # use a diff tool to update test file
+```
+
 ## Support/Contribute
 
 |||

shelltestrunner.cabal

@@ -56,6 +56,7 @@ executable shelltest
     Import
     Parse
     Preprocessor
+    Print
     Types
     Utils
     Utils.Debug

src/Parse.hs

@@ -14,7 +14,7 @@ import Preprocessor
 import System.IO hiding (stdin)
 
 
-parseFromFileWithPreprocessor :: (Parser [ShellTest]) -> PreProcessor -> FilePath -> IO (Either ParseError [ShellTest])
+parseFromFileWithPreprocessor :: Parser [ShellTest] -> PreProcessor -> FilePath -> IO (Either ParseError [ShellTest])
 parseFromFileWithPreprocessor p preproc fname =
     do
         h <- openFile fname ReadMode
@@ -69,11 +69,12 @@ shelltestfile = do
         <|>
         (many $ try format1test)
   ptrace_ "shelltestfile 1"
-  skipMany whitespaceorcommentline
+  trailingComments <- many whitespaceorcommentline
   ptrace_ "shelltestfile 2"
   eof
-  ptrace "shelltestfile ." ts
-  return ts
+  let ts' = appendTrailingComments trailingComments ts
+  ptrace "shelltestfile ." ts'
+  return ts'
 
 
 ----------------------------------------------------------------------
@@ -82,7 +83,7 @@ shelltestfile = do
 
 format1test = do
   ptrace_ " format1test 0"
-  skipMany whitespaceorcommentline
+  comments <- many whitespaceorcommentline
   ptrace_ " format1test 1"
   ln <- sourceLine <$> getPosition
   c <- command1 <?> "command line"
@@ -97,7 +98,7 @@ format1test = do
   ptrace " format1test x" x
   when (null (show c) && (isNothing i) && (null $ catMaybes [o,e]) && null (show x)) $ fail ""
   f <- sourceName . statePos <$> getParserState
-  let t = ShellTest{testname=f,command=c,stdin=i,stdoutExpected=o,stderrExpected=e,exitCodeExpected=x,lineNumber=ln}
+  let t = ShellTest{testname=f,command=c,stdin=i,stdoutExpected=o,stderrExpected=e,exitCodeExpected=x,lineNumber=ln,comments=comments,trailingComments=[]}
   ptrace " format1test ." t
   return t
 
@@ -151,7 +152,7 @@ format2testgroup inputRequiresDelimiter = do
 format2test :: Maybe String -> Parser ShellTest
 format2test i = do
   ptrace_ "  format2test 0"
-  skipMany whitespaceorcommentline
+  comments <- many whitespaceorcommentline
   ptrace_ "  format2test 1"
   ln <- sourceLine <$> getPosition
   c <- command2 <?> "command line"
@@ -166,7 +167,7 @@ format2test i = do
   ptrace "  format2test x" x
   when (null (show c) && (isNothing i) && (null $ catMaybes [o,e]) && null (show x)) $ fail ""
   f <- sourceName . statePos <$> getParserState
-  let t = ShellTest{testname=f,command=c,stdin=i,stdoutExpected=o,stderrExpected=e,exitCodeExpected=x,lineNumber=ln}
+  let t = ShellTest{testname=f,command=c,stdin=i,stdoutExpected=o,stderrExpected=e,exitCodeExpected=x,lineNumber=ln,comments=comments,trailingComments=[]}
   ptrace "  format2test ." t
   return t
 
@@ -249,7 +250,7 @@ format3testgroup inputRequiresDelimiter = do
 format3test :: Maybe String -> Parser ShellTest
 format3test i = do
   ptrace_ "  format3test 0"
-  skipMany whitespaceorcommentline
+  comments <- many whitespaceorcommentline
   ptrace_ "  format3test 1"
   ln <- sourceLine <$> getPosition
   c <- command3 <?> "command line"
@@ -264,7 +265,7 @@ format3test i = do
   ptrace "  format3test x" x
   when (null (show c) && (isNothing i) && (null $ catMaybes [o,e]) && null (show x)) $ fail ""
   f <- sourceName . statePos <$> getParserState
-  let t = ShellTest{testname=f,command=c,stdin=i,stdoutExpected=o,stderrExpected=e,exitCodeExpected=x,lineNumber=ln}
+  let t = ShellTest{testname=f,command=c,stdin=i,stdoutExpected=o,stderrExpected=e,exitCodeExpected=x,lineNumber=ln,comments=comments,trailingComments=[]}
   ptrace "  format3test ." t
   return t
 
@@ -327,6 +328,11 @@ delimiterNotNewTest3 = do
 ----------------------------------------------------------------------
 -- common
 
+appendTrailingComments :: [String] -> [ShellTest] -> [ShellTest]
+appendTrailingComments _ [] = [] -- in this case, trailing comment are discarded
+appendTrailingComments cs ts =
+      init ts ++ [(last ts) { trailingComments = cs }]
+
 linesBetween :: [String] -> [String] -> Parser String
 linesBetween startdelims enddelims = do
   let delimp "" = string ""
@@ -390,8 +396,14 @@ line = (anyChar `manyTill` newline) <?> "rest of line"
 whitespacechar = oneOf " \t"
 whitespace = many whitespacechar
 whitespaceline = try (newline >> return "") <|> try (whitespacechar >> whitespacechar `manyTill` newlineoreof)
+
 -- a line beginning with optional whitespace and #, or beginning with one or more * (an org node)
-commentline = try ((many1 (char '*') <|> (whitespace >> many1 (char '#'))) >> lineoreof) <?> "comments"
+commentline = try (do
+  prefix <- many1 (char '*') <|> (whitespace >> many1 (char '#'))
+  rest <- lineoreof
+  return $ prefix ++ rest
+  ) <?> "comments"
+
 whitespaceorcommentline = commentline <|> whitespaceline
 whitespaceorcommentlineoreof = choice [eofasstr, commentline, whitespaceline]
 eofasstr = eof >> return ""

src/Print.hs

@@ -0,0 +1,67 @@
+module Print
+where
+
+import Import
+import Types
+
+-- | Print a shell test. See CLI documentation for details.
+printShellTest
+  :: String               -- ^ Shelltest format. Value of option @--print[=FORMAT]@.
+  -> ShellTest            -- ^ Test to print
+  -> IO ()
+printShellTest format ShellTest{command=c,stdin=i,comments=comments,trailingComments=trailingComments,
+               stdoutExpected=o_expected,stderrExpected=e_expected,exitCodeExpected=x_expected}
+               = do
+          case format of
+            "v1" -> do
+              printComments comments
+              printCommand "" c
+              printStdin "<<<" i
+              printStdouterr ">>>" o_expected
+              printStdouterr ">>>2" e_expected
+              printExitStatus True ">>>=" x_expected
+              printComments trailingComments
+            "v2" -> do
+              printComments comments
+              printCommand "$$$ " c
+              printStdin "<<<" i
+              printStdouterr ">>>" o_expected
+              printStdouterr ">>>2" e_expected
+              printExitStatus False ">>>=" x_expected
+              printComments trailingComments
+            "v3" -> do
+              printComments comments
+              printCommand "$ "  c
+              printStdin "<" i
+              printStdouterr ">" o_expected
+              printStdouterr ">2" e_expected
+              printExitStatus False ">=" x_expected
+              printComments trailingComments
+            _ -> fail $ "Unsupported --print format: " ++ format
+
+printComments :: [String] -> IO ()
+printComments = mapM_ putStrLn
+
+printStdin :: String -> Maybe String -> IO ()
+printStdin _ (Just "") = return ()
+printStdin _ Nothing = return ()
+printStdin prefix (Just s) = printf "%s\n%s" prefix s
+
+printCommand :: String -> TestCommand -> IO ()
+printCommand prefix (ReplaceableCommand s) = printf "%s%s\n" prefix s
+printCommand prefix (FixedCommand s)       = printf "%s %s\n" prefix s
+
+printStdouterr :: String -> Maybe Matcher -> IO ()
+printStdouterr _ Nothing                    = return ()
+printStdouterr _ (Just (Lines _ ""))        = return ()
+printStdouterr _ (Just (Numeric _))         = fail "FATAL: Cannot handle Matcher (Numeric) for stdout/stderr."
+printStdouterr _ (Just (NegativeNumeric _)) = fail "FATAL: Cannot handle Matcher (NegativeNumeric) for stdout/stderr."
+printStdouterr prefix (Just (Lines _ s))    = printf "%s\n%s" prefix s
+printStdouterr prefix (Just regex)          = printf "%s %s\n" prefix (show regex)
+
+-- | Print exit status. First arg says 'alwaysPrintEvenIfZero'.
+printExitStatus :: Bool -> String -> Matcher -> IO ()
+printExitStatus _ _ (Lines _ _) = fail "FATAL: Cannot handle Matcher (Lines) for exit status."
+printExitStatus False _     (Numeric "0") = return ()
+printExitStatus True prefix (Numeric "0") = printf "%s 0\n" prefix
+printExitStatus _ prefix s = printf "%s %s\n" prefix (show s)

src/Types.hs

@@ -4,9 +4,11 @@ where
 import Import
 import Utils
 import Text.Parsec
-  
+
 data ShellTest = ShellTest {
-     command          :: TestCommand
+     comments         :: [String] -- # COMMENTS OR BLANK LINES before test
+    ,trailingComments :: [String] -- # COMMENTS OR BLANK LINES after the last test
+    ,command          :: TestCommand
     ,stdin            :: Maybe String
     ,stdoutExpected   :: Maybe Matcher
     ,stderrExpected   :: Maybe Matcher
@@ -43,8 +45,8 @@ instance Show Matcher where show = showMatcherTrimmed
 showMatcherTrimmed :: Matcher -> String
 showMatcherTrimmed (PositiveRegex r)   = "/"++(trim r)++"/"
 showMatcherTrimmed (NegativeRegex r)   = "!/"++(trim r)++"/"
-showMatcherTrimmed (Numeric s)         = trim (show s)
-showMatcherTrimmed (NegativeNumeric s) = "!"++ trim (show s)
+showMatcherTrimmed (Numeric s)         = trim s
+showMatcherTrimmed (NegativeNumeric s) = "!"++ trim s
 showMatcherTrimmed (Lines _ s)         = trim s
 
 showMatcher :: Matcher -> String

src/Utils.hs

@@ -66,3 +66,6 @@ replace old new = replace'
                           else h : replace' ts
      len = length old
 
+-- | Show a message, usage string, and terminate with exit status 1.
+warn :: String -> IO ()
+warn s = putStrLn s >> exitWith (ExitFailure 1)