[llvm-rc] Handle C preprocessor output

When preprocessing resource scripts (which can easily be done outside
of llvm-rc), included headers can leave behind C declarations (despite
preprocessing with -DRC_INVOKED), that can't be parsed by a resource
compiler.

This is handled in all of rc.exe, by parsing the preprocessor output
line markers and ignoring content from files named *.h and *.c,
documented at [1].

In addition to this filtering, strip out any other preprocessor directive
that is left behind (like pragmas) which also can't be handled by the
tokenizer.

The added test uses both standard #line markers (supported by rc.exe) and
GNU style extended line markers, thus this test doesn't pass with rc.exe,
but passes with GNU windres. (Windres on the other hand doesn't filter
out files named *.c, only *.h.)

Differential Revision: https://reviews.llvm.org/D46579

[1] https://msdn.microsoft.com/en-us/library/windows/desktop/aa381033(v=vs.85).aspx

llvm-svn: 331903

Author: mstorsjo

llvm/test/tools/llvm-rc/Inputs/cpp-output.rc

@@ -0,0 +1,28 @@
+// Test that the input originally is included.
+STRINGTABLE {
+  1 "a"
+}
+#line 2 "cpp-source.rc"
+// Content from a rc file (potentially the source file itself) is included.
+STRINGTABLE {
+  2 "b"
+}
+// Test a preprocessing directive that starts with leading whitespace.
+  #line 1 "\\some\\path\\header.h"
+// Content from .h files is ignored.
+typedef int Foo;
+#line 123 "\\some\\path\\header.h"
+void someFunc(void);
+// Check GNU style line markers.
+# 4 "cpp-source.rc" 1
+STRINGTABLE {
+  3 "c"
+}
+	# 1 "other/header.h" 1
+typedef int Bar;
+# 10 "cpp-source.rc" 2
+// Test that other preprocessor directives are ignored.
+#pragma foo
+STRINGTABLE {
+  4 "d"
+}

llvm/test/tools/llvm-rc/cpp-output.test

@@ -0,0 +1,17 @@
+; RUN: llvm-rc /FO %t %p/Inputs/cpp-output.rc
+; RUN: llvm-readobj %t | FileCheck %s
+
+; CHECK:      Resource type (int): 6
+; CHECK-NEXT: Resource name (int): 1
+; CHECK-NEXT: Data version: 0
+; CHECK-NEXT: Memory flags: 0x1030
+; CHECK-NEXT: Language ID: 1033
+; CHECK-NEXT: Version (major): 0
+; CHECK-NEXT: Version (minor): 0
+; CHECK-NEXT: Characteristics: 0
+; CHECK-NEXT: Data size: 40
+; CHECK-NEXT: Data: (
+; CHECK-NEXT:   0000: 00000100 61000100 62000100 63000100  |....a...b...c...|
+; CHECK-NEXT:   0010: 64000000 00000000 00000000 00000000  |d...............|
+; CHECK-NEXT:   0020: 00000000 00000000                    |........|
+; CHECK-NEXT: )

llvm/tools/llvm-rc/CMakeLists.txt

@@ -11,6 +11,7 @@ add_public_tablegen_target(RcTableGen)
 add_llvm_tool(llvm-rc
   llvm-rc.cpp
   ResourceFileWriter.cpp
+  ResourceScriptCppFilter.cpp
   ResourceScriptParser.cpp
   ResourceScriptStmt.cpp
   ResourceScriptToken.cpp

llvm/tools/llvm-rc/ResourceScriptCppFilter.cpp

@@ -0,0 +1,112 @@
+//===-- ResourceScriptCppFilter.cpp ----------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file implements an interface defined in ResourceScriptCppFilter.h.
+//
+//===---------------------------------------------------------------------===//
+
+#include "ResourceScriptCppFilter.h"
+#include "llvm/ADT/StringExtras.h"
+
+#include <vector>
+
+using namespace llvm;
+
+namespace {
+
+class Filter {
+public:
+  explicit Filter(StringRef Input) : Data(Input), DataLength(Input.size()) {}
+
+  std::string run();
+
+private:
+  // Parse the line, returning whether the line should be included in
+  // the output.
+  bool parseLine(StringRef Line);
+
+  bool streamEof() const;
+
+  StringRef Data;
+  size_t DataLength;
+
+  size_t Pos = 0;
+  bool Outputting = true;
+};
+
+std::string Filter::run() {
+  std::vector<StringRef> Output;
+
+  while (!streamEof() && Pos != StringRef::npos) {
+    size_t LineStart = Pos;
+    Pos = Data.find_first_of("\r\n", Pos);
+    Pos = Data.find_first_not_of("\r\n", Pos);
+    StringRef Line = Data.take_front(Pos).drop_front(LineStart);
+
+    if (parseLine(Line))
+      Output.push_back(Line);
+  }
+
+  return llvm::join(Output, "");
+}
+
+bool Filter::parseLine(StringRef Line) {
+  Line = Line.ltrim();
+
+  if (!Line.consume_front("#")) {
+    // A normal content line, filtered according to the current mode.
+    return Outputting;
+  }
+
+  // Found a preprocessing directive line. From here on, we always return
+  // false since the preprocessing directives should be filtered out.
+
+  Line.consume_front("line");
+  if (!Line.startswith(" "))
+    return false; // Not a line directive (pragma etc).
+
+  // #line 123 "path/file.h"
+  // # 123 "path/file.h" 1
+
+  Line =
+      Line.ltrim(); // There could be multiple spaces after the #line directive
+
+  size_t N;
+  if (Line.consumeInteger(10, N)) // Returns true to signify an error
+    return false;
+
+  Line = Line.ltrim();
+
+  if (!Line.consume_front("\""))
+    return false; // Malformed line, no quote found.
+
+  // Split the string at the last quote (in case the path name had
+  // escaped quotes as well).
+  Line = Line.rsplit('"').first;
+
+  StringRef Ext = Line.rsplit('.').second;
+
+  if (Ext.equals_lower("h") || Ext.equals_lower("c")) {
+    Outputting = false;
+  } else {
+    Outputting = true;
+  }
+
+  return false;
+}
+
+bool Filter::streamEof() const { return Pos == DataLength; }
+
+} // anonymous namespace
+
+namespace llvm {
+
+std::string filterCppOutput(StringRef Input) { return Filter(Input).run(); }
+
+} // namespace llvm

llvm/tools/llvm-rc/ResourceScriptCppFilter.h

@@ -0,0 +1,35 @@
+//===-- ResourceScriptCppFilter.h ------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This filters the input to llvm-rc for preprocessor markers, removing
+// preprocessing directives that a preprocessor can output or leave behind.
+//
+// It also filters out any contribution from files named *.h or *.c, based
+// on preprocessor line markers. When preprocessing RC files, the included
+// headers can leave behind C declarations, that RC doesn't understand.
+// Rc.exe simply discards anything that comes from files named *.h or *.h.
+//
+// https://msdn.microsoft.com/en-us/library/windows/desktop/aa381033(v=vs.85).aspx
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVMRC_RESOURCESCRIPTCPPFILTER_H
+#define LLVM_TOOLS_LLVMRC_RESOURCESCRIPTCPPFILTER_H
+
+#include "llvm/ADT/StringRef.h"
+
+#include <string>
+
+namespace llvm {
+
+std::string filterCppOutput(StringRef Input);
+
+} // namespace llvm
+
+#endif

llvm/tools/llvm-rc/llvm-rc.cpp

@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ResourceFileWriter.h"
+#include "ResourceScriptCppFilter.h"
 #include "ResourceScriptParser.h"
 #include "ResourceScriptStmt.h"
 #include "ResourceScriptToken.h"
@@ -111,7 +112,8 @@ int main(int Argc, const char **Argv) {
   std::unique_ptr<MemoryBuffer> FileContents = std::move(*File);
   StringRef Contents = FileContents->getBuffer();
 
-  std::vector<RCToken> Tokens = ExitOnErr(tokenizeRC(Contents));
+  std::string FilteredContents = filterCppOutput(Contents);
+  std::vector<RCToken> Tokens = ExitOnErr(tokenizeRC(FilteredContents));
 
   if (BeVerbose) {
     const Twine TokenNames[] = {