diff --git a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPMacro.java b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPMacro.java index 01315fc6d9..829990bde1 100644 --- a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPMacro.java +++ b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPMacro.java @@ -125,6 +125,16 @@ boolean isFunctionLikeMacro() { return parameterList != null; } + ArrayList getParameterNames() { + var parameterNames = new ArrayList(parameterList.size()); + if (isFunctionLikeMacro()) { + for (int i = 0; i < parameterList.size(); i++) { + parameterNames.add(parameterList.get(i).getValue()); + } + } + return parameterNames; + } + int getParameterIndex(String parameterName) { if (isFunctionLikeMacro()) { for (int i = 0; i < parameterList.size(); i++) { diff --git a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPReplace.java b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPReplace.java index a50a478985..9e12f6c83b 100644 --- a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPReplace.java +++ b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/PPReplace.java @@ -26,6 +26,7 @@ import java.util.List; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; +import org.sonar.cxx.parser.CxxKeyword; import org.sonar.cxx.parser.CxxTokenType; /** @@ -48,12 +49,11 @@ class PPReplace { * directive behaves exactly like that. */ List replaceObjectLikeMacro(PPMacro macro, String macroExpression) { - // C++ standard 16.3.4/2 Macro Replacement - Rescanning and further replacement + // C++ standard 16.3.4/2 Macro Replacement - Rescanning and further argument List tokens = pp.tokenizeMacro(macro, macroExpression); - // make sure that all expanded Tokens are marked as generated it will prevent them from being involved into - // NCLOC / complexity / highlighting - // TODO: mark tokens already in macro expansion + // make sure that all expanded Tokens are marked as generated it will prevent + // them from being involved into NCLOC / complexity / highlighting return PPGeneratedToken.markAllAsGenerated(tokens); } @@ -147,108 +147,105 @@ private static int matchArguments(List tokens, List arguments) { return 0; } - private List replaceExpresson(String macroExpression) { - List tokens = pp.tokenize(macroExpression); - return PPGeneratedToken.markAllAsGenerated(tokens); + private String expand(String replacementString) { + return TokenUtils.merge(pp.tokenize(replacementString)); } - private List replaceParams(PPMacro macro, List arguments) { - // replace all parameters by according arguments "Stringify" the argument if the according parameter is + private List replaceParams(PPMacro macro, List argumentList) { + // replace all parameterList by according argumentList "Stringify" the argument if the according parameter is // preceded by an # - var newTokens = new ArrayList(); - var body = macro.replacementList; - var parameters = macro.parameterList; - if (!body.isEmpty()) { - var tokenPastingLeftOp = false; - var tokenPastingRightOp = false; + var result = new ArrayList(macro.replacementList.size()); + replaceParams(macro.replacementList, macro.getParameterNames(), argumentList, result); + return result; + } - for (var i = 0; i < body.size(); ++i) { - var curr = body.get(i); - int index = -1; - if (curr.getType().equals(GenericTokenType.IDENTIFIER)) { - index = macro.getParameterIndex(curr.getValue()); - } - if (index == -1) { - if (curr.getValue().equals("__VA_OPT__")) { - boolean keep = parameters.size() == arguments.size(); - replaceVaOpt(body.subList(i, body.size()), keep); - } else { - if (tokenPastingRightOp && !curr.getType().equals(PPPunctuator.HASHHASH)) { - tokenPastingRightOp = false; - } - newTokens.add(curr); - } - } else if (index == arguments.size()) { - // EXTENSION: GCC's special meaning of token paste operator: - // If variable argument is left out then the comma before the paste operator will be deleted. - int j = i; - if (j > 0 && "##".equals(body.get(j - 1).getValue()) && ",".equals(body.get(j - 2).getValue())) { - newTokens.subList(newTokens.size() - 2, newTokens.size()).clear(); // remove , ## - } else if (j > 0 && ",".equals(body.get(j - 1).getValue())) { - // Got empty variadic args, remove comma - newTokens.remove(newTokens.size() - 1 + j - i); - } - } else if (index < arguments.size()) { - // token pasting operator? - int j = i + 1; - if (j < body.size() && "##".equals(body.get(j).getValue())) { - tokenPastingLeftOp = true; - } - // in case of token pasting operator do not fully expand - var replacement = arguments.get(index); - String newValue; - if (tokenPastingLeftOp) { - newValue = replacement.getValue(); - tokenPastingLeftOp = false; - tokenPastingRightOp = true; - } else if (tokenPastingRightOp) { - newValue = replacement.getValue(); - tokenPastingLeftOp = false; + private void replaceParams(List replacementList, List parameterList, List argumentList, + List result) { + // replace all parameterList by according argumentList "Stringify" the argument if the according parameter is + // preceded by an # + + var tokenPastingLeftOp = false; + var tokenPastingRightOp = false; + + for (var i = 0; i < replacementList.size(); ++i) { + var token = replacementList.get(i); + var tokenValue = token.getValue(); + var tokenType = token.getType(); + String newValue = ""; + + int parameterIndex = -1; + if (GenericTokenType.IDENTIFIER.equals(tokenType) || (tokenType instanceof CxxKeyword)) { + parameterIndex = parameterList.indexOf(tokenValue); + } + + if (parameterIndex == -1) { + if ("__VA_OPT__".equals(tokenValue)) { + boolean keep = parameterList.size() == argumentList.size(); + i += replaceVaOpt(replacementList.subList(i, replacementList.size()), parameterList, argumentList, keep, + result); + } else { + if (tokenPastingRightOp && !PPPunctuator.HASHHASH.equals(tokenType)) { tokenPastingRightOp = false; - } else { - if (i > 0 && "#".equals(body.get(i - 1).getValue())) { - // In function-like macros, a # operator before an identifier in the replacement-list runs the identifier - // through parameter replacement and encloses the result in quotes, effectively creating a string literal. - newTokens.remove(newTokens.size() - 1); - newValue = PPStringification.stringify(replacement.getValue()); - } else { - // otherwise the arguments have to be fully expanded before expanding the replacementList of the macro - newValue = TokenUtils.merge(replaceExpresson(replacement.getValue())); - } } - - if (newValue.isEmpty() && "__VA_ARGS__".equals(curr.getValue())) { - // the Visual C++ implementation will suppress a trailing comma if no arguments are passed to the ellipsis - for (var n = newTokens.size() - 1; n != 0; n = newTokens.size() - 1) { - if (newTokens.get(n).getType().equals(PPPunctuator.COMMA)) { - newTokens.remove(n); - break; - } else { - break; - } - } + result.add(token); + } + } else if (parameterIndex == argumentList.size()) { + // EXTENSION: GCC's special meaning of token paste operator: + // If variable argument is left out then the comma before the paste operator will be deleted. + if (i > 0 + && PPPunctuator.HASHHASH.equals(replacementList.get(i - 1).getType()) + && PPPunctuator.COMMA.equals(replacementList.get(i - 2).getType())) { + result.subList(result.size() - 2, result.size()).clear(); // remove , ## + } else if (i > 0 && ",".equals(replacementList.get(i - 1).getValue())) { + // Got empty variadic args, remove comma + result.remove(result.size() - 1); + } + } else if (parameterIndex < argumentList.size()) { + // token pasting operator? + int j = i + 1; + if (j < replacementList.size() && PPPunctuator.HASHHASH.equals(replacementList.get(j).getType())) { + tokenPastingLeftOp = true; + } + // in case of token pasting operator do not fully expand + var argument = argumentList.get(parameterIndex); + newValue = argument.getValue(); + if (tokenPastingLeftOp) { + tokenPastingLeftOp = false; + tokenPastingRightOp = true; + } else if (tokenPastingRightOp) { + tokenPastingLeftOp = false; + tokenPastingRightOp = false; + } else { + if (i > 0 && PPPunctuator.HASH.equals(replacementList.get(i - 1).getType())) { + // In function-like macros, a # operator before an identifier in the argument-list runs the identifier + // through parameter argument and encloses the result in quotes, effectively creating a string literal. + result.remove(result.size() - 1); + newValue = PPStringification.stringify(newValue); } else { - newTokens.add(PPGeneratedToken.build(replacement, replacement.getType(), newValue)); + // otherwise the argumentList have to be fully expanded before expanding the replacementList of the macro + newValue = expand(newValue); } } + + if (!newValue.isEmpty()) { + result.add(PPGeneratedToken.build(argument, argument.getType(), newValue)); + } } - } - // handle empty #__VA_ARGS__ => "", e.g. puts(#__VA_ARGS__) => puts("") - if (newTokens.size() > 3 - && newTokens.get(newTokens.size() - 2).getType().equals(PPPunctuator.HASH) - && newTokens.get(newTokens.size() - 1).getType().equals(PPPunctuator.BR_RIGHT)) { - for (var n = newTokens.size() - 2; n != 0; n--) { - if (newTokens.get(n).getType().equals(PPPunctuator.HASH)) { - newTokens.set(n, PPGeneratedToken.build(newTokens.get(n), CxxTokenType.STRING, "\"\"")); - break; - } else { - break; + if (newValue.isEmpty() && "__VA_ARGS__".equals(tokenValue)) { + var n = result.size() - 1; + if (n >= 0) { + if (PPPunctuator.HASH.equals(result.get(n).getType())) { + // handle empty #__VA_ARGS__ => "", e.g. puts(#__VA_ARGS__) => puts("") + result.set(n, PPGeneratedToken.build(result.get(n), CxxTokenType.STRING, "\"\"")); + } else if (PPPunctuator.COMMA.equals(result.get(n).getType())) { + // the Visual C++ implementation will suppress a trailing comma if no argumentList are passed to the ellipsis + result.remove(n); + } } } } - return newTokens; } /** @@ -259,17 +256,18 @@ private List replaceParams(PPMacro macro, List arguments) { * @param keep true means expand, false remove * * - * va-opt-replacement: - * __VA_OPT__ ( pp-tokensopt ) + * va-opt-argument: + * __VA_OPT__ ( pp-tokensopt ) * */ - private static void replaceVaOpt(List tokens, boolean keep) { + private int replaceVaOpt(List replacementList, List parameterList, List argumentList, + boolean keep, List result) { var firstIndex = -1; var lastIndex = -1; var brackets = 0; - for (int i = 0; i < tokens.size(); i++) { - var value = tokens.get(i).getValue(); + for (int i = 0; i < replacementList.size(); i++) { + var value = replacementList.get(i).getValue(); if ("(".equals(value)) { brackets++; if (firstIndex == -1) { @@ -284,16 +282,18 @@ private static void replaceVaOpt(List tokens, boolean keep) { } } - if (firstIndex > 0 && lastIndex < tokens.size()) { + if (firstIndex > 0 && lastIndex < replacementList.size()) { if (keep) { - // keep pp-tokensopt, remove ) and __VA_OPT__ ( - tokens.subList(lastIndex, lastIndex + 1).clear(); - tokens.subList(0, firstIndex).clear(); + // __VA_OPT__ ( pp-tokensopt ), keep pp-tokensopt + var ppTokens = replacementList.subList(firstIndex + 1, lastIndex); + replaceParams(ppTokens, parameterList, argumentList, result); + return 2 + ppTokens.size(); } else { - // remove from replacementList: __VA_OPT__ ( pp-tokensopt ) - tokens.subList(firstIndex - 1, lastIndex + 1).clear(); + // remove __VA_OPT__ ( pp-tokensopt ) + return 1 + lastIndex - firstIndex; } } + return 0; } } diff --git a/cxx-squid/src/test/java/org/sonar/cxx/parser/PreprocessorDirectivesTest.java b/cxx-squid/src/test/java/org/sonar/cxx/parser/PreprocessorDirectivesTest.java index a47585f7d7..3e71e7eac7 100644 --- a/cxx-squid/src/test/java/org/sonar/cxx/parser/PreprocessorDirectivesTest.java +++ b/cxx-squid/src/test/java/org/sonar/cxx/parser/PreprocessorDirectivesTest.java @@ -235,7 +235,6 @@ void variadic_macros() { "#define showlist(...) puts(#__VA_ARGS__)\n" + "showlist(1, \"x\", int);")) .isEqualTo("puts ( \"1,\\\"x\\\",int\" ) ; EOF"); - } @Test @@ -266,6 +265,72 @@ void va_opt_macros() { "#define SDEF(sname, ...) S sname __VA_OPT__(= { __VA_ARGS__ })\n" + "SDEF(bar, 1, 2);")) .isEqualTo("S bar = { 1 , 2 } ; EOF"); + + // https://marc.info/?l=gcc-patches&m=151047200615291&w=2 + assertThat(parse( + "#define CALL(F, ...) F(7 __VA_OPT__(,) __VA_ARGS__)\n" + + "CALL(f1);")) + .isEqualTo("f1 ( 7 ) ; EOF"); + + assertThat(parse( + "#define CALL(F, ...) F(7 __VA_OPT__(,) __VA_ARGS__)\n" + + "CALL(f1, );")) + .isEqualTo("f1 ( 7 ) ; EOF"); + + assertThat(parse( + "#define CALL(F, ...) F(7 __VA_OPT__(,) __VA_ARGS__)\n" + + "CALL(f1, 1);")) + .isEqualTo("f1 ( 7 , 1 ) ; EOF"); + + assertThat(parse( + "#define CP(F, X, Y, ...) F(__VA_OPT__(X ## Y,) __VA_ARGS__)\n" + + "CP(f0, one, two);")) + .isEqualTo("f0 ( ) ; EOF"); + + assertThat(parse( + "#define CP(F, X, Y, ...) F(__VA_OPT__(X ## Y,) __VA_ARGS__)\n" + + "CP(f0, one, two, );")) + .isEqualTo("f0 ( ) ; EOF"); + + assertThat(parse( + "#define CP(F, X, Y, ...) F(__VA_OPT__(X ## Y,) __VA_ARGS__)\n" + + "CP(f0, one, two, 3);")) + .isEqualTo("f0 ( onetwo , 3 ) ; EOF"); + + assertThat(parse( + "#define CS(F, ...) F(__VA_OPT__(s(# __VA_ARGS__)))\n" + + "CS(f0);")) + .isEqualTo("f0 ( ) ; EOF"); + + assertThat(parse( + "#define CS(F, ...) F(__VA_OPT__(s(# __VA_ARGS__)))\n" + + "CS(f1, 1, 2, 3, 4);")) + .isEqualTo("f1 ( s ( \"1,2,3,4\" ) ) ; EOF"); + + assertThat(parse( + "#define D(F, ...) F(__VA_OPT__(__VA_ARGS__) __VA_OPT__(,) __VA_ARGS__)\n" + + "D(f0);")) + .isEqualTo("f0 ( ) ; EOF"); + + assertThat(parse( + "#define D(F, ...) F(__VA_OPT__(__VA_ARGS__) __VA_OPT__(,) __VA_ARGS__)\n" + + "D(f2, 1);")) + .isEqualTo("f2 ( 1 , 1 ) ; EOF"); + + assertThat(parse( + "#define D(F, ...) F(__VA_OPT__(__VA_ARGS__) __VA_OPT__(,) __VA_ARGS__)\n" + + "D(f2, 1, 2);")) + .isEqualTo("f2 ( 1 , 2 , 1 , 2 ) ; EOF"); + + assertThat(parse( + "#define CALL0(...) __VA_OPT__(f2)(0 __VA_OPT__(,)__VA_ARGS__)\n" + + "int* ptr = CALL0();")) + .isEqualTo("int * ptr = ( 0 ) ; EOF"); + + assertThat(parse( + "#define CALL0(...) __VA_OPT__(f2)(0 __VA_OPT__(,)__VA_ARGS__)\n" + + "CALL0(23);")) + .isEqualTo("f2 ( 0 , 23 ) ; EOF"); } @Test diff --git a/cxx-squid/src/test/java/org/sonar/cxx/preprocessor/PPMacroTest.java b/cxx-squid/src/test/java/org/sonar/cxx/preprocessor/PPMacroTest.java index 77cce5d9ad..89e80a94ff 100644 --- a/cxx-squid/src/test/java/org/sonar/cxx/preprocessor/PPMacroTest.java +++ b/cxx-squid/src/test/java/org/sonar/cxx/preprocessor/PPMacroTest.java @@ -59,10 +59,13 @@ void testCreateVariadicMacro() { @Test void testGetParameterIndex() { - PPMacro result = PPMacro.create("#define MACRO(P1, P2) REPLACEMENT_LIST"); + PPMacro macro = PPMacro.create("#define MACRO(P1, P2) REPLACEMENT_LIST"); + assertThat(macro.getParameterIndex("P1")).isEqualTo(0); + assertThat(macro.getParameterIndex("P2")).isEqualTo(1); - assertThat(result.getParameterIndex("P1")).isEqualTo(0); - assertThat(result.getParameterIndex("P2")).isEqualTo(1); + var parameterNames = macro.getParameterNames(); + assertThat(parameterNames.indexOf("P1")).isEqualTo(0); + assertThat(parameterNames.indexOf("P2")).isEqualTo(1); } }