Fix #684: Keep line directives in tokenized output

simplecpp.cpp

@@ -640,23 +640,6 @@ static bool isStringLiteralPrefix(const std::string &str)
            str == "R" || str == "uR" || str == "UR" || str == "LR" || str == "u8R";
 }
 
-void simplecpp::TokenList::lineDirective(unsigned int fileIndex_, unsigned int line, Location &location)
-{
-    if (fileIndex_ != location.fileIndex || line >= location.line) {
-        location.fileIndex = fileIndex_;
-        location.line = line;
-        return;
-    }
-
-    if (line + 2 >= location.line) {
-        location.line = line;
-        while (cback()->op != '#')
-            deleteToken(back());
-        deleteToken(back());
-        return;
-    }
-}
-
 static const std::string COMMENT_END("*/");
 
 void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, OutputList *outputList)
@@ -665,6 +648,8 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,
 
     const Token *oldLastToken = nullptr;
 
+    bool locationchange = false;
+
     Location location(fileIndex(filename), 1, 1);
     while (stream.good()) {
         unsigned char ch = stream.readChar();
@@ -726,17 +711,14 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,
                 if (!ppTok || !ppTok->number)
                     continue;
 
-                const unsigned int line = std::atol(ppTok->str().c_str());
-                ppTok = advanceAndSkipComments(ppTok);
+                location.line = std::atol(ppTok->str().c_str());
 
-                unsigned int fileindex;
+                ppTok = advanceAndSkipComments(ppTok);
 
                 if (ppTok && ppTok->str()[0] == '\"')
-                    fileindex = fileIndex(replaceAll(ppTok->str().substr(1U, ppTok->str().size() - 2U),"\\\\","\\"));
-                else
-                    fileindex = location.fileIndex;
+                    location.fileIndex = fileIndex(replaceAll(ppTok->str().substr(1U, ppTok->str().size() - 2U),"\\\\","\\"));
 
-                lineDirective(fileindex, line, location);
+                locationchange = true;
             }
 
             continue;
@@ -955,6 +937,11 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,
 
         push_back(new Token(currentToken, location, !!std::isspace(stream.peekChar())));
 
+        if (locationchange) {
+            back()->locationchange = true;
+            locationchange = false;
+        }
+
         if (multiline)
             location.col += currentToken.size();
         else
@@ -1458,6 +1445,8 @@ const simplecpp::Token* simplecpp::TokenList::lastLineTok(int maxsize) const
     for (const Token *tok = cback(); ; tok = tok->previous) {
         if (!sameline(tok, cback()))
             break;
+        if (tok->locationchange)
+            return tok;
         if (tok->comment)
             continue;
         if (++count > maxsize)
@@ -3000,7 +2989,9 @@ static const simplecpp::Token *gotoNextLine(const simplecpp::Token *tok)
 {
     const unsigned int line = tok->location.line;
     const unsigned int file = tok->location.fileIndex;
-    while (tok && tok->location.line == line && tok->location.fileIndex == file)
+    if (tok)
+        tok = tok->next;
+    while (tok && tok->location.line == line && tok->location.fileIndex == file && !tok->locationchange)
         tok = tok->next;
     return tok;
 }
@@ -3541,7 +3532,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL
             continue;
         }
 
-        if (rawtok->op == '#' && !sameline(rawtok->previousSkipComments(), rawtok)) {
+        if (rawtok->op == '#' && (!sameline(rawtok->previousSkipComments(), rawtok) || rawtok->locationchange)) {
             if (!sameline(rawtok, rawtok->next)) {
                 rawtok = rawtok->next;
                 continue;

simplecpp.h

@@ -156,7 +156,7 @@ namespace simplecpp {
         }
 
         Token(const Token &tok) :
-            macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {}
+            macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), locationchange(tok.locationchange), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {}
 
         Token &operator=(const Token &tok) = delete;
 
@@ -182,6 +182,8 @@ namespace simplecpp {
         bool name;
         bool number;
         bool whitespaceahead;
+        /** whether token location is at a discontinuity from a #line directive */
+        bool locationchange {false};
         Location location;
         Token *previous{};
         Token *next{};
@@ -396,7 +398,6 @@ namespace simplecpp {
         void constFoldQuestionOp(Token *&tok1);
 
         std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList);
-        void lineDirective(unsigned int fileIndex_, unsigned int line, Location &location);
 
         const Token* lastLineTok(int maxsize=1000) const;
         const Token* isLastLinePreprocessor(int maxsize=1000) const;

test.cpp

@@ -2749,6 +2749,62 @@ static void nullDirective3()
     ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code));
 }
 
+static void lineDirective1()
+{
+    const char code[] = "#line 1\n"
+                        "#line 2 \"header1.h\"\n"
+                        "# 1\n"
+                        "# 2 \"header2.h\"\n"
+                        "# 3 \"header2.h\" 0\n";
+
+    std::vector<std::string> files;
+    const simplecpp::TokenList rawtokens(code, files);
+
+    ASSERT_EQUALS("1: # line 1 # line 2 \"header1.h\"\n"
+                  "#line 2 \"header1.h\"\n"
+                  "2: # 1\n"
+                  "#line 1 \"header1.h\"\n"
+                  "1: # 2 \"header2.h\"\n"
+                  "#line 2 \"header2.h\"\n"
+                  "2: # 3 \"header2.h\" 0",
+                  rawtokens.stringify(true));
+
+    simplecpp::FileDataCache cache;
+    simplecpp::TokenList out(files);
+    const simplecpp::DUI dui;
+    simplecpp::preprocess(out, rawtokens, files, cache, dui);
+
+    ASSERT_EQUALS("", out.stringify(true));
+}
+
+static void lineDirective2()
+{
+    const char code[] = "#\n"
+                        "#line 1\n"
+                        "include\n"
+                        "#line 1\n"
+                        "\"header1.h\"\n";
+
+    std::vector<std::string> files;
+    const simplecpp::TokenList rawtokens(code, files);
+
+    ASSERT_EQUALS("1: #\n"
+                  "2: # line 1\n"
+                  "#line 1 \"\"\n"
+                  "1: include\n"
+                  "2: # line 1\n"
+                  "#line 1 \"\"\n"
+                  "1: \"header1.h\"",
+                  rawtokens.stringify(true));
+
+    simplecpp::FileDataCache cache;
+    simplecpp::TokenList out(files);
+    const simplecpp::DUI dui;
+    simplecpp::preprocess(out, rawtokens, files, cache, dui);
+
+    ASSERT_EQUALS("1: include \"header1.h\"", out.stringify(true));
+}
+
 static void include1()
 {
     const char code[] = "#include \"A.h\"\n";
@@ -4167,6 +4223,9 @@ static void runTests(int argc, char **argv, Input input)
     TEST_CASE(nullDirective2);
     TEST_CASE(nullDirective3);
 
+    TEST_CASE(lineDirective1);
+    TEST_CASE(lineDirective2);
+
     TEST_CASE(include1);
     TEST_CASE(include2);
     TEST_CASE(include3);