Replace preprocessor with code from simplecpp project

Author: usiems

README.md

@@ -26,6 +26,11 @@ See https://code.qt.io/cgit/qt-labs/qtscriptgenerator.git for the original proje
 The PythonQt wrappers generated by the generator are distributed under the `LGPL`
 as well.
 
+The integrated preprocessor has been replaced with code from the simplecpp project,
+see https://github.com/danmar/simplecpp. It is licensed under the `0BSDL` license,
+see the `LICENSE` file in the generator/simplecpp directory.
+Copyright (C) 2016-2023 simplecpp team
+
 The generated wrappers are pre-generated and checked-in, so you only
 need to build and run the generator when you want to build additional wrappers
 or you want to upgrade/downgrade to another Qt version, but this requires

generator/abstractmetabuilder.cpp

@@ -1169,7 +1169,7 @@ void AbstractMetaBuilder::traverseFunctions(ScopeModelItem scope_item, AbstractM
                     ReportHandler::warning(warn);
                 }
 
-                meta_class->addFunction(meta_function);
+                meta_class->addFunction(meta_function, /*check_duplicates=*/true);
             } else if (meta_function->isDestructor()) {
                 meta_class->setDestructorException(meta_function->exception());
                 if (!meta_function->isPublic()) {

generator/generator.pri

@@ -16,7 +16,7 @@ RESOURCES += generator.qrc
 
 include($$GENERATORPATH/parser/rxx.pri)
 
-include($$GENERATORPATH/parser/rpp/rpp.pri)
+include($$GENERATORPATH/simplecpp/simplecpp.pri)
 
 CONFIG += strict_c++
 win32-msvc*{

generator/generator.qrc

@@ -15,6 +15,5 @@
 <file alias="typesystem_multimedia.txt">typesystem_multimedia.xml</file>
 <file alias="typesystem_qml.txt">typesystem_qml.xml</file>
 <file alias="typesystem_quick.txt">typesystem_quick.xml</file>
-<file>parser/rpp/pp-qt-configuration</file>
 </qresource>
 </RCC>

generator/main.cpp

@@ -48,14 +48,16 @@
 #include "generatorset.h"
 #include "fileout.h"
 #include "control.h"
-#include "pp.h"
+#include "simplecpp.h"
 
 #include <QDir>
 #include <QFileInfo>
 #include <QFile>
 #include <QTextStream>
 #include <QRegularExpression>
 
+#include <memory>
+
 void displayHelp(GeneratorSet *generatorSet);
 
 namespace
@@ -125,53 +127,86 @@ namespace
     }
 
     bool
-    preprocess(const QString &sourceFile, const QString &targetFile, const QString &commandLineIncludes = QString())
+    preprocess(const QString& sourceFile, const QString& targetFile, const QString& commandLineIncludes = QString())
     {
-      rpp::pp_environment env;
-      rpp::pp preprocess(env);
-
-      rpp::pp_null_output_iterator null_out;
+      simplecpp::DUI dui; // settings
 
-      const char *ppconfig = ":/trolltech/generator/parser/rpp/pp-qt-configuration";
+      for(QString include : getIncludeDirectories(commandLineIncludes)) {
+        dui.includePaths.push_back(QDir::toNativeSeparators(include).toStdString());
+      }
+      dui.defines.push_back("__cplusplus=1");
+      dui.defines.push_back("__STDC__");
+      dui.std = "c++20";
+      dui.removeComments = true;
 
-      QFile file(ppconfig);
+      QFile file(sourceFile);
       if (!file.open(QFile::ReadOnly))
       {
-        fprintf(stderr, "Preprocessor configuration file not found '%s'\n", ppconfig);
+        std::cerr << "Main file not found:" << sourceFile.toUtf8().constData() << std::endl;
         return false;
       }
 
       QByteArray ba = file.readAll();
       file.close();
-      preprocess.operator()(ba.constData(), ba.constData() + ba.size(), null_out);
 
-      foreach(QString
-      include, getIncludeDirectories(commandLineIncludes)) {
-        preprocess.push_include_path(QDir::toNativeSeparators(include).toStdString());
+      // Perform preprocessing (code originally taken from simplecpp/main.cpp and modified)
+      simplecpp::OutputList outputList;
+      std::vector<std::string> files;
+      std::unique_ptr<simplecpp::TokenList> rawtokens(new simplecpp::TokenList(ba.constData(), ba.size(), files, {}, &outputList));
+      rawtokens->removeComments();
+      simplecpp::TokenList outputTokens(files);
+      std::map<std::string, simplecpp::TokenList*> filedata;
+      simplecpp::preprocess(outputTokens, *rawtokens, files, filedata, dui, &outputList);
+      simplecpp::cleanup(filedata);
+      rawtokens.reset();
+
+      for (const simplecpp::Output& output : outputList) {
+        if (output.type == simplecpp::Output::MISSING_HEADER) {
+          // do not print these messages for now, we are not interested
+          continue;
+        }
+        std::cerr << output.location.file() << ':' << output.location.line << ": ";
+        switch (output.type) {
+        case simplecpp::Output::ERROR:
+          std::cerr << "#error: ";
+          break;
+        case simplecpp::Output::WARNING:
+          std::cerr << "#warning: ";
+          break;
+        case simplecpp::Output::MISSING_HEADER:
+          std::cerr << "missing header: ";
+          break;
+        case simplecpp::Output::INCLUDE_NESTED_TOO_DEEPLY:
+          std::cerr << "include nested too deeply: ";
+          break;
+        case simplecpp::Output::SYNTAX_ERROR:
+          std::cerr << "syntax error: ";
+          break;
+        case simplecpp::Output::PORTABILITY_BACKSLASH:
+          std::cerr << "portability: ";
+          break;
+        case simplecpp::Output::UNHANDLED_CHAR_ERROR:
+          std::cerr << "unhandled char error: ";
+          break;
+        case simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND:
+          std::cerr << "explicit include not found: ";
+          break;
+        case simplecpp::Output::FILE_NOT_FOUND:
+          std::cerr << "file not found: ";
+          break;
+        case simplecpp::Output::DUI_ERROR:
+          std::cerr << "dui error: ";
+          break;
+        }
+        std::cerr << output.msg << std::endl;
       }
 
-      QString currentDir = QDir::current().absolutePath();
-      QFileInfo sourceInfo(sourceFile);
-      QDir::setCurrent(sourceInfo.absolutePath());
-
-      std::string result;
-      result.reserve(20 * 1024); // 20K
-
-      result += "# 1 \"builtins\"\n";
-      result += "# 1 \"";
-      result += sourceFile.toStdString();
-      result += "\"\n";
-
-      preprocess.file(sourceInfo.fileName().toStdString(),
-                      rpp::pp_output_iterator<std::string>(result));
-
-      QDir::setCurrent(currentDir);
-
       QFile f(targetFile);
       if (!f.open(QIODevice::WriteOnly | QIODevice::Text))
       {
         fprintf(stderr, "Failed to write preprocessed file: %s\n", qPrintable(targetFile));
       }
+      std::string result = outputTokens.stringify();
       f.write(result.c_str(), result.length());
 
       return true;

generator/parser/lexer.cpp

@@ -69,6 +69,9 @@ void LocationManager::extract_line(int offset, int *line, QString *filename) con
   const unsigned char *cursor = begin_buffer + offset;
 
   ++cursor; // skip '#'
+  if (cursor[0] == 'l' && cursor[1] == 'i' && cursor[2] == 'n' && cursor[3] == 'e') {
+    cursor += 4;
+  }
   if (std::isspace(*cursor) && std::isdigit(*(cursor + 1)))
     {
       ++cursor;