added TokenList constructors with modern buffer wrappers and hide "unsafe" ones - if available (danmar#496)

Author: firewave

.github/workflows/CI-unixish.yml

@@ -62,6 +62,16 @@ jobs:
       run: |
         make -j$(nproc) selfcheck
 
+    - name: make testrunner (c++17)
+      run: |
+        make clean
+        make -j$(nproc) testrunner CXXOPTS="-std=c++17"
+
+    - name: make testrunner (c++20)
+      run: |
+        make clean
+        make -j$(nproc) testrunner CXXOPTS="-std=c++20"
+
     - name: Run CMake
       run: |
         cmake -S . -B cmake.output -DCMAKE_COMPILE_WARNING_AS_ERROR=On

simplecpp.cpp

@@ -466,20 +466,13 @@ simplecpp::TokenList::TokenList(std::istream &istr, std::vector<std::string> &fi
     readfile(stream,filename,outputList);
 }
 
-simplecpp::TokenList::TokenList(const unsigned char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename, OutputList *outputList)
+simplecpp::TokenList::TokenList(const unsigned char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename, OutputList *outputList, int /*unused*/)
     : frontToken(nullptr), backToken(nullptr), files(filenames)
 {
     StdCharBufStream stream(data, size);
     readfile(stream,filename,outputList);
 }
 
-simplecpp::TokenList::TokenList(const char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename, OutputList *outputList)
-    : frontToken(nullptr), backToken(nullptr), files(filenames)
-{
-    StdCharBufStream stream(reinterpret_cast<const unsigned char*>(data), size);
-    readfile(stream,filename,outputList);
-}
-
 simplecpp::TokenList::TokenList(const std::string &filename, std::vector<std::string> &filenames, OutputList *outputList)
     : frontToken(nullptr), backToken(nullptr), files(filenames)
 {

simplecpp.h

@@ -20,6 +20,16 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
+#if __cplusplus >= 202002L
+#  include <version>
+#endif
+
+#if defined(__cpp_lib_string_view) && !defined(__cpp_lib_span)
+#include <string_view>
+#endif
+#ifdef __cpp_lib_span
+#include <span>
+#endif
 
 #ifdef _WIN32
 #  ifdef SIMPLECPP_EXPORT
@@ -46,6 +56,15 @@
 #  pragma warning(disable : 4244)
 #endif
 
+// provide legacy (i.e. raw pointer) API for TokenList
+// note: std::istream has an overhead compared to raw pointers
+#ifndef SIMPLECPP_TOKENLIST_ALLOW_PTR
+// still provide the legacy API in case we lack the performant wrappers
+#  if !defined(__cpp_lib_string_view) && !defined(__cpp_lib_span)
+#    define SIMPLECPP_TOKENLIST_ALLOW_PTR
+#  endif
+#endif
+
 namespace simplecpp {
     /** C code standard */
     enum cstd_t { CUnknown=-1, C89, C99, C11, C17, C23 };
@@ -216,10 +235,45 @@ namespace simplecpp {
         explicit TokenList(std::vector<std::string> &filenames);
         /** generates a token list from the given std::istream parameter */
         TokenList(std::istream &istr, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr);
+#ifdef SIMPLECPP_TOKENLIST_ALLOW_PTR
+        /** generates a token list from the given buffer */
+        template<size_t size>
+        TokenList(const char (&data)[size], std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
+            : TokenList(reinterpret_cast<const unsigned char*>(data), size-1, filenames, filename, outputList, 0)
+        {}
+        /** generates a token list from the given buffer */
+        template<size_t size>
+        TokenList(const unsigned char (&data)[size], std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
+                : TokenList(data, size-1, filenames, filename, outputList, 0)
+        {}
+
         /** generates a token list from the given buffer */
-        TokenList(const unsigned char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr);
+        TokenList(const unsigned char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
+            : TokenList(data, size, filenames, filename, outputList, 0)
+        {}
         /** generates a token list from the given buffer */
-        TokenList(const char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr);
+        TokenList(const char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
+            : TokenList(reinterpret_cast<const unsigned char*>(data), size, filenames, filename, outputList, 0)
+        {}
+#endif
+#if defined(__cpp_lib_string_view) && !defined(__cpp_lib_span)
+        /** generates a token list from the given buffer */
+        TokenList(std::string_view data, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
+            : TokenList(reinterpret_cast<const unsigned char*>(data.data()), data.size(), filenames, filename, outputList, 0)
+        {}
+#endif
+#ifdef __cpp_lib_span
+        /** generates a token list from the given buffer */
+        TokenList(std::span<const char> data, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
+                : TokenList(reinterpret_cast<const unsigned char*>(data.data()), data.size(), filenames, filename, outputList, 0)
+        {}
+
+        /** generates a token list from the given buffer */
+        TokenList(std::span<const unsigned char> data, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
+                : TokenList(data.data(), data.size(), filenames, filename, outputList, 0)
+        {}
+#endif
+
         /** generates a token list from the given filename parameter */
         TokenList(const std::string &filename, std::vector<std::string> &filenames, OutputList *outputList = nullptr);
         TokenList(const TokenList &other);
@@ -295,6 +349,8 @@ namespace simplecpp {
         }
 
     private:
+        TokenList(const unsigned char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename, OutputList *outputList, int unused);
+
         void combineOperators();
 
         void constFoldUnaryNotPosNeg(Token *tok);
@@ -505,6 +561,8 @@ namespace simplecpp {
     SIMPLECPP_LIB std::string getCppStdString(cppstd_t std);
 }
 
+#undef SIMPLECPP_TOKENLIST_ALLOW_PTR
+
 #if defined(_MSC_VER)
 #  pragma warning(pop)
 #endif

test.cpp

@@ -3179,6 +3179,44 @@ static void preprocess_files()
     }
 }
 
+static void safe_api()
+{
+    // this test is to make sure the safe APIs are compiling
+#if defined(__cpp_lib_string_view) || defined(__cpp_lib_span)
+    std::vector<std::string> filenames;
+#  if defined(__cpp_lib_string_view)
+    {
+        const char input[] = "code";
+        const std::string_view sv = input;
+        // std::string_view can be implicitly converted into a std::span
+        simplecpp::TokenList(sv,filenames,"");
+    }
+#  endif
+#  ifdef __cpp_lib_span
+    {
+        char input[] = "code";
+        const std::span sp = input;
+        simplecpp::TokenList(sp,filenames,"");
+    }
+    {
+        const char input[] = "code";
+        const std::span sp = input;
+        simplecpp::TokenList(sp,filenames,"");
+    }
+    {
+        unsigned char input[] = "code";
+        const std::span sp = input;
+        simplecpp::TokenList(sp,filenames,"");
+    }
+    {
+        const unsigned char input[] = "code";
+        const std::span sp = input;
+        simplecpp::TokenList(sp,filenames,"");
+    }
+#  endif
+#endif
+}
+
 static void fuzz_crash()
 {
     {
@@ -3445,6 +3483,8 @@ int main(int argc, char **argv)
 
     TEST_CASE(preprocess_files);
 
+    TEST_CASE(safe_api);
+
     TEST_CASE(fuzz_crash);
 
     return numberOfFailedAssertions > 0 ? EXIT_FAILURE : EXIT_SUCCESS;