added additional tests and support for structs

Author: siriuslee69

.gitmodules

@@ -4,3 +4,12 @@
 [submodule "testCRepos/repos/BLAKE2"]
 	path = testCRepos/repos/BLAKE2
 	url = https://github.com/BLAKE2/BLAKE2
+[submodule "testCRepos/repos/openssl"]
+	path = testCRepos/repos/openssl
+	url = https://github.com/openssl/openssl.git
+[submodule "testCRepos/repos/libsodium"]
+	path = testCRepos/repos/libsodium
+	url = https://github.com/jedisct1/libsodium.git
+[submodule "testCRepos/repos/liboqs"]
+	path = testCRepos/repos/liboqs
+	url = https://github.com/open-quantum-safe/liboqs.git

nimAutoWrapper.nimble

@@ -11,17 +11,54 @@ task build_blake2, "Generate wrapper for BLAKE2 reference code":
   exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
   exec "nim c -r nimAutoWrapper.nim testCRepos/repos/BLAKE2/ref/blake2.h testCRepos/builds/BLAKE2/blake2_wrapper.nim"
 
-task build_repos, "Generate wrappers for all test repos":
+task build_openssl, "Build OpenSSL 3 and generate wrapper":
   exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
-  exec "nim c -r nimAutoWrapper.nim testCRepos/repos/tiny-AES-c/aes.h testCRepos/builds/tiny-AES-c/aes_wrapper.nim"
-  exec "nim c -r nimAutoWrapper.nim testCRepos/repos/BLAKE2/ref/blake2.h testCRepos/builds/BLAKE2/blake2_wrapper.nim"
+  exec "nim r tools/build_openssl.nim"
+  exec "nim c -r nimAutoWrapper.nim testCRepos/repos/openssl/include/openssl/sha.h testCRepos/builds/openssl/openssl_sha_wrapper.nim"
+
+task build_libsodium, "Build libsodium and generate wrapper":
+  exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
+  exec "nim r tools/build_libsodium.nim"
+  exec "nim r tools/prepare_libsodium_header.nim"
+  exec "nim c -r nimAutoWrapper.nim testCRepos/builds/libsodium/sodium_combined.h testCRepos/builds/libsodium/libsodium_wrapper.nim"
+
+task build_liboqs, "Build liboqs and generate wrapper":
+  exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
+  exec "nim r tools/build_liboqs.nim"
+  exec "nim r tools/prepare_liboqs_header.nim"
+  exec "nim c -r nimAutoWrapper.nim testCRepos/builds/liboqs/oqs_full_combined.h testCRepos/builds/liboqs/liboqs_wrapper.nim"
+
+task build_c_repos_basic, "Build C test repos without OpenSSL":
+  exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
+  exec "nim r tools/build_libsodium.nim"
+  exec "nim r tools/build_liboqs.nim"
+
+task build_c_repos_all, "Build all C test repos":
+  exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
+  exec "nim r tools/build_openssl.nim"
+  exec "nim r tools/build_libsodium.nim"
+  exec "nim r tools/build_liboqs.nim"
+
+task build_repos, "Generate wrappers for all test repos":
+  exec "nimble build_repos_basic"
+
+task build_repos_basic, "Generate wrappers for test repos without OpenSSL":
+  exec "nimble build_aes"
+  exec "nimble build_blake2"
+  exec "nimble build_libsodium"
+  exec "nimble build_liboqs"
+
+task build_repos_all, "Generate wrappers for all test repos":
+  exec "nimble build_repos_basic"
+  exec "nimble build_openssl"
 
 task setup, "Fetch submodules for test repos":
   exec "nim r tools/ensure_env.nim -- --submodules"
 
 task start, "Fetch submodules and build test wrappers":
   exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
-  exec "nimble build_repos"
+  exec "nimble build_c_repos_basic"
+  exec "nimble build_repos_basic"
 
 task test_functionality, "Run tokenizer and utils tests":
   exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
@@ -30,14 +67,42 @@ task test_functionality, "Run tokenizer and utils tests":
 
 task test_realworld, "Run real-world wrapper tests":
   exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
-  exec "nimble build_repos"
+  exec "nimble build_c_repos_basic"
+  exec "nimble build_repos_basic"
+  exec "nim c -r tests/realworld/tiny_aes_c_runner.nim"
+  exec "nim c -r tests/realworld/blake2_ref_runner.nim"
+  exec "nim c -r tests/realworld/libsodium_runner.nim"
+  exec "nim c -r tests/realworld/liboqs_runner.nim"
+
+task test_realworld_all, "Run all real-world wrapper tests":
+  exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
+  exec "nimble build_c_repos_all"
+  exec "nimble build_repos_all"
   exec "nim c -r tests/realworld/tiny_aes_c_runner.nim"
   exec "nim c -r tests/realworld/blake2_ref_runner.nim"
+  exec "nim c -r tests/realworld/openssl3_runner.nim"
+  exec "nim c -r tests/realworld/libsodium_runner.nim"
+  exec "nim c -r tests/realworld/liboqs_runner.nim"
 
 task test_all, "Run all tests":
   exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
-  exec "nimble build_repos"
+  exec "nimble build_c_repos_basic"
+  exec "nimble build_repos_basic"
+  exec "nim c -r --path:. tests/functionality/test_tokenizer.nim"
+  exec "nim c -r --path:. tests/functionality/test_utils.nim"
+  exec "nim c -r tests/realworld/tiny_aes_c_runner.nim"
+  exec "nim c -r tests/realworld/blake2_ref_runner.nim"
+  exec "nim c -r tests/realworld/libsodium_runner.nim"
+  exec "nim c -r tests/realworld/liboqs_runner.nim"
+
+task test_all_full, "Run all tests including OpenSSL":
+  exec "nim r tools/ensure_env.nim -- --submodules --builddirs"
+  exec "nimble build_c_repos_all"
+  exec "nimble build_repos_all"
   exec "nim c -r --path:. tests/functionality/test_tokenizer.nim"
   exec "nim c -r --path:. tests/functionality/test_utils.nim"
   exec "nim c -r tests/realworld/tiny_aes_c_runner.nim"
   exec "nim c -r tests/realworld/blake2_ref_runner.nim"
+  exec "nim c -r tests/realworld/openssl3_runner.nim"
+  exec "nim c -r tests/realworld/libsodium_runner.nim"
+  exec "nim c -r tests/realworld/liboqs_runner.nim"

src/define_parser.nim

@@ -41,6 +41,19 @@ proc hasTemplateLine*(s: ParserState, b: string): bool =
     dec i
   result = false
 
+proc isSimpleDefineBody*(a: seq[Token]): bool =
+  ## a: define body tokens
+  ## Returns true when the body has no identifiers and can be emitted as a const.
+  var
+    i: int = 0
+    l: int = a.len
+  while i < l:
+    if a[i].kind == tkIdentifier:
+      result = false
+      return
+    inc i
+  result = true
+
 proc formatTemplateParams*(a: seq[string]): string =
   ## a: parameter names
   ## Formats names into a Nim template parameter list.
@@ -53,6 +66,47 @@ proc formatTemplateParams*(a: seq[string]): string =
       parts.add a[i] & ": untyped"
   result = parts.join(", ")
 
+proc emitDefineTemplate*(s: var ParserState, b: string, c: seq[string], d: string,
+    e: bool) =
+  ## s: parser state
+  ## b: template name
+  ## c: template parameter names
+  ## d: macro body text
+  ## e: include parentheses when there are no parameters
+  ## Emits a template stub for a macro definition.
+  var
+    paramsText: string = formatTemplateParams(c)
+  if paramsText.len == 0:
+    if e:
+      emitLine(s, "template " & b & "*(): untyped =")
+    else:
+      emitLine(s, "template " & b & "*: untyped =")
+  else:
+    emitLine(s, "template " & b & "*(" & paramsText & "): untyped =")
+  if s.config.emitComments and d.len > 0:
+    emitLine(s, "  ## C macro: " & d)
+  emitLine(s, "  discard")
+
+proc collectDefineBodyTokens*(s: var ParserState): seq[Token] =
+  ## s: parser state
+  ## Collects macro body tokens across backslash-newline continuations.
+  var
+    items: seq[Token] = @[]
+    tok: Token
+    continueLine: bool = true
+  while continueLine and not isAtEnd(s):
+    continueLine = false
+    tok = peekToken(s)
+    while not isAtEnd(s) and tok.kind != tkNewline:
+      items.add advanceToken(s)
+      tok = peekToken(s)
+    if tok.kind == tkNewline:
+      discard advanceToken(s)
+    if items.len > 0 and items[^1].text == "\\":
+      discard items.pop()
+      continueLine = true
+  result = items
+
 proc tryParseDefine*(s: var ParserState): bool =
   ## s: parser state
   ## Parses C #define directives into Nim consts or templates.
@@ -66,7 +120,6 @@ proc tryParseDefine*(s: var ParserState): bool =
     bodyText: string = ""
     constLine: string = ""
     params: seq[string] = @[]
-    paramsText: string = ""
     paramName: string = ""
     tok: Token
     isFunc: bool = false
@@ -97,23 +150,16 @@ proc tryParseDefine*(s: var ParserState): bool =
         params.add paramName
       else:
         discard advanceToken(s)
-  bodyTokens = collectUntilNewline(s)
+  bodyTokens = collectDefineBodyTokens(s)
   if not isFunc:
     bodyTokens = stripLeadingCastTokens(bodyTokens)
   bodyText = tokensToText(bodyTokens)
-  if isFunc:
+  if isFunc or not isSimpleDefineBody(bodyTokens):
     name = registerName(s, name, origName, "template")
     if hasTemplateLine(s, name):
       result = true
       return
-    paramsText = formatTemplateParams(params)
-    if paramsText.len == 0:
-      emitLine(s, "template " & name & "*(): untyped =")
-    else:
-      emitLine(s, "template " & name & "*(" & paramsText & "): untyped =")
-    if s.config.emitComments and bodyText.len > 0:
-      emitLine(s, "  ## C macro: " & bodyText)
-    emitLine(s, "  discard")
+    emitDefineTemplate(s, name, params, bodyText, isFunc)
   else:
     name = registerName(s, name, origName, "const")
     if bodyText.len == 0:

src/function_parser.nim

@@ -1,6 +1,7 @@
 import strutils
 import name_mangle
 import name_registry
+import type_mapper
 import types
 import utils
 
@@ -93,34 +94,24 @@ proc paramNameFromTokens*(a: seq[Token]): string =
   else:
     result = name
 
-proc paramTypeFromTokens*(a: seq[Token]): string =
+proc paramTypeFromTokens*(s: var ParserState, a: seq[Token], b: string): string =
+  ## s: parser state
   ## a: parameter tokens
-  ## Returns the Nim type for known C types, defaulting to pointer.
-  var
-    i: int = 0
-    l: int = a.len
-    tok: Token
-    hasSizeT: bool = false
-  while i < l:
-    tok = a[i]
-    if tok.kind == tkIdentifier and tok.text == "size_t":
-      hasSizeT = true
-    inc i
-  if hasSizeT:
-    result = "csize_t"
-  else:
-    result = "pointer"
+  ## b: parameter name to skip
+  ## Returns the Nim type for the parameter.
+  result = mapTokensToNimType(s, a, b)
 
-proc paramInfoFromTokens*(a: seq[Token]): ParamInfo =
+proc paramInfoFromTokens*(s: var ParserState, a: seq[Token]): ParamInfo =
+  ## s: parser state
   ## a: parameter tokens
   ## Builds a ParamInfo with a name and Nim type.
   var
     info: ParamInfo
   info.name = paramNameFromTokens(a)
-  info.nimType = paramTypeFromTokens(a)
+  info.nimType = paramTypeFromTokens(s, a, info.name)
   result = info
 
-proc collectParamInfos*(s: ParserState): seq[ParamInfo] =
+proc collectParamInfos*(s: var ParserState): seq[ParamInfo] =
   ## s: parser state
   ## Collects parameter infos between parentheses.
   ## Example: `int foo(int a, size_t n)` yields `@["a: pointer", "n: csize_t"]`.
@@ -141,12 +132,12 @@ proc collectParamInfos*(s: ParserState): seq[ParamInfo] =
       continue
     if tok.text == ")":
       if current.len > 0:
-        info = paramInfoFromTokens(current)
+        info = paramInfoFromTokens(s, current)
         if info.name.len > 0:
           params.add info
       break
     if tok.text == ",":
-      info = paramInfoFromTokens(current)
+      info = paramInfoFromTokens(s, current)
       if info.name.len > 0:
         params.add info
       current = @[]

src/name_registry.nim

@@ -1,3 +1,4 @@
+import strutils
 import tables
 import debugger
 import types
@@ -14,6 +15,20 @@ proc altNameForKind*(a: string, b: string): string =
   else:
     result = ""
 
+proc normalizeUsedName*(a: string): string =
+  ## a: Nim identifier
+  ## Returns a style-insensitive key for Nim identifiers.
+  var
+    i: int = 0
+    l: int = a.len
+    ch: char
+  result = ""
+  while i < l:
+    ch = a[i]
+    if ch != '_':
+      result.add(toLowerAscii(ch))
+    inc i
+
 proc registerName*(s: var ParserState, a: string, b: string, c: string): string =
   ## s: parser state
   ## a: desired Nim name
@@ -26,34 +41,43 @@ proc registerName*(s: var ParserState, a: string, b: string, c: string): string
     alt: string = ""
     base: string = ""
     count: int = 0
+    baseKey: string = ""
+    altKey: string = ""
+    nameKey: string = ""
   if s.nameMap.hasKey(key):
     result = s.nameMap[key]
     return
-  if not s.usedNames.hasKey(a):
-    s.usedNames[a] = 1
+  baseKey = normalizeUsedName(a)
+  if not s.usedNames.hasKey(baseKey):
+    s.usedNames[baseKey] = 1
     s.nameMap[key] = a
     result = a
     return
   alt = altNameForKind(a, c)
-  if alt.len > 0 and not s.usedNames.hasKey(alt):
-    s.usedNames[alt] = 1
+  if alt.len > 0:
+    altKey = normalizeUsedName(alt)
+  if alt.len > 0 and not s.usedNames.hasKey(altKey):
+    s.usedNames[altKey] = 1
     s.nameMap[key] = alt
     recordCollision(s, a, b, alt)
     result = alt
     return
   base = a
   if alt.len > 0:
     base = alt
-  if s.usedNames.hasKey(base):
-    count = s.usedNames[base]
+  baseKey = normalizeUsedName(base)
+  if s.usedNames.hasKey(baseKey):
+    count = s.usedNames[baseKey]
   else:
     count = 1
   name = base & "_" & $count
-  while s.usedNames.hasKey(name):
+  nameKey = normalizeUsedName(name)
+  while s.usedNames.hasKey(nameKey):
     inc count
     name = base & "_" & $count
-  s.usedNames[base] = count + 1
-  s.usedNames[name] = 1
+    nameKey = normalizeUsedName(name)
+  s.usedNames[baseKey] = count + 1
+  s.usedNames[nameKey] = 1
   s.nameMap[key] = name
   recordCollision(s, a, b, name)
   result = name