Proper input sanitation for a2l files and removed detailed parsing of if data

remove escaped characters while importing a2l file because some A2L files declare Tab-Verb Conversions like
0 "\\"
which would otherwise lead to an error in the tokenizer while trying to detect a text in quotation marks.

As it is possible to declare custom datastructres in the A2ML Section which may be used in the if data blocks the parsing for the if data structures is not checking for unexpected keywords anymore as this would lead to potential errors while the A2ML declarations aren't implemented yet.

Author: asap2Go

a2l/a2l.go

@@ -4,8 +4,11 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"regexp"
 	"runtime"
+	"strings"
 	"time"
+	"unicode"
 
 	"github.com/rs/zerolog/log"
 )
@@ -18,7 +21,7 @@ var (
 	//numProc is used to set the amount of goroutines in case useMultithreading is true.
 	//numProc = runtime.NumCPU() * 2 has proven to be reliably fast for different cpu models.
 	//factors above 4 will generally lead to severe performance degredation due to channel and locking overhead.
-	numProc = runtime.NumCPU() * 2
+	numProc = runtime.NumCPU()
 )
 
 // A2L is the main struct returned by the a2l package.
@@ -115,6 +118,19 @@ func readFileToString(filepath string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	text := string(bytesString)
-	return text, nil
+	cleanedStr := cleanBytesString(bytesString)
+	return cleanedStr, nil
+}
+
+func cleanBytesString(bs []byte) string {
+	//remove escaped characters
+	re := regexp.MustCompile(`\\.`)
+	escapedBytesString := re.ReplaceAll(bs, []byte("$1"))
+	//convert byte array to string
+	str := string(escapedBytesString)
+	//remove unprintable chars at the start and end of the a2l file
+	str = strings.TrimFunc(str, func(r rune) bool {
+		return !unicode.IsGraphic(r)
+	})
+	return str
 }

a2l/if_data.go

@@ -27,10 +27,6 @@ forLoop:
 			id.dataSet = true
 			log.Info().Msg("ifData data successfully parsed")
 			break forLoop
-		} else if isKeyword(tok.current()) {
-			err = errors.New("unexpected token " + tok.current())
-			log.Err(err).Msg("ifData could not be parsed")
-			break forLoop
 		} else if !id.nameSet {
 			id.name = tok.current()
 			id.nameSet = true

a2l/testdata/fuzz/FuzzParseA2L/290378f40701d315e4488817302849612f93373fd409f18e9bc7c479f8abcd95

@@ -5,7 +5,6 @@ import (
 	"strconv"
 	"strings"
 	"sync/atomic"
-	"unicode"
 
 	"github.com/rs/zerolog/log"
 )
@@ -45,11 +44,6 @@ func buildTokenGeneratorFromString(str string) (tokenGenerator, error) {
 	//initialize moduleCount to zero
 	moduleCount = 0
 
-	//remove unprintable chars at the start and end of the a2l file
-	str = strings.TrimFunc(str, func(r rune) bool {
-		return !unicode.IsGraphic(r)
-	})
-
 	//replace escaped quotation marks
 	str = strings.ReplaceAll(str, `\"`, `'`)
 

a2l/token_generator.go

@@ -97,7 +97,7 @@ var (
 	}
 )
 
-//parseHex parses a hex-file given as an slice of strings and return a hex struct containing the data in byte form with all the addresses attached.
+// parseHex parses a hex-file given as an slice of strings and return a hex struct containing the data in byte form with all the addresses attached.
 func parseHex(lines []string) (map[uint32]byte, error) {
 	var h map[uint32]byte
 	var err error
@@ -200,7 +200,7 @@ func parseHex(lines []string) (map[uint32]byte, error) {
 	return h, nil
 }
 
-//readFileToString returns a string by reading a document from a given filepath.
+// readFileToString returns a string by reading a document from a given filepath.
 func readFileToString(filepath string) (string, error) {
 	bytesString, err := os.ReadFile(filepath)
 	if err != nil {
@@ -211,7 +211,7 @@ func readFileToString(filepath string) (string, error) {
 	return text, err
 }
 
-//ParseFromFile parses a hex file from a given filepath and return a hex struct containing all data as bytes with their addresses.
+// ParseFromFile parses a hex file from a given filepath and return a hex struct containing all data as bytes with their addresses.
 func ParseFromFile(filepath string) (map[uint32]byte, error) {
 	var h map[uint32]byte
 	var text string
@@ -236,7 +236,7 @@ func ParseFromFile(filepath string) (map[uint32]byte, error) {
 	return h, nil
 }
 
-//parseRecordRoutine calls the parseRecord method for each line given and return them via a channel.
+// parseRecordRoutine calls the parseRecord method for each line given and return them via a channel.
 func parseRecordRoutine(c chan []*record, lines []string) {
 	var recs []*record
 forLoop:
@@ -254,7 +254,7 @@ forLoop:
 	close(c)
 }
 
-//hexToByte converts a two character hexString to a single byte. fails if input is too long or not valid hex.
+// hexToByte converts a two character hexString to a single byte. fails if input is too long or not valid hex.
 func hexToByte(str string) (byte, error) {
 	var err error
 	var b byte
@@ -267,7 +267,7 @@ func hexToByte(str string) (byte, error) {
 	return b, err
 }
 
-//hexToByteSlice converts at least a four character hexString to a slice of several bytes. fails if input is too short or not valid hex.
+// hexToByteSlice converts at least a four character hexString to a slice of several bytes. fails if input is too short or not valid hex.
 func hexToByteSlice(hexVal string) ([]byte, error) {
 	decoded, err := hex.DecodeString(hexVal)
 	if err != nil {

ihex32/hex.go

@@ -14,7 +14,7 @@ import (
 
 var (
 	//numProc controls the number of goRoutines that are run for the parallel parts of the programm.
-	numProc = runtime.NumCPU() * 2
+	numProc = runtime.NumCPU()
 	//validateChecksums controls whether checksum validation will be executed or skipped.
 	//in case checksum validation encounters an incorrect checksum it will stop the parser and throw an error.
 	validateChecksums = true
@@ -97,7 +97,7 @@ var (
 	}
 )
 
-//parseHex parses a hex-file given as an slice of strings and return a hex struct containing the data in byte form with all the addresses attached.
+// parseHex parses a hex-file given as an slice of strings and return a hex struct containing the data in byte form with all the addresses attached.
 func parseHex(lines []string) (map[uint32]byte, error) {
 	var h map[uint32]byte
 	var err error
@@ -189,7 +189,7 @@ func parseHex(lines []string) (map[uint32]byte, error) {
 	return h, nil
 }
 
-//readFileToString returns a string by reading a document from a given filepath.
+// readFileToString returns a string by reading a document from a given filepath.
 func readFileToString(filepath string) (string, error) {
 	bytesString, err := os.ReadFile(filepath)
 	if err != nil {
@@ -200,7 +200,7 @@ func readFileToString(filepath string) (string, error) {
 	return text, err
 }
 
-//ParseFromFile parses a hex file from a given filepath and return a hex struct containing all data as bytes with their addresses.
+// ParseFromFile parses a hex file from a given filepath and return a hex struct containing all data as bytes with their addresses.
 func ParseFromFile(filepath string) (map[uint32]byte, error) {
 	var h map[uint32]byte
 	var text string
@@ -225,7 +225,7 @@ func ParseFromFile(filepath string) (map[uint32]byte, error) {
 	return h, nil
 }
 
-//parseRecordRoutine calls the parseRecord method for each line given and return them via a channel.
+// parseRecordRoutine calls the parseRecord method for each line given and return them via a channel.
 func parseRecordRoutine(c chan []*record, lines []string) {
 	var recs []*record
 forLoop:
@@ -243,7 +243,7 @@ forLoop:
 	close(c)
 }
 
-//hexToByte converts a two character hexString to a single byte. fails if input is too long or not valid hex.
+// hexToByte converts a two character hexString to a single byte. fails if input is too long or not valid hex.
 func hexToByte(str string) (byte, error) {
 	var err error
 	var b byte
@@ -256,7 +256,7 @@ func hexToByte(str string) (byte, error) {
 	return b, err
 }
 
-//hexToByteSlice converts at least a four character hexString to a slice of several bytes. fails if input is too short or not valid hex.
+// hexToByteSlice converts at least a four character hexString to a slice of several bytes. fails if input is too short or not valid hex.
 func hexToByteSlice(hexVal string) ([]byte, error) {
 	decoded, err := hex.DecodeString(hexVal)
 	if err != nil {