Various BugFixes

Fixed the deletion of all previous tokens when there is a multiline comment within a description delimited by quotation marks.
Removed RESERVED from keyword list as it is not necessary there.
S19 Parser fixed when there is a malformed entry that specifies more databytes than acutally exist which would previously cause an out of bounds error.

Author: asap2Go

a2l/a2l.go

@@ -21,16 +21,16 @@ var (
 	numProc = runtime.NumCPU() * 2
 )
 
-//A2L is the main struct returned by the a2l package.
-//it contains all datatypes parsed from the .a2l file
+// A2L is the main struct returned by the a2l package.
+// it contains all datatypes parsed from the .a2l file
 type A2L struct {
 	Asap2Version asap2Version
 	A2mlVersion  a2mlVersion
 	Project      Project
 }
 
-//ParseFromFile is the main exported function to be called from a2l package.
-//it takes an .a2l file and parses it
+// ParseFromFile is the main exported function to be called from a2l package.
+// it takes an .a2l file and parses it
 func ParseFromFile(filepath string) (A2L, error) {
 	var err error
 	var text string
@@ -64,8 +64,8 @@ func ParseFromFile(filepath string) (A2L, error) {
 	return a, nil
 }
 
-//parseA2l handles the parsing of the a2l struct.
-//as opposed to ParseFromFile which also handles creation of the tokenizer and file reading, etc.
+// parseA2l handles the parsing of the a2l struct.
+// as opposed to ParseFromFile which also handles creation of the tokenizer and file reading, etc.
 func parseA2l(tok *tokenGenerator) (A2L, error) {
 	a2l := A2L{}
 	var err error
@@ -109,7 +109,7 @@ forLoop:
 	return a2l, err
 }
 
-//readFileToString opens and reads a file, then returns a string value
+// readFileToString opens and reads a file, then returns a string value
 func readFileToString(filepath string) (string, error) {
 	bytesString, err := os.ReadFile(filepath)
 	if err != nil {

a2l/a2l_test.go

@@ -32,17 +32,38 @@ func FuzzParseA2L(f *testing.F) {
 	a2lPath := "testing/ASAP2_Demo_V171_allKeywords.a2l"
 	text, _ := readFileToString(a2lPath)
 	f.Add(text)
+	var errList []error
 
 	f.Fuzz(func(t *testing.T, orig string) {
 		tg, err := buildTokenGeneratorFromString(orig)
 		if err != nil {
-			log.Err(err).Msg("could not create tokens from a2l file")
-			log.Err(err).Msg(orig)
+			exists := false
+			for _, e := range errList {
+				if err == e {
+					exists = true
+					break
+				}
+			}
+			if !exists {
+				errList = append(errList, err)
+				log.Err(err).Msg("could not create tokens from a2l file")
+				log.Err(err).Msg(orig)
+			}
 		}
 		a, err := parseA2l(&tg)
 		if err != nil {
-			log.Err(err).Msg("failed parsing " + a.Project.Name + " with error:")
-			log.Err(err).Msg(orig)
+			exists := false
+			for _, e := range errList {
+				if err == e {
+					exists = true
+					break
+				}
+			}
+			if !exists {
+				errList = append(errList, err)
+				log.Err(err).Msg("failed parsing " + a.Project.Name + " with error:")
+				//dumps the a2l file into the log: log.Err(err).Msg(orig)
+			}
 		}
 	})
 }
@@ -65,7 +86,7 @@ func BenchmarkParseFromFile(b *testing.B) {
 	}
 }
 
-//configureLogger adds a file logger, resets previous log file and does some formatting
+// configureLogger adds a file logger, resets previous log file and does some formatting
 func configureLogger() error {
 	var err error
 	var file *os.File

a2l/testdata/fuzz/FuzzParseA2L/290378f40701d315e4488817302849612f93373fd409f18e9bc7c479f8abcd95

@@ -1,4 +1,4 @@
-ASAP2_VERSION 1 71
+ASAP2_VERSION 1 71
 A2ML_VERSION 1 31
 /begin PROJECT ASAP2_Example ""
 
@@ -2859,7 +2859,7 @@ A2ML_VERSION 1 31
   SYSTEM_CONSTANT "CONTROLLERx constant1" "0.33"
   SYSTEM_CONSTANT "CONTROLLERx constant2" "2.79"
   /begin CALIBRATION_METHOD
-    „InCircuit“
+    „InCircuit“
     2
     /begin CALIBRATION_HANDLE
       0x10000 /* start address of pointer table */
@@ -2870,6 +2870,8 @@ A2ML_VERSION 1 31
       CALIBRATION_HANDLE_TEXT "YourTextHere"
     /end CALIBRATION_HANDLE
   /end CALIBRATION_METHOD
+  SYSTEM_CONSTANT "CONTROLLERx constant3" "0                  /*blabalsdjkkasld as dsa d*/"
+  SYSTEM_CONSTANT "CONTROLLERx constant4" "0"  
 /end MOD_PAR
 
 
@@ -5729,8 +5731,8 @@ A2ML_VERSION 1 31
       NO_COMPU_METHOD
       10
       0
-      25
-    /end TYPEDEF_AXIS
+	    25
+	/end TYPEDEF_AXIS
 
     /begin INSTANCE
       Instance1

a2l/testing/ASAP2_Demo_V171_allKeywords.a2l

@@ -49,6 +49,10 @@ func buildTokenGeneratorFromString(str string) (tokenGenerator, error) {
 	str = strings.TrimFunc(str, func(r rune) bool {
 		return !unicode.IsGraphic(r)
 	})
+
+	//replace escaped quotation marks
+	str = strings.ReplaceAll(str, `\"`, `'`)
+
 	//Split text file into lines and the lines into words separated by whitespace
 	var locTokens []chan []string
 	tokenList = make([]string, 0, expectedNumberOfTokens)
@@ -114,7 +118,8 @@ func buildTokenList(str [][]string) []string {
 			return tl
 		}
 		if t != emptyToken {
-			if strings.Contains(t, "*/") {
+			if strings.Contains(t, "*/") && !strings.Contains(t, "*/") {
+				log.Warn().Msg("Triggered reset! at token: " + t)
 				//make sure that the goroutine did not start in the middle of a multiline comment.
 				//if so then delete all previous tokens as they are part of the comment.
 				//the buildNextValidToken routine only checks for the start of a multiline comment.
@@ -210,7 +215,7 @@ start:
 	}
 }
 
-//skipLineComment if a "//" is detected the token generator moves to the next line (currentOuterIndex++)
+// skipLineComment if a "//" is detected the token generator moves to the next line (currentOuterIndex++)
 func skipLineComment(currentOuterIndex *int, currentInnerIndex *int, str [][]string) (string, error) {
 	if strings.Contains(str[*currentOuterIndex][*currentInnerIndex], beginLineCommentToken) {
 		//move to the next line and reset inner index to 0 (first word in new line)
@@ -227,8 +232,8 @@ func skipLineComment(currentOuterIndex *int, currentInnerIndex *int, str [][]str
 	}
 }
 
-//skipLineComment if a "/*" is detected then token generator gets the next token until a "*/" is reached.
-//it will then return the next raw value token after the comment
+// skipLineComment if a "/*" is detected then token generator gets the next token until a "*/" is reached.
+// it will then return the next raw value token after the comment
 func skipMultilineComment(currentOuterIndex *int, currentInnerIndex *int, str [][]string) (string, error) {
 	var err error
 	t := str[*currentOuterIndex][*currentInnerIndex]
@@ -289,10 +294,10 @@ func getTextInQuotationMarks(currentOuterIndex *int, currentInnerIndex *int, str
 	}
 }
 
-//moveToNextRawValue returns back the next valid, white space separated value.
-//in case the line ends it restarts on the next line
-//in case there are no lines left and no words within the last line it will return an empty token
-//which is used to signal the parser that the eof has been reached.
+// moveToNextRawValue returns back the next valid, white space separated value.
+// in case the line ends it restarts on the next line
+// in case there are no lines left and no words within the last line it will return an empty token
+// which is used to signal the parser that the eof has been reached.
 func moveToNextRawValue(currentOuterIndex *int, currentInnerIndex *int, str [][]string) (string, error) {
 	//If there are still tokens left in the current line
 	if len(str[*currentOuterIndex])-1 > *currentInnerIndex && len(str[*currentOuterIndex]) > 0 {
@@ -320,17 +325,17 @@ func moveToNextRawValue(currentOuterIndex *int, currentInnerIndex *int, str [][]
 	}
 }
 
-//isKeyword is used in the matrixDim parser to detected when there are no dimensions left to parse
-//this is necessary because not every version of the a2l standard has a clear rule about how many dimensions should be expected
-//or are necessary to define ("1 0 0" and "1" are both valid descriptions for a curve)
+// isKeyword is used in the matrixDim parser to detected when there are no dimensions left to parse
+// this is necessary because not every version of the a2l standard has a clear rule about how many dimensions should be expected
+// or are necessary to define ("1 0 0" and "1" are both valid descriptions for a curve)
 func isKeyword(str string) bool {
 	//look whether the given string is contained in the list of
 	//valid a2l tokens as defined in tokens.go
 	_, exists := keywordMap[str]
 	return exists
 }
 
-//getTwoWordedToken handles keywords that contain a / like e.g. "/begin CHARACTERISTIC"
+// getTwoWordedToken handles keywords that contain a / like e.g. "/begin CHARACTERISTIC"
 func getTwoWordedToken(currentOuterIndex *int, currentInnerIndex *int, str [][]string) (string, error) {
 	var err error
 	t := str[*currentOuterIndex][*currentInnerIndex]

a2l/token_generator.go

@@ -367,9 +367,9 @@ const variablesToken = "VARIABLES"
 const versionToken = "VERSION"
 const wordToken = "WORD"
 
-//keywordMap is used in the isKeyword-Funtion.
-//Its usage is to efficiently check whether a token is also a keyword.
-//Note: enum values and Standalone Keywords like "READ_ONLY" are excluded here.
+// keywordMap is used in the isKeyword-Funtion.
+// Its usage is to efficiently check whether a token is also a keyword.
+// Note: enum values and Standalone Keywords like "READ_ONLY" are excluded here.
 var keywordMap = map[string]uint8{
 	"A2ML_VERSION":                    0,
 	"ADDR_EPK":                        0,
@@ -609,7 +609,6 @@ var keywordMap = map[string]uint8{
 	"PROJECT_NO":                      0,
 	"REF_MEMORY_SEGMENT":              0,
 	"REF_UNIT":                        0,
-	"RESERVED":                        0,
 	"RIGHT_SHIFT":                     0,
 	"RIP_ADDR_4":                      0,
 	"RIP_ADDR_5":                      0,

a2l/tokens.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -32,6 +33,7 @@ func FuzzParseHex(f *testing.F) {
 	zerolog.SetGlobalLevel(zerolog.WarnLevel)
 	hexPath := "testing/ASAP2_Demo_V171.hex"
 	text, _ := readFileToString(hexPath)
+	var errList []error
 	f.Add(text)
 
 	f.Fuzz(func(t *testing.T, orig string) {
@@ -41,11 +43,21 @@ func FuzzParseHex(f *testing.F) {
 			//in case unix line terminator is used.
 			lines = strings.Split(orig, "\n")
 		}
-		_, err := parseHex(lines)
-		if err != nil && err.Error() != "invalid checksums detected" {
-			log.Err(err).Msg("could not parse hex-file")
-			log.Err(err).Msg(orig)
-			t.Error()
+		h, err := parseHex(lines)
+		if err != nil {
+			exists := false
+			for _, e := range errList {
+				if err == e {
+					exists = true
+					break
+				}
+			}
+			if !exists {
+				errList = append(errList, err)
+				fmt.Println(len(h))
+				log.Err(err).Msg("could not parse hex-file with length " + strconv.Itoa(len(h)))
+				log.Err(err).Msg(orig)
+			}
 		}
 	})
 }

ihex32/hex_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -32,6 +33,7 @@ func FuzzParseHex(f *testing.F) {
 	zerolog.SetGlobalLevel(zerolog.WarnLevel)
 	hexPath := "testing/ASAP2_Demo_V171.s19"
 	text, _ := readFileToString(hexPath)
+	var errList []error
 	f.Add(text)
 
 	f.Fuzz(func(t *testing.T, orig string) {
@@ -41,11 +43,21 @@ func FuzzParseHex(f *testing.F) {
 			//in case unix line terminator is used.
 			lines = strings.Split(orig, "\n")
 		}
-		_, err := parseHex(lines)
-		if err != nil && err.Error() != "invalid checksums detected" {
-			log.Err(err).Msg("could not parse hex-file")
-			log.Err(err).Msg(orig)
-			t.Error()
+		h, err := parseHex(lines)
+		if err != nil {
+			exists := false
+			for _, e := range errList {
+				if err == e {
+					exists = true
+					break
+				}
+			}
+			if !exists {
+				errList = append(errList, err)
+				fmt.Println(len(h))
+				log.Err(err).Msg("could not parse s19-file with length " + strconv.Itoa(len(h)))
+				log.Err(err).Msg(orig)
+			}
 		}
 	})
 }

ihex32/testdata/fuzz/FuzzParseFromFile/b1c5ee6ddaa28de61bbe1205a5bf51459dbfa0d1fd12b7b4150665ffa0010e56

@@ -26,7 +26,7 @@ type record struct {
 func parseRecord(line string) (*record, error) {
 	r := record{}
 	r.recordType = line[1:2]
-	if line[0] == beginLineToken[0] && r.recordType == "3" {
+	if line[0] == beginLineToken[0] && r.recordType == "3" && len(line) >= 12 {
 		r.byteCount = line[2:4]
 		r.addressField = line[4:12]
 		for i := 12; i < len(line)-3; i += 2 {