tagFormat develop

Author: CppCXY

LuaParser/CMakeLists.txt

@@ -32,6 +32,7 @@ target_sources(LuaParser PUBLIC
 	${LuaParser_SOURCE_DIR}/src/LuaFile.cpp
 	${LuaParser_SOURCE_DIR}/src/LuaIdentify.cpp
 	${LuaParser_SOURCE_DIR}/src/LuaCustomParser.cpp
+	${LuaParser_SOURCE_DIR}/src/LuaDocParser.cpp
 )
 
 target_link_libraries(LuaParser Util)

LuaParser/src/LuaDocParser.cpp

@@ -0,0 +1,2 @@
+#include "LuaParser/LuaDocParser.h"
+

LuaParser/src/LuaFile.cpp

@@ -133,6 +133,7 @@ bool LuaFile::IsEmptyLine(int line)
 void LuaFile::PushLine(int offset)
 {
 	_lineOffsetVec.push_back(offset);
+	_linenumber++;
 }
 
 bool LuaFile::OnlyEmptyCharBefore(int offset)

LuaParser/src/LuaParser.cpp

@@ -33,7 +33,7 @@ std::shared_ptr<LuaParser> LuaParser::LoadFromFile(std::string_view filename)
 std::shared_ptr<LuaParser> LuaParser::LoadFromBuffer(std::string&& buffer)
 {
 	auto file = std::make_shared<LuaFile>("chunk", std::move(buffer));
-	
+
 	auto tokenParser = std::make_shared<LuaTokenParser>(file);
 
 	return std::make_shared<LuaParser>(tokenParser);
@@ -205,7 +205,7 @@ void LuaParser::StatementList(std::shared_ptr<LuaAstNode> blockNode)
 
 void LuaParser::Statement(std::shared_ptr<LuaAstNode> blockNode)
 {
-	switch (_tokenParser->Current().TokenType)
+	switch (_tokenParser->CurrentWithComment().TokenType)
 	{
 	case ';':
 		{
@@ -276,6 +276,18 @@ void LuaParser::Statement(std::shared_ptr<LuaAstNode> blockNode)
 			GotoStatement(blockNode);
 			break;
 		}
+	case TK_LONG_COMMENT:
+	case TK_SHORT_COMMENT:
+	case TK_SHEBANG:
+		{
+			Comment(blockNode);
+			break;
+		}
+	case TK_DOC_COMMENT:
+		{
+			_tokenParser->Next();
+			break;
+		}
 	default:
 		{
 			ExpressionStatement(blockNode);
@@ -553,6 +565,35 @@ void LuaParser::AssignStatement(std::shared_ptr<LuaAstNode> expressionListNode,
 	}
 }
 
+void LuaParser::Comment(std::shared_ptr<LuaAstNode> block)
+{
+	std::shared_ptr<LuaAstNode> commentAst = CreateAstNode(LuaAstNodeType::Comment);
+	if (_tokenParser->Current().TokenType == TK_SHORT_COMMENT)
+	{
+		commentAst->AddChild(CreateAstNodeFromCurrentToken(LuaAstNodeType::ShortComment));
+	}
+	else if (_tokenParser->Current().TokenType == TK_LONG_COMMENT)
+	{
+		commentAst->AddChild(CreateAstNodeFromCurrentToken(LuaAstNodeType::LongComment));
+	}
+	else
+	{
+		commentAst->AddChild(CreateAstNodeFromCurrentToken(LuaAstNodeType::ShebangComment));
+	}
+	_tokenParser->Next();
+
+	block->AddChild(commentAst);
+}
+
+void LuaParser::DocStatement(std::shared_ptr<LuaAstNode> blockNode)
+{
+
+	// 退化情况
+	std::shared_ptr<LuaAstNode> commentAst = CreateAstNode(LuaAstNodeType::Comment);
+	commentAst->AddChild(CreateAstNodeFromCurrentToken(LuaAstNodeType::ShortComment));
+
+	blockNode->AddChild(commentAst);
+}
 
 void LuaParser::Condition(std::shared_ptr<LuaAstNode> parent)
 {

LuaParser/src/LuaTokenParser.cpp

@@ -68,16 +68,8 @@ bool LuaTokenParser::Parse()
 
 		if (!text.empty())
 		{
-			if (type == TK_LONG_COMMENT || type == TK_SHEBANG || type == TK_SHORT_COMMENT)
-			{
-				_commentTokens.emplace_back(type, text, TextRange(
-					                            static_cast<int>(_buffStart), static_cast<int>(_buffIndex)));
-			}
-			else
-			{
-				_tokens.emplace_back(type, text, TextRange(
-					                     static_cast<int>(_buffStart), static_cast<int>(_buffIndex)));
-			}
+			_tokens.emplace_back(type, text, TextRange(
+				static_cast<int>(_buffStart), static_cast<int>(_buffIndex)));
 		}
 		else
 		{
@@ -119,6 +111,37 @@ LuaToken& LuaTokenParser::LookAhead()
 }
 
 LuaToken& LuaTokenParser::Current()
+{
+	if (_currentIndex < _tokens.size())
+	{
+		do
+		{
+			switch (_tokens[_currentIndex].TokenType)
+			{
+			case TK_DOC_COMMENT:
+			case TK_SHORT_COMMENT:
+			case TK_LONG_COMMENT:
+			case TK_SHEBANG:
+				{
+					_commentTokens.push_back(_tokens[_currentIndex]);
+					Next();
+					break;
+				}
+			default:
+				{
+					goto endLoop;
+				}
+			}
+		}
+		while (true);
+	endLoop:
+		return _tokens[_currentIndex];
+	}
+
+	return _eosToken;
+}
+
+LuaToken& LuaTokenParser::CurrentWithComment()
 {
 	if (_currentIndex < _tokens.size())
 	{
@@ -228,6 +251,7 @@ LuaTokenType LuaTokenParser::Lex()
 				// is comment
 				SaveAndNext();
 
+				LuaTokenType type = TK_SHORT_COMMENT;
 				if (GetCurrentChar() == '[')
 				{
 					std::size_t sep = SkipSep();
@@ -237,14 +261,19 @@ LuaTokenType LuaTokenParser::Lex()
 						return TK_LONG_COMMENT;
 					}
 				}
+				else if (GetCurrentChar() == '-')
+				{
+					SaveAndNext();
+					type = TK_DOC_COMMENT;
+				}
 
 				// is short comment
 				while (!CurrentIsNewLine() && GetCurrentChar() != EOZ)
 				{
 					SaveAndNext();
 				}
 
-				return TK_SHORT_COMMENT;
+				return type;
 			}
 		case '[':
 			{
@@ -343,7 +372,7 @@ LuaTokenType LuaTokenParser::Lex()
 			}
 		case '"':
 		case '\'':
-			// extend support
+		// extend support
 		case '`':
 			{
 				ReadString(ch);
@@ -812,6 +841,17 @@ bool LuaTokenParser::IsReserved(std::string_view text)
 	return LuaReserved.find(text) != LuaReserved.end();
 }
 
+bool LuaTokenParser::IsInlineComment()
+{
+	if (_tokens.empty())
+	{
+		return false;
+	}
+
+	auto lastTokenLine = _file->GetLine(_tokens.back().Range.EndOffset);
+	return _file->GetLine(static_cast<int>(_buffStart)) == lastTokenLine;
+}
+
 void LuaTokenParser::PushLuaError(std::string_view message, TextRange range)
 {
 	_errors.emplace_back(message, range);

include/LuaParser/LuaDocParser.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <string_view>
+
+class LuaDocParser
+{
+public:
+	LuaDocParser(std::string_view doc);
+	~LuaDocParser();
+
+private:
+	std::string_view _source;
+
+};

include/LuaParser/LuaParser.h

@@ -79,6 +79,10 @@ class LuaParser
 
 	void AssignStatement(std::shared_ptr<LuaAstNode> expressionListNode, std::shared_ptr<LuaAstNode> assignStatementNode);
 
+	void Comment(std::shared_ptr<LuaAstNode> block);
+
+	void DocStatement(std::shared_ptr<LuaAstNode> blockNode);
+
 	void ForNumber(std::shared_ptr<LuaAstNode> forStatement);
 
 	void ForList(std::shared_ptr<LuaAstNode> forStatement);

include/LuaParser/LuaTokenParser.h

@@ -28,6 +28,8 @@ class LuaTokenParser
 
 	LuaToken& Current();
 
+	LuaToken& CurrentWithComment();
+
 	int LastValidOffset();
 
 	std::string_view GetSource();
@@ -84,6 +86,8 @@ class LuaTokenParser
 
 	bool IsReserved(std::string_view text);
 
+	bool IsInlineComment();
+
 	void PushLuaError(std::string_view message, TextRange range);
 
 	int _linenumber;

include/LuaParser/LuaTokenTypeDetail.h

@@ -47,6 +47,7 @@ enum LuaTokenTypeDetail : int
 	TK_STRING,
 	// custom
 	TK_SHORT_COMMENT,// 短注释
+	TK_DOC_COMMENT, // emmylua doc
 	TK_LONG_COMMENT, //长注释
 	TK_SHEBANG, //shebang
 	TK_UNKNOWN //对自定义tokenParser来说是未知