支持非标准token

Author: CppCXY

CodeFormatLib/src/CodeFormatLib.cpp

@@ -361,6 +361,47 @@ int set_default_config(lua_State* L)
 	return 0;
 }
 
+int set_nonstandard_symbol(lua_State* L)
+{
+	int top = lua_gettop(L);
+
+	if (top != 2)
+	{
+		return 0;
+	}
+
+	if (lua_isstring(L, 1) && lua_istable(L, 2))
+	{
+		try
+		{
+			std::string type = lua_tostring(L, 1);
+			std::vector<std::string> tokens;
+			lua_pushnil(L);
+			while (lua_next(L, -2) != 0)
+			{
+				auto value = luaToString(L, -1);
+				tokens.push_back(value);
+				lua_pop(L, 1);
+			}
+			lua_settop(L, top);
+
+			LuaCodeFormat::GetInstance().SetSupportNonStandardSymbol(type, tokens);
+			lua_pushboolean(L, true);
+			return 1;
+		}
+		catch (std::exception& e)
+		{
+			std::string err = e.what();
+			lua_settop(L, top);
+			lua_pushboolean(L, false);
+			lua_pushlstring(L, err.c_str(), err.size());
+			return 2;
+		}
+	}
+
+	return 0;
+}
+
 int spell_load_dictionary_from_path(lua_State* L)
 {
 	int top = lua_gettop(L);

CodeFormatLib/src/LuaCodeFormat.cpp

@@ -14,7 +14,8 @@ LuaCodeFormat& LuaCodeFormat::GetInstance()
 
 LuaCodeFormat::LuaCodeFormat()
 	: _defaultOptions(std::make_shared<LuaCodeStyleOptions>()),
-	  _codeSpellChecker(std::make_shared<CodeSpellChecker>())
+	  _codeSpellChecker(std::make_shared<CodeSpellChecker>()),
+	  _customParser(std::make_shared<LuaCustomParser>())
 {
 }
 
@@ -63,6 +64,14 @@ void LuaCodeFormat::SetDefaultCodeStyle(ConfigMap& configMap)
 	}
 }
 
+void LuaCodeFormat::SetSupportNonStandardSymbol(const std::string& tokenType,const std::vector<std::string>& tokens)
+{
+	if (tokenType.size() == 1)
+	{
+		_customParser->SetTokens(tokenType.front(), tokens); 
+	}
+}
+
 void LuaCodeFormat::LoadSpellDictionary(const std::string& path)
 {
 	_codeSpellChecker->LoadDictionary(path);
@@ -76,6 +85,9 @@ void LuaCodeFormat::LoadSpellDictionaryFromBuffer(const std::string& buffer)
 std::string LuaCodeFormat::Reformat(const std::string& uri, std::string&& text, ConfigMap& configMap)
 {
 	auto parser = LuaParser::LoadFromBuffer(std::move(text));
+	if (_customParser->IsSupportCustomTokens()) {
+		parser->GetTokenParser()->SetCustomParser(_customParser);
+	}
 	parser->BuildAstWithComment();
 
 	if (!parser->GetErrors().empty())
@@ -93,6 +105,9 @@ std::string LuaCodeFormat::RangeFormat(const std::string& uri, LuaFormatRange& r
                                        ConfigMap& configMap)
 {
 	auto parser = LuaParser::LoadFromBuffer(std::move(text));
+	if (_customParser->IsSupportCustomTokens()) {
+		parser->GetTokenParser()->SetCustomParser(_customParser);
+	}
 	parser->BuildAstWithComment();
 
 	if (!parser->GetErrors().empty())
@@ -111,6 +126,9 @@ std::string LuaCodeFormat::RangeFormat(const std::string& uri, LuaFormatRange& r
 std::pair<bool, std::vector<LuaDiagnosisInfo>> LuaCodeFormat::Diagnose(const std::string& uri, std::string&& text)
 {
 	auto parser = LuaParser::LoadFromBuffer(std::move(text));
+	if (_customParser->IsSupportCustomTokens()) {
+		parser->GetTokenParser()->SetCustomParser(_customParser);
+	}
 	parser->BuildAstWithComment();
 
 	if (!parser->GetErrors().empty())
@@ -137,6 +155,10 @@ std::vector<LuaDiagnosisInfo> LuaCodeFormat::SpellCheck(const std::string& uri,
                                                         const CodeSpellChecker::CustomDictionary& tempDict)
 {
 	auto parser = LuaParser::LoadFromBuffer(std::move(text));
+	if (_customParser->IsSupportCustomTokens()) {
+		parser->GetTokenParser()->SetCustomParser(_customParser);
+	}
+	parser->GetTokenParser()->Parse();
 
 	auto options = GetOptions(uri);
 

CodeFormatLib/src/LuaCodeFormat.h

@@ -21,6 +21,7 @@ class LuaCodeFormat
 	void UpdateCodeStyle(const std::string& workspaceUri, const std::string& configPath);
 	void RemoveCodeStyle(const std::string& workspaceUri);
 	void SetDefaultCodeStyle(ConfigMap& configMap);
+	void SetSupportNonStandardSymbol(const std::string& tokenType, const std::vector<std::string>& tokens);
 
 	void LoadSpellDictionary(const std::string& path);
 
@@ -46,4 +47,6 @@ class LuaCodeFormat
 	std::shared_ptr<LuaCodeStyleOptions> _defaultOptions;
 
 	std::shared_ptr<CodeSpellChecker> _codeSpellChecker;
+
+	std::shared_ptr<LuaCustomParser> _customParser;
 };

LuaParser/src/LuaCustomParser.cpp

@@ -98,10 +98,25 @@ LuaTokenType LuaCustomParser::Lex(std::string_view source, std::size_t start, st
 	return it->second;
 }
 
-void LuaCustomParser::SetEqToken(std::vector<std::string>& customTokens)
+void LuaCustomParser::SetTokens(LuaTokenType type, const std::vector<std::string>& customTokens)
 {
+	for (auto it = _customTokens.begin(); it != _customTokens.end();)
+	{
+		if (it->second == type)
+		{
+			_customTokens.erase(it++);
+			continue;
+		}
+		++it;
+	}
+
 	for (auto& token : customTokens)
 	{
-		_customTokens.insert({token, '='});
+		_customTokens.insert({token, type});
 	}
 }
+
+bool LuaCustomParser::IsSupportCustomTokens() const
+{
+	return !_customTokens.empty();
+}

LuaParser/src/LuaParser.cpp

@@ -33,15 +33,15 @@ std::shared_ptr<LuaParser> LuaParser::LoadFromFile(std::string_view filename)
 std::shared_ptr<LuaParser> LuaParser::LoadFromBuffer(std::string&& buffer)
 {
 	auto file = std::make_shared<LuaFile>("chunk", std::move(buffer));
-
+	
 	auto tokenParser = std::make_shared<LuaTokenParser>(file);
-	tokenParser->Parse();
 
 	return std::make_shared<LuaParser>(tokenParser);
 }
 
 void LuaParser::BuildAst()
 {
+	_tokenParser->Parse();
 	_chunkAstNode = CreateAstNode(LuaAstNodeType::Chunk);
 	try
 	{

include/LuaParser/LuaCustomParser.h

@@ -5,14 +5,16 @@
 #include <string_view>
 #include "LuaTokenType.h"
 #include <map>
+#include <set>
 
 class LuaCustomParser
 {
 public:
 	LuaTokenType Lex(std::string_view source, std::size_t start, std::size_t& consumeSize);
 
-	void SetEqToken(std::vector<std::string>& customTokens);
-private:
+	void SetTokens(LuaTokenType type, const std::vector<std::string>& customTokens);
 
+	bool IsSupportCustomTokens() const;
+private:
 	std::map<std::string, LuaTokenType, std::less<>> _customTokens;
 };