Skip to content

Commit 9c04037

Browse files
authored
Merge pull request swiftlang#21762 from akyrtzi/syntax-parser-clib
Introduce C parser library
2 parents 52aa0c1 + 70c954e commit 9c04037

22 files changed

+1003
-29
lines changed

cmake/modules/SwiftComponents.cmake

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
# * toolchain-dev-tools -- install development tools useful in a shared toolchain
6666
# * dev -- headers and libraries required to use Swift compiler as a library.
6767
set(_SWIFT_DEFINED_COMPONENTS
68-
"autolink-driver;compiler;clang-builtin-headers;clang-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;stdlib;stdlib-experimental;sdk-overlay;editor-integration;tools;testsuite-tools;toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers")
68+
"autolink-driver;compiler;clang-builtin-headers;clang-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;stdlib;stdlib-experimental;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers")
6969

7070
macro(swift_configure_components)
7171
# Set the SWIFT_INSTALL_COMPONENTS variable to the default value if it is not passed in via -D
Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
//===--- SwiftSyntaxParser.h - C API for Swift Syntax Parsing -----*- C -*-===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
//
13+
// This C API is primarily intended to serve as the Swift parsing component
14+
// of SwiftSyntax (https://github.com/apple/swift-syntax).
15+
//
16+
//===----------------------------------------------------------------------===//
17+
18+
#ifndef SWIFT_C_SYNTAX_PARSER_H
19+
#define SWIFT_C_SYNTAX_PARSER_H
20+
21+
#include <stdbool.h>
22+
#include <stddef.h>
23+
#include <stdint.h>
24+
25+
/// The version constants for the SwiftSyntaxParser C API.
26+
/// SWIFTPARSE_VERSION_MINOR should increase when there are API additions.
27+
/// SWIFTPARSE_VERSION_MAJOR is intended for "major" source/ABI breaking changes.
28+
#define SWIFTPARSE_VERSION_MAJOR 0
29+
#define SWIFTPARSE_VERSION_MINOR 1
30+
31+
#define SWIFTPARSE_VERSION_ENCODE(major, minor) ( \
32+
((major) * 10000) \
33+
+ ((minor) * 1))
34+
35+
#define SWIFTPARSE_VERSION SWIFTPARSE_VERSION_ENCODE( \
36+
SWIFTPARSE_VERSION_MAJOR, \
37+
SWIFTPARSE_VERSION_MINOR )
38+
39+
#define SWIFTPARSE_VERSION_STRINGIZE_(major, minor) \
40+
#major"."#minor
41+
#define SWIFTPARSE_VERSION_STRINGIZE(major, minor) \
42+
SWIFTPARSE_VERSION_STRINGIZE_(major, minor)
43+
44+
#define SWIFTPARSE_VERSION_STRING SWIFTPARSE_VERSION_STRINGIZE( \
45+
SWIFTPARSE_VERSION_MAJOR, \
46+
SWIFTPARSE_VERSION_MINOR)
47+
48+
#ifdef __cplusplus
49+
# define SWIFTPARSE_BEGIN_DECLS extern "C" {
50+
# define SWIFTPARSE_END_DECLS }
51+
#else
52+
# define SWIFTPARSE_BEGIN_DECLS
53+
# define SWIFTPARSE_END_DECLS
54+
#endif
55+
56+
#ifndef SWIFTPARSE_PUBLIC
57+
# ifdef _WIN32
58+
# ifdef libSwiftSyntaxParser_EXPORTS
59+
# define SWIFTPARSE_PUBLIC __declspec(dllexport)
60+
# else
61+
# define SWIFTPARSE_PUBLIC __declspec(dllimport)
62+
# endif
63+
# else
64+
# define SWIFTPARSE_PUBLIC
65+
# endif
66+
#endif
67+
68+
#ifndef __has_feature
69+
# define __has_feature(x) 0
70+
#endif
71+
72+
#if !__has_feature(blocks)
73+
# error -fblocks is a requirement to use this library
74+
#endif
75+
76+
SWIFTPARSE_BEGIN_DECLS
77+
78+
//=== Syntax Data Types ---------------------------------------------------===//
79+
80+
/// Offset+length in UTF8 bytes.
81+
typedef struct {
82+
uint32_t offset;
83+
uint32_t length;
84+
} swiftparse_range_t;
85+
86+
typedef uint8_t swiftparse_trivia_kind_t;
87+
typedef uint8_t swiftparse_token_kind_t;
88+
typedef uint16_t swiftparse_syntax_kind_t;
89+
90+
/// This is for the client to provide an opaque pointer that the parser will
91+
/// associate with a syntax node.
92+
typedef void *swiftparse_client_node_t;
93+
94+
typedef struct {
95+
/// The length in source this trivia piece occupies, in UTF8 bytes.
96+
uint32_t length;
97+
swiftparse_trivia_kind_t kind;
98+
} swiftparse_trivia_piece_t;
99+
100+
typedef struct {
101+
const swiftparse_trivia_piece_t *leading_trivia;
102+
const swiftparse_trivia_piece_t *trailing_trivia;
103+
uint16_t leading_trivia_count;
104+
uint16_t trailing_trivia_count;
105+
swiftparse_token_kind_t kind;
106+
} swiftparse_token_data_t;
107+
108+
typedef struct {
109+
const swiftparse_client_node_t *nodes;
110+
uint32_t nodes_count;
111+
} swiftparse_layout_data_t;
112+
113+
typedef struct {
114+
union {
115+
swiftparse_token_data_t token_data;
116+
swiftparse_layout_data_t layout_data;
117+
};
118+
/// Represents the range for the node. For a token node the range includes
119+
/// the trivia associated with it.
120+
swiftparse_range_t range;
121+
/// The syntax kind. A value of '0' means this is a token node.
122+
swiftparse_syntax_kind_t kind;
123+
bool present;
124+
} swiftparse_syntax_node_t;
125+
126+
//=== Parser Functions ----------------------------------------------------===//
127+
128+
/// Container of the configuration state for parsing.
129+
///
130+
/// It keeps track of the callback blocks for parsing. Any other additional
131+
/// configuration option (like parsing options) could be set by a adding a
132+
/// function that accepts a \c swiftparse_parser_t object and modifies its
133+
/// state.
134+
typedef void *swiftparse_parser_t;
135+
136+
SWIFTPARSE_PUBLIC swiftparse_parser_t
137+
swiftparse_parser_create(void);
138+
139+
SWIFTPARSE_PUBLIC void
140+
swiftparse_parser_dispose(swiftparse_parser_t);
141+
142+
/// Invoked by the parser when a syntax node is parsed. The client should
143+
/// return a pointer to associate with that particular node.
144+
typedef swiftparse_client_node_t
145+
(^swiftparse_node_handler_t)(const swiftparse_syntax_node_t *);
146+
147+
/// Set the \c swiftparse_node_handler_t block to be used by the parser.
148+
///
149+
/// It is required to set a \c swiftparse_node_handler_t block before any calls
150+
/// to \c swiftparse_parse_string. \c swiftparse_parser_set_node_handler can be
151+
/// called multiple times to change the block before subsequent parses.
152+
SWIFTPARSE_PUBLIC void
153+
swiftparse_parser_set_node_handler(swiftparse_parser_t,
154+
swiftparse_node_handler_t);
155+
156+
typedef struct {
157+
/// Length of the source region in UTF8 bytes that the parser should skip.
158+
/// If it is set to 0 it indicates that the parser should continue parsing
159+
/// and the \c node object is ignored.
160+
size_t length;
161+
/// Node to associate for the skipped source region. It will be ignored if
162+
/// \c length is 0.
163+
swiftparse_client_node_t node;
164+
} swiftparse_lookup_result_t;
165+
166+
/// Invoked by the parser at certain points to query whether a source region
167+
/// should be skipped. See \c swiftparse_lookup_result_t.
168+
typedef swiftparse_lookup_result_t
169+
(^swiftparse_node_lookup_t)(size_t offset, swiftparse_syntax_kind_t);
170+
171+
/// Set the \c swiftparse_node_lookup_t block to be used by the parser.
172+
///
173+
/// It is not required to set a \c swiftparse_node_lookup_t block before calling
174+
/// \c swiftparse_parse_string. Not setting a \c swiftparse_node_lookup_t block
175+
/// has same semantics as never skipping any source regions.
176+
/// \c swiftparse_parser_set_node_lookup can be called multiple times to change
177+
/// the block before subsequent parses.
178+
SWIFTPARSE_PUBLIC void
179+
swiftparse_parser_set_node_lookup(swiftparse_parser_t,
180+
swiftparse_node_lookup_t);
181+
182+
/// Parse the provided \p source and invoke the callback that was set via
183+
/// \c swiftparse_parser_set_node_handler as each syntax node is parsed.
184+
///
185+
/// Syntax nodes are provided in a depth-first, source order. For example,
186+
/// token nodes will be provided ahead of the syntax node whose layout they are
187+
/// a part of. The memory that \c swiftparse_syntax_node_t points to is only
188+
/// valid to access for the duration of the \c swiftparse_node_handler_t block
189+
/// execution, the client should copy the data if it wants to persist it beyond
190+
/// the duration of the block.
191+
///
192+
/// The client provides \c swiftparse_client_node_t pointers to associate with
193+
/// each syntax node and the parser will pass back these pointers as part of the
194+
/// \c swiftparse_layout_data_t of the syntax node that they are a part of.
195+
/// \c swiftparse_client_node_t pointers are completely opaque to the parser,
196+
/// it doesn't try to interpret them in any way. There is no requirement that
197+
/// each node gets a unique \c swiftparse_client_node_t, it is up to the client
198+
/// to reuse \c swiftparse_client_node_t pointers as it deems necessary.
199+
///
200+
/// If the \c swiftparse_client_node_t pointers represent managed memory, the
201+
/// semantics of interacting with the parser should be considered as follows:
202+
///
203+
/// * \c swiftparse_node_handler_t and \c swiftparse_node_lookup_t return a
204+
/// \c swiftparse_client_node_t and transfer ownership of the pointer to the
205+
/// parser.
206+
///
207+
/// * The array of \c swiftparse_client_node_t pointers in
208+
/// \c swiftparse_layout_data_t should be considered as the parser transferring
209+
/// ownership of those pointers back to the client.
210+
///
211+
/// * The \c swiftparse_client_node_t returned by \c swiftparse_parse_string
212+
/// should be considered as the parser transferring back ownership of the
213+
/// pointer.
214+
///
215+
/// The parser guarantees that any \c swiftparse_client_node_t, given to the
216+
/// parser by \c swiftparse_node_handler_t or \c swiftparse_node_lookup_t, will
217+
/// be returned back to the client, either via \c swiftparse_layout_data_t or
218+
/// via the return value of \c swiftparse_parse_string.
219+
///
220+
/// \param source a null-terminated UTF8 string buffer.
221+
SWIFTPARSE_PUBLIC swiftparse_client_node_t
222+
swiftparse_parse_string(swiftparse_parser_t, const char *source);
223+
224+
SWIFTPARSE_END_DECLS
225+
226+
#endif
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
module _InternalSwiftSyntaxParser {
2+
header "SwiftSyntaxParser.h"
3+
link "_InternalSwiftSyntaxParser"
4+
}

test/CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ function(get_test_dependencies SDK result_var_name)
5050
llvm-bcanalyzer llvm-nm llvm-readobj llvm-profdata count not
5151
llvm-strings)
5252
endif()
53+
if(SWIFT_BUILD_SYNTAXPARSERLIB)
54+
list(APPEND deps_binaries swift-syntax-parser-test)
55+
endif()
5356
if(SWIFT_BUILD_SOURCEKIT)
5457
list(APPEND deps_binaries sourcekitd-test complete-test)
5558
endif()
@@ -131,6 +134,7 @@ normalize_boolean_spelling(LLVM_ENABLE_ASSERTIONS)
131134
normalize_boolean_spelling(SWIFT_STDLIB_ASSERTIONS)
132135
normalize_boolean_spelling(SWIFT_AST_VERIFIER)
133136
normalize_boolean_spelling(SWIFT_ASAN_BUILD)
137+
normalize_boolean_spelling(SWIFT_BUILD_SYNTAXPARSERLIB)
134138
normalize_boolean_spelling(SWIFT_ENABLE_SOURCEKIT_TESTS)
135139
is_build_type_optimized("${SWIFT_STDLIB_BUILD_TYPE}" SWIFT_OPTIMIZED)
136140

test/Syntax/Parser/tree.swift

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// REQUIRES: syntax_parser_lib
2+
// RUN: %swift-syntax-parser-test %s -dump-tree > %t.result
3+
// RUN: diff -u %s.result %t.result
4+
5+
func test() {
6+
"a\(b)c"
7+
}

test/Syntax/Parser/tree.swift.result

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<s118><s163><s92><s13><NULL/><NULL/><t6>// REQUIRES: syntax_parser_lib
2+
// RUN: %swift-syntax-parser-test %s -dump-tree > %t.result
3+
// RUN: diff -u %s.result %t.result
4+
5+
|func| </t6><t105>|test|</t105><NULL/><s110><s108><t88>|(|</t88><s174></s174><t89>|)| </t89></s108><NULL/><NULL/></s110><NULL/><s93><t90>|{|</t90><s163><s92><s64><t102>
6+
|"|</t102><s168><s104><t104>|a|</t104></s104><s105><t100>|\|</t100><t88>|(|</t88><s28><t105>|b|</t105><NULL/></s28><t101>|)|</t101></s105><s104><t104>|c|</t104></s104></s168><t102>|"|</t102></s64><NULL/><NULL/></s92></s163><t91>
7+
|}|</t91></s93></s13><NULL/><NULL/></s92></s163><t0>
8+
||</t0></s118>

test/lit.cfg

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,8 @@ config.sil_passpipeline_dumper = inferSwiftBinary('sil-passpipeline-dumper')
267267
config.lldb_moduleimport_test = inferSwiftBinary('lldb-moduleimport-test')
268268
config.swift_ide_test = inferSwiftBinary('swift-ide-test')
269269
config.swift_syntax_test = inferSwiftBinary('swift-syntax-test')
270+
if 'syntax_parser_lib' in config.available_features:
271+
config.swift_syntax_parser_test = inferSwiftBinary('swift-syntax-parser-test')
270272
config.swift_reflection_dump = inferSwiftBinary('swift-reflection-dump')
271273
config.swift_remoteast_test = inferSwiftBinary('swift-remoteast-test')
272274
config.swift_format = inferSwiftBinary('swift-format')
@@ -382,6 +384,8 @@ config.substitutions.append( ('%lldb-moduleimport-test', "%r %s" % (config.lldb_
382384
config.substitutions.append( ('%swift-ide-test_plain', config.swift_ide_test) )
383385
config.substitutions.append( ('%swift-ide-test', "%r %s %s -swift-version %s" % (config.swift_ide_test, mcp_opt, ccp_opt, swift_version)) )
384386
config.substitutions.append( ('%swift-syntax-test', config.swift_syntax_test) )
387+
if 'syntax_parser_lib' in config.available_features:
388+
config.substitutions.append( ('%swift-syntax-parser-test', config.swift_syntax_parser_test) )
385389
config.substitutions.append( ('%swift-format', config.swift_format) )
386390
config.substitutions.append( ('%llvm-link', config.llvm_link) )
387391
config.substitutions.append( ('%swift-llvm-opt', config.swift_llvm_opt) )

test/lit.site.cfg.in

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@ if "@CMAKE_GENERATOR@" == "Xcode":
8585

8686
config.available_features.add("CMAKE_GENERATOR=@CMAKE_GENERATOR@")
8787

88+
if "@SWIFT_BUILD_SYNTAXPARSERLIB@" == "TRUE":
89+
config.available_features.add('syntax_parser_lib')
90+
8891
if "@SWIFT_ENABLE_SOURCEKIT_TESTS@" == "TRUE":
8992
config.available_features.add('sourcekit')
9093

tools/CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ add_swift_tool_subdirectory(swift-llvm-opt)
3030
add_swift_tool_subdirectory(swift-api-digester)
3131
add_swift_tool_subdirectory(swift-syntax-test)
3232
add_swift_tool_subdirectory(swift-refactor)
33+
if(SWIFT_BUILD_SYNTAXPARSERLIB)
34+
add_swift_tool_subdirectory(libSwiftSyntaxParser)
35+
add_swift_tool_subdirectory(swift-syntax-parser-test)
36+
endif()
3337

3438
if(LLVM_USE_SANITIZE_COVERAGE)
3539
add_swift_tool_subdirectory(swift-demangle-fuzzer)
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Use an 'internal' name, this is primarily intended for SwiftSyntax to import.
2+
set(SYNTAX_PARSER_LIB_NAME "_InternalSwiftSyntaxParser")
3+
4+
set(LLVM_EXPORTED_SYMBOL_FILE
5+
${CMAKE_CURRENT_SOURCE_DIR}/libSwiftSyntaxParser.exports)
6+
7+
add_swift_host_library(libSwiftSyntaxParser SHARED
8+
c-include-check.c
9+
libSwiftSyntaxParser.cpp
10+
11+
LINK_LIBRARIES
12+
swiftParse
13+
)
14+
set_target_properties(libSwiftSyntaxParser
15+
PROPERTIES
16+
OUTPUT_NAME ${SYNTAX_PARSER_LIB_NAME})
17+
18+
add_llvm_symbol_exports(libSwiftSyntaxParser ${LLVM_EXPORTED_SYMBOL_FILE})
19+
20+
# Adds -dead_strip option
21+
add_link_opts(libSwiftSyntaxParser)
22+
23+
set_property(TARGET libSwiftSyntaxParser APPEND_STRING PROPERTY
24+
COMPILE_FLAGS " -fblocks")
25+
if(SWIFT_NEED_EXPLICIT_LIBDISPATCH)
26+
target_link_libraries(libSwiftSyntaxParser PRIVATE BlocksRuntime)
27+
endif()
28+
29+
swift_install_in_component(parser-lib
30+
FILES "${SWIFT_LIBRARY_OUTPUT_INTDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}${SYNTAX_PARSER_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}"
31+
DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}")
32+
swift_install_in_component(parser-lib
33+
DIRECTORY "${SWIFT_MAIN_INCLUDE_DIR}/swift-c/SyntaxParser/"
34+
DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/${SYNTAX_PARSER_LIB_NAME}")

0 commit comments

Comments
 (0)