Skip to content

Commit 2099ec0

Browse files
trevor-etrevor-e
authored
Add workflow to build tree-sitter parsers (#18)
Co-authored-by: trevor-e <trevor-e@users.noreply.github.com>
1 parent 7ff1338 commit 2099ec0

9 files changed

+168
-30
lines changed

‎.github/workflows/build_parsers.yml‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
name: Build Tree-Sitter Parsers
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- telkins/build-dylibs
8+
paths:
9+
- 'parsers.toml'
10+
- '.github/workflows/build_parsers.yml'
11+
workflow_dispatch:
12+
13+
jobs:
14+
build-parsers:
15+
runs-on: ${{ matrix.os }}
16+
strategy:
17+
matrix:
18+
include:
19+
- os: ubuntu-latest
20+
platform: linux
21+
- os: macos-latest
22+
platform: macos
23+
24+
steps:
25+
- name: Checkout repository
26+
uses: actions/checkout@v3
27+
28+
- name: Install Rust toolchain
29+
uses: actions-rs/toolchain@v1
30+
with:
31+
toolchain: stable
32+
override: true
33+
34+
- name: Install tsdl
35+
run: |
36+
cargo install tsdl
37+
38+
- name: Build tree-sitter parsers
39+
run: |
40+
tsdl build --out-dir build_parsers
41+
42+
- name: Determine platform and architecture
43+
id: platform_info
44+
run: |
45+
# Determine architecture
46+
arch=$(uname -m)
47+
# Map architecture names to standard values
48+
if [[ "$arch" == "x86_64" || "$arch" == "amd64" ]]; then
49+
arch="x86_64"
50+
elif [[ "$arch" == "arm64" || "$arch" == "aarch64" ]]; then
51+
arch="arm64"
52+
fi
53+
54+
# For macOS, adjust platform name
55+
if [[ "${{ matrix.platform }}" == "macos" ]]; then
56+
platform_name="darwin"
57+
else
58+
platform_name="${{ matrix.platform }}"
59+
fi
60+
61+
# Set outputs
62+
echo "arch=$arch" >> $GITHUB_OUTPUT
63+
echo "platform_name=$platform_name" >> $GITHUB_OUTPUT
64+
65+
- name: Rename built parsers to include platform and architecture
66+
run: |
67+
# Use outputs from previous step
68+
arch="${{ steps.platform_info.outputs.arch }}"
69+
platform_name="${{ steps.platform_info.outputs.platform_name }}"
70+
platform_arch="${platform_name}-${arch}"
71+
72+
echo "Using platform_arch=${platform_arch}"
73+
74+
# Rename parser files in the temporary directory
75+
for file in build_parsers/libtree-sitter-*; do
76+
if [ -f "$file" ]; then
77+
filename=$(basename "$file")
78+
extension="${filename##*.}"
79+
base="${filename%.*}"
80+
81+
echo "Processing file $filename"
82+
83+
# Append platform_arch before the extension
84+
new_base="${base}-${platform_arch}"
85+
echo "Renaming $file to build_parsers/${new_base}.${extension}"
86+
mv "$file" "build_parsers/${new_base}.${extension}"
87+
fi
88+
done
89+
90+
- name: Move renamed parsers to parsers directory
91+
run: |
92+
# Ensure parsers directory exists
93+
mkdir -p parsers
94+
95+
# Move files and overwrite existing ones
96+
for file in build_parsers/*; do
97+
filename=$(basename "$file")
98+
dest_file="parsers/$filename"
99+
100+
echo "Moving $file to $dest_file"
101+
mv -f "$file" "$dest_file"
102+
done
103+
104+
- name: Configure git
105+
run: |
106+
git config user.name "${{ github.actor }}"
107+
git config user.email "${{ github.actor }}@users.noreply.github.com"
108+
109+
- name: Check for changes
110+
id: changes
111+
run: |
112+
git add -A
113+
if git diff --cached --quiet; then
114+
echo "changes=false" >> $GITHUB_OUTPUT
115+
else
116+
echo "changes=true" >> $GITHUB_OUTPUT
117+
fi
118+
119+
- name: Pull latest changes
120+
if: steps.changes.outputs.changes == 'true'
121+
run: |
122+
git pull --rebase --autostash
123+
124+
- name: Commit changes
125+
if: steps.changes.outputs.changes == 'true'
126+
run: |
127+
git commit -m "Build tree-sitter parsers for ${{ steps.platform_info.outputs.platform_name }}-${{ steps.platform_info.outputs.arch }}"
128+
129+
- name: Push changes
130+
if: steps.changes.outputs.changes == 'true'
131+
run: |
132+
max_retries=5
133+
retry_count=0
134+
until git push || [ $retry_count -eq $max_retries ]; do
135+
echo "Push failed, retrying..."
136+
git pull --rebase --autostash
137+
retry_count=$((retry_count + 1))
138+
sleep 5
139+
done
140+
if [ $retry_count -eq $max_retries ]; then
141+
echo "Push failed after $max_retries attempts."
142+
exit 1
143+
fi

‎lib/reaper/ast_parser.rb‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,35 @@ def initialize(language)
3131
@language = language
3232
@current_file_contents = nil
3333

34+
platform = case RUBY_PLATFORM
35+
when /darwin/
36+
'darwin'
37+
when /linux/
38+
'linux'
39+
else
40+
raise "Unsupported platform: #{RUBY_PLATFORM}"
41+
end
42+
43+
arch = case RUBY_PLATFORM
44+
when /x86_64|amd64/
45+
'x86_64'
46+
when /arm64|aarch64/
47+
'arm64'
48+
else
49+
raise "Unsupported architecture: #{RUBY_PLATFORM}"
50+
end
51+
52+
extension = platform == 'darwin' ? 'dylib' : 'so'
53+
parser_file = "libtree-sitter-#{language}-#{platform}-#{arch}.#{extension}"
54+
parser_path = File.join('parsers', parser_file)
55+
3456
case language
3557
when 'swift'
36-
@parser.language = TreeSitter.lang('swift')
58+
@parser.language = TreeSitter::Language.load('swift', parser_path)
3759
when 'kotlin'
38-
@parser.language = TreeSitter.lang('kotlin')
60+
@parser.language = TreeSitter::Language.load('kotlin', parser_path)
3961
when 'java'
40-
@parser.language = TreeSitter.lang('java')
62+
@parser.language = TreeSitter::Language.load('java', parser_path)
4163
else
4264
raise "Unsupported language: #{language}"
4365
end

parsers/libtree-sitter-java.dylib renamed to parsers/libtree-sitter-java-darwin-arm64.dylib

437 KB
Binary file not shown.

‎parsers/libtree-sitter-java-linux-x86_64.so‎

428 KB
Binary file not shown.

parsers/libtree-sitter-kotlin.dylib renamed to parsers/libtree-sitter-kotlin-darwin-arm64.dylib

3.97 MB
Binary file not shown.

‎parsers/libtree-sitter-kotlin-linux-x86_64.so‎

3.9 MB
Binary file not shown.

parsers/libtree-sitter-swift.dylib renamed to parsers/libtree-sitter-swift-darwin-arm64.dylib

2.87 MB
Binary file not shown.

‎parsers/libtree-sitter-swift-linux-x86_64.so‎

2.78 MB
Binary file not shown.

‎test/reaper/ast_parser_test.rb‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,8 @@
22

33
module Emerge
44
module Reaper
5-
module RunTestHelper
6-
def run_test?
7-
puts "Running on #{RbConfig::CONFIG['host_os']} #{RbConfig::CONFIG['host_cpu']}"
8-
host_os = RbConfig::CONFIG['host_os']
9-
host_cpu = RbConfig::CONFIG['host_cpu']
10-
host_os =~ /darwin/ && host_cpu =~ /arm64/
11-
end
12-
end
13-
145
class AstParserTest < Minitest::Test
156
describe 'Swift' do
16-
include RunTestHelper
17-
18-
before do
19-
skip 'These tests only run on Apple Silicon (darwin arm64)' unless run_test?
20-
end
21-
227
describe 'delete_type' do
238
def test_removes_protocol_from_swift_file
249
language = 'swift'
@@ -590,12 +575,6 @@ class NestedClass {
590575
end
591576

592577
describe 'Kotlin' do
593-
include RunTestHelper
594-
595-
before do
596-
skip 'These tests only run on Apple Silicon (darwin arm64)' unless run_test?
597-
end
598-
599578
describe 'delete_type' do
600579
def test_removes_class_from_kotlin_file
601580
language = 'kotlin'
@@ -875,12 +854,6 @@ class Factory(private val bookmarkDao: BookmarkDao) : ViewModelProvider.Factory
875854
end
876855

877856
describe 'Java' do
878-
include RunTestHelper
879-
880-
before do
881-
skip 'These tests only run on Apple Silicon (darwin arm64)' unless run_test?
882-
end
883-
884857
describe 'delete_type' do
885858
def test_removes_class_from_java_file
886859
language = 'java'

0 commit comments

Comments
 (0)