Skip to content

Commit 982f889

Browse files
mohebifarmohebifar
committed
Initial commit
0 parents  commit 982f889

File tree

5 files changed

+673
-0
lines changed

5 files changed

+673
-0
lines changed

‎.github/workflows/build-all-parsers.yml‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
name: Build All Tree-sitter Parsers
2+
3+
on:
4+
push:
5+
branches: [main, master]
6+
schedule:
7+
# Run daily at 2 AM UTC
8+
- cron: '0 2 * * *'
9+
workflow_dispatch:
10+
11+
env:
12+
TREE_SITTER_ABI_VERSION: "15"
13+
14+
jobs:
15+
generate-matrix:
16+
name: Generate Build Matrix
17+
runs-on: ubuntu-latest
18+
outputs:
19+
languages: ${{ steps.set-matrix.outputs.languages }}
20+
steps:
21+
- name: Checkout repository
22+
uses: actions/checkout@v4
23+
24+
- name: Generate matrix
25+
id: set-matrix
26+
run: |
27+
languages=$(jq -c '[.languages | keys[]]' languages.json)
28+
echo "languages=$languages" >> $GITHUB_OUTPUT
29+
30+
build:
31+
name: Build ${{ matrix.language }} on ${{ matrix.os }} (${{ matrix.arch }})
32+
needs: generate-matrix
33+
runs-on: ${{ matrix.os }}
34+
strategy:
35+
fail-fast: false
36+
matrix:
37+
language: ${{ fromJson(needs.generate-matrix.outputs.languages) }}
38+
include:
39+
- { os: ubuntu-latest, arch: x64, platform: linux }
40+
- { os: ubuntu-latest, arch: arm64, platform: linux }
41+
- { os: macos-latest, arch: x64, platform: darwin }
42+
- { os: macos-latest, arch: arm64, platform: darwin }
43+
- { os: windows-latest, arch: x64, platform: win32 }
44+
45+
steps:
46+
- name: Checkout repository
47+
uses: actions/checkout@v4
48+
49+
- name: Set up QEMU (for ARM64 on Linux)
50+
if: matrix.os == 'ubuntu-latest' && matrix.arch == 'arm64'
51+
uses: docker/setup-qemu-action@v3
52+
with:
53+
platforms: arm64
54+
55+
- name: Set up Node.js
56+
uses: actions/setup-node@v4
57+
with:
58+
node-version: '18'
59+
60+
- name: Set up tree-sitter CLI
61+
uses: tree-sitter/setup-action/cli@v2
62+
63+
- name: Extract language info
64+
id: lang-info
65+
run: |
66+
repo_url=$(jq -r ".languages[\"${{ matrix.language }}\"].repo" languages.json)
67+
ref=$(jq -r ".languages[\"${{ matrix.language }}\"].ref" languages.json)
68+
echo "repo_url=$repo_url" >> $GITHUB_OUTPUT
69+
echo "ref=$ref" >> $GITHUB_OUTPUT
70+
71+
- name: Make scripts executable
72+
run: chmod +x scripts/*.sh
73+
74+
- name: Build parser (Linux ARM64)
75+
if: matrix.os == 'ubuntu-latest' && matrix.arch == 'arm64'
76+
run: |
77+
docker run --rm -v $PWD:/workspace -w /workspace \
78+
--platform linux/arm64 \
79+
node:18-alpine \
80+
sh -c "
81+
apk add --no-cache git bash tree-sitter-cli build-base &&
82+
./scripts/build-parser.sh '${{ matrix.language }}' '${{ steps.lang-info.outputs.repo_url }}' '${{ steps.lang-info.outputs.ref }}' 'artifacts'
83+
"
84+
env:
85+
TREE_SITTER_ABI_VERSION: ${{ env.TREE_SITTER_ABI_VERSION }}
86+
87+
- name: Build parser (Native)
88+
if: matrix.os != 'ubuntu-latest' || matrix.arch != 'arm64'
89+
run: |
90+
./scripts/build-parser.sh "${{ matrix.language }}" "${{ steps.lang-info.outputs.repo_url }}" "${{ steps.lang-info.outputs.ref }}" "artifacts"
91+
env:
92+
TREE_SITTER_ABI_VERSION: ${{ env.TREE_SITTER_ABI_VERSION }}
93+
94+
- name: Upload artifacts
95+
uses: actions/upload-artifact@v4
96+
with:
97+
name: parsers-${{ matrix.language }}-${{ matrix.platform }}-${{ matrix.arch }}
98+
path: artifacts/
99+
retention-days: 7
100+
101+
upload:
102+
name: Upload to S3
103+
needs: build
104+
runs-on: ubuntu-latest
105+
if: always() && needs.build.result == 'success'
106+
107+
steps:
108+
- name: Checkout repository
109+
uses: actions/checkout@v4
110+
111+
- name: Download all artifacts
112+
uses: actions/download-artifact@v4
113+
with:
114+
path: artifacts-download/
115+
pattern: parsers-*
116+
merge-multiple: true
117+
118+
- name: Configure AWS Credentials for S3 Upload
119+
uses: aws-actions/configure-aws-credentials@v4
120+
with:
121+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
122+
aws-region: us-east-1
123+
124+
- name: Upload parsers to S3
125+
run: |
126+
# Upload all parser files to S3
127+
find artifacts-download/ -type f \( -name "*.so" -o -name "*.dylib" -o -name "*.dll" -o -name "*.wasm" \) | while read -r file; do
128+
# Extract the relative path from artifacts directory
129+
relative_path=${file#artifacts-download/}
130+
131+
# Construct S3 key
132+
s3_key="tree-sitter/parsers/tree-sitter-$relative_path"
133+
134+
echo "Uploading: $file -> s3://${{ secrets.S3_BUCKET_NAME }}/$s3_key"
135+
136+
# Upload to S3
137+
aws s3 cp "$file" "s3://${{ secrets.S3_BUCKET_NAME }}/$s3_key" \
138+
--metadata "source-file=$relative_path" \
139+
--cache-control "public, max-age=31536000" \
140+
--content-type "application/octet-stream"
141+
done
142+
143+
- name: Generate and upload index
144+
run: |
145+
# Generate index file
146+
INDEX_FILE=$(mktemp)
147+
echo '{"parsers": [' > "$INDEX_FILE"
148+
149+
# Generate JSON index of all uploaded files
150+
find artifacts-download/ -type f \( -name "*.so" -o -name "*.dylib" -o -name "*.dll" -o -name "*.wasm" \) | sort | while read -r file; do
151+
relative_path=${file#artifacts-download/}
152+
153+
# Parse the path: language/sha/platform-arch.ext or language/sha/parser.wasm
154+
IFS='/' read -r language sha filename <<< "$relative_path"
155+
156+
# Determine file type and platform/arch
157+
if [[ "$filename" == "parser.wasm" ]]; then
158+
platform="wasm"
159+
arch="wasm"
160+
filetype="wasm"
161+
else
162+
# Extract platform, arch, and extension from filename
163+
filename_no_ext="${filename%.*}"
164+
filetype="${filename##*.}"
165+
166+
IFS='-' read -r platform arch <<< "$filename_no_ext"
167+
fi
168+
169+
s3_url="https://${{ secrets.S3_BUCKET_NAME }}.s3.amazonaws.com/tree-sitter/parsers/tree-sitter-$relative_path"
170+
171+
cat << EOF >> "$INDEX_FILE"
172+
{
173+
"language": "$language",
174+
"sha": "$sha",
175+
"platform": "$platform",
176+
"architecture": "$arch",
177+
"filetype": "$filetype",
178+
"url": "$s3_url",
179+
"path": "$relative_path"
180+
},
181+
EOF
182+
done
183+
184+
# Remove the last comma and close the JSON
185+
sed -i '$ s/,$//' "$INDEX_FILE"
186+
echo ']}' >> "$INDEX_FILE"
187+
188+
# Upload index file
189+
aws s3 cp "$INDEX_FILE" "s3://${{ secrets.S3_BUCKET_NAME }}/tree-sitter/parsers/index.json" \
190+
--content-type "application/json" \
191+
--cache-control "public, max-age=300"
192+
193+
# Clean up
194+
rm -f "$INDEX_FILE"
195+
196+
- name: Summary
197+
run: |
198+
echo "## 🎉 Build Summary" >> $GITHUB_STEP_SUMMARY
199+
echo "" >> $GITHUB_STEP_SUMMARY
200+
echo "### Built Parsers" >> $GITHUB_STEP_SUMMARY
201+
find artifacts-download/ -name "*.so" -o -name "*.dylib" -o -name "*.dll" -o -name "*.wasm" | sort | while read file; do
202+
echo "- \`$(basename "$file")\` ($(du -h "$file" | cut -f1))" >> $GITHUB_STEP_SUMMARY
203+
done
204+
echo "" >> $GITHUB_STEP_SUMMARY
205+
echo "### S3 Location" >> $GITHUB_STEP_SUMMARY
206+
echo "Files uploaded to: \`s3://${{ secrets.S3_BUCKET_NAME }}/tree-sitter/parsers/\`" >> $GITHUB_STEP_SUMMARY

‎.github/workflows/build-single-parser.yml‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
name: Build Single Tree-sitter Parser
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
language:
7+
description: 'Language to build parser for'
8+
required: true
9+
type: choice
10+
options:
11+
- javascript
12+
- typescript
13+
- tsx
14+
- html
15+
- css
16+
- angular
17+
- java
18+
- kotlin
19+
- scala
20+
- python
21+
- go
22+
- rust
23+
- c-sharp
24+
- cpp
25+
- c
26+
- php
27+
- ruby
28+
- elixir
29+
platforms:
30+
description: 'Platforms to build for'
31+
required: true
32+
type: choice
33+
default: 'all'
34+
options:
35+
- all
36+
- linux-only
37+
- macos-only
38+
- windows-only
39+
40+
env:
41+
TREE_SITTER_ABI_VERSION: "15"
42+
43+
jobs:
44+
generate-matrix:
45+
name: Generate Platform Matrix
46+
runs-on: ubuntu-latest
47+
outputs:
48+
matrix: ${{ steps.set-matrix.outputs.matrix }}
49+
steps:
50+
- name: Generate matrix based on platform selection
51+
id: set-matrix
52+
run: |
53+
case "${{ inputs.platforms }}" in
54+
"linux-only")
55+
matrix='{"include":[{"os":"ubuntu-latest","arch":"x64","platform":"linux"},{"os":"ubuntu-latest","arch":"arm64","platform":"linux"}]}'
56+
;;
57+
"macos-only")
58+
matrix='{"include":[{"os":"macos-latest","arch":"x64","platform":"darwin"},{"os":"macos-latest","arch":"arm64","platform":"darwin"}]}'
59+
;;
60+
"windows-only")
61+
matrix='{"include":[{"os":"windows-latest","arch":"x64","platform":"win32"}]}'
62+
;;
63+
*)
64+
matrix='{"include":[{"os":"ubuntu-latest","arch":"x64","platform":"linux"},{"os":"ubuntu-latest","arch":"arm64","platform":"linux"},{"os":"macos-latest","arch":"x64","platform":"darwin"},{"os":"macos-latest","arch":"arm64","platform":"darwin"},{"os":"windows-latest","arch":"x64","platform":"win32"}]}'
65+
;;
66+
esac
67+
echo "matrix=$matrix" >> $GITHUB_OUTPUT
68+
69+
build:
70+
name: Build ${{ inputs.language }} on ${{ matrix.os }} (${{ matrix.arch }})
71+
needs: generate-matrix
72+
runs-on: ${{ matrix.os }}
73+
strategy:
74+
fail-fast: false
75+
matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
76+
77+
steps:
78+
- name: Checkout repository
79+
uses: actions/checkout@v4
80+
81+
- name: Set up QEMU (for ARM64 on Linux)
82+
if: matrix.os == 'ubuntu-latest' && matrix.arch == 'arm64'
83+
uses: docker/setup-qemu-action@v3
84+
with:
85+
platforms: arm64
86+
87+
- name: Set up Node.js
88+
uses: actions/setup-node@v4
89+
with:
90+
node-version: '18'
91+
92+
- name: Set up tree-sitter CLI
93+
uses: tree-sitter/setup-action/cli@v2
94+
95+
- name: Extract language info
96+
id: lang-info
97+
run: |
98+
repo_url=$(jq -r ".languages[\"${{ inputs.language }}\"].repo" languages.json)
99+
ref=$(jq -r ".languages[\"${{ inputs.language }}\"].ref" languages.json)
100+
echo "repo_url=$repo_url" >> $GITHUB_OUTPUT
101+
echo "ref=$ref" >> $GITHUB_OUTPUT
102+
103+
- name: Make scripts executable
104+
run: chmod +x scripts/*.sh
105+
106+
- name: Build parser (Linux ARM64)
107+
if: matrix.os == 'ubuntu-latest' && matrix.arch == 'arm64'
108+
run: |
109+
docker run --rm -v $PWD:/workspace -w /workspace \
110+
--platform linux/arm64 \
111+
node:18-alpine \
112+
sh -c "
113+
apk add --no-cache git bash tree-sitter-cli build-base &&
114+
./scripts/build-parser.sh '${{ inputs.language }}' '${{ steps.lang-info.outputs.repo_url }}' '${{ steps.lang-info.outputs.ref }}' 'artifacts'
115+
"
116+
env:
117+
TREE_SITTER_ABI_VERSION: ${{ env.TREE_SITTER_ABI_VERSION }}
118+
119+
- name: Build parser (Native)
120+
if: matrix.os != 'ubuntu-latest' || matrix.arch != 'arm64'
121+
run: |
122+
./scripts/build-parser.sh "${{ inputs.language }}" "${{ steps.lang-info.outputs.repo_url }}" "${{ steps.lang-info.outputs.ref }}" "artifacts"
123+
env:
124+
TREE_SITTER_ABI_VERSION: ${{ env.TREE_SITTER_ABI_VERSION }}
125+
126+
- name: Upload artifacts
127+
uses: actions/upload-artifact@v4
128+
with:
129+
name: parser-${{ inputs.language }}-${{ matrix.platform }}-${{ matrix.arch }}
130+
path: artifacts/
131+
retention-days: 7
132+
133+
upload:
134+
name: Upload to S3
135+
needs: build
136+
runs-on: ubuntu-latest
137+
if: always() && needs.build.result == 'success'
138+
139+
steps:
140+
- name: Checkout repository
141+
uses: actions/checkout@v4
142+
143+
- name: Download all artifacts
144+
uses: actions/download-artifact@v4
145+
with:
146+
path: artifacts-download/
147+
pattern: parser-${{ inputs.language }}-*
148+
merge-multiple: true
149+
150+
- name: Configure AWS Credentials for S3 Upload
151+
uses: aws-actions/configure-aws-credentials@v4
152+
with:
153+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
154+
aws-region: us-east-1
155+
156+
- name: Upload parsers to S3
157+
run: |
158+
# Upload all parser files to S3
159+
find artifacts-download/ -type f \( -name "*.so" -o -name "*.dylib" -o -name "*.dll" -o -name "*.wasm" \) | while read -r file; do
160+
# Extract the relative path from artifacts directory
161+
relative_path=${file#artifacts-download/}
162+
163+
# Construct S3 key
164+
s3_key="tree-sitter/parsers/tree-sitter-$relative_path"
165+
166+
echo "Uploading: $file -> s3://${{ secrets.S3_BUCKET_NAME }}/$s3_key"
167+
168+
# Upload to S3
169+
aws s3 cp "$file" "s3://${{ secrets.S3_BUCKET_NAME }}/$s3_key" \
170+
--metadata "source-file=$relative_path" \
171+
--cache-control "public, max-age=31536000" \
172+
--content-type "application/octet-stream"
173+
done
174+
175+
- name: Summary
176+
run: |
177+
echo "## 🎉 Build Summary for ${{ inputs.language }}" >> $GITHUB_STEP_SUMMARY
178+
echo "" >> $GITHUB_STEP_SUMMARY
179+
echo "### Built Files" >> $GITHUB_STEP_SUMMARY
180+
find artifacts-download/ -name "*.so" -o -name "*.dylib" -o -name "*.dll" -o -name "*.wasm" | sort | while read file; do
181+
echo "- \`$(basename "$file")\` ($(du -h "$file" | cut -f1))" >> $GITHUB_STEP_SUMMARY
182+
done
183+
echo "" >> $GITHUB_STEP_SUMMARY
184+
echo "### S3 Location" >> $GITHUB_STEP_SUMMARY
185+
echo "Files uploaded to: \`s3://${{ secrets.S3_BUCKET_NAME }}/tree-sitter/parsers/tree-sitter-${{ inputs.language }}/\`" >> $GITHUB_STEP_SUMMARY

0 commit comments

Comments
 (0)