Skip to content

Commit dd350dd

Browse files
mohebifarmohebifar
committed
Initial commit
0 parents  commit dd350dd

File tree

5 files changed

+674
-0
lines changed

5 files changed

+674
-0
lines changed

‎.github/workflows/build-all-parsers.yml‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
name: Build All Tree-sitter Parsers
2+
3+
on:
4+
push:
5+
branches: [main, master]
6+
schedule:
7+
# Run daily at 2 AM UTC
8+
- cron: '0 2 * * *'
9+
workflow_dispatch:
10+
11+
env:
12+
TREE_SITTER_ABI_VERSION: "15"
13+
14+
jobs:
15+
generate-matrix:
16+
name: Generate Build Matrix
17+
runs-on: ubuntu-latest
18+
outputs:
19+
languages: ${{ steps.set-matrix.outputs.languages }}
20+
steps:
21+
- name: Checkout repository
22+
uses: actions/checkout@v4
23+
24+
- name: Generate matrix
25+
id: set-matrix
26+
run: |
27+
languages=$(jq -c '[.languages | keys[]]' languages.json)
28+
echo "languages=$languages" >> $GITHUB_OUTPUT
29+
30+
build:
31+
name: Build ${{ matrix.language }} on ${{ matrix.os }} (${{ matrix.arch }})
32+
needs: generate-matrix
33+
runs-on: ${{ matrix.os }}
34+
strategy:
35+
fail-fast: false
36+
matrix:
37+
language: ${{ fromJson(needs.generate-matrix.outputs.languages) }}
38+
include:
39+
- { os: ubuntu-latest, arch: x64, platform: linux }
40+
- { os: ubuntu-latest, arch: arm64, platform: linux }
41+
- { os: macos-latest, arch: x64, platform: darwin }
42+
- { os: macos-latest, arch: arm64, platform: darwin }
43+
- { os: windows-latest, arch: x64, platform: win32 }
44+
45+
steps:
46+
- name: Checkout repository
47+
uses: actions/checkout@v4
48+
49+
- name: Set up QEMU (for ARM64 on Linux)
50+
if: matrix.os == 'ubuntu-latest' && matrix.arch == 'arm64'
51+
uses: docker/setup-qemu-action@v3
52+
with:
53+
platforms: arm64
54+
55+
- name: Set up Node.js
56+
uses: actions/setup-node@v4
57+
with:
58+
node-version: '18'
59+
60+
- name: Set up tree-sitter CLI
61+
uses: tree-sitter/setup-action/cli@v2
62+
63+
- name: Extract language info
64+
id: lang-info
65+
run: |
66+
repo_url=$(jq -r ".languages[\"${{ matrix.language }}\"].repo" languages.json)
67+
ref=$(jq -r ".languages[\"${{ matrix.language }}\"].ref" languages.json)
68+
echo "repo_url=$repo_url" >> $GITHUB_OUTPUT
69+
echo "ref=$ref" >> $GITHUB_OUTPUT
70+
71+
- name: Make scripts executable
72+
run: chmod +x scripts/*.sh
73+
74+
- name: Build parser (Linux ARM64)
75+
if: matrix.os == 'ubuntu-latest' && matrix.arch == 'arm64'
76+
run: |
77+
docker run --rm -v $PWD:/workspace -w /workspace \
78+
--platform linux/arm64 \
79+
node:18-alpine \
80+
sh -c "
81+
apk add --no-cache git bash tree-sitter-cli build-base &&
82+
./scripts/build-parser.sh '${{ matrix.language }}' '${{ steps.lang-info.outputs.repo_url }}' '${{ steps.lang-info.outputs.ref }}' 'artifacts'
83+
"
84+
env:
85+
TREE_SITTER_ABI_VERSION: ${{ env.TREE_SITTER_ABI_VERSION }}
86+
87+
- name: Build parser (Native)
88+
if: matrix.os != 'ubuntu-latest' || matrix.arch != 'arm64'
89+
run: |
90+
./scripts/build-parser.sh "${{ matrix.language }}" "${{ steps.lang-info.outputs.repo_url }}" "${{ steps.lang-info.outputs.ref }}" "artifacts"
91+
env:
92+
TREE_SITTER_ABI_VERSION: ${{ env.TREE_SITTER_ABI_VERSION }}
93+
94+
- name: Upload artifacts
95+
uses: actions/upload-artifact@v4
96+
with:
97+
name: parsers-${{ matrix.language }}-${{ matrix.platform }}-${{ matrix.arch }}
98+
path: artifacts/
99+
retention-days: 7
100+
101+
upload:
102+
name: Upload to S3
103+
needs: build
104+
runs-on: ubuntu-latest
105+
if: always() && needs.build.result == 'success'
106+
107+
steps:
108+
- name: Checkout repository
109+
uses: actions/checkout@v4
110+
111+
- name: Download all artifacts
112+
uses: actions/download-artifact@v4
113+
with:
114+
path: artifacts-download/
115+
pattern: parsers-*
116+
merge-multiple: true
117+
118+
- name: Configure AWS Credentials for S3 Upload
119+
uses: aws-actions/configure-aws-credentials@v4
120+
with:
121+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
122+
aws-region: us-east-1
123+
124+
- name: Upload parsers to S3
125+
run: |
126+
# Upload all parser files to S3
127+
find artifacts-download/ -type f \( -name "*.so" -o -name "*.dylib" -o -name "*.dll" -o -name "*.wasm" \) | while read -r file; do
128+
# Extract the relative path from artifacts directory
129+
relative_path=${file#artifacts-download/}
130+
131+
# Construct S3 key
132+
s3_key="tree-sitter/parsers/tree-sitter-$relative_path"
133+
134+
echo "Uploading: $file -> s3://${{ secrets.S3_BUCKET_NAME }}/$s3_key"
135+
136+
# Upload to S3
137+
aws s3 cp "$file" "s3://${{ secrets.S3_BUCKET_NAME }}/$s3_key" \
138+
--metadata "source-file=$relative_path" \
139+
--cache-control "public, max-age=31536000" \
140+
--content-type "application/octet-stream"
141+
done
142+
143+
- name: Generate and upload index
144+
run: |
145+
# Generate index file
146+
INDEX_FILE=$(mktemp)
147+
echo '{"parsers": [' > "$INDEX_FILE"
148+
149+
# Generate JSON index of all uploaded files
150+
find artifacts-download/ -type f \( -name "*.so" -o -name "*.dylib" -o -name "*.dll" -o -name "*.wasm" \) | sort | while read -r file; do
151+
relative_path=${file#artifacts-download/}
152+
153+
# Parse the path: language/sha/platform-arch.ext or language/sha/parser.wasm
154+
IFS='/' read -r language sha filename <<< "$relative_path"
155+
156+
# Determine file type and platform/arch
157+
if [[ "$filename" == "parser.wasm" ]]; then
158+
platform="wasm"
159+
arch="wasm"
160+
filetype="wasm"
161+
else
162+
# Extract platform, arch, and extension from filename
163+
filename_no_ext="${filename%.*}"
164+
filetype="${filename##*.}"
165+
166+
IFS='-' read -r platform arch <<< "$filename_no_ext"
167+
fi
168+
169+
s3_url="https://${{ secrets.S3_BUCKET_NAME }}.s3.amazonaws.com/tree-sitter/parsers/tree-sitter-$relative_path"
170+
171+
cat << EOF >> "$INDEX_FILE"
172+
{
173+
"language": "$language",
174+
"sha": "$sha",
175+
"platform": "$platform",
176+
"architecture": "$arch",
177+
"filetype": "$filetype",
178+
"url": "$s3_url",
179+
"path": "$relative_path"
180+
},
181+
EOF
182+
done
183+
184+
# Remove the last comma and close the JSON
185+
sed -i '$ s/,$//' "$INDEX_FILE"
186+
echo ']}' >> "$INDEX_FILE"
187+
188+
# Upload index file
189+
aws s3 cp "$INDEX_FILE" "s3://${{ secrets.S3_BUCKET_NAME }}/tree-sitter/parsers/index.json" \
190+
--content-type "application/json" \
191+
--cache-control "public, max-age=300"
192+
193+
# Clean up
194+
rm -f "$INDEX_FILE"
195+
196+
- name: Summary
197+
run: |
198+
echo "## 🎉 Build Summary" >> $GITHUB_STEP_SUMMARY
199+
echo "" >> $GITHUB_STEP_SUMMARY
200+
echo "### Built Parsers" >> $GITHUB_STEP_SUMMARY
201+
find artifacts-download/ -name "*.so" -o -name "*.dylib" -o -name "*.dll" -o -name "*.wasm" | sort | while read file; do
202+
echo "- \`$(basename "$file")\` ($(du -h "$file" | cut -f1))" >> $GITHUB_STEP_SUMMARY
203+
done
204+
echo "" >> $GITHUB_STEP_SUMMARY
205+
echo "### S3 Location" >> $GITHUB_STEP_SUMMARY
206+
echo "Files uploaded to: \`s3://${{ secrets.S3_BUCKET_NAME }}/tree-sitter/parsers/\`" >> $GITHUB_STEP_SUMMARY

‎.github/workflows/build-single-parser.yml‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
name: Build Single Tree-sitter Parser
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
language:
7+
description: 'Language to build parser for'
8+
required: true
9+
type: choice
10+
options:
11+
- javascript
12+
- typescript
13+
- tsx
14+
- html
15+
- css
16+
- angular
17+
- java
18+
- kotlin
19+
- scala
20+
- python
21+
- go
22+
- rust
23+
- c-sharp
24+
- cpp
25+
- c
26+
- php
27+
- ruby
28+
- elixir
29+
platforms:
30+
description: 'Platforms to build for'
31+
required: true
32+
type: choice
33+
default: 'all'
34+
options:
35+
- all
36+
- linux-only
37+
- macos-only
38+
- windows-only
39+
40+
env:
41+
TREE_SITTER_ABI_VERSION: "15"
42+
43+
jobs:
44+
build:
45+
name: Build ${{ inputs.language }} on ${{ matrix.os }} (${{ matrix.arch }})
46+
runs-on: ${{ matrix.os }}
47+
strategy:
48+
fail-fast: false
49+
matrix:
50+
include:
51+
- os: ubuntu-latest
52+
arch: x64
53+
platform: linux
54+
- os: ubuntu-latest
55+
arch: arm64
56+
platform: linux
57+
- os: macos-latest
58+
arch: x64
59+
platform: darwin
60+
- os: macos-latest
61+
arch: arm64
62+
platform: darwin
63+
- os: windows-latest
64+
arch: x64
65+
platform: win32
66+
67+
steps:
68+
- name: Checkout repository
69+
uses: actions/checkout@v4
70+
71+
- name: Set up QEMU (for ARM64 on Linux)
72+
if: matrix.os == 'ubuntu-latest' && matrix.arch == 'arm64'
73+
uses: docker/setup-qemu-action@v3
74+
with:
75+
platforms: arm64
76+
77+
- name: Set up Node.js
78+
uses: actions/setup-node@v4
79+
with:
80+
node-version: '18'
81+
82+
- name: Set up tree-sitter CLI
83+
uses: tree-sitter/setup-action/cli@v2
84+
85+
- name: Extract language info
86+
id: lang-info
87+
shell: bash
88+
run: |
89+
repo_url=$(jq -r ".languages[\"${{ inputs.language }}\"].repo" languages.json)
90+
ref=$(jq -r ".languages[\"${{ inputs.language }}\"].ref" languages.json)
91+
echo "repo_url=$repo_url" >> $GITHUB_OUTPUT
92+
echo "ref=$ref" >> $GITHUB_OUTPUT
93+
94+
- name: Make scripts executable
95+
shell: bash
96+
run: chmod +x scripts/*.sh
97+
98+
- name: Build parser (Linux ARM64)
99+
if: matrix.os == 'ubuntu-latest' && matrix.arch == 'arm64'
100+
run: |
101+
docker run --rm -v $PWD:/workspace -w /workspace \
102+
--platform linux/arm64 \
103+
node:18-alpine \
104+
sh -c "
105+
apk add --no-cache git bash tree-sitter-cli build-base &&
106+
./scripts/build-parser.sh '${{ inputs.language }}' '${{ steps.lang-info.outputs.repo_url }}' '${{ steps.lang-info.outputs.ref }}' 'artifacts'
107+
"
108+
env:
109+
TREE_SITTER_ABI_VERSION: ${{ env.TREE_SITTER_ABI_VERSION }}
110+
111+
- name: Build parser (Native)
112+
if: matrix.os != 'ubuntu-latest' || matrix.arch != 'arm64'
113+
shell: bash
114+
run: |
115+
./scripts/build-parser.sh "${{ inputs.language }}" "${{ steps.lang-info.outputs.repo_url }}" "${{ steps.lang-info.outputs.ref }}" "artifacts"
116+
env:
117+
TREE_SITTER_ABI_VERSION: ${{ env.TREE_SITTER_ABI_VERSION }}
118+
119+
- name: Upload artifacts
120+
uses: actions/upload-artifact@v4
121+
with:
122+
name: parser-${{ inputs.language }}-${{ matrix.platform }}-${{ matrix.arch }}
123+
path: artifacts/
124+
retention-days: 7
125+
126+
upload:
127+
name: Upload to S3
128+
needs: build
129+
runs-on: ubuntu-latest
130+
if: always() && needs.build.result == 'success'
131+
132+
steps:
133+
- name: Checkout repository
134+
uses: actions/checkout@v4
135+
136+
- name: Download all artifacts
137+
uses: actions/download-artifact@v4
138+
with:
139+
path: artifacts-download/
140+
pattern: parser-${{ inputs.language }}-*
141+
merge-multiple: true
142+
143+
- name: Configure AWS Credentials for S3 Upload
144+
uses: aws-actions/configure-aws-credentials@v4
145+
with:
146+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
147+
aws-region: us-east-1
148+
149+
- name: Upload parsers to S3
150+
run: |
151+
# Upload all parser files to S3
152+
find artifacts-download/ -type f \( -name "*.so" -o -name "*.dylib" -o -name "*.dll" -o -name "*.wasm" \) | while read -r file; do
153+
# Extract the relative path from artifacts directory
154+
relative_path=${file#artifacts-download/}
155+
156+
# Construct S3 key
157+
s3_key="tree-sitter/parsers/tree-sitter-$relative_path"
158+
159+
echo "Uploading: $file -> s3://${{ secrets.S3_BUCKET_NAME }}/$s3_key"
160+
161+
# Upload to S3
162+
aws s3 cp "$file" "s3://${{ secrets.S3_BUCKET_NAME }}/$s3_key" \
163+
--metadata "source-file=$relative_path" \
164+
--cache-control "public, max-age=31536000" \
165+
--content-type "application/octet-stream"
166+
done
167+
168+
- name: Summary
169+
run: |
170+
echo "## 🎉 Build Summary for ${{ inputs.language }}" >> $GITHUB_STEP_SUMMARY
171+
echo "" >> $GITHUB_STEP_SUMMARY
172+
echo "### Built Files" >> $GITHUB_STEP_SUMMARY
173+
find artifacts-download/ -name "*.so" -o -name "*.dylib" -o -name "*.dll" -o -name "*.wasm" | sort | while read file; do
174+
echo "- \`$(basename "$file")\` ($(du -h "$file" | cut -f1))" >> $GITHUB_STEP_SUMMARY
175+
done
176+
echo "" >> $GITHUB_STEP_SUMMARY
177+
echo "### S3 Location" >> $GITHUB_STEP_SUMMARY
178+
echo "Files uploaded to: \`s3://${{ secrets.S3_BUCKET_NAME }}/tree-sitter/parsers/tree-sitter-${{ inputs.language }}/\`" >> $GITHUB_STEP_SUMMARY

0 commit comments

Comments
 (0)