Skip to content

Commit f62a9f6

Browse files
authored
Speed up fastscan package (#249)
I looked a little bit into why the `fastscan` package was slower than I expected. And it's because I accidentally left stuff in the lexer that is totally unused but is the primary source of allocations and memory usage (and thus also bad for latency since allocations aren't free). Once upon a time, in an older version of the lexer in protoparse (on which this fastscan lexer is based), this stuff was used to capture the actual raw text for a token. The new lexer in protocompile uses a completely different approach, to reduce the memory usage. But it's completely unused here. Removing it basically doubles the throughput of `fastscan` and causes it to use 1/3rd as many allocations and 1/4th as much memory. ``` -- before -- BenchmarkGoogleapisFastScan-10 5 210397558 ns/op 468123099 B/op 11217259 allocs/op BenchmarkGoogleapisFastScan-10 5 202723933 ns/op 468156548 B/op 11217275 allocs/op -- after -- BenchmarkGoogleapisFastScan-10 12 93760795 ns/op 111257610 B/op 3710000 allocs/op BenchmarkGoogleapisFastScan-10 12 94548743 ns/op 111259758 B/op 3710010 allocs/op ```
1 parent f4c4a6f commit f62a9f6

File tree

1 file changed

+0
-25
lines changed

1 file changed

+0
-25
lines changed

parser/fastscan/lexer.go

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@ func (t tokenType) describe() string {
7878

7979
type runeReader struct {
8080
rr *bufio.Reader
81-
marked []rune
8281
unread []rune
8382
err error
8483
}
@@ -90,40 +89,19 @@ func (rr *runeReader) readRune() (r rune, err error) {
9089
if len(rr.unread) > 0 {
9190
r := rr.unread[len(rr.unread)-1]
9291
rr.unread = rr.unread[:len(rr.unread)-1]
93-
if rr.marked != nil {
94-
rr.marked = append(rr.marked, r)
95-
}
9692
return r, nil
9793
}
9894
r, _, err = rr.rr.ReadRune()
9995
if err != nil {
10096
rr.err = err
101-
} else if rr.marked != nil {
102-
rr.marked = append(rr.marked, r)
10397
}
10498
return r, err
10599
}
106100

107101
func (rr *runeReader) unreadRune(r rune) {
108-
if rr.marked != nil {
109-
if rr.marked[len(rr.marked)-1] != r {
110-
panic("unread rune is not the same as last marked rune!")
111-
}
112-
rr.marked = rr.marked[:len(rr.marked)-1]
113-
}
114102
rr.unread = append(rr.unread, r)
115103
}
116104

117-
func (rr *runeReader) startMark(initial rune) {
118-
rr.marked = []rune{initial}
119-
}
120-
121-
func (rr *runeReader) endMark() string {
122-
m := string(rr.marked)
123-
rr.marked = rr.marked[:0]
124-
return m
125-
}
126-
127105
type lexer struct {
128106
input *runeReader
129107
// start of the next rune in the input
@@ -161,8 +139,6 @@ func (l *lexer) adjustPos(c rune) {
161139
}
162140

163141
func (l *lexer) Lex() (tokenType, any, error) {
164-
l.input.endMark() // reset, just in case
165-
166142
for {
167143
c, err := l.input.readRune()
168144
if err == io.EOF {
@@ -181,7 +157,6 @@ func (l *lexer) Lex() (tokenType, any, error) {
181157
continue
182158
}
183159

184-
l.input.startMark(c)
185160
l.prevTokenLine, l.prevTokenCol = l.curLine, l.curCol
186161
l.adjustPos(c)
187162
if c == '.' {

0 commit comments

Comments
 (0)