Skip to content

Commit 7b19231

Browse files
UziTechUziTech
authored
fix: only create tokens inside tokenizers (#3755)
BREAKING CHANGE: - Change how consecutive text tokens work in lists - Simplify listItem renderer - Checkbox token is added in list tokenizer - Checkbox token add type and raw property - Change loose list text tokens to paragraph type in the list tokenizer
1 parent 244941a commit 7b19231

File tree

11 files changed

+98
-96
lines changed

11 files changed

+98
-96
lines changed

‎eslint.config.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import markedEslintConfig from '@markedjs/eslint-config';
22

33
export default [
44
{
5-
ignores: ['**/lib', '**/public'],
5+
ignores: ['**/lib', '**/public', 'test.js', 'vuln.js'],
66
},
77
...markedEslintConfig,
88
];

‎src/Parser.ts‎

Lines changed: 21 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ export class _Parser<ParserOutput = string, RendererOutput = string> {
3939
/**
4040
* Parse Loop
4141
*/
42-
parse(tokens: Token[], top = true): ParserOutput {
42+
parse(tokens: Token[]): ParserOutput {
4343
let out = '';
4444

4545
for (let i = 0; i < tokens.length; i++) {
@@ -60,62 +60,51 @@ export class _Parser<ParserOutput = string, RendererOutput = string> {
6060
switch (token.type) {
6161
case 'space': {
6262
out += this.renderer.space(token);
63-
continue;
63+
break;
6464
}
6565
case 'hr': {
6666
out += this.renderer.hr(token);
67-
continue;
67+
break;
6868
}
6969
case 'heading': {
7070
out += this.renderer.heading(token);
71-
continue;
71+
break;
7272
}
7373
case 'code': {
7474
out += this.renderer.code(token);
75-
continue;
75+
break;
7676
}
7777
case 'table': {
7878
out += this.renderer.table(token);
79-
continue;
79+
break;
8080
}
8181
case 'blockquote': {
8282
out += this.renderer.blockquote(token);
83-
continue;
83+
break;
8484
}
8585
case 'list': {
8686
out += this.renderer.list(token);
87-
continue;
87+
break;
88+
}
89+
case 'checkbox': {
90+
out += this.renderer.checkbox(token);
91+
break;
8892
}
8993
case 'html': {
9094
out += this.renderer.html(token);
91-
continue;
95+
break;
9296
}
9397
case 'def': {
9498
out += this.renderer.def(token);
95-
continue;
99+
break;
96100
}
97101
case 'paragraph': {
98102
out += this.renderer.paragraph(token);
99-
continue;
103+
break;
100104
}
101105
case 'text': {
102-
let textToken = token;
103-
let body = this.renderer.text(textToken) as string;
104-
while (i + 1 < tokens.length && tokens[i + 1].type === 'text') {
105-
textToken = tokens[++i] as Tokens.Text;
106-
body += ('\n' + this.renderer.text(textToken));
107-
}
108-
if (top) {
109-
out += this.renderer.paragraph({
110-
type: 'paragraph',
111-
raw: body,
112-
text: body,
113-
tokens: [{ type: 'text', raw: body, text: body, escaped: true }],
114-
});
115-
} else {
116-
out += body;
117-
}
118-
continue;
106+
out += this.renderer.text(token);
107+
break;
119108
}
120109

121110
default: {
@@ -170,6 +159,10 @@ export class _Parser<ParserOutput = string, RendererOutput = string> {
170159
out += renderer.image(token);
171160
break;
172161
}
162+
case 'checkbox': {
163+
out += renderer.checkbox(token);
164+
break;
165+
}
173166
case 'strong': {
174167
out += renderer.strong(token);
175168
break;

‎src/Renderer.ts‎

Lines changed: 2 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -77,38 +77,13 @@ export class _Renderer<ParserOutput = string, RendererOutput = string> {
7777
}
7878

7979
listitem(item: Tokens.ListItem): RendererOutput {
80-
let itemBody = '';
81-
if (item.task) {
82-
const checkbox = this.checkbox({ checked: !!item.checked });
83-
if (item.loose) {
84-
if (item.tokens[0]?.type === 'paragraph') {
85-
item.tokens[0].text = checkbox + ' ' + item.tokens[0].text;
86-
if (item.tokens[0].tokens && item.tokens[0].tokens.length > 0 && item.tokens[0].tokens[0].type === 'text') {
87-
item.tokens[0].tokens[0].text = checkbox + ' ' + escape(item.tokens[0].tokens[0].text);
88-
item.tokens[0].tokens[0].escaped = true;
89-
}
90-
} else {
91-
item.tokens.unshift({
92-
type: 'text',
93-
raw: checkbox + ' ',
94-
text: checkbox + ' ',
95-
escaped: true,
96-
});
97-
}
98-
} else {
99-
itemBody += checkbox + ' ';
100-
}
101-
}
102-
103-
itemBody += this.parser.parse(item.tokens, !!item.loose);
104-
105-
return `<li>${itemBody}</li>\n` as RendererOutput;
80+
return `<li>${this.parser.parse(item.tokens)}</li>\n` as RendererOutput;
10681
}
10782

10883
checkbox({ checked }: Tokens.Checkbox): RendererOutput {
10984
return '<input '
11085
+ (checked ? 'checked="" ' : '')
111-
+ 'disabled="" type="checkbox">' as RendererOutput;
86+
+ 'disabled="" type="checkbox"> ' as RendererOutput;
11287
}
11388

11489
paragraph({ tokens }: Tokens.Paragraph): RendererOutput {

‎src/TextRenderer.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,8 @@ export class _TextRenderer<RendererOutput = string> {
4141
br(): RendererOutput {
4242
return '' as RendererOutput;
4343
}
44+
45+
checkbox({ raw }: Tokens.Checkbox): RendererOutput {
46+
return raw as RendererOutput;
47+
}
4448
}

‎src/Tokenizer.ts‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -381,12 +381,10 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
381381
}
382382

383383
let istask: RegExpExecArray | null = null;
384-
let ischecked: boolean | undefined;
385384
// Check for task list items
386385
if (this.options.gfm) {
387386
istask = this.rules.other.listIsTask.exec(itemContents);
388387
if (istask) {
389-
ischecked = istask[0] !== '[ ] ';
390388
itemContents = itemContents.replace(this.rules.other.listReplaceTask, '');
391389
}
392390
}
@@ -395,7 +393,6 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
395393
type: 'list_item',
396394
raw,
397395
task: !!istask,
398-
checked: ischecked,
399396
loose: false,
400397
text: itemContents,
401398
tokens: [],
@@ -416,13 +413,40 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
416413
list.raw = list.raw.trimEnd();
417414

418415
// Item child tokens handled here at end because we needed to have the final item to trim it first
419-
for (let i = 0; i < list.items.length; i++) {
416+
for (const item of list.items) {
420417
this.lexer.state.top = false;
421-
list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []);
418+
item.tokens = this.lexer.blockTokens(item.text, []);
419+
if (item.task) {
420+
const taskRaw = this.rules.other.listTaskCheckbox.exec(item.raw);
421+
if (taskRaw) {
422+
const checkboxToken: Tokens.Checkbox = {
423+
type: 'checkbox',
424+
raw: taskRaw[0] + ' ',
425+
checked: taskRaw[0] !== '[ ]',
426+
};
427+
item.checked = checkboxToken.checked;
428+
if (list.loose) {
429+
if (item.tokens[0] && ['paragraph', 'text'].includes(item.tokens[0].type) && 'tokens' in item.tokens[0] && item.tokens[0].tokens) {
430+
item.tokens[0].raw = checkboxToken.raw + item.tokens[0].raw;
431+
item.tokens[0].text = checkboxToken.raw + item.tokens[0].text;
432+
item.tokens[0].tokens.unshift(checkboxToken);
433+
} else {
434+
item.tokens.unshift({
435+
type: 'paragraph',
436+
raw: checkboxToken.raw,
437+
text: checkboxToken.raw,
438+
tokens: [checkboxToken],
439+
});
440+
}
441+
} else {
442+
item.tokens.unshift(checkboxToken);
443+
}
444+
}
445+
}
422446

423447
if (!list.loose) {
424448
// Check if list should be loose
425-
const spacers = list.items[i].tokens.filter(t => t.type === 'space');
449+
const spacers = item.tokens.filter(t => t.type === 'space');
426450
const hasMultipleLineBreaks = spacers.length > 0 && spacers.some(t => this.rules.other.anyLine.test(t.raw));
427451

428452
list.loose = hasMultipleLineBreaks;
@@ -431,8 +455,13 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
431455

432456
// Set all items to loose if list is loose
433457
if (list.loose) {
434-
for (let i = 0; i < list.items.length; i++) {
435-
list.items[i].loose = true;
458+
for (const item of list.items) {
459+
item.loose = true;
460+
for (const token of item.tokens) {
461+
if (token.type === 'text') {
462+
token.type = 'paragraph';
463+
}
464+
}
436465
}
437466
}
438467

‎src/Tokens.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
export type MarkedToken = (
44
Tokens.Blockquote
55
| Tokens.Br
6+
| Tokens.Checkbox
67
| Tokens.Code
78
| Tokens.Codespan
89
| Tokens.Def
@@ -42,6 +43,8 @@ export namespace Tokens {
4243
}
4344

4445
export interface Checkbox {
46+
type: 'checkbox';
47+
raw: string;
4548
checked: boolean;
4649
}
4750

‎src/rules.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ export const other = {
4848
listReplaceNesting: /^ {1,4}(?=( {4})*[^ ])/g,
4949
listIsTask: /^\[[ xX]\] /,
5050
listReplaceTask: /^\[[ xX]\] +/,
51+
listTaskCheckbox: /\[[ xX]\]/,
5152
anyLine: /\n.*\n/,
5253
hrefBrackets: /^<(.*)>$/,
5354
tableDelimiter: /[:|]/,

‎test/types/marked.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ class ExtendedRenderer extends marked.Renderer {
9393
hr = ({ type, raw }: Tokens.Hr): string => super.hr({ type, raw });
9494
list = ({ type, raw, ordered, start, loose, items }: Tokens.List): string => super.list({ type, raw, ordered, start, loose, items});
9595
listitem = ({ type, raw, task, checked, loose, text, tokens }: Tokens.ListItem): string => super.listitem({ type, raw, task, checked, loose, text, tokens });
96-
checkbox = ({ checked }: Tokens.Checkbox): string => super.checkbox({ checked });
96+
checkbox = ({ type, raw, checked }: Tokens.Checkbox): string => super.checkbox({ type, raw, checked });
9797
paragraph = ({ type, raw, pre, text, tokens }: Tokens.Paragraph): string => super.paragraph({ type, raw, pre, text, tokens });
9898
table = ({ type, raw, align, header, rows }: Tokens.Table): string => super.table({ type, raw, align, header, rows });
9999
tablerow = ({ text }: Tokens.TableRow): string => super.tablerow({ text });

0 commit comments

Comments
 (0)