Repository navigation
Expand file tree
/
Copy pathindex.js
More file actions
2057 lines (1856 loc) · 78.7 KB
/
Copy pathindex.js
File metadata and controls
2057 lines (1856 loc) · 78.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
const fs = require('fs-extra');
const path = require('path');
const { normalizeAiOptions } = require('./lib/ai-providers');
/**
* Docusaurus plugin to copy raw markdown files to build output
* This allows users to view markdown source by appending .md to URLs
*/
// Helper function to extract attribute value from a tag string
function extractAttribute(tagString, attrName) {
const regex = new RegExp(`${attrName}=["']([^"']*)["']`);
const match = tagString.match(regex);
return match ? match[1] : null;
}
// Convert Tabs/TabItem components to readable markdown format
// Handles both standard <Tabs> and custom tab components like <ModelDropdownTabs>
// Each tab's content is re-indented to the <Tabs> tag's column, so tabs nested
// in a list item stay in that item and their fences stay intact.
function convertTabsToMarkdown(content) {
// Match standard Tabs and any custom component ending in "Tabs"
// The body may not open another tabs element, so the innermost block
// matches first and its TabItems don't close the outer block's items.
const tabsPattern =
/^([ \t]*)<(\w*Tabs)\b[^>]*>((?:(?!<\w*Tabs\b)[\s\S])*?)<\/\2>/gm;
const processTabsBlock = (fullMatch, indent, tagName, tabsContent) => {
// Match TabItem with any attribute order by capturing the entire opening tag
const tabItemPattern = /<TabItem\s+([^>]*)>([\s\S]*?)<\/TabItem>/g;
let result = [];
let match;
while ((match = tabItemPattern.exec(tabsContent)) !== null) {
const [, attributes, itemContent] = match;
// Extract label and value from attributes (works regardless of order)
const label = extractAttribute(attributes, 'label');
const value = extractAttribute(attributes, 'value');
// Use label if available, otherwise fall back to value
const displayLabel = label || value || 'Tab';
const cleanContent = dedent(itemContent)
.replace(/^\n+|\s+$/g, '')
.split('\n')
.map((line) => (line ? indent + line : line))
.join('\n');
result.push(`${indent}**${displayLabel}:**\n\n${cleanContent}`);
}
return result.join(`\n\n${indent}---\n\n`);
};
// Process repeatedly, innermost first, to handle nested tabs
let previousContent;
let currentContent = content;
do {
previousContent = currentContent;
currentContent = currentContent.replace(tabsPattern, processTabsBlock);
} while (currentContent !== previousContent);
// Clean up any leftover TabItem or Tabs closing tags that weren't matched
currentContent = currentContent.replace(/<\/TabItem>/g, '');
currentContent = currentContent.replace(/<\/\w*Tabs>/g, '');
// Also clean up orphaned opening tags (in case content wasn't properly structured)
currentContent = currentContent.replace(/<TabItem\s+[^>]*>/g, '');
currentContent = currentContent.replace(/<\w*Tabs[^>]*>/g, '');
return currentContent;
}
// Parse JavaScript array objects into an array of key-value objects
function parseArrayObjects(arrayContent) {
const objects = [];
const objRegex = /\{([^}]+)\}/g;
let objMatch;
while ((objMatch = objRegex.exec(arrayContent)) !== null) {
const obj = {};
// Match property: 'value' or property: "value" or property: value (for booleans/numbers)
const propRegex = /(\w+)\s*:\s*(?:'([^']*)'|"([^"]*)"|([^\s,}]+))/g;
let propMatch;
while ((propMatch = propRegex.exec(objMatch[1])) !== null) {
const key = propMatch[1];
const value = propMatch[2] || propMatch[3] || propMatch[4];
// Skip internal properties like 'default: true'
if (key !== 'default') {
obj[key] = value;
}
}
if (Object.keys(obj).length > 0) objects.push(obj);
}
return objects;
}
// Find the end of an export statement starting at the given position,
// properly tracking nested {}/[] so we don't stop at inner braces.
function findExportEnd(content, startIdx) {
let i = startIdx;
let depth = 0;
let inString = false;
let stringChar = '';
while (i < content.length) {
const ch = content[i];
if (inString) {
if (ch === '\\') { i += 2; continue; }
if (ch === stringChar) inString = false;
i++;
continue;
}
if (ch === '"' || ch === "'" || ch === '`') {
inString = true;
stringChar = ch;
} else if (ch === '{' || ch === '[' || ch === '(') {
depth++;
} else if (ch === '}' || ch === ']' || ch === ')') {
depth--;
if (depth <= 0) {
i++;
if (i < content.length && content[i] === ';') i++;
return i;
}
} else if (depth === 0 && ch === ';') {
return i + 1;
}
i++;
}
return i;
}
// Convert JavaScript export const arrays to readable markdown bullet lists
// Arrays of objects are converted to lists; other exports are removed
function convertExportsToMarkdown(content) {
// First, convert array exports that contain objects to bullet lists
content = content.replace(
/export\s+const\s+(\w+)\s*=\s*\[([\s\S]*?)\]\s*;?/g,
(match, varName, arrayContent) => {
const items = parseArrayObjects(arrayContent);
if (items.length === 0) return '';
let md = `**${varName}:**\n\n`;
items.forEach(obj => {
const pairs = Object.entries(obj)
.map(([k, v]) => `**${k}:** ${v}`)
.join(', ');
md += `- ${pairs}\n`;
});
return md + '\n';
}
);
// Remove all remaining export const statements, tracking nested {}/[] depth
const exportPattern = /^export\s+const\s+\w+\s*=\s*/gm;
let match;
const regions = [];
while ((match = exportPattern.exec(content)) !== null) {
const valueStart = match.index + match[0].length;
const end = findExportEnd(content, valueStart);
regions.push([match.index, end]);
}
for (let i = regions.length - 1; i >= 0; i--) {
content = content.slice(0, regions[i][0]) + content.slice(regions[i][1]);
}
return content;
}
// Read the options of each `export const name = [{ value, default }, ...]`
// array and each `export const name = { key: arrayName, ... }` map.
function parseModelOptionExports(content) {
const arrays = {};
const maps = {};
const exportPattern = /^export\s+const\s+(\w+)\s*=\s*([[{])/gm;
let match;
while ((match = exportPattern.exec(content)) !== null) {
const start = match.index + match[0].length - 1;
const body = content.slice(start, findExportEnd(content, start));
if (match[2] === '[') {
const options = [];
for (const obj of body.match(/\{[^{}]*\}/g) || []) {
const value = obj.match(/\bvalue\s*:\s*['"]([^'"]+)['"]/);
if (value) {
options.push({ value: value[1], isDefault: /\bdefault\s*:\s*true\b/.test(obj) });
}
}
if (options.length) arrays[match[1]] = options;
} else {
const entries = {};
for (const [, key, ref] of body.matchAll(/(\w+)\s*:\s*(\w+)/g)) {
entries[key] = ref;
}
maps[match[1]] = entries;
}
}
return { arrays, maps };
}
// Map each ModelSelector variable to the model it selects on page load: the
// option marked `default: true`, else the first option.
function findModelSelectorDefaults(content) {
const { arrays, maps } = parseModelOptionExports(content);
const defaults = {};
for (const [tag] of content.matchAll(/<ModelSelector\b[^>]*?\/?>/g)) {
const optionsName = tag.match(/modelOptions=\{\s*(\w+)\s*\}/);
const variableName = extractAttribute(tag, 'variableName');
if (!optionsName || !variableName || defaults[variableName]) continue;
const name = optionsName[1];
const options = arrays[name] || arrays[maps[name]?.[variableName]];
if (!options) continue;
defaults[variableName] = (options.find((o) => o.isDefault) || options[0]).value;
}
return defaults;
}
// Convert DynamicCode components to fenced code blocks, filling each
// `{variableName}` placeholder with the model the page selects by default.
function convertDynamicCodeToMarkdown(content, modelDefaults = {}) {
return content.replace(
/^([ \t]*)<DynamicCode\s+([^>]*)>([\s\S]*?)<\/DynamicCode>/gm,
(match, indent, attrs, code) => {
const language = extractAttribute(attrs, 'language') || '';
const title = extractAttribute(attrs, 'title');
const variableName = extractAttribute(attrs, 'variableName') || 'MODEL';
const literal = code.match(/^\s*\{\s*`([\s\S]*)`\s*\}\s*$/);
let cleanCode = literal ? literal[1] : code.trim();
const model = modelDefaults[variableName];
if (model) {
cleanCode = cleanCode.split(`{${variableName}}`).join(model);
}
const opener = '```' + language + (title ? ` title="${title}"` : '');
return indentedFence(indent, opener, evaluateTemplateLiteral(cleanCode));
}
);
}
// Convert ConditionalContent components to labeled markdown sections
function convertConditionalContentToMarkdown(content) {
// Match <ConditionalContent ... condition={(model) => model.includes('Value')} > ... </ConditionalContent>
const pattern = /<ConditionalContent(?:[^>{]|\{[^}]*\})*>\s*([\s\S]*?)<\/ConditionalContent>/gi;
return content.replace(pattern, (match, innerContent) => {
// Extract the condition value from the tag (e.g., 'Llama' from model.includes('Llama'))
const conditionMatch = match.match(/\.includes\s*\(\s*['"]([^'"]+)['"]\s*\)/);
if (conditionMatch) {
const conditionValue = conditionMatch[1];
// Clean up the inner content
const cleanContent = innerContent.trim();
return `**${conditionValue} model:**\n\n${cleanContent}\n`;
}
// If no condition found, just return the inner content
return innerContent.trim();
});
}
// Convert Requirements component to a markdown link using the url attribute
function convertRequirementsToMarkdown(content) {
// Match <Requirements ... url="..." /> (self-closing)
// Handle multiline and various attribute orders
return content.replace(
/<Requirements(?:[^>]*?)url="([^"]+)"[^>]*?\/?>/gi,
(match, url) => {
return `[Read the requirements](${url})`;
}
);
}
// Placeholder tokens are wrapped in a Unicode private-use character, which
// never appears in documentation text, so restoring tokens can't collide with
// a literal `__FENCE_0__` written on the page.
const TOKEN_MARK = '\uE000';
const fenceToken = (index) => `${TOKEN_MARK}FENCE${index}${TOKEN_MARK}`;
const codeToken = (index) => `${TOKEN_MARK}CODE${index}${TOKEN_MARK}`;
const FENCE_TOKEN_PATTERN = new RegExp(`${TOKEN_MARK}FENCE(\\d+)${TOKEN_MARK}`, 'g');
const CODE_TOKEN_PATTERN = new RegExp(`${TOKEN_MARK}CODE(\\d+)${TOKEN_MARK}`, 'g');
// Restore every token in a single pass. Replacing tokens one at a time rescans
// the whole page per token, which is quadratic on pages with thousands of
// inline code spans. The function replacement keeps `$&`, `$'`, and `$$`
// inside code literal.
function restoreTokens(content, pattern, values) {
return content.replace(pattern, (token, index) => values[index] ?? token);
}
// Temporarily replace fenced code blocks so downstream regexes cannot corrupt them.
// The step-9 JSX stripper matches /<[A-Z].../ and treats heredoc markers like
// `<<EOF` as component tags (<EOF ...>) because `[A-Z]` matches the E in EOF.
// It then deletes everything until the next self-closing JSX tag (e.g.
// `<InstallOpenAI />`), leaving an unclosed ``` fence in the emitted .md file.
//
// Fences are found line by line, following CommonMark: an opening fence is 3+
// backticks or tildes (optionally indented, as inside list items), and the
// block ends at the first line holding only a fence of the same character that
// is at least as long. Unclosed fences are left alone. The whole block,
// including its indentation, is replaced by a token, so the round trip is
// lossless.
function protectFencedCodeBlocks(content) {
const blocks = [];
const lines = content.split('\n');
const out = [];
let i = 0;
while (i < lines.length) {
const open = lines[i].match(/^[ \t]*(`{3,}|~{3,})(.*)$/);
// A backtick fence's info string cannot itself contain backticks.
if (open && !(open[1][0] === '`' && open[2].includes('`'))) {
const fenceChar = open[1][0];
const fenceLen = open[1].length;
let close = -1;
for (let j = i + 1; j < lines.length; j++) {
const m = lines[j].match(/^[ \t]*(`{3,}|~{3,})\s*$/);
if (m && m[1][0] === fenceChar && m[1].length >= fenceLen) {
close = j;
break;
}
}
if (close !== -1) {
out.push(fenceToken(blocks.length));
blocks.push(lines.slice(i, close + 1).join('\n'));
i = close + 1;
continue;
}
}
out.push(lines[i]);
i++;
}
return { content: out.join('\n'), blocks };
}
function restoreFencedCodeBlocks(content, blocks) {
return restoreTokens(content, FENCE_TOKEN_PATTERN, blocks);
}
// Evaluate a JS template literal's source the way the browser would: unescape
// `\\`, `\``, and `\$`, and interpolate `${name}` for known substitutions.
// Unknown `${...}` expressions are left as written.
function evaluateTemplateLiteral(source, substitutions = {}) {
return source.replace(/\\([\\`$])|\$\{(\w+)\}/g, (match, escaped, name) => {
if (escaped !== undefined) return escaped;
return Object.prototype.hasOwnProperty.call(substitutions, name)
? String(substitutions[name])
: match;
});
}
// Docusaurus "magic comments" that only control line highlighting in the HTML
// code block (https://docusaurus.io/docs/markdown-features/code-blocks#highlighting-with-comments).
// Docusaurus removes these lines when rendering, so the .md output must too:
// a stray `# highlight-start` inside a backslash-continued shell command breaks
// it when pasted. Covers the comment styles Docusaurus recognizes.
const HIGHLIGHT_DIRECTIVE_LINE =
/^[ \t]*(?:\/\/|#|--|;;|%|\/\*|\{\/\*|<!--)[ \t]*highlight-(?:next-line|start|end)[ \t]*(?:\*\/\}?|-->)?[ \t]*\r?$/;
// Remove highlight directive lines from fenced code blocks only.
function stripHighlightDirectives(content) {
const { content: protectedContent, blocks } = protectFencedCodeBlocks(content);
const cleanedBlocks = blocks.map((block) =>
block
.split('\n')
.filter((line) => !HIGHLIGHT_DIRECTIVE_LINE.test(line))
.join('\n')
);
return restoreFencedCodeBlocks(protectedContent, cleanedBlocks);
}
// Temporarily replace inline code spans (`code`, ``co`de``) with tokens.
// Run this after protectFencedCodeBlocks. A span opens with a run of N
// backticks and closes at the next run of exactly N backticks; it may wrap
// across lines but not across a blank line (paragraph break).
function protectInlineCode(content) {
const spans = [];
const protectedContent = content.replace(
/(?<!`)(`+)(?!`)((?:[^\n]|\n(?![ \t]*\n))+?)(?<!`)\1(?!`)/g,
(match) => {
const token = codeToken(spans.length);
spans.push(match);
return token;
}
);
return { content: protectedContent, spans };
}
function restoreInlineCode(content, spans) {
return restoreTokens(content, CODE_TOKEN_PATTERN, spans);
}
// Apply `transform` to everything except fenced code blocks and inline code.
// Use this for rewrites that must not touch code, such as link rewriting:
// Mojo calls like `fn[params](args)` look exactly like markdown links.
function transformOutsideCode(content, transform) {
const fenced = protectFencedCodeBlocks(content);
const inline = protectInlineCode(fenced.content);
let result = transform(inline.content);
result = restoreInlineCode(result, inline.spans);
return restoreFencedCodeBlocks(result, fenced.blocks);
}
// Emit a fence at `indent`, so a code block inside a list item stays inside it.
// In an .mdx page, MDX strips the list container's indent from template
// literal lines, so by default up to `indent.length` leading spaces are removed
// before re-indenting. Include templates are JS components, where template
// literals are taken verbatim, so they pass `stripContainerIndent: false`.
function indentedFence(indent, opener, code, { stripContainerIndent = true } = {}) {
const body = code
.split('\n')
.map((line, i) => {
if (i > 0 && stripContainerIndent) {
const lead = line.match(/^[ \t]*/)[0].length;
line = line.slice(Math.min(lead, indent.length));
}
return line.length ? indent + line : line;
})
.join('\n');
return `${indent}${opener}\n${body}\n${indent}\`\`\``;
}
// Convert Docusaurus CodeBlock components to fenced code blocks
function convertCodeBlockToMarkdown(content, substitutions = {}, fenceOptions = {}) {
content = content.replace(
/^([ \t]*)<CodeBlock\s+language="([^"]+)"(?:\s+title="([^"]*)")?\s*>\s*\{`([\s\S]*?)`\}\s*<\/CodeBlock>/gm,
(match, indent, language, title, code) => {
const cleanCode = evaluateTemplateLiteral(code, substitutions);
const opener = title
? '```' + language + ' title="' + title + '"'
: '```' + language;
return indentedFence(indent, opener, cleanCode, fenceOptions);
}
);
return content.replace(
/^([ \t]*)<CodeBlock\s+language="([^"]+)"[^>]*>\s*\n?\s*([^<{][\s\S]*?)\s*<\/CodeBlock>/gm,
(match, indent, language, code) =>
indentedFence(indent, '```' + language, code.trim(), fenceOptions)
);
}
// The docs `current` version is the nightly channel, and every released
// version is stable. This matches isStablePage() on both Modular sites.
function versionChannel(versionName) {
return versionName === 'current' ? 'nightly' : 'stable';
}
// Keep only the branch that the HTML page renders for `activeVersion`
// ('nightly' or 'stable'). Without one, keep both branches with labels.
function convertConditionalVersionDocsToMarkdown(content, activeVersion) {
return content.replace(
/([ \t]*)<ConditionalVersionDocs\s+version=["'](\w+)["']\s*>[ \t]*\n?([\s\S]*?)[ \t]*<\/ConditionalVersionDocs>/gi,
(match, indent, version, inner) => {
const body = inner.replace(/\s+$/, '');
if (activeVersion) {
return version === activeVersion ? body : '';
}
const label = version.charAt(0).toUpperCase() + version.slice(1);
return `${indent}**${label}:**\n\n${body}\n`;
}
);
}
// Convert Docusaurus <Link to|href="..."> elements to Markdown links.
function convertLinkComponentsToMarkdown(content) {
return content.replace(
/<Link\s+(?:to|href)=["']([^"']+)["'][^>]*>([\s\S]*?)<\/Link>/g,
(match, url, text) => `[${text.replace(/\s+/g, ' ').trim()}](${url})`
);
}
// Convert <Admonition> to a blockquote at the element's own indentation.
// Each paragraph is joined onto one line, since JSX source wraps prose freely.
function convertAdmonitionToMarkdown(content) {
return content.replace(
/^([ \t]*)<Admonition\s+type="([^"]+)">\s*([\s\S]*?)<\/Admonition>/gm,
(match, indent, type, inner) => {
const paragraphs = convertLinkComponentsToMarkdown(inner)
.replace(/\{' '\}/g, ' ')
.replace(/<\/p>/g, '\n\n')
.replace(/<b>([^<]*)<\/b>/g, '**$1**')
.replace(/<code>([^<]*)<\/code>/g, '`$1`')
.replace(/<[^>]+>/g, '')
.split(/\n[ \t]*\n/)
.map((para) => para.replace(/\s+/g, ' ').trim())
.filter(Boolean);
const label = `**${type}:**`;
if (paragraphs.length && paragraphs[0].toLowerCase().startsWith(label.toLowerCase())) {
paragraphs[0] = paragraphs[0].slice(label.length).trim();
}
const quoted = paragraphs
.map((para, index) => (index === 0 ? `> ${label} ${para}` : `> ${para}`))
.join(`\n${indent}>\n${indent}`);
return `${indent}${quoted || `> ${label}`}\n`;
}
);
}
// Remove the indentation shared by every non-blank line.
function dedent(text) {
const lines = text.split('\n');
const indents = lines
.filter((line) => line.trim().length > 0)
.map((line) => line.match(/^[ \t]*/)[0].length);
const common = indents.length ? Math.min(...indents) : 0;
return lines.map((line) => (line.trim().length ? line.slice(common) : '')).join('\n');
}
// Convert JSX <ol><li> lists to a Markdown numbered list. Text between items,
// such as the `**Nightly:**` label ConditionalVersionDocs emits without an
// active version, is kept at the start of the next item instead of dropped.
function convertOrderedListsToMarkdown(content) {
return content.replace(/<ol>([\s\S]*?)<\/ol>/g, (match, inner) => {
const items = [];
let pending = '';
let last = 0;
for (const item of inner.matchAll(/<li>([\s\S]*?)<\/li>/g)) {
const between = inner.slice(last, item.index).replace(/\s+/g, ' ').trim();
if (between) pending = pending ? `${pending} ${between}` : between;
last = item.index + item[0].length;
const marker = `${items.length + 1}. `;
const pad = ' '.repeat(marker.length);
const lines = dedent(item[1]).replace(/^\n+|\s+$/g, '').split('\n');
if (pending) {
lines[0] = `${pending} ${lines[0]}`;
pending = '';
}
items.push(
lines.map((line, j) => (j === 0 ? marker + line : line ? pad + line : '')).join('\n')
);
}
const trailing = [pending, inner.slice(last).replace(/\s+/g, ' ').trim()]
.filter(Boolean)
.join(' ');
return `\n${items.join('\n\n')}\n${trailing ? `\n${trailing}\n` : ''}`;
});
}
// `<` and `>` inside fenced code are swapped for private-use characters while
// an include's JSX is stripped, so code such as `<your-token>` survives the
// tag-removal regexes. Line structure is untouched, so later dedenting and
// list re-indentation still apply to the code.
const MASKED_LT = '\uE001';
const MASKED_GT = '\uE002';
function maskCodeAngleBrackets(content) {
const { content: protectedContent, blocks } = protectFencedCodeBlocks(content);
const masked = blocks.map((block) => {
const lines = block.split('\n');
return lines
.map((line, i) =>
i === 0 || i === lines.length - 1
? line
: line.replace(/</g, MASKED_LT).replace(/>/g, MASKED_GT)
)
.join('\n');
});
return restoreFencedCodeBlocks(protectedContent, masked);
}
function unmaskCodeAngleBrackets(content) {
return content.replace(/\uE001/g, '<').replace(/\uE002/g, '>');
}
function convertFigureToMarkdown(content) {
// Emits the image path as authored (relative to the source file).
// resolveImagePaths() later turns it into a URL and copies the file.
return content.replace(
/<figure>\s*<img src=\{require\(['"]([^'"]+)['"]\)\.default\}[^>]*alt="([^"]*)"[^>]*\/>\s*<figcaption>([\s\S]*?)<\/figcaption>\s*<\/figure>/g,
(match, imagePath, alt, caption) => {
// Site-static paths are already URLs; resolveImagePaths() leaves them alone.
const imageUrl = imagePath.replace(/^@site\/static\//, '/');
const cleanCaption = caption.replace(/<\/?b>/g, '**').replace(/<\/?strong>/g, '**');
return '\n\n*' + cleanCaption + '*';
}
);
}
// Image alt text may contain one level of brackets, e.g. "OwnedPointer[Int]".
const IMAGE_ALT = String.raw`(?:[^\[\]]|\[[^\[\]]*\])*`;
// Convert remaining JSX/HTML image markup to markdown images, and keep only
// the light variant of Docusaurus light/dark image pairs.
// - <img src={require('./x.png').default} alt="..." /> (any attribute order,
// possibly multi-line) and <img src="..." /> become .
// - <img className="dark" ...> and  are dropped; the #light
// fragment is removed.
// - Bare <figure> wrappers are removed and <figcaption> becomes italic text.
function convertImagesToMarkdown(content) {
content = content.replace(/<img\b([^>]*?)\/?>/g, (match, attrs) => {
const requireSrc = attrs.match(/src=\{\s*require\(\s*['"]([^'"]+)['"]\s*\)(?:\.default)?\s*\}/);
const plainSrc = attrs.match(/src=["']([^"']+)["']/);
const src = requireSrc ? requireSrc[1] : plainSrc ? plainSrc[1] : null;
if (!src) return match;
if (/className=["'][^"']*\bdark\b/.test(attrs)) return '';
const altMatch = attrs.match(/alt=["']([^"']*)["']/);
const alt = altMatch ? altMatch[1] : '';
return `})`;
});
content = content.replace(
/<figcaption>([\s\S]*?)<\/figcaption>/g,
(match, caption) =>
`*${caption
.replace(/<\/?(?:b|strong)>/g, '**')
.replace(/<a\s+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/g, '[$2]($1)')
.replace(/<\/?(?:cite|em|i|span)\b[^>]*>/g, '')
.replace(/\s*\n\s*/g, ' ')
.trim()}*`
);
content = content.replace(/^[ \t]*<\/?figure>[ \t]*\n?/gm, '');
// Drop dark-mode variants (a whole line when the image stands alone).
content = content.replace(
new RegExp(String.raw`^[ \t]*!\[${IMAGE_ALT}\]\([^)\s]*#dark\)[ \t]*\n?`, 'gm'),
''
);
content = content.replace(new RegExp(String.raw`!\[${IMAGE_ALT}\]\([^)\s]*#dark\)`, 'g'), '');
content = content.replace(
new RegExp(String.raw`(!\[${IMAGE_ALT}\]\([^)\s#]*)#light\)`, 'g'),
'$1)'
);
return content;
}
function extractIncludeComponentBody(includeContent) {
let body = includeContent.replace(/^import[\s\S]*?(?=^export default)/m, '');
const returnMatch = body.match(/return\s*\(\s*([\s\S]*)\s*\)\s*;\s*\}\s*$/);
return returnMatch ? returnMatch[1] : body;
}
function processIncludeTemplate(includeContent, substitutions, activeVersion) {
let body = extractIncludeComponentBody(includeContent);
body = body.replace(/\(\{tooltipNightly\}\)/g, '(nightly)');
body = body.replace(/\(\{tooltipStable\}\)/g, '(stable)');
body = body.replace(/<Tooltip[\s\S]*?\/>/g, '');
for (const [key, value] of Object.entries(substitutions)) {
// `${key}` inside template literals and `{key}` in JSX text.
body = body.replace(new RegExp(`\\$?\\{${key}\\}`, 'g'), value);
}
body = convertConditionalVersionDocsToMarkdown(body, activeVersion);
body = convertCodeBlockToMarkdown(body, substitutions, { stripContainerIndent: false });
body = maskCodeAngleBrackets(body);
body = convertAdmonitionToMarkdown(body);
body = convertLinkComponentsToMarkdown(body);
body = body.replace(/<a\s+href="([^"]+)"\s*><code>([^<]*)<\/code><\/a>/g, '[`$2`]($1)');
body = body.replace(/\{' '\}/g, ' ');
body = body.replace(/^([ \t]*)<p>([\s\S]*?)<\/p>/gm, '\n$1$2\n');
body = body.replace(/<code>([^<]*)<\/code>/g, '`$1`');
body = body.replace(/<b>([^<]*)<\/b>/g, '**$1**');
body = convertOrderedListsToMarkdown(body);
body = removeDivTags(body);
body = body.replace(/<\/?>/g, '');
body = convertTabsToMarkdown(body);
body = body.replace(/<[^>]+>/g, '');
body = unmaskCodeAngleBrackets(body);
return collapseBlankLines(dedent(body)).trim();
}
// Map each shared include to the component names a page imports it as, for
// example `import InstallMAX from '../_includes/install-modular.mdx'`.
function findIncludeComponentNames(content) {
const names = {
'install-modular': ['InstallModular'],
'install-openai': ['InstallOpenAI'],
};
const importPattern =
/^import\s+(\w+)\s+from\s+['"][^'"]*\/_includes\/([\w-]+?)(?:\.mdx?)?['"]/gm;
for (const [, name, key] of content.matchAll(importPattern)) {
if (names[key] && !names[key].includes(name)) names[key].push(name);
}
return names;
}
function includeTagPattern(names) {
return new RegExp(`<(?:${names.join('|')})\\b([^>]*?)\\/>`, 'g');
}
function convertInstallModularToMarkdown(content, pluginContext, names = ['InstallModular']) {
if (!pluginContext) return content;
const includeTemplate = pluginContext.includeTemplates?.['install-modular'];
if (!includeTemplate) return content;
return content.replace(includeTagPattern(names), (match) => {
const folder = extractAttribute(match, 'folder') || 'example-project';
const extraMatch = match.match(/extraLibraries=\{(\[[^\]]*\])\}/);
let extraLibs = [];
if (extraMatch) {
try {
extraLibs = JSON.parse(extraMatch[1].replace(/'/g, '"'));
} catch (error) {
extraLibs = [];
}
}
const extraLibsString = extraLibs.length > 0 ? ' ' + extraLibs.join(' ') : '';
const extraLibsDescription =
extraLibs.length > 0 ? ' and other required packages' : '';
return processIncludeTemplate(
includeTemplate,
{ folder, extraLibsString, extraLibsDescription },
pluginContext.activeVersion
);
});
}
function convertInstallOpenAIToMarkdown(content, pluginContext, names = ['InstallOpenAI']) {
if (!pluginContext) return content;
const includeTemplate = pluginContext.includeTemplates?.['install-openai'];
if (!includeTemplate) return content;
return content.replace(includeTagPattern(names), () =>
processIncludeTemplate(includeTemplate, {}, pluginContext.activeVersion)
);
}
// Unwrap MDX components by removing their tags but preserving inner content
function unwrapMdxComponents(content) {
// List of MDX components to unwrap (keeps growing as we find more)
// Note: ConditionalContent is handled separately by convertConditionalContentToMarkdown
// Note: Requirements is handled separately by convertRequirementsToMarkdown
const components = [
'ModelSelector',
'ModelDropdownTabs',
'InstallModular',
];
for (const comp of components) {
// Remove opening tags with any attributes
// Handle JSX expressions in attributes that may contain > inside {...}
// Pattern: match <Component, then any non->{, or {...} blocks, then >
content = content.replace(new RegExp(`<${comp}(?:[^>{]|\\{[^}]*\\})*>`, 'gis'), '');
// Remove closing tags
content = content.replace(new RegExp(`</${comp}>`, 'gi'), '');
}
return content;
}
// Remove div tags while preserving their inner content
function removeDivTags(content) {
// Remove opening div tags with any attributes (including className, style, etc.)
content = content.replace(/<div[^>]*>/gi, '');
// Remove closing div tags
content = content.replace(/<\/div>/gi, '');
return content;
}
// Remove MDX/JSX comments {/* ... */}
function removeMdxComments(content) {
// Match {/* ... */} including multiline comments
return content.replace(/\{\/\*[\s\S]*?\*\/\}/g, '');
}
// Collapse multiple consecutive blank lines into a single blank line
function collapseBlankLines(content) {
// Replace 3+ consecutive newlines (2+ blank lines) with just 2 newlines (1 blank line)
return content.replace(/\n{3,}/g, '\n\n');
}
// Convert details/summary components to readable markdown format
function convertDetailsToMarkdown(content) {
const detailsPattern = /<details>\s*<summary>(<strong>)?([^<]+)(<\/strong>)?<\/summary>([\s\S]*?)<\/details>/g;
return content.replace(detailsPattern, (fullMatch, strongOpen, summaryText, strongClose, detailsContent) => {
// Remove only the indentation shared by every non-blank line, and keep
// blank lines. Trimming each line destroyed code indentation (invalid
// Mojo/Python), and dropping blank lines merged paragraphs and code.
const lines = detailsContent.split('\n');
const indents = lines
.filter((line) => line.trim().length > 0)
.map((line) => line.match(/^[ \t]*/)[0].length);
const commonIndent = indents.length > 0 ? Math.min(...indents) : 0;
const cleanContent = lines
.map((line) => (line.trim().length > 0 ? line.slice(commonIndent).trimEnd() : ''))
.join('\n')
.replace(/^\n+|\n+$/g, '');
return `### ${summaryText.trim()}\n\n${cleanContent}`;
});
}
// String attributes of a JSX tag: name="..." or name='...'. Quoted values may
// contain `>` (a description with <code>), so tags are matched with this too.
const JSX_STRING_ATTR = /([A-Za-z][\w-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
function parseJsxStringAttributes(attrText) {
const attrs = {};
for (const match of attrText.matchAll(JSX_STRING_ATTR)) {
attrs[match[1]] = match[2] ?? match[3];
}
return attrs;
}
function listingCardToMarkdown(attrs) {
const title = (attrs.title || '').trim();
const description = (attrs.description || '')
.replace(/<code>([\s\S]*?)<\/code>/g, '`$1`')
.replace(/<[^>]+>/g, '')
.replace(/\s+/g, ' ')
.trim();
const label = title && attrs.url ? `[${title}](${attrs.url.trim()})` : title;
if (!label) return description ? `- ${description}` : '';
return description ? `- ${label}: ${description}` : `- ${label}`;
}
// Convert a <ListingCards> grid into a Markdown list with one
// "- [title](url): description" item per <ListingCard />. The generic JSX
// stripper can't remove the grid: its lazy match ends at the first card's `/>`,
// which drops the cards and leaves a stray </ListingCards>.
function convertListingCardsToMarkdown(content) {
const quotedOrPlain = `(?:"[^"]*"|'[^']*'|[^"'>])*`;
const cardPattern = new RegExp(`<ListingCard\\b(${quotedOrPlain}?)\\/>`, 'g');
return content.replace(
/^([ \t]*)<ListingCards\b[^>]*>([\s\S]*?)<\/ListingCards>/gm,
(match, indent, inner) => {
const items = [...inner.matchAll(cardPattern)]
.map((card) => listingCardToMarkdown(parseJsxStringAttributes(card[1])))
.filter(Boolean)
.map((item) => indent + item);
return `\n${items.join('\n')}\n`;
}
);
}
// Convert cheat sheet components (<Sheet>, <Panel>, <Intro>, <Yes>, <No>,
// <Partial>) to Markdown. The generic JSX stripper can't handle them: its lazy
// match runs from <Sheet> to the first </Panel> and drops every panel, which
// empties the page.
// - <Sheet> is a wrapper; the page title already comes from front matter.
// - <Panel title summary source> becomes an H3 (the HTML renders an <h3>),
// followed by the summary and source as text. Its children are kept.
// - <Intro> is unwrapped into a paragraph.
// - <Yes>/<No> render "yes"/"no" plus any children; <Partial> renders its
// children, or "partial" when empty.
function convertCheatSheetsToMarkdown(content) {
const attrs = `((?:"[^"]*"|'[^']*'|[^"'>])*?)`;
const selfClosing = (name) => new RegExp(`<${name}\\b${attrs}\\/>`, 'g');
const paired = (name) =>
new RegExp(`<${name}\\b${attrs}>([\\s\\S]*?)<\\/${name}>`, 'g');
const withChildren = (word, children) => {
const text = children.trim();
return text ? `${word} ${text}` : word;
};
return content
.replace(new RegExp(`^[ \\t]*<\\/?Sheet\\b${attrs}>[ \\t]*$`, 'gm'), '')
.replace(
new RegExp(`^([ \\t]*)<Panel\\b${attrs}>[ \\t]*$`, 'gm'),
(match, indent, attrText) => {
const { title = '', summary = '', source = '' } =
parseJsxStringAttributes(attrText);
const lines = [`${indent}### ${title.trim()}`];
if (summary.trim()) lines.push('', `${indent}${summary.trim()}`);
if (source.trim()) {
lines.push('', `${indent}Source: \`${source.trim()}\``);
}
return lines.join('\n');
}
)
.replace(/^[ \t]*<\/Panel>[ \t]*$/gm, '')
.replace(paired('Intro'), (match, attrText, children) => children.trim())
.replace(selfClosing('Yes'), 'yes')
.replace(selfClosing('No'), 'no')
.replace(selfClosing('Partial'), 'partial')
.replace(paired('Yes'), (match, attrText, children) =>
withChildren('yes', children)
)
.replace(paired('No'), (match, attrText, children) =>
withChildren('no', children)
)
.replace(
paired('Partial'),
(match, attrText, children) => children.trim() || 'partial'
);
}
// Extract title from YAML frontmatter
function extractTitleFromFrontmatter(content) {
const frontmatterMatch = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
if (frontmatterMatch) {
const frontmatter = frontmatterMatch[1];
// Match title: value (with optional quotes)
const titleMatch = frontmatter.match(/^title:\s*["']?([^"'\n]+)["']?\s*$/m);
if (titleMatch) {
return titleMatch[1].trim();
}
}
return null;
}
// Clean markdown content for raw display - remove MDX/Docusaurus-specific syntax
// `assetBasePath`: URL prefix for rewriting `img/` relative images (defaults to docsPath)
function cleanMarkdownForDisplay(
content,
filepath,
docsPath = '/docs/',
assetBasePath = undefined,
pluginContext = undefined
) {
const imgUrlBase =
assetBasePath !== undefined ? assetBasePath : docsPath;
// Get the directory path for this file (relative to docs root)
const fileDir = filepath.replace(/[^/]*$/, ''); // Remove filename, keep directory
// Extract title from frontmatter before stripping it
const title = extractTitleFromFrontmatter(content);
// 1. Strip YAML front matter (--- at start, content, then ---)
content = content.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, '');
// Read import names and model defaults before imports and exports go.
const includeNames = findIncludeComponentNames(content);
const modelDefaults = findModelSelectorDefaults(content);
// 2. Remove import statements (MDX imports). The first pattern catches
// named imports split across lines: `import {\n Sheet,\n Panel,\n} from '...';`
content = content.replace(
/^import\s+(?:[\w$]+\s*,\s*)?\{[^}]*\}\s*from\s+['"][^'"]*['"];?[ \t]*$/gm,
''
);
content = content.replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, '');
// 3. Remove MDX/JSX comments {/* ... */}
content = removeMdxComments(content);
// 4. Convert DynamicCode components to fenced code blocks
content = convertDynamicCodeToMarkdown(content, modelDefaults);
// 4. Convert JavaScript export arrays to readable bullet lists
content = convertExportsToMarkdown(content);
// 5. Convert ConditionalContent to labeled sections
content = convertConditionalContentToMarkdown(content);
// 6. Convert Requirements component to link
content = convertRequirementsToMarkdown(content);
// Expand shared MDX includes into markdown (improves .md/HTML parity for agents)
content = convertInstallModularToMarkdown(
content,
pluginContext,
includeNames['install-modular']
);
content = convertInstallOpenAIToMarkdown(
content,
pluginContext,
includeNames['install-openai']
);
content = convertConditionalVersionDocsToMarkdown(
content,
pluginContext?.activeVersion
);
content = convertCodeBlockToMarkdown(content);
content = convertAdmonitionToMarkdown(content);
content = transformOutsideCode(content, convertLinkComponentsToMarkdown);
// Convert Button components with DocLink to markdown links
content = content.replace(
/<Button\s+component=\{DocLink\}\s+to=['"]([^'"]+)['"]>\s*([\s\S]*?)\s*<\/Button>/g,
'[$2]($1)'
);
// 7. Unwrap MDX components (remove tags, preserve inner content)
content = unwrapMdxComponents(content);
// 7. Remove div tags (preserve inner content)
content = removeDivTags(content);
// 7b. Strip <section> tags (preserve inner content, same as div removal)
content = content.replace(/<section[^>]*>/gi, '');
content = content.replace(/<\/section>/gi, '');
// 7c. Convert <b> tags to markdown bold
content = content.replace(/<b>([^<]*)<\/b>/g, '**$1**');
// 7d. Remove zero-width spaces (U+200B) used in API reference docs
content = content.replace(/\u200B/g, '');
// 7. Convert HTML images to markdown
// Pattern: <p align="center"><img src={require('./path').default} alt="..." width="..." /></p>
content = content.replace(
/<p align="center">\s*\n?\s*<img src=\{require\(['"]([^'"]+)['"]\)\.default\} alt="([^"]*)"(?:\s+width="[^"]*")?\s*\/>\s*\n?\s*<\/p>/g,
(match, imagePath, alt) => {
// Clean the path: remove @site/static prefix
const cleanPath = imagePath.replace('@site/static/', '/');
return ``;
}
);
// 4. Convert YouTube iframes to text links
content = content.replace(
/<iframe[^>]*src="https:\/\/www\.youtube\.com\/embed\/([a-zA-Z0-9_-]+)[^"]*"[^>]*title="([^"]*)"[^>]*>[\s\S]*?<\/iframe>/g,
'Watch the video: [$2](https://www.youtube.com/watch?v=$1)'
);
// 5. Clean HTML5 video tags - keep HTML but add fallback text
content = content.replace(
/<video[^>]*>\s*<source src=["']([^"']+)["'][^>]*>\s*<\/video>/g,
'<video controls>\n <source src="$1" type="video/mp4" />\n <p>Video demonstration: $1</p>\n</video>'
);