Merge pull request #81 from jlelong/use_yaml

Switch to yaml for base grammar
jlelong · Mar 30, 2024 · cf291a4 · cf291a4
2 parents 70d2764 + 6e49a45
commit cf291a4
Show file tree

Hide file tree

Showing 13 changed files with 6,837 additions and 5,571 deletions.
diff --git a/build/all.js b/build/all.js
@@ -1,11 +1,11 @@
 const bailout = require('./generate-bailout')
 const md = require('./latex-md')
-const codeblocks = require('./generate-grammar-blocks')
+const buildLatexGrammars = require('./generate-latex-grammars')
 
 const arg = process.argv[2]
 if (arg === undefined || arg === 'build-latex') {
     /* Make sure to generate grammar blocks first */
-    codeblocks()
+    buildLatexGrammars()
 }
 if (arg === 'update-external' || arg === undefined) {
     bailout()

diff --git a/build/generate-grammar-blocks.js → build/generate-latex-grammars.js b/build/generate-grammar-blocks.js → build/generate-latex-grammars.js
@@ -1,5 +1,7 @@
 const fs = require('fs')
 const path = require('path')
+const yaml = require('js-yaml')
+
 
 const mintedEnvs = ['minted', 'lstlisting', 'pyglist']
 const robustExternalizeEnvs = ['CacheMeCode', 'PlaceholderPathFromCode\\*?', 'PlaceholderFromCode\\*?', 'SetPlaceholderCode\\*?']
@@ -42,19 +44,33 @@ const codeLanguages = [
     {name: ['sympycode', 'sympyverbatim', 'sympyblock', 'sympyconcode', 'sympyconsole', 'sympyconverbatim'], source: 'source.python'},
 ]
 
+/**
+ * Convert an input yaml file to a json output file
+ * @param {string} inputfile a yaml file name
+ * @param {string} outputfile a json file name
+ */
+function convertYamlToJson(inputfile, outputfile) {
+    try {
+        const grammar = yaml.load(fs.readFileSync(inputfile, {encoding: 'utf-8'}))
+        fs.writeFileSync(outputfile, JSON.stringify(grammar, undefined, 4))
+    } catch (error) {
+        console.log(error)
+    }
+}
 
 /**
- * Indent text and replace spaces indentation with tabs
+ * Indent text
  * @param {number} count The number of tabs to insert at the beginning of each line
  * @param {string} text A multiline text
  */
 function indent(count, text) {
-    const indent = new Array(count + 1).join('\t')
-    return text.replace(/ {4}/gm, '\t').replace(/^/gm, indent)
+    const indent = new Array(count + 1).join(' ')
+    return text.replace(/^/gm, indent)
 }
 
 function escapeBackSlash(text) {
-    return text.replaceAll('\\', '\\\\')
+    return text
+    // return text.replaceAll('\\', '\\\\')
 }
 
 /**
@@ -75,51 +91,30 @@ function generateCodeBlock(envNames, source, contentName=undefined) {
     const beginRule = `\\s*\\\\begin\\{${envNameRegex}\\*?\\}(?:\\[[a-zA-Z0-9_-]*\\])?(?=\\[|\\{|\\s*$)`
     const endRule = `\\s*\\\\end\\{${envNameRegex}\\*?\\}`
 
-    const jsonCode = `{
-    "begin": "${beginRule}",
-    "end": "${endRule}",
-    "captures": {
-        "0": {
-            "patterns": [
-                {
-                    "include": "#begin-env-tokenizer"
-                }
-            ]
-        }
-    },
-    "patterns": [
-        {
-            "include": "#multiline-optional-arg-no-highlight"
-        },
-        {
-            "begin": "(?:\\G|(?<=\\]))(\\{)",
-            "beginCaptures": {
-                "1": {
-                    "name": "punctuation.definition.arguments.begin.latex"
-                }
-            },
-            "end": "(\\})",
-            "endCaptures": {
-                "1": {
-                    "name": "punctuation.definition.arguments.end.latex"
-                }
-            },
-            "contentName": "variable.parameter.function.latex"
-        },
-        {
-            "begin": "^(?=\\s*)",
-            "end": "^\\s*(?=\\\\end\\{${envNameRegex}\\*?\\})",
-            "contentName": "${source}",
-            "patterns": [
-                {
-                    "include": "${source}"
-                }
-            ]
-        }
-    ]
-}`
-    return escapeBackSlash(jsonCode)
-
+    const yamlCode = `- begin: ${beginRule}
+  end: ${endRule}
+  captures:
+    '0':
+      patterns:
+      - include: '#begin-env-tokenizer'
+  patterns:
+  - include: '#multiline-optional-arg-no-highlight'
+  - begin: (?:\\G|(?<=\\]))(\\{)
+    beginCaptures:
+      '1':
+        name: punctuation.definition.arguments.begin.latex
+    end: (\\})
+    endCaptures:
+      '1':
+        name: punctuation.definition.arguments.end.latex
+    contentName: variable.parameter.function.latex
+  - begin: ^(?=\\s*)
+    end: ^\\s*(?=\\\\end\\{${envNameRegex}\\*?\\})
+    contentName: ${source}
+    patterns:
+    - include: ${source}`
+
+    return escapeBackSlash(yamlCode)
 }
 
 /**
@@ -136,29 +131,20 @@ function generateMintedBlock(envNames, language, source, contentName=undefined)
     var languageRegex = '(?:' + language.join('|') + ')'
     var envNameRegex = '(?:' + envNames.join('|') + ')'
 
-    const jsonCode = `{
-    "begin": "(?:\\G|(?<=\\]))(\\{)(${languageRegex})(\\})",
-    "beginCaptures": {
-        "1": {
-            "name": "punctuation.definition.arguments.begin.latex"
-        },
-        "2": {
-            "name": "variable.parameter.function.latex"
-        },
-        "3": {
-            "name": "punctuation.definition.arguments.end.latex"
-        }
-    },
-    "end": "^\\s*(?=\\\\end\\{${envNameRegex}\\})",
-    "contentName": "${contentName}",
-    "patterns": [
-        {
-            "include": "${source}"
-        }
-    ]
-}`
-
-    return escapeBackSlash(jsonCode)
+    const yamlCode = `- begin: (?:\\G|(?<=\\]))(\\{)(${languageRegex})(\\})
+  beginCaptures:
+   '1':
+    name: punctuation.definition.arguments.begin.latex
+   '2':
+    name: variable.parameter.function.latex
+   '3':
+    name: punctuation.definition.arguments.end.latex
+  end: ^\\s*(?=\\\\end\\{${envNameRegex}\\})
+  contentName: ${contentName}
+  patterns:
+  - include: ${source}`
+
+    return escapeBackSlash(yamlCode)
 }
 
 /**
@@ -175,60 +161,54 @@ function generateRobustExternalizeBlock(envNames, language, source, contentName=
     var languageRegex = '(?i:' + language.join('|') + ')'
     var envNameRegex = '(?:RobExt)?' + '(?:' + envNames.join('|') + ')'
 
-    const jsonCode = `{
-    "begin": "\\G(\\{)(?:__|[a-z\\s]*)${languageRegex}",
-    "end": "(?=\\\\end\\{${envNameRegex}\\})",
-    "beginCaptures": {
-        "1": {
-            "name": "punctuation.definition.arguments.begin.latex"
-        }
-    },
-    "patterns": [
-        {
-            "begin": "\\G",
-            "end": "(\\})\\s*$",
-            "endCaptures": {
-                "1": {
-                    "name": "punctuation.definition.arguments.end.latex"
-                }
-            },
-            "patterns": [
-                {
-                     "include": "text.tex#braces"
-                },
-                {
-                    "include": "$base"
-                }
-            ]
-        },
-        {
-            "begin": "^(\\s*)",
-            "end": "^\\s*(?=\\\\end\\{${envNameRegex}\\})",
-            "contentName": "${contentName}",
-            "patterns": [
-                {
-                    "include": "${source}"
-                }
-            ]
-        }
-    ]
-}`
-
-    return escapeBackSlash(jsonCode)
+    const yamlCode = `- begin: \\G(\\{)(?:__|[a-z\\s]*)${languageRegex}
+  end: (?=\\\\end\\{${envNameRegex}\\})
+  beginCaptures:
+    '1':
+      name: punctuation.definition.arguments.begin.latex
+  patterns:
+  - begin: \\G
+    end: (\\})\\s*$
+    endCaptures:
+      '1':
+        name: punctuation.definition.arguments.end.latex
+    patterns:
+    - include: text.tex#braces
+    - include: $base
+  - begin: ^(\\s*)
+    end: ^\\s*(?=\\\\end\\{${envNameRegex}\\})
+    contentName: ${contentName}
+    patterns:
+    - include: ${source}`
+
+    return escapeBackSlash(yamlCode)
 }
 
 function main() {
-    console.log('Generating LaTeX.tmLanguage from data/')
-    var mintedDefinitions = mintedLanguages.map(language => generateMintedBlock(mintedEnvs, language.language, language.source, language?.contentName)).join(',\n')
-    var codeDefinitions = codeLanguages.map(language => generateCodeBlock(language.name, language.source, language?.contentName)).join(',\n')
-    var robustExternalizeDefinitions = robustExternalizeLanguages.map(language => generateRobustExternalizeBlock(robustExternalizeEnvs, language.language, language.source, language?.contentName)).join(',\n')
-
-    let text = fs.readFileSync(path.join(__dirname, '..', 'syntaxes', 'data', 'LaTeX.tmLanguage.json'), {encoding: 'utf8'})
-    text = text.replace(/^\s*\{\{includeMintedblocks\}\}/gm, indent(4, mintedDefinitions))
-    text = text.replace(/^\s*\{\{includeRobustExternalizeBlocks\}\}/gm, indent(4, robustExternalizeDefinitions))
-    text = text.replace(/^\s*\{\{includeCodeBlocks\}\}/gm, indent(2, codeDefinitions))
-    fs.writeFileSync(path.join(__dirname, '..', 'syntaxes', 'LaTeX.tmLanguage.json'), text)
+    const syntaxesDir = path.join(__dirname, '..', 'syntaxes')
+    const syntaxesSrcDir = path.join(syntaxesDir, 'src')
+
+    console.log('Generating BibTeX.tmLanguage from src/')
+    convertYamlToJson(path.join(syntaxesSrcDir, 'BibteX.tmLanguage.yaml'), path.join(syntaxesDir, 'BibteX.tmLanguage.json'))
+
+    console.log('Generating TeX.tmLanguage from src/')
+    convertYamlToJson(path.join(syntaxesSrcDir, 'TeX.tmLanguage.yaml'), path.join(syntaxesDir, 'TeX.tmLanguage.json'))
+
+    var mintedDefinitions = mintedLanguages.map(language => generateMintedBlock(mintedEnvs, language.language, language.source, language?.contentName)).join('\n')
+    var codeDefinitions = codeLanguages.map(language => generateCodeBlock(language.name, language.source, language?.contentName)).join('\n')
+    var robustExternalizeDefinitions = robustExternalizeLanguages.map(language => generateRobustExternalizeBlock(robustExternalizeEnvs, language.language, language.source, language?.contentName)).join('\n')
+
+    console.log('Generating LaTeX.tmLanguage from src/')
+    try {
+        let yamlGrammar = fs.readFileSync(path.join(syntaxesSrcDir, 'LaTeX.tmLanguage.base.yaml'), {encoding: 'utf-8'})
+        yamlGrammar = yamlGrammar.replace(/^\s{2}- includeRobustExternalizeBlocks: ''/m, indent(2, robustExternalizeDefinitions))
+        yamlGrammar = yamlGrammar.replace(/^- includeCodeBlocks: ''/m, codeDefinitions)
+        yamlGrammar = yamlGrammar.replace(/^\s{2}- includeMintedblocks: ''/m, indent(2, mintedDefinitions))
+        const latexGrammar = yaml.load(yamlGrammar)
+        fs.writeFileSync(path.join(syntaxesDir, 'LaTeX.tmLanguage.json'), JSON.stringify(latexGrammar, undefined, 4))
+    } catch (error) {
+        console.log(error)
+    }
 }
 
-
 module.exports = main
diff --git a/build/json2yaml.py b/build/json2yaml.py
@@ -0,0 +1,21 @@
+import yaml, json
+from pathlib import Path
+
+def json2yaml(infile):
+    outfile = Path(infile).with_suffix('.yaml')
+    try: 
+        with open(infile, encoding='utf8') as fd:
+            c = json.load(fd)
+    except FileNotFoundError:
+        print(f'Cannot open file {infile}')
+        return
+    try:
+        with open(outfile, mode="w", encoding="utf8") as fd:
+            yaml.dump(c, fd, indent=2)
+    except FileNotFoundError:
+        print(f'Cannot open file {infile}')
+        return
+
+json2yaml('../syntaxes/Bibtex.tmLanguage.json')
+json2yaml('../syntaxes/TeX.tmLanguage.json')
+json2yaml('../syntaxes/data/LaTeX.tmLanguage.json')
diff --git a/build/latex-md.js b/build/latex-md.js
@@ -1,6 +1,11 @@
 const fs = require('fs')
 const vel = require('vscode-extend-language')
 
+/*
+ * Generate a combined LaTeX Markdown grammar to be sued in LaTeX document
+ * using the markdown package.
+ */
+
 async function insertLaTeXGrammar(url, latexScope, newScopeName, newGrammarFile) {
     const grammar = JSON.parse(await vel.download(url))
     if(!grammar) {

diff --git a/package.json b/package.json
@@ -144,6 +144,7 @@
     "got": "12.5.2",
     "mocha": "10.2.0",
     "textmate-bailout": "1.1.0",
-    "vscode-extend-language": "^0.1.1"
+    "vscode-extend-language": "^0.1.1",
+    "js-yaml":"^4.1.0"
   }
 }