Enhancement: add / fix turndown rules, improve CLI (#43)

* Enhancement: add / fix turndown rules, improve CLI * Fix comment typo Co-authored-by: Toshihiro Kanai <i@miruc.co> * Code Refactoring Co-authored-by: Toshihiro Kanai <i@miruc.co> * Revert CLI description * Lint fix Co-authored-by: Toshihiro Kanai <i@miruc.co>
mirucon · Jul 28, 2022 · 4ead1f8 · 4ead1f8
1 parent 2856241
commit 4ead1f8
Show file tree

Hide file tree

Showing 4 changed files with 150 additions and 26 deletions.
diff --git a/README.md b/README.md
@@ -16,30 +16,38 @@ $ npm i -g wp-handbook-converter
 $ yarn global add wp-handbook-converter
 ```
 
-If you want to run the command without installing the package, use this: `$ npx wp-handbook-converter <team>`
+If you want to run the command without installing the package, use this: `$ npx wp-handbook-converter`
 
-## `wp-handbook-converter` command
-
-```bash
-$ wp-handbook-converter <team>
-```
-
-### options
+## `wp-handbook-converter` command options
 
+- `-t, --team` &lt;team&gt; Specify team name.
 - `-b, --handbook` &lt;handbook&gt; Specify handbook name. (Default "handbook")
 - `-s, --sub-domain` &lt;sub-domain&gt; Specify subdomain name. e.g. "developer" for developer.w.org, "w.org" for w.org (Default "make")
 - `-o, --output-dir` &lt;output-dir&gt; Specify the directory to save files (default `en/`)
+- `-r, --regenerate` &lt;regenerate&gt; If this option is supplied, the directory you specified as output directory will once deleted, and it will regenerate all the files in the directory
 
 ### Example
 
-Get Meetup Handbook
+Get Core Contributor Handbook
 
 ```bash
-$ wp-handbook-converter community --handbook meetup-handbook
+$ wp-handbook-converter --team core
 ```
 
-Get theme developer Handbook
+Get Meetup Organizer Handbook
+
+```bash
+$ wp-handbook-converter --team community --handbook meetup-handbook
+```
 
-```bash∑
-$ wp-handbook-converter '' --handbook theme-handbook --sub-domain developer
+Get theme Handbook
+
+```bash
+$ wp-handbook-converter --handbook theme-handbook --sub-domain developer
+```
+
+Get plugin Handbook
+
+```bash
+$ wp-handbook-converter --handbook plugin-handbook --sub-domain developer
 ```
diff --git a/cli.js b/cli.js
@@ -8,12 +8,122 @@ const { program } = require('commander')
 const mkdirp = require('mkdirp')
 const del = require('del')
 const WPAPI = require('wpapi')
-const turndown = require('turndown')
-const turndownService = new turndown({
+const TurndownService = require('turndown')
+const { tables } = require('turndown-plugin-gfm')
+
+// Languages that can be specified in the code markdown
+const codeLanguages = {
+  css: 'css',
+  bash: 'bash',
+  php: 'php',
+  yaml: 'yaml',
+  xml: 'xml',
+  jscript: 'javascript',
+}
+
+// Rules that remove escapes in code blocks
+const unEscapes = [
+  [/\\\\/g, '\\'],
+  [/\\\*/g, '*'],
+  [/\\-/g, '-'],
+  [/^\\+ /g, '+ '],
+  [/\\=/g, '='],
+  [/\\`/g, '`'],
+  [/\\~~~/g, '~~~'],
+  [/\\\[/g, '['],
+  [/\\\]/g, ']'],
+  [/\\>/g, '>'],
+  [/\\_/g, '_'],
+  [/\&quot;/g, '"'],
+  [/\&lt;/g, '<'],
+  [/\&gt;/g, '>'],
+]
+
+const turndownService = new TurndownService({
   headingStyle: 'atx',
   codeBlockStyle: 'fenced',
   emDelimiter: '*',
 })
+turndownService.use(tables)
+
+// Remove Glossary
+turndownService.addRule('glossary', {
+  filter: (node) => {
+    const classList = node.getAttribute('class')
+    if (classList) {
+      return classList === 'glossary-item-hidden-content'
+    }
+    return false
+  },
+  replacement: () => {
+    return ''
+  },
+})
+
+// Remove code trigger anchor
+turndownService.addRule('code-trigger-anchor', {
+  filter: (node) => {
+    const classList = node.getAttribute('class')
+    if (classList) {
+      return (
+        classList.includes('show-complete-source') ||
+        classList.includes(`less-complete-source`)
+      )
+    }
+    return false
+  },
+  replacement: () => {
+    return ''
+  },
+})
+
+// Transform dt tag to strong tag
+turndownService.addRule('dt-to-strong', {
+  filter: ['dt'],
+  replacement: (content, node, options) => {
+    return options.strongDelimiter + content + options.strongDelimiter
+  },
+})
+
+// Transform pre code block to code markdown
+turndownService.addRule('precode to code', {
+  filter: (node) => {
+    const classList = node.getAttribute('class')
+    const isCode =
+      node.nodeName === 'PRE' && classList && classList.includes('brush:')
+    return isCode
+  },
+  replacement: (content, node) => {
+    const classList = node.getAttribute('class')
+
+    // Search for a language that matches the list of code languages
+    const codeLanguage = Object.keys(codeLanguages).reduce(
+      (currentLanguage, language) => {
+        if (classList.includes(language)) {
+          return codeLanguages[language]
+        }
+        return currentLanguage
+      },
+      undefined
+    )
+
+    // Unescape contents
+    let newContent = unEscapes.reduce((accumulator, unEscape) => {
+      return accumulator.replace(unEscape[0], unEscape[1])
+    }, content)
+
+    // Remove br tag
+    newContent = newContent.replace(/^<br \/>\n\n|<br \/>\n/g, '\n')
+    // Remove first and last paragraph tag
+    newContent = newContent.replace(/^<\/p>|<p>$/g, '')
+    // Remove first new line
+    newContent = newContent.replace(/^\n/, '')
+    // Convert to language-aware markdown
+    newContent = '```' + (codeLanguage ?? '') + '\n' + newContent + '```'
+
+    return newContent
+  },
+})
 
 const getAll = (request) => {
   return request.then((response) => {
@@ -34,6 +144,7 @@ const generateJson = async (
   outputDir,
   regenerate
 ) => {
+  team = team ? `${team}/` : ''
   handbook = handbook ? handbook : 'handbook'
   subdomain = `${
     subdomain ? (subdomain === 'w.org' ? '' : subdomain) : 'make'
@@ -59,12 +170,13 @@ const generateJson = async (
     })
 
   const wp = new WPAPI({
-    endpoint: `https://${subdomain}wordpress.org/${team}/wp-json`,
+    endpoint: `https://${subdomain}wordpress.org/${team}wp-json`,
   })
+
   wp.handbooks = wp.registerRoute('wp/v2', `/${handbook}/(?P<id>)`)
 
   console.log(
-    `Connecting to https://${subdomain}wordpress.org/${team}/wp-json/wp/v2/${handbook}/`
+    `Connecting to https://${subdomain}wordpress.org/${team}wp-json/wp/v2/${handbook}/`
   )
 
   getAll(wp.handbooks()).then(async (allPosts) => {
@@ -82,18 +194,16 @@ const generateJson = async (
         rootPath = `https://${subdomain}wordpress.org/${team}/${handbook}/`
       }
     }
+
     for (const item of allPosts) {
       const path = item.link.split(rootPath)[1].replace(/\/$/, '') || 'index'
       const filePath =
         path.split('/').length > 1
           ? path.substring(0, path.lastIndexOf('/')) + '/'
           : ''
-      // remove <span class='glossary-item-hidden-content'>inner contents
-      const preContent = item.content.rendered.replace(
-        /<span class=\'glossary-item-hidden-content\'>.*?<\/span><\/span>/g,
-        ''
-      )
-      const markdownContent = turndownService.turndown(preContent)
+
+      const content = item.content.rendered
+      const markdownContent = turndownService.turndown(content)
       const markdown = `# ${item.title.rendered}\n\n${markdownContent}`
 
       await mkdirp(`${outputDir}/${filePath}`)
@@ -155,8 +265,8 @@ const generateJson = async (
 
 program
   .version(packageJson.version)
-  .arguments('<team>')
   .description('Generate a menu JSON file for WordPress.org handbook')
+  .option('-t, --team <team>', 'Specify team name')
   .option(
     '-b, --handbook <handbook>',
     'Specify handbook name (default "handbook")'
@@ -173,9 +283,9 @@ program
     '-r --regenerate',
     'If this option is supplied, the directory you specified as output directory will once deleted, and it will regenerate all the files in the directory'
   )
-  .action((team, options) => {
+  .action((options) => {
     generateJson(
-      team,
+      options.team,
       options.handbook,
       options.subDomain,
       options.outputDir,

diff --git a/package.json b/package.json
@@ -16,6 +16,7 @@
     "del": "^6.0.0",
     "mkdirp": "^1.0.0",
     "turndown": "^7.0.0",
+    "turndown-plugin-gfm": "^1.0.2",
     "wpapi": "^1.1.2"
   },
   "devDependencies": {

diff --git a/yarn.lock b/yarn.lock
@@ -486,6 +486,11 @@ to-regex-range@^5.0.1:
   dependencies:
     is-number "^7.0.0"
 
+turndown-plugin-gfm@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz#6f8678a361f35220b2bdf5619e6049add75bf1c7"
+  integrity sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==
+
 turndown@^7.0.0:
   version "7.1.1"
   resolved "https://registry.yarnpkg.com/turndown/-/turndown-7.1.1.tgz#96992f2d9b40a1a03d3ea61ad31b5a5c751ef77f"