513 lines
17 KiB
JavaScript
513 lines
17 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
const fs = require("fs");
|
|
const path = require("path");
|
|
const remark = require("remark");
|
|
const { Objects, Strings } = require("./lib/functions");
|
|
const languages = require("./languages");
|
|
const commandLineArgs = require("command-line-args");
|
|
|
|
const optionDefinitions = [
|
|
{
|
|
name: "input",
|
|
multiple: true,
|
|
defaultValue: ["./fpb/books", "./fpb/casts", "./fpb/courses", "./fpb/more"],
|
|
},
|
|
{ name: "output", defaultValue: "./parser/fpb.json" },
|
|
];
|
|
|
|
const excludes = [
|
|
"README.md",
|
|
"CONTRIBUTING.md",
|
|
"CODE_OF_CONDUCT.md",
|
|
"SUMMARY.md",
|
|
];
|
|
|
|
/**
|
|
* Parses the contents of a heading from remark-parse into a readable format.
|
|
*
|
|
* @param {Array<Object>} children - an array of AST items defined by remark-parse for
|
|
* the content of headings (H1..H7)
|
|
*
|
|
* @returns {string} an string with the name of the section related with the input heading
|
|
*/
|
|
function getSectionNameFromHeadingContent(children) {
|
|
// visit nodes in depth
|
|
const walk = (children, depth) =>
|
|
children.reduce((text, node, index) => {
|
|
if (!node || !node.type) return text; // not AST, maybe plain text
|
|
switch (node.type) {
|
|
//
|
|
// meaningfull nodes
|
|
//
|
|
case "emphasis":
|
|
case "strong":
|
|
text += Strings.templater(remarkTokenAST(node), {
|
|
text: walk(node.children, depth + 1),
|
|
});
|
|
break;
|
|
case "inlineCode":
|
|
case "text":
|
|
text += Strings.templater(remarkTokenAST(node), {
|
|
text: node.value,
|
|
});
|
|
break;
|
|
//
|
|
// skipped nodes
|
|
//
|
|
case "heading":
|
|
case "html":
|
|
case "link":
|
|
case "list":
|
|
case "paragraph":
|
|
default:
|
|
break;
|
|
}
|
|
return text;
|
|
}, "");
|
|
|
|
return walk(children, 0);
|
|
}
|
|
|
|
/**
|
|
* Parses the contents of a link from remark-parse into a readable format.
|
|
*
|
|
* @param {Array<Object>} children - an array of AST items defined by remark-parse for
|
|
* the content of a link (A)
|
|
*
|
|
* @returns {string} an string with the text of the related input link
|
|
*/
|
|
function getLinkTextFromLinkNodes(children) {
|
|
// visit nodes in depth
|
|
const walk = (children, depth) => {
|
|
// not AST, maybe plain text
|
|
if (!Array.isArray(children)) return Objects.toString(children);
|
|
// AST children array nodes
|
|
return children.reduce((text, node, index) => {
|
|
if (!node || !node.type) return text; // not AST, maybe plain text
|
|
switch (node.type) {
|
|
//
|
|
// rebuild meaningfull nodes
|
|
//
|
|
case "image":
|
|
text += Strings.templater(remarkTokenAST(node), {
|
|
text: node.alt || node.title,
|
|
url: node.url,
|
|
});
|
|
break;
|
|
case "inlineCode":
|
|
case "text":
|
|
text += Strings.templater(remarkTokenAST(node), {
|
|
text: node.value,
|
|
});
|
|
break;
|
|
case "emphasis":
|
|
case "strong":
|
|
text += Strings.templater(remarkTokenAST(node), {
|
|
text: walk(node.children, depth + 1),
|
|
});
|
|
break;
|
|
//
|
|
// skipped nodes
|
|
//
|
|
default:
|
|
console.log(
|
|
"getLinkTextFromLinkNodes::skipped",
|
|
depth,
|
|
node.type,
|
|
node
|
|
);
|
|
break;
|
|
}
|
|
return text;
|
|
}, "");
|
|
};
|
|
|
|
return walk(children, 0);
|
|
}
|
|
|
|
/**
|
|
* Gets the template related with AST remark-parse node.
|
|
* @param {Object} node - AST node defined by remark-parse
|
|
* @returns {string} - the template string
|
|
*/
|
|
function remarkTokenAST(node) {
|
|
if (node && node.type) {
|
|
switch (node.type) {
|
|
case "break": // {type: 'break', position: {...}}
|
|
return "<br/>";
|
|
case "emphasis": // {type: 'emphasis', children: [...], position: {...}}
|
|
return Strings.wrap("{{text}}", "_");
|
|
case "heading": // {type: 'heading', depth: 1, children: [...], position: {...}}
|
|
return ["#".repeat(node.depth || 0), "{{text}}"].join("");
|
|
case "image": // {type: 'image', title: '...', url: '...', alt: '...', position: {...}}
|
|
return "";
|
|
case "inlineCode": // {type: 'inlineCode', value: '...', position: {...}}
|
|
return Strings.wrap("{{text}}", "`");
|
|
case "link": // {type: 'link', title: '...', url: '...', children: [...], position: {...}}
|
|
return "[{{text}}]({{url}})";
|
|
case "list": // {type: 'list', ordered: false, start: null, spread: false, children: [...], position: {...}}
|
|
case "listItem": // {type: 'listItem', spread: false, checked: null, children: [...], position: {...}}
|
|
// TODO: generate token for list/listItem
|
|
break;
|
|
case "strong": // {type: 'strong', children: [...], position: {...}}
|
|
return Strings.wrap("{{text}}", "**");
|
|
case "html": // {type: 'html', value: '...', position: {...}}
|
|
case "paragraph": // {type: 'paragraph', children: [...], position: {...}}
|
|
case "text": // {type: 'text', value: '...', position: {...}}
|
|
return Strings.wrap("{{text}}"); // identity
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
throw new Error("Unrecognized remark node type: " + (node && node.type));
|
|
}
|
|
|
|
/**
|
|
* Parses a list item generated from remark-parse into a readable format.
|
|
*
|
|
* remark-parse parses a markdown file into a long, intricate json.
|
|
* Many fields in this json either give information we do not care
|
|
* about or does not go into enough detail. This function parses the
|
|
* output of remark-parse into a format preferred by this project,
|
|
* indicating authors, notes, and links etc.
|
|
*
|
|
* @param {Object} listItem - a listItem in AST format defined by remark-parse
|
|
*
|
|
* @return {Object} Returns an Object containing details about the piece of media.
|
|
*/
|
|
function parseListItem(listItem) {
|
|
let entry = {};
|
|
let s = ""; // If we need to build up a string over multiple listItem elements
|
|
let leftParen,
|
|
rightParen = -1; // If we need to parse parenthesized text
|
|
// head of listItem = url, the rest is "other stuff"
|
|
const [link, ...otherStuff] = listItem;
|
|
entry.url = link.url;
|
|
// link.children || link.value => weak way to check if link.type === "link"
|
|
entry.title = getLinkTextFromLinkNodes(link.children || link.value);
|
|
// remember to get OTHER STUFF!! remember there may be multiple links!
|
|
for (let i of otherStuff) {
|
|
if (s === "") {
|
|
// this is almost always, except for when we are parsing a multi-element note
|
|
if (i.type === "text" && i.value.slice(0, 3) === " - ") {
|
|
// author found
|
|
let parenIndex = i.value.indexOf("(");
|
|
if (parenIndex === -1) {
|
|
entry.author = i.value.slice(3).trim();
|
|
} else {
|
|
entry.author = i.value.slice(3, parenIndex).trim(); // go from " - " until the first "("
|
|
}
|
|
}
|
|
if (
|
|
i.type === "emphasis" &&
|
|
i.children[0].value.slice(0, 1) === "(" &&
|
|
i.children[0].value.slice(-1) === ")"
|
|
) {
|
|
// access notes found (currently assumes exactly one child, so far this is always the case)
|
|
entry.accessNotes = i.children[0].value.slice(1, -1);
|
|
}
|
|
if (i.type === "link") {
|
|
// other links found
|
|
if (entry.otherLinks === undefined) entry.otherLinks = [];
|
|
entry.otherLinks.push({
|
|
title: Strings.stripParens(getLinkTextFromLinkNodes(i.children)),
|
|
url: i.url,
|
|
});
|
|
// entry.otherLinks = [...entry.otherLinks, {title: i.children[0].value, url: i.url}]; // <-- i wish i could get this syntax to work with arrays
|
|
}
|
|
if (i.type === "text" && i.value.indexOf("(") !== -1) {
|
|
// notes found (currently assumes no nested parentheses)
|
|
if (entry.notes === undefined) entry.notes = [];
|
|
leftParen = i.value.indexOf("(");
|
|
while (leftParen != -1) {
|
|
rightParen = i.value.indexOf(")", leftParen);
|
|
if (rightParen === -1) {
|
|
// there must be some *emphasis* found
|
|
s += i.value.slice(leftParen);
|
|
break;
|
|
}
|
|
entry.notes.push(i.value.slice(leftParen + 1, rightParen));
|
|
leftParen = i.value.indexOf("(", rightParen);
|
|
}
|
|
}
|
|
} else {
|
|
// for now we assume that all previous ifs are mutually exclusive with this, may polish later
|
|
if (i.type === "emphasis") {
|
|
// this is the emphasis, add it in boldface and move on
|
|
s += "*" + i.children[0].value + "*";
|
|
} else if (i.type === "link") {
|
|
// something has gone terribly wrong. this book must be viewed and edited manually.
|
|
entry.manualReviewRequired = true;
|
|
break;
|
|
} else {
|
|
// hopefully this is the end of the note
|
|
let rightParen = i.value.indexOf(")");
|
|
if (rightParen === -1) {
|
|
// we have to go AGAIN
|
|
s += i.value;
|
|
} else {
|
|
// finally, we have reached the end of the note
|
|
entry.notes.push(
|
|
Strings.stripParens(s + i.value.slice(0, rightParen + 1))
|
|
);
|
|
s = "";
|
|
// this is a copypaste of another block of code. probably not a good thing tbh.
|
|
leftParen = i.value.indexOf("(");
|
|
while (leftParen != -1) {
|
|
rightParen = i.value.indexOf(")", leftParen);
|
|
if (rightParen === -1) {
|
|
// there must be some *emphasis* found
|
|
s += i.value.slice(leftParen);
|
|
break;
|
|
}
|
|
entry.notes.push(i.value.slice(leftParen + 1, rightParen));
|
|
leftParen = i.value.indexOf("(", rightParen);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return entry;
|
|
}
|
|
|
|
/**
|
|
* Determines the language a certain file is based on the format
|
|
* from the FreeEbookFoundation GitHub page
|
|
* @param {String} filename A filename in the format kept by all markdown files on the FreeProgrammingBooks Github
|
|
* @returns {String} The language the file is
|
|
*/
|
|
function getLangFromFilename(filename) {
|
|
const dash = filename.lastIndexOf("-");
|
|
const dot = filename.lastIndexOf(".");
|
|
let lang = filename.slice(dash + 1, dot).replace(/_/, "-");
|
|
let isSubject = false;
|
|
if (!languages.hasOwnProperty(lang)) {
|
|
if (/^[a-z]{2}$/.test(lang) || /^[a-z]{2}-[A-Z]{2}$/.test(lang)) {
|
|
return "";
|
|
}
|
|
// console.log(lang);
|
|
if (lang === "subjects") {
|
|
isSubject = true;
|
|
}
|
|
lang = "en";
|
|
}
|
|
return { lang: lang, isSubject: isSubject };
|
|
}
|
|
|
|
/**
|
|
* Gets all markdown files in a directory,
|
|
* @param {String} dir - A directory path
|
|
* @returns A list of all md files in a directory, excluding those in the excludes array
|
|
*/
|
|
function getFilesFromDir(dir) {
|
|
return fs
|
|
.readdirSync(dir)
|
|
.filter(
|
|
(file) => path.extname(file) === ".md" && excludes.indexOf(file) === -1
|
|
)
|
|
.map((file) => path.join(dir, file));
|
|
}
|
|
|
|
/**
|
|
* Retrieves the folder name from a string representing a directory and file
|
|
* @param {String} str - A string representing a path directory alike in the format "./directory/file"
|
|
* @returns {String} The extracted directory name
|
|
*/
|
|
function getMediaTypeFromDirectoryPath(str) {
|
|
str = path.resolve(str); // sanatize and expand (OS independent)
|
|
let type;
|
|
if (fs.lstatSync(str).isDirectory()) {
|
|
// if path is itself a directory, use it name as result
|
|
type = path.basename(str);
|
|
} else {
|
|
// if not... parent/previous slug is always a directory; extract this part
|
|
// path.sep: Windows -> "\", Unix -> "/"
|
|
type = str.split(path.sep).slice(-2, -1).join(path.sep);
|
|
}
|
|
return type;
|
|
}
|
|
|
|
/**
|
|
* Turns a single markdown file into the json structure needed
|
|
* @param {path} doc - a single file path to a markdown file
|
|
* @returns {object} Json object of entries in the md file
|
|
*/
|
|
function parseMarkdown(doc) {
|
|
let tree = remark.parse(doc).children;
|
|
let sections = []; // This will go into root object later
|
|
let errors = [];
|
|
let currentDepth = 3; // used to determine if the last heading was an h4 or h3
|
|
|
|
// find where Index ends
|
|
// probably could be done better, review later
|
|
let i = 0,
|
|
count = 0;
|
|
for (i; i < tree.length; i++) {
|
|
if (tree[i].type == "heading" && tree[i].depth == "3") count++;
|
|
if (count == 2) break;
|
|
}
|
|
|
|
tree.slice(i).forEach((item) => {
|
|
// Start iterating after Index
|
|
try {
|
|
if (item.type == "heading") {
|
|
const sectionName = getSectionNameFromHeadingContent(item.children);
|
|
if (sectionName == "Index") return;
|
|
if (item.depth == 3) {
|
|
// Heading is an h3
|
|
currentDepth = 3;
|
|
// create section record
|
|
let newSection = {
|
|
section: sectionName,
|
|
entries: [],
|
|
subsections: [],
|
|
};
|
|
// Push the section to the output array
|
|
sections.push(newSection);
|
|
} else if (item.depth == 4) {
|
|
// Heading is an h4
|
|
currentDepth = 4;
|
|
// create subsection record
|
|
let newSubsection = {
|
|
section: sectionName,
|
|
entries: [],
|
|
};
|
|
// Add to subsection array of most recent h3
|
|
sections[sections.length - 1].subsections.push(newSubsection);
|
|
}
|
|
} else if (item.type == "list") {
|
|
item.children.forEach((listItem) => {
|
|
let content = listItem.children[0].children; // gets array containing a remark-link and a remark-paragraph
|
|
// if(content[0].type !== 'link'){ // SKIPS OVER bad formatting
|
|
// return;
|
|
// }
|
|
if (currentDepth == 3) {
|
|
let contentJson = parseListItem(content);
|
|
sections[sections.length - 1].entries.push(contentJson); // add the entry to most recent h3
|
|
} else if (currentDepth == 4) {
|
|
let lastSection = sections.length - 1;
|
|
let lastSubSec = sections[lastSection].subsections.length - 1;
|
|
let contentJson = parseListItem(content);
|
|
sections[lastSection].subsections[lastSubSec].entries.push(
|
|
contentJson
|
|
); // add entry to most recent h4
|
|
}
|
|
});
|
|
}
|
|
} catch (e) {
|
|
// if there was an error while parsing, print the error to an error log
|
|
// looks really ugly, maybe try to refine output later
|
|
let errStart = JSON.stringify(item.position.start.line);
|
|
let errEnd = JSON.stringify(item.position.end.line);
|
|
str = `Error at line ${errStart} - line ${errEnd}.`;
|
|
errors.push(str);
|
|
}
|
|
});
|
|
return { sections: sections, errors: errors };
|
|
}
|
|
|
|
/**
|
|
* Parses a single directory's md files and converts them into usable json
|
|
* @param {String} directory A string pointing to a directory
|
|
* @returns {Object} An object containing two values, dirJson and dirErrors.
|
|
* dirJson contains all data that was successfully parsed from
|
|
* the markdown files. dirErrors contains all entries that had
|
|
* an error occur while parsing.
|
|
*/
|
|
function parseDirectory(directory) {
|
|
let dirChildren = []; // this will hold the output each markdown doc
|
|
let dirErrors = []; //contains error for a given directory
|
|
|
|
let mediaType = getMediaTypeFromDirectoryPath(directory);
|
|
const filenames = getFilesFromDir(path.resolve(directory));
|
|
filenames.forEach((filename) => {
|
|
const doc = fs.readFileSync(filename);
|
|
let { sections, errors } = parseMarkdown(doc); // parse the markdown document
|
|
const { lang, isSubject } = getLangFromFilename(filename);
|
|
|
|
// Entries
|
|
let docJson = {
|
|
language: {
|
|
code: lang,
|
|
name: languages[lang],
|
|
},
|
|
index: {},
|
|
sections: sections,
|
|
};
|
|
if (lang === "en") docJson.language.isSubject = isSubject;
|
|
dirChildren.push(docJson);
|
|
|
|
// Errors
|
|
if (errors.length !== 0) {
|
|
let docErrors = {
|
|
file: path.basename(filename),
|
|
errors: errors,
|
|
};
|
|
dirErrors.push(docErrors);
|
|
}
|
|
});
|
|
|
|
// File entries
|
|
let dirJson = {
|
|
type: mediaType,
|
|
index: {},
|
|
children: dirChildren,
|
|
};
|
|
|
|
// Errors
|
|
|
|
return { dirJson: dirJson, dirErrors: dirErrors };
|
|
}
|
|
|
|
/**
|
|
* Reads all given directories for markdown files and prints the parsed json in the output directory
|
|
*
|
|
* @param {Array} directories A list of strings of directories to scan for markdown files
|
|
* @param {String} output A string for the path that the output should be placed in
|
|
*/
|
|
function parseAll(directories, output) {
|
|
let rootChildren = []; // this will hold the output of each directory
|
|
let rootErrors = [];
|
|
|
|
directories.forEach((directory) => {
|
|
let { dirJson, dirErrors } = parseDirectory(directory);
|
|
rootChildren.push(dirJson);
|
|
if (dirErrors.length !== 0) {
|
|
rootErrors.push({
|
|
directory: path.basename(directory),
|
|
files: dirErrors,
|
|
});
|
|
}
|
|
});
|
|
|
|
// ALl entries
|
|
let rootJson = {
|
|
type: "root",
|
|
children: rootChildren,
|
|
};
|
|
|
|
// Errors
|
|
let allErrors = {
|
|
type: "root",
|
|
directories: rootErrors,
|
|
};
|
|
fs.writeFileSync(output, JSON.stringify(rootJson, null, 3), function (err) {
|
|
if (err) {
|
|
console.log(err);
|
|
}
|
|
});
|
|
// fs.writeFileSync(
|
|
// "./parser/fpb.log",
|
|
// JSON.stringify(allErrors, null, 3),
|
|
// function (err) {
|
|
// if (err) {
|
|
// console.log(err);
|
|
// }
|
|
// }
|
|
// );
|
|
}
|
|
|
|
let { input, output } = commandLineArgs(optionDefinitions);
|
|
parseAll(input, output);
|