Inline markdown tokenized

js/markdown.js

@@ -25,6 +25,9 @@ class _MDHAlign {
 	static Center = new _MDHAlign('Center');
 	static Right = new _MDHAlign('Right');
 
+	/** @var {String} */
+	name;
+
 	constructor(name) {
 		this.name = name;
 	}
@@ -43,6 +46,62 @@ class _MDHAlign {
 	}
 }
 
+class _MDTokenType {
+	static Text = new _MDTokenType('Text');
+	static Whitespace = new _MDTokenType('Whitespace');
+
+	static Underscore = new _MDTokenType('Underscore');
+	static Asterisk = new _MDTokenType('Asterisk');
+	static Slash = new _MDTokenType('Slash');
+	static Tilde = new _MDTokenType('Tilde');
+	static Bang = new _MDTokenType('Bang');
+	static Backtick = new _MDTokenType('Backtick');
+
+	static Label = new _MDTokenType('Label'); // content=label
+	static URL = new _MDTokenType('URL'); // content=URL, extra=title
+	static Email = new _MDTokenType('Email'); // content=email address, extra=title
+	static SimpleLink = new _MDTokenType('SimpleLink'); // content=URL
+	static SimpleEmail = new _MDTokenType('SimpleEmail'); // content=email address
+	static Footnote = new _MDTokenType('Footnote'); // content=symbol
+
+	static HTMLTag = new _MDTokenType('HTMLTag'); // content=tag string, tag=_MDHTMLTag
+
+	#name;
+
+	get name() {
+		return this.#name;
+	}
+
+	constructor(name) {
+		this.#name = name;
+	}
+
+	toString() {
+		return this.constructor.name + `.${this.#name}`;
+	}
+}
+class _MDToken {
+
+	/** @var {String} */
+	original;
+	/** @var {_MDTokenType} */
+	type;
+	/** @var {String|null} */
+	content;
+	/** @var {String|null} */
+	extra;
+	/** @var {_MDHTMLTag|null} */
+	tag;
+
+	constructor(original, type, content=null, extra=null, tag=null) {
+		this.original = original;
+		this.type = type;
+		this.content = content;
+		this.extra = extra;
+		this.tag = tag;
+	}
+}
+
 class _MDSpan {
 	toHTML(config) {
 		throw new Error(self.constructor.name + ".toHTML not implemented");
@@ -620,27 +679,30 @@ class _MDInline extends _MDBlock {
 	}
 }
 
-// Blocks that immediately start a new block
-// - Headers
-// - Blockquote
-// - Code block   ```\ncode\n```
-// Blocks that need blank line first
-// - HR   ---   - - -   ***   * * * * * *
-// - Lists
-// - Table
-// - Code block   [4+spaces]code
-// - Definition list   term\n: definition\n: alternate def
-// Unknown blocks
-// - Footnotes   some text[^1]   [^1]: first footnote content
-// - Abbreviations   *[HTML]: Hyper Text
-// Inline styles
-// - Links
-// - Italic
-// - Bold
-// - `code`
-// - Strikethrough
-// - Images   ![alt text](url){.cssclass}
-// - Literals   \*
+class _MDHTMLTag {
+	/** @var {String} */
+	fullTag;
+	/** @var {String} */
+	tagName;
+	/** @var {Boolean} */
+	isCloser;
+	/** @var {Object} */
+	attributes;
+
+	/**
+	 * @param {String} fullTag
+	 * @param {String} tagName
+	 * @param {Boolean} isCloser
+	 * @param {Object} attributes
+	 */
+	constructor(fullTag, tagName, isCloser, attributes) {
+		this.fullTag = fullTag;
+		this.tagName = tagName;
+		this.isCloser = isCloser;
+		this.attributes = attributes;
+	}
+}
+
 class _MDState {
 	/** @var {String[]} */
 	lines = [];
@@ -753,11 +815,316 @@ class Markdown {
 		return null;
 	}
 
+	static #baseURLRegex = /(?:(?:(?:[a-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[a-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[a-z0-9\.\-]+)(?:(?:\/[\+~%\/\.\w\-_]*)?\??(?:[\-\+=&;%@\.\w_]*)#?(?:[\.\!\/\\\w]*))?)/i;
+	static #baseEmailRegex = /(?:(?:[^<>()\[\]\\.,;:\s@"]+(?:\.[^<>()\[\]\\.,;:\s@"]+)*)|(?:".+"))@(?:(?:\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(?:(?:[a-z\-0-9]+\.)+[a-z]{2,}))/i;
+
+	static #footnoteWithTitleRegex = /^\[\^\s*([^\]"]+?)\s+"(.*?)"\s*\]/;
+	static #footnoteRegex = /^\[\^\s*([^\]]+?)\s*\]/;
+	static #labelRegex = /^\[(.*?)\]/;
+	static #urlWithTitleRegex = /^\((\S+?)\s+"(.*?)"\)/i;
+	static #urlRegex = /^\((\S+?)\)/i;
+	static #emailWithTitleRegex = new RegExp("^\\(\\s*(" + this.#baseEmailRegex.source + ")\\s+\"(.*?)\"\\s*\\)", "i");
+	static #emailRegex = new RegExp("^\\(\\s*(" + this.#baseEmailRegex.source + ")\\s*\\)", "i");
+	static #simpleURLRegex = new RegExp("^<" + this.#baseURLRegex.source + ">", "i");
+	static #simpleEmailRegex = new RegExp("^<" + this.#baseEmailRegex.source + ">", "i");
+
+	/**
+	 * @param {String} line
+	 * @returns {_MDHTMLTag|null} HTML tag if possible
+	 */
+	static #htmlTag(line) {
+		let expectOpenBracket = 0;
+		let expectCloserOrName = 1;
+		let expectName = 2;
+		let expectAttributeNameOrEnd = 3;
+		let expectEqualsOrAttributeOrEnd = 4;
+		let expectAttributeValue = 5;
+		let expectCloseBracket = 6;
+
+		var isCloser = false;
+		var tagName = '';
+		var attributeName = '';
+		var attributeValue = '';
+		var attributeQuote = null;
+		var attributes = {};
+		var fullTag = null;
+		let endAttribute = function() {
+			if (attributeName.length > 0) {
+				if (attributeValue.length > 0 || attributeQuote) {
+					attributes[attributeName] = attributeValue;
+				} else {
+					attributes[attributeName] = true;
+				}
+			}
+			attributeName = '';
+			attributeValue = '';
+			attributeQuote = null;
+		};
+
+		var expect = expectOpenBracket;
+		for (var p = 0; p < line.length && fullTag === null; p++) {
+			let ch = line.substring(p, p + 1);
+			switch (expect) {
+				case expectOpenBracket:
+					if (ch != '<') return null;
+					expect = expectCloserOrName;
+					break;
+				case expectCloserOrName:
+					if (ch == '/') {
+						isCloser = true;
+					} else {
+						p--;
+					}
+					expect = expectName;
+					break;
+				case expectName:
+					if (tagName.length == 0) {
+						if (/[a-z]/i.exec(ch) === null) return null;
+						tagName += ch;
+					} else {
+						if (/[a-z0-9]/i.exec(ch)) {
+							tagName += ch;
+						} else {
+							p--;
+							expect = (isCloser) ? expectCloseBracket : expectAttributeNameOrEnd;
+						}
+					}
+					break;
+				case expectAttributeNameOrEnd:
+					if (attributeName.length == 0) {
+						if (/\s/.exec(ch)) {
+							// skip whitespace
+						} else if (ch == '/') {
+							expect = expectCloseBracket;
+						} else if (ch == '>') {
+							fullTag = line.substring(0, p + 1);
+							break;
+						} else if (/[a-z0-9-]/i.exec(ch)) {
+							attributeName += ch;
+						} else {
+							return null;
+						}
+					} else if (/\s/.exec(ch)) {
+						expect = expectEqualsOrAttributeOrEnd;
+					} else if (ch == '/') {
+						endAttribute();
+						expect = expectCloseBracket;
+					} else if (ch == '>') {
+						endAttribute();
+						fullTag = line.substring(0, p + 1);
+						break;
+					} else if (ch == '=') {
+						expect = expectAttributeValue;
+					} else if (/[a-z0-9-]/i.exec(ch)) {
+						attributeName += ch;
+					} else {
+						return null;
+					}
+					break;
+				case expectEqualsOrAttributeOrEnd:
+					if (ch == '=') {
+						expect = expectAttributeValue;
+					} else if (/\s/.exec(ch)) {
+						// skip whitespace
+					} else if (ch == '/') {
+						expect = expectCloseBracket;
+					} else if (ch == '>') {
+						fullTag = line.substring(0, p + 1);
+						break;
+					} else if (/[a-z]/i.exec(ch)) {
+						endAttribute();
+						expect = expectAttributeNameOrEnd;
+						p--;
+					}
+					break;
+				case expectAttributeValue:
+					if (attributeValue.length == 0) {
+						if (attributeQuote === null) {
+							if (/\s/.exec(ch)) {
+								// skip whitespace
+							} else if (ch == '"' || ch == "'") {
+								attributeQuote = ch;
+							} else {
+								attributeQuote = '';
+								p--;
+							}
+						} else {
+							if (ch === attributeQuote) {
+								// Empty string
+								endAttribute();
+								expect = expectAttributeNameOrEnd;
+							} else if (attributeQuote === '' && (ch == '/' || ch == '>')) {
+								return null;
+							} else {
+								attributeValue += ch;
+							}
+						}
+					} else {
+						if (ch === attributeQuote) {
+							endAttribute();
+							expect = expectAttributeNameOrEnd;
+						} else if (attributeQuote === '' && /\s/.exec(ch)) {
+							endAttribute();
+							expect = expectAttributeNameOrEnd;
+						} else {
+							attributeValue += ch;
+						}
+					}
+					break;
+				case expectCloseBracket:
+					if (/\s/.exec(ch)) {
+						// ignore whitespace
+					} else if (ch == '>') {
+						fullTag = line.substring(0, p + 1);
+						break;
+					}
+					break;
+			}
+		}
+		if (fullTag === null) return null;
+		endAttribute();
+		return new _MDHTMLTag(fullTag, tagName, isCloser, attributes);
+	}
+
+	/**
+	 * @param {String} line
+	 * @returns {_MDToken[]} tokens
+	 */
+	static #tokenize(line) {
+		var tokens = [];
+		var text = '';
+		var expectLiteral = false;
+		var groups = null;
+		var tag = null;
+		const endText = function() {
+			if (text.length == 0) return;
+			let textGroups = /^(\s*)(?:(\S|\S.*\S)(\s*?))?$/.exec(text);
+			if (textGroups !== null) {
+				if (textGroups[1].length > 0) {
+					tokens.push(new _MDToken(textGroups[1], _MDTokenType.Whitespace, textGroups[1]));
+				}
+				if (textGroups[2] !== undefined && textGroups[2].length > 0) {
+					tokens.push(new _MDToken(textGroups[2], _MDTokenType.Text, textGroups[2]));
+				}
+				if (textGroups[3] !== undefined && textGroups[3].length > 0) {
+					tokens.push(new _MDToken(textGroups[3], _MDTokenType.Whitespace, textGroups[3]));
+				}
+			} else {
+				tokens.push(new _MDToken(text, _MDTokenType.Text, text));
+			}
+			text = '';
+		}
+		for (var p = 0; p < line.length; p++) {
+			let ch = line.substring(p, p + 1);
+			let remainder = line.substring(p);
+			if (expectLiteral) {
+				// TODO: Check for only allowable escapable characters
+				text += ch;
+				continue;
+			}
+			if (ch == '\\') {
+				expectLiteral = true;
+			} else if (ch == '*') {
+				endText();
+				tokens.push(new _MDToken(ch, _MDTokenType.Asterisk));
+			} else if (ch == '_') {
+				endText();
+				tokens.push(new _MDToken(ch, _MDTokenType.Underscore));
+			} else if (ch == '`') {
+				endText();
+				tokens.push(new _MDToken(ch, _MDTokenType.Backtick));
+			} else if (ch == '~') {
+				endText();
+				tokens.push(new _MDToken(ch, _MDTokenType.Tilde));
+			} else if (ch == '!') {
+				endText();
+				tokens.push(new _MDToken(ch, _MDTokenType.Bang));
+
+			} else if (groups = this.#footnoteWithTitleRegex.exec(remainder)) {
+				// Footnote with title   [^1 "Foo"]
+				endText();
+				tokens.push(new _MDToken(groups[0], _MDTokenType.Footnote, groups[1], groups[2]));
+				p += groups[0].length - 1;
+			} else if (groups = this.#footnoteRegex.exec(remainder)) {
+				// Footnote without title   [^1]
+				endText();
+				tokens.push(new _MDToken(groups[0], _MDTokenType.Footnote, groups[1]));
+				p += groups[0].length - 1;
+			} else if (groups = this.#labelRegex.exec(remainder)) {
+				// Label/ref for link/image   [Foo]
+				endText();
+				tokens.push(new _MDToken(groups[0], _MDTokenType.Label, groups[1]));
+				p += groups[0].length - 1;
+			} else if (groups = this.#urlWithTitleRegex.exec(remainder)) {
+				// URL with title   (https://foo "Bar")
+				endText();
+				tokens.push(new _MDToken(groups[0], _MDTokenType.URL, groups[1], groups[2]));
+				p += groups[0].length - 1;
+			} else if (groups = this.#emailWithTitleRegex.exec(remainder)) {
+				// Email address with title   (user@example.com  "Foo")
+				endText();
+				tokens.push(new _MDToken(groups[0], _MDTokenType.Email, groups[1]));
+				p += groups[0].length - 1;
+			} else if (groups = this.#urlRegex.exec(remainder)) {
+				// URL   (https://example.com)
+				endText();
+				tokens.push(new _MDToken(groups[0], _MDTokenType.URL, groups[1]));
+				p += groups[0].length - 1;
+			} else if (groups = this.#emailRegex.exec(remainder)) {
+				// Email   (user@example.com)
+				endText();
+				tokens.push(new _MDToken(groups[0], _MDTokenType.Email, groups[1]));
+				p += groups[0].length - 1;
+			} else if (groups = this.#simpleURLRegex.exec(remainder)) {
+				// Simple URL   <https://example.com>
+				endText();
+				tokens.push(new _MDToken(groups[0], _MDTokenType.SimpleLink, groups[1]));
+				p += groups[0].length - 1;
+			} else if (groups = this.#simpleEmailRegex.exec(remainder)) {
+				// Simple email   <user@example.com>
+				endText();
+				tokens.push(new _MDToken(groups[0], _MDTokenType.SimpleEmail, groups[1]));
+				p += groups[0].length - 1;
+			} else if (tag = this.#htmlTag(remainder)) {
+				endText();
+				tokens.push(new _MDToken(tag.fullTag, _MDTokenType.HTMLTag, tag.fullTag, null, tag));
+				p += tag.fullTag.length - 1;
+			} else {
+				text += ch;
+			}
+		}
+		endText();
+		return tokens;
+	}
+
 	/**
 	 * @param {_MDState} state
+	 * @param {String} line
 	 * @returns {_MDBlock|null}
 	 */
 	static #readInline(state, line) {
+		let tokens = this.#tokenize(line);
+		var spans = [];
+// - Link   [text](https://url)
+// - Emphasis   *emphasized* or _emphasized_ or /emphasized/
+// - Strong   **bold** or __bold__
+// - Inline code   `code` or ``code``
+// - Strikethrough   ~strike~ or ~~strike~~
+// - Image   ![alt text](https://image){.cssclass}
+// - Footnote (inline)   [^1]: footnote text
+// - Abbreviation (inline)
+//
+// Tokens:
+// _
+// *
+// /
+// ~
+// [label] or []
+// (url)
+// !
+// <>
+// `
+
 		return new _MDInline(line);
 	}