Adding PHP InlineTests and TokenTests

jstest/InlineTests.js

 
 	test_strongEmphasis_tightNesting() {
 		let markdown = 'Lorem ***ipsum*** dolor';
-		let expected1 = 'Lorem <strong><em>ipsum</em></strong> dolor';
-		let expected2 = 'Lorem <em><strong>ipsum</strong></em> dolor';
+		let expected = 'Lorem <strong><em>ipsum</em></strong> dolor';
 		let actual = this.md(markdown);
-		this.assertTrue(actual == expected1 || actual == expected2);
+		this.assertEqual(actual, expected);
 	}
 
 	test_strongEmphasis_lopsidedNesting1() {

php/markdown.php

 	public const baseEmailRegex = '(?:(?:[^<>()\\[\\]\\\\.,;:\\s@"]+(?:\\.[^<>()\\[\\]\\\\.,;:\\s@"]+)*)|(?:".+"))@(?:(?:\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}])|(?:(?:[a-z\\-0-9]+\\.)+[a-z]{2,}))';
 
 	/**
+	 * Escapes special HTML characters.
+	 *
+	 * @param string $str  string to escape
+	 * @param bool $encodeNewlinesAsBreaks  whether to convert newline characters to `<br>` tags
+	 * @return string  escaped HTML
+	 */
+	public static function escapeHTML($str, $encodeNewlinesAsBreaks=false) {
+		if (!is_string($str)) return '';
+		$html = $str;
+		$html = mb_ereg_replace('&', '&amp;', $html);
+		$html = mb_ereg_replace('<', '&lt;', $html);
+		$html = mb_ereg_replace('>', '&gt;', $html);
+		$html = mb_ereg_replace('"', '&quot;', $html);
+		if ($encodeNewlinesAsBreaks) {
+			$html = str_replace("\n", "<br>\n", $html);
+		}
+		return $html;
+	}
+
+	/**
 	 * Encodes characters as HTML numeric entities to make it marginally more
 	 * difficult for web scrapers to grab sensitive info. If `$text` starts with
 	 * `mailto:` only the email address following it will be obfuscated.
 				if ($bracketCount > 0) {
 					$bracketCount--;
 				} else {
-					return [ mb_substr($line, 0, $p + 1), mb_substr($line, 1, $p - 1) ];
+					$all = mb_substr($line, 0, $p + 1);
+					$content = mb_substr($line, 1, $p - 1);
+					return [ $all, $content ];
 				}
 			}
 		}
 		 * Flushes accumulated content in `$text` to `$tokens`.
 		 */
 		$endText = function() use (&$tokens, &$text) {
-			if (mb_strlen($text) == 0) return;
+			if (mb_strlen($text) === 0) return;
 			$textGroups = [];
 			if (mb_eregi('^(\\s+)(.*?)$', $text, $textGroups)) {
 				array_push($tokens, new MDToken($textGroups[1], MDTokenType::Whitespace, $textGroups[1]));
 			if (mb_eregi('^(.*?)(\\s+)$', $text, $textGroups)) {
 				array_push($tokens, new MDToken($textGroups[1], MDTokenType::Text, $textGroups[1]));
 				array_push($tokens, new MDToken($textGroups[2], MDTokenType::Whitespace, $textGroups[2]));
-			} else {
+			} elseif (mb_strlen($text) > 0) {
 				array_push($tokens, new MDToken($text, MDTokenType::Text, $text));
 			}
 			$text = '';
 			if ($value === true) {
 				$html .= " {$safeName}";
 			} else {
-				$escapedValue = htmlentities("{$value}");
+				$escapedValue = MDUtils::escapeHTML("{$value}");
 				$html .= " {$safeName}=\"{$escapedValue}\"";
 			}
 		}
 	protected function htmlAttributes(): string {
 		$html = '';
 		if (sizeof($this->cssClasses) > 0) {
-			$classlist = htmlentities(implode(' ', $this->cssClasses));
+			$classlist = MDUtils::escapeHTML(implode(' ', $this->cssClasses));
 			$html .= " class=\"{$classlist}\"";
 		}
 		if ($this->cssId !== null && mb_strlen($this->cssId) > 0) {
-			$html .= " id=\"" . htmlentities($this->cssId) . "\"";
+			$html .= " id=\"" . MDUtils::escapeHTML($this->cssId) . "\"";
 		}
 		$styles = [];
 		foreach ($this->cssStyles as $key => $value) {
 			array_push($styles, "{$key}: {$value};");
 		}
 		if (sizeof($styles) > 0) {
-			$escaped = htmlentities(implode(' ', $styles));
+			$escaped = MDUtils::escapeHTML(implode(' ', $styles));
 			$html .= " style=\"{$escaped}\"";
 		}
 		foreach ($this->attributes as $key => $value) {
 			if ($key === 'class' || $key === 'id' || $key === 'style') continue;
 			$cleanKey = MDUtils::scrubAttributeName($key);
 			if (mb_strlen($cleanKey) == 0) continue;
-			$cleanValue = htmlentities($value);
+			$cleanValue = MDUtils::escapeHTML($value);
 			$html .= " {$cleanKey}=\"{$cleanValue}\"";
 		}
 		return $html;
 	public function toHTML(MDState $state): string {
 		$languageModifier = ($this->language !== null) ? " class=\"language-{$this->language}\"" : '';
 		return "<pre" . $this->htmlAttributes() . "><code{$languageModifier}>" .
-			htmlentities($this->text) . "</code></pre>\n";
+			MDUtils::escapeHTML($this->text) . "</code></pre>\n";
 	}
 }
 
 	}
 
 	public function toHTML(MDState $state): string {
-		return htmlentities($this->text);
+		return MDUtils::escapeHTML($this->text);
 	}
 
 	public function toPlaintext(MDState $state): string {
 	}
 
 	public function toHTML(MDState $state): string {
-		return "<code" . $this->htmlAttributes() . ">" . htmlentities($this->text) . "</code>";
+		return "<code" . $this->htmlAttributes() . ">" . MDUtils::escapeHTML($this->text) . "</code>";
 	}
 }
 
 			return "<sup class=\"footnote\" id=\"{$state->root()->elementIdPrefix}footnoteref_{$this->occurrenceId}\"" .
 				$this->htmlAttributes() . ">" .
 				"<a href=\"#{$state->root()->elementIdPrefix}footnote_{$this->footnoteId}\">" .
-				htmlentities($this->displaySymbol ?? $this->symbol) . "</a></sup>";
+				MDUtils::escapeHTML($this->displaySymbol ?? $this->symbol) . "</a></sup>";
 		}
 		return "<!--FNREF:{{$this->symbol}}-->";
 	}
 		if (str_starts_with($this->href, 'mailto:')) {
 			$escapedLink = MDUtils::escapeObfuscated($this->href);
 		} else {
-			$escapedLink = htmlentities($this->href);
+			$escapedLink = MDUtils::escapeHTML($this->href);
 		}
 		return "<a href=\"{$escapedLink}\"" . $this->htmlAttributes() . ">" . $this->childHTML($state) . "</a>";
 	}
 	}
 
 	public function toHTML(MDState $state): string {
-		$html = "<img src=\"" . htmlentities($this->src) . "\"";
-		if ($this->alt) $html .= " alt=\"" . htmlentities($this->alt) . "\"";
+		$html = "<img src=\"" . MDUtils::escapeHTML($this->src) . "\"";
+		if ($this->alt) $html .= " alt=\"" . MDUtils::escapeHTML($this->alt) . "\"";
 		$html .= $this->htmlAttributes() . ">";
 		return $html;
 	}
 	}
 
 	public function toHTML(MDState $state): string {
-		return "<abbr" . $this->htmlAttributes() . ">" . htmlentities($this->abbreviation) . "</abbr>";
+		return "<abbr" . $this->htmlAttributes() . ">" . MDUtils::escapeHTML($this->abbreviation) . "</abbr>";
 	}
 }
 

phptest/InlineTests.php

+<?php
+declare(strict_types=1);
+
+use PHPUnit\Framework\TestCase;
+
+require_once __DIR__ . '/../php/markdown.php';
+
+final class InlineTests extends TestCase {
+	/** @type {Markdown} */
+	private ?Markdown $parser = null;
+
+	protected function setUp(): void {
+		parent::setUp();
+		$this->parser = Markdown::completeParser();
+	}
+
+	private function md(string $markdown): string {
+		return $this->normalizeWhitespace($this->parser->toHTML($markdown));
+	}
+
+	private function normalizeWhitespace(string $str): string {
+		$str = mb_eregi_replace('\\s+', ' ', $str);
+		return trim($str);
+	}
+
+	public function test_simpleText() {
+		$markdown = 'Lorem ipsum';
+		$expected = 'Lorem ipsum';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_strong() {
+		$markdown = 'Lorem **ipsum** dolor **sit**';
+		$expected = 'Lorem <strong>ipsum</strong> dolor <strong>sit</strong>';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_emphasis() {
+		$markdown = 'Lorem _ipsum_ dolor _sit_';
+		$expected = 'Lorem <em>ipsum</em> dolor <em>sit</em>';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_strongEmphasis_cleanNesting1() {
+		$markdown = 'Lorem **ipsum *dolor* sit** amet';
+		$expected = 'Lorem <strong>ipsum <em>dolor</em> sit</strong> amet';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_strongEmphasis_cleanNesting2() {
+		$markdown = 'Lorem *ipsum **dolor** sit* amet';
+		$expected = 'Lorem <em>ipsum <strong>dolor</strong> sit</em> amet';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_strongEmphasis_tightNesting() {
+		$markdown = 'Lorem ***ipsum*** dolor';
+		$expected = 'Lorem <strong><em>ipsum</em></strong> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_strongEmphasis_lopsidedNesting1() {
+		$markdown = 'Lorem ***ipsum* dolor** sit';
+		$expected = 'Lorem <strong><em>ipsum</em> dolor</strong> sit';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_strongEmphasis_lopsidedNesting2() {
+		$markdown = 'Lorem ***ipsum** dolor* sit';
+		$expected = 'Lorem <em><strong>ipsum</strong> dolor</em> sit';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_strongEmphasis_lopsidedNesting3() {
+		$markdown = 'Lorem **ipsum *dolor*** sit';
+		$expected = 'Lorem <strong>ipsum <em>dolor</em></strong> sit';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_strongEmphasis_lopsidedNesting4() {
+		$markdown = 'Lorem *ipsum **dolor*** sit';
+		$expected = 'Lorem <em>ipsum <strong>dolor</strong></em> sit';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_inlineCode() {
+		$markdown = 'Lorem `ipsum` dolor';
+		$expected = 'Lorem <code>ipsum</code> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_inlineCode_withInnerBacktick() {
+		$markdown = 'Lorem ``ip`su`m`` dolor';
+		$expected = 'Lorem <code>ip`su`m</code> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_strikethrough_double() {
+		$markdown = 'Lorem ~~ipsum~~ dolor';
+		$expected = 'Lorem <s>ipsum</s> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_subscript() {
+		$markdown = 'H~2~O';
+		$expected = 'H<sub>2</sub>O';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_superscript() {
+		$markdown = 'E=mc^2^';
+		$expected = 'E=mc<sup>2</sup>';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_highlight() {
+		$markdown = 'Lorem ==ipsum== dolor';
+		$expected = 'Lorem <mark>ipsum</mark> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_underline() {
+		$markdown = 'Lorem __ipsum__ dolor';
+		$expected = 'Lorem <u>ipsum</u> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_link_fullyQualified() {
+		$markdown = 'Lorem [ipsum](https://example.com/path/page.html) dolor';
+		$expected = 'Lorem <a href="https://example.com/path/page.html">ipsum</a> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_link_relative() {
+		$markdown = 'Lorem [ipsum](page.html) dolor';
+		$expected = 'Lorem <a href="page.html">ipsum</a> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_link_title() {
+		$markdown = 'Lorem [ipsum](page.html "link title") dolor';
+		$expected = 'Lorem <a href="page.html" title="link title">ipsum</a> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_link_literal() {
+		$markdown = 'Lorem <https://example.com> dolor';
+		$expected = 'Lorem <a href="https://example.com">https://example.com</a> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_link_ref() {
+		$markdown = "Lorem [ipsum][ref] dolor\n\n[ref]: https://example.com";
+		$expected = '<p>Lorem <a href="https://example.com">ipsum</a> dolor</p>';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_link_email() {
+		$markdown = 'Lorem [ipsum](user@example.com) dolor';
+		$expected = 'Lorem <a href="mailto:&#117;&#115;&#101;&#114;&#64;&#101;&#120;&#97;&#109;&#112;&#108;&#101;&#46;&#99;&#111;&#109;">ipsum</a> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_link_email_withTitle() {
+		$markdown = 'Lorem [ipsum](user@example.com "title") dolor';
+		$expected = 'Lorem <a href="mailto:&#117;&#115;&#101;&#114;&#64;&#101;&#120;&#97;&#109;&#112;&#108;&#101;&#46;&#99;&#111;&#109;" title="title">ipsum</a> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_link_literalEmail() {
+		$markdown = 'Lorem <user@example.com> dolor';
+		$expected = 'Lorem <a href="mailto:&#117;&#115;&#101;&#114;&#64;&#101;&#120;&#97;&#109;&#112;&#108;&#101;&#46;&#99;&#111;&#109;">&#117;&#115;&#101;&#114;&#64;&#101;&#120;&#97;&#109;&#112;&#108;&#101;&#46;&#99;&#111;&#109;</a> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_link_image() {
+		$markdown = 'Lorem [![alt](image.jpg)](page.html) ipsum';
+		$expected = 'Lorem <a href="page.html"><img src="image.jpg" alt="alt"></a> ipsum';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_link_image_complex() {
+		$markdown = 'Lorem [![alt] (image.jpg "image title")] (page.html "link title") ipsum';
+		$expected = 'Lorem <a href="page.html" title="link title"><img src="image.jpg" alt="alt" title="image title"></a> ipsum';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_image() {
+		$markdown = 'Lorem ![alt text](image.jpg) dolor';
+		$expected = 'Lorem <img src="image.jpg" alt="alt text"> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_image_noAlt() {
+		$markdown = 'Lorem ![](image.jpg) dolor';
+		$expected = 'Lorem <img src="image.jpg"> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_image_withTitle() {
+		$markdown = 'Lorem ![alt text](image.jpg "image title") dolor';
+		$expected = 'Lorem <img src="image.jpg" alt="alt text" title="image title"> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_image_ref() {
+		$markdown = "Lorem ![alt text][ref] dolor\n\n" .
+			'[ref]: image.jpg "image title"';
+		$expected = '<p>Lorem <img src="image.jpg" alt="alt text" title="image title"> dolor</p>';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+
+	public function test_htmlTags() {
+		$markdown = 'Lorem <strong title=\'with " quote\' id="with \' apostrophe" lang=unquoted translate forbidden="true">ipsum</strong> dolor';
+		$expected = 'Lorem <strong title="with &quot; quote" id="with \' apostrophe" lang="unquoted" translate>ipsum</strong> dolor';
+		$actual = $this->md($markdown);
+		$this->assertSame($expected, $actual);
+	}
+}

phptest/TokenTests.php

+<?php
+declare(strict_types=1);
+
+use PHPUnit\Framework\TestCase;
+
+require_once __DIR__ . '/../php/markdown.php';
+
+final class TokenTests extends TestCase {
+	public function test_tokenizeLabel() {
+		// Escapes are preserved
+		$this->assertSame([ '[foo]', 'foo' ], MDToken::tokenizeLabel('[foo] bar'));
+		$this->assertSame([ '[foo\\[]', 'foo\\[' ], MDToken::tokenizeLabel('[foo\\[] bar'));
+		$this->assertSame([ '[foo\\]]', 'foo\\]' ], MDToken::tokenizeLabel('[foo\\]] bar'));
+		$this->assertSame([ '[foo[]]', 'foo[]' ], MDToken::tokenizeLabel('[foo[]] bar'));
+		$this->assertSame([ '[foo\\(]', 'foo\\(' ], MDToken::tokenizeLabel('[foo\\(] bar'));
+		$this->assertSame([ '[foo\\)]', 'foo\\)' ], MDToken::tokenizeLabel('[foo\\)] bar'));
+		$this->assertSame([ '[foo()]', 'foo()' ], MDToken::tokenizeLabel('[foo()] bar'));
+
+		$this->assertSame(null, MDToken::tokenizeLabel('foo bar'));
+		$this->assertSame(null, MDToken::tokenizeLabel('[foo\\] bar'));
+		$this->assertSame(null, MDToken::tokenizeLabel('[foo bar'));
+		$this->assertSame(null, MDToken::tokenizeLabel('[foo[] bar'));
+	}
+
+	public function test_tokenizeURL() {
+		$this->assertSame([ '(page.html)', 'page.html', null ], MDToken::tokenizeURL('(page.html) foo'));
+		$this->assertSame([ '(page.html "link title")', 'page.html', 'link title' ], MDToken::tokenizeURL('(page.html "link title") foo'));
+		$this->assertSame([ '(https://example.com/path/page.html?query=foo&bar=baz#fragment)', 'https://example.com/path/page.html?query=foo&bar=baz#fragment', null ], MDToken::tokenizeURL('(https://example.com/path/page.html?query=foo&bar=baz#fragment) foo'));
+
+		$this->assertSame(null, MDToken::tokenizeURL('page.html foo'));
+		$this->assertSame(null, MDToken::tokenizeURL('(page.html foo'));
+		$this->assertSame(null, MDToken::tokenizeURL('page.html) foo'));
+		$this->assertSame(null, MDToken::tokenizeURL('(page.html "title) foo'));
+		$this->assertSame(null, MDToken::tokenizeURL('(page .html) foo'));
+		$this->assertSame(null, MDToken::tokenizeURL('(user@example.com) foo'));
+		$this->assertSame(null, MDToken::tokenizeURL('(user@example.com "title") foo'));
+	}
+
+	public function test_tokenizeEmail() {
+		$this->assertSame([ '(user@example.com)', 'user@example.com', null ], MDToken::tokenizeEmail('(user@example.com)'));
+		$this->assertSame([ '(user@example.com "link title")', 'user@example.com', 'link title' ], MDToken::tokenizeEmail('(user@example.com "link title")'));
+
+		$this->assertSame(null, MDToken::tokenizeEmail('(https://example.com) foo'));
+		$this->assertSame(null, MDToken::tokenizeEmail('(https://example.com "link title") foo'));
+		$this->assertSame(null, MDToken::tokenizeEmail('(user@example.com "link title) foo'));
+		$this->assertSame(null, MDToken::tokenizeEmail('(user@example.com foo'));
+		$this->assertSame(null, MDToken::tokenizeEmail('user@example.com) foo'));
+	}
+
+	public function test_findFirstTokens() {
+		$tokens = [
+			new MDToken('Lorem', MDTokenType::Text),
+			new MDToken(' ', MDTokenType::Whitespace),
+			new MDToken('_', MDTokenType::Underscore),
+			new MDToken('ipsum', MDTokenType::Text),
+			new MDToken('_', MDTokenType::Underscore),
+			new MDToken(' ', MDTokenType::Whitespace),
+			new MDToken('dolor', MDTokenType::Text),
+			new MDToken('_', MDTokenType::Underscore),
+			new MDToken('sit', MDTokenType::Text),
+			new MDToken('_', MDTokenType::Underscore),
+		];
+		$pattern = [
+			MDTokenType::Underscore,
+		];
+		$result = MDToken::findFirstTokens($tokens, $pattern);
+		$expected = new MDTokenMatch([ $tokens[2] ], 2);
+		$this->assertEquals($expected, $result);
+	}
+
+	public function test_findFirstTokens_optionalWhitespace1() {
+		$tokens = [
+			new MDToken('Lorem', MDTokenType::Text),
+			new MDToken(' ', MDTokenType::Whitespace),
+			new MDToken('[ipsum]', MDTokenType::Label, 'ipsum'),
+			new MDToken('(link.html)', MDTokenType::URL, 'link.html'),
+			new MDToken(' ', MDTokenType::Whitespace),
+			new MDToken('dolor', MDTokenType::Text),
+		];
+		$pattern = [
+			MDTokenType::Label,
+			MDTokenType::META_OptionalWhitespace,
+			MDTokenType::URL,
+		];
+		$result = MDToken::findFirstTokens($tokens, $pattern);
+		$expected = new MDTokenMatch([ $tokens[2], $tokens[3] ], 2);
+		$this->assertEquals($expected, $result);
+	}
+
+	public function test_findFirstTokens_optionalWhitespace2() {
+		$tokens = [
+			new MDToken('Lorem', MDTokenType::Text),
+			new MDToken(' ', MDTokenType::Whitespace),
+			new MDToken('[ipsum]', MDTokenType::Label, 'ipsum'),
+			new MDToken(' ', MDTokenType::Whitespace),
+			new MDToken('(link.html)', MDTokenType::URL, 'link.html'),
+			new MDToken(' ', MDTokenType::Whitespace),
+			new MDToken('dolor', MDTokenType::Text),
+		];
+		$pattern = [
+			MDTokenType::Label,
+			MDTokenType::META_OptionalWhitespace,
+			MDTokenType::URL,
+		];
+		$result = MDToken::findFirstTokens($tokens, $pattern);
+		$expected = new MDTokenmatch([ $tokens[2], $tokens[3], $tokens[4] ], 2);
+		$this->assertEquals($expected, $result);
+	}
+
+	public function test_findPairedTokens() {
+		$tokens = [
+			new MDToken('Lorem', MDTokenType::Text),
+			new MDToken(' ', MDTokenType::Whitespace),
+			new MDToken('_', MDTokenType::Underscore),
+			new MDToken('ipsum', MDTokenType::Text),
+			new MDToken('_', MDTokenType::Underscore),
+			new MDToken(' ', MDTokenType::Whitespace),
+			new MDToken('dolor', MDTokenType::Text),
+			new MDToken('_', MDTokenType::Underscore),
+			new MDToken('sit', MDTokenType::Text),
+			new MDToken('_', MDTokenType::Underscore),
+		];
+		$pattern = [
+			MDTokenType::Underscore,
+		];
+		$result = MDToken::findPairedTokens($tokens, $pattern, $pattern);
+		$expected = new MDPairedTokenMatch(
+			[ $tokens[2] ], // startTokens
+			[ $tokens[3] ], // contentTokens
+			[ $tokens[4] ], // endTokens
+			2, // startIndex
+			3, // contentIndex
+			4, // endIndex
+			3 // totalLength
+		);
+		$this->assertEquals($expected, $result);
+	}
+}
+?>

phptest/UtilsTests.php

 		$this->assertSame(1, MDUtils::countIndents("\tfoo", true));
 		$this->assertSame(2, MDUtils::countIndents("\t\tfoo", true));
 	}
-
-	public function test_tokenizeLabel() {
-		// Escapes are preserved
-		$this->assertSame([ '[foo]', 'foo' ], MDToken::tokenizeLabel('[foo] bar'));
-		$this->assertSame([ '[foo\\[]', 'foo\\[' ], MDToken::tokenizeLabel('[foo\\[] bar'));
-		$this->assertSame([ '[foo\\]]', 'foo\\]' ], MDToken::tokenizeLabel('[foo\\]] bar'));
-		$this->assertSame([ '[foo[]]', 'foo[]' ], MDToken::tokenizeLabel('[foo[]] bar'));
-		$this->assertSame([ '[foo\\(]', 'foo\\(' ], MDToken::tokenizeLabel('[foo\\(] bar'));
-		$this->assertSame([ '[foo\\)]', 'foo\\)' ], MDToken::tokenizeLabel('[foo\\)] bar'));
-		$this->assertSame([ '[foo()]', 'foo()' ], MDToken::tokenizeLabel('[foo()] bar'));
-
-		$this->assertSame(null, MDToken::tokenizeLabel('foo bar'));
-		$this->assertSame(null, MDToken::tokenizeLabel('[foo\\] bar'));
-		$this->assertSame(null, MDToken::tokenizeLabel('[foo bar'));
-		$this->assertSame(null, MDToken::tokenizeLabel('[foo[] bar'));
-	}
-
-	public function test_tokenizeURL() {
-		$this->assertSame([ '(page.html)', 'page.html', null ], MDToken::tokenizeURL('(page.html) foo'));
-		$this->assertSame([ '(page.html "link title")', 'page.html', 'link title' ], MDToken::tokenizeURL('(page.html "link title") foo'));
-		$this->assertSame([ '(https://example.com/path/page.html?query=foo&bar=baz#fragment)', 'https://example.com/path/page.html?query=foo&bar=baz#fragment', null ], MDToken::tokenizeURL('(https://example.com/path/page.html?query=foo&bar=baz#fragment) foo'));
-
-		$this->assertSame(null, MDToken::tokenizeURL('page.html foo'));
-		$this->assertSame(null, MDToken::tokenizeURL('(page.html foo'));
-		$this->assertSame(null, MDToken::tokenizeURL('page.html) foo'));
-		$this->assertSame(null, MDToken::tokenizeURL('(page.html "title) foo'));
-		$this->assertSame(null, MDToken::tokenizeURL('(page .html) foo'));
-		$this->assertSame(null, MDToken::tokenizeURL('(user@example.com) foo'));
-		$this->assertSame(null, MDToken::tokenizeURL('(user@example.com "title") foo'));
-	}
-
-	public function test_tokenizeEmail() {
-		$this->assertSame([ '(user@example.com)', 'user@example.com', null ], MDToken::tokenizeEmail('(user@example.com)'));
-		$this->assertSame([ '(user@example.com "link title")', 'user@example.com', 'link title' ], MDToken::tokenizeEmail('(user@example.com "link title")'));
-
-		$this->assertSame(null, MDToken::tokenizeEmail('(https://example.com) foo'));
-		$this->assertSame(null, MDToken::tokenizeEmail('(https://example.com "link title") foo'));
-		$this->assertSame(null, MDToken::tokenizeEmail('(user@example.com "link title) foo'));
-		$this->assertSame(null, MDToken::tokenizeEmail('(user@example.com foo'));
-		$this->assertSame(null, MDToken::tokenizeEmail('user@example.com) foo'));
-	}
 }
 ?>

runphptests.sh

 #!/bin/sh
-php lib/phpunit.phar phptest/UtilsTests.php
+php lib/phpunit.phar \
+	phptest/UtilsTests.php \
+	phptest/TokenTests.php \
+	phptest/InlineTests.php \
+