Adds Markdown support, separates smiley parsing from BBC parsing, adds SMF\Parser base class and subclasses

Languages/en_US/Admin.php

@@ -586,9 +586,10 @@
 $txt['manageposts_settings'] = 'Post Settings';
 $txt['manageposts_settings_description'] = 'Here you can set everything related to posts and posting.';
 
-$txt['manageposts_bbc_settings'] = 'Bulletin Board Code';
-$txt['manageposts_bbc_settings_description'] = 'Bulletin board code can be used to add markup to forum messages. For example, to highlight the word "house" you can type [b]house[/b]. All Bulletin board code tags are surrounded by square brackets ("[" and "]").';
+$txt['manageposts_bbc_settings'] = 'Bulletin Board Code & Markdown';
+$txt['manageposts_bbc_settings_description'] = 'Bulletin board code and Markdown can be used to add markup to forum messages. For example, to highlight the word "house" you can type [b]house[/b] (using BBC) or **house** (using Markdown). All bulletin board code tags are surrounded by square brackets ("[" and "]"). <a href="https://commonmark.org/help/" target="_blank">Markdown syntax</a> is a little more complicated, but not too hard.';
 $txt['manageposts_bbc_settings_title'] = 'Bulletin Board Code settings';
+$txt['manageposts_markdown_settings_title'] = 'Markdown settings';
 
 $txt['manageposts_topic_settings'] = 'Topic Settings';
 $txt['manageposts_topic_settings_description'] = 'Here you can set all settings involving topics.';
@@ -630,6 +631,10 @@
 $txt['enabled_bbc_select_all'] = 'Select all tags';
 $txt['groups_can_use'] = 'Membergroups allowed to use {0}';
 
+$txt['enableMarkdown'] = 'Enable Markdown';
+$txt['collapse_blank_lines'] = 'Collapse extra blank lines';
+$txt['collapse_single_breaks'] = 'Clean up line breaks inside paragraphs';
+
 $txt['enableParticipation'] = 'Enable participation icons';
 $txt['oldTopicDays'] = 'Time before topic is warned as old on reply';
 $txt['defaultMaxTopics'] = 'Number of topics per page in the message index';

Languages/en_US/Editor.php

@@ -33,6 +33,7 @@
 $editortxt['insert_table'] = 'Insert a table';
 $editortxt['insert_horizontal_rule'] = 'Insert a horizontal rule';
 $editortxt['code'] = 'Code';
+$editortxt['tt'] = 'Inline code';
 $editortxt['insert_quote'] = 'Insert a Quote';
 $editortxt['width'] = 'Width (optional):';
 $editortxt['height'] = 'Height (optional):';
@@ -60,5 +61,6 @@
 $editortxt['float_right'] = 'Float right';
 $editortxt['maximize'] = 'Maximize';
 $editortxt['dateformat'] = 'month/day/year';
+$editortxt['heading'] = 'Heading';
 
 ?>

Languages/en_US/Help.php

@@ -337,6 +337,10 @@
 		<li>&lt;pre&gt;, &lt;blockquote&gt;</li>
 	</ul>';
 
+$helptxt['enableMarkdown'] = 'Enabling this setting will allow your members to use Markdown throughout the forum, providing them with an alternative syntax to format their posts.<br><br>Note that, unlike pure Markdown, SMF’s implementation does not allow authors to embed raw HTML into their posts except as permitted by the "Enable basic HTML in posts" setting.';
+$helptxt['collapse_blank_lines'] = 'Enabling this setting will remove unnecessary blank lines in post content, resulting in more consistent formatting in the output. Leave it disabled to preserve the blank lines.<br><br>This feature is only available when Markdown support is enabled.';
+$helptxt['collapse_single_breaks'] = 'Enabling this setting will remove single line breaks inside paragraphs, resulting in more consistent formatting in the output. Leave it disabled to preserve the line breaks.<br><br>Even when this feature is enabled, authors can still force line breaks to happen by using the [br] BBCode or by ending the line with a backslash character or with two or more spaces.<br><br>This feature is only available when Markdown support is enabled.';
+
 $helptxt['themes_manage'] = 'Here you can install new themes and select which themes your users can choose from, the default theme that new users and guests will use, as well as other theme selection settings.';
 $helptxt['theme_install'] = 'This allows you to install new themes. You can do this from an existing directory, by uploading an archive for the theme, or by copying the default theme.<br><br>Note that the archive or directory must have a <pre>theme_info.xml</pre> definition file.';
 $helptxt['xmlnews_enable'] = 'Allows people to link to <a href="{scripturl}?action=.xml;sa=news" target="_blank" rel="noopener">Recent news</a>

Sources/Actions/Admin/ACP.php

@@ -19,7 +19,6 @@
 use SMF\Actions\MessageIndex;
 use SMF\Actions\Notify;
 use SMF\ActionTrait;
-use SMF\BBCodeParser;
 use SMF\Cache\CacheApi;
 use SMF\Config;
 use SMF\Db\DatabaseApi as Db;
@@ -28,6 +27,7 @@
 use SMF\Lang;
 use SMF\Mail;
 use SMF\Menu;
+use SMF\Parser;
 use SMF\SecurityToken;
 use SMF\Theme;
 use SMF\Url;
@@ -976,7 +976,7 @@ public static function prepareDBSettingContext(array &$config_vars): void
 		// What about any BBC selection boxes?
 		if (!empty($bbcChoice)) {
 			// What are the options, eh?
-			$temp = BBCodeParser::getCodes();
+			$temp = Parser::getBBCodes();
 			$bbcTags = [];
 
 			foreach ($temp as $tag) {
@@ -1351,7 +1351,7 @@ public static function saveDBSettings(array &$config_vars): void
 			elseif ($var[0] == 'bbc') {
 				$bbcTags = [];
 
-				foreach (BBCodeParser::getCodes() as $tag) {
+				foreach (Parser::getBBCodes() as $tag) {
 					$bbcTags[] = $tag['tag'];
 				}
 

Sources/Actions/Admin/Boards.php

@@ -18,7 +18,6 @@
 use SMF\ActionInterface;
 use SMF\Actions\BackwardCompatibility;
 use SMF\ActionTrait;
-use SMF\BBCodeParser;
 use SMF\Board;
 use SMF\Category;
 use SMF\Config;
@@ -28,6 +27,7 @@
 use SMF\IntegrationHook;
 use SMF\Lang;
 use SMF\Menu;
+use SMF\Parser;
 use SMF\SecurityToken;
 use SMF\Theme;
 use SMF\Url;
@@ -366,7 +366,7 @@ public function editCategory2(): void
 
 			// Try to get any valid HTML to BBC first, add a naive attempt to strip it off, htmlspecialchars for the rest
 			$catOptions['cat_name'] = Utils::htmlspecialchars(strip_tags($_POST['cat_name']));
-			$catOptions['cat_desc'] = Utils::htmlspecialchars(strip_tags(BBCodeParser::load()->unparse($_POST['cat_desc'])));
+			$catOptions['cat_desc'] = Utils::htmlspecialchars(strip_tags(Parser::transform($_POST['cat_desc'], Parser::OUTPUT_BBC)));
 			$catOptions['is_collapsible'] = isset($_POST['collapse']);
 
 			if (isset($_POST['add'])) {
@@ -677,7 +677,7 @@ public function editBoard2(): void
 
 			// Try to get any valid HTML to BBC first, add a naive attempt to strip it off, htmlspecialchars for the rest
 			$boardOptions['board_name'] = Utils::htmlspecialchars(strip_tags($_POST['board_name']));
-			$boardOptions['board_description'] = Utils::htmlspecialchars(strip_tags(BBCodeParser::load()->unparse($_POST['desc'])));
+			$boardOptions['board_description'] = Utils::htmlspecialchars(strip_tags(Parser::transform($_POST['desc'], Parser::OUTPUT_BBC)));
 
 			$boardOptions['moderator_string'] = $_POST['moderators'];
 

Sources/Actions/Admin/ErrorLog.php

@@ -17,13 +17,13 @@
 
 use SMF\ActionInterface;
 use SMF\ActionTrait;
-use SMF\BBCodeParser;
 use SMF\Config;
 use SMF\Db\DatabaseApi as Db;
 use SMF\ErrorHandler;
 use SMF\IP;
 use SMF\Lang;
 use SMF\PageIndex;
+use SMF\Parser;
 use SMF\SecurityToken;
 use SMF\Theme;
 use SMF\Time;
@@ -318,11 +318,11 @@ public function view(): void
 			} elseif ($this->filter['variable'] == 'url') {
 				Utils::$context['filter']['value']['html'] = '\'' . strtr(Utils::htmlspecialchars((str_starts_with($this->filter['value']['sql'], '?') ? Config::$scripturl : '') . $this->filter['value']['sql']), ['\\_' => '_']) . '\'';
 			} elseif ($this->filter['variable'] == 'message') {
-				Utils::$context['filter']['value']['html'] = '\'' . strtr(Utils::htmlspecialchars($this->filter['value']['sql']), ["\n" => '<br>', '&lt;br /&gt;' => '<br>', "\t" => '&nbsp;&nbsp;&nbsp;', '\\_' => '_', '\\%' => '%', '\\\\' => '\\']) . '\'';
+				Utils::$context['filter']['value']['html'] = '\'' . strtr(Utils::htmlspecialchars($this->filter['value']['sql']), ["\n" => '<br>', '&lt;br /&gt;' => '<br>', "\t" => Utils::TAB_SUBSTITUTE, '\\_' => '_', '\\%' => '%', '\\\\' => '\\']) . '\'';
 
 				Utils::$context['filter']['value']['html'] = preg_replace('~&amp;lt;span class=&amp;quot;remove&amp;quot;&amp;gt;(.+?)&amp;lt;/span&amp;gt;~', '$1', Utils::$context['filter']['value']['html']);
 			} elseif ($this->filter['variable'] == 'error_type') {
-				Utils::$context['filter']['value']['html'] = '\'' . strtr(Utils::htmlspecialchars($this->filter['value']['sql']), ["\n" => '<br>', '&lt;br /&gt;' => '<br>', "\t" => '&nbsp;&nbsp;&nbsp;', '\\_' => '_', '\\%' => '%', '\\\\' => '\\']) . '\'';
+				Utils::$context['filter']['value']['html'] = '\'' . strtr(Utils::htmlspecialchars($this->filter['value']['sql']), ["\n" => '<br>', '&lt;br /&gt;' => '<br>', "\t" => Utils::TAB_SUBSTITUTE, '\\_' => '_', '\\%' => '%', '\\\\' => '\\']) . '\'';
 			} else {
 				Utils::$context['filter']['value']['html'] = &$this->filter['value']['sql'];
 			}
@@ -430,7 +430,7 @@ public function viewFile(): void
 			ErrorHandler::fatalLang('error_bad_line');
 		}
 
-		$file_data = explode('<br />', BBCodeParser::highlightPhpCode(Utils::htmlspecialchars(file_get_contents($file))));
+		$file_data = explode('<br />', Parser::highlightPhpCode(Utils::htmlspecialchars(file_get_contents($file))));
 
 		// We don't want to slice off too many so lets make sure we stop at the last one
 		$max = min($max, max(array_keys($file_data)));

Sources/Actions/Admin/Features.php

@@ -19,7 +19,6 @@
 use SMF\Actions\BackwardCompatibility;
 use SMF\Actions\Profile\Notification;
 use SMF\ActionTrait;
-use SMF\BBCodeParser;
 use SMF\Config;
 use SMF\Db\DatabaseApi as Db;
 use SMF\ErrorHandler;
@@ -28,6 +27,8 @@
 use SMF\ItemList;
 use SMF\Lang;
 use SMF\Menu;
+use SMF\Parser;
+use SMF\Parsers\MarkdownParser;
 use SMF\Profile;
 use SMF\Sapi;
 use SMF\SecurityToken;
@@ -165,6 +166,10 @@ public function bbc(): void
 		// Legacy BBC are listed separately, but we use the same info in both cases
 		Config::$modSettings['bbc_disabled_legacyBBC'] = Config::$modSettings['bbc_disabled_disabledBBC'];
 
+		// The Markdown settings for handling line breaks are actually a single bitmask.
+		Config::$modSettings['collapse_blank_lines'] = (int) !((Config::$modSettings['markdown_brs'] ?? 0) & MarkdownParser::BR_LINES);
+		Config::$modSettings['collapse_single_breaks'] = (int) !((Config::$modSettings['markdown_brs'] ?? 0) & MarkdownParser::BR_IN_PARAGRAPHS);
+
 		$extra = '';
 
 		if (isset($_REQUEST['cowsay'])) {
@@ -180,7 +185,7 @@ public function bbc(): void
 			$bbcTags = [];
 			$bbcTagsChildren = [];
 
-			foreach (BBCodeParser::getCodes() as $tag) {
+			foreach (Parser::getBBCodes() as $tag) {
 				$bbcTags[] = $tag['tag'];
 
 				if (isset($tag['require_children'])) {
@@ -189,8 +194,8 @@ public function bbc(): void
 			}
 
 			// Clean up tags with children
-			foreach($bbcTagsChildren as $parent_tag => $children) {
-				foreach($children as $index => $child_tag) {
+			foreach ($bbcTagsChildren as $parent_tag => $children) {
+				foreach ($children as $index => $child_tag) {
 					// Remove entries where parent and child tag is the same
 					if ($child_tag == $parent_tag) {
 						unset($bbcTagsChildren[$parent_tag][$index]);
@@ -239,6 +244,12 @@ function ($config_var) {
 				},
 			);
 
+			// Save the Markdown collapse_* settings as a bitmask.
+			$config_vars[] = ['int', 'markdown_brs'];
+			$_POST['markdown_brs'] = (!empty($_POST['collapse_blank_lines']) ? 0 : MarkdownParser::BR_LINES);
+			$_POST['markdown_brs'] |= (!empty($_POST['collapse_single_breaks']) ? 0 : MarkdownParser::BR_IN_PARAGRAPHS);
+			unset($_POST['collapse_blank_lines'], $_POST['collapse_single_breaks']);
+
 			IntegrationHook::call('integrate_save_bbc_settings', [$bbcTags]);
 
 			ACP::saveDBSettings($config_vars);
@@ -581,7 +592,7 @@ public function signature(): void
 			// Clean up the tag stuff!
 			$bbcTags = [];
 
-			foreach (BBCodeParser::getCodes() as $tag) {
+			foreach (Parser::getBBCodes() as $tag) {
 				$bbcTags[] = $tag['tag'];
 			}
 
@@ -1024,7 +1035,7 @@ public function profileEdit(): void
 				[],
 			);
 
-			while($row = Db::$db->fetch_assoc($request)) {
+			while ($row = Db::$db->fetch_assoc($request)) {
 				$fields[] = $row['id_field'];
 			}
 			Db::$db->free_result($request);
@@ -1613,6 +1624,12 @@ public static function bbcConfigVars(): array
 
 			// This one is actually pretend...
 			['bbc', 'legacyBBC', 'help' => 'legacy_bbc'],
+
+			// Markdown settings
+			['title', 'markdown_settings', 'text_label' => Lang::$txt['manageposts_markdown_settings_title']],
+			['check', 'enableMarkdown', 'onchange' => 'document.getElementById(\'collapse_blank_lines\').disabled = !this.checked; document.getElementById(\'collapse_single_breaks\').disabled = !this.checked;'],
+			['check', 'collapse_blank_lines', 'disabled' => empty(Config::$modSettings['enableMarkdown'])],
+			['check', 'collapse_single_breaks', 'disabled' => empty(Config::$modSettings['enableMarkdown'])],
 		];
 
 		// Permissions for restricted BBC

Sources/Actions/Admin/News.php

@@ -19,7 +19,6 @@
 use SMF\Actions\BackwardCompatibility;
 use SMF\Actions\Notify;
 use SMF\ActionTrait;
-use SMF\BBCodeParser;
 use SMF\Config;
 use SMF\Db\DatabaseApi as Db;
 use SMF\Editor;
@@ -31,6 +30,7 @@
 use SMF\Mail;
 use SMF\Menu;
 use SMF\Msg;
+use SMF\Parser;
 use SMF\PersonalMessage\PM;
 use SMF\SecurityToken;
 use SMF\Theme;
@@ -197,7 +197,7 @@ function addNewsItem ()
 
 						' + last_preview + ' .
 
-						'{js_escape:" style="overflow: auto; width: 100%; height: 10ex;"></div>
+						'{js_escape:" style="overflow: auto; width: 100%; min-height: 10ex;"></div>
 					</td>
 					<td></td>
 				</tr>}' .
@@ -1106,7 +1106,7 @@ public static function list_getNews(): array
 			$admin_current_news[$id] = [
 				'id' => $id,
 				'unparsed' => Msg::un_preparsecode($line),
-				'parsed' => preg_replace('~<([/]?)form[^>]*?[>]*>~i', '<em class="smalltext">&lt;$1form&gt;</em>', BBCodeParser::load()->parse($line)),
+				'parsed' => preg_replace('~<([/]?)form[^>]*?[>]*>~i', '<em class="smalltext">&lt;$1form&gt;</em>', Parser::transform($line)),
 			];
 		}
 
@@ -1141,7 +1141,7 @@ public static function list_getNewsTextarea(array $news): string
 	 */
 	public static function list_getNewsPreview(array $news): string
 	{
-		return '<div id="box_preview_' . $news['id'] . '" style="overflow: auto; width: 100%; height: 10ex;">' . $news['parsed'] . '</div>';
+		return '<div id="box_preview_' . $news['id'] . '" style="overflow: auto; width: 100%; min-height: 10ex;">' . Utils::adjustHeadingLevels($news['parsed'], null) . '</div>';
 	}
 
 	/**
@@ -1193,7 +1193,7 @@ public static function prepareMailingForPreview(): void
 			if (!empty(Utils::$context['send_html'])) {
 				$enablePostHTML = Config::$modSettings['enablePostHTML'];
 				Config::$modSettings['enablePostHTML'] = Utils::$context['send_html'];
-				Utils::$context[$key] = BBCodeParser::load()->parse(Utils::$context[$key]);
+				Utils::$context[$key] = Parser::transform(Utils::$context[$key]);
 				Config::$modSettings['enablePostHTML'] = $enablePostHTML;
 			}
 

Sources/Actions/Admin/Smileys.php

@@ -21,7 +21,6 @@
 use SMF\Actions\BackwardCompatibility;
 use SMF\Actions\MessageIndex;
 use SMF\ActionTrait;
-use SMF\BBCodeParser;
 use SMF\Cache\CacheApi;
 use SMF\Config;
 use SMF\Db\DatabaseApi as Db;
@@ -33,6 +32,7 @@
 use SMF\Menu;
 use SMF\Msg;
 use SMF\PackageManager\SubsPackage;
+use SMF\Parser;
 use SMF\SecurityToken;
 use SMF\Theme;
 use SMF\User;
@@ -1622,7 +1622,7 @@ public function install(): void
 
 				if (!empty($action['parse_bbc'])) {
 					Msg::preparsecode(Utils::$context[$type]);
-					Utils::$context[$type] = BBCodeParser::load()->parse(Utils::$context[$type]);
+					Utils::$context[$type] = Parser::transform(Utils::$context[$type]);
 				} else {
 					Utils::$context[$type] = nl2br(Utils::$context[$type]);
 				}
@@ -1671,7 +1671,7 @@ public function install(): void
 			Utils::$context['is_installed'] = false;
 			Utils::$context['package_name'] = $smileyInfo['name'];
 
-			loadTemplate('Packages');
+			Theme::loadTemplate('Packages');
 		}
 		// Do the actual install
 		else {
@@ -2248,7 +2248,7 @@ protected function __construct()
 		User::$me->isAllowedTo('manage_smileys');
 
 		Lang::load('ManageSmileys');
-		loadTemplate('ManageSmileys');
+		Theme::loadTemplate('ManageSmileys');
 
 		// If customized smileys is disabled don't show the setting page
 		if (empty(Config::$modSettings['smiley_enable'])) {

Sources/Actions/Agreement.php

@@ -17,10 +17,10 @@
 
 use SMF\ActionInterface;
 use SMF\ActionTrait;
-use SMF\BBCodeParser;
 use SMF\Config;
 use SMF\ErrorHandler;
 use SMF\Lang;
+use SMF\Parser;
 use SMF\Theme;
 use SMF\User;
 use SMF\Utils;
@@ -148,7 +148,10 @@ protected function prepareAgreementContext(): void
 
 			if (!empty(Utils::$context['agreement_file'])) {
 				$cache_id = strtr(Utils::$context['agreement_file'], [Config::$languagesdir => '', '.txt' => '', '.' => '_']);
-				Utils::$context['agreement'] = BBCodeParser::load()->parse(file_get_contents(Utils::$context['agreement_file']), true, $cache_id);
+				Utils::$context['agreement'] = Parser::transform(
+					string: file_get_contents(Utils::$context['agreement_file']),
+					options: ['cache_id' => $cache_id, 'hard_breaks' => 0],
+				);
 			} elseif (Utils::$context['can_accept_agreement']) {
 				ErrorHandler::fatalLang('error_no_agreement', false);
 			}
@@ -157,9 +160,15 @@ protected function prepareAgreementContext(): void
 		if (!Utils::$context['accept_doc'] || Utils::$context['can_accept_privacy_policy']) {
 			// Have we got a localized policy?
 			if (!empty(Config::$modSettings['policy_' . User::$me->language])) {
-				Utils::$context['privacy_policy'] = BBCodeParser::load()->parse(Config::$modSettings['policy_' . User::$me->language]);
+				Utils::$context['privacy_policy'] = Parser::transform(
+					string: Config::$modSettings['policy_' . User::$me->language],
+					options: ['hard_breaks' => 0],
+				);
 			} elseif (!empty(Config::$modSettings['policy_' . Lang::$default])) {
-				Utils::$context['privacy_policy'] = BBCodeParser::load()->parse(Config::$modSettings['policy_' . Lang::$default]);
+				Utils::$context['privacy_policy'] = Parser::transform(
+					string: Config::$modSettings['policy_' . Lang::$default],
+					options: ['hard_breaks' => 0],
+				);
 			}
 			// Then I guess we've got nothing
 			elseif (Utils::$context['can_accept_privacy_policy']) {