diff options
Diffstat (limited to 'markdown/extensions')
23 files changed, 2686 insertions, 1335 deletions
diff --git a/markdown/extensions/__init__.py b/markdown/extensions/__init__.py index e69de29..2d8d72a 100644 --- a/markdown/extensions/__init__.py +++ b/markdown/extensions/__init__.py @@ -0,0 +1,86 @@ +""" +Python Markdown + +A Python implementation of John Gruber's Markdown. + +Documentation: https://python-markdown.github.io/ +GitHub: https://github.com/Python-Markdown/markdown/ +PyPI: https://pypi.org/project/Markdown/ + +Started by Manfred Stienstra (http://www.dwerg.net/). +Maintained for a few years by Yuri Takhteyev (http://www.freewisdom.org). +Currently maintained by Waylan Limberg (https://github.com/waylan), +Dmitry Shachnev (https://github.com/mitya57) and Isaac Muse (https://github.com/facelessuser). + +Copyright 2007-2018 The Python Markdown Project (v. 1.7 and later) +Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) +Copyright 2004 Manfred Stienstra (the original version) + +License: BSD (see LICENSE.md for details). +""" + +from ..util import parseBoolValue + + +class Extension: + """ Base class for extensions to subclass. """ + + # Default config -- to be overridden by a subclass + # Must be of the following format: + # { + # 'key': ['value', 'description'] + # } + # Note that Extension.setConfig will raise a KeyError + # if a default is not set here. + config = {} + + def __init__(self, **kwargs): + """ Initiate Extension and set up configs. """ + self.setConfigs(kwargs) + + def getConfig(self, key, default=''): + """ Return a setting for the given key or an empty string. """ + if key in self.config: + return self.config[key][0] + else: + return default + + def getConfigs(self): + """ Return all configs settings as a dict. """ + return {key: self.getConfig(key) for key in self.config.keys()} + + def getConfigInfo(self): + """ Return all config descriptions as a list of tuples. """ + return [(key, self.config[key][1]) for key in self.config.keys()] + + def setConfig(self, key, value): + """ Set a config setting for `key` with the given `value`. """ + if isinstance(self.config[key][0], bool): + value = parseBoolValue(value) + if self.config[key][0] is None: + value = parseBoolValue(value, preserve_none=True) + self.config[key][0] = value + + def setConfigs(self, items): + """ Set multiple config settings given a dict or list of tuples. """ + if hasattr(items, 'items'): + # it's a dict + items = items.items() + for key, value in items: + self.setConfig(key, value) + + def extendMarkdown(self, md): + """ + Add the various processors and patterns to the Markdown Instance. + + This method must be overridden by every extension. + + Keyword arguments: + + * md: The Markdown instance. + + """ + raise NotImplementedError( + 'Extension "%s.%s" must define an "extendMarkdown"' + 'method.' % (self.__class__.__module__, self.__class__.__name__) + ) diff --git a/markdown/extensions/abbr.py b/markdown/extensions/abbr.py index 783220e..9879314 100644 --- a/markdown/extensions/abbr.py +++ b/markdown/extensions/abbr.py @@ -4,67 +4,74 @@ Abbreviation Extension for Python-Markdown This extension adds abbreviation handling to Python-Markdown. -Simple Usage: - - >>> import markdown - >>> text = """ - ... Some text with an ABBR and a REF. Ignore REFERENCE and ref. - ... - ... *[ABBR]: Abbreviation - ... *[REF]: Abbreviation Reference - ... """ - >>> markdown.markdown(text, ['abbr']) - u'<p>Some text with an <abbr title="Abbreviation">ABBR</abbr> and a <abbr title="Abbreviation Reference">REF</abbr>. Ignore REFERENCE and ref.</p>' - -Copyright 2007-2008 -* [Waylan Limberg](http://achinghead.com/) -* [Seemant Kulleen](http://www.kulleen.org/) - +See <https://Python-Markdown.github.io/extensions/abbreviations> +for documentation. + +Oringinal code Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/) and + [Seemant Kulleen](http://www.kulleen.org/) + +All changes Copyright 2008-2014 The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) ''' -import markdown, re -from markdown import etree +from . import Extension +from ..blockprocessors import BlockProcessor +from ..inlinepatterns import InlineProcessor +from ..util import AtomicString +import re +import xml.etree.ElementTree as etree -# Global Vars -ABBR_REF_RE = re.compile(r'[*]\[(?P<abbr>[^\]]*)\][ ]?:\s*(?P<title>.*)') -class AbbrExtension(markdown.Extension): +class AbbrExtension(Extension): """ Abbreviation Extension for Python-Markdown. """ - def extendMarkdown(self, md, md_globals): + def extendMarkdown(self, md): """ Insert AbbrPreprocessor before ReferencePreprocessor. """ - md.preprocessors.add('abbr', AbbrPreprocessor(md), '<reference') - - -class AbbrPreprocessor(markdown.preprocessors.Preprocessor): + md.parser.blockprocessors.register(AbbrPreprocessor(md.parser), 'abbr', 16) + + +class AbbrPreprocessor(BlockProcessor): """ Abbreviation Preprocessor - parse text for abbr references. """ - def run(self, lines): + RE = re.compile(r'^[*]\[(?P<abbr>[^\]]*)\][ ]?:[ ]*\n?[ ]*(?P<title>.*)$', re.MULTILINE) + + def test(self, parent, block): + return True + + def run(self, parent, blocks): ''' Find and remove all Abbreviation references from the text. Each reference is set as a new AbbrPattern in the markdown instance. - + ''' - new_text = [] - for line in lines: - m = ABBR_REF_RE.match(line) - if m: - abbr = m.group('abbr').strip() - title = m.group('title').strip() - self.markdown.inlinePatterns['abbr-%s'%abbr] = \ - AbbrPattern(self._generate_pattern(abbr), title) - else: - new_text.append(line) - return new_text - + block = blocks.pop(0) + m = self.RE.search(block) + if m: + abbr = m.group('abbr').strip() + title = m.group('title').strip() + self.parser.md.inlinePatterns.register( + AbbrInlineProcessor(self._generate_pattern(abbr), title), 'abbr-%s' % abbr, 2 + ) + if block[m.end():].strip(): + # Add any content after match back to blocks as separate block + blocks.insert(0, block[m.end():].lstrip('\n')) + if block[:m.start()].strip(): + # Add any content before match back to blocks as separate block + blocks.insert(0, block[:m.start()].rstrip('\n')) + return True + # No match. Restore block. + blocks.insert(0, block) + return False + def _generate_pattern(self, text): ''' - Given a string, returns an regex pattern to match that string. - - 'HTML' -> r'(?P<abbr>[H][T][M][L])' - - Note: we force each char as a literal match (in brackets) as we don't + Given a string, returns an regex pattern to match that string. + + 'HTML' -> r'(?P<abbr>[H][T][M][L])' + + Note: we force each char as a literal match (in brackets) as we don't know what they will be beforehand. ''' @@ -74,22 +81,19 @@ class AbbrPreprocessor(markdown.preprocessors.Preprocessor): return r'(?P<abbr>\b%s\b)' % (r''.join(chars)) -class AbbrPattern(markdown.inlinepatterns.Pattern): +class AbbrInlineProcessor(InlineProcessor): """ Abbreviation inline pattern. """ def __init__(self, pattern, title): - markdown.inlinepatterns.Pattern.__init__(self, pattern) + super().__init__(pattern) self.title = title - def handleMatch(self, m): + def handleMatch(self, m, data): abbr = etree.Element('abbr') - abbr.text = m.group('abbr') + abbr.text = AtomicString(m.group('abbr')) abbr.set('title', self.title) - return abbr + return abbr, m.start(0), m.end(0) -def makeExtension(configs=None): - return AbbrExtension(configs=configs) -if __name__ == "__main__": - import doctest - doctest.testmod() +def makeExtension(**kwargs): # pragma: no cover + return AbbrExtension(**kwargs) diff --git a/markdown/extensions/admonition.py b/markdown/extensions/admonition.py new file mode 100644 index 0000000..cb8d901 --- /dev/null +++ b/markdown/extensions/admonition.py @@ -0,0 +1,170 @@ +""" +Admonition extension for Python-Markdown +======================================== + +Adds rST-style admonitions. Inspired by [rST][] feature with the same name. + +[rST]: http://docutils.sourceforge.net/docs/ref/rst/directives.html#specific-admonitions # noqa + +See <https://Python-Markdown.github.io/extensions/admonition> +for documentation. + +Original code Copyright [Tiago Serafim](https://www.tiagoserafim.com/). + +All changes Copyright The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) + +""" + +from . import Extension +from ..blockprocessors import BlockProcessor +import xml.etree.ElementTree as etree +import re + + +class AdmonitionExtension(Extension): + """ Admonition extension for Python-Markdown. """ + + def extendMarkdown(self, md): + """ Add Admonition to Markdown instance. """ + md.registerExtension(self) + + md.parser.blockprocessors.register(AdmonitionProcessor(md.parser), 'admonition', 105) + + +class AdmonitionProcessor(BlockProcessor): + + CLASSNAME = 'admonition' + CLASSNAME_TITLE = 'admonition-title' + RE = re.compile(r'(?:^|\n)!!! ?([\w\-]+(?: +[\w\-]+)*)(?: +"(.*?)")? *(?:\n|$)') + RE_SPACES = re.compile(' +') + + def __init__(self, parser): + """Initialization.""" + + super().__init__(parser) + + self.current_sibling = None + self.content_indention = 0 + + def parse_content(self, parent, block): + """Get sibling admonition. + + Retrieve the appropriate sibling element. This can get tricky when + dealing with lists. + + """ + + old_block = block + the_rest = '' + + # We already acquired the block via test + if self.current_sibling is not None: + sibling = self.current_sibling + block, the_rest = self.detab(block, self.content_indent) + self.current_sibling = None + self.content_indent = 0 + return sibling, block, the_rest + + sibling = self.lastChild(parent) + + if sibling is None or sibling.get('class', '').find(self.CLASSNAME) == -1: + sibling = None + else: + # If the last child is a list and the content is sufficiently indented + # to be under it, then the content's sibling is in the list. + last_child = self.lastChild(sibling) + indent = 0 + while last_child: + if ( + sibling and block.startswith(' ' * self.tab_length * 2) and + last_child and last_child.tag in ('ul', 'ol', 'dl') + ): + + # The expectation is that we'll find an <li> or <dt>. + # We should get its last child as well. + sibling = self.lastChild(last_child) + last_child = self.lastChild(sibling) if sibling else None + + # Context has been lost at this point, so we must adjust the + # text's indentation level so it will be evaluated correctly + # under the list. + block = block[self.tab_length:] + indent += self.tab_length + else: + last_child = None + + if not block.startswith(' ' * self.tab_length): + sibling = None + + if sibling is not None: + indent += self.tab_length + block, the_rest = self.detab(old_block, indent) + self.current_sibling = sibling + self.content_indent = indent + + return sibling, block, the_rest + + def test(self, parent, block): + + if self.RE.search(block): + return True + else: + return self.parse_content(parent, block)[0] is not None + + def run(self, parent, blocks): + block = blocks.pop(0) + m = self.RE.search(block) + + if m: + if m.start() > 0: + self.parser.parseBlocks(parent, [block[:m.start()]]) + block = block[m.end():] # removes the first line + block, theRest = self.detab(block) + else: + sibling, block, theRest = self.parse_content(parent, block) + + if m: + klass, title = self.get_class_and_title(m) + div = etree.SubElement(parent, 'div') + div.set('class', '{} {}'.format(self.CLASSNAME, klass)) + if title: + p = etree.SubElement(div, 'p') + p.text = title + p.set('class', self.CLASSNAME_TITLE) + else: + # Sibling is a list item, but we need to wrap it's content should be wrapped in <p> + if sibling.tag in ('li', 'dd') and sibling.text: + text = sibling.text + sibling.text = '' + p = etree.SubElement(sibling, 'p') + p.text = text + + div = sibling + + self.parser.parseChunk(div, block) + + if theRest: + # This block contained unindented line(s) after the first indented + # line. Insert these lines as the first block of the master blocks + # list for future processing. + blocks.insert(0, theRest) + + def get_class_and_title(self, match): + klass, title = match.group(1).lower(), match.group(2) + klass = self.RE_SPACES.sub(' ', klass) + if title is None: + # no title was provided, use the capitalized classname as title + # e.g.: `!!! note` will render + # `<p class="admonition-title">Note</p>` + title = klass.split(' ', 1)[0].capitalize() + elif title == '': + # an explicit blank title should not be rendered + # e.g.: `!!! warning ""` will *not* render `p` with a title + title = None + return klass, title + + +def makeExtension(**kwargs): # pragma: no cover + return AdmonitionExtension(**kwargs) diff --git a/markdown/extensions/attr_list.py b/markdown/extensions/attr_list.py new file mode 100644 index 0000000..9a67551 --- /dev/null +++ b/markdown/extensions/attr_list.py @@ -0,0 +1,166 @@ +""" +Attribute List Extension for Python-Markdown +============================================ + +Adds attribute list syntax. Inspired by +[maruku](http://maruku.rubyforge.org/proposal.html#attribute_lists)'s +feature of the same name. + +See <https://Python-Markdown.github.io/extensions/attr_list> +for documentation. + +Original code Copyright 2011 [Waylan Limberg](http://achinghead.com/). + +All changes Copyright 2011-2014 The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) + +""" + +from . import Extension +from ..treeprocessors import Treeprocessor +import re + + +def _handle_double_quote(s, t): + k, v = t.split('=', 1) + return k, v.strip('"') + + +def _handle_single_quote(s, t): + k, v = t.split('=', 1) + return k, v.strip("'") + + +def _handle_key_value(s, t): + return t.split('=', 1) + + +def _handle_word(s, t): + if t.startswith('.'): + return '.', t[1:] + if t.startswith('#'): + return 'id', t[1:] + return t, t + + +_scanner = re.Scanner([ + (r'[^ =]+=".*?"', _handle_double_quote), + (r"[^ =]+='.*?'", _handle_single_quote), + (r'[^ =]+=[^ =]+', _handle_key_value), + (r'[^ =]+', _handle_word), + (r' ', None) +]) + + +def get_attrs(str): + """ Parse attribute list and return a list of attribute tuples. """ + return _scanner.scan(str)[0] + + +def isheader(elem): + return elem.tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'] + + +class AttrListTreeprocessor(Treeprocessor): + + BASE_RE = r'\{\:?[ ]*([^\}\n ][^\}\n]*)[ ]*\}' + HEADER_RE = re.compile(r'[ ]+{}[ ]*$'.format(BASE_RE)) + BLOCK_RE = re.compile(r'\n[ ]*{}[ ]*$'.format(BASE_RE)) + INLINE_RE = re.compile(r'^{}'.format(BASE_RE)) + NAME_RE = re.compile(r'[^A-Z_a-z\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u02ff' + r'\u0370-\u037d\u037f-\u1fff\u200c-\u200d' + r'\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff' + r'\uf900-\ufdcf\ufdf0-\ufffd' + r'\:\-\.0-9\u00b7\u0300-\u036f\u203f-\u2040]+') + + def run(self, doc): + for elem in doc.iter(): + if self.md.is_block_level(elem.tag): + # Block level: check for attrs on last line of text + RE = self.BLOCK_RE + if isheader(elem) or elem.tag in ['dt', 'td', 'th']: + # header, def-term, or table cell: check for attrs at end of element + RE = self.HEADER_RE + if len(elem) and elem.tag == 'li': + # special case list items. children may include a ul or ol. + pos = None + # find the ul or ol position + for i, child in enumerate(elem): + if child.tag in ['ul', 'ol']: + pos = i + break + if pos is None and elem[-1].tail: + # use tail of last child. no ul or ol. + m = RE.search(elem[-1].tail) + if m: + self.assign_attrs(elem, m.group(1)) + elem[-1].tail = elem[-1].tail[:m.start()] + elif pos is not None and pos > 0 and elem[pos-1].tail: + # use tail of last child before ul or ol + m = RE.search(elem[pos-1].tail) + if m: + self.assign_attrs(elem, m.group(1)) + elem[pos-1].tail = elem[pos-1].tail[:m.start()] + elif elem.text: + # use text. ul is first child. + m = RE.search(elem.text) + if m: + self.assign_attrs(elem, m.group(1)) + elem.text = elem.text[:m.start()] + elif len(elem) and elem[-1].tail: + # has children. Get from tail of last child + m = RE.search(elem[-1].tail) + if m: + self.assign_attrs(elem, m.group(1)) + elem[-1].tail = elem[-1].tail[:m.start()] + if isheader(elem): + # clean up trailing #s + elem[-1].tail = elem[-1].tail.rstrip('#').rstrip() + elif elem.text: + # no children. Get from text. + m = RE.search(elem.text) + if m: + self.assign_attrs(elem, m.group(1)) + elem.text = elem.text[:m.start()] + if isheader(elem): + # clean up trailing #s + elem.text = elem.text.rstrip('#').rstrip() + else: + # inline: check for attrs at start of tail + if elem.tail: + m = self.INLINE_RE.match(elem.tail) + if m: + self.assign_attrs(elem, m.group(1)) + elem.tail = elem.tail[m.end():] + + def assign_attrs(self, elem, attrs): + """ Assign attrs to element. """ + for k, v in get_attrs(attrs): + if k == '.': + # add to class + cls = elem.get('class') + if cls: + elem.set('class', '{} {}'.format(cls, v)) + else: + elem.set('class', v) + else: + # assign attr k with v + elem.set(self.sanitize_name(k), v) + + def sanitize_name(self, name): + """ + Sanitize name as 'an XML Name, minus the ":"'. + See https://www.w3.org/TR/REC-xml-names/#NT-NCName + """ + return self.NAME_RE.sub('_', name) + + +class AttrListExtension(Extension): + def extendMarkdown(self, md): + md.treeprocessors.register(AttrListTreeprocessor(md), 'attr_list', 8) + md.registerExtension(self) + + +def makeExtension(**kwargs): # pragma: no cover + return AttrListExtension(**kwargs) diff --git a/markdown/extensions/codehilite.py b/markdown/extensions/codehilite.py index c5d496b..a54ba21 100644 --- a/markdown/extensions/codehilite.py +++ b/markdown/extensions/codehilite.py @@ -1,156 +1,216 @@ -#!/usr/bin/python - """ CodeHilite Extension for Python-Markdown ======================================== Adds code/syntax highlighting to standard Python-Markdown code blocks. -Copyright 2006-2008 [Waylan Limberg](http://achinghead.com/). +See <https://Python-Markdown.github.io/extensions/code_hilite> +for documentation. + +Original code Copyright 2006-2008 [Waylan Limberg](http://achinghead.com/). + +All changes Copyright 2008-2014 The Python Markdown Project -Project website: <http://www.freewisdom.org/project/python-markdown/CodeHilite> -Contact: markdown@freewisdom.org - -License: BSD (see ../docs/LICENSE for details) - -Dependencies: -* [Python 2.3+](http://python.org/) -* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) -* [Pygments](http://pygments.org/) +License: [BSD](https://opensource.org/licenses/bsd-license.php) """ -import markdown +from . import Extension +from ..treeprocessors import Treeprocessor +from ..util import parseBoolValue + +try: # pragma: no cover + from pygments import highlight + from pygments.lexers import get_lexer_by_name, guess_lexer + from pygments.formatters import get_formatter_by_name + from pygments.util import ClassNotFound + pygments = True +except ImportError: # pragma: no cover + pygments = False + -# --------------- CONSTANTS YOU MIGHT WANT TO MODIFY ----------------- +def parse_hl_lines(expr): + """Support our syntax for emphasizing certain lines of code. -try: - TAB_LENGTH = markdown.TAB_LENGTH -except AttributeError: - TAB_LENGTH = 4 + expr should be like '1 2' to emphasize lines 1 and 2 of a code block. + Returns a list of ints, the line numbers to emphasize. + """ + if not expr: + return [] + + try: + return list(map(int, expr.split())) + except ValueError: # pragma: no cover + return [] # ------------------ The Main CodeHilite Class ---------------------- class CodeHilite: """ - Determine language of source code, and pass it into the pygments hilighter. + Determine language of source code, and pass it on to the Pygments highlighter. + + Usage: + code = CodeHilite(src=some_code, lang='python') + html = code.hilite() - Basic Usage: - >>> code = CodeHilite(src = 'some text') - >>> html = code.hilite() - + Arguments: * src: Source string or any object with a .readline attribute. - - * linenos: (Boolen) Turn line numbering 'on' or 'off' (off by default). - - * css_class: Set class name of wrapper div ('codehilite' by default). - - Low Level Usage: - >>> code = CodeHilite() - >>> code.src = 'some text' # String or anything with a .readline attr. - >>> code.linenos = True # True or False; Turns line numbering on or of. - >>> html = code.hilite() - + + * lang: String name of Pygments lexer to use for highlighting. Default: `None`. + + * guess_lang: Auto-detect which lexer to use. Ignored if `lang` is set to a valid + value. Default: `True`. + + * use_pygments: Pass code to pygments for code highlighting. If `False`, the code is + instead wrapped for highlighting by a JavaScript library. Default: `True`. + + * pygments_formatter: The name of a Pygments formatter or a formatter class used for + highlighting the code blocks. Default: `html`. + + * linenums: An alias to Pygments `linenos` formatter option. Default: `None`. + + * css_class: An alias to Pygments `cssclass` formatter option. Default: 'codehilite'. + + * lang_prefix: Prefix prepended to the language. Default: "language-". + + Other Options: + Any other options are accepted and passed on to the lexer and formatter. Therefore, + valid options include any options which are accepted by the `html` formatter or + whichever lexer the code's language uses. Note that most lexers do not have any + options. However, a few have very useful options, such as PHP's `startinline` option. + Any invalid options are ignored without error. + + Formatter options: https://pygments.org/docs/formatters/#HtmlFormatter + Lexer Options: https://pygments.org/docs/lexers/ + + Additionally, when Pygments is enabled, the code's language is passed to the + formatter as an extra option `lang_str`, whose value being `{lang_prefix}{lang}`. + This option has no effect to the Pygments's builtin formatters. + + Advanced Usage: + code = CodeHilite( + src = some_code, + lang = 'php', + startinline = True, # Lexer option. Snippet does not start with `<?php`. + linenostart = 42, # Formatter option. Snippet starts on line 42. + hl_lines = [45, 49, 50], # Formatter option. Highlight lines 45, 49, and 50. + linenos = 'inline' # Formatter option. Avoid alignment problems. + ) + html = code.hilite() + """ - def __init__(self, src=None, linenos=False, css_class="codehilite"): + def __init__(self, src, **options): self.src = src - self.lang = None - self.linenos = linenos - self.css_class = css_class - - def hilite(self): + self.lang = options.pop('lang', None) + self.guess_lang = options.pop('guess_lang', True) + self.use_pygments = options.pop('use_pygments', True) + self.lang_prefix = options.pop('lang_prefix', 'language-') + self.pygments_formatter = options.pop('pygments_formatter', 'html') + + if 'linenos' not in options: + options['linenos'] = options.pop('linenums', None) + if 'cssclass' not in options: + options['cssclass'] = options.pop('css_class', 'codehilite') + if 'wrapcode' not in options: + # Override pygments default + options['wrapcode'] = True + # Disallow use of `full` option + options['full'] = False + + self.options = options + + def hilite(self, shebang=True): """ - Pass code to the [Pygments](http://pygments.pocoo.org/) highliter with - optional line numbers. The output should then be styled with css to - your liking. No styles are applied by default - only styling hooks - (i.e.: <span class="k">). + Pass code to the [Pygments](http://pygments.pocoo.org/) highliter with + optional line numbers. The output should then be styled with css to + your liking. No styles are applied by default - only styling hooks + (i.e.: <span class="k">). returns : A string of html. - + """ self.src = self.src.strip('\n') - - self._getLang() - - try: - from pygments import highlight - from pygments.lexers import get_lexer_by_name, guess_lexer, \ - TextLexer - from pygments.formatters import HtmlFormatter - except ImportError: - # just escape and pass through - txt = self._escape(self.src) - if self.linenos: - txt = self._number(txt) - else : - txt = '<div class="%s"><pre>%s</pre></div>\n'% \ - (self.css_class, txt) - return txt - else: + + if self.lang is None and shebang: + self._parseHeader() + + if pygments and self.use_pygments: try: - lexer = get_lexer_by_name(self.lang) + lexer = get_lexer_by_name(self.lang, **self.options) except ValueError: try: - lexer = guess_lexer(self.src) - except ValueError: - lexer = TextLexer() - formatter = HtmlFormatter(linenos=self.linenos, - cssclass=self.css_class) + if self.guess_lang: + lexer = guess_lexer(self.src, **self.options) + else: + lexer = get_lexer_by_name('text', **self.options) + except ValueError: # pragma: no cover + lexer = get_lexer_by_name('text', **self.options) + if not self.lang: + # Use the guessed lexer's language instead + self.lang = lexer.aliases[0] + lang_str = f'{self.lang_prefix}{self.lang}' + if isinstance(self.pygments_formatter, str): + try: + formatter = get_formatter_by_name(self.pygments_formatter, **self.options) + except ClassNotFound: + formatter = get_formatter_by_name('html', **self.options) + else: + formatter = self.pygments_formatter(lang_str=lang_str, **self.options) return highlight(self.src, lexer, formatter) - - def _escape(self, txt): - """ basic html escaping """ - txt = txt.replace('&', '&') - txt = txt.replace('<', '<') - txt = txt.replace('>', '>') - txt = txt.replace('"', '"') - return txt - - def _number(self, txt): - """ Use <ol> for line numbering """ - # Fix Whitespace - txt = txt.replace('\t', ' '*TAB_LENGTH) - txt = txt.replace(" "*4, " ") - txt = txt.replace(" "*3, " ") - txt = txt.replace(" "*2, " ") - - # Add line numbers - lines = txt.splitlines() - txt = '<div class="codehilite"><pre><ol>\n' - for line in lines: - txt += '\t<li>%s</li>\n'% line - txt += '</ol></pre></div>\n' - return txt - - - def _getLang(self): - """ - Determines language of a code block from shebang lines and whether said - line should be removed or left in place. If the sheband line contains a - path (even a single /) then it is assumed to be a real shebang lines and - left alone. However, if no path is given (e.i.: #!python or :::python) - then it is assumed to be a mock shebang for language identifitation of a - code fragment and removed from the code block prior to processing for - code highlighting. When a mock shebang (e.i: #!python) is found, line - numbering is turned on. When colons are found in place of a shebang - (e.i.: :::python), line numbering is left in the current state - off - by default. - + else: + # just escape and build markup usable by JS highlighting libs + txt = self.src.replace('&', '&') + txt = txt.replace('<', '<') + txt = txt.replace('>', '>') + txt = txt.replace('"', '"') + classes = [] + if self.lang: + classes.append('{}{}'.format(self.lang_prefix, self.lang)) + if self.options['linenos']: + classes.append('linenums') + class_str = '' + if classes: + class_str = ' class="{}"'.format(' '.join(classes)) + return '<pre class="{}"><code{}>{}\n</code></pre>\n'.format( + self.options['cssclass'], + class_str, + txt + ) + + def _parseHeader(self): + """ + Determines language of a code block from shebang line and whether the + said line should be removed or left in place. If the sheband line + contains a path (even a single /) then it is assumed to be a real + shebang line and left alone. However, if no path is given + (e.i.: #!python or :::python) then it is assumed to be a mock shebang + for language identification of a code fragment and removed from the + code block prior to processing for code highlighting. When a mock + shebang (e.i: #!python) is found, line numbering is turned on. When + colons are found in place of a shebang (e.i.: :::python), line + numbering is left in the current state - off by default. + + Also parses optional list of highlight lines, like: + + :::python hl_lines="1 3" """ import re - - #split text into lines + + # split text into lines lines = self.src.split("\n") - #pull first line to examine + # pull first line to examine fl = lines.pop(0) - + c = re.compile(r''' - (?:(?:::+)|(?P<shebang>[#]!)) # Shebang or 2 or more colons. - (?P<path>(?:/\w+)*[/ ])? # Zero or 1 path - (?P<lang>[\w+-]*) # The language + (?:(?:^::+)|(?P<shebang>^[#]!)) # Shebang or 2 or more colons + (?P<path>(?:/\w+)*[/ ])? # Zero or 1 path + (?P<lang>[\w#.+-]*) # The language + \s* # Arbitrary whitespace + # Optional highlight lines, single- or double-quote-delimited + (hl_lines=(?P<quot>"|')(?P<hl_lines>.*?)(?P=quot))? ''', re.VERBOSE) # search first line for shebang m = c.search(fl) @@ -158,67 +218,113 @@ class CodeHilite: # we have a match try: self.lang = m.group('lang').lower() - except IndexError: + except IndexError: # pragma: no cover self.lang = None if m.group('path'): # path exists - restore first line lines.insert(0, fl) - if m.group('shebang'): - # shebang exists - use line numbers - self.linenos = True + if self.options['linenos'] is None and m.group('shebang'): + # Overridable and Shebang exists - use line numbers + self.options['linenos'] = True + + self.options['hl_lines'] = parse_hl_lines(m.group('hl_lines')) else: # No match lines.insert(0, fl) - - self.src = "\n".join(lines).strip("\n") + self.src = "\n".join(lines).strip("\n") # ------------------ The Markdown Extension ------------------------------- -class HiliteTreeprocessor(markdown.treeprocessors.Treeprocessor): - """ Hilight source code in code blocks. """ + + +class HiliteTreeprocessor(Treeprocessor): + """ Highlight source code in code blocks. """ + + def code_unescape(self, text): + """Unescape code.""" + text = text.replace("<", "<") + text = text.replace(">", ">") + # Escaped '&' should be replaced at the end to avoid + # conflicting with < and >. + text = text.replace("&", "&") + return text def run(self, root): """ Find code blocks and store in htmlStash. """ - blocks = root.getiterator('pre') + blocks = root.iter('pre') for block in blocks: - children = block.getchildren() - if len(children) == 1 and children[0].tag == 'code': - code = CodeHilite(children[0].text, - linenos=self.config['force_linenos'][0], - css_class=self.config['css_class'][0]) - placeholder = self.markdown.htmlStash.store(code.hilite(), - safe=True) + if len(block) == 1 and block[0].tag == 'code': + local_config = self.config.copy() + code = CodeHilite( + self.code_unescape(block[0].text), + tab_length=self.md.tab_length, + style=local_config.pop('pygments_style', 'default'), + **local_config + ) + placeholder = self.md.htmlStash.store(code.hilite()) # Clear codeblock in etree instance block.clear() - # Change to p element which will later + # Change to p element which will later # be removed when inserting raw html block.tag = 'p' block.text = placeholder -class CodeHiliteExtension(markdown.Extension): - """ Add source code hilighting to markdown codeblocks. """ +class CodeHiliteExtension(Extension): + """ Add source code highlighting to markdown codeblocks. """ - def __init__(self, configs): + def __init__(self, **kwargs): # define default configs self.config = { - 'force_linenos' : [False, "Force line numbers - Default: False"], - 'css_class' : ["codehilite", - "Set class name for wrapper <div> - Default: codehilite"], + 'linenums': [None, + "Use lines numbers. True|table|inline=yes, False=no, None=auto"], + 'guess_lang': [True, + "Automatic language detection - Default: True"], + 'css_class': ["codehilite", + "Set class name for wrapper <div> - " + "Default: codehilite"], + 'pygments_style': ['default', + 'Pygments HTML Formatter Style ' + '(Colorscheme) - Default: default'], + 'noclasses': [False, + 'Use inline styles instead of CSS classes - ' + 'Default false'], + 'use_pygments': [True, + 'Use Pygments to Highlight code blocks. ' + 'Disable if using a JavaScript library. ' + 'Default: True'], + 'lang_prefix': [ + 'language-', + 'Prefix prepended to the language when use_pygments is false. Default: "language-"' + ], + 'pygments_formatter': ['html', + 'Use a specific formatter for Pygments highlighting.' + 'Default: "html"', + ], } - - # Override defaults with user settings - for key, value in configs: - self.setConfig(key, value) - def extendMarkdown(self, md, md_globals): + for key, value in kwargs.items(): + if key in self.config: + self.setConfig(key, value) + else: + # manually set unknown keywords. + if isinstance(value, str): + try: + # Attempt to parse str as a bool value + value = parseBoolValue(value, preserve_none=True) + except ValueError: + pass # Assume it's not a bool value. Use as-is. + self.config[key] = [value, ''] + + def extendMarkdown(self, md): """ Add HilitePostprocessor to Markdown instance. """ hiliter = HiliteTreeprocessor(md) - hiliter.config = self.config - md.treeprocessors.add("hilite", hiliter, "_begin") + hiliter.config = self.getConfigs() + md.treeprocessors.register(hiliter, 'hilite', 30) + md.registerExtension(self) -def makeExtension(configs={}): - return CodeHiliteExtension(configs=configs) +def makeExtension(**kwargs): # pragma: no cover + return CodeHiliteExtension(**kwargs) diff --git a/markdown/extensions/def_list.py b/markdown/extensions/def_list.py index 73a1c85..17549f0 100644 --- a/markdown/extensions/def_list.py +++ b/markdown/extensions/def_list.py @@ -1,61 +1,71 @@ -#!/usr/bin/env Python """ Definition List Extension for Python-Markdown ============================================= -Added parsing of Definition Lists to Python-Markdown. +Adds parsing of Definition Lists to Python-Markdown. -A simple example: +See <https://Python-Markdown.github.io/extensions/definition_lists> +for documentation. - Apple - : Pomaceous fruit of plants of the genus Malus in - the family Rosaceae. - : An american computer company. +Original code Copyright 2008 [Waylan Limberg](http://achinghead.com) - Orange - : The fruit of an evergreen tree of the genus Citrus. +All changes Copyright 2008-2014 The Python Markdown Project -Copyright 2008 - [Waylan Limberg](http://achinghead.com) +License: [BSD](https://opensource.org/licenses/bsd-license.php) """ -import markdown, re -from markdown import etree +from . import Extension +from ..blockprocessors import BlockProcessor, ListIndentProcessor +import xml.etree.ElementTree as etree +import re -class DefListProcessor(markdown.blockprocessors.BlockProcessor): +class DefListProcessor(BlockProcessor): """ Process Definition Lists. """ RE = re.compile(r'(^|\n)[ ]{0,3}:[ ]{1,3}(.*?)(\n|$)') + NO_INDENT_RE = re.compile(r'^[ ]{0,3}[^ :]') def test(self, parent, block): return bool(self.RE.search(block)) def run(self, parent, blocks): - block = blocks.pop(0) - m = self.RE.search(block) - terms = [l.strip() for l in block[:m.start()].split('\n') if l.strip()] - d, theRest = self.detab(block[m.end():]) + + raw_block = blocks.pop(0) + m = self.RE.search(raw_block) + terms = [term.strip() for term in + raw_block[:m.start()].split('\n') if term.strip()] + block = raw_block[m.end():] + no_indent = self.NO_INDENT_RE.match(block) + if no_indent: + d, theRest = (block, None) + else: + d, theRest = self.detab(block) if d: - d = '%s\n%s' % (m.group(2), d) + d = '{}\n{}'.format(m.group(2), d) else: d = m.group(2) - #import ipdb; ipdb.set_trace() sibling = self.lastChild(parent) + if not terms and sibling is None: + # This is not a definition item. Most likely a paragraph that + # starts with a colon at the beginning of a document or list. + blocks.insert(0, raw_block) + return False if not terms and sibling.tag == 'p': # The previous paragraph contains the terms state = 'looselist' terms = sibling.text.split('\n') parent.remove(sibling) - # Aquire new sibling + # Acquire new sibling sibling = self.lastChild(parent) else: state = 'list' - if sibling and sibling.tag == 'dl': + if sibling is not None and sibling.tag == 'dl': # This is another item on an existing list dl = sibling - if len(dl) and dl[-1].tag == 'dd' and len(dl[-1]): + if not terms and len(dl) and dl[-1].tag == 'dd' and len(dl[-1]): state = 'looselist' else: # This is a new list @@ -73,32 +83,29 @@ class DefListProcessor(markdown.blockprocessors.BlockProcessor): if theRest: blocks.insert(0, theRest) -class DefListIndentProcessor(markdown.blockprocessors.ListIndentProcessor): + +class DefListIndentProcessor(ListIndentProcessor): """ Process indented children of definition list items. """ - ITEM_TYPES = ['dd'] - LIST_TYPES = ['dl'] + # Definition lists need to be aware of all list types + ITEM_TYPES = ['dd', 'li'] + LIST_TYPES = ['dl', 'ol', 'ul'] - def create_item(parent, block): - """ Create a new dd and parse the block with it as the parent. """ - dd = markdown.etree.SubElement(parent, 'dd') + def create_item(self, parent, block): + """ Create a new dd or li (depending on parent) and parse the block with it as the parent. """ + + dd = etree.SubElement(parent, 'dd') self.parser.parseBlocks(dd, [block]) - -class DefListExtension(markdown.Extension): +class DefListExtension(Extension): """ Add definition lists to Markdown. """ - def extendMarkdown(self, md, md_globals): + def extendMarkdown(self, md): """ Add an instance of DefListProcessor to BlockParser. """ - md.parser.blockprocessors.add('defindent', - DefListIndentProcessor(md.parser), - '>indent') - md.parser.blockprocessors.add('deflist', - DefListProcessor(md.parser), - '>ulist') - + md.parser.blockprocessors.register(DefListIndentProcessor(md.parser), 'defindent', 85) + md.parser.blockprocessors.register(DefListProcessor(md.parser), 'deflist', 25) -def makeExtension(configs={}): - return DefListExtension(configs=configs) +def makeExtension(**kwargs): # pragma: no cover + return DefListExtension(**kwargs) diff --git a/markdown/extensions/extra.py b/markdown/extensions/extra.py index 4a2ffbf..909ba07 100644 --- a/markdown/extensions/extra.py +++ b/markdown/extensions/extra.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Python-Markdown Extra Extension =============================== @@ -7,43 +6,53 @@ A compilation of various Python-Markdown extensions that imitates [PHP Markdown Extra](http://michelf.com/projects/php-markdown/extra/). Note that each of the individual extensions still need to be available -on your PYTHONPATH. This extension simply wraps them all up as a +on your PYTHONPATH. This extension simply wraps them all up as a convenience so that only one extension needs to be listed when initiating Markdown. See the documentation for each individual extension for specifics about that extension. -In the event that one or more of the supported extensions are not -available for import, Markdown will issue a warning and simply continue -without that extension. - -There may be additional extensions that are distributed with +There may be additional extensions that are distributed with Python-Markdown that are not included here in Extra. Those extensions are not part of PHP Markdown Extra, and therefore, not part of Python-Markdown Extra. If you really would like Extra to include additional extensions, we suggest creating your own clone of Extra -under a differant name. You could also edit the `extensions` global -variable defined below, but be aware that such changes may be lost +under a different name. You could also edit the `extensions` global +variable defined below, but be aware that such changes may be lost when you upgrade to any future version of Python-Markdown. +See <https://Python-Markdown.github.io/extensions/extra> +for documentation. + +Copyright The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) + """ -import markdown +from . import Extension -extensions = ['fenced_code', - 'footnotes', - 'headerid', - 'def_list', - 'tables', - 'abbr', - ] - +extensions = [ + 'fenced_code', + 'footnotes', + 'attr_list', + 'def_list', + 'tables', + 'abbr', + 'md_in_html' +] -class ExtraExtension(markdown.Extension): + +class ExtraExtension(Extension): """ Add various extensions to Markdown class.""" - def extendMarkdown(self, md, md_globals): + def __init__(self, **kwargs): + """ config is a dumb holder which gets passed to actual ext later. """ + self.config = kwargs + + def extendMarkdown(self, md): """ Register extension instances. """ md.registerExtensions(extensions, self.config) -def makeExtension(configs={}): - return ExtraExtension(configs=dict(configs)) + +def makeExtension(**kwargs): # pragma: no cover + return ExtraExtension(**kwargs) diff --git a/markdown/extensions/fenced_code.py b/markdown/extensions/fenced_code.py index 307b1dc..409166a 100644 --- a/markdown/extensions/fenced_code.py +++ b/markdown/extensions/fenced_code.py @@ -1,104 +1,166 @@ -#!/usr/bin/env python - """ Fenced Code Extension for Python Markdown ========================================= This extension adds Fenced Code Blocks to Python-Markdown. - >>> import markdown - >>> text = ''' - ... A paragraph before a fenced code block: - ... - ... ~~~ - ... Fenced code block - ... ~~~ - ... ''' - >>> html = markdown.markdown(text, extensions=['fenced_code']) - >>> html - u'<p>A paragraph before a fenced code block:</p>\\n<pre><code>Fenced code block\\n</code></pre>' - -Works with safe_mode also (we check this because we are using the HtmlStash): - - >>> markdown.markdown(text, extensions=['fenced_code'], safe_mode='replace') - u'<p>A paragraph before a fenced code block:</p>\\n<pre><code>Fenced code block\\n</code></pre>' - -Include tilde's in a code block and wrap with blank lines: - - >>> text = ''' - ... ~~~~~~~~ - ... - ... ~~~~ - ... - ... ~~~~~~~~''' - >>> markdown.markdown(text, extensions=['fenced_code']) - u'<pre><code>\\n~~~~\\n\\n</code></pre>' - -Multiple blocks and language tags: - - >>> text = ''' - ... ~~~~{.python} - ... block one - ... ~~~~ - ... - ... ~~~~.html - ... <p>block two</p> - ... ~~~~''' - >>> markdown.markdown(text, extensions=['fenced_code']) - u'<pre><code class="python">block one\\n</code></pre>\\n\\n<pre><code class="html"><p>block two</p>\\n</code></pre>' - -Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/). - -Project website: <http://www.freewisdom.org/project/python-markdown/Fenced__Code__Blocks> -Contact: markdown@freewisdom.org - -License: BSD (see ../docs/LICENSE for details) - -Dependencies: -* [Python 2.3+](http://python.org) -* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) +See <https://Python-Markdown.github.io/extensions/fenced_code_blocks> +for documentation. + +Original code Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/). + + +All changes Copyright 2008-2014 The Python Markdown Project +License: [BSD](https://opensource.org/licenses/bsd-license.php) """ -import markdown, re -# Global vars -FENCED_BLOCK_RE = re.compile( \ - r'(?P<fence>^~{3,})[ ]*(\{?\.(?P<lang>[a-zA-Z0-9_-]*)\}?)?[ ]*\n(?P<code>.*?)(?P=fence)[ ]*$', - re.MULTILINE|re.DOTALL - ) -CODE_WRAP = '<pre><code%s>%s</code></pre>' -LANG_TAG = ' class="%s"' +from textwrap import dedent +from . import Extension +from ..preprocessors import Preprocessor +from .codehilite import CodeHilite, CodeHiliteExtension, parse_hl_lines +from .attr_list import get_attrs, AttrListExtension +from ..util import parseBoolValue +from ..serializers import _escape_attrib_html +import re -class FencedCodeExtension(markdown.Extension): +class FencedCodeExtension(Extension): + def __init__(self, **kwargs): + self.config = { + 'lang_prefix': ['language-', 'Prefix prepended to the language. Default: "language-"'] + } + super().__init__(**kwargs) - def extendMarkdown(self, md, md_globals): + def extendMarkdown(self, md): """ Add FencedBlockPreprocessor to the Markdown instance. """ + md.registerExtension(self) + + md.preprocessors.register(FencedBlockPreprocessor(md, self.getConfigs()), 'fenced_code_block', 25) + + +class FencedBlockPreprocessor(Preprocessor): + FENCED_BLOCK_RE = re.compile( + dedent(r''' + (?P<fence>^(?:~{3,}|`{3,}))[ ]* # opening fence + ((\{(?P<attrs>[^\}\n]*)\})| # (optional {attrs} or + (\.?(?P<lang>[\w#.+-]*)[ ]*)? # optional (.)lang + (hl_lines=(?P<quot>"|')(?P<hl_lines>.*?)(?P=quot)[ ]*)?) # optional hl_lines) + \n # newline (end of opening fence) + (?P<code>.*?)(?<=\n) # the code block + (?P=fence)[ ]*$ # closing fence + '''), + re.MULTILINE | re.DOTALL | re.VERBOSE + ) - md.preprocessors.add('fenced_code_block', - FencedBlockPreprocessor(md), - "_begin") - + def __init__(self, md, config): + super().__init__(md) + self.config = config + self.checked_for_deps = False + self.codehilite_conf = {} + self.use_attr_list = False + # List of options to convert to bool values + self.bool_options = [ + 'linenums', + 'guess_lang', + 'noclasses', + 'use_pygments' + ] -class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): - def run(self, lines): """ Match and store Fenced Code Blocks in the HtmlStash. """ + + # Check for dependent extensions + if not self.checked_for_deps: + for ext in self.md.registeredExtensions: + if isinstance(ext, CodeHiliteExtension): + self.codehilite_conf = ext.getConfigs() + if isinstance(ext, AttrListExtension): + self.use_attr_list = True + + self.checked_for_deps = True + text = "\n".join(lines) while 1: - m = FENCED_BLOCK_RE.search(text) + m = self.FENCED_BLOCK_RE.search(text) if m: - lang = '' - if m.group('lang'): - lang = LANG_TAG % m.group('lang') - code = CODE_WRAP % (lang, self._escape(m.group('code'))) - placeholder = self.markdown.htmlStash.store(code, safe=True) - text = '%s\n%s\n%s'% (text[:m.start()], placeholder, text[m.end():]) + lang, id, classes, config = None, '', [], {} + if m.group('attrs'): + id, classes, config = self.handle_attrs(get_attrs(m.group('attrs'))) + if len(classes): + lang = classes.pop(0) + else: + if m.group('lang'): + lang = m.group('lang') + if m.group('hl_lines'): + # Support hl_lines outside of attrs for backward-compatibility + config['hl_lines'] = parse_hl_lines(m.group('hl_lines')) + + # If config is not empty, then the codehighlite extension + # is enabled, so we call it to highlight the code + if self.codehilite_conf and self.codehilite_conf['use_pygments'] and config.get('use_pygments', True): + local_config = self.codehilite_conf.copy() + local_config.update(config) + # Combine classes with cssclass. Ensure cssclass is at end + # as pygments appends a suffix under certain circumstances. + # Ignore ID as Pygments does not offer an option to set it. + if classes: + local_config['css_class'] = '{} {}'.format( + ' '.join(classes), + local_config['css_class'] + ) + highliter = CodeHilite( + m.group('code'), + lang=lang, + style=local_config.pop('pygments_style', 'default'), + **local_config + ) + + code = highliter.hilite(shebang=False) + else: + id_attr = lang_attr = class_attr = kv_pairs = '' + if lang: + prefix = self.config.get('lang_prefix', 'language-') + lang_attr = f' class="{prefix}{_escape_attrib_html(lang)}"' + if classes: + class_attr = f' class="{_escape_attrib_html(" ".join(classes))}"' + if id: + id_attr = f' id="{_escape_attrib_html(id)}"' + if self.use_attr_list and config and not config.get('use_pygments', False): + # Only assign key/value pairs to code element if attr_list ext is enabled, key/value pairs + # were defined on the code block, and the `use_pygments` key was not set to True. The + # `use_pygments` key could be either set to False or not defined. It is omitted from output. + kv_pairs = ''.join( + f' {k}="{_escape_attrib_html(v)}"' for k, v in config.items() if k != 'use_pygments' + ) + code = self._escape(m.group('code')) + code = f'<pre{id_attr}{class_attr}><code{lang_attr}{kv_pairs}>{code}</code></pre>' + + placeholder = self.md.htmlStash.store(code) + text = f'{text[:m.start()]}\n{placeholder}\n{text[m.end():]}' else: break return text.split("\n") + def handle_attrs(self, attrs): + """ Return tuple: (id, [list, of, classes], {configs}) """ + id = '' + classes = [] + configs = {} + for k, v in attrs: + if k == 'id': + id = v + elif k == '.': + classes.append(v) + elif k == 'hl_lines': + configs[k] = parse_hl_lines(v) + elif k in self.bool_options: + configs[k] = parseBoolValue(v, fail_on_errors=False, preserve_none=True) + else: + configs[k] = v + return id, classes, configs + def _escape(self, txt): """ basic html escaping """ txt = txt.replace('&', '&') @@ -108,10 +170,5 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): return txt -def makeExtension(configs=None): - return FencedCodeExtension() - - -if __name__ == "__main__": - import doctest - doctest.testmod() +def makeExtension(**kwargs): # pragma: no cover + return FencedCodeExtension(**kwargs) diff --git a/markdown/extensions/footnotes.py b/markdown/extensions/footnotes.py index e1a9cda..96ed5c2 100644 --- a/markdown/extensions/footnotes.py +++ b/markdown/extensions/footnotes.py @@ -1,81 +1,126 @@ """ -========================= FOOTNOTES ================================= +Footnotes Extension for Python-Markdown +======================================= -This section adds footnote handling to markdown. It can be used as -an example for extending python-markdown with relatively complex -functionality. While in this case the extension is included inside -the module itself, it could just as easily be added from outside the -module. Not that all markdown classes above are ignorant about -footnotes. All footnote functionality is provided separately and -then added to the markdown instance at the run time. +Adds footnote handling to Python-Markdown. -Footnote functionality is attached by calling extendMarkdown() -method of FootnoteExtension. The method also registers the -extension to allow it's state to be reset by a call to reset() -method. +See <https://Python-Markdown.github.io/extensions/footnotes> +for documentation. -Example: - Footnotes[^1] have a label[^label] and a definition[^!DEF]. +Copyright The Python Markdown Project - [^1]: This is a footnote - [^label]: A footnote on "label" - [^!DEF]: The footnote for definition +License: [BSD](https://opensource.org/licenses/bsd-license.php) """ -import re, markdown -from markdown import etree +from . import Extension +from ..blockprocessors import BlockProcessor +from ..inlinepatterns import InlineProcessor +from ..treeprocessors import Treeprocessor +from ..postprocessors import Postprocessor +from .. import util +from collections import OrderedDict +import re +import copy +import xml.etree.ElementTree as etree -FN_BACKLINK_TEXT = "zz1337820767766393qq" -NBSP_PLACEHOLDER = "qq3936677670287331zz" -DEF_RE = re.compile(r'(\ ?\ ?\ ?)\[\^([^\]]*)\]:\s*(.*)') -TABBED_RE = re.compile(r'((\t)|( ))(.*)') +FN_BACKLINK_TEXT = util.STX + "zz1337820767766393qq" + util.ETX +NBSP_PLACEHOLDER = util.STX + "qq3936677670287331zz" + util.ETX +RE_REF_ID = re.compile(r'(fnref)(\d+)') -class FootnoteExtension(markdown.Extension): + +class FootnoteExtension(Extension): """ Footnote Extension. """ - def __init__ (self, configs): + def __init__(self, **kwargs): """ Setup configs. """ - self.config = {'PLACE_MARKER': - ["///Footnotes Go Here///", - "The text string that marks where the footnotes go"], - 'UNIQUE_IDS': - [False, - "Avoid name collisions across " - "multiple calls to reset()."]} - for key, value in configs: - self.config[key][0] = value + self.config = { + 'PLACE_MARKER': + ["///Footnotes Go Here///", + "The text string that marks where the footnotes go"], + 'UNIQUE_IDS': + [False, + "Avoid name collisions across " + "multiple calls to reset()."], + "BACKLINK_TEXT": + ["↩", + "The text string that links from the footnote " + "to the reader's place."], + "SUPERSCRIPT_TEXT": + ["{}", + "The text string that links from the reader's place " + "to the footnote."], + "BACKLINK_TITLE": + ["Jump back to footnote %d in the text", + "The text string used for the title HTML attribute " + "of the backlink. %d will be replaced by the " + "footnote number."], + "SEPARATOR": + [":", + "Footnote separator."] + } + super().__init__(**kwargs) # In multiple invocations, emit links that don't get tangled. self.unique_prefix = 0 + self.found_refs = {} + self.used_refs = set() self.reset() - def extendMarkdown(self, md, md_globals): + def extendMarkdown(self, md): """ Add pieces to Markdown. """ md.registerExtension(self) self.parser = md.parser - # Insert a preprocessor before ReferencePreprocessor - md.preprocessors.add("footnote", FootnotePreprocessor(self), - "<reference") + self.md = md + # Insert a blockprocessor before ReferencePreprocessor + md.parser.blockprocessors.register(FootnoteBlockProcessor(self), 'footnote', 17) + # Insert an inline pattern before ImageReferencePattern - FOOTNOTE_RE = r'\[\^([^\]]*)\]' # blah blah [^1] blah - md.inlinePatterns.add("footnote", FootnotePattern(FOOTNOTE_RE, self), - "<reference") + FOOTNOTE_RE = r'\[\^([^\]]*)\]' # blah blah [^1] blah + md.inlinePatterns.register(FootnoteInlineProcessor(FOOTNOTE_RE, self), 'footnote', 175) # Insert a tree-processor that would actually add the footnote div - # This must be before the inline treeprocessor so inline patterns - # run on the contents of the div. - md.treeprocessors.add("footnote", FootnoteTreeprocessor(self), - "<inline") - # Insert a postprocessor after amp_substitute oricessor - md.postprocessors.add("footnote", FootnotePostprocessor(self), - ">amp_substitute") + # This must be before all other treeprocessors (i.e., inline and + # codehilite) so they can run on the the contents of the div. + md.treeprocessors.register(FootnoteTreeprocessor(self), 'footnote', 50) + + # Insert a tree-processor that will run after inline is done. + # In this tree-processor we want to check our duplicate footnote tracker + # And add additional backrefs to the footnote pointing back to the + # duplicated references. + md.treeprocessors.register(FootnotePostTreeprocessor(self), 'footnote-duplicate', 15) + + # Insert a postprocessor after amp_substitute processor + md.postprocessors.register(FootnotePostprocessor(self), 'footnote', 25) def reset(self): - """ Clear the footnotes on reset, and prepare for a distinct document. """ - self.footnotes = markdown.odict.OrderedDict() + """ Clear footnotes on reset, and prepare for distinct document. """ + self.footnotes = OrderedDict() self.unique_prefix += 1 + self.found_refs = {} + self.used_refs = set() + + def unique_ref(self, reference, found=False): + """ Get a unique reference if there are duplicates. """ + if not found: + return reference + + original_ref = reference + while reference in self.used_refs: + ref, rest = reference.split(self.get_separator(), 1) + m = RE_REF_ID.match(ref) + if m: + reference = '%s%d%s%s' % (m.group(1), int(m.group(2))+1, self.get_separator(), rest) + else: + reference = '%s%d%s%s' % (ref, 2, self.get_separator(), rest) + + self.used_refs.add(reference) + if original_ref in self.found_refs: + self.found_refs[original_ref] += 1 + else: + self.found_refs[original_ref] = 1 + return reference def findFootnotesPlaceholder(self, root): """ Return ElementTree Element that contains Footnote placeholder. """ @@ -83,13 +128,15 @@ class FootnoteExtension(markdown.Extension): for child in element: if child.text: if child.text.find(self.getConfig("PLACE_MARKER")) > -1: - return child, True + return child, element, True if child.tail: if child.tail.find(self.getConfig("PLACE_MARKER")) > -1: - return (child, element), False - finder(child) + return child, element, False + child_res = finder(child) + if child_res is not None: + return child_res return None - + res = finder(root) return res @@ -97,43 +144,59 @@ class FootnoteExtension(markdown.Extension): """ Store a footnote for later retrieval. """ self.footnotes[id] = text + def get_separator(self): + """ Get the footnote separator. """ + return self.getConfig("SEPARATOR") + def makeFootnoteId(self, id): """ Return footnote link id. """ if self.getConfig("UNIQUE_IDS"): - return 'fn:%d-%s' % (self.unique_prefix, id) + return 'fn%s%d-%s' % (self.get_separator(), self.unique_prefix, id) else: - return 'fn:%s' % id + return 'fn{}{}'.format(self.get_separator(), id) - def makeFootnoteRefId(self, id): + def makeFootnoteRefId(self, id, found=False): """ Return footnote back-link id. """ if self.getConfig("UNIQUE_IDS"): - return 'fnref:%d-%s' % (self.unique_prefix, id) + return self.unique_ref('fnref%s%d-%s' % (self.get_separator(), self.unique_prefix, id), found) else: - return 'fnref:%s' % id + return self.unique_ref('fnref{}{}'.format(self.get_separator(), id), found) def makeFootnotesDiv(self, root): """ Return div of footnotes as et Element. """ - if not self.footnotes.keys(): + if not list(self.footnotes.keys()): return None div = etree.Element("div") div.set('class', 'footnote') - hr = etree.SubElement(div, "hr") + etree.SubElement(div, "hr") ol = etree.SubElement(div, "ol") + surrogate_parent = etree.Element("div") + + # Backward compatibility with old '%d' placeholder + backlink_title = self.getConfig("BACKLINK_TITLE").replace("%d", "{}") - for id in self.footnotes.keys(): + for index, id in enumerate(self.footnotes.keys(), start=1): li = etree.SubElement(ol, "li") li.set("id", self.makeFootnoteId(id)) - self.parser.parseChunk(li, self.footnotes[id]) + # Parse footnote with surrogate parent as li cannot be used. + # List block handlers have special logic to deal with li. + # When we are done parsing, we will copy everything over to li. + self.parser.parseChunk(surrogate_parent, self.footnotes[id]) + for el in list(surrogate_parent): + li.append(el) + surrogate_parent.remove(el) backlink = etree.Element("a") backlink.set("href", "#" + self.makeFootnoteRefId(id)) - backlink.set("rev", "footnote") - backlink.set("title", "Jump back to footnote %d in the text" % \ - (self.footnotes.index(id)+1)) + backlink.set("class", "footnote-backref") + backlink.set( + "title", + backlink_title.format(index) + ) backlink.text = FN_BACKLINK_TEXT - if li.getchildren(): + if len(li): node = li[-1] if node.tag == "p": node.text = node.text + NBSP_PLACEHOLDER @@ -144,164 +207,205 @@ class FootnoteExtension(markdown.Extension): return div -class FootnotePreprocessor(markdown.preprocessors.Preprocessor): +class FootnoteBlockProcessor(BlockProcessor): """ Find all footnote references and store for later use. """ - def __init__ (self, footnotes): - self.footnotes = footnotes - - def run(self, lines): - lines = self._handleFootnoteDefinitions(lines) - text = "\n".join(lines) - return text.split("\n") + RE = re.compile(r'^[ ]{0,3}\[\^([^\]]*)\]:[ ]*(.*)$', re.MULTILINE) - def _handleFootnoteDefinitions(self, lines): - """ - Recursively find all footnote definitions in lines. - - Keywords: + def __init__(self, footnotes): + super().__init__(footnotes.parser) + self.footnotes = footnotes - * lines: A list of lines of text - - Return: A list of lines with footnote definitions removed. - - """ - i, id, footnote = self._findFootnoteDefinition(lines) - - if id : - plain = lines[:i] - detabbed, theRest = self.detectTabbed(lines[i+1:]) - self.footnotes.setFootnote(id, - footnote + "\n" - + "\n".join(detabbed)) - more_plain = self._handleFootnoteDefinitions(theRest) - return plain + [""] + more_plain - else : - return lines - - def _findFootnoteDefinition(self, lines): - """ - Find the parts of a footnote definition. + def test(self, parent, block): + return True + + def run(self, parent, blocks): + """ Find, set, and remove footnote definitions. """ + block = blocks.pop(0) + m = self.RE.search(block) + if m: + id = m.group(1) + fn_blocks = [m.group(2)] + + # Handle rest of block + therest = block[m.end():].lstrip('\n') + m2 = self.RE.search(therest) + if m2: + # Another footnote exists in the rest of this block. + # Any content before match is continuation of this footnote, which may be lazily indented. + before = therest[:m2.start()].rstrip('\n') + fn_blocks[0] = '\n'.join([fn_blocks[0], self.detab(before)]).lstrip('\n') + # Add back to blocks everything from beginning of match forward for next iteration. + blocks.insert(0, therest[m2.start():]) + else: + # All remaining lines of block are continuation of this footnote, which may be lazily indented. + fn_blocks[0] = '\n'.join([fn_blocks[0], self.detab(therest)]).strip('\n') - Keywords: + # Check for child elements in remaining blocks. + fn_blocks.extend(self.detectTabbed(blocks)) - * lines: A list of lines of text. + footnote = "\n\n".join(fn_blocks) + self.footnotes.setFootnote(id, footnote.rstrip()) - Return: A three item tuple containing the index of the first line of a - footnote definition, the id of the definition and the body of the - definition. - - """ - counter = 0 - for line in lines: - m = DEF_RE.match(line) - if m: - return counter, m.group(2), m.group(3) - counter += 1 - return counter, None, None + if block[:m.start()].strip(): + # Add any content before match back to blocks as separate block + blocks.insert(0, block[:m.start()].rstrip('\n')) + return True + # No match. Restore block. + blocks.insert(0, block) + return False - def detectTabbed(self, lines): + def detectTabbed(self, blocks): """ Find indented text and remove indent before further proccesing. - Keyword arguments: - - * lines: an array of strings - - Returns: a list of post processed items and the unused - remainder of the original list - + Returns: a list of blocks with indentation removed. """ - items = [] - item = -1 - i = 0 # to keep track of where we are - - def detab(line): - match = TABBED_RE.match(line) - if match: - return match.group(4) - - for line in lines: - if line.strip(): # Non-blank line - line = detab(line) - if line: - items.append(line) - i += 1 - continue - else: - return items, lines[i:] - - else: # Blank line: _maybe_ we are done. - i += 1 # advance - - # Find the next non-blank line - for j in range(i, len(lines)): - if lines[j].strip(): - next_line = lines[j]; break + fn_blocks = [] + while blocks: + if blocks[0].startswith(' '*4): + block = blocks.pop(0) + # Check for new footnotes within this block and split at new footnote. + m = self.RE.search(block) + if m: + # Another footnote exists in this block. + # Any content before match is continuation of this footnote, which may be lazily indented. + before = block[:m.start()].rstrip('\n') + fn_blocks.append(self.detab(before)) + # Add back to blocks everything from beginning of match forward for next iteration. + blocks.insert(0, block[m.start():]) + # End of this footnote. + break else: - break # There is no more text; we are done. + # Entire block is part of this footnote. + fn_blocks.append(self.detab(block)) + else: + # End of this footnote. + break + return fn_blocks - # Check if the next non-blank line is tabbed - if detab(next_line): # Yes, more work to do. - items.append("") - continue - else: - break # No, we are done. - else: - i += 1 + def detab(self, block): + """ Remove one level of indent from a block. - return items, lines[i:] + Preserve lazily indented blocks by only removing indent from indented lines. + """ + lines = block.split('\n') + for i, line in enumerate(lines): + if line.startswith(' '*4): + lines[i] = line[4:] + return '\n'.join(lines) -class FootnotePattern(markdown.inlinepatterns.Pattern): +class FootnoteInlineProcessor(InlineProcessor): """ InlinePattern for footnote markers in a document's body text. """ def __init__(self, pattern, footnotes): - markdown.inlinepatterns.Pattern.__init__(self, pattern) + super().__init__(pattern) self.footnotes = footnotes - def handleMatch(self, m): - sup = etree.Element("sup") - a = etree.SubElement(sup, "a") - id = m.group(2) - sup.set('id', self.footnotes.makeFootnoteRefId(id)) - a.set('href', '#' + self.footnotes.makeFootnoteId(id)) - a.set('rel', 'footnote') - a.text = str(self.footnotes.footnotes.index(id) + 1) - return sup + def handleMatch(self, m, data): + id = m.group(1) + if id in self.footnotes.footnotes.keys(): + sup = etree.Element("sup") + a = etree.SubElement(sup, "a") + sup.set('id', self.footnotes.makeFootnoteRefId(id, found=True)) + a.set('href', '#' + self.footnotes.makeFootnoteId(id)) + a.set('class', 'footnote-ref') + a.text = self.footnotes.getConfig("SUPERSCRIPT_TEXT").format( + list(self.footnotes.footnotes.keys()).index(id) + 1 + ) + return sup, m.start(0), m.end(0) + else: + return None, None, None + + +class FootnotePostTreeprocessor(Treeprocessor): + """ Amend footnote div with duplicates. """ + + def __init__(self, footnotes): + self.footnotes = footnotes + def add_duplicates(self, li, duplicates): + """ Adjust current li and add the duplicates: fnref2, fnref3, etc. """ + for link in li.iter('a'): + # Find the link that needs to be duplicated. + if link.attrib.get('class', '') == 'footnote-backref': + ref, rest = link.attrib['href'].split(self.footnotes.get_separator(), 1) + # Duplicate link the number of times we need to + # and point the to the appropriate references. + links = [] + for index in range(2, duplicates + 1): + sib_link = copy.deepcopy(link) + sib_link.attrib['href'] = '%s%d%s%s' % (ref, index, self.footnotes.get_separator(), rest) + links.append(sib_link) + self.offset += 1 + # Add all the new duplicate links. + el = list(li)[-1] + for link in links: + el.append(link) + break + + def get_num_duplicates(self, li): + """ Get the number of duplicate refs of the footnote. """ + fn, rest = li.attrib.get('id', '').split(self.footnotes.get_separator(), 1) + link_id = '{}ref{}{}'.format(fn, self.footnotes.get_separator(), rest) + return self.footnotes.found_refs.get(link_id, 0) + + def handle_duplicates(self, parent): + """ Find duplicate footnotes and format and add the duplicates. """ + for li in list(parent): + # Check number of duplicates footnotes and insert + # additional links if needed. + count = self.get_num_duplicates(li) + if count > 1: + self.add_duplicates(li, count) -class FootnoteTreeprocessor(markdown.treeprocessors.Treeprocessor): + def run(self, root): + """ Crawl the footnote div and add missing duplicate footnotes. """ + self.offset = 0 + for div in root.iter('div'): + if div.attrib.get('class', '') == 'footnote': + # Footnotes should be under the first ordered list under + # the footnote div. So once we find it, quit. + for ol in div.iter('ol'): + self.handle_duplicates(ol) + break + + +class FootnoteTreeprocessor(Treeprocessor): """ Build and append footnote div to end of document. """ - def __init__ (self, footnotes): + def __init__(self, footnotes): self.footnotes = footnotes def run(self, root): footnotesDiv = self.footnotes.makeFootnotesDiv(root) - if footnotesDiv: + if footnotesDiv is not None: result = self.footnotes.findFootnotesPlaceholder(root) if result: - node, isText = result + child, parent, isText = result + ind = list(parent).index(child) if isText: - node.text = None - node.getchildren().insert(0, footnotesDiv) + parent.remove(child) + parent.insert(ind, footnotesDiv) else: - child, element = node - ind = element.getchildren().find(child) - element.getchildren().insert(ind + 1, footnotesDiv) + parent.insert(ind + 1, footnotesDiv) child.tail = None - fnPlaceholder.parent.replaceChild(fnPlaceholder, footnotesDiv) else: root.append(footnotesDiv) -class FootnotePostprocessor(markdown.postprocessors.Postprocessor): + +class FootnotePostprocessor(Postprocessor): """ Replace placeholders with html entities. """ + def __init__(self, footnotes): + self.footnotes = footnotes def run(self, text): - text = text.replace(FN_BACKLINK_TEXT, "↩") + text = text.replace( + FN_BACKLINK_TEXT, self.footnotes.getConfig("BACKLINK_TEXT") + ) return text.replace(NBSP_PLACEHOLDER, " ") -def makeExtension(configs=[]): - """ Return an instance of the FootnoteExtension """ - return FootnoteExtension(configs=configs) +def makeExtension(**kwargs): # pragma: no cover + """ Return an instance of the FootnoteExtension """ + return FootnoteExtension(**kwargs) diff --git a/markdown/extensions/headerid.py b/markdown/extensions/headerid.py deleted file mode 100644 index f70a7a9..0000000 --- a/markdown/extensions/headerid.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/python - -""" -HeaderID Extension for Python-Markdown -====================================== - -Adds ability to set HTML IDs for headers. - -Basic usage: - - >>> import markdown - >>> text = "# Some Header # {#some_id}" - >>> md = markdown.markdown(text, ['headerid']) - >>> md - u'<h1 id="some_id">Some Header</h1>' - -All header IDs are unique: - - >>> text = ''' - ... #Header - ... #Another Header {#header} - ... #Third Header {#header}''' - >>> md = markdown.markdown(text, ['headerid']) - >>> md - u'<h1 id="header">Header</h1>\\n<h1 id="header_1">Another Header</h1>\\n<h1 id="header_2">Third Header</h1>' - -To fit within a html template's hierarchy, set the header base level: - - >>> text = ''' - ... #Some Header - ... ## Next Level''' - >>> md = markdown.markdown(text, ['headerid(level=3)']) - >>> md - u'<h3 id="some_header">Some Header</h3>\\n<h4 id="next_level">Next Level</h4>' - -Turn off auto generated IDs: - - >>> text = ''' - ... # Some Header - ... # Header with ID # { #foo }''' - >>> md = markdown.markdown(text, ['headerid(forceid=False)']) - >>> md - u'<h1>Some Header</h1>\\n<h1 id="foo">Header with ID</h1>' - -Use with MetaData extension: - - >>> text = '''header_level: 2 - ... header_forceid: Off - ... - ... # A Header''' - >>> md = markdown.markdown(text, ['headerid', 'meta']) - >>> md - u'<h2>A Header</h2>' - -Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/). - -Project website: <http://www.freewisdom.org/project/python-markdown/HeaderId> -Contact: markdown@freewisdom.org - -License: BSD (see ../docs/LICENSE for details) - -Dependencies: -* [Python 2.3+](http://python.org) -* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) - -""" - -import markdown -from markdown import etree -import re -from string import ascii_lowercase, digits, punctuation - -ID_CHARS = ascii_lowercase + digits + '-_' -IDCOUNT_RE = re.compile(r'^(.*)_([0-9]+)$') - - -class HeaderIdProcessor(markdown.blockprocessors.BlockProcessor): - """ Replacement BlockProcessor for Header IDs. """ - - # Detect a header at start of any line in block - RE = re.compile(r"""(^|\n) - (?P<level>\#{1,6}) # group('level') = string of hashes - (?P<header>.*?) # group('header') = Header text - \#* # optional closing hashes - (?:[ \t]*\{[ \t]*\#(?P<id>[-_:a-zA-Z0-9]+)[ \t]*\})? - (\n|$) # ^^ group('id') = id attribute - """, - re.VERBOSE) - - IDs = [] - - def test(self, parent, block): - return bool(self.RE.search(block)) - - def run(self, parent, blocks): - block = blocks.pop(0) - m = self.RE.search(block) - if m: - before = block[:m.start()] # All lines before header - after = block[m.end():] # All lines after header - if before: - # As the header was not the first line of the block and the - # lines before the header must be parsed first, - # recursively parse this lines as a block. - self.parser.parseBlocks(parent, [before]) - # Create header using named groups from RE - start_level, force_id = self._get_meta() - level = len(m.group('level')) + start_level - if level > 6: - level = 6 - h = markdown.etree.SubElement(parent, 'h%d' % level) - h.text = m.group('header').strip() - if m.group('id'): - h.set('id', self._unique_id(m.group('id'))) - elif force_id: - h.set('id', self._create_id(m.group('header').strip())) - if after: - # Insert remaining lines as first block for future parsing. - blocks.insert(0, after) - else: - # This should never happen, but just in case... - message(CRITICAL, "We've got a problem header!") - - def _get_meta(self): - """ Return meta data suported by this ext as a tuple """ - level = int(self.config['level'][0]) - 1 - force = self._str2bool(self.config['forceid'][0]) - if hasattr(self.md, 'Meta'): - if self.md.Meta.has_key('header_level'): - level = int(self.md.Meta['header_level'][0]) - 1 - if self.md.Meta.has_key('header_forceid'): - force = self._str2bool(self.md.Meta['header_forceid'][0]) - return level, force - - def _str2bool(self, s, default=False): - """ Convert a string to a booleen value. """ - s = str(s) - if s.lower() in ['0', 'f', 'false', 'off', 'no', 'n']: - return False - elif s.lower() in ['1', 't', 'true', 'on', 'yes', 'y']: - return True - return default - - def _unique_id(self, id): - """ Ensure ID is unique. Append '_1', '_2'... if not """ - while id in self.IDs: - m = IDCOUNT_RE.match(id) - if m: - id = '%s_%d'% (m.group(1), int(m.group(2))+1) - else: - id = '%s_%d'% (id, 1) - self.IDs.append(id) - return id - - def _create_id(self, header): - """ Return ID from Header text. """ - h = '' - for c in header.lower().replace(' ', '_'): - if c in ID_CHARS: - h += c - elif c not in punctuation: - h += '+' - return self._unique_id(h) - - -class HeaderIdExtension (markdown.Extension): - def __init__(self, configs): - # set defaults - self.config = { - 'level' : ['1', 'Base level for headers.'], - 'forceid' : ['True', 'Force all headers to have an id.'] - } - - for key, value in configs: - self.setConfig(key, value) - - def extendMarkdown(self, md, md_globals): - md.registerExtension(self) - self.processor = HeaderIdProcessor(md.parser) - self.processor.md = md - self.processor.config = self.config - # Replace existing hasheader in place. - md.parser.blockprocessors['hashheader'] = self.processor - - def reset(self): - self.processor.IDs = [] - - -def makeExtension(configs=None): - return HeaderIdExtension(configs=configs) - -if __name__ == "__main__": - import doctest - doctest.testmod() - diff --git a/markdown/extensions/html_tidy.py b/markdown/extensions/html_tidy.py deleted file mode 100644 index 5105e33..0000000 --- a/markdown/extensions/html_tidy.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python - -""" -HTML Tidy Extension for Python-Markdown -======================================= - -Runs [HTML Tidy][] on the output of Python-Markdown using the [uTidylib][] -Python wrapper. Both libtidy and uTidylib must be installed on your system. - -Note than any Tidy [options][] can be passed in as extension configs. So, -for example, to output HTML rather than XHTML, set ``output_xhtml=0``. To -indent the output, set ``indent=auto`` and to have Tidy wrap the output in -``<html>`` and ``<body>`` tags, set ``show_body_only=0``. - -[HTML Tidy]: http://tidy.sourceforge.net/ -[uTidylib]: http://utidylib.berlios.de/ -[options]: http://tidy.sourceforge.net/docs/quickref.html - -Copyright (c)2008 [Waylan Limberg](http://achinghead.com) - -License: [BSD](http://www.opensource.org/licenses/bsd-license.php) - -Dependencies: -* [Python2.3+](http://python.org) -* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) -* [HTML Tidy](http://utidylib.berlios.de/) -* [uTidylib](http://utidylib.berlios.de/) - -""" - -import markdown -import tidy - -class TidyExtension(markdown.Extension): - - def __init__(self, configs): - # Set defaults to match typical markdown behavior. - self.config = dict(output_xhtml=1, - show_body_only=1, - ) - # Merge in user defined configs overriding any present if nessecary. - for c in configs: - self.config[c[0]] = c[1] - - def extendMarkdown(self, md, md_globals): - # Save options to markdown instance - md.tidy_options = self.config - # Add TidyProcessor to postprocessors - md.postprocessors['tidy'] = TidyProcessor(md) - - -class TidyProcessor(markdown.postprocessors.Postprocessor): - - def run(self, text): - # Pass text to Tidy. As Tidy does not accept unicode we need to encode - # it and decode its return value. - return unicode(tidy.parseString(text.encode('utf-8'), - **self.markdown.tidy_options)) - - -def makeExtension(configs=None): - return TidyExtension(configs=configs) diff --git a/markdown/extensions/imagelinks.py b/markdown/extensions/imagelinks.py deleted file mode 100644 index ee0b708..0000000 --- a/markdown/extensions/imagelinks.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -========================= IMAGE LINKS ================================= - - -Turns paragraphs like - -<~~~~~~~~~~~~~~~~~~~~~~~~ -dir/subdir -dir/subdir -dir/subdir -~~~~~~~~~~~~~~ -dir/subdir -dir/subdir -dir/subdir -~~~~~~~~~~~~~~~~~~~> - -Into mini-photo galleries. - -""" - -import re, markdown -import url_manager - - -IMAGE_LINK = """<a href="%s"><img src="%s" title="%s"/></a>""" -SLIDESHOW_LINK = """<a href="%s" target="_blank">[slideshow]</a>""" -ALBUM_LINK = """ <a href="%s">[%s]</a>""" - - -class ImageLinksExtension(markdown.Extension): - - def extendMarkdown(self, md, md_globals): - - md.preprocessors.add("imagelink", ImageLinkPreprocessor(md), "_begin") - - -class ImageLinkPreprocessor(markdown.preprocessors.Preprocessor): - - def run(self, lines): - - url = url_manager.BlogEntryUrl(url_manager.BlogUrl("all"), - "2006/08/29/the_rest_of_our") - - - all_images = [] - blocks = [] - in_image_block = False - - new_lines = [] - - for line in lines: - - if line.startswith("<~~~~~~~"): - albums = [] - rows = [] - in_image_block = True - - if not in_image_block: - - new_lines.append(line) - - else: - - line = line.strip() - - if line.endswith("~~~~~~>") or not line: - in_image_block = False - new_block = "<div><br/><center><span class='image-links'>\n" - - album_url_hash = {} - - for row in rows: - for photo_url, title in row: - new_block += " " - new_block += IMAGE_LINK % (photo_url, - photo_url.get_thumbnail(), - title) - - album_url_hash[str(photo_url.get_album())] = 1 - - new_block += "<br/>" - - new_block += "</span>" - new_block += SLIDESHOW_LINK % url.get_slideshow() - - album_urls = album_url_hash.keys() - album_urls.sort() - - if len(album_urls) == 1: - new_block += ALBUM_LINK % (album_urls[0], "complete album") - else : - for i in range(len(album_urls)) : - new_block += ALBUM_LINK % (album_urls[i], - "album %d" % (i + 1) ) - - new_lines.append(new_block + "</center><br/></div>") - - elif line[1:6] == "~~~~~" : - rows.append([]) # start a new row - else : - parts = line.split() - line = parts[0] - title = " ".join(parts[1:]) - - album, photo = line.split("/") - photo_url = url.get_photo(album, photo, - len(all_images)+1) - all_images.append(photo_url) - rows[-1].append((photo_url, title)) - - if not album in albums : - albums.append(album) - - return new_lines - - -def makeExtension(configs): - return ImageLinksExtension(configs) - diff --git a/markdown/extensions/legacy_attrs.py b/markdown/extensions/legacy_attrs.py new file mode 100644 index 0000000..445aba1 --- /dev/null +++ b/markdown/extensions/legacy_attrs.py @@ -0,0 +1,67 @@ +""" +Python Markdown + +A Python implementation of John Gruber's Markdown. + +Documentation: https://python-markdown.github.io/ +GitHub: https://github.com/Python-Markdown/markdown/ +PyPI: https://pypi.org/project/Markdown/ + +Started by Manfred Stienstra (http://www.dwerg.net/). +Maintained for a few years by Yuri Takhteyev (http://www.freewisdom.org). +Currently maintained by Waylan Limberg (https://github.com/waylan), +Dmitry Shachnev (https://github.com/mitya57) and Isaac Muse (https://github.com/facelessuser). + +Copyright 2007-2018 The Python Markdown Project (v. 1.7 and later) +Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) +Copyright 2004 Manfred Stienstra (the original version) + +License: BSD (see LICENSE.md for details). + +Legacy Attributes Extension +=========================== + +An extension to Python Markdown which implements legacy attributes. + +Prior to Python-Markdown version 3.0, the Markdown class had an `enable_attributes` +keyword which was on by default and provided for attributes to be defined for elements +using the format `{@key=value}`. This extension is provided as a replacement for +backward compatibility. New documents should be authored using attr_lists. However, +numerious documents exist which have been using the old attribute format for many +years. This extension can be used to continue to render those documents correctly. +""" + +import re +from markdown.treeprocessors import Treeprocessor, isString +from markdown.extensions import Extension + + +ATTR_RE = re.compile(r'\{@([^\}]*)=([^\}]*)}') # {@id=123} + + +class LegacyAttrs(Treeprocessor): + def run(self, doc): + """Find and set values of attributes ({@key=value}). """ + for el in doc.iter(): + alt = el.get('alt', None) + if alt is not None: + el.set('alt', self.handleAttributes(el, alt)) + if el.text and isString(el.text): + el.text = self.handleAttributes(el, el.text) + if el.tail and isString(el.tail): + el.tail = self.handleAttributes(el, el.tail) + + def handleAttributes(self, el, txt): + """ Set attributes and return text without definitions. """ + def attributeCallback(match): + el.set(match.group(1), match.group(2).replace('\n', ' ')) + return ATTR_RE.sub(attributeCallback, txt) + + +class LegacyAttrExtension(Extension): + def extendMarkdown(self, md): + md.treeprocessors.register(LegacyAttrs(md), 'legacyattrs', 15) + + +def makeExtension(**kwargs): # pragma: no cover + return LegacyAttrExtension(**kwargs) diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py new file mode 100644 index 0000000..360988b --- /dev/null +++ b/markdown/extensions/legacy_em.py @@ -0,0 +1,49 @@ +''' +Legacy Em Extension for Python-Markdown +======================================= + +This extension provides legacy behavior for _connected_words_. + +Copyright 2015-2018 The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) + +''' + +from . import Extension +from ..inlinepatterns import UnderscoreProcessor, EmStrongItem, EM_STRONG2_RE, STRONG_EM2_RE +import re + +# _emphasis_ +EMPHASIS_RE = r'(_)([^_]+)\1' + +# __strong__ +STRONG_RE = r'(_{2})(.+?)\1' + +# __strong_em___ +STRONG_EM_RE = r'(_)\1(?!\1)([^_]+?)\1(?!\1)(.+?)\1{3}' + + +class LegacyUnderscoreProcessor(UnderscoreProcessor): + """Emphasis processor for handling strong and em matches inside underscores.""" + + PATTERNS = [ + EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), + EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), + EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), + EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), + EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') + ] + + +class LegacyEmExtension(Extension): + """ Add legacy_em extension to Markdown class.""" + + def extendMarkdown(self, md): + """ Modify inline patterns. """ + md.inlinePatterns.register(LegacyUnderscoreProcessor(r'_'), 'em_strong2', 50) + + +def makeExtension(**kwargs): # pragma: no cover + """ Return an instance of the LegacyEmExtension """ + return LegacyEmExtension(**kwargs) diff --git a/markdown/extensions/md_in_html.py b/markdown/extensions/md_in_html.py new file mode 100644 index 0000000..ec7dcba --- /dev/null +++ b/markdown/extensions/md_in_html.py @@ -0,0 +1,364 @@ +""" +Python-Markdown Markdown in HTML Extension +=============================== + +An implementation of [PHP Markdown Extra](http://michelf.com/projects/php-markdown/extra/)'s +parsing of Markdown syntax in raw HTML. + +See <https://Python-Markdown.github.io/extensions/raw_html> +for documentation. + +Copyright The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) + +""" + +from . import Extension +from ..blockprocessors import BlockProcessor +from ..preprocessors import Preprocessor +from ..postprocessors import RawHtmlPostprocessor +from .. import util +from ..htmlparser import HTMLExtractor, blank_line_re +import xml.etree.ElementTree as etree + + +class HTMLExtractorExtra(HTMLExtractor): + """ + Override HTMLExtractor and create etree Elements for any elements which should have content parsed as Markdown. + """ + + def __init__(self, md, *args, **kwargs): + # All block-level tags. + self.block_level_tags = set(md.block_level_elements.copy()) + # Block-level tags in which the content only gets span level parsing + self.span_tags = set( + ['address', 'dd', 'dt', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'legend', 'li', 'p', 'summary', 'td', 'th'] + ) + # Block-level tags which never get their content parsed. + self.raw_tags = set(['canvas', 'math', 'option', 'pre', 'script', 'style', 'textarea']) + + super().__init__(md, *args, **kwargs) + + # Block-level tags in which the content gets parsed as blocks + self.block_tags = set(self.block_level_tags) - (self.span_tags | self.raw_tags | self.empty_tags) + self.span_and_blocks_tags = self.block_tags | self.span_tags + + def reset(self): + """Reset this instance. Loses all unprocessed data.""" + self.mdstack = [] # When markdown=1, stack contains a list of tags + self.treebuilder = etree.TreeBuilder() + self.mdstate = [] # one of 'block', 'span', 'off', or None + super().reset() + + def close(self): + """Handle any buffered data.""" + super().close() + # Handle any unclosed tags. + if self.mdstack: + # Close the outermost parent. handle_endtag will close all unclosed children. + self.handle_endtag(self.mdstack[0]) + + def get_element(self): + """ Return element from treebuilder and reset treebuilder for later use. """ + element = self.treebuilder.close() + self.treebuilder = etree.TreeBuilder() + return element + + def get_state(self, tag, attrs): + """ Return state from tag and `markdown` attr. One of 'block', 'span', or 'off'. """ + md_attr = attrs.get('markdown', '0') + if md_attr == 'markdown': + # `<tag markdown>` is the same as `<tag markdown='1'>`. + md_attr = '1' + parent_state = self.mdstate[-1] if self.mdstate else None + if parent_state == 'off' or (parent_state == 'span' and md_attr != '0'): + # Only use the parent state if it is more restrictive than the markdown attribute. + md_attr = parent_state + if ((md_attr == '1' and tag in self.block_tags) or + (md_attr == 'block' and tag in self.span_and_blocks_tags)): + return 'block' + elif ((md_attr == '1' and tag in self.span_tags) or + (md_attr == 'span' and tag in self.span_and_blocks_tags)): + return 'span' + elif tag in self.block_level_tags: + return 'off' + else: # pragma: no cover + return None + + def handle_starttag(self, tag, attrs): + # Handle tags that should always be empty and do not specify a closing tag + if tag in self.empty_tags and (self.at_line_start() or self.intail): + attrs = {key: value if value is not None else key for key, value in attrs} + if "markdown" in attrs: + attrs.pop('markdown') + element = etree.Element(tag, attrs) + data = etree.tostring(element, encoding='unicode', method='html') + else: + data = self.get_starttag_text() + self.handle_empty_tag(data, True) + return + + if tag in self.block_level_tags and (self.at_line_start() or self.intail): + # Valueless attr (ex: `<tag checked>`) results in `[('checked', None)]`. + # Convert to `{'checked': 'checked'}`. + attrs = {key: value if value is not None else key for key, value in attrs} + state = self.get_state(tag, attrs) + if self.inraw or (state in [None, 'off'] and not self.mdstack): + # fall back to default behavior + attrs.pop('markdown', None) + super().handle_starttag(tag, attrs) + else: + if 'p' in self.mdstack and tag in self.block_level_tags: + # Close unclosed 'p' tag + self.handle_endtag('p') + self.mdstate.append(state) + self.mdstack.append(tag) + attrs['markdown'] = state + self.treebuilder.start(tag, attrs) + else: + # Span level tag + if self.inraw: + super().handle_starttag(tag, attrs) + else: + text = self.get_starttag_text() + if self.mdstate and self.mdstate[-1] == "off": + self.handle_data(self.md.htmlStash.store(text)) + else: + self.handle_data(text) + if tag in self.CDATA_CONTENT_ELEMENTS: + # This is presumably a standalone tag in a code span (see #1036). + self.clear_cdata_mode() + + def handle_endtag(self, tag): + if tag in self.block_level_tags: + if self.inraw: + super().handle_endtag(tag) + elif tag in self.mdstack: + # Close element and any unclosed children + while self.mdstack: + item = self.mdstack.pop() + self.mdstate.pop() + self.treebuilder.end(item) + if item == tag: + break + if not self.mdstack: + # Last item in stack is closed. Stash it + element = self.get_element() + # Get last entry to see if it ends in newlines + # If it is an element, assume there is no newlines + item = self.cleandoc[-1] if self.cleandoc else '' + # If we only have one newline before block element, add another + if not item.endswith('\n\n') and item.endswith('\n'): + self.cleandoc.append('\n') + self.cleandoc.append(self.md.htmlStash.store(element)) + self.cleandoc.append('\n\n') + self.state = [] + # Check if element has a tail + if not blank_line_re.match( + self.rawdata[self.line_offset + self.offset + len(self.get_endtag_text(tag)):]): + # More content exists after endtag. + self.intail = True + else: + # Treat orphan closing tag as a span level tag. + text = self.get_endtag_text(tag) + if self.mdstate and self.mdstate[-1] == "off": + self.handle_data(self.md.htmlStash.store(text)) + else: + self.handle_data(text) + else: + # Span level tag + if self.inraw: + super().handle_endtag(tag) + else: + text = self.get_endtag_text(tag) + if self.mdstate and self.mdstate[-1] == "off": + self.handle_data(self.md.htmlStash.store(text)) + else: + self.handle_data(text) + + def handle_startendtag(self, tag, attrs): + if tag in self.empty_tags: + attrs = {key: value if value is not None else key for key, value in attrs} + if "markdown" in attrs: + attrs.pop('markdown') + element = etree.Element(tag, attrs) + data = etree.tostring(element, encoding='unicode', method='html') + else: + data = self.get_starttag_text() + else: + data = self.get_starttag_text() + self.handle_empty_tag(data, is_block=self.md.is_block_level(tag)) + + def handle_data(self, data): + if self.intail and '\n' in data: + self.intail = False + if self.inraw or not self.mdstack: + super().handle_data(data) + else: + self.treebuilder.data(data) + + def handle_empty_tag(self, data, is_block): + if self.inraw or not self.mdstack: + super().handle_empty_tag(data, is_block) + else: + if self.at_line_start() and is_block: + self.handle_data('\n' + self.md.htmlStash.store(data) + '\n\n') + else: + self.handle_data(self.md.htmlStash.store(data)) + + def parse_pi(self, i): + if self.at_line_start() or self.intail or self.mdstack: + # The same override exists in HTMLExtractor without the check + # for mdstack. Therefore, use HTMLExtractor's parent instead. + return super(HTMLExtractor, self).parse_pi(i) + # This is not the beginning of a raw block so treat as plain data + # and avoid consuming any tags which may follow (see #1066). + self.handle_data('<?') + return i + 2 + + def parse_html_declaration(self, i): + if self.at_line_start() or self.intail or self.mdstack: + # The same override exists in HTMLExtractor without the check + # for mdstack. Therefore, use HTMLExtractor's parent instead. + return super(HTMLExtractor, self).parse_html_declaration(i) + # This is not the beginning of a raw block so treat as plain data + # and avoid consuming any tags which may follow (see #1066). + self.handle_data('<!') + return i + 2 + + +class HtmlBlockPreprocessor(Preprocessor): + """Remove html blocks from the text and store them for later retrieval.""" + + def run(self, lines): + source = '\n'.join(lines) + parser = HTMLExtractorExtra(self.md) + parser.feed(source) + parser.close() + return ''.join(parser.cleandoc).split('\n') + + +class MarkdownInHtmlProcessor(BlockProcessor): + """Process Markdown Inside HTML Blocks which have been stored in the HtmlStash.""" + + def test(self, parent, block): + # ALways return True. `run` will return `False` it not a valid match. + return True + + def parse_element_content(self, element): + """ + Recursively parse the text content of an etree Element as Markdown. + + Any block level elements generated from the Markdown will be inserted as children of the element in place + of the text content. All `markdown` attributes are removed. For any elements in which Markdown parsing has + been disabled, the text content of it and its chidlren are wrapped in an `AtomicString`. + """ + + md_attr = element.attrib.pop('markdown', 'off') + + if md_attr == 'block': + # Parse content as block level + # The order in which the different parts are parsed (text, children, tails) is important here as the + # order of elements needs to be preserved. We can't be inserting items at a later point in the current + # iteration as we don't want to do raw processing on elements created from parsing Markdown text (for + # example). Therefore, the order of operations is children, tails, text. + + # Recursively parse existing children from raw HTML + for child in list(element): + self.parse_element_content(child) + + # Parse Markdown text in tail of children. Do this separate to avoid raw HTML parsing. + # Save the position of each item to be inserted later in reverse. + tails = [] + for pos, child in enumerate(element): + if child.tail: + block = child.tail.rstrip('\n') + child.tail = '' + # Use a dummy placeholder element. + dummy = etree.Element('div') + self.parser.parseBlocks(dummy, block.split('\n\n')) + children = list(dummy) + children.reverse() + tails.append((pos + 1, children)) + + # Insert the elements created from the tails in reverse. + tails.reverse() + for pos, tail in tails: + for item in tail: + element.insert(pos, item) + + # Parse Markdown text content. Do this last to avoid raw HTML parsing. + if element.text: + block = element.text.rstrip('\n') + element.text = '' + # Use a dummy placeholder element as the content needs to get inserted before existing children. + dummy = etree.Element('div') + self.parser.parseBlocks(dummy, block.split('\n\n')) + children = list(dummy) + children.reverse() + for child in children: + element.insert(0, child) + + elif md_attr == 'span': + # Span level parsing will be handled by inlineprocessors. + # Walk children here to remove any `markdown` attributes. + for child in list(element): + self.parse_element_content(child) + + else: + # Disable inline parsing for everything else + if element.text is None: + element.text = '' + element.text = util.AtomicString(element.text) + for child in list(element): + self.parse_element_content(child) + if child.tail: + child.tail = util.AtomicString(child.tail) + + def run(self, parent, blocks): + m = util.HTML_PLACEHOLDER_RE.match(blocks[0]) + if m: + index = int(m.group(1)) + element = self.parser.md.htmlStash.rawHtmlBlocks[index] + if isinstance(element, etree.Element): + # We have a matched element. Process it. + blocks.pop(0) + self.parse_element_content(element) + parent.append(element) + # Cleanup stash. Replace element with empty string to avoid confusing postprocessor. + self.parser.md.htmlStash.rawHtmlBlocks.pop(index) + self.parser.md.htmlStash.rawHtmlBlocks.insert(index, '') + # Confirm the match to the blockparser. + return True + # No match found. + return False + + +class MarkdownInHTMLPostprocessor(RawHtmlPostprocessor): + def stash_to_string(self, text): + """ Override default to handle any etree elements still in the stash. """ + if isinstance(text, etree.Element): + return self.md.serializer(text) + else: + return str(text) + + +class MarkdownInHtmlExtension(Extension): + """Add Markdown parsing in HTML to Markdown class.""" + + def extendMarkdown(self, md): + """ Register extension instances. """ + + # Replace raw HTML preprocessor + md.preprocessors.register(HtmlBlockPreprocessor(md), 'html_block', 20) + # Add blockprocessor which handles the placeholders for etree elements + md.parser.blockprocessors.register( + MarkdownInHtmlProcessor(md.parser), 'markdown_block', 105 + ) + # Replace raw HTML postprocessor + md.postprocessors.register(MarkdownInHTMLPostprocessor(md), 'raw_html', 30) + + +def makeExtension(**kwargs): # pragma: no cover + return MarkdownInHtmlExtension(**kwargs) diff --git a/markdown/extensions/meta.py b/markdown/extensions/meta.py index 1b555b2..10dee11 100644 --- a/markdown/extensions/meta.py +++ b/markdown/extensions/meta.py @@ -1,75 +1,68 @@ -#!usr/bin/python - """ Meta Data Extension for Python-Markdown ======================================= This extension adds Meta Data handling to markdown. -Basic Usage: - - >>> import markdown - >>> text = '''Title: A Test Doc. - ... Author: Waylan Limberg - ... John Doe - ... Blank_Data: - ... - ... The body. This is paragraph one. - ... ''' - >>> md = markdown.Markdown(['meta']) - >>> md.convert(text) - u'<p>The body. This is paragraph one.</p>' - >>> md.Meta - {u'blank_data': [u''], u'author': [u'Waylan Limberg', u'John Doe'], u'title': [u'A Test Doc.']} - -Make sure text without Meta Data still works (markdown < 1.6b returns a <p>). +See <https://Python-Markdown.github.io/extensions/meta_data> +for documentation. - >>> text = ' Some Code - not extra lines of meta data.' - >>> md = markdown.Markdown(['meta']) - >>> md.convert(text) - u'<pre><code>Some Code - not extra lines of meta data.\\n</code></pre>' - >>> md.Meta - {} +Original code Copyright 2007-2008 [Waylan Limberg](http://achinghead.com). -Copyright 2007-2008 [Waylan Limberg](http://achinghead.com). +All changes Copyright 2008-2014 The Python Markdown Project -Project website: <http://www.freewisdom.org/project/python-markdown/Meta-Data> -Contact: markdown@freewisdom.org - -License: BSD (see ../docs/LICENSE for details) +License: [BSD](https://opensource.org/licenses/bsd-license.php) """ -import markdown, re +from . import Extension +from ..preprocessors import Preprocessor +import re +import logging + +log = logging.getLogger('MARKDOWN') # Global Vars META_RE = re.compile(r'^[ ]{0,3}(?P<key>[A-Za-z0-9_-]+):\s*(?P<value>.*)') META_MORE_RE = re.compile(r'^[ ]{4,}(?P<value>.*)') +BEGIN_RE = re.compile(r'^-{3}(\s.*)?') +END_RE = re.compile(r'^(-{3}|\.{3})(\s.*)?') + -class MetaExtension (markdown.Extension): +class MetaExtension (Extension): """ Meta-Data extension for Python-Markdown. """ - def extendMarkdown(self, md, md_globals): + def extendMarkdown(self, md): """ Add MetaPreprocessor to Markdown instance. """ + md.registerExtension(self) + self.md = md + md.preprocessors.register(MetaPreprocessor(md), 'meta', 27) - md.preprocessors.add("meta", MetaPreprocessor(md), "_begin") + def reset(self): + self.md.Meta = {} -class MetaPreprocessor(markdown.preprocessors.Preprocessor): +class MetaPreprocessor(Preprocessor): """ Get Meta-Data. """ def run(self, lines): """ Parse Meta-Data and store in Markdown.Meta. """ meta = {} key = None - while 1: + if lines and BEGIN_RE.match(lines[0]): + lines.pop(0) + while lines: line = lines.pop(0) - if line.strip() == '': - break # blank line - done m1 = META_RE.match(line) + if line.strip() == '' or END_RE.match(line): + break # blank line or end of YAML header - done if m1: key = m1.group('key').lower().strip() - meta[key] = [m1.group('value').strip()] + value = m1.group('value').strip() + try: + meta[key].append(value) + except KeyError: + meta[key] = [value] else: m2 = META_MORE_RE.match(line) if m2 and key: @@ -77,14 +70,10 @@ class MetaPreprocessor(markdown.preprocessors.Preprocessor): meta[key].append(m2.group('value').strip()) else: lines.insert(0, line) - break # no meta data - done - self.markdown.Meta = meta + break # no meta data - done + self.md.Meta = meta return lines - -def makeExtension(configs={}): - return MetaExtension(configs=configs) -if __name__ == "__main__": - import doctest - doctest.testmod() +def makeExtension(**kwargs): # pragma: no cover + return MetaExtension(**kwargs) diff --git a/markdown/extensions/nl2br.py b/markdown/extensions/nl2br.py new file mode 100644 index 0000000..6c7491b --- /dev/null +++ b/markdown/extensions/nl2br.py @@ -0,0 +1,33 @@ +""" +NL2BR Extension +=============== + +A Python-Markdown extension to treat newlines as hard breaks; like +GitHub-flavored Markdown does. + +See <https://Python-Markdown.github.io/extensions/nl2br> +for documentation. + +Oringinal code Copyright 2011 [Brian Neal](https://deathofagremmie.com/) + +All changes Copyright 2011-2014 The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) + +""" + +from . import Extension +from ..inlinepatterns import SubstituteTagInlineProcessor + +BR_RE = r'\n' + + +class Nl2BrExtension(Extension): + + def extendMarkdown(self, md): + br_tag = SubstituteTagInlineProcessor(BR_RE, 'br') + md.inlinePatterns.register(br_tag, 'nl', 5) + + +def makeExtension(**kwargs): # pragma: no cover + return Nl2BrExtension(**kwargs) diff --git a/markdown/extensions/rss.py b/markdown/extensions/rss.py deleted file mode 100644 index 1274da2..0000000 --- a/markdown/extensions/rss.py +++ /dev/null @@ -1,114 +0,0 @@ -import markdown -from markdown import etree - -DEFAULT_URL = "http://www.freewisdom.org/projects/python-markdown/" -DEFAULT_CREATOR = "Yuri Takhteyev" -DEFAULT_TITLE = "Markdown in Python" -GENERATOR = "http://www.freewisdom.org/projects/python-markdown/markdown2rss" - -month_map = { "Jan" : "01", - "Feb" : "02", - "March" : "03", - "April" : "04", - "May" : "05", - "June" : "06", - "July" : "07", - "August" : "08", - "September" : "09", - "October" : "10", - "November" : "11", - "December" : "12" } - -def get_time(heading): - - heading = heading.split("-")[0] - heading = heading.strip().replace(",", " ").replace(".", " ") - - month, date, year = heading.split() - month = month_map[month] - - return rdftime(" ".join((month, date, year, "12:00:00 AM"))) - -def rdftime(time): - - time = time.replace(":", " ") - time = time.replace("/", " ") - time = time.split() - return "%s-%s-%sT%s:%s:%s-08:00" % (time[0], time[1], time[2], - time[3], time[4], time[5]) - - -def get_date(text): - return "date" - -class RssExtension (markdown.Extension): - - def extendMarkdown(self, md, md_globals): - - self.config = { 'URL' : [DEFAULT_URL, "Main URL"], - 'CREATOR' : [DEFAULT_CREATOR, "Feed creator's name"], - 'TITLE' : [DEFAULT_TITLE, "Feed title"] } - - md.xml_mode = True - - # Insert a tree-processor that would actually add the title tag - treeprocessor = RssTreeProcessor(md) - treeprocessor.ext = self - md.treeprocessors['rss'] = treeprocessor - md.stripTopLevelTags = 0 - md.docType = '<?xml version="1.0" encoding="utf-8"?>\n' - -class RssTreeProcessor(markdown.treeprocessors.Treeprocessor): - - def run (self, root): - - rss = etree.Element("rss") - rss.set("version", "2.0") - - channel = etree.SubElement(rss, "channel") - - for tag, text in (("title", self.ext.getConfig("TITLE")), - ("link", self.ext.getConfig("URL")), - ("description", None)): - - element = etree.SubElement(channel, tag) - element.text = text - - for child in root: - - if child.tag in ["h1", "h2", "h3", "h4", "h5"]: - - heading = child.text.strip() - item = etree.SubElement(channel, "item") - link = etree.SubElement(item, "link") - link.text = self.ext.getConfig("URL") - title = etree.SubElement(item, "title") - title.text = heading - - guid = ''.join([x for x in heading if x.isalnum()]) - guidElem = etree.SubElement(item, "guid") - guidElem.text = guid - guidElem.set("isPermaLink", "false") - - elif child.tag in ["p"]: - try: - description = etree.SubElement(item, "description") - except UnboundLocalError: - # Item not defined - moving on - pass - else: - if len(child): - content = "\n".join([etree.tostring(node) - for node in child]) - else: - content = child.text - pholder = self.markdown.htmlStash.store( - "<![CDATA[ %s]]>" % content) - description.text = pholder - - return rss - - -def makeExtension(configs): - - return RssExtension(configs) diff --git a/markdown/extensions/sane_lists.py b/markdown/extensions/sane_lists.py new file mode 100644 index 0000000..e27eb18 --- /dev/null +++ b/markdown/extensions/sane_lists.py @@ -0,0 +1,54 @@ +""" +Sane List Extension for Python-Markdown +======================================= + +Modify the behavior of Lists in Python-Markdown to act in a sane manor. + +See <https://Python-Markdown.github.io/extensions/sane_lists> +for documentation. + +Original code Copyright 2011 [Waylan Limberg](http://achinghead.com) + +All changes Copyright 2011-2014 The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) + +""" + +from . import Extension +from ..blockprocessors import OListProcessor, UListProcessor +import re + + +class SaneOListProcessor(OListProcessor): + + SIBLING_TAGS = ['ol'] + LAZY_OL = False + + def __init__(self, parser): + super().__init__(parser) + self.CHILD_RE = re.compile(r'^[ ]{0,%d}((\d+\.))[ ]+(.*)' % + (self.tab_length - 1)) + + +class SaneUListProcessor(UListProcessor): + + SIBLING_TAGS = ['ul'] + + def __init__(self, parser): + super().__init__(parser) + self.CHILD_RE = re.compile(r'^[ ]{0,%d}(([*+-]))[ ]+(.*)' % + (self.tab_length - 1)) + + +class SaneListExtension(Extension): + """ Add sane lists to Markdown. """ + + def extendMarkdown(self, md): + """ Override existing Processors. """ + md.parser.blockprocessors.register(SaneOListProcessor(md.parser), 'olist', 40) + md.parser.blockprocessors.register(SaneUListProcessor(md.parser), 'ulist', 30) + + +def makeExtension(**kwargs): # pragma: no cover + return SaneListExtension(**kwargs) diff --git a/markdown/extensions/smarty.py b/markdown/extensions/smarty.py new file mode 100644 index 0000000..c4bfd58 --- /dev/null +++ b/markdown/extensions/smarty.py @@ -0,0 +1,257 @@ +''' +Smarty extension for Python-Markdown +==================================== + +Adds conversion of ASCII dashes, quotes and ellipses to their HTML +entity equivalents. + +See <https://Python-Markdown.github.io/extensions/smarty> +for documentation. + +Author: 2013, Dmitry Shachnev <mitya57@gmail.com> + +All changes Copyright 2013-2014 The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) + +SmartyPants license: + + Copyright (c) 2003 John Gruber <https://daringfireball.net/> + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name "SmartyPants" nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + + This software is provided by the copyright holders and contributors "as + is" and any express or implied warranties, including, but not limited + to, the implied warranties of merchantability and fitness for a + particular purpose are disclaimed. In no event shall the copyright + owner or contributors be liable for any direct, indirect, incidental, + special, exemplary, or consequential damages (including, but not + limited to, procurement of substitute goods or services; loss of use, + data, or profits; or business interruption) however caused and on any + theory of liability, whether in contract, strict liability, or tort + (including negligence or otherwise) arising in any way out of the use + of this software, even if advised of the possibility of such damage. + + +smartypants.py license: + + smartypants.py is a derivative work of SmartyPants. + Copyright (c) 2004, 2007 Chad Miller <http://web.chad.org/> + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + This software is provided by the copyright holders and contributors "as + is" and any express or implied warranties, including, but not limited + to, the implied warranties of merchantability and fitness for a + particular purpose are disclaimed. In no event shall the copyright + owner or contributors be liable for any direct, indirect, incidental, + special, exemplary, or consequential damages (including, but not + limited to, procurement of substitute goods or services; loss of use, + data, or profits; or business interruption) however caused and on any + theory of liability, whether in contract, strict liability, or tort + (including negligence or otherwise) arising in any way out of the use + of this software, even if advised of the possibility of such damage. + +''' + + +from . import Extension +from ..inlinepatterns import HtmlInlineProcessor, HTML_RE +from ..treeprocessors import InlineProcessor +from ..util import Registry + + +# Constants for quote education. +punctClass = r"""[!"#\$\%'()*+,-.\/:;<=>?\@\[\\\]\^_`{|}~]""" +endOfWordClass = r"[\s.,;:!?)]" +closeClass = r"[^\ \t\r\n\[\{\(\-\u0002\u0003]" + +openingQuotesBase = ( + r'(\s' # a whitespace char + r'| ' # or a non-breaking space entity + r'|--' # or dashes + r'|–|—' # or unicode + r'|&[mn]dash;' # or named dash entities + r'|–|—' # or decimal entities + r')' +) + +substitutions = { + 'mdash': '—', + 'ndash': '–', + 'ellipsis': '…', + 'left-angle-quote': '«', + 'right-angle-quote': '»', + 'left-single-quote': '‘', + 'right-single-quote': '’', + 'left-double-quote': '“', + 'right-double-quote': '”', +} + + +# Special case if the very first character is a quote +# followed by punctuation at a non-word-break. Close the quotes by brute force: +singleQuoteStartRe = r"^'(?=%s\B)" % punctClass +doubleQuoteStartRe = r'^"(?=%s\B)' % punctClass + +# Special case for double sets of quotes, e.g.: +# <p>He said, "'Quoted' words in a larger quote."</p> +doubleQuoteSetsRe = r""""'(?=\w)""" +singleQuoteSetsRe = r"""'"(?=\w)""" + +# Special case for decade abbreviations (the '80s): +decadeAbbrRe = r"(?<!\w)'(?=\d{2}s)" + +# Get most opening double quotes: +openingDoubleQuotesRegex = r'%s"(?=\w)' % openingQuotesBase + +# Double closing quotes: +closingDoubleQuotesRegex = r'"(?=\s)' +closingDoubleQuotesRegex2 = '(?<=%s)"' % closeClass + +# Get most opening single quotes: +openingSingleQuotesRegex = r"%s'(?=\w)" % openingQuotesBase + +# Single closing quotes: +closingSingleQuotesRegex = r"(?<=%s)'(?!\s|s\b|\d)" % closeClass +closingSingleQuotesRegex2 = r"(?<=%s)'(\s|s\b)" % closeClass + +# All remaining quotes should be opening ones +remainingSingleQuotesRegex = r"'" +remainingDoubleQuotesRegex = r'"' + +HTML_STRICT_RE = HTML_RE + r'(?!\>)' + + +class SubstituteTextPattern(HtmlInlineProcessor): + def __init__(self, pattern, replace, md): + """ Replaces matches with some text. """ + HtmlInlineProcessor.__init__(self, pattern) + self.replace = replace + self.md = md + + def handleMatch(self, m, data): + result = '' + for part in self.replace: + if isinstance(part, int): + result += m.group(part) + else: + result += self.md.htmlStash.store(part) + return result, m.start(0), m.end(0) + + +class SmartyExtension(Extension): + def __init__(self, **kwargs): + self.config = { + 'smart_quotes': [True, 'Educate quotes'], + 'smart_angled_quotes': [False, 'Educate angled quotes'], + 'smart_dashes': [True, 'Educate dashes'], + 'smart_ellipses': [True, 'Educate ellipses'], + 'substitutions': [{}, 'Overwrite default substitutions'], + } + super().__init__(**kwargs) + self.substitutions = dict(substitutions) + self.substitutions.update(self.getConfig('substitutions', default={})) + + def _addPatterns(self, md, patterns, serie, priority): + for ind, pattern in enumerate(patterns): + pattern += (md,) + pattern = SubstituteTextPattern(*pattern) + name = 'smarty-%s-%d' % (serie, ind) + self.inlinePatterns.register(pattern, name, priority-ind) + + def educateDashes(self, md): + emDashesPattern = SubstituteTextPattern( + r'(?<!-)---(?!-)', (self.substitutions['mdash'],), md + ) + enDashesPattern = SubstituteTextPattern( + r'(?<!-)--(?!-)', (self.substitutions['ndash'],), md + ) + self.inlinePatterns.register(emDashesPattern, 'smarty-em-dashes', 50) + self.inlinePatterns.register(enDashesPattern, 'smarty-en-dashes', 45) + + def educateEllipses(self, md): + ellipsesPattern = SubstituteTextPattern( + r'(?<!\.)\.{3}(?!\.)', (self.substitutions['ellipsis'],), md + ) + self.inlinePatterns.register(ellipsesPattern, 'smarty-ellipses', 10) + + def educateAngledQuotes(self, md): + leftAngledQuotePattern = SubstituteTextPattern( + r'\<\<', (self.substitutions['left-angle-quote'],), md + ) + rightAngledQuotePattern = SubstituteTextPattern( + r'\>\>', (self.substitutions['right-angle-quote'],), md + ) + self.inlinePatterns.register(leftAngledQuotePattern, 'smarty-left-angle-quotes', 40) + self.inlinePatterns.register(rightAngledQuotePattern, 'smarty-right-angle-quotes', 35) + + def educateQuotes(self, md): + lsquo = self.substitutions['left-single-quote'] + rsquo = self.substitutions['right-single-quote'] + ldquo = self.substitutions['left-double-quote'] + rdquo = self.substitutions['right-double-quote'] + patterns = ( + (singleQuoteStartRe, (rsquo,)), + (doubleQuoteStartRe, (rdquo,)), + (doubleQuoteSetsRe, (ldquo + lsquo,)), + (singleQuoteSetsRe, (lsquo + ldquo,)), + (decadeAbbrRe, (rsquo,)), + (openingSingleQuotesRegex, (1, lsquo)), + (closingSingleQuotesRegex, (rsquo,)), + (closingSingleQuotesRegex2, (rsquo, 1)), + (remainingSingleQuotesRegex, (lsquo,)), + (openingDoubleQuotesRegex, (1, ldquo)), + (closingDoubleQuotesRegex, (rdquo,)), + (closingDoubleQuotesRegex2, (rdquo,)), + (remainingDoubleQuotesRegex, (ldquo,)) + ) + self._addPatterns(md, patterns, 'quotes', 30) + + def extendMarkdown(self, md): + configs = self.getConfigs() + self.inlinePatterns = Registry() + if configs['smart_ellipses']: + self.educateEllipses(md) + if configs['smart_quotes']: + self.educateQuotes(md) + if configs['smart_angled_quotes']: + self.educateAngledQuotes(md) + # Override HTML_RE from inlinepatterns.py so that it does not + # process tags with duplicate closing quotes. + md.inlinePatterns.register(HtmlInlineProcessor(HTML_STRICT_RE, md), 'html', 90) + if configs['smart_dashes']: + self.educateDashes(md) + inlineProcessor = InlineProcessor(md) + inlineProcessor.inlinePatterns = self.inlinePatterns + md.treeprocessors.register(inlineProcessor, 'smarty', 2) + md.ESCAPED_CHARS.extend(['"', "'"]) + + +def makeExtension(**kwargs): # pragma: no cover + return SmartyExtension(**kwargs) diff --git a/markdown/extensions/tables.py b/markdown/extensions/tables.py index 1d3c920..c8b1024 100644 --- a/markdown/extensions/tables.py +++ b/markdown/extensions/tables.py @@ -1,44 +1,88 @@ -#!/usr/bin/env Python """ Tables Extension for Python-Markdown ==================================== Added parsing of tables to Python-Markdown. -A simple example: +See <https://Python-Markdown.github.io/extensions/tables> +for documentation. - First Header | Second Header - ------------- | ------------- - Content Cell | Content Cell - Content Cell | Content Cell +Original code Copyright 2009 [Waylan Limberg](http://achinghead.com) + +All changes Copyright 2008-2014 The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) -Copyright 2009 - [Waylan Limberg](http://achinghead.com) """ -import markdown -from markdown import etree + +from . import Extension +from ..blockprocessors import BlockProcessor +import xml.etree.ElementTree as etree +import re +PIPE_NONE = 0 +PIPE_LEFT = 1 +PIPE_RIGHT = 2 -class TableProcessor(markdown.blockprocessors.BlockProcessor): +class TableProcessor(BlockProcessor): """ Process Tables. """ + RE_CODE_PIPES = re.compile(r'(?:(\\\\)|(\\`+)|(`+)|(\\\|)|(\|))') + RE_END_BORDER = re.compile(r'(?<!\\)(?:\\\\)*\|$') + + def __init__(self, parser, config): + self.border = False + self.separator = '' + self.config = config + + super().__init__(parser) + def test(self, parent, block): - rows = block.split('\n') - return (len(rows) > 2 and '|' in rows[0] and - '|' in rows[1] and '-' in rows[1] and - rows[1][0] in ['|', ':', '-']) + """ + Ensure first two rows (column header and separator row) are valid table rows. + + Keep border check and separator row do avoid repeating the work. + """ + is_table = False + rows = [row.strip(' ') for row in block.split('\n')] + if len(rows) > 1: + header0 = rows[0] + self.border = PIPE_NONE + if header0.startswith('|'): + self.border |= PIPE_LEFT + if self.RE_END_BORDER.search(header0) is not None: + self.border |= PIPE_RIGHT + row = self._split_row(header0) + row0_len = len(row) + is_table = row0_len > 1 + + # Each row in a single column table needs at least one pipe. + if not is_table and row0_len == 1 and self.border: + for index in range(1, len(rows)): + is_table = rows[index].startswith('|') + if not is_table: + is_table = self.RE_END_BORDER.search(rows[index]) is not None + if not is_table: + break + + if is_table: + row = self._split_row(rows[1]) + is_table = (len(row) == row0_len) and set(''.join(row)) <= set('|:- ') + if is_table: + self.separator = row + + return is_table def run(self, parent, blocks): """ Parse a table block and build table. """ block = blocks.pop(0).split('\n') - header = block[:2] - rows = block[2:] - # Get format type (bordered by pipes or not) - border = False - if header[0].startswith('|'): - border = True + header = block[0].strip(' ') + rows = [] if len(block) < 3 else block[2:] + # Get alignment of columns align = [] - for c in self._split_row(header[1], border): + for c in self.separator: + c = c.strip(' ') if c.startswith(':') and c.endswith(':'): align.append('center') elif c.startswith(':'): @@ -47,51 +91,146 @@ class TableProcessor(markdown.blockprocessors.BlockProcessor): align.append('right') else: align.append(None) + # Build table table = etree.SubElement(parent, 'table') thead = etree.SubElement(table, 'thead') - self._build_row(header[0], thead, align, border) + self._build_row(header, thead, align) tbody = etree.SubElement(table, 'tbody') - for row in rows: - self._build_row(row, tbody, align, border) + if len(rows) == 0: + # Handle empty table + self._build_empty_row(tbody, align) + else: + for row in rows: + self._build_row(row.strip(' '), tbody, align) + + def _build_empty_row(self, parent, align): + """Build an empty row.""" + tr = etree.SubElement(parent, 'tr') + count = len(align) + while count: + etree.SubElement(tr, 'td') + count -= 1 - def _build_row(self, row, parent, align, border): + def _build_row(self, row, parent, align): """ Given a row of text, build table cells. """ tr = etree.SubElement(parent, 'tr') tag = 'td' if parent.tag == 'thead': tag = 'th' - cells = self._split_row(row, border) - # We use align here rather than cells to ensure every row + cells = self._split_row(row) + # We use align here rather than cells to ensure every row # contains the same number of columns. for i, a in enumerate(align): c = etree.SubElement(tr, tag) try: - c.text = cells[i].strip() - except IndexError: + c.text = cells[i].strip(' ') + except IndexError: # pragma: no cover c.text = "" if a: - c.set('align', a) + if self.config['use_align_attribute']: + c.set('align', a) + else: + c.set('style', f'text-align: {a};') - def _split_row(self, row, border): + def _split_row(self, row): """ split a row of text into list of cells. """ - if border: + if self.border: if row.startswith('|'): row = row[1:] - if row.endswith('|'): - row = row[:-1] - return row.split('|') + row = self.RE_END_BORDER.sub('', row) + return self._split(row) + def _split(self, row): + """ split a row of text with some code into a list of cells. """ + elements = [] + pipes = [] + tics = [] + tic_points = [] + tic_region = [] + good_pipes = [] + + # Parse row + # Throw out \\, and \| + for m in self.RE_CODE_PIPES.finditer(row): + # Store ` data (len, start_pos, end_pos) + if m.group(2): + # \`+ + # Store length of each tic group: subtract \ + tics.append(len(m.group(2)) - 1) + # Store start of group, end of group, and escape length + tic_points.append((m.start(2), m.end(2) - 1, 1)) + elif m.group(3): + # `+ + # Store length of each tic group + tics.append(len(m.group(3))) + # Store start of group, end of group, and escape length + tic_points.append((m.start(3), m.end(3) - 1, 0)) + # Store pipe location + elif m.group(5): + pipes.append(m.start(5)) + + # Pair up tics according to size if possible + # Subtract the escape length *only* from the opening. + # Walk through tic list and see if tic has a close. + # Store the tic region (start of region, end of region). + pos = 0 + tic_len = len(tics) + while pos < tic_len: + try: + tic_size = tics[pos] - tic_points[pos][2] + if tic_size == 0: + raise ValueError + index = tics[pos + 1:].index(tic_size) + 1 + tic_region.append((tic_points[pos][0], tic_points[pos + index][1])) + pos += index + 1 + except ValueError: + pos += 1 -class TableExtension(markdown.Extension): + # Resolve pipes. Check if they are within a tic pair region. + # Walk through pipes comparing them to each region. + # - If pipe position is less that a region, it isn't in a region + # - If it is within a region, we don't want it, so throw it out + # - If we didn't throw it out, it must be a table pipe + for pipe in pipes: + throw_out = False + for region in tic_region: + if pipe < region[0]: + # Pipe is not in a region + break + elif region[0] <= pipe <= region[1]: + # Pipe is within a code region. Throw it out. + throw_out = True + break + if not throw_out: + good_pipes.append(pipe) + + # Split row according to table delimiters. + pos = 0 + for pipe in good_pipes: + elements.append(row[pos:pipe]) + pos = pipe + 1 + elements.append(row[pos:]) + return elements + + +class TableExtension(Extension): """ Add tables to Markdown. """ - def extendMarkdown(self, md, md_globals): + def __init__(self, **kwargs): + self.config = { + 'use_align_attribute': [False, 'True to use align attribute instead of style.'], + } + + super().__init__(**kwargs) + + def extendMarkdown(self, md): """ Add an instance of TableProcessor to BlockParser. """ - md.parser.blockprocessors.add('table', - TableProcessor(md.parser), - '<hashheader') + if '|' not in md.ESCAPED_CHARS: + md.ESCAPED_CHARS.append('|') + processor = TableProcessor(md.parser, self.getConfigs()) + md.parser.blockprocessors.register(processor, 'table', 75) -def makeExtension(configs={}): - return TableExtension(configs=configs) +def makeExtension(**kwargs): # pragma: no cover + return TableExtension(**kwargs) diff --git a/markdown/extensions/toc.py b/markdown/extensions/toc.py index 1d9489c..1ded18d 100644 --- a/markdown/extensions/toc.py +++ b/markdown/extensions/toc.py @@ -1,136 +1,384 @@ """ Table of Contents Extension for Python-Markdown -* * * +=============================================== -(c) 2008 [Jack Miller](http://codezen.org) +See <https://Python-Markdown.github.io/extensions/toc> +for documentation. -Dependencies: -* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) +Oringinal code Copyright 2008 [Jack Miller](https://codezen.org/) + +All changes Copyright 2008-2014 The Python Markdown Project + +License: [BSD](https://opensource.org/licenses/bsd-license.php) """ -import markdown -from markdown import etree + +from . import Extension +from ..treeprocessors import Treeprocessor +from ..util import code_escape, parseBoolValue, AMP_SUBSTITUTE, HTML_PLACEHOLDER_RE, AtomicString +from ..treeprocessors import UnescapeTreeprocessor import re +import html +import unicodedata +import xml.etree.ElementTree as etree -class TocTreeprocessor(markdown.treeprocessors.Treeprocessor): - # Iterator wrapper to get parent and child all at once - def iterparent(self, root): - for parent in root.getiterator(): - for child in parent: - yield parent, child - def run(self, doc): - div = etree.Element("div") - div.attrib["class"] = "toc" - last_li = None +def slugify(value, separator, unicode=False): + """ Slugify a string, to make it URL friendly. """ + if not unicode: + # Replace Extended Latin characters with ASCII, i.e. žlutý → zluty + value = unicodedata.normalize('NFKD', value) + value = value.encode('ascii', 'ignore').decode('ascii') + value = re.sub(r'[^\w\s-]', '', value).strip().lower() + return re.sub(r'[{}\s]+'.format(separator), separator, value) - # Add title to the div - if self.config["title"][0]: - header = etree.SubElement(div, "span") - header.attrib["class"] = "toctitle" - header.text = self.config["title"][0] - level = 0 - list_stack=[div] - header_rgx = re.compile("[Hh][123456]") +def slugify_unicode(value, separator): + """ Slugify a string, to make it URL friendly while preserving Unicode characters. """ + return slugify(value, separator, unicode=True) - # Get a list of id attributes - used_ids = [] - for c in doc.getiterator(): - if "id" in c.attrib: - used_ids.append(c.attrib["id"]) - for (p, c) in self.iterparent(doc): - if not c.text: +IDCOUNT_RE = re.compile(r'^(.*)_([0-9]+)$') + + +def unique(id, ids): + """ Ensure id is unique in set of ids. Append '_1', '_2'... if not """ + while id in ids or not id: + m = IDCOUNT_RE.match(id) + if m: + id = '%s_%d' % (m.group(1), int(m.group(2))+1) + else: + id = '%s_%d' % (id, 1) + ids.add(id) + return id + + +def get_name(el): + """Get title name.""" + + text = [] + for c in el.itertext(): + if isinstance(c, AtomicString): + text.append(html.unescape(c)) + else: + text.append(c) + return ''.join(text).strip() + + +def stashedHTML2text(text, md, strip_entities=True): + """ Extract raw HTML from stash, reduce to plain text and swap with placeholder. """ + def _html_sub(m): + """ Substitute raw html with plain text. """ + try: + raw = md.htmlStash.rawHtmlBlocks[int(m.group(1))] + except (IndexError, TypeError): # pragma: no cover + return m.group(0) + # Strip out tags and/or entities - leaving text + res = re.sub(r'(<[^>]+>)', '', raw) + if strip_entities: + res = re.sub(r'(&[\#a-zA-Z0-9]+;)', '', res) + return res + + return HTML_PLACEHOLDER_RE.sub(_html_sub, text) + + +def unescape(text): + """ Unescape escaped text. """ + c = UnescapeTreeprocessor() + return c.unescape(text) + + +def nest_toc_tokens(toc_list): + """Given an unsorted list with errors and skips, return a nested one. + [{'level': 1}, {'level': 2}] + => + [{'level': 1, 'children': [{'level': 2, 'children': []}]}] + + A wrong list is also converted: + [{'level': 2}, {'level': 1}] + => + [{'level': 2, 'children': []}, {'level': 1, 'children': []}] + """ + + ordered_list = [] + if len(toc_list): + # Initialize everything by processing the first entry + last = toc_list.pop(0) + last['children'] = [] + levels = [last['level']] + ordered_list.append(last) + parents = [] + + # Walk the rest nesting the entries properly + while toc_list: + t = toc_list.pop(0) + current_level = t['level'] + t['children'] = [] + + # Reduce depth if current level < last item's level + if current_level < levels[-1]: + # Pop last level since we know we are less than it + levels.pop() + + # Pop parents and levels we are less than or equal to + to_pop = 0 + for p in reversed(parents): + if current_level <= p['level']: + to_pop += 1 + else: # pragma: no cover + break + if to_pop: + levels = levels[:-to_pop] + parents = parents[:-to_pop] + + # Note current level as last + levels.append(current_level) + + # Level is the same, so append to + # the current parent (if available) + if current_level == levels[-1]: + (parents[-1]['children'] if parents + else ordered_list).append(t) + + # Current level is > last item's level, + # So make last item a parent and append current as child + else: + last['children'].append(t) + parents.append(last) + levels.append(current_level) + last = t + + return ordered_list + + +class TocTreeprocessor(Treeprocessor): + def __init__(self, md, config): + super().__init__(md) + + self.marker = config["marker"] + self.title = config["title"] + self.base_level = int(config["baselevel"]) - 1 + self.slugify = config["slugify"] + self.sep = config["separator"] + self.toc_class = config["toc_class"] + self.use_anchors = parseBoolValue(config["anchorlink"]) + self.anchorlink_class = config["anchorlink_class"] + self.use_permalinks = parseBoolValue(config["permalink"], False) + if self.use_permalinks is None: + self.use_permalinks = config["permalink"] + self.permalink_class = config["permalink_class"] + self.permalink_title = config["permalink_title"] + self.header_rgx = re.compile("[Hh][123456]") + if isinstance(config["toc_depth"], str) and '-' in config["toc_depth"]: + self.toc_top, self.toc_bottom = [int(x) for x in config["toc_depth"].split('-')] + else: + self.toc_top = 1 + self.toc_bottom = int(config["toc_depth"]) + + def iterparent(self, node): + ''' Iterator wrapper to get allowed parent and child all at once. ''' + + # We do not allow the marker inside a header as that + # would causes an enless loop of placing a new TOC + # inside previously generated TOC. + for child in node: + if not self.header_rgx.match(child.tag) and child.tag not in ['pre', 'code']: + yield node, child + yield from self.iterparent(child) + + def replace_marker(self, root, elem): + ''' Replace marker with elem. ''' + for (p, c) in self.iterparent(root): + text = ''.join(c.itertext()).strip() + if not text: continue # To keep the output from screwing up the # validation by putting a <div> inside of a <p> # we actually replace the <p> in its entirety. - # We do not allow the marker inside a header as that - # would causes an enless loop of placing a new TOC - # inside previously generated TOC. - if c.text.find(self.config["marker"][0]) > -1 and not header_rgx.match(c.tag): + # The <p> element may contain more than a single text content + # (nl2br can introduce a <br>). In this situation, c.text returns + # the very first content, ignore children contents or tail content. + # len(c) == 0 is here to ensure there is only text in the <p>. + if c.text and c.text.strip() == self.marker and len(c) == 0: for i in range(len(p)): if p[i] == c: - p[i] = div + p[i] = elem break - - if header_rgx.match(c.tag): - tag_level = int(c.tag[-1]) - - while tag_level < level: - list_stack.pop() - level -= 1 - - if tag_level > level: - newlist = etree.Element("ul") - if last_li: - last_li.append(newlist) - else: - list_stack[-1].append(newlist) - list_stack.append(newlist) - level += 1 - - # Do not override pre-existing ids - if not "id" in c.attrib: - id = self.config["slugify"][0](c.text) - if id in used_ids: - ctr = 1 - while "%s_%d" % (id, ctr) in used_ids: - ctr += 1 - id = "%s_%d" % (id, ctr) - used_ids.append(id) - c.attrib["id"] = id - else: - id = c.attrib["id"] + def set_level(self, elem): + ''' Adjust header level according to base level. ''' + level = int(elem.tag[-1]) + self.base_level + if level > 6: + level = 6 + elem.tag = 'h%d' % level + + def add_anchor(self, c, elem_id): # @ReservedAssignment + anchor = etree.Element("a") + anchor.text = c.text + anchor.attrib["href"] = "#" + elem_id + anchor.attrib["class"] = self.anchorlink_class + c.text = "" + for elem in c: + anchor.append(elem) + while len(c): + c.remove(c[0]) + c.append(anchor) + + def add_permalink(self, c, elem_id): + permalink = etree.Element("a") + permalink.text = ("%spara;" % AMP_SUBSTITUTE + if self.use_permalinks is True + else self.use_permalinks) + permalink.attrib["href"] = "#" + elem_id + permalink.attrib["class"] = self.permalink_class + if self.permalink_title: + permalink.attrib["title"] = self.permalink_title + c.append(permalink) + + def build_toc_div(self, toc_list): + """ Return a string div given a toc list. """ + div = etree.Element("div") + div.attrib["class"] = self.toc_class + + # Add title to the div + if self.title: + header = etree.SubElement(div, "span") + header.attrib["class"] = "toctitle" + header.text = self.title + + def build_etree_ul(toc_list, parent): + ul = etree.SubElement(parent, "ul") + for item in toc_list: # List item link, to be inserted into the toc div - last_li = etree.Element("li") - link = etree.SubElement(last_li, "a") - link.text = c.text - link.attrib["href"] = '#' + id - - if int(self.config["anchorlink"][0]): - anchor = etree.SubElement(c, "a") - anchor.text = c.text - anchor.attrib["href"] = "#" + id - anchor.attrib["class"] = "toclink" - c.text = "" - - list_stack[-1].append(last_li) - -class TocExtension(markdown.Extension): - def __init__(self, configs): - self.config = { "marker" : ["[TOC]", - "Text to find and replace with Table of Contents -" - "Defaults to \"[TOC]\""], - "slugify" : [self.slugify, - "Function to generate anchors based on header text-" - "Defaults to a built in slugify function."], - "title" : [None, - "Title to insert into TOC <div> - " - "Defaults to None"], - "anchorlink" : [0, - "1 if header should be a self link" - "Defaults to 0"]} - - for key, value in configs: - self.setConfig(key, value) - - # This is exactly the same as Django's slugify - def slugify(self, value): - """ Slugify a string, to make it URL friendly. """ - import unicodedata - value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') - value = unicode(re.sub('[^\w\s-]', '', value).strip().lower()) - return re.sub('[-\s]+','-',value) - - def extendMarkdown(self, md, md_globals): - tocext = TocTreeprocessor(md) - tocext.config = self.config - md.treeprocessors.add("toc", tocext, "_begin") - -def makeExtension(configs={}): - return TocExtension(configs=configs) + li = etree.SubElement(ul, "li") + link = etree.SubElement(li, "a") + link.text = item.get('name', '') + link.attrib["href"] = '#' + item.get('id', '') + if item['children']: + build_etree_ul(item['children'], li) + return ul + + build_etree_ul(toc_list, div) + + if 'prettify' in self.md.treeprocessors: + self.md.treeprocessors['prettify'].run(div) + + return div + + def run(self, doc): + # Get a list of id attributes + used_ids = set() + for el in doc.iter(): + if "id" in el.attrib: + used_ids.add(el.attrib["id"]) + + toc_tokens = [] + for el in doc.iter(): + if isinstance(el.tag, str) and self.header_rgx.match(el.tag): + self.set_level(el) + text = get_name(el) + + # Do not override pre-existing ids + if "id" not in el.attrib: + innertext = unescape(stashedHTML2text(text, self.md)) + el.attrib["id"] = unique(self.slugify(innertext, self.sep), used_ids) + + if int(el.tag[-1]) >= self.toc_top and int(el.tag[-1]) <= self.toc_bottom: + toc_tokens.append({ + 'level': int(el.tag[-1]), + 'id': el.attrib["id"], + 'name': stashedHTML2text( + code_escape(el.attrib.get('data-toc-label', text)), + self.md, strip_entities=False + ) + }) + + # Remove the data-toc-label attribute as it is no longer needed + if 'data-toc-label' in el.attrib: + del el.attrib['data-toc-label'] + + if self.use_anchors: + self.add_anchor(el, el.attrib["id"]) + if self.use_permalinks not in [False, None]: + self.add_permalink(el, el.attrib["id"]) + + toc_tokens = nest_toc_tokens(toc_tokens) + div = self.build_toc_div(toc_tokens) + if self.marker: + self.replace_marker(doc, div) + + # serialize and attach to markdown instance. + toc = self.md.serializer(div) + for pp in self.md.postprocessors: + toc = pp.run(toc) + self.md.toc_tokens = toc_tokens + self.md.toc = toc + + +class TocExtension(Extension): + + TreeProcessorClass = TocTreeprocessor + + def __init__(self, **kwargs): + self.config = { + "marker": ['[TOC]', + 'Text to find and replace with Table of Contents - ' + 'Set to an empty string to disable. Defaults to "[TOC]"'], + "title": ["", + "Title to insert into TOC <div> - " + "Defaults to an empty string"], + "toc_class": ['toc', + 'CSS class(es) used for the link. ' + 'Defaults to "toclink"'], + "anchorlink": [False, + "True if header should be a self link - " + "Defaults to False"], + "anchorlink_class": ['toclink', + 'CSS class(es) used for the link. ' + 'Defaults to "toclink"'], + "permalink": [0, + "True or link text if a Sphinx-style permalink should " + "be added - Defaults to False"], + "permalink_class": ['headerlink', + 'CSS class(es) used for the link. ' + 'Defaults to "headerlink"'], + "permalink_title": ["Permanent link", + "Title attribute of the permalink - " + "Defaults to 'Permanent link'"], + "baselevel": ['1', 'Base level for headers.'], + "slugify": [slugify, + "Function to generate anchors based on header text - " + "Defaults to the headerid ext's slugify function."], + 'separator': ['-', 'Word separator. Defaults to "-".'], + "toc_depth": [6, + 'Define the range of section levels to include in' + 'the Table of Contents. A single integer (b) defines' + 'the bottom section level (<h1>..<hb>) only.' + 'A string consisting of two digits separated by a hyphen' + 'in between ("2-5"), define the top (t) and the' + 'bottom (b) (<ht>..<hb>). Defaults to `6` (bottom).'], + } + + super().__init__(**kwargs) + + def extendMarkdown(self, md): + md.registerExtension(self) + self.md = md + self.reset() + tocext = self.TreeProcessorClass(md, self.getConfigs()) + # Headerid ext is set to '>prettify'. With this set to '_end', + # it should always come after headerid ext (and honor ids assigned + # by the header id extension) if both are used. Same goes for + # attr_list extension. This must come last because we don't want + # to redefine ids after toc is created. But we do want toc prettified. + md.treeprocessors.register(tocext, 'toc', 5) + + def reset(self): + self.md.toc = '' + self.md.toc_tokens = [] + + +def makeExtension(**kwargs): # pragma: no cover + return TocExtension(**kwargs) diff --git a/markdown/extensions/wikilinks.py b/markdown/extensions/wikilinks.py index df44e1c..cddee7a 100644 --- a/markdown/extensions/wikilinks.py +++ b/markdown/extensions/wikilinks.py @@ -1,155 +1,87 @@ -#!/usr/bin/env python - ''' WikiLinks Extension for Python-Markdown ====================================== -Converts [[WikiLinks]] to relative links. Requires Python-Markdown 2.0+ - -Basic usage: - - >>> import markdown - >>> text = "Some text with a [[WikiLink]]." - >>> html = markdown.markdown(text, ['wikilinks']) - >>> html - u'<p>Some text with a <a class="wikilink" href="/WikiLink/">WikiLink</a>.</p>' - -Whitespace behavior: - - >>> markdown.markdown('[[ foo bar_baz ]]', ['wikilinks']) - u'<p><a class="wikilink" href="/foo_bar_baz/">foo bar_baz</a></p>' - >>> markdown.markdown('foo [[ ]] bar', ['wikilinks']) - u'<p>foo bar</p>' - -To define custom settings the simple way: - - >>> markdown.markdown(text, - ... ['wikilinks(base_url=/wiki/,end_url=.html,html_class=foo)'] - ... ) - u'<p>Some text with a <a class="foo" href="/wiki/WikiLink.html">WikiLink</a>.</p>' - -Custom settings the complex way: - - >>> md = markdown.Markdown( - ... extensions = ['wikilinks'], - ... extension_configs = {'wikilinks': [ - ... ('base_url', 'http://example.com/'), - ... ('end_url', '.html'), - ... ('html_class', '') ]}, - ... safe_mode = True) - >>> md.convert(text) - u'<p>Some text with a <a href="http://example.com/WikiLink.html">WikiLink</a>.</p>' - -Use MetaData with mdx_meta.py (Note the blank html_class in MetaData): - - >>> text = """wiki_base_url: http://example.com/ - ... wiki_end_url: .html - ... wiki_html_class: - ... - ... Some text with a [[WikiLink]].""" - >>> md = markdown.Markdown(extensions=['meta', 'wikilinks']) - >>> md.convert(text) - u'<p>Some text with a <a href="http://example.com/WikiLink.html">WikiLink</a>.</p>' - -MetaData should not carry over to next document: +Converts [[WikiLinks]] to relative links. - >>> md.convert("No [[MetaData]] here.") - u'<p>No <a class="wikilink" href="/MetaData/">MetaData</a> here.</p>' +See <https://Python-Markdown.github.io/extensions/wikilinks> +for documentation. -Define a custom URL builder: +Original code Copyright [Waylan Limberg](http://achinghead.com/). - >>> def my_url_builder(label, base, end): - ... return '/bar/' - >>> md = markdown.Markdown(extensions=['wikilinks'], - ... extension_configs={'wikilinks' : [('build_url', my_url_builder)]}) - >>> md.convert('[[foo]]') - u'<p><a class="wikilink" href="/bar/">foo</a></p>' +All changes Copyright The Python Markdown Project -From the command line: +License: [BSD](https://opensource.org/licenses/bsd-license.php) - python markdown.py -x wikilinks(base_url=http://example.com/,end_url=.html,html_class=foo) src.txt - -By [Waylan Limberg](http://achinghead.com/). - -License: [BSD](http://www.opensource.org/licenses/bsd-license.php) - -Dependencies: -* [Python 2.3+](http://python.org) -* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) ''' -import markdown +from . import Extension +from ..inlinepatterns import InlineProcessor +import xml.etree.ElementTree as etree import re + def build_url(label, base, end): """ Build a url from the label, a base, and an end. """ clean_label = re.sub(r'([ ]+_)|(_[ ]+)|([ ]+)', '_', label) - return '%s%s%s'% (base, clean_label, end) + return '{}{}{}'.format(base, clean_label, end) + +class WikiLinkExtension(Extension): -class WikiLinkExtension(markdown.Extension): - def __init__(self, configs): - # set extension defaults + def __init__(self, **kwargs): self.config = { - 'base_url' : ['/', 'String to append to beginning or URL.'], - 'end_url' : ['/', 'String to append to end of URL.'], - 'html_class' : ['wikilink', 'CSS hook. Leave blank for none.'], - 'build_url' : [build_url, 'Callable formats URL from label.'], + 'base_url': ['/', 'String to append to beginning or URL.'], + 'end_url': ['/', 'String to append to end of URL.'], + 'html_class': ['wikilink', 'CSS hook. Leave blank for none.'], + 'build_url': [build_url, 'Callable formats URL from label.'], } - - # Override defaults with user settings - for key, value in configs : - self.setConfig(key, value) - - def extendMarkdown(self, md, md_globals): + + super().__init__(**kwargs) + + def extendMarkdown(self, md): self.md = md - + # append to end of inline patterns - WIKILINK_RE = r'\[\[([A-Za-z0-9_ -]+)\]\]' - wikilinkPattern = WikiLinks(WIKILINK_RE, self.config) + WIKILINK_RE = r'\[\[([\w0-9_ -]+)\]\]' + wikilinkPattern = WikiLinksInlineProcessor(WIKILINK_RE, self.getConfigs()) wikilinkPattern.md = md - md.inlinePatterns.add('wikilink', wikilinkPattern, "<not_strong") + md.inlinePatterns.register(wikilinkPattern, 'wikilink', 75) -class WikiLinks(markdown.inlinepatterns.Pattern): +class WikiLinksInlineProcessor(InlineProcessor): def __init__(self, pattern, config): - markdown.inlinepatterns.Pattern.__init__(self, pattern) + super().__init__(pattern) self.config = config - - def handleMatch(self, m): - if m.group(2).strip(): + + def handleMatch(self, m, data): + if m.group(1).strip(): base_url, end_url, html_class = self._getMeta() - label = m.group(2).strip() - url = self.config['build_url'][0](label, base_url, end_url) - a = markdown.etree.Element('a') - a.text = label + label = m.group(1).strip() + url = self.config['build_url'](label, base_url, end_url) + a = etree.Element('a') + a.text = label a.set('href', url) if html_class: a.set('class', html_class) else: a = '' - return a + return a, m.start(0), m.end(0) def _getMeta(self): """ Return meta data or config data. """ - base_url = self.config['base_url'][0] - end_url = self.config['end_url'][0] - html_class = self.config['html_class'][0] + base_url = self.config['base_url'] + end_url = self.config['end_url'] + html_class = self.config['html_class'] if hasattr(self.md, 'Meta'): - if self.md.Meta.has_key('wiki_base_url'): + if 'wiki_base_url' in self.md.Meta: base_url = self.md.Meta['wiki_base_url'][0] - if self.md.Meta.has_key('wiki_end_url'): + if 'wiki_end_url' in self.md.Meta: end_url = self.md.Meta['wiki_end_url'][0] - if self.md.Meta.has_key('wiki_html_class'): + if 'wiki_html_class' in self.md.Meta: html_class = self.md.Meta['wiki_html_class'][0] return base_url, end_url, html_class - - -def makeExtension(configs=None) : - return WikiLinkExtension(configs=configs) - -if __name__ == "__main__": - import doctest - doctest.testmod() +def makeExtension(**kwargs): # pragma: no cover + return WikiLinkExtension(**kwargs) |