Source code for myst_parser.parsers.docutils_

"""MyST Markdown parser for docutils."""
from dataclasses import Field
from typing import (

import yaml
from docutils import frontend, nodes
from docutils.core import default_description, publish_cmdline, publish_string
from docutils.frontend import filter_settings_spec
from docutils.parsers.rst import Parser as RstParser
from docutils.writers.html5_polyglot import HTMLTranslator, Writer

from myst_parser._compat import Literal, get_args, get_origin
from myst_parser.config.main import (
from myst_parser.mdit_to_docutils.base import DocutilsRenderer
from myst_parser.mdit_to_docutils.transforms import ResolveAnchorIds
from myst_parser.parsers.mdit import create_md_parser
from myst_parser.warnings_ import MystWarnings, create_warning

def _validate_int(
    setting, value, option_parser, config_parser=None, config_section=None
) -> int:
    """Validate an integer setting."""
    return int(value)

def _validate_comma_separated_set(
    setting, value, option_parser, config_parser=None, config_section=None
) -> Set[str]:
    """Validate an integer setting."""
    value = frontend.validate_comma_separated_list(
        setting, value, option_parser, config_parser, config_section
    return set(value)

def _create_validate_tuple(length: int) -> Callable[..., Tuple[str, ...]]:
    """Create a validator for a tuple of length `length`."""

    def _validate(
        setting, value, option_parser, config_parser=None, config_section=None
        string_list = frontend.validate_comma_separated_list(
            setting, value, option_parser, config_parser, config_section
        if len(string_list) != length:
            raise ValueError(
                f"Expecting {length} items in {setting}, got {len(string_list)}."
        return tuple(string_list)

    return _validate

[docs]class Unset: """A sentinel class for unset settings.""" def __repr__(self): return "UNSET" def __bool__(self): # this allows to check if the setting is unset/falsy return False
DOCUTILS_UNSET = Unset() """Sentinel for arguments not set through docutils.conf.""" def _create_validate_yaml(field: Field): """Create a deserializer/validator for a json setting.""" def _validate_yaml( setting, value, option_parser, config_parser=None, config_section=None ): """Check/normalize a key-value pair setting. Items delimited by `,`, and key-value pairs delimited by `=`. """ try: output = yaml.safe_load(value) except Exception: raise ValueError("Invalid YAML string") if not isinstance(output, dict): raise ValueError("Expecting a YAML dictionary") return output return _validate_yaml def _validate_url_schemes( setting, value, option_parser, config_parser=None, config_section=None ): """Validate a url_schemes setting. This is a tricky one, because it can be either a comma-separated list or a YAML dictionary. """ try: output = yaml.safe_load(value) except Exception: raise ValueError("Invalid YAML string") if isinstance(output, str): output = {k: None for k in output.split(",")} if not isinstance(output, dict): raise ValueError("Expecting a comma-delimited str or YAML dictionary") return output def _attr_to_optparse_option(at: Field, default: Any) -> Tuple[dict, str]: """Convert a field into a Docutils optparse options dict. :returns: (option_dict, default) """ if == "url_schemes": return { "metavar": "<comma-delimited>|<yaml-dict>", "validator": _validate_url_schemes, }, ",".join(default) if at.type is int: return {"metavar": "<int>", "validator": _validate_int}, str(default) if at.type is bool: return { "metavar": "<boolean>", "validator": frontend.validate_boolean, }, str(default) if at.type is str or == "heading_slug_func": return { "metavar": "<str>", }, f"(default: '{default}')" if get_origin(at.type) is Literal and all( isinstance(a, str) for a in get_args(at.type) ): args = get_args(at.type) return { "metavar": f"<{'|'.join(repr(a) for a in args)}>", "type": "choice", "choices": args, }, repr(default) if at.type in (Iterable[str], Sequence[str]): return { "metavar": "<comma-delimited>", "validator": frontend.validate_comma_separated_list, }, ",".join(default) if at.type == Set[str]: return { "metavar": "<comma-delimited>", "validator": _validate_comma_separated_set, }, ",".join(default) if at.type == Tuple[str, str]: return { "metavar": "<str,str>", "validator": _create_validate_tuple(2), }, ",".join(default) if at.type == Union[int, type(None)]: return { "metavar": "<null|int>", "validator": _validate_int, }, str(default) if at.type == Union[Iterable[str], type(None)]: return { "metavar": "<null|comma-delimited>", "validator": frontend.validate_comma_separated_list, }, ",".join(default) if default else "" if get_origin(at.type) is dict: return { "metavar": "<yaml-dict>", "validator": _create_validate_yaml(at), }, str(default) if default else "" raise AssertionError( f"Configuration option {} not set up for use in docutils.conf." )
[docs]def attr_to_optparse_option( attribute: Field, default: Any, prefix: str = "myst_" ) -> Tuple[str, List[str], Dict[str, Any]]: """Convert an ``MdParserConfig`` attribute into a Docutils setting tuple. :returns: A tuple of ``(help string, option flags, optparse kwargs)``. """ name = f"{prefix}{}" flag = "--" + name.replace("_", "-") options = {"dest": name, "default": DOCUTILS_UNSET} at_options, default_str = _attr_to_optparse_option(attribute, default) options.update(at_options) help_str = attribute.metadata.get("help", "") if attribute.metadata else "" if default_str: help_str += f" (default: {default_str})" return (help_str, [flag], options)
[docs]def create_myst_settings_spec(config_cls=MdParserConfig, prefix: str = "myst_"): """Return a list of Docutils setting for the docutils MyST section.""" defaults = config_cls() return tuple( attr_to_optparse_option(at, getattr(defaults,, prefix) for at in config_cls.get_fields() if ("docutils" not in at.metadata.get("omit", [])) )
[docs]def create_myst_config( settings: frontend.Values, config_cls=MdParserConfig, prefix: str = "myst_", ): """Create a configuration instance from the given settings.""" values = {} for attribute in config_cls.get_fields(): if "docutils" in attribute.metadata.get("omit", []): continue setting = f"{prefix}{}" val = getattr(settings, setting, DOCUTILS_UNSET) if val is not DOCUTILS_UNSET: values[] = val return config_cls(**values)
[docs]class Parser(RstParser): """Docutils parser for Markedly Structured Text (MyST).""" supported: Tuple[str, ...] = ("md", "markdown", "myst") """Aliases this parser supports.""" settings_spec = ( "MyST options", None, create_myst_settings_spec(), *RstParser.settings_spec, ) """Runtime settings specification.""" config_section = "myst parser" config_section_dependencies = ("parsers",) translate_section_name = None
[docs] def get_transforms(self): return super().get_transforms() + [ResolveAnchorIds]
[docs] def parse(self, inputstring: str, document: nodes.document) -> None: """Parse source text. :param inputstring: The source string to parse :param document: The root docutils node to add AST elements to """ from docutils.writers._html_base import HTMLTranslator HTMLTranslator.visit_rubric = visit_rubric_html HTMLTranslator.depart_rubric = depart_rubric_html HTMLTranslator.visit_container = visit_container_html HTMLTranslator.depart_container = depart_container_html self.setup_parse(inputstring, document) # check for exorbitantly long lines if hasattr(document.settings, "line_length_limit"): for i, line in enumerate(inputstring.split("\n")): if len(line) > document.settings.line_length_limit: error = document.reporter.error( f"Line {i+1} exceeds the line-length-limit:" f" {document.settings.line_length_limit}." ) document.append(error) return # create parsing configuration from the global config try: config = create_myst_config(document.settings) except Exception as exc: error = document.reporter.error(f"Global myst configuration invalid: {exc}") document.append(error) config = MdParserConfig() if "attrs_image" in config.enable_extensions: create_warning( document, "The `attrs_image` extension is deprecated, " "please use `attrs_inline` instead.", MystWarnings.DEPRECATED, ) # update the global config with the file-level config try: topmatter = read_topmatter(inputstring) except TopmatterReadError: pass # this will be reported during the render else: if topmatter: warning = lambda wtype, msg: create_warning( # noqa: E731 document, msg, wtype, line=1, append_to=document ) config = merge_file_level(config, topmatter, warning) # parse content parser = create_md_parser(config, DocutilsRenderer) parser.options["document"] = document parser.render(inputstring) # post-processing # replace raw nodes if raw is not allowed if not getattr(document.settings, "raw_enabled", True): for node in document.traverse(nodes.raw): warning = document.reporter.warning("Raw content disabled.") node.parent.replace(node, warning) self.finish_parse()
[docs]class SimpleTranslator(HTMLTranslator):
[docs] def stylesheet_call(self, *args, **kwargs): return ""
[docs]class SimpleWriter(Writer): settings_spec = filter_settings_spec( Writer.settings_spec, "template", )
[docs] def apply_template(self): subs = self.interpolation_dict() return "%(body)s\n" % subs
def __init__(self): = {} self.translator_class = SimpleTranslator
def _run_cli(writer_name: str, writer_description: str, argv: Optional[List[str]]): """Run the command line interface for a particular writer.""" publish_cmdline( parser=Parser(), writer_name=writer_name, description=( f"Generates {writer_description} from standalone MyST sources.\n{default_description}" ), argv=argv, )
[docs]def cli_html(argv: Optional[List[str]] = None) -> None: """Cmdline entrypoint for converting MyST to HTML.""" _run_cli("html", "(X)HTML documents", argv)
[docs]def cli_html5(argv: Optional[List[str]] = None): """Cmdline entrypoint for converting MyST to HTML5.""" _run_cli("html5", "HTML5 documents", argv)
[docs]def cli_html5_demo(argv: Optional[List[str]] = None): """Cmdline entrypoint for converting MyST to simple HTML5 demonstrations. This is a special case of the HTML5 writer, that only outputs the body of the document. """ publish_cmdline( parser=Parser(), writer=SimpleWriter(), description=( f"Generates body HTML5 from standalone MyST sources.\n{default_description}" ), settings_overrides={ "doctitle_xform": False, "sectsubtitle_xform": False, "initial_header_level": 1, }, argv=argv, )
[docs]def to_html5_demo(inputstring: str, **kwargs) -> str: """Convert a MyST string to HTML5.""" overrides = { "doctitle_xform": False, "sectsubtitle_xform": False, "initial_header_level": 1, "output_encoding": "unicode", } overrides.update(kwargs) return publish_string( inputstring, parser=Parser(), writer=SimpleWriter(), settings_overrides=overrides, )
[docs]def cli_latex(argv: Optional[List[str]] = None): """Cmdline entrypoint for converting MyST to LaTeX.""" _run_cli("latex", "LaTeX documents", argv)
[docs]def cli_xml(argv: Optional[List[str]] = None): """Cmdline entrypoint for converting MyST to XML.""" _run_cli("xml", "Docutils-native XML", argv)
[docs]def cli_pseudoxml(argv: Optional[List[str]] = None): """Cmdline entrypoint for converting MyST to pseudo-XML.""" _run_cli("pseudoxml", "pseudo-XML", argv)
[docs]def visit_rubric_html(self, node): """Override the default HTML visit method for rubric nodes. docutils structures a document, based on the headings, into nested sections:: # h1 ## h2 ### h3 <section> <title> h1 <section> <title> h2 <section> <title> h3 This means that it is not possible to have "standard" headings nested inside other components, such as blockquotes, because it would break the structure:: # h1 > ## h2 ### h3 <section> <title> h1 <blockquote> <section> <title> h2 <section> <title> h3 we work around this shortcoming, in `DocutilsRenderer.render_heading`, by identifying if a heading is inside another component and instead outputting it as a "non-structural" rubric node, and capture the level:: <section> <title> h1 <blockquote> <rubric level=2> h2 <section> <title> h3 However, docutils natively just outputs rubrics as <p> tags, and does not "honor" the heading level. So here we override the visit/depart methods to output the correct <h> element """ if "level" in node: self.body.append(self.starttag(node, f'h{node["level"]}', "", CLASS="rubric")) else: self.body.append(self.starttag(node, "p", "", CLASS="rubric"))
[docs]def depart_rubric_html(self, node): """Override the default HTML visit method for rubric nodes. See explanation in `visit_rubric_html` """ if "level" in node: self.body.append(f'</h{node["level"]}>\n') else: self.body.append("</p>\n")
[docs]def visit_container_html(self, node: nodes.Node): """Override the default HTML visit method for container nodes. to remove the "container" class for divs this avoids CSS clashes with the bootstrap theme """ classes = "docutils container" attrs = {} if node.get("is_div", False): # we don't want the CSS for container for these nodes classes = "docutils" if "style" in node: attrs["style"] = node["style"] self.body.append(self.starttag(node, "div", CLASS=classes, **attrs))
[docs]def depart_container_html(self, node: nodes.Node): """Override the default HTML depart method for container nodes. See explanation in `visit_container_html` """ self.body.append("</div>\n")