convert.py

#!/usr/bin/env python3
# /// script
# requires-python = ">=3.13"
# dependencies = [
#     "lxml",
#     "markdown-it-py",
#     "mistletoe",
#     "pypandoc",
# ]
# ///
import sys
import os
import re
import html
import argparse

import mistletoe.block_token
import mistletoe.span_token
from lxml import etree
import mistletoe.token
import pypandoc
import markdown_it
from markdown_it.token import Token
import mistletoe
from mistletoe.ast_renderer import ASTRenderer
from mistletoe.markdown_renderer import MarkdownRenderer
import urllib.request
import urllib.parse

os.environ["PYPANDOC_PANDOC"] = os.path.abspath("pandoc/pandoc-3.7.0.2/bin/pandoc")

# This may cause issue if used with github pages repo
enable_permalink = False

WEIRD_KEYWORDS = {
    "Feed": "1284192381092u4123"
}

BLACKLIST_FILTER = [
    "Huvudsida-ny"
]

def new_link(matches):
    if '|' not in matches[1]:
        link_text = matches[1]
        new_link = matches[1]
    else:
        link_text = matches[1].split('|')[1].strip()
        new_link = matches[1].split('|')[0].strip()
    
    if '#' in new_link:
        base, anchor = new_link.split('#', 1)
        anchor = anchor.lower().replace(' ', '-')
        new_link = f"{base}#{anchor}"

    new_link = new_link.replace(' ', '_')
    new_link = new_link.replace('?', '')

    if enable_permalink:
        return f"[[/{new_link}|{link_text}]]"

    return f"[[{new_link}|{link_text}]]"

def normalize_path(path):
    parts = []
    path = path.replace('\\', '/')
    path = re.sub(r'\/+', '/', path)
    segments = path.split('/')
    test = ''
    for segment in segments:
        if segment != '.':
            test = parts.pop() if parts else None
            if test is None:
                parts.append(segment)
            else:
                if segment == '..':
                    if test == '..':
                        parts.append(test)
                    if test == '..' or test == '':
                        parts.append(segment)
                else:
                    parts.append(test)
                    parts.append(segment)
    return '/'.join(parts)

def main():
    args = parse_arguments()
    filter: Optional[str] = args.filter

    if not args.filename:
        print("No input file specified. Use --filename=mediawiki.xml\n\n")
        exit(1)

    if args.output:
        output_path = args.output
        if not os.path.exists(output_path):
            print(f"Creating output directory {output_path}\n\n")
            os.makedirs(output_path)
    else:
        output_path = os.getcwd()

    format = args.format
    add_meta = args.fm or (not args.fm and format == 'gfm')

    with open(args.filename, 'r', encoding='utf-8') as file:
        xml_string = file.read()

    format_ = args.format or 'gfm'
    add_meta = args.fm or (not args.fm and format_ == 'gfm')

    # Register the default namespace
    nsmap = {'mw': 'http://www.mediawiki.org/xml/export-0.11/'}

    # Replace Swedish mediawiki syntax with English equivalents
    xml_string = re.sub(r'\bFil:', 'File:', xml_string, flags=re.IGNORECASE)
    xml_string = re.sub(r'\bMedia:', 'File:', xml_string, flags=re.IGNORECASE)
    xml_string = re.sub(r'\bKategori:', 'Category:', xml_string, flags=re.IGNORECASE)
    xml_string = re.sub(r'\bMall:', 'Template:', xml_string, flags=re.IGNORECASE)
    xml_string = re.sub(r'\bBild:', 'Image:', xml_string, flags=re.IGNORECASE)


    xml = etree.fromstring(xml_string.encode('utf-8'))
    result = xml.xpath('//mw:page', namespaces=nsmap)

    count = 0
    directory_list = {}

    sitewide_contributors = []
    all_titles = set()

    for node in result:
        title = node.xpath('mw:title', namespaces=nsmap)[0].text
        url = title.replace(' ', '_')
        url = url.replace('?', '')
        slash = url.rfind('/')

        # Skip talk pages and file pages and other special pages
        if ':' in title:
            print(f"skipping {title}")
            continue

        if filter and not re.search(filter, title):
            continue

        if title in BLACKLIST_FILTER:
            print(f"skipping {title} (blacklisted)")
            continue
    
        directory = ''
        
        filename = title
        filename = filename.replace('/', ' or ')
        
        print(f"Processing {title} ({url})")
        
        print(etree.tostring(node, pretty_print=True, encoding='unicode'))

        # Replace &lt;gallery&gt;...&lt;/gallery&gt; or <gallery>...</gallery> with normal mediawiki file links
        # Handle both escaped and unescaped gallery tags in the text content
        def replace_gallery_tags(text):
            # Match both <gallery>...</gallery> and &lt;gallery&gt;...&lt;/gallery&gt;
            gallery_pattern = re.compile(
            r'<gallery(?: mode="packed-overlay")?[^>]*?>(.*?)(?:</gallery>|&lt;/gallery&gt;)', re.DOTALL | re.IGNORECASE
            )
            def gallery_replacer(match):
                content = match.group(1)
                files = [line.strip() for line in content.split('\n') if line.strip()]
                # Accept both File: and Image: prefixes
                files = [f for f in files if f.startswith('File:') or f.startswith('Image:')]
                files = [f.replace('File:', '').replace('Image:', '').strip() for f in files]
                links = '\n\n'.join(f'[[Image:{file}]]' for file in files)
                r =  "\n" + links + "\n"
                return r
            return gallery_pattern.sub(gallery_replacer, text)

        revision_elements = node.xpath('mw:revision', namespaces=nsmap)
        revision_elements.sort(key=lambda elem: elem.findtext('mw:timestamp', namespaces=nsmap))

        # Iterate all the revision, collecting all the useful metadata we want to preserve (especially the contributors)
        contributors = []
        for revision_element in revision_elements:
            contributor = revision_element.findtext('mw:contributor/mw:username', namespaces=nsmap)
            # Check if contributor is not None and not already in the list
            if contributor and contributor not in contributors:
                contributors.append(contributor)
            if contributor and contributor not in sitewide_contributors:
                sitewide_contributors.append(contributor)

        # Extract relevant information from the latest revision
        latest_revision = revision_elements[-1] # Get the last (highest) <revision> element
        latest_id = latest_revision.findtext('mw:id', namespaces=nsmap)
        latest_timestamp = latest_revision.findtext('mw:timestamp', namespaces=nsmap)
        latest_contributor = latest_revision.findtext('mw:contributor/mw:username', namespaces=nsmap)
        latest_content = latest_revision.findtext('mw:text', namespaces=nsmap)
        print(f"{title} ({latest_id} {latest_timestamp} {latest_contributor})")

        latest_content = re.sub(r'<!--.*?-->', '', latest_content, flags=re.DOTALL)

        # Replace links like https://www.makerspace.se/wiki/images/e/e7/{filename} with File:filename
        latest_content = re.sub(
            r'https?://www.makerspace.se/wiki/images/[a-z0-9]/[a-z0-9]{2}/([^ \n\]\|]+)',
            r'File:\1',
            latest_content,
            flags=re.IGNORECASE
        )

        # Remove all [[Category:XXX]] from the document
        latest_content = re.sub(r'\[\[Category:[^\]]+\]\]', '', latest_content)

        # Replace image links with only filename and title
        def simplify_image_link(match):
            parts = [p.strip() for p in match.group(1).split('|') if p.strip()]
            if not parts:
                return match.group(0)
            filename = parts[0].replace(" ", "_")
            if len(parts) == 1:
                return f'[[{filename}]]'
            known_options = {'none', 'ingen', 'border', 'kantlinje', 'miniature', 'miniatyr', 'left', 'right', 'center', 'centered', 'centrerad', 'frameless', 'ramlös', 'thumb', 'thumbnail', 'vänster', 'höger', 'center'}
            
            def is_known(opt):
                return opt in known_options or re.match(r'^\d+\s?(px|em)', opt) or re.match(r'^\d+x\d+(px|em)', opt) or opt.startswith("alt=") or opt.startswith("länk=")
            # The title is always the last item
            title = parts[-1]
            options = parts[1:-1]
            if is_known(title.lower()):
                # If the title is a known option, we treat it as an option and remove it from the title
                title = None
                options = parts[1:]

            # Check all options are known
            for opt in options:
                if not is_known(opt.lower()):
                    raise ValueError(f"Unknown image option: '{opt}' in {match.group(0)}")
            if title:
                all_titles.add(title)
                # return f'[[{filename}|{title}]]'
                return f"%%%{filename}12341253{title}%%%"
            else:
                return f"%%%{filename}%%%"
                # return f'[[{filename}]]'

        latest_content = re.sub(
            r'\[\[((?:File|Image):[^|\]]+(?:\|[^\]]*)*)\]\]',
            simplify_image_link,
            latest_content,
            flags=re.IGNORECASE
        )

        # Convert absolute links like [https://wiki.makerspace.se/Hello English] to relative links
        def absolute_to_relative(match):
            url = match.group(1)
            latest_content = match.group(2)
            # Only convert links to wiki.makerspace.se that are not files/images
            m = re.match(r'https?://wiki\.makerspace\.se/(?!wiki/images/|Special:Filepath/)([^ ]+)', url)
            if m:
                page = m.group(1)
                page = page.replace('_', ' ')
                # Remove any fragment
                page = page.split('#')[0]
                return f'[[{page}|{latest_content}]]'
            return match.group(0)

        latest_content = re.sub(
            r'\[(https?://wiki\.makerspace\.se/[^ ]+) ([^\]]+)\]',
            absolute_to_relative,
            latest_content
        )

        text = html.unescape(latest_content)

        text = replace_gallery_tags(text)

        # Replace all <br> tags with two newlines, except if the line starts with a pipe (|), which is used for tables
        # Adding newlines inside tables will mess up the syntax
        text = re.sub(r'^(\|.*)<br\s*/?\s*>', r'\1', text, flags=re.MULTILINE | re.IGNORECASE)
        text = re.sub(r'<br\s*/?\s*>', r'\n\n', text, flags=re.MULTILINE | re.IGNORECASE)

        # Some machine pages use a picture as a caption, instead of a row. This looks bad when converted to Markdown.
        # So replace it with a regular row.
        text = re.sub(r"\|\+'''Snabbfakta:''' ''(.*?)''", "| ", text)

        # text = re.sub(r'\[\[(.+?)\]\]', new_link, text)

        # Convert file and image links to an internal format, to avoid some pandoc bugs when a cell only contains an image
        # text = re.sub(
        #     r'\[\[((?:File|Image):[^|\]]+(?:\|[^\]]*)*)\]\]',
        #     lambda m: f"%%%{m.group(1).replace('|', '5234234')}%%%",
        #     text,
        #     flags=re.IGNORECASE
        # )
        text = text.replace("__NOTOC__", "")

        if add_meta:
            # frontmatter = "---\n"
            # frontmatter += f"title: {title}\n"
            # if enable_permalink:
            #     frontmatter += f"permalink: /{url}/\n"
            # # Add contributors to the front matter
            # if contributors:
            #     frontmatter += "contributors:\n"
            #     for contributor in contributors:
            #         frontmatter += f"  - {contributor}\n"
            # frontmatter += "---\n\n"
            pass

        has_museum = "{{Museum}}" in text

        # Pandoc interprets some keywords in weird ways. For example "Feed:blah" will become a link to 'blah' for some reason.
        for keyword, replacement in WEIRD_KEYWORDS.items():
            text = text.replace(keyword, replacement)

        # Remove leading tabs from all lines, as they are ignored in MediaWiki, but have meaning in Markdown
        text = re.sub(r'^\t+', '', text, flags=re.MULTILINE)

        print("=========================================\n")
        print(text)
        print("=========================================\n")
        internal = pypandoc.convert_text(text, format='mediawiki-auto_identifiers', to="native", extra_args=["--columns=10000"])

        with open("native.txt", 'w') as f:
            f.write(internal)

        internal = internal.replace("[ \"wikilink\" ]", "[]")
        text = pypandoc.convert_text(internal, format='native', to="markdown-multiline_tables-grid_tables+pipe_tables-table_captions-simple_tables-auto_identifiers", extra_args=["--columns=10000"])

        with open("pandoc_out.txt", 'w') as f:
            f.write(text)

        print(text)
        # exit(0)

        # Convert %%%File:...%%% or %%%Image:...%%% to markdown image syntax
        def file_to_markdown_image(match):
            parts = match.group(1).strip().split('12341253')
            filename = parts[0].strip()
            if len(parts) > 1:
                title = parts[1].strip()
                return f'![{title}]({filename})'
            else:
                return f'![]({filename})'

        text = re.sub(r'%%%((?:File|Image):[^%]+)%%%', file_to_markdown_image, text, flags=re.IGNORECASE)
        
        
        text = re.sub(r'\\?<\s*\\?<\s*TODO\s*\\?>\s*\\?>', '\n:::warning\nTODO\n:::\n', text, flags=re.IGNORECASE)

        # Add the .md extension to wikilinks if the --mdbook argument is passed
        # This is similar to running this sed function changing all instances of `[Text](Text "wikilink")` to `[Text](Text.md)`
        # ergo: `find . -name '*.md' -exec sed -i 's/\(\[.*\]\)(\(.*\)\s"wikilink")/\1(\2.md)/g' {} \;`
        if args.mdbook:
            text = re.sub(r'\[([^\]]+)\]\(([^ )]+)( "wikilink"\))', r'[\1](\2.md)', text)

        text = text.replace('\\_', '_')
        text = text.replace('\\. ', '. ')

        # Replace <youtube>HASH</youtube> with a Markdown YouTube link
        def youtube_replacer(match):
            video_id = match.group(1).strip()
            url = f"https://www.youtube.com/watch?v={video_id}"
            return f"[{url}]({url})"

        text = re.sub(r'<youtube>(.*?)</youtube>', youtube_replacer, text, flags=re.IGNORECASE)

        # Remove <!-- end list --> comments from the text
        text = re.sub(r'<!--\s*end list\s*-->', '', text, flags=re.IGNORECASE)

        # Replace <sup>...</sup> with unicode superscript equivalents
        def sup_to_unicode(match):
            sup_map = {
                '0': '⁰', '1': '¹', '2': '²', '3': '³', '4': '⁴',
                '5': '⁵', '6': '⁶', '7': '⁷', '8': '⁸', '9': '⁹',
                '+': '⁺', '-': '⁻', '=': '⁼', '(': '⁽', ')': '⁾',
                'n': 'ⁿ', 'i': 'ⁱ', 'a': 'ᵃ', 'b': 'ᵇ', 'c': 'ᶜ',
                'd': 'ᵈ', 'e': 'ᵉ', 'f': 'ᶠ', 'g': 'ᵍ', 'h': 'ʰ',
                'j': 'ʲ', 'k': 'ᵏ', 'l': 'ˡ', 'm': 'ᵐ', 'o': 'ᵒ',
                'p': 'ᵖ', 'r': 'ʳ', 's': 'ˢ', 't': 'ᵗ', 'u': 'ᵘ',
                'v': 'ᵛ', 'w': 'ʷ', 'x': 'ˣ', 'y': 'ʸ', 'z': 'ᶻ',
                'A': 'ᴬ', 'B': 'ᴮ', 'D': 'ᴰ', 'E': 'ᴱ', 'G': 'ᴳ',
                'H': 'ᴴ', 'I': 'ᴵ', 'J': 'ᴶ', 'K': 'ᴷ', 'L': 'ᴸ',
                'M': 'ᴹ', 'N': 'ᴺ', 'O': 'ᴼ', 'P': 'ᴾ', 'R': 'ᴿ',
                'T': 'ᵀ', 'U': 'ᵁ', 'V': 'ⱽ', 'W': 'ᵂ'
            }
            text = match.group(1)
            return ''.join(list(sup_map.get(c, c) for c in text))

        text = re.sub(r'<sup>(.*?)</sup>', sup_to_unicode, text, flags=re.DOTALL)

        # Remove <big> and </big> tags from the text
        text = re.sub(r'</?big>', '', text, flags=re.IGNORECASE)

        # Replace <s> and </s> tags with Markdown strikethrough (~~)
        text = re.sub(r'</?s>', '~~', text, flags=re.IGNORECASE)

        if not output_path.endswith('/'):
            output_path += '/'

        # Find all image and media links in the markdown and download them if not already present

        media_folder = os.path.join(output_path, "uploads")
        if not os.path.exists(media_folder):
            os.makedirs(media_folder, exist_ok=True)

        def download_media_file(filename):
            if filename.startswith(":"):
                raise ValueError(f"Invalid filename: {filename}")

            safe_filename = filename.replace(' ', '_')
            encoded_filename = urllib.parse.quote(safe_filename)
            # https://wiki.makerspace.se/Special:Filepath/AI_AMP1.pdf
            url = f"https://wiki.makerspace.se/Special:Filepath/{encoded_filename}"
            local_path = os.path.join(media_folder, safe_filename)
            if not os.path.exists(local_path):
                try:
                    print(f"Downloading {url} -> {local_path}")
                    urllib.request.urlretrieve(url, local_path)
                except Exception as e:
                    print(f"Failed to download media: {filename} ({e})")
                    return filename  # fallback
            return os.path.relpath(local_path, os.path.dirname(normalize_path(directory + filename + '.md')))

        def patch_html(md_text):
            changed = False

            def is_media_file(link):
                return re.search(r'\.(png|jpg|jpeg|gif|svg|mp3|mp4|ogg|webm|pdf)$', link, re.IGNORECASE)

            changed = False
            lowest_heading_level = 6

            def strong(text):
                return mistletoe.Document("*" + text + "*").children[0].children[0]

            def raw_text(text):
                return mistletoe.span_token.RawText(text)

            def walk(node: mistletoe.token.Token):
                nonlocal changed
                nonlocal lowest_heading_level
                # print(type(node))
                if isinstance(node, mistletoe.span_token.Image):
                    # print(f"Walking node: {node}")
                    # Remove titles that are just the filename
                    print(node.title, os.path.basename(node.src))
                    if node.title and node.title.replace(" ", "_").replace("Image:", "").replace("File:", "") == os.path.basename(node.src):
                        print(f"Removing title from image: {node.src}")
                        node.title = None
                        changed = True
                    
                    assert node.children is not None
                    if len(node.children) == 1 and isinstance(node.children[0], mistletoe.span_token.RawText):
                        if node.children[0].content.replace(" ", "_").replace("Image:", "").replace("File:", "") == os.path.basename(node.src):
                            node.children = []
                            changed = True

                    src = node.src
                    print(src)
                    if src and not (src.startswith("http://") or src.startswith("https://")) and is_media_file(src):
                        src_clean = src.replace("File:", "").replace("Image:", "")
                        local_rel_path = download_media_file(os.path.basename(src_clean))
                        node.src = f'uploads/{os.path.basename(local_rel_path)}'
                        changed = True
                elif isinstance(node, mistletoe.block_token.Heading):
                    lowest_heading_level = min(lowest_heading_level, node.level)
                elif isinstance(node, mistletoe.span_token.Link):
                    # If there's a leading colon, this just means to link to the file instead of showing the content inline on the page.
                    if node.label:
                        node.label = re.sub(":?(File|Image):", "", node.label, flags=re.IGNORECASE)
                    if node.title:
                        node.title = re.sub(":?(File|Image):", "", node.title, flags=re.IGNORECASE)

                    if node.target and not (node.target.startswith("http://") or node.target.startswith("https://")) and is_media_file(node.target):
                        target_clean =  re.sub(":?(File|Image):", "", node.target, flags=re.IGNORECASE)
                        local_rel_path = download_media_file(os.path.basename(target_clean))
                        node.target = f'uploads/{os.path.basename(local_rel_path)}'
                        changed = True
                    # Remove link titles that are exactly "wikilink"
                    if node.title and node.title.strip().lower() == 'wikilink':
                        node.title = None
                        print(f"Removed title from link: {node.target}")
                        changed = True
                elif isinstance(node, mistletoe.block_token.Table):
                    # Check for single-row "You must complete" warning table
                    if len(node.children) == 1:
                        row = node.children[0]
                        if isinstance(row, mistletoe.block_token.TableRow) and row.children:
                            cell = row.children[0]
                            if isinstance(cell, mistletoe.block_token.TableCell) and cell.children:
                                first_child = cell.children[0]
                                if isinstance(first_child, mistletoe.span_token.RawText) and (first_child.content.strip().startswith("You must complete") or first_child.content.strip().startswith("Kräver genomförd")):
                                    replacement = ([
                                        raw_text(":::warning\n")] +
                                        cell.children +
                                        [raw_text("\n:::\n")]
                                    )
                                    print(replacement)
                                    p = mistletoe.block_token.Paragraph([])
                                    p.children = replacement
                                    replacement = [p]

                                    # Replace the table node in its parent's children with the replacement nodes
                                    idx = node.parent.children.index(node)
                                    # Remove the table node
                                    node.parent.children.pop(idx)
                                    # Insert the replacement nodes at the same position
                                    for i, rep in enumerate(replacement):
                                        node.parent.children.insert(idx + i, rep)
                                    changed = True

                                    print(f"Replaced table with warning: {first_child.content.strip()}")
                                    print(node.parent)
                                    # walk(cell)
                                    return


                    single_col = True
                    parsed_rows = []
                    for row in node.children:
                        if isinstance(row, mistletoe.block_token.TableRow):
                            if len(row.children) != 1:
                                single_col = False
                                break

                            cell = row.children[0]
                            assert isinstance(cell, mistletoe.block_token.TableCell), "Expected TableCell in TableRow"

                            if len(cell.children) == 0:
                                # If the cell is empty, skip it
                                continue

                            # Check if the cell contains a single <strong> tag with "key:" and a value
                            bold = cell.children[0]
                            if isinstance(bold, mistletoe.span_token.Strong):
                                if len(cell.children) == 1 and len(parsed_rows) == 0:
                                    # Probably a header row
                                    # Skip it
                                    continue


                                for c in bold.children:
                                    if isinstance(c, mistletoe.span_token.RawText):
                                        c.content = c.content.replace(':', '').strip()
                                
                                key = bold.children
                                value = cell.children[1:]
                                parsed_rows.append((key, value))
                            elif all(isinstance(child, mistletoe.span_token.Image) for child in cell.children):
                                # If all children are images, treat as "Image" key
                                key = [raw_text("Image")]
                                value = cell.children
                                parsed_rows.append((key, value))
                            else:
                                single_col = False
                                break
                    
                    if single_col and parsed_rows:
                        # Remove all children from the table
                        node.children.clear()
                        node.column_align = [None]
                        # Add header row
                        header = mistletoe.block_token.TableRow("")
                        th1 = mistletoe.block_token.TableCell("")
                        th1.children.append(raw_text("Fact"))
                        th2 = mistletoe.block_token.TableCell("")
                        th2.children.append(raw_text(""))
                        header.children = [th1, th2]
                        node.header = header
                        # node.children.append(header)
                        # Add parsed rows as two columns
                        for key, value in parsed_rows:
                            tr = mistletoe.block_token.TableRow("")
                            td1 = mistletoe.block_token.TableCell("")
                            td1.children.extend(key)
                            td2 = mistletoe.block_token.TableCell("")
                            td2.children.extend(value)
                            tr.children = [td1, td2]
                            node.children.append(tr)
                        
                        changed = True
                elif isinstance(node, mistletoe.block_token.Paragraph):
                    # Add a newline at the end of paragraphs
                    node.children.append(mistletoe.span_token.RawText('\n\n'))

                    # If the paragraph starts with a dash, replace it with a star
                    if node.children and isinstance(node.children[0], mistletoe.span_token.EscapeSequence):
                        raw = node.children[0].children[0]
                        if raw.content == '-':
                            node.children[0] = mistletoe.span_token.RawText('*')
                            changed = True

                if node.children:
                    # Walk children
                    for v in node.children:
                        walk(v)

            def raise_heading_levels(node: mistletoe.token.Token, level: int):
                if isinstance(node, mistletoe.block_token.Heading):
                    node.level += level
                    assert node.level >= 1
                if node.children:
                    for child in node.children:
                        raise_heading_levels(child, level)

            # Convert markdown to AST
            doc = mistletoe.Document(md_text)
            walk(doc)
            raise_heading_levels(doc, 1 - lowest_heading_level)

            # Convert AST back to markdown
            with MarkdownRenderer(normalize_whitespace=False, max_line_length=10000) as md_renderer:
                old_render_table = md_renderer.render_table
                old_render_paragraph = md_renderer.render_paragraph
                def render_table(token: mistletoe.block_token.Table, max_line_length: int) -> list[str]:
                    return ["", ""] + old_render_table(token, max_line_length) + ["", ""]
                
                def render_paragraph(token: mistletoe.block_token.Paragraph, max_line_length: int) -> list[str]:
                    # Add a newline at the end of paragraphs
                    res = list(old_render_paragraph(token, max_line_length))
                    
                    if not isinstance(token.parent, mistletoe.block_token.ListItem):
                        res += ["", ""]

                    # print(type(token.parent) if token.parent else "<ROOT>", "\n".join(res)[:50])
                    return res

                md_renderer.render_map[mistletoe.block_token.Table.__name__] = render_table
                md_renderer.render_map[mistletoe.block_token.Paragraph.__name__] = render_paragraph

                patched_md = md_renderer.render(doc)
            

            changed = True
            while changed:
                orig = patched_md
                # Convert green spans to Markdown tip blocks
                patched_md = re.sub(
                    r'<(?:span|b)\s+style=["\'](?:color|background):\s*#33cc33;?(?: font-size: \d+px;?)?\s*["\']\s*>(.*?)</(?:span|b)>',
                    r'\n:::tip\n\1\n:::\n\n',
                    patched_md,
                    flags=re.DOTALL | re.IGNORECASE
                )

                # Convert red spans to Markdown warning blocks
                patched_md = re.sub(
                    r'<(?:span|b)\s+style=["\'](?:color|background):\s*(?:red|yellow|#FF0000);?(?: font-size: \d+px;?)?\s*["\']\s*>(.*?)</(?:span|b)>',
                    r'\n:::warning\n\1\n:::\n\n',
                    patched_md,
                    flags=re.DOTALL | re.IGNORECASE
                )

                # Convert broken red spans that don't have a closing tag
                patched_md = re.sub(
                    r'<(?:span|b)\s+style=["\'](?:color|background):\s*(?:red|yellow|#FF0000);?(?: font-size: \d+px;?)?\s*["\']\s*>(.*?)$',
                    r'\n:::warning\n\1\n:::\n\n',
                    patched_md,
                    flags=re.DOTALL | re.IGNORECASE | re.MULTILINE
                )

                changed = patched_md != orig

            return patched_md

        print("-----------------------------------\n")
        print(text)
        print("-----------------------------------\n")
        
        text = patch_html(text)
        if has_museum:
            text = ":::warning\nThis page is no longer relevant, and is only kept for nostalgic reasons.\n:::\n\n" + text

        text = f"# {title}\n\n" + text

        for key, value in WEIRD_KEYWORDS.items():
            text = text.replace(value, key)

        print("\n----------------------------------\n")
        print(text)
        print("\n----------------------------------\n")

        # Check for any remaining HTML tags and log errors
        html_tags = re.findall(r'<[^>]+>', text)
        # Filter out tags that are just http/https urls, as they are sometimes generated by the markdown converter
        html_tags = [tag for tag in html_tags if not re.match(r'<(?:https?|s?ftp)://[^>]+>|<[^\s]+@[^\s]+>', tag)]
        html_tags = [tag for tag in html_tags if tag not in ["<u>", "</u>"]]
    
        if html_tags:
            print(f"ERROR: Remaining HTML tags found in {title}:")
            for tag in set(html_tags):
                print(f"  - {tag}")
            # exit(1)

        directory = output_path + directory

        if directory:
            if not os.path.exists(directory):
                os.makedirs(directory, 0o755, True)
            directory += '/'

        with open(normalize_path(directory + filename + '.md'), 'w') as file:
            file.write(text)

        count += 1

    if sitewide_contributors:
        print("sitewide_contributors:")
        for contributor in sitewide_contributors:
            print(f"  - {contributor}")

    print("all_titles:")
    for title in sorted(all_titles):
        print(f"  - {title}")
    

    if directory_list and args.indexes:
        directory_list = list(directory_list.keys())

        for directory_name in directory_list:
            if os.path.exists(output_path + directory_name + '.md'):
                os.rename(output_path + directory_name + '.md', output_path + directory_name + '/index.md')

    if count > 0:
        print(f"{count} files converted\n\n")

def parse_arguments():
    parser = argparse.ArgumentParser(description="Convert MediaWiki XML to Markdown")
    parser.add_argument('--filename', required=True, help="Input XML file")
    parser.add_argument('--output', help="Output directory")
    parser.add_argument('--format', default='gfm', help="Output format (default: gfm)")
    parser.add_argument('--fm', action='store_true', help="Add front matter")
    parser.add_argument('--indexes', action='store_true', help="Rename and move files with the same name as directories")
    parser.add_argument('--mdbook', action='store_true', help="Convert wikilinks to .md links for mdBook compatibility")
    parser.add_argument('--filter', help="Filter pages by title (regex)")
    return parser.parse_args()

if __name__ == "__main__":
    main()