html_convert.py

import os
import requests
from lxml import html
import argparse
import re
import urllib.parse
import json
from dataclasses import dataclass, field
from typing import Optional, Any, Tuple
from dataclasses_json import DataClassJsonMixin
import uuid
import unicodedata
from datetime import datetime
import zipfile
import mimetypes
import webcolors
import minify_html
import re
import colorsys
from colormath.color_objects import sRGBColor, LabColor
from colormath.color_conversions import convert_color
import colormath.color_diff
from progressbar import progressbar

pages = open("all_pages.txt", "r").read().strip().split("\n")

BASE = "https://wiki.makerspace.se"
BASE_HOSTNAME = urllib.parse.urlparse(BASE).hostname

CACHE_DIR = "cache_html"
OUTPUT_DIR = "output"
os.makedirs(CACHE_DIR, exist_ok=True)

VALID_COLORS = [
    "#FDEA9B",
    "#FED46A",
    "#FA551E",
    "#B4DC19",
    "#C8AFF0",
    "#3CBEFC",
]

BLOCK_ELEMENTS = [
    "div",
    "p",
    "h1",
    "h2",
    "h3",
    "h4",
    "h5",
    "h6",
    "blockquote",
    "ul",
    "ol",
    "li",
    "table",
    "tr",
    "td",
    "th",
]

MIRRORED_EXTENSIONS = [
    ".svg",
    ".png",
    ".jpg",
    ".jpeg",
    ".gif",
    ".pdf",
    ".webp",
    ".txt",
    ".md",
    ".zip",
    ".stl",
    ".gcode",
    ".mp4",
    ".webm",
    ".ogv",
    ".mov",
]


# Replace <sup>...</sup> with unicode superscript equivalents
def sup_to_unicode(match):
    sup_map = {
        "0": "⁰",
        "1": "¹",
        "2": "²",
        "3": "³",
        "4": "⁴",
        "5": "⁵",
        "6": "⁶",
        "7": "⁷",
        "8": "⁸",
        "9": "⁹",
        "+": "⁺",
        "-": "⁻",
        "=": "⁼",
        "(": "⁽",
        ")": "⁾",
        "n": "ⁿ",
        "i": "ⁱ",
        "a": "ᵃ",
        "b": "ᵇ",
        "c": "ᶜ",
        "d": "ᵈ",
        "e": "ᵉ",
        "f": "ᶠ",
        "g": "ᵍ",
        "h": "ʰ",
        "j": "ʲ",
        "k": "ᵏ",
        "l": "ˡ",
        "m": "ᵐ",
        "o": "ᵒ",
        "p": "ᵖ",
        "r": "ʳ",
        "s": "ˢ",
        "t": "ᵗ",
        "u": "ᵘ",
        "v": "ᵛ",
        "w": "ʷ",
        "x": "ˣ",
        "y": "ʸ",
        "z": "ᶻ",
        "A": "ᴬ",
        "B": "ᴮ",
        "D": "ᴰ",
        "E": "ᴱ",
        "G": "ᴳ",
        "H": "ᴴ",
        "I": "ᴵ",
        "J": "ᴶ",
        "K": "ᴷ",
        "L": "ᴸ",
        "M": "ᴹ",
        "N": "ᴺ",
        "O": "ᴼ",
        "P": "ᴾ",
        "R": "ᴿ",
        "T": "ᵀ",
        "U": "ᵁ",
        "V": "ⱽ",
        "W": "ᵂ",
    }
    text = match.group(1)
    return "".join(list(sup_map.get(c, c) for c in text))


def heading_to_slug(heading: str) -> str:
    replacement = "-"
    heading = re.sub(
        r"[\s!\"#$%&'\.()*+,\/:;<=>?@\[\]\\^_`{|}~]+",
        replacement,
        heading,
        flags=re.UNICODE,
    )
    heading = heading.lower()
    heading = heading.strip(replacement)
    return heading


def delta_e_cie2000(color1, color2, Kl=1, Kc=1, Kh=1):
    """
    Calculates the Delta E (CIE2000) of two colors.
    """
    color1_vector = colormath.color_diff._get_lab_color1_vector(color1)
    color2_matrix = colormath.color_diff._get_lab_color2_matrix(color2)
    delta_e = colormath.color_diff.color_diff_matrix.delta_e_cie2000(
        color1_vector, color2_matrix, Kl=Kl, Kc=Kc, Kh=Kh
    )[0]
    return float(delta_e)


def nearest_valid_color(hex_color):
    def hex_to_rgb(hexstr):
        hexstr = hexstr.lstrip("#")
        if len(hexstr) == 3:
            hexstr = "".join([c * 2 for c in hexstr])
        return tuple(int(hexstr[i : i + 2], 16) for i in (0, 2, 4))

    try:
        rgb = hex_to_rgb(hex_color)
        color1_rgb = sRGBColor(rgb[0], rgb[1], rgb[2], is_upscaled=True)
        color1_lab = convert_color(color1_rgb, LabColor)
    except Exception:
        return VALID_COLORS[0]  # fallback
    min_dist = float("inf")
    nearest = VALID_COLORS[0]
    for valid in VALID_COLORS:
        v_rgb = hex_to_rgb(valid)
        color2_rgb = sRGBColor(v_rgb[0], v_rgb[1], v_rgb[2], is_upscaled=True)
        color2_lab = convert_color(color2_rgb, LabColor)
        dist = delta_e_cie2000(color1_lab, color2_lab)
        if dist < min_dist:
            min_dist = dist
            nearest = valid
    return nearest


@dataclass
class DocumentStructureItem(DataClassJsonMixin):
    id: str  # UUID of the document
    url: str
    title: str
    children: list["DocumentStructureItem"]


@dataclass
class Sort(DataClassJsonMixin):
    field: str
    direction: str


@dataclass
class OutlineCollection(DataClassJsonMixin):
    id: str  # UUID of the collection
    urlId: str  # Example: um8KoiqAPm
    name: str
    data: dict[Any, Any]
    icon: Optional[str]
    permission: Optional[Any]
    commenting: bool
    sharing: bool
    documentStructure: list[DocumentStructureItem]
    index: str = ","
    color: Optional[str] = None
    sort: Sort = field(default_factory=lambda: Sort(field="index", direction="asc"))
    createdAt: Optional[str] = None  # ISO 8601 format
    updatedAt: Optional[str] = None  # ISO 8601 format
    deletedAt: Optional[str] = None  # ISO 8601 format
    archivedAt: Optional[str] = None  # ISO 8601 format


@dataclass
class OutlinePage(DataClassJsonMixin):
    createdById: str  # UUID of the user who created the page
    createdByName: str
    createdByEmail: str
    createdAt: Optional[str]  # ISO 8601 format
    updatedAt: Optional[str]  # ISO 8601 format
    publishedAt: Optional[str]  # ISO 8601 format
    fullWidth: bool
    template: bool
    parentDocumentId: Optional[str]
    id: str  #
    urlId: str
    title: str
    icon: Optional[str]
    color: Optional[str]
    data: dict[Any, Any]


@dataclass
class OutlineExport(DataClassJsonMixin):
    collection: OutlineCollection
    documents: dict[str, OutlinePage]
    attachments: dict[Any, Any]


# {
#     "exportVersion": 1,
#     "version": "0.84.0",
#     "createdAt": "2025-07-06T19:19:21.827Z",
#     "createdById": "f74221f0-4749-41f7-b327-5ed81752ef12",
#     "createdByEmail": "aron.granberg@gmail.com"
# }


@dataclass
class OutlineExportMetadata(DataClassJsonMixin):
    exportVersion: int = 1
    version: str = "0.84.0"
    createdAt: Optional[str] = None  # ISO 8601 format
    createdById: Optional[str] = None  # UUID of the user who created the export
    createdByEmail: Optional[str] = None  # Email of the user who created the export


def mediawiki_slugify(value):
    value = re.sub(r"[\s]+", "_", value)
    return value


def outline_doc_url(doc: OutlinePage) -> str:
    return f"/doc/{outline_slugify(doc.title)}-{doc.urlId}"


# Make a safe urlId from page name
def outline_slugify(value):
    value = unicodedata.normalize("NFKD", value)
    value = value.encode("ascii", "ignore").decode("ascii")
    value = re.sub(r"[\s_]+", "-", value)
    value = re.sub(r"[^\w\d-]", "", value)
    return value.lower()


# Write an old-wiki-path -> Outline-URL map consumed by nginx as
# `map $uri $wiki_redirect { include wiki-redirects.map; }`.
#
# Key coverage: the wiki serves pages as "/Title" ($wgArticlePath = "/$1") and
# "/index.php/Title" ($wgUsePathInfo). The "/index.php?title=Title" query form
# is invisible in $uri; the vhost config must normalize it to "/Title" itself.
# Each title is emitted with underscores (MediaWiki canonical) and with
# spaces, because nginx percent-decodes $uri so "%20" links arrive as literal
# spaces. Case differences cost nothing: nginx map string matching is
# case-insensitive, which also mirrors MediaWiki's case-insensitive first
# letter.
#
# Contract: the map is only valid for an Outline instance imported from the
# export.zip of the same run, because urlIds are regenerated every run.
def write_nginx_redirect_map(
    export: OutlineExport, redirects: dict[str, str], real_page_ids: set[str]
):
    def nginx_quote(value: str) -> str:
        return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'

    # Lowercased key -> (key as emitted, target URL). nginx rejects duplicate
    # map keys, and matching is case-insensitive, so dedup must be too.
    entries: dict[str, tuple[str, str]] = {}

    def add(title_variant: str, target_url: str):
        for key in ("/" + title_variant, "/index.php/" + title_variant):
            existing = entries.get(key.lower())
            if existing is not None:
                if existing[1] != target_url:
                    print(
                        f"Warning: conflicting redirect map entries for {key!r}: "
                        f"{existing[1]} vs {target_url}"
                    )
                continue
            entries[key.lower()] = (key, target_url)

    def add_title_variants(title: str, target_url: str):
        canonical = mediawiki_slugify(title)
        add(canonical, target_url)
        spaced = canonical.replace("_", " ")
        if spaced != canonical:
            add(spaced, target_url)

    slug_to_doc: dict[str, OutlinePage] = {}
    for doc in export.documents.values():
        slug_to_doc.setdefault(mediawiki_slugify(doc.title), doc)

    for doc in export.documents.values():
        if doc.id in real_page_ids:
            add_title_variants(doc.title, outline_doc_url(doc))
        else:
            # Synthetic category folder: point the old category page URLs at it.
            add_title_variants("Kategori:" + doc.title, outline_doc_url(doc))
            add_title_variants("Category:" + doc.title, outline_doc_url(doc))

    # Redirect pages never become documents; send them to their (already
    # chain-resolved) target's URL.
    for src_slug, target_slug in redirects.items():
        target_doc = slug_to_doc.get(target_slug)
        if target_doc is None:
            print(
                f"Warning: redirect target {target_slug!r} has no document; "
                f"skipping map entry for {src_slug!r}"
            )
            continue
        add_title_variants(src_slug, outline_doc_url(target_doc))

    map_path = os.path.join(OUTPUT_DIR, "wiki-redirects.map")
    with open(map_path, "w", encoding="utf-8") as f:
        for key, target_url in sorted(entries.values()):
            f.write(f"{nginx_quote(key)} {nginx_quote(target_url)};\n")
    print(f"Created {map_path} ({len(entries)} entries)")


def get_inner_html(page):
    cache_path = os.path.join(CACHE_DIR, f"{page.replace('/', '_')}.html")
    if os.path.exists(cache_path):
        # print(f"Using cached: {cache_path}")
        with open(cache_path, "r", encoding="utf-8") as f:
            return f.read()

    url = BASE + "/" + page
    response = requests.get(url + "?redirect=no")
    if response.status_code == 200:
        tree = html.fromstring(response.text)
        content_div = tree.xpath('//div[@id="mw-content-text"]')
        if content_div:
            html_content = html.tostring(
                content_div[0], pretty_print=True, encoding="unicode"
            )
            with open(cache_path, "w", encoding="utf-8") as f:
                f.write(html_content)
            print(f"Saved content for {page} to {cache_path}")
            return html_content
        else:
            print(f"Content not found for {page}")
            return None
    else:
        print(f"Failed to fetch {url} (status {response.status_code})")
        return None


def process_images(tree):
    # Find all image, video, and link elements
    element = tree.xpath(".//img[@src]|.//video[@src]|.//video/source[@src]|.//a[@href]")
    for element in element:
        src = element.get("href") if element.tag == "a" else element.get("src")
        if not src:
            continue

        if src.startswith("/"):
            src = BASE + src

        if src.startswith(f"{BASE}/images/"):
            # Extract filename from src
            if "/images/thumb/" in src:
                filename = src.split("/")[-2]
            else:
                filename = os.path.basename(src)

            # Download from Special:Filepath for full resolution
            img_url = f"{BASE}/Special:Filepath/{filename}"
        elif src.startswith(f"{BASE}/File:") or src.startswith(f"{BASE}/Image:"):
            img_url = src
            img_url = img_url.replace(f"{BASE}/File:", f"{BASE}/Special:Filepath/")
            img_url = img_url.replace(f"{BASE}/Image:", f"{BASE}/Special:Filepath/")
            filename = os.path.basename(img_url)
        elif src.startswith(f"{BASE}/Special:Filepath/"):
            img_url = src
            filename = os.path.basename(img_url)
        elif os.path.splitext(src)[1].lower() in MIRRORED_EXTENSIONS:
            filename = "mirror_" + os.path.basename(src)
            img_url = src
        else:
            # Leave other links as is
            continue

        filename = urllib.parse.unquote(filename)
        local_dir = os.path.join(OUTPUT_DIR, "uploads")
        os.makedirs(local_dir, exist_ok=True)
        local_path = os.path.join(local_dir, filename)

        if os.path.exists(local_path + ".failed"):
            print(f"Skipping previously failed file: {img_url}")
            continue

        # Download if not already present
        if not os.path.exists(local_path):
            try:
                print(f"Downloading {img_url}...")

                # Use a browser-like User-Agent to avoid some pages (like banggood) blocking the request
                headers = {
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36"
                }

                resp = requests.get(img_url, stream=True, headers=headers)
                if resp.status_code == 200:
                    with open(local_path, "wb") as f:
                        for chunk in resp.iter_content(1024):
                            f.write(chunk)
                    print(f"\tDownloaded {img_url} -> {local_path}")
                else:
                    raise Exception(
                        f"Failed to download {img_url} (status {resp.status_code})"
                    )
            except Exception as e:
                print(f"Error downloading {img_url}: {e}")
                with open(local_path + ".failed", "w", encoding="utf-8") as f:
                    f.write(f"Error downloading image: {img_url}. {e}")
                continue

        # Replace src with local url
        element.set(
            "href" if element.tag == "a" else "src", f"/uploads/{filename}"
        )
        if element.tag == "img":
            # Remove all other attributes except 'src', 'width', 'height', 'alt', and 'title'
            for attr in list(element.attrib.keys()):
                if attr not in ("src", "width", "height", "alt", "title"):
                    del element.attrib[attr]


def convert_galleries_to_imgs(tree):
    GALLERY_MODE = "table"  # Or "flat"

    # Convert gallery nodes to flat img tags
    galleries = tree.xpath('.//ul[contains(@class, "gallery")]')
    for gallery in galleries:
        img_tags = []
        for li in gallery.xpath('.//li[contains(@class, "gallerybox")]'):
            img = li.xpath(".//img")
            if img:
                img_tags.append(img[0])

        if GALLERY_MODE == "table":
            # Replace gallery with a table of images (up to 3 per row)
            parent = gallery.getparent()
            if parent is not None:
                table = html.Element("table")
                row = None
                for i, img in enumerate(img_tags):
                    if i % 3 == 0:
                        row = html.Element("tr")
                        table.append(row)

                    # Set height to 200, preserve aspect ratio, assuming width and height are already set
                    orig_width = int(img.attrib.get("width", "0"))
                    orig_height = int(img.attrib.get("height", "0"))
                    DESIRED_CELL_HEIGHT = 150
                    if orig_width > 0 and orig_height > 0:
                        new_height = DESIRED_CELL_HEIGHT
                        new_width = int(round(orig_width * (new_height / orig_height)))
                        img.set("width", str(new_width))
                        img.set("height", str(new_height))
                    else:
                        img.set("height", str(DESIRED_CELL_HEIGHT))
                        img.set("width", str(DESIRED_CELL_HEIGHT))

                    td = html.Element("td")
                    td.append(img)
                    row.append(td)
                parent.insert(parent.index(gallery), table)
                parent.remove(gallery)
        else:
            # Replace gallery with flat img tags
            parent = gallery.getparent()
            if parent is not None:
                for img in img_tags:
                    img.tail = "\n"
                    parent.insert(parent.index(gallery), img)
                parent.remove(gallery)


def unwrap_span(span):
    parent = span.getparent()
    assert parent is not None, "Span must have a parent to unwrap"
    # Replace the <span> with all its children and text
    idx = parent.index(span)
    # Insert span's text if present
    if span.text:
        if idx == 0:
            if parent.text:
                parent.text += span.text
            else:
                parent.text = span.text
        else:
            prev = parent[idx - 1]
            if prev.tail:
                prev.tail += span.text
            else:
                prev.tail = span.text
    # Insert all children of span
    for child in list(span):
        parent.insert(idx, child)
        idx += 1
    # Insert span.tail if present
    if span.tail:
        if idx == 0:
            if parent.text:
                parent.text += span.tail
            else:
                parent.text = span.tail
        else:
            node = parent[idx - 1]
            if node.tail:
                node.tail += span.tail
            else:
                node.tail = span.tail

    parent.remove(span)


def convert_figures_to_imgs(tree) -> None:
    # Convert <figure> elements to <img> with caption as alt text
    for figure in tree.xpath(".//figure"):
        img = figure.find(".//img")
        caption_elem = figure.find(".//figcaption")
        if img is not None:
            # Set alt text from caption if present (text_content flattens any
            # inline markup like <b>/<i> inside the caption)
            if caption_elem is not None and caption_elem.text_content().strip():
                img.set("alt", caption_elem.text_content().strip())
            # Replace <figure> with <img>
            parent = figure.getparent()
            if parent is not None:
                parent.insert(parent.index(figure), img)
                parent.remove(figure)


def normalize_heading_levels(tree):
    # Find all headings (h1-h6)
    headings = tree.xpath(".//h1 | .//h2 | .//h3 | .//h4 | .//h5 | .//h6")
    if not headings:
        return

    # Find the lowest heading level
    min_level = min(int(h.tag[1]) for h in headings)

    # Raise all headings so that the lowest is 1
    for h in headings:
        old_level = int(h.tag[1])
        new_level = old_level - min_level + 1
        new_level = min(max(new_level, 1), 6)  # Clamp between 1 and 6
        h.tag = f"h{new_level}"


# Convert CSS color names to hex values if present
def css_color_to_hex(color):
    color = color.strip().lower()
    if color == "yellow" or color.lower() == "#ffff00":
        # Of the valid colors, the greenish color is actually perceptually closer than the yellowish color.
        # Which is annoying. So convert yellow to a yellowish color even though it's perceptually not as close.
        return "#FDEA9B"
    try:
        # Try named color
        return webcolors.name_to_hex(color)
    except ValueError:
        pass
    try:
        # Try rgb/rgba/hsl/hex
        return webcolors.normalize_hex(color)
    except ValueError:
        pass
    return color  # fallback


def unwrap_single_cell_notice_tables(nodes):
    """Replace tables that contain only a single cell, and that cell is a container_notice. Replace the table with just the container_notice node."""
    result = []
    for node in nodes:
        if (
            node.get("type") == "table"
            and len(node.get("content", [])) == 1
            and node["content"][0].get("type") == "tr"
            and len(node["content"][0].get("content", [])) == 1
            and node["content"][0]["content"][0].get("type") in ("td", "th")
        ):
            cell = node["content"][0]["content"][0]
            cell_content = cell.get("content", [])
            if (
                len(cell_content) == 1
                and isinstance(cell_content[0], dict)
                and cell_content[0].get("type") == "container_notice"
            ):
                # Replace table with the container_notice node
                result.append(cell_content[0])
                continue
        # Recursively process children if present
        if "content" in node and isinstance(node["content"], list):
            node = dict(node)
            node["content"] = unwrap_single_cell_notice_tables(node["content"])
        result.append(node)
    return result


def color_marks(node):
    if isinstance(node, str):
        return []

    if node.tag in ["table"]:
        return []

    # Check for marks in style attribute
    marks = []
    style = node.attrib.get("style", "")
    if "text-decoration: underline" in style:
        marks.append({"type": "underline"})  # Correct?
    if "text-decoration: line-through" in style:
        marks.append({"type": "strikethrough"})  # Correct?
    # Check for color or background-color in style
    bgcolor_match = re.search(
        r"(?:background-color|color|background):\s*([^;]+)", style
    )

    if bgcolor_match:
        orig = css_color_to_hex(bgcolor_match.group(1).strip())
        rgb = _hex_to_rgb01(orig)
        if rgb is not None:
            r, g, b = rgb
            # Skip near-neutral light backgrounds (e.g. #f5faff infobox
            # framing): they are layout decoration, not semantic color.
            if max(r, g, b) - min(r, g, b) >= 0.08 or max(r, g, b) < 0.85:
                color = nearest_valid_color(orig)
                # "original" is consumed by wrap_highlight_paragraphs to pick a
                # notice style and stripped before export.
                marks.append(
                    {"type": "highlight", "attrs": {"color": color}, "original": orig}
                )

    return marks


def _hex_to_rgb01(hex_color) -> Optional[Tuple[float, float, float]]:
    if not isinstance(hex_color, str):
        return None
    m = re.fullmatch(r"#?([0-9a-fA-F]{6})", hex_color.strip())
    if not m:
        return None
    v = m.group(1)
    return tuple(int(v[i : i + 2], 16) / 255 for i in (0, 2, 4))


def notice_style_for_color(hex_color) -> str:
    """Pick a container_notice style from a CSS color: green means ok/success,
    yellow/orange means caution/tip, red means forbidden/warning."""
    rgb = _hex_to_rgb01(hex_color)
    if rgb is None:
        return "warning"
    h, _l, _s = colorsys.rgb_to_hls(*rgb)
    deg = h * 360
    if 70 <= deg <= 170:
        return "success"
    if 20 <= deg < 70:
        return "tip"
    if 170 < deg <= 260:
        return "info"
    return "warning"


# Remove all None children from nodes recursively
def remove_none_children(nodes):
    if not isinstance(nodes, list):
        return nodes
    cleaned = []
    for node in nodes:
        if node is None:
            continue
        if (
            isinstance(node, dict)
            and "content" in node
            and isinstance(node["content"], list)
        ):
            node = dict(node)
            node["content"] = remove_none_children(node["content"])
        cleaned.append(node)
    return cleaned


def fill_empty_block_elements(nodes):
    """Ensure all block elements have at least one paragraph with an empty text node if their content is empty"""
    for node in nodes:
        if node.get("type") in ("td", "li", "header") and (
            not node.get("content") or len(node["content"]) == 0
        ):
            node["content"] = [{"type": "paragraph"}]
        elif "content" in node:
            fill_empty_block_elements(node["content"])


def replace_nostalgia_template(content):
    """Find all headings with the content "Nostalgi" and replace both the heading and the following paragraph"""
    idx = 0
    found = False
    while idx < len(content):
        node = content[idx]
        if (
            isinstance(node, dict)
            and node.get("type") == "heading"
            and node.get("content")
            and len(node["content"]) == 1
            and node["content"][0].get("type") == "text"
            and node["content"][0].get("text", "").strip() == "Nostalgi"
        ):
            # Assert that the next element is a paragraph
            if idx + 1 >= len(content):
                raise AssertionError(
                    "Heading 'Nostalgi' is not followed by a paragraph"
                )
            next_elem = content[idx + 1]
            if not (
                isinstance(next_elem, dict) and next_elem.get("type") == "paragraph"
            ):
                raise AssertionError(
                    f"Heading 'Nostalgi' is not followed by a paragraph, found {next_elem.get('type') if isinstance(next_elem, dict) else type(next_elem)}"
                )

            assert content[idx - 1]["type"] == "hr"
            assert content[idx + 2]["type"] == "hr"

            # Replace both with a single container_notice
            notice = {
                "type": "container_notice",
                "attrs": {"style": "warning"},
                "content": [
                    {
                        "type": "text",
                        "text": "This page is no longer relevant, it is only kept for nostalgic reasons",
                    }
                ],
            }
            found = True
            # Remove the heading and paragraph, insert notice
            content[idx - 1 : idx + 3] = [notice]
            # Do not increment idx, in case there are multiple in a row
        else:
            idx += 1

    return found


def wrap_text_and_image_nodes(nodes, parent_type=None):
    wrapped = []
    buffer = []

    def flush_buffer():
        if buffer:
            wrapped.append({"type": "paragraph", "content": buffer[:]})
            buffer.clear()

    for node in nodes:
        if (
            isinstance(node, dict)
            and node.get("type") in ("text", "image")
            and parent_type not in ("heading", "paragraph", "code_fence")
        ):
            buffer.append(node)
        elif (
            isinstance(node, dict)
            and "content" in node
            and isinstance(node["content"], list)
        ):
            # Flush buffer before handling non-text/image node
            flush_buffer()
            node_copy = dict(node)
            node_copy["content"] = wrap_text_and_image_nodes(
                node["content"], parent_type=node.get("type")
            )
            wrapped.append(node_copy)
        else:
            flush_buffer()
            wrapped.append(node)
    flush_buffer()
    return wrapped


def unwrap_embed_paragraphs(nodes):
    """Replace paragraphs that only contain a single embed node with the embed node itself"""
    result = []
    for node in nodes:
        if (
            isinstance(node, dict)
            and node.get("type") == "paragraph"
            and isinstance(node.get("content"), list)
            and len(node["content"]) == 1
            and isinstance(node["content"][0], dict)
            and node["content"][0].get("type") == "embed"
        ):
            result.append(node["content"][0])
        else:
            # Recursively process children if present
            if (
                isinstance(node, dict)
                and "content" in node
                and isinstance(node["content"], list)
            ):
                node = dict(node)
                node["content"] = unwrap_embed_paragraphs(node["content"])
            result.append(node)
    return result


def collapse_adjacent_text_nodes(nodes):
    """Collapse adjacent text nodes with no marks into the same text node"""
    if not nodes or not isinstance(nodes, list):
        return nodes
    collapsed = []
    prev = None
    for node in nodes:
        if (
            isinstance(node, dict)
            and node.get("type") == "text"
            and not node.get("marks")
            and prev
            and isinstance(prev, dict)
            and prev.get("type") == "text"
            and not prev.get("marks")
        ):
            prev["text"] += node["text"]
        else:
            collapsed.append(node)
            prev = node
    # Recursively apply to children
    for node in collapsed:
        if (
            isinstance(node, dict)
            and "content" in node
            and isinstance(node["content"], list)
        ):
            node["content"] = collapse_adjacent_text_nodes(node["content"])
    return collapsed


def wrap_highlight_paragraphs(nodes):
    """Wrap paragraphs - where all child text nodes are highlighted the same - in a container_notice node"""
    result = []
    for node in nodes:
        # Check for paragraph where all text nodes have the same highlight mark
        if (
            node.get("type") == "paragraph"
            and "content" in node
            and all(
                c.get("type") == "text"
                and "marks" in c
                and any(m.get("type") == "highlight" for m in c["marks"])
                for c in node["content"]
                if c.get("type") == "text"
            )
            and any(c.get("type") == "text" for c in node["content"])
        ):
            # Get the highlight color from the first text node
            first_marks = next(
                c["marks"] for c in node["content"] if c.get("type") == "text"
            )
            highlight_mark = next(
                (m for m in first_marks if m.get("type") == "highlight"), None
            )
            if highlight_mark is not None:
                # Ensure all text nodes have the same highlight color
                same_highlight = all(
                    any(
                        m.get("type") == "highlight"
                        and m.get("attrs", {}).get("color")
                        == highlight_mark.get("attrs", {}).get("color")
                        for m in c["marks"]
                    )
                    for c in node["content"]
                    if c.get("type") == "text"
                )
                if same_highlight:
                    # Prefer the original CSS color (pre-preset-snapping) for
                    # classifying the notice style, so pale reds/yellows keep
                    # their meaning.
                    color = highlight_mark.get("original") or highlight_mark.get(
                        "attrs", {}
                    ).get("color")
                    notice_type = notice_style_for_color(color)

                    # Remove the highlight mark from all text nodes
                    new_content = []
                    for c in node["content"]:
                        if c.get("type") == "text" and "marks" in c:
                            new_marks = [
                                m for m in c["marks"] if m.get("type") != "highlight"
                            ]
                            text_node = dict(c)
                            if new_marks:
                                text_node["marks"] = new_marks
                            else:
                                text_node.pop("marks", None)
                            new_content.append(text_node)
                        else:
                            new_content.append(c)
                    # Wrap in container_notice with style "warning"
                    result.append(
                        {
                            "type": "container_notice",
                            "attrs": {"style": notice_type},
                            "content": [{"type": "paragraph", "content": new_content}],
                        }
                    )
                    continue

        # Recursively process children
        if "content" in node and node["content"] is not None:
            node = dict(node)
            node["content"] = wrap_highlight_paragraphs(node["content"])
        result.append(node)
    return result


def remove_empty_text_nodes(nodes):
    """Drop text nodes with empty strings; ProseMirror rejects them. (The
    per-node guard misses empties that acquire marks during processing.)"""
    if not isinstance(nodes, list):
        return nodes
    result = []
    for node in nodes:
        if isinstance(node, dict):
            if node.get("type") == "text" and node.get("text", "") == "":
                continue
            if isinstance(node.get("content"), list):
                node["content"] = remove_empty_text_nodes(node["content"])
        result.append(node)
    return result


def strip_highlight_original(nodes):
    """Remove the internal "original" key from highlight marks before export."""
    if not isinstance(nodes, list):
        return nodes
    for node in nodes:
        if not isinstance(node, dict):
            continue
        for mark in node.get("marks", []) or []:
            mark.pop("original", None)
        if isinstance(node.get("content"), list):
            strip_highlight_original(node["content"])
    return nodes


# Block-level node types that must not appear inside a paragraph. Notably
# patch_attachments turns file-link text nodes into attachment blocks in place.
BLOCK_TYPES_IN_PARAGRAPH = {
    "attachment",
    "code_fence",
    "embed",
    "table",
    "blockquote",
    "bullet_list",
    "ordered_list",
    "checkbox_list",
    "container_notice",
    "hr",
}


def split_blocks_out_of_paragraphs(nodes):
    """Hoist block nodes out of paragraph content by splitting the paragraph,
    since ProseMirror paragraphs may only contain inline content."""
    if not isinstance(nodes, list):
        return nodes
    result = []
    for node in nodes:
        if isinstance(node, dict) and isinstance(node.get("content"), list):
            node = dict(node)
            node["content"] = split_blocks_out_of_paragraphs(node["content"])

        if (
            isinstance(node, dict)
            and node.get("type") == "paragraph"
            and node.get("content")
            and any(
                c.get("type") in BLOCK_TYPES_IN_PARAGRAPH for c in node["content"]
            )
        ):
            inline_run = []
            for child in node["content"]:
                if child.get("type") in BLOCK_TYPES_IN_PARAGRAPH:
                    if inline_run:
                        result.append({"type": "paragraph", "content": inline_run})
                        inline_run = []
                    result.append(child)
                else:
                    inline_run.append(child)
            if inline_run:
                result.append({"type": "paragraph", "content": inline_run})
        else:
            result.append(node)
    return result


def replace_stub_notice(content):
    """
    Replace container_notice nodes that contain a paragraph with a text node containing
    " Den här artikeln är bara påbörjad. Du kan hjälpa till genom att "
    with a new container_notice of type "tip" and the specified English text.
    """
    result = []
    for node in content:
        replaced = False
        if (
            isinstance(node, dict)
            and node.get("type") == "container_notice"
            and isinstance(node.get("content"), list)
        ):
            for child in node["content"]:
                if (
                    isinstance(child, dict)
                    and child.get("type") == "paragraph"
                    and isinstance(child.get("content"), list)
                ):
                    for text_node in child["content"]:
                        if (
                            isinstance(text_node, dict)
                            and text_node.get("type") == "text"
                            and " Den här artikeln är bara påbörjad. Du kan hjälpa till genom att "
                            in text_node.get("text", "")
                        ):
                            # Replace with new container_notice
                            result.append(
                                {
                                    "type": "container_notice",
                                    "attrs": {"style": "tip"},
                                    "content": [
                                        {
                                            "type": "paragraph",
                                            "content": [
                                                {
                                                    "type": "text",
                                                    "text": "This article is still in its early stages. Feel free to help build it out!",
                                                }
                                            ],
                                        }
                                    ],
                                }
                            )
                            replaced = True
                            break
                    if replaced:
                        break
        if not replaced:
            # Recursively process children if present
            if (
                isinstance(node, dict)
                and "content" in node
                and isinstance(node["content"], list)
            ):
                node = dict(node)
                node["content"] = replace_stub_notice(node["content"])
            result.append(node)
    return result


def split_paragraphs_with_attachments(nodes):
    """Replace paragraphs that contain an attachment node with up to three nodes:
    - a paragraph with the prefix text/content,
    - the attachment node,
    - a paragraph with the suffix text/content.
    The attachment node must be outside the paragraph.
    """
    result = []
    retry = False
    for node in nodes:
        if (
            isinstance(node, dict)
            and node.get("type") == "paragraph"
            and isinstance(node.get("content"), list)
        ):
            content = node["content"]
            # Find the first attachment node
            for idx, child in enumerate(content):
                if isinstance(child, dict) and child.get("type") == "attachment":
                    # Split content into prefix, attachment, suffix
                    prefix = content[:idx]
                    if all(
                        x["type"] == "text" and x["text"].strip() == "" for x in prefix
                    ):
                        prefix = []
                    attachment = child
                    suffix = content[idx + 1 :]
                    if all(
                        x["type"] == "text" and x["text"].strip() == "" for x in suffix
                    ):
                        suffix = []
                    if prefix:
                        result.append({"type": "paragraph", "content": prefix})
                    result.append(attachment)
                    if suffix:
                        result.append({"type": "paragraph", "content": suffix})
                    retry = True
                    break
            else:
                # No attachment found, keep as is
                result.append(node)
        elif (
            isinstance(node, dict)
            and "content" in node
            and isinstance(node["content"], list)
        ):
            node_copy = dict(node)
            node_copy["content"] = split_paragraphs_with_attachments(node["content"])
            result.append(node_copy)
        else:
            result.append(node)

    if retry:
        # If we split any paragraphs, we need to retry to ensure all paragraphs are processed
        # in case there are multiple attachments in a single paragraph.
        return split_paragraphs_with_attachments(result)

    return result


def unwrap_single_attachment_lists(nodes):
    """Replace lists (ordered or bullet) that contain only a single attachment node with the attachment node itself."""
    result = []
    for node in nodes:
        if (
            isinstance(node, dict)
            and node.get("type") in ("ordered_list", "bullet_list")
            and isinstance(node.get("content"), list)
            and len(node["content"]) == 1
            and isinstance(node["content"][0], dict)
            and node["content"][0].get("type") == "list_item"
            and isinstance(node["content"][0].get("content"), list)
            and len(node["content"][0]["content"]) == 1
            and isinstance(node["content"][0]["content"][0], dict)
            and node["content"][0]["content"][0].get("type") == "attachment"
        ):
            # Replace the list with the attachment node
            result.append(node["content"][0]["content"][0])
        else:
            # Recursively process children if present
            if (
                isinstance(node, dict)
                and "content" in node
                and isinstance(node["content"], list)
            ):
                node = dict(node)
                node["content"] = unwrap_single_attachment_lists(node["content"])
            result.append(node)
    return result


def convert_to_outline_json(tree):
    # Visitor pattern for HTML to outline JSON AST
    class Visitor:
        def __init__(self):
            self.marks_stack = []

        def visit(self, node, parent, check_style=True):
            if check_style:
                temp_marks = color_marks(node)
                if temp_marks:
                    # If there are marks, push them onto the stack
                    s = len(self.marks_stack)
                    self.marks_stack.extend(temp_marks)
                    res = self.visit(node, parent, check_style=False)
                    self.marks_stack = self.marks_stack[: -len(temp_marks)]
                    assert len(self.marks_stack) == s, "Marks stack should be restored"
                    return res

            if isinstance(node, str):
                return self.text_node(node)
            if node.tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
                return self.heading(node)
            if node.tag == "p":
                return self.paragraph(node)
            if node.tag == "ul":
                return self.list(node, ordered=False)
            if node.tag == "ol":
                return self.list(node, ordered=True)
            if node.tag == "li":
                return self.list_item(node)
            if node.tag == "blockquote":
                return self.blockquote(node)
            if node.tag == "table":
                table_json = self.table(node)
                # Collapsible wiki tables become collapsed toggle blocks. The
                # toggle's first child is its always-visible summary line.
                if node.attrib.get("data-collapsible") == "true":
                    return {
                        "type": "container_toggle",
                        "content": [
                            {
                                "type": "paragraph",
                                "content": [{"type": "text", "text": "Details"}],
                            },
                            table_json,
                        ],
                    }
                return table_json
            if node.tag == "tr":
                return self.tr(node)
            if node.tag == "td":
                return self.td(node)
            if node.tag == "th":
                return self.th(node)
            if node.tag == "img":
                return self.image(node)
            if node.tag == "video":
                return self.video(node)
            if node.tag == "a":
                return self.link(node)
            if node.tag == "b" or node.tag == "strong" or node.tag == "big":
                return self.mark(node, {"type": "strong"})
            if node.tag == "i" or node.tag == "em":
                return self.mark(node, {"type": "em"})
            if node.tag == "del" or node.tag == "s" or node.tag == "strike":
                return self.mark(node, {"type": "strikethrough"})
            elif node.tag == "u":
                return self.mark(node, {"type": "underline"})
            if node.tag == "pre":
                return self.pre(node)
            if node.tag == "code":
                # Inline <code> becomes a code mark, keeping the sentence
                # intact, unlike block-level <pre> which becomes a code fence.
                return self.mark(node, {"type": "code_inline"})
            if node.tag == "hr":
                return {"type": "hr", "attrs": {"markup": "---"}}
            if node.tag == "br":
                return {
                    "type": "text",
                    "text": "\n",
                    "marks": list(self.marks_stack),
                }  # TODO: Use paragraph instead?
            if node.tag == "span":
                return self.visit_children(node)

            if node.tag == "div":
                # Notices: info/warning
                cls = node.attrib.get("class", "")
                if "notice" in cls or "messagebox" in cls:
                    style = "info"
                    if "warning" in cls:
                        style = "warning"
                    return {
                        "type": "container_notice",
                        "attrs": {"style": style},
                        "content": self.visit_children(node),
                    }
                # Otherwise, flatten
                return self.visit_children(node)
            if node.tag == "iframe":
                src = node.attrib.get("src")
                # Wiki embeds often end with a bare "?" which would end up
                # glued to the video id below.
                src = src.rstrip("?")
                src = src.replace("youtube-nocookie.com", "youtube.com")
                src = src.replace("youtube.com/embed/", "youtube.com/watch?v=")
                src = src.replace("youtu.be/", "youtube.com/watch?v=")
                if src.startswith("//"):
                    src = "https:" + src
                # If the original src had query params, the embed->watch
                # rewrite leaves a second "?"; turn it into "&".
                if src.count("?") > 1:
                    head, _, tail = src.partition("?")
                    src = head + "?" + tail.replace("?", "&")
                if src and (
                    "youtube.com" in src
                    or "youtu.be" in src
                    or "youtube-nocookie.com" in src
                ):
                    return {
                        "type": "embed",
                        "attrs": {
                            "href": src,
                            "width": None,
                            "height": None,
                            # "width": node.attrib.get("width"),
                            # "height": node.attrib.get("height"),
                        },
                    }
            # Default: flatten children
            print(
                "Unknown tag:",
                node.tag,
                "in",
                parent.tag if parent is not None else "root",
            )
            return self.visit_children(node)

        def visit_children(self, node):
            result = []
            if node.text and node.text.strip():
                result.append(self.text_node(node.text))
            for child in node:
                child_json = self.visit(child, parent=node)
                if child_json:
                    if isinstance(child_json, list):
                        result.extend(child_json)
                    else:
                        result.append(child_json)
                if child.tail:
                    result.append(self.text_node(child.tail))
            return result

        def text_node(self, text):
            if text == "":
                return None

            node = {"type": "text", "text": text}
            if self.marks_stack:
                # Remove duplicates, keep the last of every type
                seen = {}
                for m in reversed(self.marks_stack):
                    t = m.get("type")
                    if t not in seen:
                        seen[t] = m
                node["marks"] = list(reversed(list(seen.values())))

            return node

        def pre(self, node):
            lines = node.text.strip().splitlines() if node.text else []
            checklist_pattern = re.compile(r"^\s*\[(.| ?)\]\s*(.*)$", re.IGNORECASE)
            if len(node) == 0 and all(checklist_pattern.match(x) for x in lines):
                # Convert to a checklist instead
                items = []
                for line in lines:
                    m = checklist_pattern.match(line)
                    if not m:
                        assert False, f"Unexpected checklist line format: {line}"
                    checked = m.group(1).lower().strip() != ""
                    label = m.group(2)
                    items.append(
                        {
                            "type": "checkbox_item",
                            "attrs": {"checked": checked},
                            "content": [{"type": "text", "text": label}],
                        }
                    )
                return {"type": "checkbox_list", "content": items}

            return {
                "type": "code_fence",
                "attrs": {
                    "language": "none",
                },
                "content": [self.text_node(node.text_content())],
            }

        def heading(self, node):
            # Outline only supports heading levels 1-4
            level = min(int(node.tag[1]), 4)
            content = self.visit_children(node)
            return {"type": "heading", "attrs": {"level": level}, "content": content}

        def paragraph(self, node):
            content = self.visit_children(node)
            return (
                {"type": "paragraph", "content": content}
                if content
                else {"type": "paragraph"}
            )

        def list(self, node, ordered):
            items = []
            for child in node:
                if child.tag == "li":
                    item = self.visit(child, parent=node)
                    if item:
                        items.append(item)
            res = {
                "type": "ordered_list" if ordered else "bullet_list",
                "content": items,
            }
            if ordered:
                start = node.attrib.get("start", "")
                order = int(start) if start.lstrip("-").isdigit() else 1
                # Outline supports number/lower-alpha/upper-alpha list styles.
                # Roman numerals (type=i/I) have no equivalent and fall back
                # to numbers.
                list_type = node.attrib.get("type", "")
                style = node.attrib.get("style", "")
                style_match = re.search(r"list-style-type\s*:\s*([\w-]+)", style)
                list_style_type = style_match.group(1) if style_match else ""
                if list_type == "a" or list_style_type in ("lower-alpha", "lower-latin"):
                    list_style = "lower-alpha"
                elif list_type == "A" or list_style_type in ("upper-alpha", "upper-latin"):
                    list_style = "upper-alpha"
                else:
                    list_style = "number"
                res["attrs"] = {"order": order, "listStyle": list_style}
            return res

        def list_item(self, node):
            content = self.visit_children(node)
            return {"type": "list_item", "content": content}

        def blockquote(self, node):
            content = self.visit_children(node)
            return {"type": "blockquote", "content": content}

        def table(self, node):
            rows = []
            for tr in node.xpath("./tr"):
                row = self.visit(tr, parent=node)
                if row:
                    rows.append(row)
            # Also handle <tbody>/<thead>
            for part in node.xpath("./tbody|./thead"):
                for tr in part.xpath("./tr"):
                    row = self.visit(tr, parent=node)
                    if row:
                        rows.append(row)
            return {"type": "table", "attrs": {"layout": None}, "content": rows}

        def tr(self, node):
            cells = []
            for cell in node:
                if cell.tag in ["td", "th"]:
                    cell_json = self.visit(cell, parent=node)
                    if cell_json:
                        cells.append(cell_json)
            return {"type": "tr", "content": cells}

        def td(self, node):
            colspan = int(node.attrib.get("colspan", 1))
            rowspan = int(node.attrib.get("rowspan", 1))
            align = node.attrib.get("align") or ""
            content = self.visit_children(node)
            return {
                "type": "td",
                "attrs": {
                    "colspan": colspan,
                    "rowspan": rowspan,
                    "alignment": align,
                    "colwidth": None,
                },
                "content": content,
            }

        def th(self, node):
            colspan = int(node.attrib.get("colspan", 1))
            rowspan = int(node.attrib.get("rowspan", 1))
            align = node.attrib.get("align") or ""
            colwidth = (
                node.attrib.get("colwidth") if "colwidth" in node.attrib else None
            )
            content = self.visit_children(node)
            attrs = {"colspan": colspan, "rowspan": rowspan, "alignment": align}
            if colwidth is not None:
                attrs["colwidth"] = colwidth
            return {"type": "th", "attrs": attrs, "content": content}

        def image(self, node):
            src = node.attrib.get("src")
            width = (
                int(node.attrib.get("width"))
                if node.attrib.get("width", "").isdigit()
                else None
            )
            height = (
                int(node.attrib.get("height"))
                if node.attrib.get("height", "").isdigit()
                else None
            )
            alt = node.attrib.get("alt")
            title = node.attrib.get("title") or ""
            return {
                "type": "image",
                "attrs": {
                    "src": src,
                    "width": width,
                    "height": height,
                    "alt": alt,
                    "layoutClass": None,
                    "title": title,
                },
            }

        def video(self, node):
            # MediaWiki (TimedMediaHandler) renders videos as <video> with the
            # source on the element itself or on child <source> elements.
            src = node.attrib.get("src")
            if not src:
                for source in node.xpath("./source[@src]"):
                    src = source.attrib.get("src")
                    break
            if not src:
                return self.visit_children(node)
            width = (
                int(node.attrib.get("width"))
                if node.attrib.get("width", "").isdigit()
                else None
            )
            height = (
                int(node.attrib.get("height"))
                if node.attrib.get("height", "").isdigit()
                else None
            )
            title = node.attrib.get("title") or os.path.basename(
                urllib.parse.unquote(src)
            )
            return {
                "type": "video",
                "attrs": {
                    "id": None,
                    "src": src,
                    "width": width,
                    "height": height,
                    "title": title,
                },
            }

        def link(self, node):
            href = node.attrib.get("href")
            title = node.attrib.get("title")
            content = self.visit_children(node)
            return {
                "type": "text",
                "marks": [{"type": "link", "attrs": {"href": href, "title": title}}]
                + self.marks_stack,
                "text": (
                    "".join(
                        [c["text"] for c in content if c and c.get("type") == "text"]
                    )
                    if content
                    else ""
                ),
            }

        def mark(self, node, mark):
            assert isinstance(mark, dict), "Mark must be a dictionary"
            s = len(self.marks_stack)
            self.marks_stack.append(mark)
            content = self.visit_children(node)
            self.marks_stack.pop()
            assert (
                len(self.marks_stack) == s
            ), "Marks stack should be restored after visit"
            return content

    visitor = Visitor()
    content = []
    icon = None
    for child in tree:
        node = visitor.visit(child, parent=tree)
        if node:
            if isinstance(node, list):
                content.extend(node)
            else:
                content.append(node)

    content = remove_none_children(content)

    fill_empty_block_elements(content)

    if replace_nostalgia_template(content):
        icon = "🪦"

    content = wrap_text_and_image_nodes(content)

    content = unwrap_embed_paragraphs(content)

    content = collapse_adjacent_text_nodes(content)

    content = wrap_highlight_paragraphs(content)

    content = unwrap_single_cell_notice_tables(content)

    content = replace_stub_notice(content)

    content = split_blocks_out_of_paragraphs(content)
    content = strip_highlight_original(content)

    # Left trim the first text node in the first paragraph
    if content:
        node = content[0]
        if (
            isinstance(node, dict)
            and node.get("type") == "paragraph"
            and "content" in node
        ):
            for child in node["content"]:
                if isinstance(child, dict) and child.get("type") == "text":
                    child["text"] = child["text"].lstrip()
                    break

    # Must run last: the left-trim above can reduce a whitespace-only text
    # node to an empty string, which ProseMirror rejects.
    content = remove_empty_text_nodes(content)

    # Outline's importer cannot handle documents with empty content (the
    # ProseMirror round-trip drops the empty content array entirely), so
    # represent an empty page as a single empty paragraph like Outline does.
    if not content:
        content = [{"type": "paragraph"}]

    return ({"type": "doc", "content": content}, icon)


# Get the metadata of the first and last revision: user, created, updated
# Also fetch the categories the page is part of
def get_page_revision_metadata(page):
    print("Fetching first and last revision metadata for:", page)
    api_url = f"{BASE}/api.php"
    params = {
        "action": "query",
        "prop": "revisions|categories",
        "titles": page,
        "rvlimit": 1,
        "rvdir": "newer",
        "rvprop": "user|timestamp",
        "cllimit": "max",
        "format": "json",
    }
    try:
        resp = requests.get(api_url, params=params)
        resp.raise_for_status()
        data = resp.json()
        pages_data = data.get("query", {}).get("pages", {})
        first_user = None
        first_timestamp = None
        categories = []
        for pageid, pagedata in pages_data.items():
            revs = pagedata.get("revisions", [])
            if revs:
                first_user = revs[0].get("user")
                first_timestamp = revs[0].get("timestamp")
            cats = pagedata.get("categories", [])
            categories = [cat.get("title", "") for cat in cats]
            break
    except Exception as e:
        print(f"Failed to fetch first revision metadata for {page}: {e}")
        first_user = None
        first_timestamp = None
        categories = []

    # Get last revision
    params["rvdir"] = "older"
    try:
        resp = requests.get(api_url, params=params)
        resp.raise_for_status()
        data = resp.json()
        pages_data = data.get("query", {}).get("pages", {})
        last_user = None
        last_timestamp = None
        for pageid, pagedata in pages_data.items():
            revs = pagedata.get("revisions", [])
            if revs:
                last_user = revs[0].get("user")
                last_timestamp = revs[0].get("timestamp")
                break
    except Exception as e:
        print(f"Failed to fetch last revision metadata for {page}: {e}")
        last_user = None
        last_timestamp = None

    return {
        "first_user": first_user,
        "first_timestamp": first_timestamp,
        "last_user": last_user,
        "last_timestamp": last_timestamp,
        "categories": [
            x.replace("Kategori:", "").replace("Category:", "") for x in categories
        ],
    }


def get_page_revision_metadata_cached(page):
    cache_file = os.path.join(CACHE_DIR, f"{page.replace('/', '_')}_first_user.txt")
    if os.path.exists(cache_file):
        with open(cache_file, "r", encoding="utf-8") as f:
            return json.loads(f.read())
    user = get_page_revision_metadata(page)
    with open(cache_file, "w", encoding="utf-8") as f:
        f.write(json.dumps(user))
    return user


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Convert MediaWiki HTML pages.")
    parser.add_argument("--filter", type=str, help="Regex to filter page names")
    parser.add_argument(
        "--no-attachments", action="store_true", help="Skip attachments"
    )
    args = parser.parse_args()

    page_filter = None
    if args.filter:
        page_filter = re.compile(args.filter)

    export = OutlineExport(
        collection=OutlineCollection(
            id=str(uuid.uuid4()),
            urlId="um8KoiqAPm",
            name="Makerspace Wiki",
            data={"type": "doc", "content": [{"type": "paragraph"}]},
            icon=None,
            permission=None,
            commenting=False,
            sharing=False,
            documentStructure=[],
            color=None,
        ),
        documents={},
        attachments={},
    )
    include_attachments = not args.no_attachments

    attachments = {}
    included_attachment_paths: list[Tuple[str, str]] = []
    page_heading_id_mapping: dict[str, dict[str, str]] = {}
    redirects = {}

    for page in pages:
        if page_filter and not page_filter.search(page):
            continue

        html_content = get_inner_html(page)
        if not html_content:
            continue

        # Replace Swedish mediawiki syntax with English equivalents
        html_content = re.sub(r"\bFil:", "File:", html_content, flags=re.IGNORECASE)
        html_content = re.sub(r"\bMedia:", "File:", html_content, flags=re.IGNORECASE)
        html_content = re.sub(
            r"\bKategori:", "Category:", html_content, flags=re.IGNORECASE
        )
        html_content = re.sub(
            r"\bMall:", "Template:", html_content, flags=re.IGNORECASE
        )
        html_content = re.sub(r"\bBild:", "Image:", html_content, flags=re.IGNORECASE)

        html_content = minify_html.minify(
            html_content,
            minify_js=False,
            remove_processing_instructions=True,
            keep_closing_tags=True,
            keep_html_and_head_opening_tags=True,
        )

        html_content = re.sub(
            r"<sup>(.*?)</sup>", sup_to_unicode, html_content, flags=re.DOTALL
        )

        tree = html.fromstring(html_content)

        # Find a link that is the child of an element with the class redirectMsg (assume redirectMsg is the only class)
        redirect_link = None
        redirect_elem = tree.xpath('.//*[@class="redirectMsg"]')
        if redirect_elem:
            # Find the first <a> child within the redirectMsg element
            link = redirect_elem[0].xpath(".//a")
            assert len(link) == 1, f"Expected exactly one redirect link in {page}"

            redirect_href = urllib.parse.unquote(link[0].get("href").lstrip("/"))

            # Alternative url pattern used sometimes
            redirect_href = re.sub(
                r"index.php\?title=([^\&]+)(?:&redirect=no)?", r"\1", redirect_href
            )

            # Some redirect pages carry real legacy content below the redirect
            # message (e.g. an old machine page turned into a redirect after
            # the machine was sold). Import those as regular pages instead of
            # dropping the content: strip the redirect message and fall
            # through to normal conversion.
            redirect_msg = redirect_elem[0]
            redirect_msg.getparent().remove(redirect_msg)
            remaining_text = tree.text_content().strip()
            if len(remaining_text) > 150:
                print(f"Redirect page has its own content, importing as page: {page}")
            else:
                if "Användare:" in redirect_href or "User:" in redirect_href:
                    continue  # Skip user pages

                if redirect_href not in [mediawiki_slugify(x) for x in pages]:
                    # Redirect to a page that does not exist (redlink) or is
                    # outside the imported namespaces: skip it.
                    print(
                        f"Warning: skipping redirect {page!r} -> {redirect_href!r} (target not found)"
                    )
                    continue

                redirects[mediawiki_slugify(page)] = redirect_href
                continue

        # Find all URLs in text nodes (not inside <a>) and wrap them in <a> tags
        # Improved URL regex: matches http(s) and avoids trailing punctuation, does not assume www
        url_regex = re.compile(
            r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)",
        )

        def wrap_implicit_urls_in_text(parent):
            if parent.tag == "a":
                return

            # Helper to wrap URLs in a string with <a> tags
            def wrap_urls(text):
                result = []
                last = 0
                for m in url_regex.finditer(text):
                    end = m.end()
                    while text[end - 1] in ".,;:!":
                        end -= 1  # Remove trailing punctuation

                    if m.start() > last:
                        result.append(text[last : m.start()])
                    href = text[m.start() : end]
                    a = html.Element("a", href=href)
                    a.text = href
                    result.append(a)
                    last = end
                if last < len(text):
                    result.append(text[last:])
                return result

            def patch_text(new_text):
                text = None
                children = []
                for part in new_text:
                    if isinstance(part, str):
                        if len(children) == 0:
                            assert text is None
                            text = part
                        else:
                            prev = children[-1]
                            if prev.tail:
                                prev.tail += part
                            else:
                                prev.tail = part
                    else:
                        children.append(part)

                return text, children

            # Recursively process children
            for elem in list(parent):
                wrap_implicit_urls_in_text(elem)
                if elem.tail:
                    new_tail = wrap_urls(elem.tail)
                    if len(new_tail) > 1:
                        elem.tail, children = patch_text(new_tail)
                        idx = parent.index(elem) + 1
                        for i, child in enumerate(children):
                            parent.insert(idx + i, child)

            # Process this node's text if not <a>
            if parent.text:
                new_text = wrap_urls(parent.text)
                if len(new_text) > 1:
                    parent.text, children = patch_text(new_text)
                    for i, child in enumerate(children):
                        parent.insert(i, child)

        wrap_implicit_urls_in_text(tree)

        tree = tree.find('.//div[@class="mw-parser-output"]')
        assert tree is not None, f"Could not find mw-parser-output in {page}"

        # Remove all HTML comments (<!-- ... -->)
        comments = tree.xpath("//comment()")
        for c in comments:
            parent = c.getparent()
            if parent is not None:
                parent.remove(c)

        # Remove all <meta> elements
        for meta in tree.xpath(".//meta"):
            parent = meta.getparent()
            if parent is not None:
                parent.remove(meta)

        # Find all <a> tags that have an <img> as their only content and replace them with just the <img>
        for a in tree.xpath(".//a"):
            # Check if <a> has exactly one child and it's an <img>, and no text
            if (
                len(a) == 1
                and a[0].tag == "img"
                and (a.text is None or a.text.strip() == "")
                and (a[0].tail is None or a[0].tail.strip() == "")
            ):
                img = a[0]
                parent = a.getparent()
                if parent is not None:
                    idx = parent.index(a)
                    parent.insert(idx, img)
                    parent.remove(a)

        if include_attachments:
            process_images(tree)

        convert_galleries_to_imgs(tree)

        # Preserve collapsibility before classes are stripped below
        for elem in tree.xpath('//table[contains(@class, "mw-collapsible")]'):
            elem.set("data-collapsible", "true")

        # Remove all classes from all elements
        for elem in tree.xpath("//*[@class or @id]"):
            elem.attrib.pop("class", None)
            # elem.attrib.pop("id", None)

        heading_id_mapping: dict[str, str] = {}

        for heading in tree.xpath(".//h1 | .//h2 | .//h3 | .//h4 | .//h5 | .//h6"):
            # Find ids on the heading element and all ids on any child elements
            ids = []
            if heading.get("id"):
                ids.append(heading.get("id"))
            for elem in heading.iterdescendants():
                if elem.get("id"):
                    ids.append(elem.get("id"))
                    elem.attrib.pop("id", None)

            if ids:
                text = heading.text_content().strip()
                if not text:
                    print(
                        "Warning: Heading with ID but no text:",
                        ids,
                        " on page ",
                        page,
                    )
                    continue

                # This is exactly what Outline does, so we do it too
                new_id = heading_to_slug(text)
                for id in ids:
                    heading_id_mapping[id.lower()] = new_id
                heading.set("id", new_id)

        # Remove all <span> elements that are empty (no text and no child elements)
        for span in tree.xpath(".//span"):
            if (span.text is None or span.text.strip() == "") and len(span) == 0:
                parent = span.getparent()
                if parent is not None:
                    parent.remove(span)

        # Find all headings that contain a single span. Replace the heading contents with the span's contents.
        for heading in tree.xpath(".//h1 | .//h2 | .//h3 | .//h4 | .//h5 | .//h6"):
            # Only one child, and it's a span, and no text before/after
            if (
                len(heading) == 1
                and heading[0].tag == "span"
                and (heading.text is None or heading.text.strip() == "")
                and (heading[0].tail is None or heading[0].tail.strip() == "")
            ):
                span = heading[0]
                # Remove the span from heading
                heading.remove(span)
                # Clear heading text
                heading.text = None
                # Move all children and text from span to heading
                if span.text:
                    heading.text = span.text
                for child in list(span):
                    span.remove(child)
                    heading.append(child)
                # Set heading id to span id if present
                if span.get("id"):
                    heading.set("id", span.get("id"))

        # Replace <span> elements that only wrap text with their text content,
        # but only if the span has no style, id, or other classes
        for span in tree.xpath(".//span"):
            if (
                span.text
                and not span.attrib.get("style")
                and not span.attrib.get("id")
                and not span.attrib.get("class")
            ):
                unwrap_span(span)

        # Remove all <p> elements that only contain whitespace
        for p in tree.xpath(".//p"):
            # Check if the <p> has no children and its text is only whitespace (or empty)
            if p.text_content().strip() == "" and all(c.tag == "br" for c in p):
                parent = p.getparent()
                if parent is not None:
                    parent.remove(p)

        # Trim ":" from the end of all headings
        for heading in tree.xpath(".//h1 | .//h2 | .//h3 | .//h4 | .//h5 | .//h6"):
            if heading.text and heading.text.strip().endswith(":"):
                heading.text = heading.text.rstrip(":").rstrip()

            if heading.text and heading.text.strip().startswith("-"):
                heading.text = heading.text.lstrip("-").lstrip()

        convert_figures_to_imgs(tree)
        normalize_heading_levels(tree)

        # Remove all <tbody> and <thead> elements, replacing them with their children
        for tag in tree.xpath(".//tbody | .//thead"):
            parent = tag.getparent()
            if parent is not None:
                idx = parent.index(tag)
                # Insert all children of tag into parent at the tag's position
                for child in list(tag):
                    parent.insert(idx, child)
                    idx += 1
                parent.remove(tag)

        # Convert <caption> in tables to a header row (assuming 1 column)
        for table in tree.xpath(".//table"):
            caption = table.find("caption")
            if caption is not None:
                if caption.text_content().strip() == "":
                    # If caption is empty, remove it
                    table.remove(caption)
                    continue

                # Create <thead> with <tr><th>
                tr = html.Element("tr")
                th = html.Element("th")
                # Move all children of caption into th
                for node in list(caption):
                    caption.remove(node)
                    th.append(node)
                # Add any text in caption before children
                if caption.text and caption.text.strip():
                    th.text = caption.text.strip()
                # Add any tail text after children
                if caption.tail and caption.tail.strip():
                    if th.tail:
                        th.tail += caption.tail.strip()
                    else:
                        th.tail = caption.tail.strip()
                tr.append(th)
                table.remove(caption)
                table.insert(0, tr)

        # Convert 1-column tables of the form <b>X:</b>Y to 2-column tables X | Y
        for table in tree.xpath(".//table"):
            # Find all rows
            rows = table.xpath(".//tr")

            converted = True
            new_rows = []
            for row in rows:
                # Allow both <td> and <th> as the single cell
                cells = row.xpath("./td|./th")

                if len(cells) != 1:
                    converted = False
                    break

                cell = cells[0]
                if len(cell) >= 1:
                    if cell[0].tag == "b":
                        # Check for <b>X:</b>Y pattern
                        b = cell[0]
                        if b.text.endswith(":") or b.tail:
                            x = b.text.rstrip(":")  # Remove trailing colon
                            assert len(b) == 0

                            # Get Y: text after <b>...</b>
                            y = (b.tail or "").strip()
                            # If there are more elements after <b>, append their text to y
                            for extra in list(cell)[1:]:
                                y += html.tostring(
                                    extra, encoding="unicode", with_tail=True
                                )

                            td1 = html.Element(cell.tag)
                            td1.text = x
                            td2 = html.fromstring(f"<{cell.tag}>{y}</{cell.tag}>")
                            tr = html.Element("tr")
                            tr.append(td1)
                            tr.append(td2)
                            new_rows.append(tr)
                        else:
                            # Could just be a single header row that's unimportant
                            if (
                                "Snabbfakta" in b.text
                                or "Innehåll" in b.text
                                or (b.text.startswith("“") and b.text.endswith("’’"))
                                or row == rows[0]  # First row is often a header row
                            ):
                                continue

                            converted = False
                            break
                    elif cell[0].tag == "img":
                        img = cell[0]
                        # https://github.com/outline/outline/issues/9575
                        OUTLINE_FIXED_ISSUE_9575 = True
                        if OUTLINE_FIXED_ISSUE_9575:
                            tr = html.Element("tr")
                            td = html.Element(cell.tag)
                            td.set("colspan", "2")
                            td.append(img)
                            tr.append(td)

                            # Set max width/height to 400, keep aspect ratio, set both width and height
                            orig_width = int(img.attrib.get("width", "0"))
                            orig_height = int(img.attrib.get("height", "0"))
                            DESIRED_SIZE = 400
                            if orig_width > 0 and orig_height > 0:
                                scale = min(
                                    DESIRED_SIZE / orig_width,
                                    DESIRED_SIZE / orig_height,
                                )
                                new_width = int(round(orig_width * scale))
                                new_height = int(round(orig_height * scale))
                                img.set("width", str(new_width))
                                img.set("height", str(new_height))
                            else:
                                img.set("width", str(DESIRED_SIZE))
                                img.set("height", str(DESIRED_SIZE))
                            new_rows.append(tr)
                        else:
                            # Add a row: left cell "Image", right cell is the <img>
                            tr = html.Element("tr")
                            td1 = html.Element(cell.tag)
                            td1.text = "Image"
                            td2 = html.Element(cell.tag)
                            td2.append(img)
                            tr.append(td1)
                            tr.append(td2)
                            new_rows.append(tr)
                    else:
                        # Not a <b>...</b> or <img> cell, cannot convert
                        converted = False
                        break
                else:
                    if row.text_content().strip() == "":
                        # Skip empty rows
                        continue

                    # Could just be a single header row that's unimportant
                    if cell.text and (
                        "Snabbfakta" in cell.text
                        or (cell.text.startswith("“") and cell.text.endswith("’’"))
                    ):
                        continue

                    # Not a <b>...</b> or <img> cell, cannot convert
                    converted = False
                    break

            if converted:
                # Remove all previous rows
                for row in rows:
                    table.remove(row)

                # Insert new rows
                for tr in new_rows:
                    table.append(tr)

            # Optionally, set a header row if desired
            # (not required by prompt)

        # Trim whitespace at the start and end of each node's text and tail
        # This is mostly done by the minifier anyway, but it skips some places
        # before block elements, I suppose since css could technically make it important.
        def trim_whitespace(node):
            # Trim node.text
            if node.text is not None:
                node.text = node.text.lstrip()
                if len(node) == 0:
                    node.text = node.text.rstrip()
            # Trim node.tail
            if node.tail is not None:
                if node.getnext() is None or node.getnext().tag in BLOCK_ELEMENTS:
                    node.tail = node.tail.rstrip()
            # Recursively trim children
            for child in node:
                trim_whitespace(child)

        trim_whitespace(tree)

        # Save the modified HTML
        output_path = os.path.join(OUTPUT_DIR, f"{page.replace('/', '_')}.html")
        output_path_raw = os.path.join(OUTPUT_DIR, f"{page.replace('/', '_')}.raw.html")
        os.makedirs(os.path.dirname(output_path), exist_ok=True)
        with open(output_path, "w", encoding="utf-8") as f:
            res = "\n".join(
                [
                    html.tostring(child, pretty_print=True, encoding="unicode").strip()
                    for child in tree
                ]
            )
            f.write(res)

        with open(output_path_raw, "w", encoding="utf-8") as f:
            res = "".join(
                [
                    html.tostring(child, pretty_print=False, encoding="unicode").strip()
                    for child in tree
                ]
            )
            f.write(res)

        (outline_json, document_icon) = convert_to_outline_json(tree)
        outline_page_id = str(uuid.uuid4())

        def convert_to_outline_attachment_path(src: str) -> Tuple[str, int]:
            # Remove leading slash
            rel_path = src.lstrip("/")
            abs_path = os.path.join(OUTPUT_DIR, rel_path)
            if not os.path.exists(abs_path):
                raise FileNotFoundError(f"Attachment file not found: {abs_path}")
            name = os.path.basename(rel_path)
            size = os.path.getsize(abs_path)
            content_type, _ = mimetypes.guess_type(abs_path)
            if not content_type:
                content_type = "application/octet-stream"
            att_id = str(uuid.uuid4())
            key = f"uploads/f74221f0-4749-41f7-b327-5ed81752ef12/{att_id}/{name}"
            attachments[att_id] = {
                "userId": "f74221f0-4749-41f7-b327-5ed81752ef12",
                "documentId": outline_page_id,
                "contentType": content_type,
                "name": name,
                "id": att_id,
                "size": str(size),
                "key": key,
            }
            included_attachment_paths.append((abs_path, key))

            return f"/api/attachments.redirect?id={att_id}", size

        def patch_attachments(node):
            if node.get("type") in ("image", "video") and node.get("attrs", {}).get(
                "src", ""
            ).startswith("/uploads/"):
                node["attrs"]["src"] = convert_to_outline_attachment_path(
                    urllib.parse.unquote(node["attrs"]["src"])
                )[0]
            elif node.get("type") == "text" and node.get("marks"):
                for mark in node["marks"]:
                    if mark.get("type") == "link" and mark.get("attrs", {}).get(
                        "href", ""
                    ).startswith("/uploads/"):
                        href, size = convert_to_outline_attachment_path(
                            urllib.parse.unquote(mark["attrs"]["href"])
                        )
                        title = (
                            os.path.basename(node["text"])
                            if "http" in node["text"]
                            else node["text"]
                        )
                        title = title.replace("File:", "").replace("Image:", "").strip()
                        new_node = {
                            "type": "attachment",
                            "attrs": {
                                "id": None,
                                "href": href,
                                "title": title,
                                "size": size,
                            },
                        }
                        node.clear()
                        node.update(new_node)
                        break

            contents = node.get("content")
            if contents:
                for child in contents:
                    patch_attachments(child)

        if include_attachments:
            patch_attachments(outline_json)
            # patch_attachments converts inline file links into attachment
            # blocks in place; hoist those out of their paragraphs.
            outline_json["content"] = split_blocks_out_of_paragraphs(
                outline_json["content"]
            )

        for _ in range(
            2
        ):  # Not quite sure if we can have a paragraph with a list with a paragraph with attachments, but let's be safe
            outline_json["content"] = split_paragraphs_with_attachments(
                outline_json["content"]
            )
            outline_json["content"] = unwrap_single_attachment_lists(
                outline_json["content"]
            )

        wiki_page_meta = get_page_revision_metadata_cached(page)

        outline_page = OutlinePage(
            createdById=str(uuid.uuid4()),
            createdByName=wiki_page_meta["first_user"] or "Unknown",
            createdByEmail="info@makerspace.se",
            createdAt=wiki_page_meta["first_timestamp"],
            updatedAt=wiki_page_meta["last_timestamp"],
            publishedAt=wiki_page_meta["first_timestamp"],
            fullWidth=False,
            template=False,
            parentDocumentId=None,
            id=outline_page_id,
            urlId="".join([uuid.uuid4().hex[:10]]),
            title=page,
            icon=document_icon,
            color=None,
            data=outline_json,
        )
        export.documents[outline_page.id] = outline_page

        page_heading_id_mapping[outline_page.id] = heading_id_mapping

    if include_attachments:
        export.attachments = attachments

    def build_page_structure(export: OutlineExport):
        # Build category -> [pages] mapping
        category_to_pages: dict[str, list[OutlinePage]] = {}
        for doc, meta in zip(
            export.documents.values(),
            [
                get_page_revision_metadata_cached(page)
                for page in pages
                if not (page_filter and not page_filter.search(page))
            ],
        ):
            for cat in meta["categories"]:
                category_to_pages.setdefault(cat, []).append(doc)

        # Find the largest category for each page (if any)
        page_to_main_category = {}
        main_category_to_pages: dict[str, list[OutlinePage]] = {}
        for doc in export.documents.values():
            meta = get_page_revision_metadata_cached(doc.title)
            cats = meta["categories"]
            if not cats:
                continue
            # Pick the category with the most pages
            largest_cat = max(cats, key=lambda c: len(category_to_pages.get(c, [])))
            page_to_main_category[doc.id] = largest_cat
            main_category_to_pages.setdefault(largest_cat, []).append(doc)

        # Create dummy pages for categories with sub-pages
        category_docs = {}
        for cat, docs in main_category_to_pages.items():
            assert len(docs) > 0
            print(cat, [d.title for d in docs])
            cat_id = str(uuid.uuid4())
            cat_title = cat.replace("Category:", "")
            # Date the synthetic category folder like its oldest/newest child
            # so it doesn't show up as brand new ("New" badge) after import.
            child_created = [d.createdAt for d in docs if d.createdAt]
            child_updated = [d.updatedAt for d in docs if d.updatedAt]
            dummy_doc = OutlinePage(
                createdById="f74221f0-4749-41f7-b327-5ed81752ef12",
                createdByName="system",
                createdByEmail="info@makerspace.se",
                createdAt=min(child_created) if child_created else None,
                updatedAt=max(child_updated) if child_updated else None,
                publishedAt=None,
                fullWidth=False,
                template=False,
                parentDocumentId=None,
                id=cat_id,
                urlId="".join([uuid.uuid4().hex[:10]]),
                title=cat_title,
                icon="📁",
                color=None,
                data={
                    "type": "doc",
                    "content": [{"type": "paragraph"}],
                },
            )
            category_docs[cat] = dummy_doc
            for doc in docs:
                # Set the parentDocumentId to the category doc
                doc.parentDocumentId = cat_id

        # Build children lists for each category
        cat_children: dict[str, list[DocumentStructureItem]] = {
            cat: [] for cat in category_docs
        }
        root_children = []

        for doc in export.documents.values():
            main_cat = page_to_main_category.get(doc.id)
            item = DocumentStructureItem(
                id=doc.id,
                url=outline_doc_url(doc),
                title=doc.title,
                children=[],
            )
            if main_cat:
                cat_children[main_cat].append(item)
            else:
                root_children.append(item)

        # Now build the top-level documentStructure
        export.collection.documentStructure = []
        for cat, dummy_doc in category_docs.items():
            export.collection.documentStructure.append(
                DocumentStructureItem(
                    id=dummy_doc.id,
                    url=outline_doc_url(dummy_doc),
                    title=dummy_doc.title,
                    children=cat_children[cat],
                )
            )
        export.collection.documentStructure.extend(root_children)

        # Sort the whole structure alphabetically. The category folders are
        # otherwise emitted in discovery order, and page children in
        # all_pages.txt order, which mostly but not always matches.
        def sort_structure(items: list[DocumentStructureItem]):
            items.sort(key=lambda item: item.title.casefold())
            for item in items:
                sort_structure(item.children)

        sort_structure(export.collection.documentStructure)

        # Add dummy category docs to export
        # The importer currently depends on insertion order.
        # parent documents need to come before their children.
        # https://github.com/outline/outline/issues/9604
        # prev_docs = export.documents
        # export.documents = {}
        for d in category_docs.values():
            export.documents[d.id] = d

        # Add the original documents back
        # export.documents.update(prev_docs)

        for doc in export.documents.values():
            # Ensure each document appears only once in the document structure
            def doc_in_structure(doc_id, structure):
                res = 0
                if isinstance(structure, list):
                    res = sum(doc_in_structure(doc_id, item) for item in structure)
                if isinstance(structure, DocumentStructureItem):
                    if structure.id == doc_id:
                        res += 1
                    res += doc_in_structure(doc_id, structure.children)
                return res

            assert doc_in_structure(doc.id, export.collection.documentStructure) == 1

    # Snapshot before build_page_structure appends synthetic category folders,
    # so the redirect map can tell real wiki pages from folders.
    real_page_ids = set(export.documents.keys())
    build_page_structure(export)

    # Build a mapping from page title (case-insensitive, slugified) to documentStructure url
    doc_url_map = {}

    def add_doc_to_map(doc):
        # Map both the raw title and the slugified version for robustness
        doc_url_map[doc.title.lower()] = doc
        doc_url_map[outline_slugify(doc.title)] = doc
        doc_url_map[doc.id] = doc  # Also map by ID for direct access

    for structure_doc in export.collection.documentStructure:
        add_doc_to_map(structure_doc)
        if structure_doc.children:
            for child in structure_doc.children:
                add_doc_to_map(child)

    with open(os.path.join(OUTPUT_DIR, "doc_url_map.json"), "w", encoding="utf-8") as f:
        json.dump(
            {"page_heading_ids": page_heading_id_mapping, "redirects": redirects},
            f,
            indent=4,
        )

    def patch_internal_links(document, node):
        if isinstance(node, dict):
            # Patch link marks in text nodes
            if node.get("type") == "text" and node.get("marks"):
                for mark in node["marks"]:
                    if mark.get("type") == "link":
                        href = mark["attrs"].get("href", "")

                        if "@" in href:
                            # Skip email links
                            continue

                        parsed = urllib.parse.urlparse(href)
                        # Check if relative or BASE-prefixed
                        if (
                            parsed.hostname is None
                            and not parsed.path.startswith("/uploads/")
                        ) or parsed.hostname == BASE_HOSTNAME:

                            if parsed.path != "":
                                page_part = urllib.parse.unquote(parsed.path)
                                page_part = page_part.lstrip("/")

                                if page_part == "index.php":
                                    t0 = parsed.query.split("&")[0]
                                    if t0.startswith("title="):
                                        page_part = urllib.parse.unquote(t0[6:])
                                    else:
                                        mark["attrs"]["href"] = "/home"
                                        return

                                if page_part in redirects:
                                    page_part = redirects[page_part]

                                if "Användare:" in page_part:
                                    # Links to user pages cannot be patched
                                    return

                                if "Category:" in page_part:
                                    # Links to categories cannot be patched
                                    return

                                if "Template:" in page_part:
                                    # Links to templates cannot be patched
                                    return

                                if "Special:" in page_part:
                                    # Links to special pages cannot be patched
                                    return

                                # Try to match against doc_url_map
                                # Try slugified, then lower-case, then as-is
                                key_candidates = [
                                    outline_slugify(page_part),
                                    page_part.lower(),
                                ]
                                target_doc = None
                                for key in key_candidates:
                                    if key in doc_url_map:
                                        target_doc = doc_url_map[key]
                                        break

                                if not target_doc:
                                    if "sidan existerar inte" in (
                                        mark["attrs"].get("title") or ""
                                    ):
                                        mark["attrs"]["href"] = "/" + page_part
                                        node["text"] = node["text"] + " (broken link)"
                                    else:
                                        print(
                                            f"Warning: Could not find document for link {page_part} in (found in {document.title})"
                                        )
                            else:
                                target_doc = doc_url_map[document.id]

                            if target_doc:
                                fragment = urllib.parse.unquote(parsed.fragment)
                                if fragment:
                                    new_fragment = page_heading_id_mapping[
                                        target_doc.id
                                    ].get(fragment.lower())
                                    if not new_fragment:
                                        print(
                                            f"Failed to find heading {fragment} in {target_doc.title}"
                                        )
                                    else:
                                        fragment = new_fragment
                                url = (
                                    f"{target_doc.url}#h-{outline_slugify(fragment)}"
                                    if fragment
                                    else target_doc.url
                                )
                                mark["attrs"]["href"] = url

                                if BASE in node["text"]:
                                    node["text"] = target_doc.title

            # Recurse into children
            if "content" in node and isinstance(node["content"], list):
                for child in node["content"]:
                    patch_internal_links(document, child)
        elif isinstance(node, list):
            for child in node:
                patch_internal_links(document, child)

    for k, v in redirects.items():
        while v in redirects:
            v = redirects[v]  # Resolve redirects to their final target

        redirects[k] = v

    write_nginx_redirect_map(export, redirects, real_page_ids)

    for document in export.documents.values():
        patch_internal_links(document, document.data)

    for document in export.documents.values():
        json_str = json.dumps(document.data, ensure_ascii=False, indent=4)
        if "/uploads/" in json_str:
            print(
                f"Warning: Found /uploads/ in outline JSON for {document.title}. This should not happen."
            )

        if BASE in json_str:
            urls = re.findall(rf"{re.escape(BASE)}[^\s\"']*", json_str)
            for url in urls:
                print(
                    f"Warning: found {url} in export for {document.title}. This is likely a URL that couldn't be converted."
                )

        outline_path = os.path.join(
            OUTPUT_DIR, f"{document.title.replace('/', '_')}_outline.json"
        )
        with open(outline_path, "w", encoding="utf-8") as f:
            f.write(json_str)

    meta = OutlineExportMetadata(
        createdAt=datetime.now().isoformat(),
        createdById="f74221f0-4749-41f7-b327-5ed81752ef12",  # Example UUID, adjust as needed
        createdByEmail="aron.granberg@gmail.com",
    )

    export_path = os.path.join(OUTPUT_DIR, "export.json")
    with open(export_path, "w", encoding="utf-8") as f:
        json.dump(export.to_dict(), f, ensure_ascii=False, indent=4)

    meta_path = os.path.join(OUTPUT_DIR, "export_meta.json")
    with open(meta_path, "w", encoding="utf-8") as f:
        json.dump(meta.to_dict(), f, ensure_ascii=False, indent=4)

    zip_path = os.path.join(OUTPUT_DIR, "export.zip")
    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zipf:
        zipf.write(export_path, arcname="collection.json")
        zipf.write(meta_path, arcname="metadata.json")

        if include_attachments:
            for abs_path, key in progressbar(
                included_attachment_paths, prefix="Zipping attachments"
            ):
                # Use the key as the path in the zip
                zipf.write(abs_path, arcname=key)
    print(f"Created {zip_path}")