feat(alphabet): emoji extension — settings-gated emoji group for every alphabet (RFC 0020)

settings_manifest.json

@@ -1403,6 +1403,31 @@
       "subgroup": "History",
       "uiType": "Enum"
     },
+    {
+      "key": "BP_EMOJI_GROUP",
+      "storageName": "EmojiGroup",
+      "type": "bool",
+      "default": true,
+      "label": "Emoji in Alphabet",
+      "description": "Add an emoji group to every alphabet, so emoji are available in the node tree alongside your language (RFC 0020). Emoji are learned in context as you type.",
+      "uiType": "Switch",
+      "tier": "common",
+      "group": "Language",
+      "subgroup": "Emoji"
+    },
+    {
+      "key": "SP_EMOJI_SKIN_TONE",
+      "storageName": "EmojiSkinTone",
+      "type": "string",
+      "default": "none",
+      "label": "Preferred Emoji Skin Tone",
+      "description": "When set, only the base gesture and the preferred skin-tone variant are added to the emoji group (RFC 0020). 'None' adds all variants; the tone you use most rises through adaptation either way.",
+      "tier": "common",
+      "group": "Language",
+      "subgroup": "Emoji",
+      "uiType": "Enum",
+      "values": ["none", "light", "medium-light", "medium", "medium-dark", "dark"]
+    },
     {
       "key": "SP_COLOUR_ID",
       "storageName": "ColourID",

src/DasherCore/Alphabet/AlphIO.cpp

@@ -120,7 +120,8 @@ CAlphIO::CAlphIO(CMessageDisplay* pMsgs) : AbstractXMLParser(pMsgs) {
 }
 
 SGroupInfo* CAlphIO::ParseGroupRecursive(pugi::xml_node& group_node, CAlphInfo* CurrentAlphabet,
-                                         SGroupInfo* previous_sibling, std::vector<SGroupInfo*> ancestors) {
+                                         SGroupInfo* previous_sibling, std::vector<SGroupInfo*> ancestors,
+                                         const std::string* toneFilter) {
     SGroupInfo* pNewGroup = new SGroupInfo();
     pNewGroup->iNumChildNodes = 0;
     pNewGroup->strName = group_node.attribute("name").as_string("");
@@ -139,6 +140,13 @@ SGroupInfo* CAlphIO::ParseGroupRecursive(pugi::xml_node& group_node, CAlphInfo*
     for (auto node : group_node.children()) {
         // symbol (v6 "node" or v5 "s")
         if (std::strcmp(node.name(), "node") == 0 || std::strcmp(node.name(), "s") == 0) {
+            // RFC 0020 skin-tone filter: nodes carrying a tone= attribute are
+            // variants; a filter keeps the base (unmarked) nodes plus exactly
+            // the preferred variant.
+            if (toneFilter) {
+                pugi::xml_attribute tone = node.attribute("tone");
+                if (!tone.empty() && tone.as_string() != *toneFilter) continue;
+            }
             CurrentAlphabet->m_vCharacters.resize(CurrentAlphabet->m_vCharacters.size() + 1); // new char
             CurrentAlphabet->m_vCharacterDoActions.resize(CurrentAlphabet->m_vCharacterDoActions.size() +
                                                           1); // new Do Actions
@@ -153,7 +161,7 @@ SGroupInfo* CAlphIO::ParseGroupRecursive(pugi::xml_node& group_node, CAlphInfo*
         // group
         if (std::strcmp(node.name(), "group") == 0) {
             SGroupInfo* newChildGroup =
-                ParseGroupRecursive(node, CurrentAlphabet, previous_subgroup_sibling, new_ancestors);
+                ParseGroupRecursive(node, CurrentAlphabet, previous_subgroup_sibling, new_ancestors, toneFilter);
             if (newChildGroup == nullptr) continue;
             pNewGroup->iNumChildNodes++;
             pNewGroup->pChild = newChildGroup;
@@ -190,6 +198,30 @@ bool Dasher::CAlphIO::Parse(pugi::xml_document& document, const std::string, boo
 
     if (std::strcmp(alphabet.name(), "alphabet") != 0) return false; // a non <alphabet ...> node
 
+    CAlphInfo* CurrentAlphabet = ParseAlphabet(alphabet, isV5);
+
+    auto it = Alphabets.find(CurrentAlphabet->AlphID);
+    if (it != Alphabets.end()) {
+        // v5 alphabets are legacy (e.g. the bundled oldAlphabets/ files). Never
+        // let them overwrite a v6 alphabet that already loaded under the same
+        // name — the v6 version is authoritative. User-authored v5 files with
+        // unique names still load normally.
+        if (isV5) {
+            delete CurrentAlphabet;
+            return true;
+        }
+        delete it->second;
+    }
+    Alphabets[CurrentAlphabet->AlphID] = CurrentAlphabet;
+
+    return true;
+}
+
+// Parse one <alphabet> element into a fresh, caller-owned CAlphInfo.
+// Extracted from Parse so MakeExtendedInfo (RFC 0020) can obtain a private
+// copy — CAlphInfo's destructor owns its ControlActions, so sharing action
+// pointers between the shared base info and a derived one would double-free.
+CAlphInfo* CAlphIO::ParseAlphabet(pugi::xml_node& alphabet, bool isV5) {
     CAlphInfo* CurrentAlphabet = new CAlphInfo();
     CurrentAlphabet->AlphID = alphabet.attribute("name").as_string();
     CurrentAlphabet->TrainingFile = alphabet.attribute("trainingFilename").as_string();
@@ -305,21 +337,7 @@ bool Dasher::CAlphIO::Parse(pugi::xml_document& document, const std::string, boo
     // child groups were added (to linked list) in reverse order. Put them in (iStart/iEnd) order...
     ReverseChildList(CurrentAlphabet->pChild);
 
-    auto it = Alphabets.find(CurrentAlphabet->AlphID);
-    if (it != Alphabets.end()) {
-        // v5 alphabets are legacy (e.g. the bundled oldAlphabets/ files). Never
-        // let them overwrite a v6 alphabet that already loaded under the same
-        // name — the v6 version is authoritative. User-authored v5 files with
-        // unique names still load normally.
-        if (isV5) {
-            delete CurrentAlphabet;
-            return true;
-        }
-        delete it->second;
-    }
-    Alphabets[CurrentAlphabet->AlphID] = CurrentAlphabet;
-
-    return true;
+    return CurrentAlphabet;
 }
 
 void CAlphIO::GetAlphabets(std::vector<std::string>* AlphabetList) const {
@@ -371,6 +389,76 @@ bool CAlphIO::LoadAlphabetFile(const std::string& filename) {
     return true;
 }
 
+// ── Emoji extension (RFC 0020) ─────────────────────────────────────────────
+
+bool CAlphIO::LoadEmojiExtension(const std::string& filename) {
+    m_emojiExtensionGroups.clear();
+    if (filename.empty()) return false;
+    std::ifstream in(filename.c_str(), std::ios::binary);
+    if (!in.good()) return false;
+    pugi::xml_parse_result result = m_emojiExtensionDoc.load(in);
+    if (!result) return false;
+    pugi::xml_node root = m_emojiExtensionDoc.document_element();
+    if (std::strcmp(root.name(), "emoji-extension") != 0) return false;
+    for (pugi::xml_node& child : root.children())
+        if (std::strcmp(child.name(), "group") == 0) m_emojiExtensionGroups.push_back(child);
+    return !m_emojiExtensionGroups.empty();
+}
+
+const CAlphInfo* CAlphIO::MakeExtendedInfo(const std::string& AlphID, const std::string& skinTone) {
+    const CAlphInfo* base = GetInfo(AlphID);
+    if (m_emojiExtensionGroups.empty()) return base;
+
+    // A derived info owns its ControlActions (the base's destructor deletes
+    // its own), so re-parse the alphabet's file into a private copy rather
+    // than cloning the shared info. The emergency built-in "Default" has no
+    // file — it stays unextended (documented RFC 0020 limitation).
+    const std::string file = FileNameFor(AlphID);
+    if (file.empty() || AlphID == "Default") return base;
+
+    std::ifstream in(file.c_str(), std::ios::binary);
+    if (!in.good()) return base;
+    pugi::xml_document doc;
+    if (!doc.load(in)) return base;
+    pugi::xml_node alphabet = doc.document_element();
+    bool isV5 = (std::strcmp(alphabet.name(), "alphabets") == 0);
+    if (isV5) alphabet = alphabet.child("alphabet");
+    if (!alphabet || std::strcmp(alphabet.name(), "alphabet") != 0) return base;
+
+    CAlphInfo* derived = ParseAlphabet(alphabet, isV5);
+    // Guard against index/registry divergence (e.g. a user-dir v5 file
+    // shadowing a v6 name in the filename index but not the registry):
+    // the re-parse must produce the requested alphabet, else serve the
+    // registered base rather than a divergent copy.
+    if (derived->AlphID != AlphID) {
+        delete derived;
+        return base;
+    }
+
+    // Append the extension's groups after the alphabet's own (ParseGroupRecursive
+    // appends characters at the end and links groups as reverse siblings).
+    std::string toneFilter(skinTone != "none" ? skinTone : "");
+    const std::string* pToneFilter = toneFilter.empty() ? nullptr : &toneFilter;
+    // Append the extension's groups AFTER the alphabet's own chain: start
+    // at the base chain's tail so the reverse below yields
+    // base→…→base→ext→…→ext. Starting at nullptr (the original bug) orphaned
+    // the base group chain — leaking its SGroupInfo tree and dropping every
+    // base symbol's group coverage.
+    SGroupInfo* previous_sibling = derived->pChild;
+    if (previous_sibling)
+        for (; previous_sibling->pNext; previous_sibling = previous_sibling->pNext);
+    for (pugi::xml_node& group : m_emojiExtensionGroups) {
+        SGroupInfo* newGroup = ParseGroupRecursive(group, derived, previous_sibling, {}, pToneFilter);
+        if (!newGroup) continue;
+        derived->iNumChildNodes++;
+        derived->pChild = newGroup; // last parsed; the reverse below restores order
+        previous_sibling = newGroup;
+    }
+    derived->iEnd = static_cast<int>(derived->m_vCharacters.size()) + 1;
+    ReverseChildList(derived->pChild);
+    return derived;
+}
+
 std::string CAlphIO::GetDefault() const {
     std::string DefaultExternalAlphabet = "English with limited punctuation";
     if (Alphabets.count(DefaultExternalAlphabet) != 0) {

src/DasherCore/Alphabet/AlphIO.h

@@ -76,16 +76,37 @@ class Dasher::CAlphIO : public AbstractXMLParser {
     /// Full-parse one alphabet file by (absolute or pattern) filename.
     bool LoadAlphabetFile(const std::string& filename);
 
+    /// ── Emoji extension (RFC 0020) ─────────────────────────────────────
+    /// Parse and cache the groups-only extension definition
+    /// (Data/emoji/extension.xml). Safe to call when absent.
+    /// \return true when an extension was loaded.
+    bool LoadEmojiExtension(const std::string& filename);
+
+    /// True when an emoji extension definition is loaded.
+    bool HasEmojiExtension() const { return !m_emojiExtensionGroups.empty(); }
+
+    /// Merge the emoji extension into a DEEP COPY of the named alphabet.
+    /// \param skinTone "none" keeps all tone variants; any other value
+    ///  keeps only base gestures plus that tone's variants.
+    /// \return a caller-owned CAlphInfo (delete when the alphabet
+    ///  changes), or the shared base info when no extension is loaded —
+    ///  callers must treat the result as const and never delete it then.
+    const CAlphInfo* MakeExtendedInfo(const std::string& AlphID, const std::string& skinTone);
+
   private:
     std::map<std::string, const CAlphInfo*> Alphabets; // map AlphabetID to AlphabetInfo.
     std::map<std::string, std::string> AlphabetFiles;  // name index: AlphID → filename
     static CAlphInfo*
     CreateDefault(); // Give the user an English alphabet rather than nothing if anything goes horribly wrong.
+    std::vector<pugi::xml_node> m_emojiExtensionGroups; // cached <group> nodes
+    pugi::xml_document m_emojiExtensionDoc;             // owns the cached nodes
 
     void ReadCharAttributes(pugi::xml_node xml_node, CAlphInfo::character& alphabet_character, SGroupInfo* parentGroup,
                             std::vector<ControlAction*>& DoActions, std::vector<ControlAction*>& UndoActions);
+    CAlphInfo* ParseAlphabet(pugi::xml_node& alphabet, bool isV5);
     SGroupInfo* ParseGroupRecursive(pugi::xml_node& group_node, CAlphInfo* CurrentAlphabet,
-                                    SGroupInfo* previous_sibling, std::vector<SGroupInfo*> ancestors);
+                                    SGroupInfo* previous_sibling, std::vector<SGroupInfo*> ancestors,
+                                    const std::string* toneFilter = nullptr);
     void ReverseChildList(SGroupInfo*& pList);
     // Alphabet types:
     std::map<std::string, Options::AlphabetTypes> AlphabetStringToType;

src/DasherCore/AlphabetManager.cpp

@@ -602,8 +602,16 @@ void CAlphabetManager::IterateChildGroups(CAlphNode* pParent, const SGroupInfo*
     DASHER_ASSERT((*pCProb)[0] == 0);
     const int iMin(pParentGroup->iStart);
     const int iMax(pParentGroup->iEnd);
-    unsigned int iRange(pParentGroup == m_pBaseGroup ? CDasherModel::NORMALIZATION
-                                                     : ((*pCProb)[iMax - 1] - (*pCProb)[iMin - 1]));
+    // Scale children against the LM's ACTUAL cumulative total rather than
+    // assuming it fills NORMALIZATION: PPM reserves escape/uniform mass for
+    // unseen text, and with a large untrained symbol set (RFC 0020's merged
+    // emoji extension, before adaptation lifts what the user types) that
+    // residual is no longer negligible — assuming 65536 left the last child
+    // short and the tree didn't fill the screen. For fully-trained
+    // alphabets the total IS 65536 and this is a no-op.
+    const unsigned int iActualRange((*pCProb)[iMax - 1] - (*pCProb)[iMin - 1]);
+    DASHER_ASSERT(iActualRange > 0);
+    unsigned int iRange(iActualRange);
 
     // TODO: Think through alphabet file formats etc. to make this class easier.
     // TODO: Throw a warning if parent node already has children

src/DasherCore/DasherInterfaceBase.cpp

@@ -117,6 +117,15 @@ void CDasherInterfaceBase::Realize(unsigned long ulTime) {
     // THIS realize read the other bundle (greptile P1 #2 on #88 — pinned by
     // retired_default_heal_uses_own_context_data_dir).
     m_AlphIO->ScanNameIndex(m_dataDir);
+    // RFC 0020: load the emoji extension definition (groups-only, merged
+    // into every alphabet when BP_EMOJI_GROUP is on). The engine resolves
+    // data against {dataDir, dataDir/Data} — try both, tolerate absence.
+    {
+        const std::string sep = (m_dataDir.empty() || m_dataDir.back() == '/') ? "" : "/";
+        for (const std::string& sub : {std::string("emoji/extension.xml"), std::string("Data/emoji/extension.xml")}) {
+            if (m_AlphIO->LoadEmojiExtension(m_dataDir + sep + sub)) break;
+        }
+    }
     const auto loadById = [this](const std::string& alphId) { LoadAlphabetById(alphId); };
     {
         std::string alphId = m_pSettingsStore->GetStringParameter(SP_ALPHABET_ID);
@@ -219,6 +228,13 @@ CDasherInterfaceBase::~CDasherInterfaceBase() {
 
     // Clean up cached lock label (created by Redraw when locked)
     delete m_pLockLabel;
+
+    // RFC 0020: derived infos own their ControlActions — free them once the
+    // model (which referenced them) is gone.
+    for (const CAlphInfo* info : m_vExtendedAlphInfos)
+        delete info;
+    m_vExtendedAlphInfos.clear();
+    m_pExtendedAlphInfo = nullptr;
 }
 
 void CDasherInterfaceBase::HandleParameterChange(Parameter parameter) {
@@ -241,6 +257,11 @@ void CDasherInterfaceBase::HandleParameterChange(Parameter parameter) {
         ChangeAlphabet();
         ScheduleRedraw();
         break;
+    case BP_EMOJI_GROUP:     // RFC 0020 — the merged emoji group changes the
+    case SP_EMOJI_SKIN_TONE: // alphabet's symbol set; rebuild like a switch
+        ChangeAlphabet();
+        ScheduleRedraw();
+        break;
     case SP_COLOUR_ID:
         ChangeColors();
         ScheduleRedraw();
@@ -646,7 +667,24 @@ double CDasherInterfaceBase::GetCurFPS() {
 }
 
 const CAlphInfo* CDasherInterfaceBase::GetActiveAlphabet() {
-    return m_AlphIO->GetInfo(m_pSettingsStore->GetStringParameter(SP_ALPHABET_ID));
+    const std::string alphId = m_pSettingsStore->GetStringParameter(SP_ALPHABET_ID);
+    if (!m_pSettingsStore->GetBoolParameter(BP_EMOJI_GROUP) || !m_AlphIO->HasEmojiExtension())
+        return m_AlphIO->GetInfo(alphId);
+
+    // RFC 0020: merge the emoji extension into a derived copy. Cached per
+    // (alphabet, tone): GetActiveAlphabet is called per frame by CAPI
+    // consumers, and each merge re-parses the alphabet file.
+    const std::string tone = m_pSettingsStore->GetStringParameter(SP_EMOJI_SKIN_TONE);
+    const std::string key = alphId + '\x1f' + tone;
+    if (m_pExtendedAlphInfo && m_strExtendedKey == key) return m_pExtendedAlphInfo;
+
+    const CAlphInfo* base = m_AlphIO->GetInfo(alphId);
+    const CAlphInfo* derived = m_AlphIO->MakeExtendedInfo(alphId, tone);
+    m_pExtendedAlphInfo = derived;
+    m_strExtendedKey = key;
+    if (derived != base)                         // MakeExtendedInfo falls back to the shared base
+        m_vExtendedAlphInfos.push_back(derived); // info when nothing merged
+    return derived;
 }
 
 // int CDasherInterfaceBase::GetAutoOffset() {

src/DasherCore/DasherInterfaceBase.h

@@ -546,6 +546,17 @@ class Dasher::CDasherInterfaceBase : public CMessageDisplay, private NoClones {
     std::unique_ptr<CColorIO> m_ColorIO;
     std::unique_ptr<CNodeCreationManager> m_pNCManager;
 
+    // ── Emoji extension (RFC 0020) ─────────────────────────────────────
+    // Derived alphabet infos (base + merged emoji groups) are cached per
+    // (alphabetId, skinTone) and kept alive for the process lifetime:
+    // node trees built against a previous info may outlive the switch, and
+    // each derived info owns its ControlActions (the shared base infos'
+    // destructors would double-free them). Bounded by user-driven
+    // alphabet/tone changes.
+    std::vector<const CAlphInfo*> m_vExtendedAlphInfos;
+    const CAlphInfo* m_pExtendedAlphInfo = nullptr;
+    std::string m_strExtendedKey;
+
     // the game mode module - only
     // initialized if game mode is enabled
     std::unique_ptr<CGameModule> m_pGameModule;

src/DasherCore/NodeCreationManager.cpp

@@ -70,7 +70,13 @@ CNodeCreationManager::CNodeCreationManager(CSettingsStore* pSettingsStore, CDash
     : m_pInterface(pInterface), m_pScreen(nullptr), m_pSettingsStore(pSettingsStore) {
     m_pSettingsStore->OnParameterChanged.Subscribe(this, [this](const Parameter p) { HandleParameterChange(p); });
 
-    const Dasher::CAlphInfo* pAlphInfo(pAlphIO->GetInfo(m_pSettingsStore->GetStringParameter(SP_ALPHABET_ID)));
+    // RFC 0020: the ACTIVE alphabet (base + emoji extension when
+    // BP_EMOJI_GROUP is on) — NOT the registered base. GetActiveAlphabet
+    // serves the derived, merged info; building from the registered base
+    // here would render and train a tree the extension never touches,
+    // leaving the CAPI split-brained (symbol count includes emoji, node
+    // tree doesn't).
+    const Dasher::CAlphInfo* pAlphInfo(pInterface->GetActiveAlphabet());
 
     switch (pAlphInfo->m_iConversionID) {
     case CAlphInfo::None: // No conversion required