feat(ddi)!: cells as authored, unlabelled groups, messages without a constraint (#160)

ddi-validation/schematron/ddi_custom_rules.sch

@@ -216,7 +216,7 @@
 
     <pattern id="cdl_notes">
     <rule context="ddi:notes[starts-with(@type, 'cdl:')]">
-        <assert test="@type = 'cdl:appearance' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
+        <assert test="@type = 'cdl:appearance' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:no_label' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
         <assert test="not(@type = 'cdl:constraint') or @subject = 'xlsform-xpath'">A cdl:constraint note needs subject="xlsform-xpath": its text is an expression in that syntax.</assert>
         <assert test="not(@type = 'cdl:language') or normalize-space(@subject) != ''">A cdl:language note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:position') or normalize-space(@subject) != ''">A cdl:position note needs a subject: the name of what it holds.</assert>
@@ -229,7 +229,7 @@
         <assert test="not(@type = 'cdl:setting') or normalize-space(@subject) != ''">A cdl:setting note needs a subject: the name of what it holds.</assert>
     </rule>
     <rule context="notes[starts-with(@type, 'cdl:')]">
-        <assert test="@type = 'cdl:appearance' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
+        <assert test="@type = 'cdl:appearance' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:no_label' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
         <assert test="not(@type = 'cdl:constraint') or @subject = 'xlsform-xpath'">A cdl:constraint note needs subject="xlsform-xpath": its text is an expression in that syntax.</assert>
         <assert test="not(@type = 'cdl:language') or normalize-space(@subject) != ''">A cdl:language note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:position') or normalize-space(@subject) != ''">A cdl:position note needs a subject: the name of what it holds.</assert>

registry/conventions/ddiFields.jsonld

@@ -63,7 +63,7 @@
               "var",
               "varGrp"
             ],
-            "text": "the parameters not in a standard element, space-separated key=value; a range's start and end as authored (valrng/range has them too)"
+            "text": "the parameters cell as authored (a range's start and end are also valrng/range, a guidance_hint inside it also ivuInstr)"
           },
           "hint": {
             "type": "cdl:hint",
@@ -180,6 +180,13 @@
             ],
             "localized": true,
             "text": "a select_multiple's authored other choice's label: the pair has no binary var for it"
+          },
+          "no_label": {
+            "type": "cdl:no_label",
+            "on": [
+              "varGrp[@type='section' or @type='grid']"
+            ],
+            "text": "yes: the group has no label; its txt is its name, for readers that show one"
           }
         },
         "fields": {

registry/conventions/logicMapping.jsonld

@@ -76,7 +76,7 @@
             "lsSyntax": "Y/N",
             "ddi": {
               "element": "var/notes[@type='cdl:required']",
-              "note": "Written only for a required question, with the text `yes`. It tells 'not asked' (universe) from 'refused' apart.",
+              "note": "Written only for a required question, with the required cell as authored (`yes` when it is `yes` or the source has none, e.g. `TRUE` as written). It tells 'not asked' (universe) from 'refused' apart.",
               "lossy": false
             }
           }

src/ddi/codebook.ts

@@ -33,6 +33,7 @@ import {
   addListNote,
   addNoteNames,
   addRowFieldNotes,
+  addRowLabel,
   addRowNotes,
   addSettingNotes,
   references,
@@ -585,6 +586,8 @@ interface StudyNotes {
   rows: Variable[];
   /** Every note row, for its fields. */
   notes: Variable[];
+  /** Inline notes that need their own text (`cdl:row_label`). */
+  unsplittable: Variable[];
   /** Where the orphans and rows are. */
   positions: Position[];
   /** Language tag → the form's name for it. */
@@ -633,6 +636,7 @@ function addStudyDscr(
     kept.add(row.name);
   }
   for (const note of study.notes) addRowFieldNotes(stdy, note);
+  for (const note of study.unsplittable) addRowLabel(stdy, note);
   for (const p of study.positions) {
     if (!p.name || !(p.group || kept.has(p.name))) continue;
     stdy.textChild('notes', `in=${p.in} after=${p.after}`, {
@@ -994,6 +998,7 @@ export function buildDdiCodebook(
     orphans: classified.orphanNotes,
     rows: classified.rows,
     notes: variables.filter((v) => v.type === 'note' && v.row === undefined),
+    unsplittable: classified.unsplittable,
     positions: classified.positions,
     languageNames,
   });

src/ddi/fields.ts

@@ -13,21 +13,6 @@ import type { XmlElement } from './xml.js';
 
 const NOTES = conventions.conventions.ddiFields.notes;
 
-/**
- * The `parameters` DDI has no element for, as `key=value` tokens: without
- * `guidance_hint` (`ivuInstr`). A range's `start`/`end` stay as authored
- * (#160), though its `valrng` has them too: a bound equal to the default is
- * otherwise not told apart from none.
- */
-export function otherParameters(v: Variable): string {
-  const tokens: string[] = [];
-  for (const part of (v.parameters ?? '').split(';')) {
-    if (/^\s*guidance_hint\s*=/.test(part)) continue;
-    tokens.push(...part.split(/[\s,]+/).filter(Boolean));
-  }
-  return tokens.join(' ');
-}
-
 /** A `range`'s bounds, the registry's defaults where not authored. */
 export function rangeBounds(v: Variable): { min: string; max: string } {
   const values = {
@@ -45,9 +30,10 @@ export function addFieldNotes(el: XmlElement, v: Variable): void {
   if (v.appearance) {
     el.textChild('notes', v.appearance, { type: NOTES.appearance.type });
   }
-  const parameters = otherParameters(v);
-  if (parameters) {
-    el.textChild('notes', parameters, { type: NOTES.parameters.type });
+  // As authored (#160): a range's bounds are its valrng too, and a
+  // guidance_hint inside is its ivuInstr too, but only the cell says so.
+  if (v.parameters) {
+    el.textChild('notes', v.parameters, { type: NOTES.parameters.type });
   }
 }
 
@@ -76,6 +62,10 @@ export function addGroupFieldNotes(
   if (group.appearance && !(grid && group.appearance === GRID_APPEARANCE)) {
     el.textChild('notes', group.appearance, { type: NOTES.appearance.type });
   }
+  // Its txt is its name, for readers that show one (#160).
+  if (group.unlabelled) {
+    el.textChild('notes', 'yes', { type: NOTES.no_label.type });
+  }
 }
 
 /** Settings in a standard element: `titl`, `IDNo`, `verStmt/version`. */
@@ -138,13 +128,18 @@ export function addRowNotes(stdy: XmlElement, v: Variable): void {
     type: NOTES.row.type,
     subject: v.name,
   });
+  addRowLabel(stdy, v);
+  addRowFieldNotes(stdy, v);
+}
+
+/** A row's own label by its name (`cdl:row_label`), in every language. */
+export function addRowLabel(stdy: XmlElement, v: Variable): void {
   if (v.label) {
     localizedChild(stdy, 'notes', v.label, textsOf(v, 'label'), {
       type: NOTES.row_label.type,
       subject: v.name,
     });
   }
-  addRowFieldNotes(stdy, v);
 }
 
 /** A note row's or data-less row's hint, relevant and appearance, by its name. */

src/ddi/fromInstrument.ts

@@ -124,6 +124,13 @@ function groupsOf(ctx: GroupContext): Pick<Variable, 'groups'> {
   return ctx.groups.length ? { groups: ctx.groups } : {};
 }
 
+/** The `required` cell as authored, when it isn't the plain `yes` (#160). */
+function requiredCell(q: QuestionItem): Pick<Variable, 'requiredCell'> {
+  const cell = q.row['required'];
+  const text = typeof cell === 'string' ? cell.trim() : '';
+  return text && text !== 'yes' ? { requiredCell: text } : {};
+}
+
 /** A question's own logic (#151) and form fields (#153), absent when empty. */
 function logicOf(
   q: QuestionItem,
@@ -134,6 +141,7 @@ function logicOf(
   | 'constraint'
   | 'constraintMessage'
   | 'required'
+  | 'requiredCell'
   | 'default'
   | 'appearance'
   | 'parameters'
@@ -143,8 +151,8 @@ function logicOf(
   return {
     ...(relevant ? { relevant } : {}),
     ...(q.constraint ? { constraint: q.constraint } : {}),
-    ...(q.constraint && message ? { constraintMessage: message } : {}),
-    ...(q.required ? { required: true } : {}),
+    ...(message ? { constraintMessage: message } : {}),
+    ...(q.required ? { required: true, ...requiredCell(q) } : {}),
     ...(q.default ? { default: q.default } : {}),
     ...(q.appearance ? { appearance: q.appearance } : {}),
     ...(q.parameters.trim() ? { parameters: q.parameters.trim() } : {}),
@@ -392,7 +400,7 @@ function pushQuestion(
       hint: q.hint,
       guidanceHint: q.guidanceHint,
       groupLabel: ctx.labelText,
-      ...(q.constraint ? { constraintMessage: q.constraintMessage } : {}),
+      constraintMessage: q.constraintMessage,
     },
     state.others,
   );
@@ -438,6 +446,9 @@ function project(items: Item[], ctx: GroupContext, state: ProjectState): void {
       relevant: item.relevant.trim(),
       hint: pick(item.hint, state.lang).trim(),
       ...(hintTranslations ? { hintTranslations } : {}),
+      ...(Object.values(item.label).some((t) => t.trim())
+        ? {}
+        : { unlabelled: true }),
     };
     project(
       item.children,

src/ddi/logic.ts

@@ -386,6 +386,8 @@ export function addLogicNotes(el: XmlElement, v: Variable): void {
     });
   }
   if (v.required) {
-    el.textChild('notes', NOTES.required.text, { type: NOTES.required.type });
+    el.textChild('notes', v.requiredCell ?? NOTES.required.text, {
+      type: NOTES.required.type,
+    });
   }
 }

src/ddi/notes.ts

@@ -29,6 +29,12 @@ export interface ClassifiedNotes {
   inlinePreqtxtTranslations: Record<string, Translations>;
   /** `variable.name` → the names of the notes in its `inlinePreqtxt`. */
   inlineNames: Record<string, string[]>;
+  /**
+   * Inline notes whose joined text can't be split back at its blank lines
+   * (one has a blank line, or lacks a language another has): their own
+   * texts go in `cdl:row_label` (#160).
+   */
+  unsplittable: Variable[];
   /** Notes with no data-carrying successor in the same group. */
   orphanNotes: Variable[];
   /** Rows without data: metadata rows, matrix headers. */
@@ -75,6 +81,19 @@ function positionOf(
   return { name, in: group, after: '' };
 }
 
+/** Whether notes joined with blank lines split back into the same notes. */
+function splittable(notes: Variable[]): boolean {
+  if (notes.length < 2) return true;
+  const texts = notes.map((n): Record<string, string> => ({
+    '': n.label,
+    ...textsOf(n, 'label'),
+  }));
+  const langs = new Set(texts.flatMap((t) => Object.keys(t)));
+  return texts.every((t) =>
+    [...langs].every((l) => !!t[l] && !t[l].includes('\n\n')),
+  );
+}
+
 /** Split notes into inline (`<preQTxt>`) and orphan (`<notes>`) buckets. */
 export function classifyNotes(variables: Variable[]): ClassifiedNotes {
   const dataVars: Variable[] = [];
@@ -109,7 +128,9 @@ export function classifyNotes(variables: Variable[]): ClassifiedNotes {
   const inlinePreqtxt: Record<string, string> = {};
   const inlinePreqtxtTranslations: Record<string, Translations> = {};
   const inlineNames: Record<string, string[]> = {};
+  const unsplittable: Variable[] = [];
   for (const [name, notes] of Object.entries(inline)) {
+    if (!splittable(notes)) unsplittable.push(...notes);
     inlinePreqtxt[name] = notes.map((n) => n.label).join('\n\n');
     inlinePreqtxtTranslations[name] = joinTranslations(
       notes.map((n) => textsOf(n, 'label')),
@@ -125,6 +146,7 @@ export function classifyNotes(variables: Variable[]): ClassifiedNotes {
     inlinePreqtxt,
     inlinePreqtxtTranslations,
     inlineNames,
+    unsplittable,
     orphanNotes: kept.filter((v) => v.type === 'note'),
     rows: kept.filter((v) => v.row !== undefined),
     positions,

src/ddi/types.ts

@@ -41,6 +41,8 @@ export interface DdiGroup {
   relevant: string;
   /** Its `hint`: a `cdl:hint` note (#153); `''` when none. */
   hint: string;
+  /** It has no label of its own: `label` is its name (`cdl:no_label`, #160). */
+  unlabelled?: boolean;
   /** `hint` in the form's other languages. */
   hintTranslations?: Translations;
 }
@@ -94,6 +96,8 @@ export interface Variable {
   constraintMessage?: string;
   /** XLSForm `required`: a `cdl:required` note. */
   required?: boolean;
+  /** The `required` cell when it isn't `yes` (`TRUE`), the note's text (#160). */
+  requiredCell?: string;
   /** XLSForm `default`: a `cdl:default` note (#153). */
   default?: string;
   /** The question's own lowercased `appearance`: a `cdl:appearance` note. */

src/generated/conventions.json

@@ -35,7 +35,7 @@
             "var",
             "varGrp"
           ],
-          "text": "the parameters not in a standard element, space-separated key=value; a range's start and end as authored (valrng/range has them too)"
+          "text": "the parameters cell as authored (a range's start and end are also valrng/range, a guidance_hint inside it also ivuInstr)"
         },
         "hint": {
           "type": "cdl:hint",
@@ -152,6 +152,13 @@
           ],
           "localized": true,
           "text": "a select_multiple's authored other choice's label: the pair has no binary var for it"
+        },
+        "no_label": {
+          "type": "cdl:no_label",
+          "on": [
+            "varGrp[@type='section' or @type='grid']"
+          ],
+          "text": "yes: the group has no label; its txt is its name, for readers that show one"
         }
       },
       "fields": {
@@ -525,7 +532,7 @@
           "lsSyntax": "Y/N",
           "ddi": {
             "element": "var/notes[@type='cdl:required']",
-            "note": "Written only for a required question, with the text `yes`. It tells 'not asked' (universe) from 'refused' apart.",
+            "note": "Written only for a required question, with the required cell as authored (`yes` when it is `yes` or the source has none, e.g. `TRUE` as written). It tells 'not asked' (universe) from 'refused' apart.",
             "lossy": false
           }
         }

src/generated/conventions.ts

@@ -44,7 +44,7 @@ const conventions = {
             "var",
             "varGrp"
           ],
-          "text": "the parameters not in a standard element, space-separated key=value; a range's start and end as authored (valrng/range has them too)"
+          "text": "the parameters cell as authored (a range's start and end are also valrng/range, a guidance_hint inside it also ivuInstr)"
         },
         "hint": {
           "type": "cdl:hint",
@@ -161,6 +161,13 @@ const conventions = {
           ],
           "localized": true,
           "text": "a select_multiple's authored other choice's label: the pair has no binary var for it"
+        },
+        "no_label": {
+          "type": "cdl:no_label",
+          "on": [
+            "varGrp[@type='section' or @type='grid']"
+          ],
+          "text": "yes: the group has no label; its txt is its name, for readers that show one"
         }
       },
       "fields": {
@@ -534,7 +541,7 @@ const conventions = {
           "lsSyntax": "Y/N",
           "ddi": {
             "element": "var/notes[@type='cdl:required']",
-            "note": "Written only for a required question, with the text `yes`. It tells 'not asked' (universe) from 'refused' apart.",
+            "note": "Written only for a required question, with the required cell as authored (`yes` when it is `yes` or the source has none, e.g. `TRUE` as written). It tells 'not asked' (universe) from 'refused' apart.",
             "lossy": false
           }
         }