feat(ddi)!: empty groups, end_group cells, appearance case, every setting (#160)

ddi-validation/schematron/ddi_custom_rules.sch

@@ -216,10 +216,11 @@
 
     <pattern id="cdl_notes">
     <rule context="ddi:notes[starts-with(@type, 'cdl:')]">
-        <assert test="@type = 'cdl:appearance' or @type = 'cdl:choice_column' or @type = 'cdl:column' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:no_label' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_column' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:choice_column, cdl:column, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_column, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
+        <assert test="@type = 'cdl:appearance' or @type = 'cdl:choice_column' or @type = 'cdl:column' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:end_column' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:no_label' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_column' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:choice_column, cdl:column, cdl:constraint, cdl:constraint_message, cdl:default, cdl:end_column, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_column, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
         <assert test="not(@type = 'cdl:choice_column') or normalize-space(@subject) != ''">A cdl:choice_column note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:column') or normalize-space(@subject) != ''">A cdl:column note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:constraint') or @subject = 'xlsform-xpath'">A cdl:constraint note needs subject="xlsform-xpath": its text is an expression in that syntax.</assert>
+        <assert test="not(@type = 'cdl:end_column') or normalize-space(@subject) != ''">A cdl:end_column note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:language') or normalize-space(@subject) != ''">A cdl:language note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:position') or normalize-space(@subject) != ''">A cdl:position note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:relevant') or @subject = 'xlsform-xpath'">A cdl:relevant note needs subject="xlsform-xpath": its text is an expression in that syntax.</assert>
@@ -232,10 +233,11 @@
         <assert test="not(@type = 'cdl:setting') or normalize-space(@subject) != ''">A cdl:setting note needs a subject: the name of what it holds.</assert>
     </rule>
     <rule context="notes[starts-with(@type, 'cdl:')]">
-        <assert test="@type = 'cdl:appearance' or @type = 'cdl:choice_column' or @type = 'cdl:column' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:no_label' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_column' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:choice_column, cdl:column, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_column, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
+        <assert test="@type = 'cdl:appearance' or @type = 'cdl:choice_column' or @type = 'cdl:column' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:end_column' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:no_label' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_column' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:choice_column, cdl:column, cdl:constraint, cdl:constraint_message, cdl:default, cdl:end_column, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_column, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
         <assert test="not(@type = 'cdl:choice_column') or normalize-space(@subject) != ''">A cdl:choice_column note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:column') or normalize-space(@subject) != ''">A cdl:column note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:constraint') or @subject = 'xlsform-xpath'">A cdl:constraint note needs subject="xlsform-xpath": its text is an expression in that syntax.</assert>
+        <assert test="not(@type = 'cdl:end_column') or normalize-space(@subject) != ''">A cdl:end_column note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:language') or normalize-space(@subject) != ''">A cdl:language note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:position') or normalize-space(@subject) != ''">A cdl:position note needs a subject: the name of what it holds.</assert>
         <assert test="not(@type = 'cdl:relevant') or @subject = 'xlsform-xpath'">A cdl:relevant note needs subject="xlsform-xpath": its text is an expression in that syntax.</assert>

registry/conventions/ddiFields.jsonld

@@ -55,7 +55,7 @@
               "var",
               "varGrp"
             ],
-            "text": "the appearance cell, lowercased"
+            "text": "the appearance cell as authored (readers compare it lowercased)"
           },
           "parameters": {
             "type": "cdl:parameters",
@@ -213,6 +213,14 @@
             ],
             "subject": "the row's name, a space, the column",
             "text": "the cell of a column the model doesn't lift, of a note row or cdl:row"
+          },
+          "end_column": {
+            "type": "cdl:end_column",
+            "on": [
+              "varGrp[@type='section' or @type='grid']"
+            ],
+            "subject": "the column",
+            "text": "a cell of the group's end_group row but its type (a name, Kobo's $kuid)"
           }
         },
         "fields": {
@@ -318,6 +326,9 @@
           },
           "InstrumentChoice.columns": {
             "cdlNote": "cdl:choice_column"
+          },
+          "GroupItem.endColumns": {
+            "cdlNote": "cdl:end_column"
           }
         },
         "liftedColumns": {

src/conventions/columns.ts

@@ -29,13 +29,12 @@ function cellText(value: unknown): string {
  */
 export function otherColumns(
   row: Record<string, unknown>,
-  sheet: Sheet,
+  sheet: Sheet | null,
 ): Record<string, string> {
   const out: Record<string, string> = {};
   for (const [key, value] of Object.entries(row)) {
-    if (key.startsWith('_') || liftedBy[sheet].has(key.split('::')[0])) {
-      continue;
-    }
+    if (key.startsWith('_')) continue;
+    if (sheet && liftedBy[sheet].has(key.split('::')[0])) continue;
     const cells: Array<[string, unknown]> =
       value !== null && typeof value === 'object'
         ? Object.entries(value).map(([lang, v]) => [`${key}::${lang}`, v])
@@ -47,3 +46,13 @@ export function otherColumns(
   }
   return out;
 }
+
+/**
+ * A row's every cell but the loader's, by column name, `{ lang: text }`
+ * cells one per language: a settings row, an `end_group` row.
+ */
+export function allColumns(
+  row: Record<string, unknown>,
+): Record<string, string> {
+  return otherColumns(row, null);
+}

src/ddi/codebook.ts

@@ -1009,7 +1009,9 @@ export function buildDdiCodebook(
 
   const dataDscr = root.child('dataDscr');
   const buckets = splitDataVars(dataVars);
-  const described = variables.filter((v) => v.row === undefined);
+  const described = variables.filter(
+    (v) => v.row === undefined && !v.emptyGroup,
+  );
   const ctx = logicContext(described, lang, questionIds(buckets.units));
   addVarGroups(
     dataDscr,

src/ddi/fields.ts

@@ -4,6 +4,7 @@
  * `<notes type="cdl:<field>">`.
  */
 import conventions from '../generated/conventions.js';
+import { allColumns } from '../conventions/columns.js';
 import { GRID_APPEARANCE } from '../conventions/grid.js';
 import { parseParameters } from '../utils/parameters.js';
 import { TYPE_MAPPINGS } from '../generated/TypeMappings.js';
@@ -28,7 +29,9 @@ export function addFieldNotes(el: XmlElement, v: Variable): void {
     el.textChild('notes', v.default, { type: NOTES.default.type });
   }
   if (v.appearance) {
-    el.textChild('notes', v.appearance, { type: NOTES.appearance.type });
+    el.textChild('notes', v.appearanceCell ?? v.appearance, {
+      type: NOTES.appearance.type,
+    });
   }
   // As authored (#160): a range's bounds are its valrng too, and a
   // guidance_hint inside is its ivuInstr too, but only the cell says so.
@@ -60,14 +63,19 @@ export function addGroupFieldNotes(
       type: NOTES.hint.type,
     });
   }
-  if (group.appearance && !(grid && group.appearance === GRID_APPEARANCE)) {
-    el.textChild('notes', group.appearance, { type: NOTES.appearance.type });
+  const gridOnly =
+    grid && group.appearance === GRID_APPEARANCE && !group.appearanceCell;
+  if (group.appearance && !gridOnly) {
+    el.textChild('notes', group.appearanceCell ?? group.appearance, {
+      type: NOTES.appearance.type,
+    });
   }
   // Its txt is its name, for readers that show one (#160).
   if (group.unlabelled) {
     el.textChild('notes', 'yes', { type: NOTES.no_label.type });
   }
   addColumnNotes(el, group.columns);
+  addColumnNotes(el, group.endColumns, NOTES.end_column.type);
 }
 
 /** Settings in a standard element: `titl`, `IDNo`, `verStmt/version`. */
@@ -82,13 +90,19 @@ export function addSettingNotes(
   stdy: XmlElement,
   settings: Record<string, unknown>,
 ): void {
-  const keys = Object.keys(settings).sort();
-  for (const [key, value] of keys.map((k) => [k, settings[k]] as const)) {
-    if (STANDARD_SETTINGS.has(key)) continue;
-    if (typeof value !== 'string' && typeof value !== 'number') continue;
-    const text = String(value).trim();
-    if (!text) continue;
-    stdy.textChild('notes', text, { type: NOTES.setting.type, subject: key });
+  // A setting per language is one note per `<key>::<language>` (#160).
+  const cells = allColumns(settings);
+  for (const column of Object.keys(cells).sort()) {
+    const base = column.split('::')[0];
+    // A standard setting, or one of its languages when titl/parTitl has them.
+    if (STANDARD_SETTINGS.has(column)) continue;
+    if (STANDARD_SETTINGS.has(base) && typeof settings[base] === 'object') {
+      continue;
+    }
+    stdy.textChild('notes', cells[column], {
+      type: NOTES.setting.type,
+      subject: column,
+    });
   }
 }
 
@@ -161,7 +175,7 @@ export function addRowFieldNotes(stdy: XmlElement, v: Variable): void {
     });
   }
   if (v.appearance) {
-    stdy.textChild('notes', v.appearance, {
+    stdy.textChild('notes', v.appearanceCell ?? v.appearance, {
       type: NOTES.row_appearance.type,
       subject,
     });

src/ddi/fromInstrument.ts

@@ -352,6 +352,7 @@ function pushRow(q: QuestionItem, ctx: GroupContext, state: ProjectState) {
     }),
     ...(q.relevant.trim() ? { relevant: q.relevant.trim() } : {}),
     ...(q.appearance ? { appearance: q.appearance } : {}),
+    ...appearanceCell(q),
     ...(translations ? { translations } : {}),
     ...columnsOf(q),
     group: ctx.path,
@@ -364,6 +365,15 @@ function pushRow(q: QuestionItem, ctx: GroupContext, state: ProjectState) {
   });
 }
 
+/** The appearance cell as authored, when its case isn't the model's (#160). */
+function appearanceCell(item: Item): { appearanceCell?: string } {
+  const cell = item.row['appearance'];
+  const raw = typeof cell === 'string' ? cell.trim() : '';
+  return raw && raw !== item.appearance && raw.toLowerCase() === item.appearance
+    ? { appearanceCell: raw }
+    : {};
+}
+
 /** An item's or choice's columns the model doesn't lift, absent when none. */
 function columnsOf(x: { columns?: Record<string, string> }): {
   columns?: Record<string, string>;
@@ -432,6 +442,7 @@ function pushQuestion(
     ...logicOf(q, state.lang),
     ...(translations ? { translations } : {}),
     ...(added ? { orOther: otherOrigin(q) } : {}),
+    ...appearanceCell(q),
     ...columnsOf(q),
   });
 
@@ -461,7 +472,12 @@ function project(items: Item[], ctx: GroupContext, state: ProjectState): void {
         ? {}
         : { unlabelled: true }),
       ...columnsOf(item),
+      ...appearanceCell(item),
+      ...(item.endColumns && Object.keys(item.endColumns).length
+        ? { endColumns: item.endColumns }
+        : {}),
     };
+    const before = state.variables.length;
     project(
       item.children,
       {
@@ -473,6 +489,22 @@ function project(items: Item[], ctx: GroupContext, state: ProjectState): void {
       },
       state,
     );
+    // Nothing in it the codebook has: still a group, placed where it was.
+    if (state.variables.length === before) {
+      state.variables.push({
+        name: item.name,
+        type: 'begin_group',
+        label: '',
+        group: path,
+        groupLabel: label,
+        groupAppearance: item.appearance,
+        listName: '',
+        vocab: '',
+        choices: [],
+        groups: [...ctx.groups, group],
+        emptyGroup: true,
+      });
+    }
   }
 }
 

src/ddi/notes.ts

@@ -103,7 +103,7 @@ export function classifyNotes(variables: Variable[]): ClassifiedNotes {
   let pending: number[] = [];
 
   variables.forEach((v, i) => {
-    if (v.row !== undefined) {
+    if (v.row !== undefined || v.emptyGroup) {
       placed.push(i);
       return;
     }
@@ -181,7 +181,7 @@ function placements(
       });
       emptyGroups.push(chain.slice(0, depth + 1));
     });
-    positions.push(positionOf(variables, i));
+    if (!variables[i].emptyGroup) positions.push(positionOf(variables, i));
   }
   return { positions, emptyGroups };
 }

src/ddi/types.ts

@@ -47,6 +47,10 @@ export interface DdiGroup {
   unlabelled?: boolean;
   /** Its columns the model doesn't lift: `cdl:column` notes (#160). */
   columns?: Record<string, string>;
+  /** Its `end_group` row's cells: `cdl:end_column` notes (#160). */
+  endColumns?: Record<string, string>;
+  /** The appearance cell when its case isn't `appearance`'s (#160). */
+  appearanceCell?: string;
   /** `hint` in the form's other languages. */
   hintTranslations?: Translations;
 }
@@ -119,6 +123,13 @@ export interface Variable {
   orOther?: 'shorthand' | 'added';
   /** The `or_other` companion the projection added, not an authored row. */
   synthesized?: boolean;
+  /** The appearance cell when its case isn't `appearance`'s (#160). */
+  appearanceCell?: string;
+  /**
+   * Stands for a group with nothing in it the codebook has (`groups` ends
+   * with it): a section `varGrp` placed by `cdl:position` (#160).
+   */
+  emptyGroup?: boolean;
   /**
    * Its columns the model doesn't lift: `cdl:column` notes, a row's without
    * data or a note's `cdl:row_column` (#160).

src/generated/conventions.json

@@ -27,7 +27,7 @@
             "var",
             "varGrp"
           ],
-          "text": "the appearance cell, lowercased"
+          "text": "the appearance cell as authored (readers compare it lowercased)"
         },
         "parameters": {
           "type": "cdl:parameters",
@@ -185,6 +185,14 @@
           ],
           "subject": "the row's name, a space, the column",
           "text": "the cell of a column the model doesn't lift, of a note row or cdl:row"
+        },
+        "end_column": {
+          "type": "cdl:end_column",
+          "on": [
+            "varGrp[@type='section' or @type='grid']"
+          ],
+          "subject": "the column",
+          "text": "a cell of the group's end_group row but its type (a name, Kobo's $kuid)"
         }
       },
       "fields": {
@@ -290,6 +298,9 @@
         },
         "InstrumentChoice.columns": {
           "cdlNote": "cdl:choice_column"
+        },
+        "GroupItem.endColumns": {
+          "cdlNote": "cdl:end_column"
         }
       },
       "liftedColumns": {

src/generated/conventions.ts

@@ -36,7 +36,7 @@ const conventions = {
             "var",
             "varGrp"
           ],
-          "text": "the appearance cell, lowercased"
+          "text": "the appearance cell as authored (readers compare it lowercased)"
         },
         "parameters": {
           "type": "cdl:parameters",
@@ -194,6 +194,14 @@ const conventions = {
           ],
           "subject": "the row's name, a space, the column",
           "text": "the cell of a column the model doesn't lift, of a note row or cdl:row"
+        },
+        "end_column": {
+          "type": "cdl:end_column",
+          "on": [
+            "varGrp[@type='section' or @type='grid']"
+          ],
+          "subject": "the column",
+          "text": "a cell of the group's end_group row but its type (a name, Kobo's $kuid)"
         }
       },
       "fields": {
@@ -299,6 +307,9 @@ const conventions = {
         },
         "InstrumentChoice.columns": {
           "cdlNote": "cdl:choice_column"
+        },
+        "GroupItem.endColumns": {
+          "cdlNote": "cdl:end_column"
         }
       },
       "liftedColumns": {