Skip to content

feat(ddi): columns the model doesn't lift ride along (#160) #164

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jstet merged 1 commit into from
Sep 27, 2026
Merged

feat(ddi): columns the model doesn't lift ride along (#160) #164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions codegen/schematron.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,11 @@ def generate_schematron(registry: dict[str, Any], output: Path) -> None:
{subject_rules}
</rule>
"""
# At most one note of each cdl: type per element and language (on stdyDscr,
# where the subject names what a note is about: per subject too).
# At most one note of each cdl: type per element, subject and language
# (a subject names what the note is about: a column, a row).
cdl_note_uniqueness = """\
<rule context="%P%var | %P%varGrp">
<assert test="every $t in distinct-values(%P%notes[starts-with(@type, 'cdl:')]/@type) satisfies every $l in distinct-values(%P%notes[@type = $t]/string(@xml:lang)) satisfies count(%P%notes[@type = $t][string(@xml:lang) = $l]) &lt;= 1"><value-of select="@name"/> has more than one note of one cdl: type in one language.</assert>
<assert test="every $t in distinct-values(%P%notes[starts-with(@type, 'cdl:')]/@type) satisfies every $s in distinct-values(%P%notes[@type = $t]/string(@subject)) satisfies every $l in distinct-values(%P%notes[@type = $t][string(@subject) = $s]/string(@xml:lang)) satisfies count(%P%notes[@type = $t][string(@subject) = $s][string(@xml:lang) = $l]) &lt;= 1"><value-of select="@name"/> has more than one note of one cdl: type in one language.</assert>
</rule>
<rule context="%P%stdyDscr">
<assert test="every $t in distinct-values(%P%notes[starts-with(@type, 'cdl:')]/@type) satisfies every $s in distinct-values(%P%notes[@type = $t]/string(@subject)) satisfies every $l in distinct-values(%P%notes[@type = $t][string(@subject) = $s]/string(@xml:lang)) satisfies count(%P%notes[@type = $t][string(@subject) = $s][string(@xml:lang) = $l]) &lt;= 1">The study has more than one note of one cdl: type about one subject in one language.</assert>
Expand Down
14 changes: 10 additions & 4 deletions ddi-validation/schematron/ddi_custom_rules.sch
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,26 +216,32 @@

<pattern id="cdl_notes">
<rule context="ddi:notes[starts-with(@type, 'cdl:')]">
<assert test="@type = 'cdl:appearance' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:no_label' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
<assert test="@type = 'cdl:appearance' or @type = 'cdl:choice_column' or @type = 'cdl:column' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:no_label' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_column' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:choice_column, cdl:column, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_column, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
<assert test="not(@type = 'cdl:choice_column') or normalize-space(@subject) != ''">A cdl:choice_column note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:column') or normalize-space(@subject) != ''">A cdl:column note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:constraint') or @subject = 'xlsform-xpath'">A cdl:constraint note needs subject="xlsform-xpath": its text is an expression in that syntax.</assert>
<assert test="not(@type = 'cdl:language') or normalize-space(@subject) != ''">A cdl:language note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:position') or normalize-space(@subject) != ''">A cdl:position note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:relevant') or @subject = 'xlsform-xpath'">A cdl:relevant note needs subject="xlsform-xpath": its text is an expression in that syntax.</assert>
<assert test="not(@type = 'cdl:row') or normalize-space(@subject) != ''">A cdl:row note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:row_appearance') or normalize-space(@subject) != ''">A cdl:row_appearance note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:row_column') or normalize-space(@subject) != ''">A cdl:row_column note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:row_hint') or normalize-space(@subject) != ''">A cdl:row_hint note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:row_label') or normalize-space(@subject) != ''">A cdl:row_label note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:row_relevant') or normalize-space(@subject) != ''">A cdl:row_relevant note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:setting') or normalize-space(@subject) != ''">A cdl:setting note needs a subject: the name of what it holds.</assert>
</rule>
<rule context="notes[starts-with(@type, 'cdl:')]">
<assert test="@type = 'cdl:appearance' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:no_label' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
<assert test="@type = 'cdl:appearance' or @type = 'cdl:choice_column' or @type = 'cdl:column' or @type = 'cdl:constraint' or @type = 'cdl:constraint_message' or @type = 'cdl:default' or @type = 'cdl:exclusive' or @type = 'cdl:hint' or @type = 'cdl:language' or @type = 'cdl:list' or @type = 'cdl:no_label' or @type = 'cdl:note_names' or @type = 'cdl:or_other' or @type = 'cdl:other_label' or @type = 'cdl:parameters' or @type = 'cdl:position' or @type = 'cdl:relevant' or @type = 'cdl:required' or @type = 'cdl:row' or @type = 'cdl:row_appearance' or @type = 'cdl:row_column' or @type = 'cdl:row_hint' or @type = 'cdl:row_label' or @type = 'cdl:row_relevant' or @type = 'cdl:setting'">Note type "<value-of select="@type"/>" is not in the CDL vocabulary (cdl:appearance, cdl:choice_column, cdl:column, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_column, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting).</assert>
<assert test="not(@type = 'cdl:choice_column') or normalize-space(@subject) != ''">A cdl:choice_column note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:column') or normalize-space(@subject) != ''">A cdl:column note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:constraint') or @subject = 'xlsform-xpath'">A cdl:constraint note needs subject="xlsform-xpath": its text is an expression in that syntax.</assert>
<assert test="not(@type = 'cdl:language') or normalize-space(@subject) != ''">A cdl:language note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:position') or normalize-space(@subject) != ''">A cdl:position note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:relevant') or @subject = 'xlsform-xpath'">A cdl:relevant note needs subject="xlsform-xpath": its text is an expression in that syntax.</assert>
<assert test="not(@type = 'cdl:row') or normalize-space(@subject) != ''">A cdl:row note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:row_appearance') or normalize-space(@subject) != ''">A cdl:row_appearance note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:row_column') or normalize-space(@subject) != ''">A cdl:row_column note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:row_hint') or normalize-space(@subject) != ''">A cdl:row_hint note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:row_label') or normalize-space(@subject) != ''">A cdl:row_label note needs a subject: the name of what it holds.</assert>
<assert test="not(@type = 'cdl:row_relevant') or normalize-space(@subject) != ''">A cdl:row_relevant note needs a subject: the name of what it holds.</assert>
Expand All @@ -245,13 +251,13 @@

<pattern id="cdl_note_uniqueness">
<rule context="ddi:var | ddi:varGrp">
<assert test="every $t in distinct-values(ddi:notes[starts-with(@type, 'cdl:')]/@type) satisfies every $l in distinct-values(ddi:notes[@type = $t]/string(@xml:lang)) satisfies count(ddi:notes[@type = $t][string(@xml:lang) = $l]) &lt;= 1"><value-of select="@name"/> has more than one note of one cdl: type in one language.</assert>
<assert test="every $t in distinct-values(ddi:notes[starts-with(@type, 'cdl:')]/@type) satisfies every $s in distinct-values(ddi:notes[@type = $t]/string(@subject)) satisfies every $l in distinct-values(ddi:notes[@type = $t][string(@subject) = $s]/string(@xml:lang)) satisfies count(ddi:notes[@type = $t][string(@subject) = $s][string(@xml:lang) = $l]) &lt;= 1"><value-of select="@name"/> has more than one note of one cdl: type in one language.</assert>
</rule>
<rule context="ddi:stdyDscr">
<assert test="every $t in distinct-values(ddi:notes[starts-with(@type, 'cdl:')]/@type) satisfies every $s in distinct-values(ddi:notes[@type = $t]/string(@subject)) satisfies every $l in distinct-values(ddi:notes[@type = $t][string(@subject) = $s]/string(@xml:lang)) satisfies count(ddi:notes[@type = $t][string(@subject) = $s][string(@xml:lang) = $l]) &lt;= 1">The study has more than one note of one cdl: type about one subject in one language.</assert>
</rule>
<rule context="var | varGrp">
<assert test="every $t in distinct-values(notes[starts-with(@type, 'cdl:')]/@type) satisfies every $l in distinct-values(notes[@type = $t]/string(@xml:lang)) satisfies count(notes[@type = $t][string(@xml:lang) = $l]) &lt;= 1"><value-of select="@name"/> has more than one note of one cdl: type in one language.</assert>
<assert test="every $t in distinct-values(notes[starts-with(@type, 'cdl:')]/@type) satisfies every $s in distinct-values(notes[@type = $t]/string(@subject)) satisfies every $l in distinct-values(notes[@type = $t][string(@subject) = $s]/string(@xml:lang)) satisfies count(notes[@type = $t][string(@subject) = $s][string(@xml:lang) = $l]) &lt;= 1"><value-of select="@name"/> has more than one note of one cdl: type in one language.</assert>
</rule>
<rule context="stdyDscr">
<assert test="every $t in distinct-values(notes[starts-with(@type, 'cdl:')]/@type) satisfies every $s in distinct-values(notes[@type = $t]/string(@subject)) satisfies every $l in distinct-values(notes[@type = $t][string(@subject) = $s]/string(@xml:lang)) satisfies count(notes[@type = $t][string(@subject) = $s][string(@xml:lang) = $l]) &lt;= 1">The study has more than one note of one cdl: type about one subject in one language.</assert>
Expand Down
60 changes: 58 additions & 2 deletions registry/conventions/ddiFields.jsonld
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,32 @@
"varGrp[@type='section' or @type='grid']"
],
"text": "yes: the group has no label; its txt is its name, for readers that show one"
},
"column": {
"type": "cdl:column",
"on": [
"var",
"varGrp"
],
"subject": "the column, as the sheet names it (media::image::Deutsch (de))",
"text": "the cell of a survey column the model doesn't lift (liftedColumns)"
},
"choice_column": {
"type": "cdl:choice_column",
"on": [
"var",
"varGrp"
],
"subject": "the choice's code, a space, the column",
"text": "the cell of a choices column the model doesn't lift; on the first question that uses the list"
},
"row_column": {
"type": "cdl:row_column",
"on": [
"stdyDscr"
],
"subject": "the row's name, a space, the column",
"text": "the cell of a column the model doesn't lift, of a note row or cdl:row"
}
},
"fields": {
Expand All @@ -208,7 +234,7 @@
"note": "On a var (a select_multiple's or pair's on its varGrp), and on a section; a grid's table-list is its type='grid', so only a grid appearance other than that is noted."
},
"ItemBase.row": {
"loss": "Columns the model does not lift (media, calculation, …) are not carried."
"loss": "The source row itself; its columns the model doesn't lift are `columns`."
},
"GroupItem.kind": {
"ddi": "a group is a varGrp"
Expand Down Expand Up @@ -268,7 +294,7 @@
},
"InstrumentChoice.row": {
"ddi": "exclusive: a cdl:exclusive note on the select_multiple's varGrp listing the codes",
"note": "The choice sheet's other columns are not carried."
"note": "Its other columns are `columns`."
},
"Instrument.languages": {
"ddi": "xml:lang siblings of every text (convention:languageTagging); a language's own name in a cdl:language note"
Expand All @@ -285,7 +311,37 @@
},
"Instrument.body": {
"ddi": "dataDscr: varGrp and var, in survey order; note rows in preQTxt (names: cdl:note_names) or as stdyDscr notes type='instruction'; rows without data as cdl:row; their fields in cdl:row_*; where the stdyDscr ones are: cdl:position"
},
"ItemBase.columns": {
"cdlNote": "cdl:column",
"note": "On the var (a select_multiple's or pair's varGrp) or the group's varGrp; a note's or data-less row's as cdl:row_column."
},
"InstrumentChoice.columns": {
"cdlNote": "cdl:choice_column"
}
},
"liftedColumns": {
"description": "The sheet columns the model reads into its own fields (a column with a language suffix, label::de, by its base). Every other non-empty column is an item's or choice's `columns`, carried as a cdl:column note (#160). A column starting with `_` is the loader's, not the form's.",
"survey": [
"type",
"name",
"label",
"hint",
"guidance_hint",
"relevant",
"appearance",
"constraint",
"constraint_message",
"required",
"default",
"parameters"
],
"choices": [
"list_name",
"name",
"label",
"exclusive"
]
}
}
}
Expand Down
49 changes: 49 additions & 0 deletions src/conventions/columns.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/**
* The sheet columns the model doesn't lift (`convention:ddiFields`
* `liftedColumns`, #160): kept by name, so a codebook can carry them and an
* XLSForm get them back.
*/
import conventions from '../generated/conventions.js';

const LIFTED = conventions.conventions.ddiFields.liftedColumns;

type Sheet = 'survey' | 'choices';

const liftedBy: Record<Sheet, Set<string>> = {
survey: new Set(LIFTED.survey),
choices: new Set(LIFTED.choices),
};

function cellText(value: unknown): string {
if (typeof value === 'string') return value.trim();
if (typeof value === 'number' || typeof value === 'boolean') {
return String(value);
}
return '';
}

/**
* A row's columns the model doesn't lift, non-empty, by column name. A
* `{ lang: text }` cell is one column per language (`media::image::de`). A
* column starting with `_` is the loader's, not the form's.
*/
export function otherColumns(
row: Record<string, unknown>,
sheet: Sheet,
): Record<string, string> {
const out: Record<string, string> = {};
for (const [key, value] of Object.entries(row)) {
if (key.startsWith('_') || liftedBy[sheet].has(key.split('::')[0])) {
continue;
}
const cells: Array<[string, unknown]> =
value !== null && typeof value === 'object'
? Object.entries(value).map(([lang, v]) => [`${key}::${lang}`, v])
: [[key, value]];
for (const [column, cell] of cells) {
const text = cellText(cell);
if (text) out[column] = text;
}
}
return out;
}
3 changes: 3 additions & 0 deletions src/ddi/codebook.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import conventions from '../generated/conventions.js';
import { Choice, DdiGroup, Translations, Variable } from './types.js';
import { localizedChild, textsOf } from './translations.js';
import {
addChoiceColumnNotes,
addExclusiveNote,
addFieldNotes,
addGroupFieldNotes,
Expand Down Expand Up @@ -203,6 +204,7 @@ function addVarElement(parent: XmlElement, spec: AddVarSpec): XmlElement {
addLogicNotes(varEl, spec.logic.v);
addFieldNotes(varEl, spec.logic.v);
addListNote(varEl, spec.logic.v, spec.opts?.listDefault ?? name);
addChoiceColumnNotes(varEl, spec.logic.v, spec.logic.ctx.listsWritten);
}
addNoteNames(varEl, spec.opts?.noteNames);

Expand Down Expand Up @@ -296,6 +298,7 @@ function addGroupLogic(
addFieldNotes(grpEl, v);
addExclusiveNote(grpEl, v.choices);
addListNote(grpEl, v, v.name);
addChoiceColumnNotes(grpEl, v, ctx.listsWritten);
}

/** A semi-open pair written with the `or_other` shorthand (#160). */
Expand Down
Loading
Loading