From bd85087e8cf63fcbc9c08643b175b548db905aa7 Mon Sep 17 00:00:00 2001 From: jstet Date: Sun, 27 Sep 2026 20:25:06 +0200 Subject: [PATCH] feat(ddi)!: cells as authored, unlabelled groups, messages without a constraint (#160) Closes the small losses #162 left: - constraint_message is kept without a constraint - cdl:required holds the cell as authored (TRUE, true), yes by default - cdl:parameters holds the cell as authored, guidance_hint included (ivuInstr still has it), not only the parameters DDI has no element for - cdl:no_label marks a group without a label (its txt stays its name) - notes whose joined preQTxt can't be split at blank lines (a blank line in one, a language only some have) keep their own text in cdl:row_label - the form model's languages come from every text column, not only label: a language only a hint has is no longer dropped The generated forms cover all of these. BREAKING CHANGE: cdl:parameters now includes a guidance_hint, and cdl:constraint_message appears without a constraint. ddi2xlsform gives back a guidance_hint inside parameters as it was, which pyxform rejects when it has spaces (hints_survey, as its source). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../schematron/ddi_custom_rules.sch | 4 +- registry/conventions/ddiFields.jsonld | 9 ++- registry/conventions/logicMapping.jsonld | 2 +- src/ddi/codebook.ts | 5 ++ src/ddi/fields.ts | 33 ++++---- src/ddi/fromInstrument.ts | 17 ++++- src/ddi/logic.ts | 4 +- src/ddi/notes.ts | 22 ++++++ src/ddi/types.ts | 4 + src/generated/conventions.json | 11 ++- src/generated/conventions.ts | 11 ++- src/instrument/fromDdi.ts | 24 +++++- src/instrument/fromXlsform.ts | 10 ++- src/pipelines/ddi2xlsform/README.md | 21 ++--- src/xlsform/fromInstrument.ts | 8 +- tests/fixtures/surveys/hints_survey/ddi.xml | 1 + .../surveys/hints_survey/ddi2xlsform.json | 2 +- tests/fixtures/surveys/testA/ddi.xml | 30 +++++--- tests/fixtures/surveys/testA/ddi2xlsform.json | 36 ++++----- tests/fixtures/surveys/testB/ddi.xml | 76 +++++++++---------- tests/fixtures/surveys/testB/ddi2xlsform.json | 76 +++++++++---------- tests/ts/contract/canonicalInstrument.ts | 26 +++---- .../ts/contract/ddiRoundtripGenerated.test.ts | 46 +++++++++-- tests/ts/unit/ddi/fields.test.ts | 6 +- tests/ts/unit/ddi/fromDdi.test.ts | 33 ++++++++ tests/ts/unit/ddi/logic.test.ts | 6 +- tests/validation/test_xlsform_pyxform.py | 19 ++++- 27 files changed, 352 insertions(+), 190 deletions(-) diff --git a/ddi-validation/schematron/ddi_custom_rules.sch b/ddi-validation/schematron/ddi_custom_rules.sch index 1d60f10..88dd66d 100644 --- a/ddi-validation/schematron/ddi_custom_rules.sch +++ b/ddi-validation/schematron/ddi_custom_rules.sch @@ -216,7 +216,7 @@ - Note type "" is not in the CDL vocabulary (cdl:appearance, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting). + Note type "" is not in the CDL vocabulary (cdl:appearance, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting). A cdl:constraint note needs subject="xlsform-xpath": its text is an expression in that syntax. A cdl:language note needs a subject: the name of what it holds. A cdl:position note needs a subject: the name of what it holds. @@ -229,7 +229,7 @@ A cdl:setting note needs a subject: the name of what it holds. - Note type "" is not in the CDL vocabulary (cdl:appearance, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting). + Note type "" is not in the CDL vocabulary (cdl:appearance, cdl:constraint, cdl:constraint_message, cdl:default, cdl:exclusive, cdl:hint, cdl:language, cdl:list, cdl:no_label, cdl:note_names, cdl:or_other, cdl:other_label, cdl:parameters, cdl:position, cdl:relevant, cdl:required, cdl:row, cdl:row_appearance, cdl:row_hint, cdl:row_label, cdl:row_relevant, cdl:setting). A cdl:constraint note needs subject="xlsform-xpath": its text is an expression in that syntax. A cdl:language note needs a subject: the name of what it holds. A cdl:position note needs a subject: the name of what it holds. diff --git a/registry/conventions/ddiFields.jsonld b/registry/conventions/ddiFields.jsonld index 96fbfba..8f1848f 100644 --- a/registry/conventions/ddiFields.jsonld +++ b/registry/conventions/ddiFields.jsonld @@ -63,7 +63,7 @@ "var", "varGrp" ], - "text": "the parameters not in a standard element, space-separated key=value; a range's start and end as authored (valrng/range has them too)" + "text": "the parameters cell as authored (a range's start and end are also valrng/range, a guidance_hint inside it also ivuInstr)" }, "hint": { "type": "cdl:hint", @@ -180,6 +180,13 @@ ], "localized": true, "text": "a select_multiple's authored other choice's label: the pair has no binary var for it" + }, + "no_label": { + "type": "cdl:no_label", + "on": [ + "varGrp[@type='section' or @type='grid']" + ], + "text": "yes: the group has no label; its txt is its name, for readers that show one" } }, "fields": { diff --git a/registry/conventions/logicMapping.jsonld b/registry/conventions/logicMapping.jsonld index d5d3505..69d7218 100644 --- a/registry/conventions/logicMapping.jsonld +++ b/registry/conventions/logicMapping.jsonld @@ -76,7 +76,7 @@ "lsSyntax": "Y/N", "ddi": { "element": "var/notes[@type='cdl:required']", - "note": "Written only for a required question, with the text `yes`. It tells 'not asked' (universe) from 'refused' apart.", + "note": "Written only for a required question, with the required cell as authored (`yes` when it is `yes` or the source has none, e.g. `TRUE` as written). It tells 'not asked' (universe) from 'refused' apart.", "lossy": false } } diff --git a/src/ddi/codebook.ts b/src/ddi/codebook.ts index 2274e48..0756e19 100644 --- a/src/ddi/codebook.ts +++ b/src/ddi/codebook.ts @@ -33,6 +33,7 @@ import { addListNote, addNoteNames, addRowFieldNotes, + addRowLabel, addRowNotes, addSettingNotes, references, @@ -585,6 +586,8 @@ interface StudyNotes { rows: Variable[]; /** Every note row, for its fields. */ notes: Variable[]; + /** Inline notes that need their own text (`cdl:row_label`). */ + unsplittable: Variable[]; /** Where the orphans and rows are. */ positions: Position[]; /** Language tag → the form's name for it. */ @@ -633,6 +636,7 @@ function addStudyDscr( kept.add(row.name); } for (const note of study.notes) addRowFieldNotes(stdy, note); + for (const note of study.unsplittable) addRowLabel(stdy, note); for (const p of study.positions) { if (!p.name || !(p.group || kept.has(p.name))) continue; stdy.textChild('notes', `in=${p.in} after=${p.after}`, { @@ -994,6 +998,7 @@ export function buildDdiCodebook( orphans: classified.orphanNotes, rows: classified.rows, notes: variables.filter((v) => v.type === 'note' && v.row === undefined), + unsplittable: classified.unsplittable, positions: classified.positions, languageNames, }); diff --git a/src/ddi/fields.ts b/src/ddi/fields.ts index cc35225..f011d36 100644 --- a/src/ddi/fields.ts +++ b/src/ddi/fields.ts @@ -13,21 +13,6 @@ import type { XmlElement } from './xml.js'; const NOTES = conventions.conventions.ddiFields.notes; -/** - * The `parameters` DDI has no element for, as `key=value` tokens: without - * `guidance_hint` (`ivuInstr`). A range's `start`/`end` stay as authored - * (#160), though its `valrng` has them too: a bound equal to the default is - * otherwise not told apart from none. - */ -export function otherParameters(v: Variable): string { - const tokens: string[] = []; - for (const part of (v.parameters ?? '').split(';')) { - if (/^\s*guidance_hint\s*=/.test(part)) continue; - tokens.push(...part.split(/[\s,]+/).filter(Boolean)); - } - return tokens.join(' '); -} - /** A `range`'s bounds, the registry's defaults where not authored. */ export function rangeBounds(v: Variable): { min: string; max: string } { const values = { @@ -45,9 +30,10 @@ export function addFieldNotes(el: XmlElement, v: Variable): void { if (v.appearance) { el.textChild('notes', v.appearance, { type: NOTES.appearance.type }); } - const parameters = otherParameters(v); - if (parameters) { - el.textChild('notes', parameters, { type: NOTES.parameters.type }); + // As authored (#160): a range's bounds are its valrng too, and a + // guidance_hint inside is its ivuInstr too, but only the cell says so. + if (v.parameters) { + el.textChild('notes', v.parameters, { type: NOTES.parameters.type }); } } @@ -76,6 +62,10 @@ export function addGroupFieldNotes( if (group.appearance && !(grid && group.appearance === GRID_APPEARANCE)) { el.textChild('notes', group.appearance, { type: NOTES.appearance.type }); } + // Its txt is its name, for readers that show one (#160). + if (group.unlabelled) { + el.textChild('notes', 'yes', { type: NOTES.no_label.type }); + } } /** Settings in a standard element: `titl`, `IDNo`, `verStmt/version`. */ @@ -138,13 +128,18 @@ export function addRowNotes(stdy: XmlElement, v: Variable): void { type: NOTES.row.type, subject: v.name, }); + addRowLabel(stdy, v); + addRowFieldNotes(stdy, v); +} + +/** A row's own label by its name (`cdl:row_label`), in every language. */ +export function addRowLabel(stdy: XmlElement, v: Variable): void { if (v.label) { localizedChild(stdy, 'notes', v.label, textsOf(v, 'label'), { type: NOTES.row_label.type, subject: v.name, }); } - addRowFieldNotes(stdy, v); } /** A note row's or data-less row's hint, relevant and appearance, by its name. */ diff --git a/src/ddi/fromInstrument.ts b/src/ddi/fromInstrument.ts index f221172..fcc826b 100644 --- a/src/ddi/fromInstrument.ts +++ b/src/ddi/fromInstrument.ts @@ -124,6 +124,13 @@ function groupsOf(ctx: GroupContext): Pick { return ctx.groups.length ? { groups: ctx.groups } : {}; } +/** The `required` cell as authored, when it isn't the plain `yes` (#160). */ +function requiredCell(q: QuestionItem): Pick { + const cell = q.row['required']; + const text = typeof cell === 'string' ? cell.trim() : ''; + return text && text !== 'yes' ? { requiredCell: text } : {}; +} + /** A question's own logic (#151) and form fields (#153), absent when empty. */ function logicOf( q: QuestionItem, @@ -134,6 +141,7 @@ function logicOf( | 'constraint' | 'constraintMessage' | 'required' + | 'requiredCell' | 'default' | 'appearance' | 'parameters' @@ -143,8 +151,8 @@ function logicOf( return { ...(relevant ? { relevant } : {}), ...(q.constraint ? { constraint: q.constraint } : {}), - ...(q.constraint && message ? { constraintMessage: message } : {}), - ...(q.required ? { required: true } : {}), + ...(message ? { constraintMessage: message } : {}), + ...(q.required ? { required: true, ...requiredCell(q) } : {}), ...(q.default ? { default: q.default } : {}), ...(q.appearance ? { appearance: q.appearance } : {}), ...(q.parameters.trim() ? { parameters: q.parameters.trim() } : {}), @@ -392,7 +400,7 @@ function pushQuestion( hint: q.hint, guidanceHint: q.guidanceHint, groupLabel: ctx.labelText, - ...(q.constraint ? { constraintMessage: q.constraintMessage } : {}), + constraintMessage: q.constraintMessage, }, state.others, ); @@ -438,6 +446,9 @@ function project(items: Item[], ctx: GroupContext, state: ProjectState): void { relevant: item.relevant.trim(), hint: pick(item.hint, state.lang).trim(), ...(hintTranslations ? { hintTranslations } : {}), + ...(Object.values(item.label).some((t) => t.trim()) + ? {} + : { unlabelled: true }), }; project( item.children, diff --git a/src/ddi/logic.ts b/src/ddi/logic.ts index c2168a8..a8891df 100644 --- a/src/ddi/logic.ts +++ b/src/ddi/logic.ts @@ -386,6 +386,8 @@ export function addLogicNotes(el: XmlElement, v: Variable): void { }); } if (v.required) { - el.textChild('notes', NOTES.required.text, { type: NOTES.required.type }); + el.textChild('notes', v.requiredCell ?? NOTES.required.text, { + type: NOTES.required.type, + }); } } diff --git a/src/ddi/notes.ts b/src/ddi/notes.ts index c2a5b47..a978264 100644 --- a/src/ddi/notes.ts +++ b/src/ddi/notes.ts @@ -29,6 +29,12 @@ export interface ClassifiedNotes { inlinePreqtxtTranslations: Record; /** `variable.name` → the names of the notes in its `inlinePreqtxt`. */ inlineNames: Record; + /** + * Inline notes whose joined text can't be split back at its blank lines + * (one has a blank line, or lacks a language another has): their own + * texts go in `cdl:row_label` (#160). + */ + unsplittable: Variable[]; /** Notes with no data-carrying successor in the same group. */ orphanNotes: Variable[]; /** Rows without data: metadata rows, matrix headers. */ @@ -75,6 +81,19 @@ function positionOf( return { name, in: group, after: '' }; } +/** Whether notes joined with blank lines split back into the same notes. */ +function splittable(notes: Variable[]): boolean { + if (notes.length < 2) return true; + const texts = notes.map((n): Record => ({ + '': n.label, + ...textsOf(n, 'label'), + })); + const langs = new Set(texts.flatMap((t) => Object.keys(t))); + return texts.every((t) => + [...langs].every((l) => !!t[l] && !t[l].includes('\n\n')), + ); +} + /** Split notes into inline (``) and orphan (``) buckets. */ export function classifyNotes(variables: Variable[]): ClassifiedNotes { const dataVars: Variable[] = []; @@ -109,7 +128,9 @@ export function classifyNotes(variables: Variable[]): ClassifiedNotes { const inlinePreqtxt: Record = {}; const inlinePreqtxtTranslations: Record = {}; const inlineNames: Record = {}; + const unsplittable: Variable[] = []; for (const [name, notes] of Object.entries(inline)) { + if (!splittable(notes)) unsplittable.push(...notes); inlinePreqtxt[name] = notes.map((n) => n.label).join('\n\n'); inlinePreqtxtTranslations[name] = joinTranslations( notes.map((n) => textsOf(n, 'label')), @@ -125,6 +146,7 @@ export function classifyNotes(variables: Variable[]): ClassifiedNotes { inlinePreqtxt, inlinePreqtxtTranslations, inlineNames, + unsplittable, orphanNotes: kept.filter((v) => v.type === 'note'), rows: kept.filter((v) => v.row !== undefined), positions, diff --git a/src/ddi/types.ts b/src/ddi/types.ts index d44196a..486dfc5 100644 --- a/src/ddi/types.ts +++ b/src/ddi/types.ts @@ -41,6 +41,8 @@ export interface DdiGroup { relevant: string; /** Its `hint`: a `cdl:hint` note (#153); `''` when none. */ hint: string; + /** It has no label of its own: `label` is its name (`cdl:no_label`, #160). */ + unlabelled?: boolean; /** `hint` in the form's other languages. */ hintTranslations?: Translations; } @@ -94,6 +96,8 @@ export interface Variable { constraintMessage?: string; /** XLSForm `required`: a `cdl:required` note. */ required?: boolean; + /** The `required` cell when it isn't `yes` (`TRUE`), the note's text (#160). */ + requiredCell?: string; /** XLSForm `default`: a `cdl:default` note (#153). */ default?: string; /** The question's own lowercased `appearance`: a `cdl:appearance` note. */ diff --git a/src/generated/conventions.json b/src/generated/conventions.json index 03de26d..27056b1 100644 --- a/src/generated/conventions.json +++ b/src/generated/conventions.json @@ -35,7 +35,7 @@ "var", "varGrp" ], - "text": "the parameters not in a standard element, space-separated key=value; a range's start and end as authored (valrng/range has them too)" + "text": "the parameters cell as authored (a range's start and end are also valrng/range, a guidance_hint inside it also ivuInstr)" }, "hint": { "type": "cdl:hint", @@ -152,6 +152,13 @@ ], "localized": true, "text": "a select_multiple's authored other choice's label: the pair has no binary var for it" + }, + "no_label": { + "type": "cdl:no_label", + "on": [ + "varGrp[@type='section' or @type='grid']" + ], + "text": "yes: the group has no label; its txt is its name, for readers that show one" } }, "fields": { @@ -525,7 +532,7 @@ "lsSyntax": "Y/N", "ddi": { "element": "var/notes[@type='cdl:required']", - "note": "Written only for a required question, with the text `yes`. It tells 'not asked' (universe) from 'refused' apart.", + "note": "Written only for a required question, with the required cell as authored (`yes` when it is `yes` or the source has none, e.g. `TRUE` as written). It tells 'not asked' (universe) from 'refused' apart.", "lossy": false } } diff --git a/src/generated/conventions.ts b/src/generated/conventions.ts index 4e26b98..4e3bfb4 100644 --- a/src/generated/conventions.ts +++ b/src/generated/conventions.ts @@ -44,7 +44,7 @@ const conventions = { "var", "varGrp" ], - "text": "the parameters not in a standard element, space-separated key=value; a range's start and end as authored (valrng/range has them too)" + "text": "the parameters cell as authored (a range's start and end are also valrng/range, a guidance_hint inside it also ivuInstr)" }, "hint": { "type": "cdl:hint", @@ -161,6 +161,13 @@ const conventions = { ], "localized": true, "text": "a select_multiple's authored other choice's label: the pair has no binary var for it" + }, + "no_label": { + "type": "cdl:no_label", + "on": [ + "varGrp[@type='section' or @type='grid']" + ], + "text": "yes: the group has no label; its txt is its name, for readers that show one" } }, "fields": { @@ -534,7 +541,7 @@ const conventions = { "lsSyntax": "Y/N", "ddi": { "element": "var/notes[@type='cdl:required']", - "note": "Written only for a required question, with the text `yes`. It tells 'not asked' (universe) from 'refused' apart.", + "note": "Written only for a required question, with the required cell as authored (`yes` when it is `yes` or the source has none, e.g. `TRUE` as written). It tells 'not asked' (universe) from 'refused' apart.", "lossy": false } } diff --git a/src/instrument/fromDdi.ts b/src/instrument/fromDdi.ts index c51c98f..8c90407 100644 --- a/src/instrument/fromDdi.ts +++ b/src/instrument/fromDdi.ts @@ -64,6 +64,8 @@ interface ReadState { names: Set; /** A CDL codebook: its lists are named (`cdl:list`), not deduplicated. */ cdl: boolean; + /** `cdl:row_label` notes by subject. */ + rowLabels: Map; /** Section / grid items by `varGrp/@name`, as the tree builds them. */ groupItems: Map; onWarning?: WarningHandler; @@ -229,7 +231,12 @@ function readNotes(q: QuestionItem, node: XmlNode, state: ReadState): void { notesOf(node, LOGIC.constraint_message.type), state, ); - q.required = noteText(node, LOGIC.required.type, state) !== ''; + const required = noteText(node, LOGIC.required.type, state); + q.required = required !== ''; + // A cell other than `yes` (`TRUE`), as the form had it (#160). + if (required && required !== LOGIC.required.text) { + q.row = { ...q.row, required }; + } q.default = noteText(node, FIELDS.default.type, state); q.appearance = noteText(node, FIELDS.appearance.type, state); q.parameters = noteText(node, FIELDS.parameters.type, state); @@ -476,7 +483,12 @@ function leadIn( if (!Object.keys(label).length) return []; const names = ids(noteText(el, FIELDS.note_names.type, state)); if (names.length < 2) return [noteItem(label, near, state, names[0])]; - return splitNotes(label, names) + // Texts the blank lines can't separate are each in a cdl:row_label. + const own = names.map((n) => state.rowLabels.get(n)); + const parts = own.every(Boolean) + ? own.map((nodes) => texts(nodes!, state)) + : splitNotes(label, names); + return parts .map((text, i) => ({ text, name: names[i] })) .filter(({ text }) => Object.keys(text).length) .map(({ text, name }) => noteItem(text, near, state, name)); @@ -592,7 +604,9 @@ function groupItem(id: string, state: ReadState): GroupItem { const item: GroupItem = { kind: 'group', name: gridName(id, state), - label: childTexts(grp, 'txt', state), + label: notesOf(grp, FIELDS.no_label.type).length + ? {} + : childTexts(grp, 'txt', state), hint: texts(notesOf(grp, FIELDS.hint.type), state), relevant: noteText(grp, LOGIC.relevant.type, state), appearance: @@ -709,7 +723,7 @@ function studyRows( ): Map { const rows = new Map(); if (!stdy) return rows; - const labels = bySubject(stdy, FIELDS.row_label.type); + const labels = state.rowLabels; for (const [subject, notes] of bySubject(stdy, 'instruction')) { rows.set(subject, studyItem(subject, 'note', texts(notes, state))); } @@ -937,6 +951,7 @@ export function instrumentFromDdi( listByKey: new Map(), names: new Set(), cdl: false, + rowLabels: new Map(), groupItems: new Map(), onWarning: options.onWarning, }; @@ -944,6 +959,7 @@ export function instrumentFromDdi( const names = languageNames(stdy); // The form's languages in its order (cdl:language), else as they come. if (names.size) state.languages = [...names.keys()]; + if (stdy) state.rowLabels = bySubject(stdy, FIELDS.row_label.type); const elements = dataElements(roots); indexStructure(elements, state); state.cdl = isCdl(elements); diff --git a/src/instrument/fromXlsform.ts b/src/instrument/fromXlsform.ts index 8fa2467..a59e2ba 100644 --- a/src/instrument/fromXlsform.ts +++ b/src/instrument/fromXlsform.ts @@ -55,9 +55,15 @@ export function readText(row: Row, base: string): Text { return text; } -/** Language tags a row's label uses, in column order. */ +/** The columns whose texts the model reads, in every language. */ +const TEXT_COLUMNS = ['label', 'hint', 'guidance_hint', 'constraint_message']; + +/** + * Language tags a row's texts use, label first, in column order: a language + * only a hint has is still one of the form's (#160). + */ function labelLanguages(row: Row): string[] { - return Object.keys(readText(row, 'label')); + return TEXT_COLUMNS.flatMap((column) => Object.keys(readText(row, column))); } /** Parse `parameters`' `guidance_hint=` (qwacback's encoding). */ diff --git a/src/pipelines/ddi2xlsform/README.md b/src/pipelines/ddi2xlsform/README.md index ccfe57c..12e3419 100644 --- a/src/pipelines/ddi2xlsform/README.md +++ b/src/pipelines/ddi2xlsform/README.md @@ -25,15 +25,15 @@ Standard DDI first, `cdl:` notes where DDI has no element |---|---| | type | `qstn/@responseDomainType`; `varFormat/@category` (date, time); `var/@dcml="0"` (integer); `valrng` without a constraint (range); `concept/@vocab` (`select_*_from_file`); the `or_other` shorthand: `cdl:or_other` | | label / hint / guidance_hint | `qstnLit` / `postQTxt` / `ivuInstr`, every `xml:lang` | -| a note row before a question | its `preQTxt` (a grid member's is the grid's text) or its group's untyped `notes`; the rows' names in `cdl:note_names` | +| a note row before a question | its `preQTxt` (a grid member's is the grid's text) or its group's untyped `notes`; the rows' names in `cdl:note_names`, and each one's own text in `cdl:row_label` where the blank lines joining them can't separate them | | a note row with no question after it in its group | `stdyDscr/notes[@type='instruction']`, placed by `cdl:position` | | a note row's hint, relevant, appearance | `cdl:row_hint`, `cdl:row_relevant`, `cdl:row_appearance` on `stdyDscr` | | rows without data (`start`, `deviceid`, …, a matrix header) | `cdl:row` (the type cell) and `cdl:row_label` on `stdyDscr`, placed by `cdl:position` | | choices | `catgry` (`catValu`, `labl`); a select_multiple's binary `var`s; the list's name in `cdl:list` when it isn't the question's; a select_multiple pair's other label in `cdl:other_label` | -| groups | `varGrp type="section"` / `"grid"`, nested by `@varGrp`; label `txt`, hint `cdl:hint`; a group of notes only placed by `cdl:position` | +| groups | `varGrp type="section"` / `"grid"`, nested by `@varGrp`; label `txt` (none: `cdl:no_label`), hint `cdl:hint`; a group of notes only placed by `cdl:position` | | order | `var` order, `qstn/@seqNo` | -| relevant, constraint, constraint_message, required | `cdl:relevant`, `cdl:constraint`, `cdl:constraint_message`, `cdl:required` | -| default, appearance, parameters | `cdl:default`, `cdl:appearance`, `cdl:parameters` (a range's bounds also `valrng/range`) | +| relevant, constraint, constraint_message, required | `cdl:relevant`, `cdl:constraint`, `cdl:constraint_message`, `cdl:required` (the cell as authored) | +| default, appearance, parameters | `cdl:default`, `cdl:appearance`, `cdl:parameters` as authored (a range's bounds also `valrng/range`, a `guidance_hint` also `ivuInstr`) | | exclusive | `cdl:exclusive` on the select_multiple's `varGrp` | | settings | `titl` (+ `parTitl` per language), `IDNo`, `verStmt/version`, `codeBook/@xml:lang`, every other setting a `cdl:setting` | | language columns | `xml:lang`; the form's name for each (`label::Deutsch (de)`) in `cdl:language` | @@ -62,16 +62,9 @@ out): 1. **Rows no registry type covers** (`calculate`, …) and **groups with nothing in them**. 2. **Columns the model doesn't lift**: on the survey sheet (media, - `calculation`, `choice_filter`, …), on the choices sheet every column but - `exclusive`. Settings that are neither a string nor a number. -3. **A group without a label** comes back labelled with its name. -4. **A `constraint_message` without a `constraint`** is not in the DDI. -5. **Cell spellings**: `required` is `yes` or absent, a `guidance_hint` inside - `parameters` comes back as the `guidance_hint` column, whitespace in the - type cell and `parameters` is one space. -6. **Consecutive note rows** before a question are split back at the blank - lines that joined them. A note whose text has a blank line itself, or a - language only some of them have, gives the whole text to the first. + `choice_filter`, …), on the choices sheet every column but `exclusive`. + Settings that are neither a string nor a number. +3. **Whitespace in the type cell** comes back as one space. ## Tests diff --git a/src/xlsform/fromInstrument.ts b/src/xlsform/fromInstrument.ts index e864165..7e37e65 100644 --- a/src/xlsform/fromInstrument.ts +++ b/src/xlsform/fromInstrument.ts @@ -282,7 +282,7 @@ function questionRow(q: QuestionItem, ctx: EmitCtx): SurveyRow { if (value) row[column] = htmlLabel(value); } const plain: Array<[string, string]> = [ - ['required', q.required ? 'yes' : ''], + ['required', q.required ? requiredCell(q) : ''], ['default', q.default], ['appearance', q.appearance], ['parameters', q.parameters], @@ -293,6 +293,12 @@ function questionRow(q: QuestionItem, ctx: EmitCtx): SurveyRow { return row; } +/** The `required` cell: as the source row had it (`TRUE`), else `yes`. */ +function requiredCell(q: QuestionItem): string { + const cell = q.row['required']; + return typeof cell === 'string' && cell.trim() ? cell.trim() : 'yes'; +} + function perLanguageOtherLabel(languages: string[]): LabelValue { if (languages.length <= 1) return otherLabelFor(languages[0] ?? 'en'); const obj: Record = {}; diff --git a/tests/fixtures/surveys/hints_survey/ddi.xml b/tests/fixtures/surveys/hints_survey/ddi.xml index cfa03a6..a32f7f3 100644 --- a/tests/fixtures/surveys/hints_survey/ddi.xml +++ b/tests/fixtures/surveys/hints_survey/ddi.xml @@ -61,6 +61,7 @@ Sind Sie ehrenamtlich tätig? + guidance_hint=Auch gelegentliches Engagement zählt. janein diff --git a/tests/fixtures/surveys/hints_survey/ddi2xlsform.json b/tests/fixtures/surveys/hints_survey/ddi2xlsform.json index 099fec9..2ddeac7 100644 --- a/tests/fixtures/surveys/hints_survey/ddi2xlsform.json +++ b/tests/fixtures/surveys/hints_survey/ddi2xlsform.json @@ -22,7 +22,7 @@ "type": "select_one janein", "name": "ehrenamt", "label": "Sind Sie ehrenamtlich tätig?", - "guidance_hint": "Auch gelegentliches Engagement zählt." + "parameters": "guidance_hint=Auch gelegentliches Engagement zählt." }, { "type": "select_multiple medien", diff --git a/tests/fixtures/surveys/testA/ddi.xml b/tests/fixtures/surveys/testA/ddi.xml index 5356a99..77167f4 100644 --- a/tests/fixtures/surveys/testA/ddi.xml +++ b/tests/fixtures/surveys/testA/ddi.xml @@ -35,27 +35,33 @@ teilnahme teilnahme + yes wirkung wirkung Bitte gib an, wie sehr du zustimmst (1 = stimme gar nicht zu, 5 = stimme voll zu): + yes erfahrungen erfahrungen + yes empfehlung empfehlung + yes demografie demografie + yes intro intro + yes Welche Bereiche nutzt du überhaupt? (Mehrfachauswahl möglich) @@ -87,7 +93,7 @@ Wie lange kommst du schon zu TestWerk? - yes + true @@ -111,7 +117,7 @@ Wie oft kommst du ungefähr? - yes + true @@ -147,7 +153,7 @@ In welcher Werkstatt bist du hauptsächlich? - yes + true @@ -223,7 +229,7 @@ Hattest du vorher schon handwerkliche Erfahrung? - yes + true @@ -251,7 +257,7 @@ Ich fühle mich bei TestWerk wohl und akzeptiert. - yes + true likert likert5 @@ -281,7 +287,7 @@ Ich habe bei TestWerk Freund*innen gefunden. - yes + true likert likert5 @@ -311,7 +317,7 @@ Ich traue mir zu, eigene Ideen und Projekte umzusetzen. - yes + true likert likert5 @@ -341,7 +347,7 @@ Wenn bei einem Projekt etwas nicht klappt, finde ich eine Lösung. - yes + true likert likert5 @@ -371,7 +377,7 @@ Ich habe bei TestWerk handwerkliche Fähigkeiten gelernt. - yes + true likert likert5 @@ -402,7 +408,7 @@ Bevor ich zu TestWerk kam, hatte ich eine klare Vorstellung von meiner beruflichen Zukunft. - yes + true likert likert5 beruf_note @@ -433,7 +439,7 @@ Aktuell habe ich eine klare Vorstellung von meiner beruflichen Zukunft. - yes + true likert likert5 @@ -524,7 +530,7 @@ Würdest du TestWerk Freund*innen empfehlen? - yes + true likert nps diff --git a/tests/fixtures/surveys/testA/ddi2xlsform.json b/tests/fixtures/surveys/testA/ddi2xlsform.json index 76f8c2f..2503f39 100644 --- a/tests/fixtures/surveys/testA/ddi2xlsform.json +++ b/tests/fixtures/surveys/testA/ddi2xlsform.json @@ -3,7 +3,7 @@ { "type": "begin_group", "name": "intro", - "label": "intro" + "label": "" }, { "type": "note", @@ -16,19 +16,19 @@ { "type": "begin_group", "name": "teilnahme", - "label": "teilnahme" + "label": "" }, { "type": "select_one dauer", "name": "dauer", "label": "Wie lange kommst du schon zu TestWerk?", - "required": "yes" + "required": "true" }, { "type": "select_one haeufigkeit", "name": "haeufigkeit", "label": "Wie oft kommst du ungefähr?", - "required": "yes" + "required": "true" }, { "type": "integer", @@ -40,7 +40,7 @@ "type": "select_one hauptwerkstatt", "name": "hauptwerkstatt", "label": "In welcher Werkstatt bist du hauptsächlich?", - "required": "yes" + "required": "true" }, { "type": "select_multiple bereiche", @@ -51,7 +51,7 @@ "type": "select_one vorwissen", "name": "vorwissen", "label": "Hattest du vorher schon handwerkliche Erfahrung?", - "required": "yes" + "required": "true" }, { "type": "end_group" @@ -59,42 +59,42 @@ { "type": "begin_group", "name": "wirkung", - "label": "wirkung", + "label": "", "hint": "Bitte gib an, wie sehr du zustimmst (1 = stimme gar nicht zu, 5 = stimme voll zu):" }, { "type": "select_one likert5", "name": "wohlfuehlen", "label": "Ich fühle mich bei TestWerk wohl und akzeptiert.", - "required": "yes", + "required": "true", "appearance": "likert" }, { "type": "select_one likert5", "name": "freundschaften", "label": "Ich habe bei TestWerk Freund*innen gefunden.", - "required": "yes", + "required": "true", "appearance": "likert" }, { "type": "select_one likert5", "name": "eigene_ideen", "label": "Ich traue mir zu, eigene Ideen und Projekte umzusetzen.", - "required": "yes", + "required": "true", "appearance": "likert" }, { "type": "select_one likert5", "name": "problemloesung", "label": "Wenn bei einem Projekt etwas nicht klappt, finde ich eine Lösung.", - "required": "yes", + "required": "true", "appearance": "likert" }, { "type": "select_one likert5", "name": "skills_gelernt", "label": "Ich habe bei TestWerk handwerkliche Fähigkeiten gelernt.", - "required": "yes", + "required": "true", "appearance": "likert" }, { @@ -106,14 +106,14 @@ "type": "select_one likert5", "name": "beruf_pre", "label": "Bevor ich zu TestWerk kam, hatte ich eine klare Vorstellung von meiner beruflichen Zukunft.", - "required": "yes", + "required": "true", "appearance": "likert" }, { "type": "select_one likert5", "name": "beruf_post", "label": "Aktuell habe ich eine klare Vorstellung von meiner beruflichen Zukunft.", - "required": "yes", + "required": "true", "appearance": "likert" }, { @@ -130,7 +130,7 @@ { "type": "begin_group", "name": "erfahrungen", - "label": "erfahrungen" + "label": "" }, { "type": "text", @@ -153,14 +153,14 @@ { "type": "begin_group", "name": "empfehlung", - "label": "empfehlung" + "label": "" }, { "type": "select_one nps", "name": "nps_score", "label": "Würdest du TestWerk Freund*innen empfehlen?", "hint": "0 = auf keinen Fall, 10 = auf jeden Fall", - "required": "yes", + "required": "true", "appearance": "likert" }, { @@ -169,7 +169,7 @@ { "type": "begin_group", "name": "demografie", - "label": "demografie" + "label": "" }, { "type": "select_one alter", diff --git a/tests/fixtures/surveys/testB/ddi.xml b/tests/fixtures/surveys/testB/ddi.xml index 6693c01..98fe123 100644 --- a/tests/fixtures/surveys/testB/ddi.xml +++ b/tests/fixtures/surveys/testB/ddi.xml @@ -112,7 +112,7 @@ In accordance with the GDPR, you have the right to access and delete your person Für welche Projekte möchtest Du dich bewerben? Which project(s) do you want to apply for? Für welche Projekte möchtest Du dich bewerben? - yes + true mb5co98 @@ -122,7 +122,7 @@ In accordance with the GDPR, you have the right to access and delete your person Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Alpha Only if “Which project(s) do you want to apply for?” includes Project Alpha selected(${project_id}, 'project-alpha') - yes + true cl-role-project-alpha @@ -132,7 +132,7 @@ In accordance with the GDPR, you have the right to access and delete your person Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Beta Only if “Which project(s) do you want to apply for?” includes Project Beta selected(${project_id}, 'project-beta') - yes + true cl-role-project-beta @@ -142,7 +142,7 @@ In accordance with the GDPR, you have the right to access and delete your person Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Gamma Only if “Which project(s) do you want to apply for?” includes Project Gamma selected(${project_id}, 'project-gamma') - yes + true cl-role-project-gamma @@ -452,7 +452,7 @@ In accordance with the GDPR, you have the right to access and delete your person SoSci Survey selected(${project_role_project-alpha}, 'role-survey-design') - yes + true list-nolabel iy6st81 @@ -487,7 +487,7 @@ In accordance with the GDPR, you have the right to access and delete your person Python selected(${project_role_project-alpha}, 'role-data-analysis') - yes + true list-nolabel iy6st81 @@ -522,7 +522,7 @@ In accordance with the GDPR, you have the right to access and delete your person R selected(${project_role_project-alpha}, 'role-data-analysis') - yes + true list-nolabel iy6st81 @@ -557,7 +557,7 @@ In accordance with the GDPR, you have the right to access and delete your person Power BI selected(${project_role_project-beta}, 'role-visualization') - yes + true list-nolabel iy6st81 @@ -592,7 +592,7 @@ In accordance with the GDPR, you have the right to access and delete your person Microsoft Excel selected(${project_role_project-beta}, 'role-visualization') - yes + true list-nolabel iy6st81 @@ -627,7 +627,7 @@ In accordance with the GDPR, you have the right to access and delete your person SQL selected(${project_role_project-beta}, 'role-data-engineering') - yes + true list-nolabel iy6st81 @@ -662,7 +662,7 @@ In accordance with the GDPR, you have the right to access and delete your person Git selected(${project_role_project-beta}, 'role-data-engineering') - yes + true list-nolabel iy6st81 @@ -697,7 +697,7 @@ In accordance with the GDPR, you have the right to access and delete your person Jupyter Notebooks selected(${project_role_project-gamma}, 'role-machine-learning') - yes + true list-nolabel iy6st81 @@ -732,7 +732,7 @@ In accordance with the GDPR, you have the right to access and delete your person TensorFlow / PyTorch selected(${project_role_project-gamma}, 'role-machine-learning') - yes + true list-nolabel iy6st81 @@ -778,7 +778,7 @@ In accordance with the GDPR, you have the right to access and delete your person Entwicklung von Fragebögen selected(${project_role_project-alpha}, 'role-survey-design') - yes + true list-nolabel xe9mo32 @@ -813,7 +813,7 @@ In accordance with the GDPR, you have the right to access and delete your person Entwicklung von Indikatoren selected(${project_role_project-alpha}, 'role-survey-design') - yes + true list-nolabel xe9mo32 @@ -848,7 +848,7 @@ In accordance with the GDPR, you have the right to access and delete your person Datenerhebung selected(${project_role_project-alpha}, 'role-survey-design') - yes + true list-nolabel xe9mo32 @@ -883,7 +883,7 @@ In accordance with the GDPR, you have the right to access and delete your person Datenbereinigung selected(${project_role_project-alpha}, 'role-data-analysis') - yes + true list-nolabel xe9mo32 @@ -918,7 +918,7 @@ In accordance with the GDPR, you have the right to access and delete your person Deskriptive Statistik selected(${project_role_project-alpha}, 'role-data-analysis') - yes + true list-nolabel xe9mo32 @@ -953,7 +953,7 @@ In accordance with the GDPR, you have the right to access and delete your person Datenvisualisierung selected(${project_role_project-beta}, 'role-visualization') - yes + true list-nolabel xe9mo32 @@ -988,7 +988,7 @@ In accordance with the GDPR, you have the right to access and delete your person Data Engineering selected(${project_role_project-beta}, 'role-data-engineering') - yes + true list-nolabel xe9mo32 @@ -1023,7 +1023,7 @@ In accordance with the GDPR, you have the right to access and delete your person Automatisierung selected(${project_role_project-beta}, 'role-data-engineering') - yes + true list-nolabel xe9mo32 @@ -1058,7 +1058,7 @@ In accordance with the GDPR, you have the right to access and delete your person Projektplanung selected(${project_role_project-gamma}, 'role-project-management') - yes + true list-nolabel xe9mo32 @@ -1093,7 +1093,7 @@ In accordance with the GDPR, you have the right to access and delete your person ML-Modellierung selected(${project_role_project-gamma}, 'role-machine-learning') - yes + true list-nolabel xe9mo32 @@ -1128,7 +1128,7 @@ In accordance with the GDPR, you have the right to access and delete your person Wirkungsmessung selected(${project_role_project-alpha}, 'role-survey-design') - yes + true list-nolabel dm1uu76 @@ -1163,7 +1163,7 @@ In accordance with the GDPR, you have the right to access and delete your person Research Design selected(${project_role_project-alpha}, 'role-survey-design') - yes + true list-nolabel dm1uu76 @@ -1198,7 +1198,7 @@ In accordance with the GDPR, you have the right to access and delete your person Umfrageforschung selected(${project_role_project-alpha}, 'role-survey-design') - yes + true list-nolabel dm1uu76 @@ -1233,7 +1233,7 @@ In accordance with the GDPR, you have the right to access and delete your person Datenschutz selected(${project_role_project-alpha}, 'role-survey-design') - yes + true list-nolabel dm1uu76 @@ -1268,7 +1268,7 @@ In accordance with the GDPR, you have the right to access and delete your person Resilienz und Mentale Gesundheit selected(${project_role_project-alpha}, 'role-survey-design') - yes + true list-nolabel dm1uu76 @@ -1303,7 +1303,7 @@ In accordance with the GDPR, you have the right to access and delete your person Bildungsforschung o.ä. selected(${project_role_project-alpha}, 'role-survey-design') - yes + true list-nolabel dm1uu76 @@ -1316,7 +1316,7 @@ In accordance with the GDPR, you have the right to access and delete your person Bitte beschreibe hier, welche Deiner Fähigkeiten und Erfahrungen Dich besonders für die Teilnahme an diesem Projekt qualifizieren. - yes + true multiline @@ -1328,7 +1328,7 @@ In accordance with the GDPR, you have the right to access and delete your person Bitte beschreibe hier, warum Du Dich für dieses Projekt engagieren möchtest. - yes + true multiline @@ -1355,7 +1355,7 @@ In accordance with the GDPR, you have the right to access and delete your person Hast du dich in der Vergangenheit bereits auf CorrelAid-Projekte beworben? - yes + true il2wc73 @@ -1379,7 +1379,7 @@ In accordance with the GDPR, you have the right to access and delete your person Wie lautet dein Vorname? - yes + true @@ -1388,7 +1388,7 @@ In accordance with the GDPR, you have the right to access and delete your person Wie lautet dein Nachname? - yes + true @@ -1399,7 +1399,7 @@ In accordance with the GDPR, you have the right to access and delete your person regex(., '^[\w-\.]+@([\w-]+\.)+[\w-]{2,}$') Please specify a valid email address. - yes + true @@ -1435,7 +1435,7 @@ In accordance with the GDPR, you have the right to access and delete your person Was ist dein Geschlecht? - yes + true iy7os66 @@ -1451,7 +1451,7 @@ In accordance with the GDPR, you have the right to access and delete your person Mein Geschlecht ist: ${gender} = 'self_identification' - yes + true @@ -1467,7 +1467,7 @@ In accordance with the GDPR, you have the right to access and delete your person Einwilligung in die Datenschutzerklärung - yes + true mj8ty33 diff --git a/tests/fixtures/surveys/testB/ddi2xlsform.json b/tests/fixtures/surveys/testB/ddi2xlsform.json index 6336021..f98ca83 100644 --- a/tests/fixtures/surveys/testB/ddi2xlsform.json +++ b/tests/fixtures/surveys/testB/ddi2xlsform.json @@ -51,7 +51,7 @@ "de": "Für welche Projekte möchtest Du dich bewerben?", "en": "Which project(s) do you want to apply for?" }, - "required": "yes" + "required": "true" }, { "type": "select_multiple cl-role-project-alpha", @@ -60,7 +60,7 @@ "de": "In welcher Rolle siehst du dich im Projekt Projekt Alpha?", "en": "Which role do you think you could fill in Project Alpha?" }, - "required": "yes", + "required": "true", "relevant": "selected(${project_id}, 'project-alpha')" }, { @@ -70,7 +70,7 @@ "de": "In welcher Rolle siehst du dich im Projekt Beta?", "en": "Which role do you see yourself in for Project Beta?" }, - "required": "yes", + "required": "true", "relevant": "selected(${project_id}, 'project-beta')" }, { @@ -80,7 +80,7 @@ "de": "In welcher Rolle siehst du dich im Projekt Gamma?", "en": "Which role do you see yourself in for Project Gamma?" }, - "required": "yes", + "required": "true", "relevant": "selected(${project_id}, 'project-gamma')" }, { @@ -119,7 +119,7 @@ "de": "SoSci Survey", "en": "SoSci Survey" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-survey-design')" }, @@ -130,7 +130,7 @@ "de": "Python", "en": "Python" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-data-analysis')" }, @@ -141,7 +141,7 @@ "de": "R", "en": "R" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-data-analysis')" }, @@ -152,7 +152,7 @@ "de": "Power BI", "en": "Power BI" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-beta}, 'role-visualization')" }, @@ -163,7 +163,7 @@ "de": "Microsoft Excel", "en": "Microsoft Excel" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-beta}, 'role-visualization')" }, @@ -174,7 +174,7 @@ "de": "SQL", "en": "SQL" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-beta}, 'role-data-engineering')" }, @@ -185,7 +185,7 @@ "de": "Git", "en": "Git" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-beta}, 'role-data-engineering')" }, @@ -196,7 +196,7 @@ "de": "Jupyter Notebooks", "en": "Jupyter Notebooks" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-gamma}, 'role-machine-learning')" }, @@ -207,7 +207,7 @@ "de": "TensorFlow / PyTorch", "en": "TensorFlow / PyTorch" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-gamma}, 'role-machine-learning')" }, @@ -256,7 +256,7 @@ "de": "Entwicklung von Fragebögen", "en": "Survey design" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-survey-design')" }, @@ -267,7 +267,7 @@ "de": "Entwicklung von Indikatoren", "en": "Development of indicators" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-survey-design')" }, @@ -278,7 +278,7 @@ "de": "Datenerhebung", "en": "Data collection" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-survey-design')" }, @@ -289,7 +289,7 @@ "de": "Datenbereinigung", "en": "Data cleaning" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-data-analysis')" }, @@ -300,7 +300,7 @@ "de": "Deskriptive Statistik", "en": "Descriptive statistics" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-data-analysis')" }, @@ -311,7 +311,7 @@ "de": "Datenvisualisierung", "en": "Data visualization" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-beta}, 'role-visualization')" }, @@ -322,7 +322,7 @@ "de": "Data Engineering", "en": "Data engineering" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-beta}, 'role-data-engineering')" }, @@ -333,7 +333,7 @@ "de": "Automatisierung", "en": "Automation" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-beta}, 'role-data-engineering')" }, @@ -344,7 +344,7 @@ "de": "Projektplanung", "en": "Project planning" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-gamma}, 'role-project-management')" }, @@ -355,7 +355,7 @@ "de": "ML-Modellierung", "en": "ML modeling" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-gamma}, 'role-machine-learning')" }, @@ -391,7 +391,7 @@ "de": "Wirkungsmessung", "en": "Impact Measurement" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-survey-design')" }, @@ -402,7 +402,7 @@ "de": "Research Design", "en": "Research Design" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-survey-design')" }, @@ -413,7 +413,7 @@ "de": "Umfrageforschung", "en": "Survey Research" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-survey-design')" }, @@ -424,7 +424,7 @@ "de": "Datenschutz", "en": "Data protection" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-survey-design')" }, @@ -435,7 +435,7 @@ "de": "Resilienz und Mentale Gesundheit", "en": "Resilience and mental health" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-survey-design')" }, @@ -446,7 +446,7 @@ "de": "Bildungsforschung o.ä.", "en": "Education research (or similar)" }, - "required": "yes", + "required": "true", "appearance": "list-nolabel", "relevant": "selected(${project_role_project-alpha}, 'role-survey-design')" }, @@ -472,7 +472,7 @@ "de": "Bitte schreibe maximal 5 Sätze pro Projekt, auf das du dich bewerben möchtest.", "en": "Please write maximum 5 sentences per project that you are applying to." }, - "required": "yes", + "required": "true", "appearance": "multiline" }, { @@ -486,7 +486,7 @@ "de": "Bitte schreibe maximal 5 Sätze pro Projekt, auf das du dich bewerben möchtest.", "en": "Please write maximum 5 sentences per project that you are applying to." }, - "required": "yes", + "required": "true", "appearance": "multiline" }, { @@ -500,7 +500,7 @@ "de": "Um möglichst vielen Personen die Gelegenheit zu geben, an unseren Projekten teilzunehmen, werden Bewerbungen von bisher unerfolgreichen Personen bei gleicher Eignung bevorzugt behandelt.", "en": "In order to give as many people as possible the opportunity to participate in one of our projects, applications of individuals who have unsuccessfully applied in the past are preferred given equal suitability for the project." }, - "required": "yes" + "required": "true" }, { "type": "text", @@ -536,7 +536,7 @@ "de": "Wie lautet dein Vorname?", "en": "What is your first name?" }, - "required": "yes" + "required": "true" }, { "type": "text", @@ -545,7 +545,7 @@ "de": "Wie lautet dein Nachname?", "en": "What is your last name?" }, - "required": "yes" + "required": "true" }, { "type": "text", @@ -557,7 +557,7 @@ "constraint_message": { "de": "Please specify a valid email address." }, - "required": "yes", + "required": "true", "constraint": "regex(., '^[\\w-\\.]+@([\\w-]+\\.)+[\\w-]{2,}$')" }, { @@ -571,7 +571,7 @@ "de": "Hinweis: Da wir bei CorrelAid nach dem Grundprinzip von Geschlechtergleichberechtigung arbeiten, ist diese Frage für uns besonders wichtig.", "en": "Note: We ask this question because we want to foster gender diversity within CorrelAid." }, - "required": "yes" + "required": "true" }, { "type": "text", @@ -584,7 +584,7 @@ "de": "Bitte gebe hier dein Geschlecht an.", "en": "Please specify your gender here." }, - "required": "yes", + "required": "true", "relevant": "${gender} = 'self_identification'" }, { @@ -601,7 +601,7 @@ "de": "Ich habe den Disclaimer gelesen und erteile hiermit meine Zustimmung zur Erhebung, Verarbeitung und Nutzung meiner personenbezogener Daten (Geschlecht und persönliche Fähigkeiten) zu internen und externen Reporting- und Präsentationszwecken. Daneben erteile ich ausdrücklich die Erlaubnis meine Kontaktdaten (Name und E-Mail) zur Kontaktaufnahme zu nutzen und diese bei erfolgreicher Bewerbung mit dem Projektteam zu teilen.", "en": "I have read the disclaimer and hereby give my consent to the collection, processing and use of my personal data (gender and personal skills) for the team selection process and for internal and external reporting as well as presentation purposes. In addition, I expressly grant permission to use my contact details (name and e-mail) to contact me and to share them with the project team if the application is successful." }, - "required": "yes" + "required": "true" } ], "choices": [ diff --git a/tests/ts/contract/canonicalInstrument.ts b/tests/ts/contract/canonicalInstrument.ts index 88c328c..1db0bfd 100644 --- a/tests/ts/contract/canonicalInstrument.ts +++ b/tests/ts/contract/canonicalInstrument.ts @@ -45,15 +45,11 @@ function text(t: Text | undefined, ctx: Ctx): Record { return out; } -/** Parameters without guidance_hint: DDI has it as ivuInstr. */ -function parameters(q: QuestionItem): string { - return q.parameters - .split(';') - .filter((p) => !/^\s*guidance_hint\s*=/.test(p)) - .join(' ') - .split(/[\s,]+/) - .filter(Boolean) - .join(' '); +/** The `required` cell a required question has: as authored, else `yes`. */ +function required(q: QuestionItem): string | false { + if (!q.required) return false; + const cell = q.row['required']; + return typeof cell === 'string' && cell.trim() ? cell.trim() : 'yes'; } /** Whether the DDI has it: a variable, a note or a row without data. */ @@ -66,7 +62,6 @@ function kept(q: QuestionItem): boolean { function question(q: QuestionItem, ctx: Ctx): Canon { const type = ALIASES[q.type] ?? q.type; if (q.list) ctx.lists.add(q.list); - const hasConstraint = !!q.constraint.trim(); return { name: q.name, type, @@ -78,12 +73,11 @@ function question(q: QuestionItem, ctx: Ctx): Canon { guidance: text(q.guidanceHint, ctx), relevant: q.relevant.trim(), constraint: q.constraint.trim(), - // A message without a constraint is not in the DDI. - constraintMessage: hasConstraint ? text(q.constraintMessage, ctx) : {}, - required: q.required, + constraintMessage: text(q.constraintMessage, ctx), + required: required(q), default: q.default, appearance: q.appearance, - parameters: parameters(q), + parameters: q.parameters.trim(), }; } @@ -93,11 +87,9 @@ function items(list: Item[], ctx: Ctx): Canon[] { if (item.kind === 'group') { // A group with nothing in it is not carried. if (!item.children.length) continue; - const label = text(item.label, ctx); out.push({ group: item.name, - // A group without a label is labelled with its name. - label: Object.keys(label).length ? label : text({ '': item.name }, ctx), + label: text(item.label, ctx), hint: text(item.hint, ctx), relevant: item.relevant.trim(), appearance: item.appearance, diff --git a/tests/ts/contract/ddiRoundtripGenerated.test.ts b/tests/ts/contract/ddiRoundtripGenerated.test.ts index 4bca1b8..fa8a429 100644 --- a/tests/ts/contract/ddiRoundtripGenerated.test.ts +++ b/tests/ts/contract/ddiRoundtripGenerated.test.ts @@ -2,8 +2,9 @@ * The DDI round trips on generated forms (#154, #160): random valid * XLSForms from the registry's types, with groups, grids, matrices, * languages, logic, fields, named and shared lists, `or_other` and explicit - * other pairs, runs of notes, groups of notes only, metadata rows and - * settings. Compared as `ddiRoundtrip.test.ts` and + * other pairs, runs of notes (with blank lines, in only some languages), + * groups of notes only or without a label, metadata rows, `required` + * spellings, messages without a constraint and settings. Compared as `ddiRoundtrip.test.ts` and * `lstsvDdiRoundtrip.test.ts` do. */ import { describe, test, expect } from 'vitest'; @@ -57,9 +58,11 @@ type Question = type: string; t: Texts; hint?: Texts; - required: boolean; + required?: string; ref: boolean; constraint: boolean; + /** A constraint_message without a constraint. */ + message: boolean; appearance: boolean; params?: string; } @@ -74,7 +77,16 @@ type Question = exclusive: boolean; ref: boolean; } - | { kind: 'note'; t: Texts; hint?: Texts; ref: boolean } + | { + kind: 'note'; + t: Texts; + hint?: Texts; + ref: boolean; + /** Its text has a blank line. */ + paragraphs: boolean; + /** Only the first language has it. */ + partial: boolean; + } | { kind: 'metadata'; type: string }; const question: fc.Arbitrary = fc.oneof( @@ -85,9 +97,12 @@ const question: fc.Arbitrary = fc.oneof( type: fc.constantFrom(...SIMPLE), t: texts, hint: fc.option(texts, { nil: undefined }), - required: fc.boolean(), + required: fc.option(fc.constantFrom('yes', 'TRUE', 'true'), { + nil: undefined, + }), ref: fc.boolean(), constraint: fc.boolean(), + message: fc.boolean(), appearance: fc.boolean(), params: fc.option(fc.constantFrom(...RANGE_PARAMETERS), { nil: undefined, @@ -122,6 +137,8 @@ const question: fc.Arbitrary = fc.oneof( t: texts, hint: fc.option(texts, { nil: undefined }), ref: fc.boolean(), + paragraphs: fc.boolean(), + partial: fc.boolean(), }), }, fc.record({ @@ -135,6 +152,7 @@ type Block = | { kind: 'group'; t: Texts; + unlabelled: boolean; relevant: boolean; inner: Question[]; nested: Question[]; @@ -152,6 +170,7 @@ const block: fc.Arbitrary = fc.oneof( arbitrary: fc.record({ kind: fc.constant('group' as const), t: texts, + unlabelled: fc.boolean(), relevant: fc.boolean(), inner: fc.array(question, { minLength: 1, maxLength: 3 }), nested: fc.array(question, { maxLength: 2 }), @@ -255,8 +274,16 @@ function sheets(f: Form) { return; } const row: Row = { name }; - put(row, 'label', q.t); if (q.kind === 'note') { + const t = q.paragraphs + ? { + one: `${q.t.one}\n\n${q.t.en}`, + de: `${q.t.de}\n\n${q.t.en}`, + en: q.t.en, + } + : q.t; + if (q.partial && f.multilingual) row[`label::${LANGS[0]}`] = t.de; + else put(row, 'label', t); row.type = 'note'; if (q.hint) put(row, 'hint', q.hint); const c = q.ref ? condition() : undefined; @@ -264,14 +291,17 @@ function sheets(f: Form) { survey.push(row); return; } + put(row, 'label', q.t); row.type = q.type; if (q.hint) put(row, 'hint', q.hint); - if (q.required) row.required = 'yes'; + if (q.required) row.required = q.required; const c = q.ref ? condition() : undefined; if (c) row.relevant = c; if (q.constraint && q.type === 'integer') { row.constraint = '. >= 1 and . <= 10'; put(row, 'constraint_message', q.t); + } else if (q.message) { + put(row, 'constraint_message', q.t); } if (q.appearance && q.type === 'text') row.appearance = 'multiline'; if (q.params && q.type === 'range') row.parameters = q.params; @@ -304,7 +334,7 @@ function sheets(f: Form) { if (b.kind === 'question') addQuestion(b.q); else if (b.kind === 'group') { const row: Row = { type: 'begin_group', name: `g${++n}` }; - put(row, 'label', b.t); + if (!b.unlabelled) put(row, 'label', b.t); if (b.relevant && answered.length) row.relevant = `\${${answered[0]}} != ''`; survey.push(row); diff --git a/tests/ts/unit/ddi/fields.test.ts b/tests/ts/unit/ddi/fields.test.ts index b2da947..0a5471e 100644 --- a/tests/ts/unit/ddi/fields.test.ts +++ b/tests/ts/unit/ddi/fields.test.ts @@ -139,8 +139,10 @@ describe('cdl: notes', () => { const v = varXml(xml, 'q'); expect(v).toContain('no'); expect(v).toContain('minimal'); - // guidance_hint is its ivuInstr, not a parameter. - expect(v).toContain('randomize=true'); + // The cell as authored (#160); its guidance_hint is the ivuInstr too. + expect(v).toContain( + 'randomize=true; guidance_hint=Only once', + ); expect(v).toContain('Only once'); }); diff --git a/tests/ts/unit/ddi/fromDdi.test.ts b/tests/ts/unit/ddi/fromDdi.test.ts index 15f2ad4..1281e55 100644 --- a/tests/ts/unit/ddi/fromDdi.test.ts +++ b/tests/ts/unit/ddi/fromDdi.test.ts @@ -9,6 +9,7 @@ import type { Diagnostic } from '../../../../src/diagnostics.js'; import { ConversionError } from '../../../../src/diagnostics.js'; import { instrumentFromDdi } from '../../../../src/instrument/fromDdi.js'; import { ddiToXlsform } from '../../../../src/pipelines/ddi2xlsform/index.js'; +import { buildDdiXml } from '../../../../src/pipelines/xlsform2ddi/index.js'; import type { QuestionItem } from '../../../../src/instrument/types.js'; import { parseXml, textContent } from '../../../../src/utils/xmlParse.js'; @@ -148,3 +149,35 @@ describe('choice lists', () => { }); }); }); + +describe('cells as authored (#160)', () => { + const back = (survey: Record[]) => + ddiToXlsform(buildDdiXml(survey, [], { prodDate: '2020-01-01' })).survey; + + test('a required cell other than yes', () => { + expect( + back([{ type: 'text', name: 't', label: 'T', required: 'TRUE' }])[0], + ).toMatchObject({ required: 'TRUE' }); + }); + + test('a group without a label has none', () => { + const [group] = back([ + { type: 'begin_group', name: 'g' }, + { type: 'text', name: 't', label: 'T' }, + { type: 'end_group' }, + ]); + expect(group).toMatchObject({ type: 'begin_group', name: 'g', label: '' }); + }); + + test('notes the blank lines can not tell apart keep their own texts', () => { + const survey = back([ + { type: 'note', name: 'a', label: 'One\n\nTwo' }, + { type: 'note', name: 'b', label: 'Three' }, + { type: 'text', name: 't', label: 'T' }, + ]); + expect(survey.slice(0, 2)).toEqual([ + { type: 'note', name: 'a', label: 'One\n\nTwo' }, + { type: 'note', name: 'b', label: 'Three' }, + ]); + }); +}); diff --git a/tests/ts/unit/ddi/logic.test.ts b/tests/ts/unit/ddi/logic.test.ts index 18a0d2c..f53b817 100644 --- a/tests/ts/unit/ddi/logic.test.ts +++ b/tests/ts/unit/ddi/logic.test.ts @@ -279,11 +279,13 @@ describe('constraint', () => { expect(v).toContain(' { + test('a constraint_message without a constraint is kept too (#160)', () => { const xml = ddi([ { type: 'text', name: 't', label: 'T', constraint_message: 'Oops' }, ]); - expect(varXml(xml, 't')).not.toContain('cdl:constraint_message'); + expect(varXml(xml, 't')).toContain( + 'Oops', + ); }); }); diff --git a/tests/validation/test_xlsform_pyxform.py b/tests/validation/test_xlsform_pyxform.py index 5f66ae6..cebfb38 100644 --- a/tests/validation/test_xlsform_pyxform.py +++ b/tests/validation/test_xlsform_pyxform.py @@ -137,10 +137,25 @@ def test_survey_fixture_is_valid_xlsform(survey_dir: Path, tmp_path: Path) -> No _BACK = [d for d in _SURVEYS if (d / "ddi2xlsform.json").exists()] -@pytest.mark.parametrize("survey_dir", _BACK, ids=[d.name for d in _BACK]) +@pytest.mark.parametrize( + "survey_dir", + [ + pytest.param( + d, + marks=pytest.mark.xfail( + strict=True, + reason=f"gives back its source's cells (#160), which pyxform rejects: {_PYXFORM_INVALID[d.name]}", + ), + ) + if d.name in _PYXFORM_INVALID + else d + for d in _BACK + ], + ids=[d.name for d in _BACK], +) def test_ddi2xlsform_output_is_valid_xlsform(survey_dir: Path, tmp_path: Path) -> None: """The form ddi2xlsform gives back from each blessed codebook is valid - XLSForm: qwacback exports it to Kobo.""" + XLSForm when its source is: qwacback exports it to Kobo.""" for csv in (REPO_ROOT / "registry" / "vocab").glob("*.csv"): shutil.copy(csv, tmp_path / csv.name) xlsx = _json_xlsx(survey_dir / "ddi2xlsform.json", tmp_path / f"{survey_dir.name}.xlsx")