From 67b63ca549dca3cda6c922b96d60ae3b76b6bbf9 Mon Sep 17 00:00:00 2001 From: jstet Date: Sun, 27 Sep 2026 17:54:01 +0200 Subject: [PATCH] feat(ddi): skip logic and validation in the codebook (#151) DDI Codebook 2.5 has no expression syntax, so each condition goes in twice (convention:logicMapping, ddiEncoding): - readable: `` prose from the expression and the referenced labels, in the base language and each other language with a template (en, de); a simple numeric range also as ``; - exact: typed notes, last in var/varGrp: `cdl:relevant` and `cdl:constraint` (subject="xlsform-xpath"), `cdl:constraint_message` per xml:lang, `cdl:required`. A select_multiple's logic is on its varGrp (the semi-open pair's on its `other` varGrp), with the prose also on each binary var. Enclosing groups' conditions are ANDed in until groups get a varGrp (#152). The or_other companion gets pyxform's `${q} = 'other'` / `selected(${q}, 'other')`. lstsv2ddi now reverses relevance and constraints; one outside the dialect gets an `em-unsupported` warning and no note instead of failing. The CDL Schematron allows any number of typed notes and one untyped note per language (was: one notes element per var). Co-Authored-By: Claude Opus 5.5 (1M context) --- ARCHITECTURE.md | 11 +- README.md | 11 +- codegen/schematron.py | 2 +- .../schematron/ddi_custom_rules.sch | 4 +- registry/conventions/logicMapping.jsonld | 106 +++++- .../entities/select_multiple_other/ddi.xml | 2 + registry/entities/select_one_other/ddi.xml | 2 + src/api.ts | 5 +- src/cli.ts | 2 + src/ddi/codebook.ts | 90 +++-- src/ddi/fromInstrument.ts | 59 ++- src/ddi/logic.ts | 344 ++++++++++++++++++ src/ddi/types.ts | 12 + src/generated/conventions.json | 106 +++++- src/generated/conventions.ts | 106 +++++- src/instrument/fromLstsv.ts | 44 ++- src/pipelines/README.md | 38 +- src/pipelines/lstsv2ddi/index.ts | 7 +- src/pipelines/lstsv2ddi/toVariables.ts | 16 +- .../fixtures/surveys/all_types_survey/ddi.xml | 15 + tests/fixtures/surveys/basic_survey/ddi.xml | 3 + .../fixtures/surveys/bilingual_survey/ddi.xml | 3 + tests/fixtures/surveys/complex_survey/ddi.xml | 16 + .../surveys/complex_xpath_survey/ddi.xml | 10 + .../fixtures/surveys/settings_survey/ddi.xml | 2 + tests/fixtures/surveys/testA/ddi.xml | 16 + tests/fixtures/surveys/testB/ddi.xml | 154 ++++++++ .../validation_relevance_survey/ddi.xml | 28 ++ tests/ts/unit/ddi/logic.test.ts | 324 +++++++++++++++++ .../test_registry_schematron_conformance.py | 24 +- 30 files changed, 1451 insertions(+), 111 deletions(-) create mode 100644 src/ddi/logic.ts create mode 100644 tests/ts/unit/ddi/logic.test.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 00bbe5b..b888b86 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -151,15 +151,14 @@ The migration runs in phases, each keeping every snapshot byte-identical: ### Why there is no `ddi2xlsform` or `ddi2lstsv` -Deliberate design decision. **DDI is the terminus of the pipeline graph** — it describes a *dataset*, not an *instrument*, so it does not carry the information a survey needs to run. +**DDI is the terminus of the pipeline graph for now.** It describes a *dataset*, not an *instrument*, and a reverse path needs the whole instrument. #155 plans one. -The canonical `Variable` (`src/ddi/types.ts`) is what survives an emit. Everything that makes a form behave is absent: +The canonical `Variable` (`src/ddi/types.ts`) is what survives an emit: -- **no `relevant`** — DDI Codebook 2.5 has no machine-readable expression syntax at all, so skip logic is dropped on the way in -- **no `constraint`** — same reason -- **no `required`, `default`, per-question `appearance`, `calculation`** (a question's `hint` and `guidance_hint` do survive, as `` and ``) +- **`relevant`, `constraint`, `constraint_message`, `required`** survive (#151). DDI Codebook 2.5 has no expression syntax, so each goes in twice: readable (`` prose, `` for a simple numeric range) and exact, in a typed ``. `convention:logicMapping` (`ddiEncoding`) defines the notes. Until every group has a `varGrp` (#152), a group's condition is ANDed into each variable's own. +- **no `default`, per-question `appearance`, `calculation`, plain group structure** yet (#152, #153). A question's `hint` and `guidance_hint` do survive, as `` and ``. -Compare `lstsv2xlsform`, which *is* implemented: a LimeSurvey structure TSV carries `relevance`, `em_validation_q`, `mandatory`, `default` and the `!`/`T` type overrides. It is a form definition in a different dialect, so reversing it is a translation problem. Reversing DDI is not — it is a *reconstruction* problem, and the missing pieces cannot be inferred from a codebook. +Compare `lstsv2xlsform`, which *is* implemented: a LimeSurvey structure TSV carries `relevance`, `em_validation_q`, `mandatory`, `default` and the `!`/`T` type overrides. It is a form definition in a different dialect, so reversing it is a translation problem. Reversing a codebook without those pieces is a *reconstruction* problem, and they cannot be inferred from it. ## Development Workflow diff --git a/README.md b/README.md index f8d0498..39e8d09 100644 --- a/README.md +++ b/README.md @@ -190,10 +190,13 @@ types: plain/nested groups flatten, choice codes over 5 chars truncate, `select_multiple` becomes N binary variables, and the reverse paths cannot recover a select's authored `list_name` or tell `integer` from `decimal`. -**DDI is the terminus:** there is no `ddi2xlsform` or `ddi2lstsv`, by design. A -codebook describes a *dataset*, not an *instrument* — it carries no relevance, -constraint, required, default or appearance, so reversing it would emit a survey -that looks right and behaves wrongly. +**DDI has no way back yet:** there is no `ddi2xlsform` or `ddi2lstsv`. A CDL +codebook carries skip logic, validation and `required`: each condition as a +readable `` sentence (and a simple numeric range as ``), plus +the exact expression in a typed note such as `` (`convention:logicMapping`). Defaults, appearances +and the group structure are not in it yet, so a reversed form would still look +right and behave wrongly. #155 tracks the rest. ## Errors and warnings diff --git a/codegen/schematron.py b/codegen/schematron.py index 419df22..d890c29 100644 --- a/codegen/schematron.py +++ b/codegen/schematron.py @@ -83,7 +83,7 @@ def generate_schematron(registry: dict[str, Any], output: Path) -> None: Variable is missing technical format (varFormat). Variable is missing a concept element. Variable uses labl — use concept instead. labl is only for catgry elements. - Variable has multiple notes elements. Only one notes element per variable is allowed. + Variable has more than one untyped notes element in one language. Only one untyped note per language is allowed; typed notes (convention:logicMapping, e.g. type="cdl:relevant") are not limited. diff --git a/ddi-validation/schematron/ddi_custom_rules.sch b/ddi-validation/schematron/ddi_custom_rules.sch index e406f86..8635861 100644 --- a/ddi-validation/schematron/ddi_custom_rules.sch +++ b/ddi-validation/schematron/ddi_custom_rules.sch @@ -29,7 +29,7 @@ Variable is missing technical format (varFormat). Variable is missing a concept element. Variable uses labl — use concept instead. labl is only for catgry elements. - Variable has multiple notes elements. Only one notes element per variable is allowed. + Variable has more than one untyped notes element in one language. Only one untyped note per language is allowed; typed notes (convention:logicMapping, e.g. type="cdl:relevant") are not limited. @@ -54,7 +54,7 @@ Variable is missing technical format (varFormat). Variable is missing a concept element. Variable uses labl — use concept instead. labl is only for catgry elements. - Variable has multiple notes elements. Only one notes element per variable is allowed. + Variable has more than one untyped notes element in one language. Only one untyped note per language is allowed; typed notes (convention:logicMapping, e.g. type="cdl:relevant") are not limited. diff --git a/registry/conventions/logicMapping.jsonld b/registry/conventions/logicMapping.jsonld index 7727000..9e0410b 100644 --- a/registry/conventions/logicMapping.jsonld +++ b/registry/conventions/logicMapping.jsonld @@ -30,7 +30,7 @@ "@type": "GlobalConvention", "skos:prefLabel": "Relevance & validation logic: XLSForm ↔ LimeSurvey ↔ DDI", "rule": { - "description": "XLSForm expresses survey logic in an XPath subset; LimeSurvey uses ExpressionScript (EM). The transformation transpiles between them (implemented in src/pipelines/xlsform2lstsv/xpathTranspiler.ts). DDI Codebook has no machine-readable expression syntax — logic fields are dropped when emitting DDI and tools MUST report this as loss.", + "description": "XLSForm expresses survey logic in an XPath subset; LimeSurvey uses ExpressionScript (EM). The transformation transpiles between them (implemented in src/pipelines/xlsform2lstsv/xpathTranspiler.ts). DDI Codebook 2.5 has no expression syntax of its own, so a CDL codebook carries the logic in two parts (formtransform#151): readable prose in the standard element (`universe`, `valrng`) and the expression itself in a typed ``, the extension point the DDI XSD provides (\"The attributes for notes permit a controlled vocabulary to be developed ('type' and 'subject')\"). See ddiEncoding.", "fields": [ { "concept": "relevance (conditional display)", @@ -39,9 +39,9 @@ "lsTsvColumn": "relevance", "lsSyntax": "ExpressionScript (EM)", "ddi": { - "closestElement": "var/universe", - "note": "DDI universe describes the subpopulation as prose, not as machine syntax. Currently NOT emitted — relevance is dropped in DDI output.", - "lossy": true + "element": "var/universe[@clusion='I'] + var/notes[@type='cdl:relevant']", + "note": "The universe is prose formtransform generates from the expression and the referenced questions' labels (see ddiEncoding.universe); the typed note carries the XPath verbatim. A select_multiple's go on its varGrp[@type='multipleResp'] (the semi-open pair's on its varGrp[@type='other']), with the universe prose also on each binary var. Until every group has a varGrp (formtransform#152), the enclosing groups' conditions are ANDed into each variable's own.", + "lossy": false } }, { @@ -51,9 +51,9 @@ "lsTsvColumn": "em_validation_q", "lsSyntax": "ExpressionScript (EM)", "ddi": { - "closestElement": "var/valrng", - "note": "DDI valrng can express simple numeric ranges only. Currently NOT emitted — constraints are dropped in DDI output.", - "lossy": true + "element": "var/notes[@type='cdl:constraint'] (+ var/valrng/range for a simple numeric range)", + "note": "The typed note carries the XPath verbatim. On an integer or decimal, a constraint that is only bounds on `.` (`. >= 1 and . <= 10`, `. > 0`) is also written as the standard valrng/range (min/max, minExclusive/maxExclusive).", + "lossy": false } }, { @@ -63,9 +63,21 @@ "lsTsvColumn": "em_validation_q_tip", "lsSyntax": "plain text", "ddi": { - "closestElement": null, - "note": "No DDI counterpart; dropped.", - "lossy": true + "element": "var/notes[@type='cdl:constraint_message']", + "note": "The form's own text, one note per language (base untagged, others with xml:lang), like every other form text.", + "lossy": false + } + }, + { + "concept": "required answer", + "xlsformColumn": "required", + "xlsformSyntax": "yes/no", + "lsTsvColumn": "mandatory", + "lsSyntax": "Y/N", + "ddi": { + "element": "var/notes[@type='cdl:required']", + "note": "Written only for a required question, with the text `yes`. It tells 'not asked' (universe) from 'refused' apart.", + "lossy": false } } ], @@ -111,7 +123,79 @@ "today", "now" ], - "unsupportedFunctionBehavior": "error — the transpiler throws on any XPath function outside supportedXPathFunctions" + "unsupportedFunctionBehavior": "error — the transpiler throws on any XPath function outside supportedXPathFunctions", + "ddiEncoding": { + "noteSubject": "xlsform-xpath", + "noteSubjectMeaning": "The note's text is an XLSForm XPath expression. `${name}` refers to a var/@name or varGrp/@name of the same codebook; `.` to the variable's own answer.", + "notes": { + "relevant": { + "type": "cdl:relevant", + "subject": "xlsform-xpath", + "on": [ + "var", + "varGrp" + ] + }, + "constraint": { + "type": "cdl:constraint", + "subject": "xlsform-xpath", + "on": [ + "var", + "varGrp" + ] + }, + "constraint_message": { + "type": "cdl:constraint_message", + "on": [ + "var", + "varGrp" + ], + "localized": true + }, + "required": { + "type": "cdl:required", + "on": [ + "var", + "varGrp" + ], + "text": "yes" + } + }, + "order": "The typed notes come last in var/varGrp, after any untyped note; universe comes after qstn and valrng, before catgry. At most one untyped note per language stays allowed (a citation).", + "universe": { + "clusion": "I", + "untaggedLanguage": "en", + "use": "The prose covers `=`, `!=`, `<`, `<=`, `>`, `>=`, `selected()`, `and`, `or`, `not()`, and a comparison with '' (answered / not answered). A condition using anything else gets no universe; its typed note still carries it. It is written in the codebook's base language and each other language with a template here (an untagged codebook uses untaggedLanguage), and only where every referenced label exists in that language. The connecting words are formtransform's metadata, not the form's text; labels and choice labels are the form's own.", + "templates": { + "en": { + "prefix": "Only if ", + "and": "and", + "or": "or", + "not": "not ({})", + "selected": "{q} includes {v}", + "answered": "{q} is answered", + "unanswered": "{q} is not answered", + "quotes": [ + "“", + "”" + ] + }, + "de": { + "prefix": "Nur wenn ", + "and": "und", + "or": "oder", + "not": "nicht ({})", + "selected": "{q} enthält {v}", + "answered": "{q} ist beantwortet", + "unanswered": "{q} ist nicht beantwortet", + "quotes": [ + "„", + "“" + ] + } + } + } + } } } ] diff --git a/registry/entities/select_multiple_other/ddi.xml b/registry/entities/select_multiple_other/ddi.xml index 74fb4d4..ff917ab 100644 --- a/registry/entities/select_multiple_other/ddi.xml +++ b/registry/entities/select_multiple_other/ddi.xml @@ -75,8 +75,10 @@ Sonstiges (bitte angeben) + Only if “Welche dieser Geräte besitzen Sie?” includes Sonstiges Sonstiges (bitte angeben) + selected(${geraetebesitz}, 'other') diff --git a/registry/entities/select_one_other/ddi.xml b/registry/entities/select_one_other/ddi.xml index df7fdb2..3dcd138 100644 --- a/registry/entities/select_one_other/ddi.xml +++ b/registry/entities/select_one_other/ddi.xml @@ -52,8 +52,10 @@ Sonstiges (bitte angeben) + Only if “Wie sind Sie auf unser Angebot aufmerksam geworden?” = Sonstiges Sonstiges (bitte angeben) + ${aufmerksam} = 'other' diff --git a/src/api.ts b/src/api.ts index 5056945..a3c0d9d 100644 --- a/src/api.ts +++ b/src/api.ts @@ -162,5 +162,8 @@ export function lstsvToDdi( tsv: string, options: LstsvToDdiOptions = {}, ): string { - return lstsvToDdiXml(tsv, options); + return lstsvToDdiXml(tsv, { + ...options, + onWarning: onceEach(options.onWarning), + }); } diff --git a/src/cli.ts b/src/cli.ts index 67e7fdc..c0fdc0f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -425,6 +425,8 @@ function cmdLstsv2ddi(argv: string[]): void { datasetFilename, prodDate: values['prod-date'] as string | undefined, submissions, + onWarning: (w) => + process.stderr.write(`${PROG}: warning: ${w.message}\n`), }); if (submissions) { csv = lstsvToDataCsv(tsv, submissions, { diff --git a/src/ddi/codebook.ts b/src/ddi/codebook.ts index e3c5945..703aaca 100644 --- a/src/ddi/codebook.ts +++ b/src/ddi/codebook.ts @@ -22,6 +22,13 @@ import { XmlElement } from './xml.js'; import { classifyNotes } from './notes.js'; import { Choice, Translations, Variable } from './types.js'; import { joinTranslations, localizedChild, textsOf } from './translations.js'; +import { + addLogicNotes, + addUniverse, + addValrng, + logicContext, + type LogicContext, +} from './logic.js'; import { registeredVocabCodes } from '../conventions/fromFile.js'; import { languageTagOf } from '../utils/languageUtils.js'; @@ -89,6 +96,8 @@ interface AddVarSpec { labelTranslations?: Translations; hintTranslations?: Translations; guidanceHintTranslations?: Translations; + /** The variable whose logic (#151) this `` carries. */ + logic?: { v: Variable; ctx: LogicContext }; } /** An {@link AddVarSpec}'s translations, read off its source variable. */ @@ -129,7 +138,8 @@ function categoricalFormat( } /** - * Append one ``. Element order: `qstn → catgry* → concept → varFormat`. + * Append one ``. Element order: `qstn → valrng → universe → catgry* → + * concept → varFormat → notes`. * With `vocab`, no `` is emitted and `` carries `@vocab`. */ function addVarElement(parent: XmlElement, spec: AddVarSpec): XmlElement { @@ -167,6 +177,11 @@ function addVarElement(parent: XmlElement, spec: AddVarSpec): XmlElement { } } + if (spec.logic) { + addValrng(varEl, spec.logic.v); + addUniverse(varEl, spec.logic.v, spec.logic.ctx); + } + if (!vocab) { for (const choice of choices) { const catgry = varEl.child('catgry'); @@ -177,22 +192,35 @@ function addVarElement(parent: XmlElement, spec: AddVarSpec): XmlElement { varEl.textChild('concept', label, vocab ? { vocab } : {}); varEl.child('varFormat', { type: fmtType, schema: 'other' }); + if (spec.logic) addLogicNotes(varEl, spec.logic.v); return varEl; } +/** A group-level question's logic on its ``: universe, then notes. */ +function addGroupLogic( + grpEl: XmlElement, + v: Variable, + ctx: LogicContext, + notes?: { note: InlineNotes; name: string }, +): void { + addUniverse(grpEl, v, ctx); + if (notes) addGroupNote(grpEl, notes.note, notes.name); + addLogicNotes(grpEl, v); +} + /** Append a binary 0/1 `` for one `select_multiple` option. */ function addBinaryVar( parent: XmlElement, - varId: string, name: string, question: Variable, choice: Choice, + ctx: LogicContext, ): XmlElement { const questionLabel = question.label; const choiceLabel = choice.label; const varEl = parent.child('var', { - ID: varId, + ID: makeVarId(name), name, intrvl: 'discrete', files: 'F1', @@ -201,6 +229,8 @@ function addBinaryVar( const qstn = varEl.child('qstn', { responseDomainType: 'multiple' }); localizedChild(qstn, 'preQTxt', questionLabel, textsOf(question, 'label')); localizedChild(qstn, 'qstnLit', choiceLabel, choice.translations); + // The question's notes are on its varGrp; the prose also here, for readers. + addUniverse(varEl, question, ctx); for (const val of ['0', '1']) { varEl.child('catgry').textChild('catValu', val); @@ -241,7 +271,11 @@ function detectOtherPatterns(variables: Variable[]): Map { } /** Emit the parent `` (+ child multipleResp for multi). */ -function emitOtherPattern(dataDscr: XmlElement, p: OtherPattern): void { +function emitOtherPattern( + dataDscr: XmlElement, + p: OtherPattern, + ctx: LogicContext, +): void { const { base, otherVar } = p; const label = base.label; const labelTranslations = textsOf(base, 'label'); @@ -264,6 +298,8 @@ function emitOtherPattern(dataDscr: XmlElement, p: OtherPattern): void { }); localizedChild(parentEl, 'txt', label, labelTranslations); parentEl.textChild('concept', label); + // No var has the question's name: the parent group carries its logic. + addGroupLogic(parentEl, base, ctx); const childEl = dataDscr.child('varGrp', { ID: childId, @@ -286,20 +322,18 @@ function emitOtherPattern(dataDscr: XmlElement, p: OtherPattern): void { } /** Emit the `` elements associated with an `_other` pattern. */ -function emitOtherPatternVars(dataDscr: XmlElement, p: OtherPattern): void { +function emitOtherPatternVars( + dataDscr: XmlElement, + p: OtherPattern, + ctx: LogicContext, +): void { const { base, otherVar } = p; const baseName = base.name; if (p.isMulti) { for (const choice of base.choices) { if (choice.name === OTHER_CODE) continue; - addBinaryVar( - dataDscr, - makeVarId(`${baseName}_${choice.name}`), - `${baseName}_${choice.name}`, - base, - choice, - ); + addBinaryVar(dataDscr, `${baseName}_${choice.name}`, base, choice, ctx); } } else { addVarElement(dataDscr, { @@ -311,6 +345,7 @@ function emitOtherPatternVars(dataDscr: XmlElement, p: OtherPattern): void { hint: base.hint, guidanceHint: base.guidanceHint, ...specTranslations(base), + logic: { v: base, ctx }, }); } @@ -324,6 +359,7 @@ function emitOtherPatternVars(dataDscr: XmlElement, p: OtherPattern): void { hint: otherVar.hint, guidanceHint: otherVar.guidanceHint, ...specTranslations(otherVar), + logic: { v: otherVar, ctx }, }); } @@ -456,9 +492,9 @@ function addVarGroups( dataVars: Variable[], notes: InlineNotes, buckets: DataVarBuckets, - otherPatterns: Map, + ctx: LogicContext, ): void { - const { gridGroups, multiRespGroups } = buckets; + const { gridGroups, multiRespGroups, otherPatterns } = buckets; for (const [groupName, members] of gridGroups) { const group = getGroupLabel(dataVars, groupName); @@ -483,11 +519,11 @@ function addVarGroups( }); localizedChild(grpEl, 'txt', smVar.label, textsOf(smVar, 'label')); grpEl.textChild('concept', smVar.label); - addGroupNote(grpEl, notes, smName); + addGroupLogic(grpEl, smVar, ctx, { note: notes, name: smName }); } for (const p of otherPatterns.values()) { - emitOtherPattern(dataDscr, p); + emitOtherPattern(dataDscr, p, ctx); } } @@ -497,9 +533,10 @@ function addVars( dataVars: Variable[], notes: InlineNotes, buckets: DataVarBuckets, - otherPatterns: Map, + ctx: LogicContext, ): void { - const { gridGroups, multiRespGroups, standaloneVars } = buckets; + const { gridGroups, multiRespGroups, standaloneVars, otherPatterns } = + buckets; for (const [groupName, members] of gridGroups) { const group = getGroupLabel(dataVars, groupName); @@ -515,24 +552,19 @@ function addVars( opts: { preQTxt: group.label, preQTxtTranslations: group.translations }, guidanceHint: v.guidanceHint, ...specTranslations(v, false), + logic: { v, ctx }, }); } } for (const [smName, smVar] of multiRespGroups) { for (const choice of smVar.choices) { - addBinaryVar( - dataDscr, - makeVarId(`${smName}_${choice.name}`), - `${smName}_${choice.name}`, - smVar, - choice, - ); + addBinaryVar(dataDscr, `${smName}_${choice.name}`, smVar, choice, ctx); } } for (const p of otherPatterns.values()) { - emitOtherPatternVars(dataDscr, p); + emitOtherPatternVars(dataDscr, p, ctx); } for (const v of standaloneVars) { @@ -550,6 +582,7 @@ function addVars( hint: v.hint, guidanceHint: v.guidanceHint, ...specTranslations(v), + logic: { v, ctx }, }); } } @@ -596,8 +629,9 @@ export function buildDdiCodebook( const dataDscr = root.child('dataDscr'); const buckets = splitDataVars(dataVars); - addVarGroups(dataDscr, dataVars, notes, buckets, buckets.otherPatterns); - addVars(dataDscr, dataVars, notes, buckets, buckets.otherPatterns); + const ctx = logicContext(variables, lang); + addVarGroups(dataDscr, dataVars, notes, buckets, ctx); + addVars(dataDscr, dataVars, notes, buckets, ctx); return root; } diff --git a/src/ddi/fromInstrument.ts b/src/ddi/fromInstrument.ts index 841870b..2ed54f7 100644 --- a/src/ddi/fromInstrument.ts +++ b/src/ddi/fromInstrument.ts @@ -11,6 +11,7 @@ import { OTHER_COMPANION_TYPE, OTHER_SUFFIX, limesurveyOtherText, + otherCompanionRelevance, otherLabelFor, } from '../conventions/other.js'; import { isFromFileType, vocabFromFilename } from '../conventions/fromFile.js'; @@ -107,6 +108,38 @@ interface GroupContext { /** The group's label in every language, for its translations. */ labelText: Text; appearance: string; + /** The enclosing groups' `relevant` expressions, outermost first. */ + relevant: string[]; +} + +/** + * All of `conditions` as one XPath: each parenthesized, ANDed. Until every + * group has a `varGrp` (#152), a variable carries its groups' conditions. + */ +function allOf(conditions: string[]): string { + const parts = conditions.map((c) => c.trim()).filter(Boolean); + return parts.length === 1 + ? parts[0] + : parts.map((c) => `(${c})`).join(' and '); +} + +/** A question's logic fields, absent when empty (#151). */ +function logicOf( + q: QuestionItem, + ctx: GroupContext, + lang: string | undefined, +): Pick< + Variable, + 'relevant' | 'constraint' | 'constraintMessage' | 'required' +> { + const relevant = allOf([...ctx.relevant, q.relevant]); + const message = pick(q.constraintMessage, lang).trim(); + return { + ...(relevant ? { relevant } : {}), + ...(q.constraint ? { constraint: q.constraint } : {}), + ...(q.constraint && message ? { constraintMessage: message } : {}), + ...(q.required ? { required: true } : {}), + }; } /** @@ -242,15 +275,23 @@ function emitsVariable(q: QuestionItem): boolean { return !NO_DATA_APPEARANCES.has(q.appearance); } -/** The or_other shorthand's `_other` text variable, unless authored. */ +/** + * The or_other shorthand's `_other` text variable, unless authored. + * It is asked when "other" is chosen, as pyxform's (convention:other). + */ function pushCompanion( - name: string, + q: QuestionItem, + stdType: string, other: { label: string; translations?: Translations }, - group: Pick, ctx: GroupContext, state: ProjectState, ): void { + const name = q.name + OTHER_SUFFIX; if (state.authoredNames.has(name)) return; + const relevant = allOf([ + ...ctx.relevant, + otherCompanionRelevance(stdType, q.name), + ]); const translations: Record = {}; const groupLabels = translationsOf(ctx.labelText, state.others) ?? {}; for (const [, tag] of state.others) { @@ -263,10 +304,13 @@ function pushCompanion( name, type: OTHER_COMPANION_TYPE, label: other.label, - ...group, + group: ctx.path, + groupLabel: ctx.label, + groupAppearance: ctx.appearance, listName: '', vocab: '', choices: [], + relevant, ...(Object.keys(translations).length ? { translations } : {}), }); } @@ -296,6 +340,7 @@ function pushQuestion( hint: q.hint, guidanceHint: q.guidanceHint, groupLabel: ctx.labelText, + ...(q.constraint ? { constraintMessage: q.constraintMessage } : {}), }, state.others, ); @@ -312,10 +357,11 @@ function pushQuestion( hint: pick(q.hint, state.lang).trim(), guidanceHint: pick(q.guidanceHint, state.lang).trim(), }), + ...logicOf(q, ctx, state.lang), ...(translations ? { translations } : {}), }); - if (other) pushCompanion(q.name + OTHER_SUFFIX, other, group, ctx, state); + if (other) pushCompanion(q, stdType, other, ctx, state); } function project(items: Item[], ctx: GroupContext, state: ProjectState): void { @@ -331,6 +377,7 @@ function project(items: Item[], ctx: GroupContext, state: ProjectState): void { label: pick(item.label, state.lang), labelText: item.label, appearance: item.appearance, + relevant: [...ctx.relevant, item.relevant], }, state, ); @@ -393,7 +440,7 @@ export function variablesFromInstrument( }; project( instrument.body, - { path: '', label: '', labelText: {}, appearance: '' }, + { path: '', label: '', labelText: {}, appearance: '', relevant: [] }, state, ); return state.variables; diff --git a/src/ddi/logic.ts b/src/ddi/logic.ts new file mode 100644 index 0000000..74431d0 --- /dev/null +++ b/src/ddi/logic.ts @@ -0,0 +1,344 @@ +/** + * Skip logic and validation in the DDI (`convention:logicMapping`, + * `ddiEncoding`; #151). DDI Codebook 2.5 has no expression syntax, so each + * condition goes in twice: + * + * - readable, in the standard element: `` prose built + * from the expression and the referenced questions' labels, and a simple + * numeric range as ``; + * - exact, in a typed note the reverse direction reads: + * `${a} = 'x'`. + */ +import conventions from '../generated/conventions.js'; +import { parseXPath, type XPathNode } from '../instrument/xpathParser.js'; +import { localizedChild } from './translations.js'; +import type { Choice, Variable } from './types.js'; +import type { XmlElement } from './xml.js'; + +const ENCODING = conventions.conventions.logicMapping.ddiEncoding; +const NOTES = ENCODING.notes; + +interface ProseTemplate { + prefix: string; + and: string; + or: string; + not: string; + selected: string; + answered: string; + unanswered: string; + quotes: string[]; +} + +const TEMPLATES = ENCODING.universe.templates as Record; + +/** What the logic of one variable is written against. */ +export interface LogicContext { + /** Every variable by name, for the labels `${name}` refers to. */ + byName: Map; + /** The codebook's base language tag; `null` when undeclared. */ + base: string | null; + /** The codebook's other language tags. */ + others: string[]; +} + +/** The languages a variable's texts come in: its base and translations. */ +export function logicContext( + variables: Variable[], + base: string | null, +): LogicContext { + const others = new Set(); + for (const v of variables) { + for (const tag of Object.keys(v.translations ?? {})) others.add(tag); + } + return { + byName: new Map(variables.map((v) => [v.name, v])), + base, + others: [...others].filter((t) => t !== base), + }; +} + +/** A variable's label in `tag` (`null`: the base language). */ +function labelIn(v: Variable, tag: string | null): string | undefined { + const text = tag === null ? v.label : v.translations?.[tag]?.label; + return text?.trim() || undefined; +} + +function choiceLabelIn(c: Choice, tag: string | null): string | undefined { + const text = tag === null ? c.label : c.translations?.[tag]; + return text?.trim() || undefined; +} + +const SYMBOLS: Record = { + '=': '=', + '!=': '≠', + '<': '<', + '<=': '≤', + '>': '>', + '>=': '≥', +}; + +/** The mirror image of a comparison, for `literal op ${q}`. */ +const MIRROR: Record = { + '=': '=', + '!=': '!=', + '<': '>', + '<=': '>=', + '>': '<', + '>=': '<=', +}; + +/** Writes one condition in one language; `null` where it can't. */ +class Prose { + constructor( + private readonly t: ProseTemplate, + private readonly ctx: LogicContext, + /** The language's tag in the variables' texts (`null`: base). */ + private readonly tag: string | null, + ) {} + + private quote(text: string): string { + return `${this.t.quotes[0]}${text}${this.t.quotes[1]}`; + } + + private question(node: XPathNode): Variable | null { + return node.kind === 'path' && node.name !== null + ? (this.ctx.byName.get(node.name) ?? null) + : null; + } + + private questionText(v: Variable): string | null { + const label = labelIn(v, this.tag); + return label ? this.quote(label) : null; + } + + /** A literal compared with `v`: its choice's label, else as written. */ + private value(node: XPathNode, v: Variable | null): string | null { + if (node.kind === 'num') return node.text; + if (node.kind !== 'str') { + const other = this.question(node); + return other ? this.questionText(other) : null; + } + const choice = v?.choices.find((c) => c.name === node.value); + if (choice) return choiceLabelIn(choice, this.tag) ?? null; + return this.quote(node.value); + } + + private comparison( + op: string, + left: XPathNode, + right: XPathNode, + ): string | null { + let q = this.question(left); + let other = right; + if (!q) { + q = this.question(right); + other = left; + op = MIRROR[op]; + } + if (!q) return null; + const qText = this.questionText(q); + if (!qText) return null; + if (other.kind === 'str' && other.value === '') { + if (op === '=') return this.t.unanswered.replace('{q}', qText); + if (op === '!=') return this.t.answered.replace('{q}', qText); + return null; + } + const value = this.value(other, q); + return value === null ? null : `${qText} ${SYMBOLS[op]} ${value}`; + } + + private call(name: string, args: XPathNode[]): string | null { + if (name === 'not' && args.length === 1) { + const inner = this.write(args[0]); + return inner === null ? null : this.t.not.replace('{}', inner); + } + if (name !== 'selected' || args.length !== 2) return null; + const q = this.question(args[0]); + const qText = q ? this.questionText(q) : null; + const value = q ? this.value(args[1], q) : null; + if (!q || !qText || value === null || args[1].kind !== 'str') return null; + return q.type === 'select_multiple' + ? this.t.selected.replace('{q}', qText).replace('{v}', value) + : `${qText} = ${value}`; + } + + write(node: XPathNode): string | null { + if (node.kind === 'call') return this.call(node.name, node.args); + if (node.kind !== 'bin') return null; + if (node.op === 'and' || node.op === 'or') { + const sides = [node.left, node.right].map((side) => { + const text = this.write(side); + // A mixed and/or keeps its grouping. + const nested = + side.kind === 'bin' && + (side.op === 'and' || side.op === 'or') && + side.op !== node.op; + return text !== null && nested ? `(${text})` : text; + }); + if (sides.some((s) => s === null)) return null; + return sides.join(` ${this.t[node.op]} `); + } + if (node.op in SYMBOLS) { + return this.comparison(node.op, node.left, node.right); + } + return null; + } +} + +/** `${name}` as the parser's bare field name. */ +function parse(expression: string): XPathNode | null { + try { + return parseXPath(expression.replace(/\$\{([^}]+)\}/g, '$1')); + } catch { + return null; + } +} + +/** + * The universe sentence of a `relevant` expression in each language that + * has a template and every label it names: base (`''`) first, then by tag. + * Empty when the expression is outside what the prose covers. + */ +export function universeProse( + relevant: string, + ctx: LogicContext, +): Array<[string, string]> { + const tree = parse(relevant); + if (!tree) return []; + const out: Array<[string, string]> = []; + const baseTemplate = + TEMPLATES[ctx.base ?? ENCODING.universe.untaggedLanguage]; + const langs: Array<[string, string | null, ProseTemplate | undefined]> = [ + ['', null, baseTemplate], + ...ctx.others.map((tag): [string, string, ProseTemplate | undefined] => [ + tag, + tag, + TEMPLATES[tag], + ]), + ]; + for (const [lang, tag, t] of langs) { + if (!t) continue; + const text = new Prose(t, ctx, tag).write(tree); + if (text) out.push([lang, `${t.prefix}${text}`]); + } + return out; +} + +/** `` per language, when there's a condition. */ +export function addUniverse( + el: XmlElement, + v: Variable, + ctx: LogicContext, +): void { + if (!v.relevant) return; + for (const [lang, text] of universeProse(v.relevant, ctx)) { + el.textChild('universe', text, { + clusion: ENCODING.universe.clusion, + ...(lang ? { 'xml:lang': lang } : {}), + }); + } +} + +/** Bounds on `.` alone, as `` attributes. */ +export type Range = Partial< + Record<'min' | 'minExclusive' | 'max' | 'maxExclusive', string> +>; + +const RANGE_OPS = new Set(['<', '<=', '>', '>=']); + +/** Bounds one `. op n` (or `n op .`) sets; `null` if it isn't one. */ +function bound(node: XPathNode): Range | null { + if (node.kind !== 'bin' || !RANGE_OPS.has(node.op)) return null; + const self = (n: XPathNode) => n.kind === 'path' && n.name === null; + const number = (n: XPathNode): string | null => + n.kind === 'num' + ? n.text + : n.kind === 'neg' && n.operand.kind === 'num' + ? `-${n.operand.text}` + : null; + let op = node.op; + let n = number(node.right); + if (!(self(node.left) && n !== null)) { + n = number(node.left); + if (!(self(node.right) && n !== null)) return null; + op = MIRROR[op] as typeof op; + } + switch (op) { + case '>=': + return { min: n }; + case '>': + return { minExclusive: n }; + case '<=': + return { max: n }; + default: + return { maxExclusive: n }; + } +} + +/** + * A constraint that is only numeric bounds on the answer + * (`. >= 1 and . <= 10`), as `` attributes; `null` otherwise. + */ +export function simpleRange(constraint: string): Range | null { + const tree = parse(constraint); + if (!tree) return null; + const range: Range = {}; + const conjuncts: XPathNode[] = []; + const collect = (n: XPathNode) => { + if (n.kind === 'bin' && n.op === 'and') { + collect(n.left); + collect(n.right); + } else conjuncts.push(n); + }; + collect(tree); + for (const c of conjuncts) { + const b = bound(c); + if (!b) return null; + for (const [k, val] of Object.entries(b) as Array<[keyof Range, string]>) { + if (k in range || (k === 'min' && 'minExclusive' in range)) return null; + if (k === 'minExclusive' && 'min' in range) return null; + if (k === 'max' && 'maxExclusive' in range) return null; + if (k === 'maxExclusive' && 'max' in range) return null; + range[k] = val; + } + } + return range; +} + +const NUMERIC_TYPES = new Set(['integer', 'decimal']); + +/** `` for a numeric variable's simple constraint. */ +export function addValrng(el: XmlElement, v: Variable): void { + if (!v.constraint || !NUMERIC_TYPES.has(v.type)) return; + const range = simpleRange(v.constraint); + if (range) el.child('valrng').child('range', range); +} + +/** + * The typed notes of a variable's logic, in `convention:logicMapping`'s + * order: relevant, constraint, constraint_message (per language), required. + */ +export function addLogicNotes(el: XmlElement, v: Variable): void { + const subject = ENCODING.noteSubject; + if (v.relevant) { + el.textChild('notes', v.relevant, { type: NOTES.relevant.type, subject }); + } + if (v.constraint) { + el.textChild('notes', v.constraint, { + type: NOTES.constraint.type, + subject, + }); + } + if (v.constraintMessage) { + const translations: Record = {}; + for (const [tag, texts] of Object.entries(v.translations ?? {})) { + if (texts.constraintMessage) translations[tag] = texts.constraintMessage; + } + localizedChild(el, 'notes', v.constraintMessage, translations, { + type: NOTES.constraint_message.type, + }); + } + if (v.required) { + el.textChild('notes', NOTES.required.text, { type: NOTES.required.type }); + } +} diff --git a/src/ddi/types.ts b/src/ddi/types.ts index e4fc037..9748828 100644 --- a/src/ddi/types.ts +++ b/src/ddi/types.ts @@ -27,6 +27,7 @@ export interface VariableTexts { hint?: string; guidanceHint?: string; groupLabel?: string; + constraintMessage?: string; } /** One data-carrying (or note) row of a survey, normalized. */ @@ -53,6 +54,17 @@ export interface Variable { hint?: string; /** XLSForm `guidance_hint`, emitted as ``. */ guidanceHint?: string; + /** + * XLSForm `relevant` (XPath), with the enclosing groups' conditions ANDed + * in: `` prose plus a `cdl:relevant` note (#151). + */ + relevant?: string; + /** XLSForm `constraint` (XPath): a `cdl:constraint` note, maybe ``. */ + constraint?: string; + /** XLSForm `constraint_message`: a `cdl:constraint_message` note. */ + constraintMessage?: string; + /** XLSForm `required`: a `cdl:required` note. */ + required?: boolean; /** Its texts in the form's other languages, by language tag (#135). */ translations?: Record; } diff --git a/src/generated/conventions.json b/src/generated/conventions.json index bab9958..ddff170 100644 --- a/src/generated/conventions.json +++ b/src/generated/conventions.json @@ -225,7 +225,7 @@ } }, "logicMapping": { - "description": "XLSForm expresses survey logic in an XPath subset; LimeSurvey uses ExpressionScript (EM). The transformation transpiles between them (implemented in src/pipelines/xlsform2lstsv/xpathTranspiler.ts). DDI Codebook has no machine-readable expression syntax — logic fields are dropped when emitting DDI and tools MUST report this as loss.", + "description": "XLSForm expresses survey logic in an XPath subset; LimeSurvey uses ExpressionScript (EM). The transformation transpiles between them (implemented in src/pipelines/xlsform2lstsv/xpathTranspiler.ts). DDI Codebook 2.5 has no expression syntax of its own, so a CDL codebook carries the logic in two parts (formtransform#151): readable prose in the standard element (`universe`, `valrng`) and the expression itself in a typed ``, the extension point the DDI XSD provides (\"The attributes for notes permit a controlled vocabulary to be developed ('type' and 'subject')\"). See ddiEncoding.", "fields": [ { "concept": "relevance (conditional display)", @@ -234,9 +234,9 @@ "lsTsvColumn": "relevance", "lsSyntax": "ExpressionScript (EM)", "ddi": { - "closestElement": "var/universe", - "note": "DDI universe describes the subpopulation as prose, not as machine syntax. Currently NOT emitted — relevance is dropped in DDI output.", - "lossy": true + "element": "var/universe[@clusion='I'] + var/notes[@type='cdl:relevant']", + "note": "The universe is prose formtransform generates from the expression and the referenced questions' labels (see ddiEncoding.universe); the typed note carries the XPath verbatim. A select_multiple's go on its varGrp[@type='multipleResp'] (the semi-open pair's on its varGrp[@type='other']), with the universe prose also on each binary var. Until every group has a varGrp (formtransform#152), the enclosing groups' conditions are ANDed into each variable's own.", + "lossy": false } }, { @@ -246,9 +246,9 @@ "lsTsvColumn": "em_validation_q", "lsSyntax": "ExpressionScript (EM)", "ddi": { - "closestElement": "var/valrng", - "note": "DDI valrng can express simple numeric ranges only. Currently NOT emitted — constraints are dropped in DDI output.", - "lossy": true + "element": "var/notes[@type='cdl:constraint'] (+ var/valrng/range for a simple numeric range)", + "note": "The typed note carries the XPath verbatim. On an integer or decimal, a constraint that is only bounds on `.` (`. >= 1 and . <= 10`, `. > 0`) is also written as the standard valrng/range (min/max, minExclusive/maxExclusive).", + "lossy": false } }, { @@ -258,9 +258,21 @@ "lsTsvColumn": "em_validation_q_tip", "lsSyntax": "plain text", "ddi": { - "closestElement": null, - "note": "No DDI counterpart; dropped.", - "lossy": true + "element": "var/notes[@type='cdl:constraint_message']", + "note": "The form's own text, one note per language (base untagged, others with xml:lang), like every other form text.", + "lossy": false + } + }, + { + "concept": "required answer", + "xlsformColumn": "required", + "xlsformSyntax": "yes/no", + "lsTsvColumn": "mandatory", + "lsSyntax": "Y/N", + "ddi": { + "element": "var/notes[@type='cdl:required']", + "note": "Written only for a required question, with the text `yes`. It tells 'not asked' (universe) from 'refused' apart.", + "lossy": false } } ], @@ -306,7 +318,79 @@ "today", "now" ], - "unsupportedFunctionBehavior": "error — the transpiler throws on any XPath function outside supportedXPathFunctions" + "unsupportedFunctionBehavior": "error — the transpiler throws on any XPath function outside supportedXPathFunctions", + "ddiEncoding": { + "noteSubject": "xlsform-xpath", + "noteSubjectMeaning": "The note's text is an XLSForm XPath expression. `${name}` refers to a var/@name or varGrp/@name of the same codebook; `.` to the variable's own answer.", + "notes": { + "relevant": { + "type": "cdl:relevant", + "subject": "xlsform-xpath", + "on": [ + "var", + "varGrp" + ] + }, + "constraint": { + "type": "cdl:constraint", + "subject": "xlsform-xpath", + "on": [ + "var", + "varGrp" + ] + }, + "constraint_message": { + "type": "cdl:constraint_message", + "on": [ + "var", + "varGrp" + ], + "localized": true + }, + "required": { + "type": "cdl:required", + "on": [ + "var", + "varGrp" + ], + "text": "yes" + } + }, + "order": "The typed notes come last in var/varGrp, after any untyped note; universe comes after qstn and valrng, before catgry. At most one untyped note per language stays allowed (a citation).", + "universe": { + "clusion": "I", + "untaggedLanguage": "en", + "use": "The prose covers `=`, `!=`, `<`, `<=`, `>`, `>=`, `selected()`, `and`, `or`, `not()`, and a comparison with '' (answered / not answered). A condition using anything else gets no universe; its typed note still carries it. It is written in the codebook's base language and each other language with a template here (an untagged codebook uses untaggedLanguage), and only where every referenced label exists in that language. The connecting words are formtransform's metadata, not the form's text; labels and choice labels are the form's own.", + "templates": { + "en": { + "prefix": "Only if ", + "and": "and", + "or": "or", + "not": "not ({})", + "selected": "{q} includes {v}", + "answered": "{q} is answered", + "unanswered": "{q} is not answered", + "quotes": [ + "“", + "”" + ] + }, + "de": { + "prefix": "Nur wenn ", + "and": "und", + "or": "oder", + "not": "nicht ({})", + "selected": "{q} enthält {v}", + "answered": "{q} ist beantwortet", + "unanswered": "{q} ist nicht beantwortet", + "quotes": [ + "„", + "“" + ] + } + } + } + } }, "other": { "companionSuffix": "_other", diff --git a/src/generated/conventions.ts b/src/generated/conventions.ts index 74d6561..60762af 100644 --- a/src/generated/conventions.ts +++ b/src/generated/conventions.ts @@ -234,7 +234,7 @@ const conventions = { } }, "logicMapping": { - "description": "XLSForm expresses survey logic in an XPath subset; LimeSurvey uses ExpressionScript (EM). The transformation transpiles between them (implemented in src/pipelines/xlsform2lstsv/xpathTranspiler.ts). DDI Codebook has no machine-readable expression syntax — logic fields are dropped when emitting DDI and tools MUST report this as loss.", + "description": "XLSForm expresses survey logic in an XPath subset; LimeSurvey uses ExpressionScript (EM). The transformation transpiles between them (implemented in src/pipelines/xlsform2lstsv/xpathTranspiler.ts). DDI Codebook 2.5 has no expression syntax of its own, so a CDL codebook carries the logic in two parts (formtransform#151): readable prose in the standard element (`universe`, `valrng`) and the expression itself in a typed ``, the extension point the DDI XSD provides (\"The attributes for notes permit a controlled vocabulary to be developed ('type' and 'subject')\"). See ddiEncoding.", "fields": [ { "concept": "relevance (conditional display)", @@ -243,9 +243,9 @@ const conventions = { "lsTsvColumn": "relevance", "lsSyntax": "ExpressionScript (EM)", "ddi": { - "closestElement": "var/universe", - "note": "DDI universe describes the subpopulation as prose, not as machine syntax. Currently NOT emitted — relevance is dropped in DDI output.", - "lossy": true + "element": "var/universe[@clusion='I'] + var/notes[@type='cdl:relevant']", + "note": "The universe is prose formtransform generates from the expression and the referenced questions' labels (see ddiEncoding.universe); the typed note carries the XPath verbatim. A select_multiple's go on its varGrp[@type='multipleResp'] (the semi-open pair's on its varGrp[@type='other']), with the universe prose also on each binary var. Until every group has a varGrp (formtransform#152), the enclosing groups' conditions are ANDed into each variable's own.", + "lossy": false } }, { @@ -255,9 +255,9 @@ const conventions = { "lsTsvColumn": "em_validation_q", "lsSyntax": "ExpressionScript (EM)", "ddi": { - "closestElement": "var/valrng", - "note": "DDI valrng can express simple numeric ranges only. Currently NOT emitted — constraints are dropped in DDI output.", - "lossy": true + "element": "var/notes[@type='cdl:constraint'] (+ var/valrng/range for a simple numeric range)", + "note": "The typed note carries the XPath verbatim. On an integer or decimal, a constraint that is only bounds on `.` (`. >= 1 and . <= 10`, `. > 0`) is also written as the standard valrng/range (min/max, minExclusive/maxExclusive).", + "lossy": false } }, { @@ -267,9 +267,21 @@ const conventions = { "lsTsvColumn": "em_validation_q_tip", "lsSyntax": "plain text", "ddi": { - "closestElement": null, - "note": "No DDI counterpart; dropped.", - "lossy": true + "element": "var/notes[@type='cdl:constraint_message']", + "note": "The form's own text, one note per language (base untagged, others with xml:lang), like every other form text.", + "lossy": false + } + }, + { + "concept": "required answer", + "xlsformColumn": "required", + "xlsformSyntax": "yes/no", + "lsTsvColumn": "mandatory", + "lsSyntax": "Y/N", + "ddi": { + "element": "var/notes[@type='cdl:required']", + "note": "Written only for a required question, with the text `yes`. It tells 'not asked' (universe) from 'refused' apart.", + "lossy": false } } ], @@ -315,7 +327,79 @@ const conventions = { "today", "now" ], - "unsupportedFunctionBehavior": "error — the transpiler throws on any XPath function outside supportedXPathFunctions" + "unsupportedFunctionBehavior": "error — the transpiler throws on any XPath function outside supportedXPathFunctions", + "ddiEncoding": { + "noteSubject": "xlsform-xpath", + "noteSubjectMeaning": "The note's text is an XLSForm XPath expression. `${name}` refers to a var/@name or varGrp/@name of the same codebook; `.` to the variable's own answer.", + "notes": { + "relevant": { + "type": "cdl:relevant", + "subject": "xlsform-xpath", + "on": [ + "var", + "varGrp" + ] + }, + "constraint": { + "type": "cdl:constraint", + "subject": "xlsform-xpath", + "on": [ + "var", + "varGrp" + ] + }, + "constraint_message": { + "type": "cdl:constraint_message", + "on": [ + "var", + "varGrp" + ], + "localized": true + }, + "required": { + "type": "cdl:required", + "on": [ + "var", + "varGrp" + ], + "text": "yes" + } + }, + "order": "The typed notes come last in var/varGrp, after any untyped note; universe comes after qstn and valrng, before catgry. At most one untyped note per language stays allowed (a citation).", + "universe": { + "clusion": "I", + "untaggedLanguage": "en", + "use": "The prose covers `=`, `!=`, `<`, `<=`, `>`, `>=`, `selected()`, `and`, `or`, `not()`, and a comparison with '' (answered / not answered). A condition using anything else gets no universe; its typed note still carries it. It is written in the codebook's base language and each other language with a template here (an untagged codebook uses untaggedLanguage), and only where every referenced label exists in that language. The connecting words are formtransform's metadata, not the form's text; labels and choice labels are the form's own.", + "templates": { + "en": { + "prefix": "Only if ", + "and": "and", + "or": "or", + "not": "not ({})", + "selected": "{q} includes {v}", + "answered": "{q} is answered", + "unanswered": "{q} is not answered", + "quotes": [ + "“", + "”" + ] + }, + "de": { + "prefix": "Nur wenn ", + "and": "und", + "or": "oder", + "not": "nicht ({})", + "selected": "{q} enthält {v}", + "answered": "{q} ist beantwortet", + "unanswered": "{q} ist nicht beantwortet", + "quotes": [ + "„", + "“" + ] + } + } + } + } }, "other": { "companionSuffix": "_other", diff --git a/src/instrument/fromLstsv.ts b/src/instrument/fromLstsv.ts index fdfc825..2946a02 100644 --- a/src/instrument/fromLstsv.ts +++ b/src/instrument/fromLstsv.ts @@ -6,14 +6,18 @@ * * With `expressions`, relevance and constraints are reversed from * LimeSurvey's Expression Manager into XPath (`reverseExpressions.ts`), - * throwing `em-unsupported` on anything outside the forward dialect. Without, - * they stay `''` (the DDI carries none, so a DDI conversion never fails on - * them). + * throwing `em-unsupported` on anything outside the forward dialect (or, with + * `onWarning`, reporting it and leaving it out). Without, they stay `''`. */ import { OTHER_CODE, OTHER_SUFFIX } from '../conventions/other.js'; import { GRID_APPEARANCE } from '../conventions/grid.js'; import { fromFileTypeFor, vocabFromCssClass } from '../conventions/fromFile.js'; import { resolveType } from './lstsvTypes.js'; +import { + ConversionError, + warning, + type WarningHandler, +} from '../diagnostics.js'; import { buildSelectContext, reverseConstraint, @@ -319,6 +323,7 @@ function messageNote( function reverseAllExpressions( body: Item[], lists: Record, + onWarning?: WarningHandler, ): void { const selectCtx = buildSelectContext( allQuestions(body) @@ -331,11 +336,29 @@ function reverseAllExpressions( }; }), ); + // With a warning handler, an expression outside the dialect is reported + // and left out; without one, it throws. + const reverse = (item: Item, column: string, fn: (em: string) => string) => { + try { + return fn(cell(item.row as Row, column)); + } catch (error) { + if (!onWarning || !(error instanceof ConversionError)) throw error; + onWarning( + warning( + error.code, + `${column} of "${item.name}" is left out: ${error.message}`, + item.name, + ), + ); + return ''; + } + }; for (const item of allItems(body)) { - const row = item.row as Row; - item.relevant = reverseRelevance(cell(row, 'relevance'), selectCtx); + item.relevant = reverse(item, 'relevance', (em) => + reverseRelevance(em, selectCtx), + ); if (item.kind === 'question') { - item.constraint = reverseConstraint(cell(row, 'em_validation_q')); + item.constraint = reverse(item, 'em_validation_q', reverseConstraint); } } } @@ -343,6 +366,11 @@ function reverseAllExpressions( export interface LstsvParseOptions { /** Reverse relevance/constraints into XPath (default: leave them empty). */ expressions?: boolean; + /** + * With `expressions`: report an expression outside the dialect here and + * leave it out, instead of throwing. + */ + onWarning?: WarningHandler; } /** Parse LimeSurvey structure-TSV rows into an Instrument. */ @@ -378,7 +406,9 @@ export function instrumentFromLstsv( ...parseBody(baseRows, state), ...messageNote('end', 'surveyls_endtext', state), ]; - if (options.expressions) reverseAllExpressions(body, state.lists); + if (options.expressions) { + reverseAllExpressions(body, state.lists, options.onWarning); + } return { languages: languages.length ? languages : [''], ...(base ? { defaultLanguage: base } : {}), diff --git a/src/pipelines/README.md b/src/pipelines/README.md index 7160e27..e95dfac 100644 --- a/src/pipelines/README.md +++ b/src/pipelines/README.md @@ -52,31 +52,35 @@ path never guess at them. ## Why there is no `ddi2xlsform` or `ddi2lstsv` -Deliberate, not a gap. **DDI is the terminus of the pipeline graph** — it -describes a *dataset*, not an *instrument*, so it does not carry the information -a survey needs to run. +Not yet. **DDI is the terminus of the pipeline graph for now**: it describes a +*dataset*, not an *instrument*, and a reverse path needs the whole instrument. +The plan to carry all of it is #155 (#151–#154). The canonical `Variable` (`src/ddi/types.ts`) is what survives an emit: `name`, `type`, `label`, group path/label/appearance, `listName`, `vocab`, `choices`, -and the question's `hint` (``) and `guidance_hint` (``). -Everything that makes a form behave is absent: - -- **no `relevant`** — DDI Codebook 2.5 has no machine-readable expression syntax - at all, so skip logic is dropped on the way in. `convention:logicMapping` - ([`registry/conventions/logicMapping.jsonld`](../../registry/conventions/logicMapping.jsonld)) - records this and requires tools to report it as loss. -- **no `constraint`** — same reason. -- **no `required`, `default`, per-question `appearance`, `calculation`.** +the question's `hint` (``) and `guidance_hint` (``), and +since #151 its logic: + +- **`relevant`, `constraint`, `constraint_message`, `required`.** DDI Codebook + 2.5 has no expression syntax, so each is written twice: readable + (`` prose, `` for a simple numeric range) and exact, in a + typed note (``). + `convention:logicMapping` + ([`registry/conventions/logicMapping.jsonld`](../../registry/conventions/logicMapping.jsonld), + `ddiEncoding`) defines them. A group's condition is ANDed into each + variable's own until groups get a `varGrp` (#152). +- **still absent:** `default`, per-question `appearance`, `calculation`, plain + group structure, question order (#152, #153). Compare `lstsv2xlsform`, which *is* implemented: a LimeSurvey structure TSV carries `relevance`, `em_validation_q`, `mandatory`, `default` and the `!`/`T` type overrides. It is a form definition in a different dialect, so reversing it -is a translation problem. Reversing DDI is not — it is a *reconstruction* -problem, and the missing pieces cannot be inferred from a codebook. +is a translation problem. Reversing a codebook without those pieces is a +*reconstruction* problem, and they cannot be inferred from it. -The failure mode matters more than the missing feature. A `ddi2xlsform` would -emit a survey that looks correct and behaves wrongly: no skip logic, no -validation, nothing mandatory. Silently producing a broken instrument is worse +The failure mode matters more than the missing feature. A `ddi2xlsform` before +the rest of #155 would emit a survey that looks correct and behaves wrongly: no +defaults, no appearances, flattened groups. Silently producing a broken instrument is worse than declining to produce one — the same reasoning behind `validateLstsvSubset` rejecting out-of-subset input rather than guessing at it. diff --git a/src/pipelines/lstsv2ddi/index.ts b/src/pipelines/lstsv2ddi/index.ts index 866b8cd..5801942 100644 --- a/src/pipelines/lstsv2ddi/index.ts +++ b/src/pipelines/lstsv2ddi/index.ts @@ -18,6 +18,7 @@ import { normalizeLimeSurveyResponses } from './data.js'; import type { NormalizeResponsesOptions } from './data.js'; import { lstsvProjection, lstsvToVariables } from './toVariables.js'; import { ConversionError } from '../../diagnostics.js'; +import type { WarningHandler } from '../../diagnostics.js'; export { parseLstsv } from '../../lstsv/parser.js'; export { lstsvToVariables } from './toVariables.js'; @@ -32,6 +33,8 @@ export interface LstsvToDdiOptions extends BuildDdiOptions { * default: an unsupported code would otherwise silently mis-type a variable. */ skipValidation?: boolean; + /** Receives what the DDI leaves out, e.g. an expression it can't reverse. */ + onWarning?: WarningHandler; } /** Parse a TSV and apply the reverse subset check unless skipped. */ @@ -65,14 +68,14 @@ export function lstsvToDdiXml( tsv: string, options: LstsvToDdiOptions = {}, ): string { - const { skipValidation, ...ddiOptions } = options; + const { skipValidation, onWarning, ...ddiOptions } = options; const rows = parseChecked(tsv, skipValidation); const title = rows.find( (r) => r.class?.trim() === 'SL' && r.name?.trim() === 'surveyls_title', )?.text; - const { variables, language } = lstsvProjection(rows); + const { variables, language } = lstsvProjection(rows, onWarning); const opts: BuildDdiOptions = { ...ddiOptions }; if (!opts.assetName && title?.trim()) { opts.settings = { form_title: title.trim(), ...opts.settings }; diff --git a/src/pipelines/lstsv2ddi/toVariables.ts b/src/pipelines/lstsv2ddi/toVariables.ts index 9538425..494e296 100644 --- a/src/pipelines/lstsv2ddi/toVariables.ts +++ b/src/pipelines/lstsv2ddi/toVariables.ts @@ -12,6 +12,7 @@ import { variablesFromInstrument, } from '../../ddi/fromInstrument.js'; import { instrumentFromLstsv } from '../../instrument/fromLstsv.js'; +import type { WarningHandler } from '../../diagnostics.js'; type Row = Record; @@ -24,21 +25,30 @@ export function lstsvToVariables(rows: Row[]): Variable[] { } /** - * {@link lstsvToVariables} plus, for a survey in several languages, its base + * {@link lstsvToVariables} with relevance and constraints reversed into XPath + * for the DDI's logic (#151): one outside the dialect gets a warning and is + * left out, rather than failing the conversion. Plus, for a survey in several languages, its base * language (the `language` S row, as a BCP 47 tag): the DDI declares it as * `codeBook/@xml:lang` so the untagged texts say their language. A * single-language survey's DDI stays undeclared, as an XLSForm's without * `default_language` does. */ -export function lstsvProjection(rows: Row[]): { +export function lstsvProjection( + rows: Row[], + onWarning?: WarningHandler, +): { variables: Variable[]; language?: string; } { - const instrument = instrumentFromLstsv(rows); + const instrument = instrumentFromLstsv(rows, { + expressions: true, + onWarning: onWarning ?? (() => {}), + }); return { variables: variablesFromInstrument( instrument, choicesFromInstrument(instrument), + { onWarning }, ), language: instrument.languages.length > 1 ? instrument.defaultLanguage : undefined, diff --git a/tests/fixtures/surveys/all_types_survey/ddi.xml b/tests/fixtures/surveys/all_types_survey/ddi.xml index 7b197c6..a95dca6 100644 --- a/tests/fixtures/surveys/all_types_survey/ddi.xml +++ b/tests/fixtures/surveys/all_types_survey/ddi.xml @@ -109,8 +109,10 @@ Sonstiges: + Nur wenn „Welcher Track hat Ihnen am besten gefallen?“ = Sonstiges: Sonstiges: + ${q_sel1_other} = 'other' @@ -158,8 +160,10 @@ Sonstiges: + Nur wenn „Welche Tracks würden Sie erneut besuchen?“ enthält Sonstiges: Sonstiges: + selected(${q_selm_other}, 'other') @@ -355,6 +359,7 @@ Wie lautet Ihre E-Mail-Adresse? + yes @@ -368,8 +373,13 @@ Wie alt sind Sie? + + + Wie alt sind Sie? + . >= 18 and . <= 120 + Bitte geben Sie ein gültiges Alter zwischen 18 und 120 ein @@ -382,16 +392,21 @@ Wie sollen wir Sie kontaktieren? + Nur wenn „Wie lautet Ihre E-Mail-Adresse?“ ist beantwortet Wie sollen wir Sie kontaktieren? + ${q_mandatory} != '' Freiwillig, aber willkommen Haben Sie noch Anmerkungen? + Nur wenn „Wie lautet Ihre E-Mail-Adresse?“ ist beantwortet Haben Sie noch Anmerkungen? + ${q_mandatory} != '' + yes diff --git a/tests/fixtures/surveys/basic_survey/ddi.xml b/tests/fixtures/surveys/basic_survey/ddi.xml index bf3701a..ed2d449 100644 --- a/tests/fixtures/surveys/basic_survey/ddi.xml +++ b/tests/fixtures/surveys/basic_survey/ddi.xml @@ -29,6 +29,7 @@ What is your name? + yes @@ -36,6 +37,7 @@ What is your age? + yes @@ -51,6 +53,7 @@ Do you consent to participate? + yes diff --git a/tests/fixtures/surveys/bilingual_survey/ddi.xml b/tests/fixtures/surveys/bilingual_survey/ddi.xml index 249dd8c..bdfbcb4 100644 --- a/tests/fixtures/surveys/bilingual_survey/ddi.xml +++ b/tests/fixtures/surveys/bilingual_survey/ddi.xml @@ -151,8 +151,11 @@ Sonstiges: Other: + Nur wenn „Wie haben Sie von uns erfahren?“ = Sonstiges: + Only if “How did you hear about us?” = Other: Sonstiges: + ${quelle} = 'other' diff --git a/tests/fixtures/surveys/complex_survey/ddi.xml b/tests/fixtures/surveys/complex_survey/ddi.xml index deffef7..d3b33a1 100644 --- a/tests/fixtures/surveys/complex_survey/ddi.xml +++ b/tests/fixtures/surveys/complex_survey/ddi.xml @@ -25,12 +25,15 @@ What are your favorite colors? What are your favorite colors? + Only if “Age” ≥ 18 + ${age} >= 18 What are your favorite colors? Red + Only if “Age” ≥ 18 0 @@ -45,6 +48,7 @@ What are your favorite colors? Blue + Only if “Age” ≥ 18 0 @@ -59,6 +63,7 @@ What are your favorite colors? Green + Only if “Age” ≥ 18 0 @@ -73,6 +78,7 @@ What are your favorite colors? Yellow + Only if “Age” ≥ 18 0 @@ -88,13 +94,19 @@ Full Name + yes Age + + + Age + . >= 18 and . <= 120 + Age must be between 18 and 120 @@ -119,6 +131,7 @@ How satisfied are you? + Only if “Age” ≥ 18 happ Happy @@ -129,13 +142,16 @@ How satisfied are you? + ${age} >= 18 Any other comments? + Only if “Age” ≥ 18 and “How satisfied are you?” = “unhappy” Any other comments? + (${age} >= 18) and (${satisfaction_level} = 'unhappy') diff --git a/tests/fixtures/surveys/complex_xpath_survey/ddi.xml b/tests/fixtures/surveys/complex_xpath_survey/ddi.xml index 677b89d..d5b47f4 100644 --- a/tests/fixtures/surveys/complex_xpath_survey/ddi.xml +++ b/tests/fixtures/surveys/complex_xpath_survey/ddi.xml @@ -47,22 +47,28 @@ Adult consent verification + Only if “Do you consent to participate?” = “yes” and “What is your age?” ≥ 18 Adult consent verification + selected(${consent}, 'yes') and ${age} >= 18 Eligible participant + Only if “What is your age?” ≥ 18 and (“What country do you live in?” = “USA” or “What country do you live in?” = “Canada”) Eligible participant + ${age} >= 18 and (${country} = 'USA' or ${country} = 'Canada') Valid response check + Only if not (“Do you consent to participate?” is not answered) and “What is your age?” > 0 Valid response check + not(${consent} = '') and ${age} > 0 @@ -70,13 +76,17 @@ Age category + if(${age} > 18, true(), false()) Complex validation test + Only if “Do you consent to participate?” = “yes” and (“What is your age?” ≥ 18 or “What country do you live in?” = “USA”) Complex validation test + ${consent} = 'yes' and (${age} >= 18 or ${country} = 'USA') + . >= 5 and . <= 100 diff --git a/tests/fixtures/surveys/settings_survey/ddi.xml b/tests/fixtures/surveys/settings_survey/ddi.xml index d87e351..56408ec 100644 --- a/tests/fixtures/surveys/settings_survey/ddi.xml +++ b/tests/fixtures/surveys/settings_survey/ddi.xml @@ -51,8 +51,10 @@ Please specify your color + Only if “What is your _favorite_ color?” = Other Please specify your color + ${fav_color} = 'other' diff --git a/tests/fixtures/surveys/testA/ddi.xml b/tests/fixtures/surveys/testA/ddi.xml index b6a7b88..7754e68 100644 --- a/tests/fixtures/surveys/testA/ddi.xml +++ b/tests/fixtures/surveys/testA/ddi.xml @@ -113,6 +113,7 @@ Wie lange kommst du schon zu TestWerk? + yes @@ -136,6 +137,7 @@ Wie oft kommst du ungefähr? + yes @@ -171,6 +173,7 @@ In welcher Werkstatt bist du hauptsächlich? + yes @@ -190,6 +193,7 @@ Hattest du vorher schon handwerkliche Erfahrung? + yes @@ -217,6 +221,7 @@ Ich fühle mich bei TestWerk wohl und akzeptiert. + yes @@ -244,6 +249,7 @@ Ich habe bei TestWerk Freund*innen gefunden. + yes @@ -271,6 +277,7 @@ Ich traue mir zu, eigene Ideen und Projekte umzusetzen. + yes @@ -298,6 +305,7 @@ Wenn bei einem Projekt etwas nicht klappt, finde ich eine Lösung. + yes @@ -325,6 +333,7 @@ Ich habe bei TestWerk handwerkliche Fähigkeiten gelernt. + yes @@ -353,6 +362,7 @@ Bevor ich zu TestWerk kam, hatte ich eine klare Vorstellung von meiner beruflichen Zukunft. + yes @@ -380,14 +390,17 @@ Aktuell habe ich eine klare Vorstellung von meiner beruflichen Zukunft. + yes 0% = gar nicht, 100% = komplett Ca. Wie viel Prozent dieser Veränderung geht auf TestWerk zurück? + Only if “Aktuell habe ich eine klare Vorstellung von meiner beruflichen Zukunft.” ≠ “Bevor ich zu TestWerk kam, hatte ich eine klare Vorstellung von meiner beruflichen Zukunft.” Ca. Wie viel Prozent dieser Veränderung geht auf TestWerk zurück? + ${beruf_post} != ${beruf_pre} @@ -461,6 +474,7 @@ Würdest du TestWerk Freund*innen empfehlen? + yes @@ -516,8 +530,10 @@ Falls "Anderes": Bitte angeben (optional) + Only if “Geschlecht (optional)” = Anderes Geschlecht Falls "Anderes": Bitte angeben (optional) + ${geschlecht} = 'andere' diff --git a/tests/fixtures/surveys/testB/ddi.xml b/tests/fixtures/surveys/testB/ddi.xml index 6aed917..ba2b30c 100644 --- a/tests/fixtures/surveys/testB/ddi.xml +++ b/tests/fixtures/surveys/testB/ddi.xml @@ -32,21 +32,34 @@ Für welche Projekte möchtest Du dich bewerben? Which project(s) do you want to apply for? Für welche Projekte möchtest Du dich bewerben? + yes In welcher Rolle siehst du dich im Projekt Projekt Alpha? Which role do you think you could fill in Project Alpha? In welcher Rolle siehst du dich im Projekt Projekt Alpha? + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Alpha + Only if “Which project(s) do you want to apply for?” includes Project Alpha + selected(${project_id}, 'project-alpha') + yes In welcher Rolle siehst du dich im Projekt Beta? Which role do you see yourself in for Project Beta? In welcher Rolle siehst du dich im Projekt Beta? + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Beta + Only if “Which project(s) do you want to apply for?” includes Project Beta + selected(${project_id}, 'project-beta') + yes In welcher Rolle siehst du dich im Projekt Gamma? Which role do you see yourself in for Project Gamma? In welcher Rolle siehst du dich im Projekt Gamma? + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Gamma + Only if “Which project(s) do you want to apply for?” includes Project Gamma + selected(${project_id}, 'project-gamma') + yes @@ -103,6 +116,8 @@ Umfragedesign und Datenerhebung Survey design and data collection + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Alpha + Only if “Which project(s) do you want to apply for?” includes Project Alpha 0 @@ -119,6 +134,8 @@ Datenanalyse Data analysis + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Alpha + Only if “Which project(s) do you want to apply for?” includes Project Alpha 0 @@ -135,6 +152,8 @@ Teamkoordinator:in Team Coordinator + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Alpha + Only if “Which project(s) do you want to apply for?” includes Project Alpha 0 @@ -151,6 +170,8 @@ Team Trainee Team Trainee + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Alpha + Only if “Which project(s) do you want to apply for?” includes Project Alpha 0 @@ -167,6 +188,8 @@ Datenvisualisierung Data visualization + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Beta + Only if “Which project(s) do you want to apply for?” includes Project Beta 0 @@ -183,6 +206,8 @@ Data Engineering Data engineering + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Beta + Only if “Which project(s) do you want to apply for?” includes Project Beta 0 @@ -199,6 +224,8 @@ Teamkoordinator:in Team Coordinator + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Beta + Only if “Which project(s) do you want to apply for?” includes Project Beta 0 @@ -215,6 +242,8 @@ Team Trainee Team Trainee + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Beta + Only if “Which project(s) do you want to apply for?” includes Project Beta 0 @@ -231,6 +260,8 @@ Projektmanagement Project management + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Gamma + Only if “Which project(s) do you want to apply for?” includes Project Gamma 0 @@ -247,6 +278,8 @@ Machine Learning Machine learning + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Gamma + Only if “Which project(s) do you want to apply for?” includes Project Gamma 0 @@ -263,6 +296,8 @@ Teamkoordinator:in Team Coordinator + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Gamma + Only if “Which project(s) do you want to apply for?” includes Project Gamma 0 @@ -279,6 +314,8 @@ Team Trainee Team Trainee + Nur wenn „Für welche Projekte möchtest Du dich bewerben?“ enthält Projekt Gamma + Only if “Which project(s) do you want to apply for?” includes Project Gamma 0 @@ -293,6 +330,8 @@ SoSci Survey SoSci Survey + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Umfragedesign und Datenerhebung + Only if “Which role do you think you could fill in Project Alpha?” includes Survey design and data collection beginner Anfänger:in @@ -315,12 +354,16 @@ SoSci Survey + selected(${project_role_project-alpha}, 'role-survey-design') + yes Python Python + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Datenanalyse + Only if “Which role do you think you could fill in Project Alpha?” includes Data analysis beginner Anfänger:in @@ -343,12 +386,16 @@ Python + selected(${project_role_project-alpha}, 'role-data-analysis') + yes R R + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Datenanalyse + Only if “Which role do you think you could fill in Project Alpha?” includes Data analysis beginner Anfänger:in @@ -371,12 +418,16 @@ R + selected(${project_role_project-alpha}, 'role-data-analysis') + yes Power BI Power BI + Nur wenn „In welcher Rolle siehst du dich im Projekt Beta?“ enthält Datenvisualisierung + Only if “Which role do you see yourself in for Project Beta?” includes Data visualization beginner Anfänger:in @@ -399,12 +450,16 @@ Power BI + selected(${project_role_project-beta}, 'role-visualization') + yes Microsoft Excel Microsoft Excel + Nur wenn „In welcher Rolle siehst du dich im Projekt Beta?“ enthält Datenvisualisierung + Only if “Which role do you see yourself in for Project Beta?” includes Data visualization beginner Anfänger:in @@ -427,12 +482,16 @@ Microsoft Excel + selected(${project_role_project-beta}, 'role-visualization') + yes SQL SQL + Nur wenn „In welcher Rolle siehst du dich im Projekt Beta?“ enthält Data Engineering + Only if “Which role do you see yourself in for Project Beta?” includes Data engineering beginner Anfänger:in @@ -455,12 +514,16 @@ SQL + selected(${project_role_project-beta}, 'role-data-engineering') + yes Git Git + Nur wenn „In welcher Rolle siehst du dich im Projekt Beta?“ enthält Data Engineering + Only if “Which role do you see yourself in for Project Beta?” includes Data engineering beginner Anfänger:in @@ -483,12 +546,16 @@ Git + selected(${project_role_project-beta}, 'role-data-engineering') + yes Jupyter Notebooks Jupyter Notebooks + Nur wenn „In welcher Rolle siehst du dich im Projekt Gamma?“ enthält Machine Learning + Only if “Which role do you see yourself in for Project Gamma?” includes Machine learning beginner Anfänger:in @@ -511,12 +578,16 @@ Jupyter Notebooks + selected(${project_role_project-gamma}, 'role-machine-learning') + yes TensorFlow / PyTorch TensorFlow / PyTorch + Nur wenn „In welcher Rolle siehst du dich im Projekt Gamma?“ enthält Machine Learning + Only if “Which role do you see yourself in for Project Gamma?” includes Machine learning beginner Anfänger:in @@ -539,6 +610,8 @@ TensorFlow / PyTorch + selected(${project_role_project-gamma}, 'role-machine-learning') + yes @@ -555,6 +628,8 @@ Entwicklung von Fragebögen Survey design + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Umfragedesign und Datenerhebung + Only if “Which role do you think you could fill in Project Alpha?” includes Survey design and data collection beginner Anfänger:in @@ -577,12 +652,16 @@ Entwicklung von Fragebögen + selected(${project_role_project-alpha}, 'role-survey-design') + yes Entwicklung von Indikatoren Development of indicators + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Umfragedesign und Datenerhebung + Only if “Which role do you think you could fill in Project Alpha?” includes Survey design and data collection beginner Anfänger:in @@ -605,12 +684,16 @@ Entwicklung von Indikatoren + selected(${project_role_project-alpha}, 'role-survey-design') + yes Datenerhebung Data collection + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Umfragedesign und Datenerhebung + Only if “Which role do you think you could fill in Project Alpha?” includes Survey design and data collection beginner Anfänger:in @@ -633,12 +716,16 @@ Datenerhebung + selected(${project_role_project-alpha}, 'role-survey-design') + yes Datenbereinigung Data cleaning + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Datenanalyse + Only if “Which role do you think you could fill in Project Alpha?” includes Data analysis beginner Anfänger:in @@ -661,12 +748,16 @@ Datenbereinigung + selected(${project_role_project-alpha}, 'role-data-analysis') + yes Deskriptive Statistik Descriptive statistics + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Datenanalyse + Only if “Which role do you think you could fill in Project Alpha?” includes Data analysis beginner Anfänger:in @@ -689,12 +780,16 @@ Deskriptive Statistik + selected(${project_role_project-alpha}, 'role-data-analysis') + yes Datenvisualisierung Data visualization + Nur wenn „In welcher Rolle siehst du dich im Projekt Beta?“ enthält Datenvisualisierung + Only if “Which role do you see yourself in for Project Beta?” includes Data visualization beginner Anfänger:in @@ -717,12 +812,16 @@ Datenvisualisierung + selected(${project_role_project-beta}, 'role-visualization') + yes Data Engineering Data engineering + Nur wenn „In welcher Rolle siehst du dich im Projekt Beta?“ enthält Data Engineering + Only if “Which role do you see yourself in for Project Beta?” includes Data engineering beginner Anfänger:in @@ -745,12 +844,16 @@ Data Engineering + selected(${project_role_project-beta}, 'role-data-engineering') + yes Automatisierung Automation + Nur wenn „In welcher Rolle siehst du dich im Projekt Beta?“ enthält Data Engineering + Only if “Which role do you see yourself in for Project Beta?” includes Data engineering beginner Anfänger:in @@ -773,12 +876,16 @@ Automatisierung + selected(${project_role_project-beta}, 'role-data-engineering') + yes Projektplanung Project planning + Nur wenn „In welcher Rolle siehst du dich im Projekt Gamma?“ enthält Projektmanagement + Only if “Which role do you see yourself in for Project Gamma?” includes Project management beginner Anfänger:in @@ -801,12 +908,16 @@ Projektplanung + selected(${project_role_project-gamma}, 'role-project-management') + yes ML-Modellierung ML modeling + Nur wenn „In welcher Rolle siehst du dich im Projekt Gamma?“ enthält Machine Learning + Only if “Which role do you see yourself in for Project Gamma?” includes Machine learning beginner Anfänger:in @@ -829,12 +940,16 @@ ML-Modellierung + selected(${project_role_project-gamma}, 'role-machine-learning') + yes Wirkungsmessung Impact Measurement + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Umfragedesign und Datenerhebung + Only if “Which role do you think you could fill in Project Alpha?” includes Survey design and data collection beginner Anfänger:in @@ -857,12 +972,16 @@ Wirkungsmessung + selected(${project_role_project-alpha}, 'role-survey-design') + yes Research Design Research Design + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Umfragedesign und Datenerhebung + Only if “Which role do you think you could fill in Project Alpha?” includes Survey design and data collection beginner Anfänger:in @@ -885,12 +1004,16 @@ Research Design + selected(${project_role_project-alpha}, 'role-survey-design') + yes Umfrageforschung Survey Research + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Umfragedesign und Datenerhebung + Only if “Which role do you think you could fill in Project Alpha?” includes Survey design and data collection beginner Anfänger:in @@ -913,12 +1036,16 @@ Umfrageforschung + selected(${project_role_project-alpha}, 'role-survey-design') + yes Datenschutz Data protection + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Umfragedesign und Datenerhebung + Only if “Which role do you think you could fill in Project Alpha?” includes Survey design and data collection beginner Anfänger:in @@ -941,12 +1068,16 @@ Datenschutz + selected(${project_role_project-alpha}, 'role-survey-design') + yes Resilienz und Mentale Gesundheit Resilience and mental health + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Umfragedesign und Datenerhebung + Only if “Which role do you think you could fill in Project Alpha?” includes Survey design and data collection beginner Anfänger:in @@ -969,12 +1100,16 @@ Resilienz und Mentale Gesundheit + selected(${project_role_project-alpha}, 'role-survey-design') + yes Bildungsforschung o.ä. Education research (or similar) + Nur wenn „In welcher Rolle siehst du dich im Projekt Projekt Alpha?“ enthält Umfragedesign und Datenerhebung + Only if “Which role do you think you could fill in Project Alpha?” includes Survey design and data collection beginner Anfänger:in @@ -997,6 +1132,8 @@ Bildungsforschung o.ä. + selected(${project_role_project-alpha}, 'role-survey-design') + yes @@ -1007,6 +1144,7 @@ Bitte beschreibe hier, welche Deiner Fähigkeiten und Erfahrungen Dich besonders für die Teilnahme an diesem Projekt qualifizieren. + yes @@ -1017,6 +1155,7 @@ Bitte beschreibe hier, warum Du Dich für dieses Projekt engagieren möchtest. + yes @@ -1042,6 +1181,7 @@ Hast du dich in der Vergangenheit bereits auf CorrelAid-Projekte beworben? + yes @@ -1050,8 +1190,11 @@ Bitte gib hier an, für welche Projekte du dich beworben hast, falls du dich noch erinnern kannst. If you can remember, please specify which project(s) you have applied for. + Nur wenn „Hast du dich in der Vergangenheit bereits auf CorrelAid-Projekte beworben?“ = Ja, aber ich war nicht erfolgreich / ich wurde bis jetzt immer abgelehnt. + Only if “Have you applied to a CorrelAid project in the past?” = Yes, but I have not been successful / all my applications have been rejected so far. Bitte gib hier an, für welche Projekte du dich beworben hast, falls du dich noch erinnern kannst. + ${past_applications} = 'not_successful' @@ -1060,6 +1203,7 @@ Wie lautet dein Vorname? + yes @@ -1068,6 +1212,7 @@ Wie lautet dein Nachname? + yes @@ -1076,6 +1221,9 @@ Unter welcher E-Mail Adresse können wir dich erreichen? + regex(., '^[\w-\.]+@([\w-]+\.)+[\w-]{2,}$') + Please specify a valid email address. + yes @@ -1111,6 +1259,7 @@ Was ist dein Geschlecht? + yes @@ -1119,8 +1268,12 @@ Mein Geschlecht ist: My gender is: + Nur wenn „Was ist dein Geschlecht?“ = Mein Geschlecht ist (eigene Angabe): + Only if “What is your gender?” = My gender is (manually specify): Mein Geschlecht ist: + ${gender} = 'self_identification' + yes @@ -1136,6 +1289,7 @@ Einwilligung in die Datenschutzerklärung + yes diff --git a/tests/fixtures/surveys/validation_relevance_survey/ddi.xml b/tests/fixtures/surveys/validation_relevance_survey/ddi.xml index 8ebe618..893d001 100644 --- a/tests/fixtures/surveys/validation_relevance_survey/ddi.xml +++ b/tests/fixtures/surveys/validation_relevance_survey/ddi.xml @@ -28,20 +28,32 @@ Username + . != '' + Username cannot be empty Age + + + Age + . >= 18 and . <= 120 + Age must be between 18 and 120 Price + + + Price + . > 0 and . < 1000 + Price must be between 0 and 1000 @@ -62,36 +74,47 @@ Adult Information + Only if “Age” ≥ 18 Adult Information + ${age} >= 18 + . != '' Senior Discount Code + Only if “Age” ≥ 65 Senior Discount Code + ${age} >= 65 Additional Information + Only if “Do you consent?” = Yes Additional Information + ${consent} = 'yes' Complex Relevance Test + Only if “Age” ≥ 18 and “Do you consent?” = Yes Complex Relevance Test + ${age} >= 18 and ${consent} = 'yes' OR Relevance Test + Only if “Age” < 18 or “Do you consent?” = No OR Relevance Test + ${age} < 18 or ${consent} = 'no' @@ -99,13 +122,18 @@ String Function Test + string-length(${username}) > 3 + string-length(.) >= 5 Math Function Test + Only if “Age” > 21 Math Function Test + ${age} > 21 + . >= ${age} * 2 diff --git a/tests/ts/unit/ddi/logic.test.ts b/tests/ts/unit/ddi/logic.test.ts new file mode 100644 index 0000000..2b25747 --- /dev/null +++ b/tests/ts/unit/ddi/logic.test.ts @@ -0,0 +1,324 @@ +/** + * Skip logic and validation in the DDI (#151, convention:logicMapping + * `ddiEncoding`): `` prose and `` for readers, typed + * `cdl:` notes with the exact expression for the way back. + */ +import { describe, test, expect } from 'vitest'; + +import { buildDdiXml } from '../../../../src/pipelines/xlsform2ddi/index.js'; +import { lstsvToDdiXml } from '../../../../src/pipelines/lstsv2ddi/index.js'; +import { simpleRange } from '../../../../src/ddi/logic.js'; + +const OPTS = { prodDate: '2020-01-01' }; + +const YN = [ + { list_name: 'yn', name: 'yes', label: 'Yes' }, + { list_name: 'yn', name: 'no', label: 'No' }, +]; + +/** The `` element's XML. */ +function varXml(xml: string, name: string): string { + const start = xml.indexOf(`', start)); +} + +function escapeXml(text: string): string { + return text.replace(//g, '>'); +} + +function ddi(rows: Record[], settings = {}): string { + return buildDdiXml(rows, YN, { ...OPTS, settings }); +} + +describe('relevant', () => { + test('a universe sentence and the exact expression as a typed note', () => { + const xml = ddi([ + { type: 'select_one yn', name: 'dog', label: 'Do you have a dog?' }, + { + type: 'text', + name: 'dogname', + label: 'Its name?', + relevant: "${dog} = 'yes'", + }, + ]); + const v = varXml(xml, 'dogname'); + expect(v).toContain( + 'Only if “Do you have a dog?” = Yes', + ); + expect(v).toContain( + `\${dog} = 'yes'`, + ); + // XSD order: qstn, universe, …, concept, varFormat, notes. + expect(v.indexOf('')).toBeLessThan(v.indexOf('= 18 and ${n} != 99', 'Only if “Age” ≥ 18 and “Age” ≠ 99'], + [ + '${n} < 18 or (${n} > 65 and ${n} < 90)', + 'Only if “Age” < 18 or (“Age” > 65 and “Age” < 90)', + ], + ['18 <= ${n}', 'Only if “Age” ≥ 18'], + ["${n} != ''", 'Only if “Age” is answered'], + ["${n} = ''", 'Only if “Age” is not answered'], + ['not(${n} > 3)', 'Only if not (“Age” > 3)'], + ])('%s', (relevant, prose) => { + const xml = ddi([ + { type: 'integer', name: 'n', label: 'Age' }, + { type: 'text', name: 't', label: 'T', relevant }, + ]); + expect(varXml(xml, 't')).toContain(`>${escapeXml(prose)}`); + }); + + test('selected() on a select_multiple reads as "includes"', () => { + const xml = ddi([ + { type: 'select_multiple yn', name: 'm', label: 'Which?' }, + { + type: 'text', + name: 't', + label: 'T', + relevant: "selected(${m}, 'no')", + }, + ]); + expect(varXml(xml, 't')).toContain('Only if “Which?” includes No'); + }); + + test('outside the prose: no universe, the note still carries it', () => { + const xml = ddi([ + { type: 'text', name: 'a', label: 'A' }, + { + type: 'text', + name: 't', + label: 'T', + relevant: 'string-length(${a}) > 3', + }, + ]); + const v = varXml(xml, 't'); + expect(v).not.toContain(''); + }); + + test("a group's condition is ANDed into each member's", () => { + const xml = ddi([ + { type: 'select_one yn', name: 'dog', label: 'Dog?' }, + { + type: 'begin_group', + name: 'g', + label: 'G', + relevant: "${dog} = 'yes'", + }, + { type: 'integer', name: 'age', label: 'Age' }, + { type: 'text', name: 't', label: 'T', relevant: '${age} > 1' }, + { type: 'end_group' }, + ]); + expect(varXml(xml, 'age')).toContain( + `subject="xlsform-xpath">\${dog} = 'yes'`, + ); + expect(varXml(xml, 't')).toContain( + `subject="xlsform-xpath">(\${dog} = 'yes') and (\${age} > 1)`, + ); + }); + + test("a select_multiple's logic is on its varGrp, the prose also on each binary var", () => { + const xml = ddi([ + { type: 'integer', name: 'n', label: 'N' }, + { + type: 'select_multiple yn', + name: 'm', + label: 'Which?', + relevant: '${n} > 1', + required: 'yes', + }, + ]); + const grp = xml.slice( + xml.indexOf(''), + ); + expect(grp).toContain( + 'Only if “N” > 1', + ); + expect(grp).toContain('yes'); + expect(varXml(xml, 'm_yes')).toContain(' { + const xml = ddi([ + { type: 'select_one yn or_other', name: 'src', label: 'Source?' }, + ]); + expect(varXml(xml, 'src_other')).toContain( + `subject="xlsform-xpath">\${src} = 'other'`, + ); + }); +}); + +describe('multilingual prose', () => { + const rows = [ + { + type: 'select_one yn', + name: 'dog', + 'label::Deutsch (de)': 'Hund?', + 'label::English (en)': 'Dog?', + }, + { + type: 'text', + name: 't', + 'label::Deutsch (de)': 'Name?', + 'label::English (en)': 'Name?', + relevant: "${dog} = 'yes'", + }, + ]; + const choices = [ + { + list_name: 'yn', + name: 'yes', + 'label::Deutsch (de)': 'Ja', + 'label::English (en)': 'Yes', + }, + ]; + + test('one universe per language with a template, in its own labels', () => { + const xml = buildDdiXml(rows, choices, { + ...OPTS, + settings: { default_language: 'Deutsch (de)' }, + }); + const v = varXml(xml, 't'); + expect(v).toContain( + 'Nur wenn „Hund?“ = Ja', + ); + expect(v).toContain( + 'Only if “Dog?” = Yes', + ); + }); + + test('a language lacking a label it needs gets no universe', () => { + const xml = buildDdiXml( + rows, + [{ list_name: 'yn', name: 'yes', 'label::Deutsch (de)': 'Ja' }], + { ...OPTS, settings: { default_language: 'Deutsch (de)' } }, + ); + const v = varXml(xml, 't'); + expect(v).toContain('Nur wenn'); + expect(v).not.toContain('Only if'); + }); + + test('constraint_message: one note per language', () => { + const xml = buildDdiXml( + [ + { + type: 'integer', + name: 'n', + 'label::Deutsch (de)': 'Zahl', + 'label::English (en)': 'Number', + constraint: '. > 0', + 'constraint_message::Deutsch (de)': 'Positiv', + 'constraint_message::English (en)': 'Positive', + }, + ], + [], + { ...OPTS, settings: { default_language: 'Deutsch (de)' } }, + ); + const v = varXml(xml, 'n'); + expect(v).toContain('Positiv'); + expect(v).toContain( + 'Positive', + ); + }); +}); + +describe('constraint', () => { + test.each([ + ['. >= 1 and . <= 10', { min: '1', max: '10' }], + ['. > 0', { minExclusive: '0' }], + ['100 > .', { maxExclusive: '100' }], + ['. >= -5', { min: '-5' }], + ['. >= 1 and . >= 2', null], + ['. >= 1 or . <= 10', null], + ['. != 3', null], + ['string-length(.) < 5', null], + ])('simpleRange(%s)', (constraint, range) => { + expect(simpleRange(constraint)).toEqual(range); + }); + + test('a simple range on a number is also a valrng, before universe', () => { + const xml = ddi([ + { type: 'integer', name: 'k', label: 'K' }, + { + type: 'integer', + name: 'n', + label: 'N', + constraint: '. >= 1 and . <= 10', + relevant: '${k} > 0', + }, + ]); + const v = varXml(xml, 'n'); + expect(v).toContain('\n '); + expect(v.indexOf(' { + const xml = ddi([ + { type: 'text', name: 't', label: 'T', constraint: '. > 0' }, + ]); + const v = varXml(xml, 't'); + expect(v).not.toContain(' { + const xml = ddi([ + { type: 'text', name: 't', label: 'T', constraint_message: 'Oops' }, + ]); + expect(varXml(xml, 't')).not.toContain('cdl:constraint_message'); + }); +}); + +describe('lstsv → DDI logic', () => { + const header = [ + 'class', + 'type/scale', + 'name', + 'relevance', + 'text', + 'help', + 'language', + 'mandatory', + 'em_validation_q', + ].join('\t'); + const line = (...cells: string[]) => cells.join('\t'); + + test('relevance is reversed into XPath', () => { + const tsv = [ + header, + line('S', '', 'language', '1', 'en', '', '', '', ''), + line('G', '1', 'G', '1', '', '', 'en', '', ''), + line('Q', 'N', 'age', '1', 'Age?', '', 'en', 'Y', ''), + line('Q', 'S', 'job', 'age > 17', 'Job?', '', 'en', 'N', ''), + ].join('\n'); + const xml = lstsvToDdiXml(tsv, OPTS); + expect(varXml(xml, 'job')).toContain( + 'subject="xlsform-xpath">${age} > 17', + ); + expect(varXml(xml, 'age')).toContain( + 'yes', + ); + }); + + test('an expression outside the dialect: a warning, no note', () => { + const tsv = [ + header, + line('S', '', 'language', '1', 'en', '', '', '', ''), + line('G', '1', 'G', '1', '', '', 'en', '', ''), + line('Q', 'S', 'job', 'strtoupper(x) == "A"', 'Job?', '', 'en', 'N', ''), + ].join('\n'); + const warnings: string[] = []; + const xml = lstsvToDdiXml(tsv, { + ...OPTS, + onWarning: (w) => warnings.push(w.code), + }); + expect(varXml(xml, 'job')).not.toContain('cdl:relevant'); + expect(warnings).toEqual(['em-unsupported']); + }); +}); diff --git a/tests/validation/test_registry_schematron_conformance.py b/tests/validation/test_registry_schematron_conformance.py index 439d9eb..89f5626 100644 --- a/tests/validation/test_registry_schematron_conformance.py +++ b/tests/validation/test_registry_schematron_conformance.py @@ -247,7 +247,11 @@ def mutate(xml: str) -> str | None: "missing a concept element", ), ("type:select_one", _insert_after_first("", "x"), "uses labl"), - ("type:select_one", _insert_after_first("", "ab"), "multiple notes elements"), + ( + "type:select_one", + _insert_after_first("", "ab"), + "more than one untyped notes element", + ), ( "composite:grid", _sub1(r'(]*?)\s+name="[^"]+"', r"\1"), @@ -380,6 +384,24 @@ def test_several_concepts_are_allowed(worker_jar, java_bin, tmp_path): assert rc == 1 and any("missing a concept element" in m for m in messages), messages +def test_typed_notes_and_one_untyped_note_per_language(worker_jar, java_bin, tmp_path): + """Typed notes (convention:logicMapping, #151) are not limited, and one + untyped note (a citation) is allowed per language.""" + from .fixtures import load_registry + + variant = next(e for e in load_registry() if e.get("@id") == "type:select_one") + xml = load_example_ddi(variant) + notes = ( + '1 = 1' + 'yes' + "Quelle: X" + 'Source: X' + ) + ok = xml.replace("", notes + "", 1) + rc, out = _validate(java_bin, worker_jar, ok.encode(), tmp_path) + assert rc == 0, out[out.index("{") :][:800] + + def test_companion_of_a_name_that_ends_in_other(worker_jar, java_bin, tmp_path): """The base of `_other_other` is `_other` (the trailing suffix), not `` (the first one); the rule used to cut at the first (#86)."""