From 69a60849cf1093b8a3366625859627090eb78578 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Sun, 20 Sep 2026 23:11:19 -0700 Subject: [PATCH 1/5] fix(power-bi): fix SkuMeterName column error in storage reports for FOCUS 1.2 exports Co-Authored-By: Claude Sonnet 5 --- docs-mslearn/toolkit/changelog.md | 4 +++- .../storage/Shared.Dataset/definition/tables/Costs.tmdl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 6ca4dd7c3..18b927b83 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 09/11/2026 +ms.date: 09/20/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -58,6 +58,8 @@ The following section lists features and enhancements that are currently in deve - **Changed** - Switched the InstanceSizeFlexibility table in the storage and KQL shared datasets from the retired `ccmstorageprod` AutofitComboMeterData.csv to the FinOps toolkit [Instance size flexibility](open-data.md#instance-size-flexibility) open data file, joined to reservation recommendations on the unique ARM SKU name ([#2090](https://github.com/microsoft/finops-toolkit/issues/2090)). +- **Fixed** + - Fixed the storage reports failing to refresh with "The column 'SkuMeterName' of the table wasn't found" when the Cost Management export uses the FOCUS 1.2 schema ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)). ### [Optimization Engine](optimization-engine/overview.md) diff --git a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl index 75880048f..04dadea32 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl @@ -1469,7 +1469,7 @@ table Costs then Table.ReplaceValue(InvoiceId, null, each [x_PricingCurrency], Replacer.ReplaceValue, {"PricingCurrency"}) else Table.AddColumn( InvoiceId, "PricingCurrency", each [x_PricingCurrency]), SkuMeter = if Has12 - then Table.ReplaceValue(PricingCurrency, null, each [x_SkuMeterName], Replacer.ReplaceValue, {"SkuMeterName"}) + then Table.ReplaceValue(PricingCurrency, null, each [x_SkuMeterName], Replacer.ReplaceValue, {"SkuMeter"}) else Table.AddColumn( PricingCurrency, "SkuMeter", each [x_SkuMeterName]), // Copy 1.2 values to 1.0 columns Align12 = Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(SkuMeter, From add6080bc7a58f4a473a18333035771d1de1fff4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Sep 2026 06:11:54 +0000 Subject: [PATCH 2/5] chore: Update ms.date in docs-mslearn files --- docs-mslearn/toolkit/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 18b927b83..ff77643c7 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 09/20/2026 +ms.date: 09/21/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit From 42eea93f758af2bb112f3c9354536c173979e5bb Mon Sep 17 00:00:00 2001 From: Roland Krummenacher Date: Mon, 21 Sep 2026 08:49:37 +0200 Subject: [PATCH 3/5] test(power-bi): add a lint rule for Power Query column names Nothing in CI runs Power Query, so a mistyped column name in a .tmdl partition ships and surfaces only as a refresh failure on a customer's report. That is how issue #2332 happened: the FOCUS 1.2 branch of the Costs query searched {"SkuMeterName"}, a column no FOCUS version emits. Two static rules: - Every columnsToSearch literal in Table.ReplaceValue must name a column that exists - from a Cost Management export schema, created by the query itself, or declared on the model table. - References to columns FOCUS 1.2 drops (x_InvoiceId, x_PricingCurrency, x_SkuMeterName) need a Table.HasColumns or _exists guard, because ftk_Storage does no backfill for focuscost. The dropped set is derived from the export schemas rather than hard-coded, so a future FOCUS version updates the rule on its own. Scoped to focuscost partitions: the pricesheet and reservationtransactions datasets reuse some of these names legitimately. Rule 2 baselines the three existing unguarded references in Costs.tmdl as a ratchet. Verified the rule catches the original typo: reverting the one-line fix fails rule 1 with "got 'SkuMeterName'". Co-Authored-By: Claude Opus 5 --- docs-mslearn/toolkit/changelog.md | 2 +- .../Lint/PowerQueryColumnNames.Tests.ps1 | 169 ++++++++++++++++++ 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index ff77643c7..8e7efbf35 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -59,7 +59,7 @@ The following section lists features and enhancements that are currently in deve - **Changed** - Switched the InstanceSizeFlexibility table in the storage and KQL shared datasets from the retired `ccmstorageprod` AutofitComboMeterData.csv to the FinOps toolkit [Instance size flexibility](open-data.md#instance-size-flexibility) open data file, joined to reservation recommendations on the unique ARM SKU name ([#2090](https://github.com/microsoft/finops-toolkit/issues/2090)). - **Fixed** - - Fixed the storage reports failing to refresh with "The column 'SkuMeterName' of the table wasn't found" when the Cost Management export uses the FOCUS 1.2 schema ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)). + - Fixed the storage reports failing to refresh with "The column 'SkuMeterName' of the table wasn't found" when the Cost Management export uses the FOCUS 1.2 schema, and added a lint rule that fails the build when a Power Query column name doesn't exist ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)). ### [Optimization Engine](optimization-engine/overview.md) diff --git a/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 b/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 new file mode 100644 index 000000000..fdba8f0b0 --- /dev/null +++ b/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 @@ -0,0 +1,169 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + Lint rule: Power Query column names in Power BI datasets must refer to columns that exist. + + Nothing in CI runs Power Query, so a mistyped column name in a .tmdl partition ships and only + surfaces as a refresh failure on a customer's report. That is exactly what happened with + issue #2332: the FOCUS 1.2 branch of the Costs query passed {"SkuMeterName"} as the + columnsToSearch of Table.ReplaceValue, but the column FOCUS 1.2 emits is SkuMeter. The typo + was introduced in fa19ad68 and survived because the 1.2 branch was never exercised. + + Two rules, both static: + + 1. Known columns - every columnsToSearch literal in Table.ReplaceValue must be a column that + exists: one of the Cost Management export schemas, one the query creates itself via + Table.AddColumn / Table.RenameColumns, or a column declared on the model table. + + 2. Version-dropped columns - FOCUS 1.2 removes three columns that earlier versions carry + (x_InvoiceId and x_PricingCurrency were promoted to InvoiceId / PricingCurrency, + x_SkuMeterName was renamed to SkuMeter). ftk_Storage does no backfill for focuscost + (CleanColumns passes ExtractColumns straight through), so the columns available are exactly + the union of what the export files contain. Referencing a dropped column without a + Table.HasColumns / _exists guard fails on a container that holds only 1.2 exports. + + Verified 2026-09-21 against a real FocusCost export (dataVersion 1.2-preview, 105 columns): + InvoiceId, PricingCurrency and SkuMeter are present; x_InvoiceId, x_PricingCurrency and + x_SkuMeterName are absent. Note that docs-mslearn/focus/metadata.md is not a reliable + source for this - it listed x_SkuMeterName under 1.2-preview until this change corrected it. + The ADF mappings under Exports/schemas are authoritative and are what this rule reads. + + Rule 2 is baselined per file as a ratchet: + - Guarding a reference REQUIRES lowering the count here (the test fails on stale entries). + - Adding a new unguarded reference is never allowed. +#> + +Describe 'PowerQueryColumnNames' { + + BeforeDiscovery { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + + # Power Query lives in the `source = ```...``` ` partition blocks of the dataset .tmdl files. + $partitionPattern = [regex]'(?s)source = \x60{3}(.*?)\x60{3}' + + $scanFiles = @( + Get-ChildItem -Path (Join-Path $repoRoot 'src/power-bi') -Filter '*.tmdl' -Recurse -File -ErrorAction SilentlyContinue | + Where-Object { $partitionPattern.IsMatch((Get-Content -Path $_.FullName -Raw)) } | + Sort-Object FullName | + ForEach-Object { + @{ FullName = $_.FullName; RelPath = $_.FullName.Substring($repoRoot.Length + 1).Replace('\', '/') } + } + ) + } + + BeforeAll { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + $schemaDir = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas' + + # Every column name any Cost Management export can deliver, across all dataset types and + # FOCUS versions. Both sides of the mapping count: `source` is the export column, `sink` is + # the name the hub stores it under, and queries legitimately reference either. + function Get-SchemaColumn([string] $path) + { + $mappings = (Get-Content -Path $path -Raw | ConvertFrom-Json).translator.mappings + return @($mappings | ForEach-Object { $_.source.name; $_.sink.name } | Where-Object { $_ } | Sort-Object -Unique) + } + + $schemaFiles = @(Get-ChildItem -Path $schemaDir -Filter '*.json' -File) + $knownColumns = [System.Collections.Generic.HashSet[string]]::new([string[]]@($schemaFiles | ForEach-Object { Get-SchemaColumn $_.FullName }), [StringComparer]::Ordinal) + + # Columns an earlier FOCUS version carries that 1.2 does not. Derived from the schemas + # rather than hard-coded so a future FOCUS version updates the rule automatically. + $focus10 = Get-SchemaColumn (Join-Path $schemaDir 'focuscost_1.0r2.json') + $focus12 = @( + (Get-SchemaColumn (Join-Path $schemaDir 'focuscost_1.2-preview.json')) + (Get-SchemaColumn (Join-Path $schemaDir 'focuscost_1.2.json')) + ) | Sort-Object -Unique + $droppedInFocus12 = @($focus10 | Where-Object { $_ -notin $focus12 }) + + $partitionPattern = [regex]'(?s)source = \x60{3}(.*?)\x60{3}' + # Last argument of Table.ReplaceValue: the columns the replacement searches. + $columnsToSearchPattern = [regex]'Replacer\.\w+\s*,\s*\{([^}]*)\}' + $quotedNamePattern = [regex]'"([A-Za-z_][A-Za-z0-9_]*)"' + $addColumnPattern = [regex]'Table\.AddColumn\(\s*[^,]+,\s*"([^"]+)"' + $renamePairPattern = [regex]'\{\s*"([^"]+)"\s*,\s*"([^"]+)"\s*\}' + $modelColumnPattern = [regex]'(?m)^\s*sourceColumn:\s*(\S+)' + + # Unguarded references to columns FOCUS 1.2 drops, counted per repo-relative path. + # Ratchet only: lower on fix, never raise. + # + # Costs.tmdl: the "Handle columns renamed in FOCUS 1.2 gracefully" block reads + # [x_InvoiceId], [x_PricingCurrency] and [x_SkuMeterName], and Align12 passes all three as + # columnsToSearch. On a container holding only 1.2 exports none of them exist. Guard the + # block with the _exists / _swapCol helpers already defined at the top of the query, then + # lower this entry. Tracked on issue #2332 / PR #2333. + $baseline = @{ + 'src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl' = 3 + } + } + + It 'Should find the export schemas and the dataset partitions' { + $knownColumns.Count | Should -BeGreaterThan 100 -Because 'the Cost Management export schemas define the column vocabulary this rule checks against' + $droppedInFocus12.Count | Should -BeGreaterThan 0 -Because 'FOCUS 1.2 renames or promotes columns that earlier versions carry; if this is empty the schema files moved or changed shape' + } + + It 'Should only search columns that exist: ' -ForEach $scanFiles { + $content = Get-Content -Path $FullName -Raw + $partitions = @($partitionPattern.Matches($content) | ForEach-Object { $_.Groups[1].Value }) + + # Columns the query builds for itself are just as valid as exported ones. + $created = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($partition in $partitions) + { + foreach ($match in $addColumnPattern.Matches($partition)) { [void] $created.Add($match.Groups[1].Value) } + foreach ($match in $renamePairPattern.Matches($partition)) { [void] $created.Add($match.Groups[2].Value) } + } + foreach ($match in $modelColumnPattern.Matches($content)) { [void] $created.Add($match.Groups[1].Value) } + + $searched = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($partition in $partitions) + { + foreach ($match in $columnsToSearchPattern.Matches($partition)) + { + foreach ($name in $quotedNamePattern.Matches($match.Groups[1].Value)) { [void] $searched.Add($name.Groups[1].Value) } + } + } + + $unknown = @($searched | Where-Object { -not $knownColumns.Contains($_) -and -not $created.Contains($_) } | Sort-Object) + + $unknown -join ', ' | Should -BeNullOrEmpty -Because 'Table.ReplaceValue fails with "The column '''' of the table wasn''t found" when columnsToSearch names a column that is not in the table. Every name must be an export column, a column the query adds itself, or a column on the model table (see issue #2332).' + } + + It 'Should not reference columns FOCUS 1.2 drops without a guard: ' -ForEach $scanFiles { + $content = Get-Content -Path $FullName -Raw + + # FOCUS versioning only applies to the focuscost dataset. The other datasets have their own + # schemas that happen to share column names: Prices reads "pricesheet" and renames MeterName + # to x_SkuMeterName, ReservationTransactions reads "reservationtransactions". Those uses are + # correct and must not be flagged. + $partitions = @( + $partitionPattern.Matches($content) | + ForEach-Object { $_.Groups[1].Value } | + Where-Object { $_ -match 'ftk_Storage\(\s*"focuscost"' } + ) + $allowed = if ($baseline.ContainsKey($RelPath)) { $baseline[$RelPath] } else { 0 } + + $unguarded = @( + foreach ($column in $droppedInFocus12) + { + $referenced = $false + $guarded = $false + foreach ($partition in $partitions) + { + if ($partition -match ('\[{0}\]|"{0}"' -f [regex]::Escape($column))) { $referenced = $true } + if ($partition -match ('(Table\.HasColumns|_exists)\([^)]*"{0}"' -f [regex]::Escape($column))) { $guarded = $true } + } + if ($referenced -and -not $guarded) { $column } + } + ) + + $unguarded.Count | Should -BeLessOrEqual $allowed -Because ("FOCUS 1.2 does not deliver these columns and ftk_Storage does not backfill them, so a container holding only 1.2 exports fails to refresh. Guard the reference with Table.HasColumns or the _exists helper. Unguarded here: $($unguarded -join ', ')") + + if ($unguarded.Count -le $allowed) + { + # Ratchet: if a reference was guarded, the baseline must be lowered so it cannot return. + $unguarded.Count | Should -Be $allowed -Because ("the unguarded reference count in this file dropped below the baseline ($allowed); lower the baseline entry for '$RelPath' in PowerQueryColumnNames.Tests.ps1 to $($unguarded.Count) (or remove it if 0) so the fix is locked in.") + } + } +} From 9a22d267aaa9c2a2310c263dcd667265563a49f2 Mon Sep 17 00:00:00 2001 From: Roland Krummenacher Date: Mon, 21 Sep 2026 08:49:38 +0200 Subject: [PATCH 4/5] docs(focus): drop x_SkuMeterName from the FOCUS 1.2 column list The FocusCost 1.2-preview table listed x_SkuMeterName alongside the SkuMeter column that replaced it. Verified against a real FocusCost export (dataVersion 1.2-preview, 105 columns): x_SkuMeterName is not delivered. The 1.0 and 1.0-preview(v1) tables keep the column. The ADF mappings in Exports/schemas already had this right. Co-Authored-By: Claude Opus 5 --- docs-mslearn/focus/metadata.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs-mslearn/focus/metadata.md b/docs-mslearn/focus/metadata.md index 93160875e..9a4d68d94 100644 --- a/docs-mslearn/focus/metadata.md +++ b/docs-mslearn/focus/metadata.md @@ -3,7 +3,7 @@ title: FOCUS metadata description: This article provides general information about the FOCUS dataset including the data generator, schema version, and columns included in the dataset. author: flanakin ms.author: micflan -ms.date: 04/01/2026 +ms.date: 09/21/2026 ms.topic: reference ms.service: finops ms.subservice: finops-learning-resources @@ -128,7 +128,6 @@ Columns include: | `x_SkuIsCreditEligible` | Boolean | Indicates if the charge is eligible for Azure credits. | | `x_SkuMeterCategory` | String | Name of the service the SKU falls within. | | `x_SkuMeterId` | String | Unique identifier (sometimes a GUID, but not always) for the usage meter. It usually maps to a specific SKU or range of SKUs that have a specific price. | -| `x_SkuMeterName` | String | Name of the usage meter. It usually maps to a specific SKU or range of SKUs that have a specific price. Not applicable for purchases. | | `x_SkuMeterSubcategory` | String | Group of SKU Classes that address the same core need within the SKU Group. | | `x_SkuOfferId` | String | Microsoft Cloud subscription type. | | `x_SkuOrderId` | String | Unique identifier of the entitlement product for this charge. Same as MCA ProductOrderId. Not applicable for EA. | From e7e11e01437201d9d9db5a464b490138115c078e Mon Sep 17 00:00:00 2001 From: Roland Krummenacher Date: Mon, 21 Sep 2026 09:06:35 +0200 Subject: [PATCH 5/5] fix(power-bi): handle FOCUS 1.2 exports that carry no 1.0 columns The "Handle columns renamed in FOCUS 1.2 gracefully" block read [x_InvoiceId], [x_PricingCurrency] and [x_SkuMeterName] unconditionally, and Align12 passed all three as columnsToSearch. FOCUS 1.2 does not deliver them - verified against a real export (dataVersion 1.2-preview, 105 columns): InvoiceId, PricingCurrency and SkuMeter are present, the three x_ columns are not. ftk_Storage does no backfill for focuscost, so a container holding only 1.2 exports has none of them and the refresh fails with "The column 'x_InvoiceId' of the table wasn't found" - the same class of error as #2332, just one step later. EnsureRenamedCols now adds whichever side is missing as a null column before Align12 copies values in both directions, so 1.0-only, 1.2-only and mixed containers all work. Has12 is gone: the per-column checks replace it and it had no other caller. Lowers the PowerQueryColumnNames rule 2 baseline to zero. Co-Authored-By: Claude Opus 5 --- docs-mslearn/toolkit/changelog.md | 2 +- .../definition/tables/Costs.tmdl | 31 ++++++++++++------- .../Lint/PowerQueryColumnNames.Tests.ps1 | 14 +++------ 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 8e7efbf35..364ec3ac4 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -59,7 +59,7 @@ The following section lists features and enhancements that are currently in deve - **Changed** - Switched the InstanceSizeFlexibility table in the storage and KQL shared datasets from the retired `ccmstorageprod` AutofitComboMeterData.csv to the FinOps toolkit [Instance size flexibility](open-data.md#instance-size-flexibility) open data file, joined to reservation recommendations on the unique ARM SKU name ([#2090](https://github.com/microsoft/finops-toolkit/issues/2090)). - **Fixed** - - Fixed the storage reports failing to refresh with "The column 'SkuMeterName' of the table wasn't found" when the Cost Management export uses the FOCUS 1.2 schema, and added a lint rule that fails the build when a Power Query column name doesn't exist ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)). + - Fixed the storage reports failing to refresh when the Cost Management export uses the FOCUS 1.2 schema. The FOCUS 1.2 branch searched for a column named `SkuMeterName` instead of `SkuMeter`, and the surrounding block assumed the FOCUS 1.0 columns (`x_InvoiceId`, `x_PricingCurrency`, `x_SkuMeterName`) were always present, which fails for a storage container that holds only FOCUS 1.2 exports. Added a lint rule that fails the build when a Power Query column name doesn't exist ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)). ### [Optimization Engine](optimization-engine/overview.md) diff --git a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl index 04dadea32..27ce027af 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl @@ -1431,7 +1431,6 @@ table Costs PerformDeprecatedOptimizations = #"Deprecated: Perform Extra Query Optimizations", HasPreview = Table.HasColumns(RawData, {"ChargeSubcategory"}), Has10 = Table.HasColumns(RawData, {"ChargeClass"}), - Has12 = Table.HasColumns(RawData, {"InvoiceId"}), HandlePeview = if HasPreview = false then RawData else if PerformDeprecatedOptimizations = false then error "LegacyFocusVersion: Please enable the 'Perform Extra Query Optimizations' parameter" else ( @@ -1462,17 +1461,25 @@ table Costs RemovePreviewCols ), // Handle columns renamed in FOCUS 1.2 gracefully -- Copy values to/from 1.0 and 1.2 columns - InvoiceId = if Has12 - then Table.ReplaceValue(HandlePeview, null, each [x_InvoiceId], Replacer.ReplaceValue, {"InvoiceId"}) - else Table.AddColumn( HandlePeview, "InvoiceId", each [x_InvoiceId]), - PricingCurrency = if Has12 - then Table.ReplaceValue(InvoiceId, null, each [x_PricingCurrency], Replacer.ReplaceValue, {"PricingCurrency"}) - else Table.AddColumn( InvoiceId, "PricingCurrency", each [x_PricingCurrency]), - SkuMeter = if Has12 - then Table.ReplaceValue(PricingCurrency, null, each [x_SkuMeterName], Replacer.ReplaceValue, {"SkuMeter"}) - else Table.AddColumn( PricingCurrency, "SkuMeter", each [x_SkuMeterName]), - // Copy 1.2 values to 1.0 columns - Align12 = Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(SkuMeter, + // FOCUS 1.2 promotes x_InvoiceId/x_PricingCurrency to InvoiceId/PricingCurrency and renames + // x_SkuMeterName to SkuMeter. ftk_Storage does not backfill focuscost columns, so a container + // holding only 1.2 exports has none of the 1.0 columns (and a 1.0-only container none of the + // 1.2 ones). Add whichever side is missing as null first, then copy values in both directions. + EnsureRenamedCols = + let + a = if Table.HasColumns(HandlePeview, {"x_InvoiceId"}) then HandlePeview else Table.AddColumn(HandlePeview, "x_InvoiceId", each null), + b = if Table.HasColumns(a, {"x_PricingCurrency"}) then a else Table.AddColumn(a, "x_PricingCurrency", each null), + c = if Table.HasColumns(b, {"x_SkuMeterName"}) then b else Table.AddColumn(b, "x_SkuMeterName", each null), + d = if Table.HasColumns(c, {"InvoiceId"}) then c else Table.AddColumn(c, "InvoiceId", each null), + e = if Table.HasColumns(d, {"PricingCurrency"}) then d else Table.AddColumn(d, "PricingCurrency", each null), + f = if Table.HasColumns(e, {"SkuMeter"}) then e else Table.AddColumn(e, "SkuMeter", each null) + in + f, + // Copy 1.0 values to the 1.2 columns, then 1.2 values back to the 1.0 columns + Align12 = Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(EnsureRenamedCols, + null, each [x_InvoiceId], Replacer.ReplaceValue, {"InvoiceId"}), + null, each [x_PricingCurrency], Replacer.ReplaceValue, {"PricingCurrency"}), + null, each [x_SkuMeterName], Replacer.ReplaceValue, {"SkuMeter"}), null, each [InvoiceId], Replacer.ReplaceValue, {"x_InvoiceId"}), null, each [PricingCurrency], Replacer.ReplaceValue, {"x_PricingCurrency"}), null, each [SkuMeter], Replacer.ReplaceValue, {"x_SkuMeterName"}), diff --git a/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 b/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 index fdba8f0b0..fc3bf0408 100644 --- a/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 +++ b/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 @@ -86,16 +86,10 @@ Describe 'PowerQueryColumnNames' { $modelColumnPattern = [regex]'(?m)^\s*sourceColumn:\s*(\S+)' # Unguarded references to columns FOCUS 1.2 drops, counted per repo-relative path. - # Ratchet only: lower on fix, never raise. - # - # Costs.tmdl: the "Handle columns renamed in FOCUS 1.2 gracefully" block reads - # [x_InvoiceId], [x_PricingCurrency] and [x_SkuMeterName], and Align12 passes all three as - # columnsToSearch. On a container holding only 1.2 exports none of them exist. Guard the - # block with the _exists / _swapCol helpers already defined at the top of the query, then - # lower this entry. Tracked on issue #2332 / PR #2333. - $baseline = @{ - 'src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl' = 3 - } + # Ratchet only: lower on fix, never raise. Empty today - Costs.tmdl guards all three with + # Table.HasColumns in its EnsureRenamedCols step. Add an entry only for a reference that + # genuinely cannot be guarded, and say why. + $baseline = @{} } It 'Should find the export schemas and the dataset partitions' {