[Power BI] Fix SkuMeterName column error in storage reports for FOCUS 1.2 exports

docs-mslearn/focus/metadata.md

@@ -3,7 +3,7 @@ title: FOCUS metadata
 description: This article provides general information about the FOCUS dataset including the data generator, schema version, and columns included in the dataset.
 author: flanakin
 ms.author: micflan
-ms.date: 04/01/2026
+ms.date: 09/21/2026
 ms.topic: reference
 ms.service: finops
 ms.subservice: finops-learning-resources
@@ -128,7 +128,6 @@ Columns include:
 | `x_SkuIsCreditEligible`      | Boolean  | Indicates if the charge is eligible for Azure credits.                                                                                                                                                                                         |
 | `x_SkuMeterCategory`         | String   | Name of the service the SKU falls within.                                                                                                                                                                                                      |
 | `x_SkuMeterId`               | String   | Unique identifier (sometimes a GUID, but not always) for the usage meter. It usually maps to a specific SKU or range of SKUs that have a specific price.                                                                                       |
-| `x_SkuMeterName`             | String   | Name of the usage meter. It usually maps to a specific SKU or range of SKUs that have a specific price. Not applicable for purchases.                                                                                                          |
 | `x_SkuMeterSubcategory`      | String   | Group of SKU Classes that address the same core need within the SKU Group.                                                                                                                                                                     |
 | `x_SkuOfferId`               | String   | Microsoft Cloud subscription type.                                                                                                                                                                                                             |
 | `x_SkuOrderId`               | String   | Unique identifier of the entitlement product for this charge. Same as MCA ProductOrderId. Not applicable for EA.                                                                                                                               |

docs-mslearn/toolkit/changelog.md

@@ -3,7 +3,7 @@ title: FinOps toolkit changelog
 description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more.
 author: MSBrett
 ms.author: brettwil
-ms.date: 09/11/2026
+ms.date: 09/21/2026
 ms.topic: reference
 ms.service: finops
 ms.subservice: finops-toolkit
@@ -58,6 +58,8 @@ The following section lists features and enhancements that are currently in deve
 
 - **Changed**
   - Switched the InstanceSizeFlexibility table in the storage and KQL shared datasets from the retired `ccmstorageprod` AutofitComboMeterData.csv to the FinOps toolkit [Instance size flexibility](open-data.md#instance-size-flexibility) open data file, joined to reservation recommendations on the unique ARM SKU name ([#2090](https://github.com/microsoft/finops-toolkit/issues/2090)).
+- **Fixed**
+  - Fixed the storage reports failing to refresh when the Cost Management export uses the FOCUS 1.2 schema. The FOCUS 1.2 branch searched for a column named `SkuMeterName` instead of `SkuMeter`, and the surrounding block assumed the FOCUS 1.0 columns (`x_InvoiceId`, `x_PricingCurrency`, `x_SkuMeterName`) were always present, which fails for a storage container that holds only FOCUS 1.2 exports. Added a lint rule that fails the build when a Power Query column name doesn't exist ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)).
 
 ### [Optimization Engine](optimization-engine/overview.md)
 

src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl

@@ -1431,7 +1431,6 @@ table Costs
 					PerformDeprecatedOptimizations = #"Deprecated: Perform Extra Query Optimizations",
 					HasPreview = Table.HasColumns(RawData, {"ChargeSubcategory"}),
 					Has10      = Table.HasColumns(RawData, {"ChargeClass"}),
-					Has12      = Table.HasColumns(RawData, {"InvoiceId"}),
 					HandlePeview = if HasPreview = false then RawData
 						else if PerformDeprecatedOptimizations = false then error "LegacyFocusVersion: Please enable the 'Perform Extra Query Optimizations' parameter"
 						else (
@@ -1462,17 +1461,25 @@ table Costs
 								RemovePreviewCols
 						),
 					// Handle columns renamed in FOCUS 1.2 gracefully -- Copy values to/from 1.0 and 1.2 columns
-					InvoiceId       = if Has12
-						then Table.ReplaceValue(HandlePeview, null,        each [x_InvoiceId], Replacer.ReplaceValue, {"InvoiceId"})
-						else Table.AddColumn(   HandlePeview, "InvoiceId", each [x_InvoiceId]),
-					PricingCurrency = if Has12
-						then Table.ReplaceValue(InvoiceId, null,              each [x_PricingCurrency], Replacer.ReplaceValue, {"PricingCurrency"})
-						else Table.AddColumn(   InvoiceId, "PricingCurrency", each [x_PricingCurrency]),
-					SkuMeter        = if Has12
-						then Table.ReplaceValue(PricingCurrency, null,       each [x_SkuMeterName], Replacer.ReplaceValue, {"SkuMeterName"})
-						else Table.AddColumn(   PricingCurrency, "SkuMeter", each [x_SkuMeterName]),
-					// Copy 1.2 values to 1.0 columns
-					Align12 = Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(SkuMeter,
+					// FOCUS 1.2 promotes x_InvoiceId/x_PricingCurrency to InvoiceId/PricingCurrency and renames
+					// x_SkuMeterName to SkuMeter. ftk_Storage does not backfill focuscost columns, so a container
+					// holding only 1.2 exports has none of the 1.0 columns (and a 1.0-only container none of the
+					// 1.2 ones). Add whichever side is missing as null first, then copy values in both directions.
+					EnsureRenamedCols =
+						let
+							a = if Table.HasColumns(HandlePeview, {"x_InvoiceId"})       then HandlePeview else Table.AddColumn(HandlePeview, "x_InvoiceId",       each null),
+							b = if Table.HasColumns(a,            {"x_PricingCurrency"}) then a            else Table.AddColumn(a,            "x_PricingCurrency", each null),
+							c = if Table.HasColumns(b,            {"x_SkuMeterName"})    then b            else Table.AddColumn(b,            "x_SkuMeterName",    each null),
+							d = if Table.HasColumns(c,            {"InvoiceId"})         then c            else Table.AddColumn(c,            "InvoiceId",         each null),
+							e = if Table.HasColumns(d,            {"PricingCurrency"})   then d            else Table.AddColumn(d,            "PricingCurrency",   each null),
+							f = if Table.HasColumns(e,            {"SkuMeter"})          then e            else Table.AddColumn(e,            "SkuMeter",          each null)
+						in
+							f,
+					// Copy 1.0 values to the 1.2 columns, then 1.2 values back to the 1.0 columns
+					Align12 = Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(EnsureRenamedCols,
+						null, each [x_InvoiceId],       Replacer.ReplaceValue, {"InvoiceId"}),
+						null, each [x_PricingCurrency], Replacer.ReplaceValue, {"PricingCurrency"}),
+						null, each [x_SkuMeterName],    Replacer.ReplaceValue, {"SkuMeter"}),
 						null, each [InvoiceId],         Replacer.ReplaceValue, {"x_InvoiceId"}),
 						null, each [PricingCurrency],   Replacer.ReplaceValue, {"x_PricingCurrency"}),
 						null, each [SkuMeter],          Replacer.ReplaceValue, {"x_SkuMeterName"}),

src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1

@@ -0,0 +1,163 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+<#
+    Lint rule: Power Query column names in Power BI datasets must refer to columns that exist.
+
+    Nothing in CI runs Power Query, so a mistyped column name in a .tmdl partition ships and only
+    surfaces as a refresh failure on a customer's report. That is exactly what happened with
+    issue #2332: the FOCUS 1.2 branch of the Costs query passed {"SkuMeterName"} as the
+    columnsToSearch of Table.ReplaceValue, but the column FOCUS 1.2 emits is SkuMeter. The typo
+    was introduced in fa19ad68 and survived because the 1.2 branch was never exercised.
+
+    Two rules, both static:
+
+    1. Known columns - every columnsToSearch literal in Table.ReplaceValue must be a column that
+       exists: one of the Cost Management export schemas, one the query creates itself via
+       Table.AddColumn / Table.RenameColumns, or a column declared on the model table.
+
+    2. Version-dropped columns - FOCUS 1.2 removes three columns that earlier versions carry
+       (x_InvoiceId and x_PricingCurrency were promoted to InvoiceId / PricingCurrency,
+       x_SkuMeterName was renamed to SkuMeter). ftk_Storage does no backfill for focuscost
+       (CleanColumns passes ExtractColumns straight through), so the columns available are exactly
+       the union of what the export files contain. Referencing a dropped column without a
+       Table.HasColumns / _exists guard fails on a container that holds only 1.2 exports.
+
+       Verified 2026-09-21 against a real FocusCost export (dataVersion 1.2-preview, 105 columns):
+       InvoiceId, PricingCurrency and SkuMeter are present; x_InvoiceId, x_PricingCurrency and
+       x_SkuMeterName are absent. Note that docs-mslearn/focus/metadata.md is not a reliable
+       source for this - it listed x_SkuMeterName under 1.2-preview until this change corrected it.
+       The ADF mappings under Exports/schemas are authoritative and are what this rule reads.
+
+    Rule 2 is baselined per file as a ratchet:
+    - Guarding a reference REQUIRES lowering the count here (the test fails on stale entries).
+    - Adding a new unguarded reference is never allowed.
+#>
+
+Describe 'PowerQueryColumnNames' {
+
+    BeforeDiscovery {
+        $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path
+
+        # Power Query lives in the `source = ```...``` ` partition blocks of the dataset .tmdl files.
+        $partitionPattern = [regex]'(?s)source = \x60{3}(.*?)\x60{3}'
+
+        $scanFiles = @(
+            Get-ChildItem -Path (Join-Path $repoRoot 'src/power-bi') -Filter '*.tmdl' -Recurse -File -ErrorAction SilentlyContinue |
+                Where-Object { $partitionPattern.IsMatch((Get-Content -Path $_.FullName -Raw)) } |
+                Sort-Object FullName |
+                ForEach-Object {
+                    @{ FullName = $_.FullName; RelPath = $_.FullName.Substring($repoRoot.Length + 1).Replace('\', '/') }
+                }
+        )
+    }
+
+    BeforeAll {
+        $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path
+        $schemaDir = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas'
+
+        # Every column name any Cost Management export can deliver, across all dataset types and
+        # FOCUS versions. Both sides of the mapping count: `source` is the export column, `sink` is
+        # the name the hub stores it under, and queries legitimately reference either.
+        function Get-SchemaColumn([string] $path)
+        {
+            $mappings = (Get-Content -Path $path -Raw | ConvertFrom-Json).translator.mappings
+            return @($mappings | ForEach-Object { $_.source.name; $_.sink.name } | Where-Object { $_ } | Sort-Object -Unique)
+        }
+
+        $schemaFiles = @(Get-ChildItem -Path $schemaDir -Filter '*.json' -File)
+        $knownColumns = [System.Collections.Generic.HashSet[string]]::new([string[]]@($schemaFiles | ForEach-Object { Get-SchemaColumn $_.FullName }), [StringComparer]::Ordinal)
+
+        # Columns an earlier FOCUS version carries that 1.2 does not. Derived from the schemas
+        # rather than hard-coded so a future FOCUS version updates the rule automatically.
+        $focus10 = Get-SchemaColumn (Join-Path $schemaDir 'focuscost_1.0r2.json')
+        $focus12 = @(
+            (Get-SchemaColumn (Join-Path $schemaDir 'focuscost_1.2-preview.json'))
+            (Get-SchemaColumn (Join-Path $schemaDir 'focuscost_1.2.json'))
+        ) | Sort-Object -Unique
+        $droppedInFocus12 = @($focus10 | Where-Object { $_ -notin $focus12 })
+
+        $partitionPattern = [regex]'(?s)source = \x60{3}(.*?)\x60{3}'
+        # Last argument of Table.ReplaceValue: the columns the replacement searches.
+        $columnsToSearchPattern = [regex]'Replacer\.\w+\s*,\s*\{([^}]*)\}'
+        $quotedNamePattern = [regex]'"([A-Za-z_][A-Za-z0-9_]*)"'
+        $addColumnPattern = [regex]'Table\.AddColumn\(\s*[^,]+,\s*"([^"]+)"'
+        $renamePairPattern = [regex]'\{\s*"([^"]+)"\s*,\s*"([^"]+)"\s*\}'
+        $modelColumnPattern = [regex]'(?m)^\s*sourceColumn:\s*(\S+)'
+
+        # Unguarded references to columns FOCUS 1.2 drops, counted per repo-relative path.
+        # Ratchet only: lower on fix, never raise. Empty today - Costs.tmdl guards all three with
+        # Table.HasColumns in its EnsureRenamedCols step. Add an entry only for a reference that
+        # genuinely cannot be guarded, and say why.
+        $baseline = @{}
+    }
+
+    It 'Should find the export schemas and the dataset partitions' {
+        $knownColumns.Count | Should -BeGreaterThan 100 -Because 'the Cost Management export schemas define the column vocabulary this rule checks against'
+        $droppedInFocus12.Count | Should -BeGreaterThan 0 -Because 'FOCUS 1.2 renames or promotes columns that earlier versions carry; if this is empty the schema files moved or changed shape'
+    }
+
+    It 'Should only search columns that exist: <RelPath>' -ForEach $scanFiles {
+        $content = Get-Content -Path $FullName -Raw
+        $partitions = @($partitionPattern.Matches($content) | ForEach-Object { $_.Groups[1].Value })
+
+        # Columns the query builds for itself are just as valid as exported ones.
+        $created = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
+        foreach ($partition in $partitions)
+        {
+            foreach ($match in $addColumnPattern.Matches($partition)) { [void] $created.Add($match.Groups[1].Value) }
+            foreach ($match in $renamePairPattern.Matches($partition)) { [void] $created.Add($match.Groups[2].Value) }
+        }
+        foreach ($match in $modelColumnPattern.Matches($content)) { [void] $created.Add($match.Groups[1].Value) }
+
+        $searched = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
+        foreach ($partition in $partitions)
+        {
+            foreach ($match in $columnsToSearchPattern.Matches($partition))
+            {
+                foreach ($name in $quotedNamePattern.Matches($match.Groups[1].Value)) { [void] $searched.Add($name.Groups[1].Value) }
+            }
+        }
+
+        $unknown = @($searched | Where-Object { -not $knownColumns.Contains($_) -and -not $created.Contains($_) } | Sort-Object)
+
+        $unknown -join ', ' | Should -BeNullOrEmpty -Because 'Table.ReplaceValue fails with "The column ''<name>'' of the table wasn''t found" when columnsToSearch names a column that is not in the table. Every name must be an export column, a column the query adds itself, or a column on the model table (see issue #2332).'
+    }
+
+    It 'Should not reference columns FOCUS 1.2 drops without a guard: <RelPath>' -ForEach $scanFiles {
+        $content = Get-Content -Path $FullName -Raw
+
+        # FOCUS versioning only applies to the focuscost dataset. The other datasets have their own
+        # schemas that happen to share column names: Prices reads "pricesheet" and renames MeterName
+        # to x_SkuMeterName, ReservationTransactions reads "reservationtransactions". Those uses are
+        # correct and must not be flagged.
+        $partitions = @(
+            $partitionPattern.Matches($content) |
+                ForEach-Object { $_.Groups[1].Value } |
+                Where-Object { $_ -match 'ftk_Storage\(\s*"focuscost"' }
+        )
+        $allowed = if ($baseline.ContainsKey($RelPath)) { $baseline[$RelPath] } else { 0 }
+
+        $unguarded = @(
+            foreach ($column in $droppedInFocus12)
+            {
+                $referenced = $false
+                $guarded = $false
+                foreach ($partition in $partitions)
+                {
+                    if ($partition -match ('\[{0}\]|"{0}"' -f [regex]::Escape($column))) { $referenced = $true }
+                    if ($partition -match ('(Table\.HasColumns|_exists)\([^)]*"{0}"' -f [regex]::Escape($column))) { $guarded = $true }
+                }
+                if ($referenced -and -not $guarded) { $column }
+            }
+        )
+
+        $unguarded.Count | Should -BeLessOrEqual $allowed -Because ("FOCUS 1.2 does not deliver these columns and ftk_Storage does not backfill them, so a container holding only 1.2 exports fails to refresh. Guard the reference with Table.HasColumns or the _exists helper. Unguarded here: $($unguarded -join ', ')")
+
+        if ($unguarded.Count -le $allowed)
+        {
+            # Ratchet: if a reference was guarded, the baseline must be lowered so it cannot return.
+            $unguarded.Count | Should -Be $allowed -Because ("the unguarded reference count in this file dropped below the baseline ($allowed); lower the baseline entry for '$RelPath' in PowerQueryColumnNames.Tests.ps1 to $($unguarded.Count) (or remove it if 0) so the fix is locked in.")
+        }
+    }
+}