diff --git a/ConvertOneNote2MarkDown-v2.Tests.ps1 b/ConvertOneNote2MarkDown-v2.Tests.ps1 index 72d1ae7..5c89e9b 100644 --- a/ConvertOneNote2MarkDown-v2.Tests.ps1 +++ b/ConvertOneNote2MarkDown-v2.Tests.ps1 @@ -885,6 +885,27 @@ Describe 'New-SectionGroupConversionConfig' -Tag 'Unit' { } } + It "Should generate unique media file names for each page, even for subpages and identically-named pages of a Section" { + foreach ($medialocation in 1, 2) { + $params['Config']['medialocation']['value'] = $medialocation + foreach ($fakeHierarchy in @( (Get-FakeOneNoteHierarchy), (Get-FakeOneNoteHierarchyWithDuplicatePageNames) )) { + $params['SectionGroups'] = $fakeHierarchy.Notebooks.Notebook + $result = @( New-SectionGroupConversionConfig @params 6>$null ) + + # Media file names must be unique within each media folder + $result | Group-Object { [io.path]::combine( $_['mediaPath'], $_['mediaNamePrefix'] ) } | Where-Object { $_.Count -gt 1 } | ForEach-Object { $_.Name } | Should -Be $null + } + } + + # Media file names of first level pages that are not identically-named are unchanged + $params['Config']['medialocation']['value'] = 1 + $params['SectionGroups'] = (Get-FakeOneNoteHierarchy).Notebooks.Notebook + $result = @( New-SectionGroupConversionConfig @params 6>$null ) + foreach ($pageCfg in @( $result | Where-Object { $_['pageLevel'] -eq 1 } )) { + $pageCfg['mediaNamePrefix'] | Should -Be $pageCfg['pathFromRootCompat'] + } + } + It "Should honor config notesdestpath" { if ($env:OS -match 'windows') { $params['Config']['notesdestpath']['value'] = 'D:\foo\bar' @@ -1116,6 +1137,92 @@ Describe 'New-SectionGroupConversionConfig' -Tag 'Unit' { } } + It "Should honor config mediaLinkFormat" { + $params['Config']['mediaLinkFormat']['value'] = 2 + $params['Config']['mdFileNameAndFolderNameMaxLength']['value'] = 100 # So that page names are truncated to 100 characters + $fakeHierarchy = Get-FakeOneNoteHierarchy + $pages = $fakeHierarchy.SelectNodes("//*[local-name()='Page']") + $pages[0].SetAttribute('name', 'C# notes ^1 f(g(x))') + $pages[1].SetAttribute('name', 'a' * 100) # A subpage of the first page, so that the prefix of media file names is longer than 100 characters + $pages[2].SetAttribute('name', 'b' * 100) # A subpage of the subpage + $pages[3].SetAttribute('name', 'a) exercise') # A subpage of the first page, with an unbalanced parenthesis + $params['SectionGroups'] = $fakeHierarchy.Notebooks.Notebook + Mock Get-OneNotePageContent { + $pageContent = Get-FakeOneNotePageContent + $pageContent.SelectNodes("//*[local-name()='InsertedFile']")[0].SetAttribute('preferredName', "$( 'c' * 140 ).pdf") # So that the attachment's name is longer than 150 characters + $pageContent + } + + $result = @( New-SectionGroupConversionConfig @params 6>$null ) + + # 15 pages from 'test' notebook, 15 pages from 'test2' notebook + $result.Count | Should -Be 48 + + # Media file names should be unique across notebooks, start with the notebook name, not contain '#' or '^', and their prefix should be no longer than 100 characters + $result | Group-Object { $_['mediaNamePrefix'] } | Where-Object { $_.Count -gt 1 } | ForEach-Object { $_.Name } | Should -Be $null + foreach ($pageCfg in $result) { + $pageCfg['mediaNamePrefix'] | Should -Match "^$( [regex]::Escape($pageCfg['notebookName']) )-" + $pageCfg['mediaNamePrefix'] | Should -Not -Match '[#^]' + $pageCfg['mediaNamePrefix'].Length | Should -BeLessOrEqual 100 + foreach ($attachmentCfg in $pageCfg['insertedAttachments']) { + # Attachment names should be prefixed, and no longer than 150 characters + $name = "$( $pageCfg['mediaNamePrefix'] )-$( $attachmentCfg['object'].preferredName | Remove-InvalidFileNameCharsInsertedFiles )" + if ($name.Length -le 150) { + $attachmentCfg['nameCompat'] | Should -Be $name + }else { + $attachmentCfg['nameCompat'].Length | Should -BeLessOrEqual 150 + $attachmentCfg['nameCompat'] | Should -Match "^$( [regex]::Escape($pageCfg['mediaNamePrefix']) )-c+-[0-9a-f]{8}\.pdf$" + } + Split-Path $attachmentCfg['destination'] -Leaf | Should -Be $attachmentCfg['nameCompat'] + } + } + $result[0]['mediaNamePrefix'] | Should -Be 'test-s0-C--notes--1-f(g(x))' + $result[1]['mediaNamePrefix'] | Should -Match '^test-s0-C--notes--1-f\(g\(x\)\)-[0-9a-f]{8}-a{50}$' # Keeps the start and the end, cut at a dash if possible + $result[2]['mediaNamePrefix'] | Should -Match '^test-s0-C--notes--1-f\(g\(x\)\)-[0-9a-f]{8}-b{50}$' + $result[3]['mediaNamePrefix'] | Should -Be 'test-s0-C--notes--1-f(g(x))-a)-exercise' + + # Mutate + $pageCfg = $result[3] + $fakeMarkdownContent = @( + "![some \[text\]]($( $pageCfg['mediaPathPandoc'] )/$( $pageCfg['mediaNamePrefix'] )-image1.png){width=`"1in`" height=`"2in`"}" + "![]($( $pageCfg['mediaPathPandoc'] )/$( $pageCfg['mediaNamePrefix'] )-image2.png)![]($( $pageCfg['mediaPathPandoc'] )/$( $pageCfg['mediaNamePrefix'] )-image3.png)" + $pageCfg['insertedAttachments'][0]['object'].preferredName + ) -join "`n" + $mutated = $fakeMarkdownContent + foreach ($m in $pageCfg['mutations']) { + foreach ($r in $m['replacements']) { + $mutated = $mutated -replace $r['searchRegex'], $r['replacement'] + } + } + + # Should use wikilinks for images and attachments. Ignore first 8 lines for page header + $split = $mutated -split "`n" + $split[8] | Should -Be "![[$( $pageCfg['mediaNamePrefix'] )-image1.png]]" + $split[9] | Should -Be "![[$( $pageCfg['mediaNamePrefix'] )-image2.png]]![[$( $pageCfg['mediaNamePrefix'] )-image3.png]]" + $split[10] | Should -Be "[[$( $pageCfg['insertedAttachments'][0]['nameCompat'] )]]" + + $params['Config']['mediaLinkFormat']['value'] = 1 + + $result = @( New-SectionGroupConversionConfig @params 6>$null ) + + # Should use markdown links with relative paths + $pageCfg = $result[0] + $pageCfg['mediaNamePrefix'] | Should -Be 's0-C#-notes-^1-f(g(x))' + $pageCfg['insertedAttachments'][0]['nameCompat'] | Should -Be ($pageCfg['insertedAttachments'][0]['object'].preferredName | Remove-InvalidFileNameCharsInsertedFiles) + $mutated = @( + "![some \[text\]]($( $pageCfg['mediaPathPandoc'] )/$( $pageCfg['mediaNamePrefix'] )-image1.png){width=`"1in`" height=`"2in`"}" + $pageCfg['insertedAttachments'][0]['object'].preferredName + ) -join "`n" + foreach ($m in $pageCfg['mutations']) { + foreach ($r in $m['replacements']) { + $mutated = $mutated -replace $r['searchRegex'], $r['replacement'] + } + } + $split = $mutated -split "`n" + $split[8] | Should -Be "![some [text]]($( $pageCfg['levelsPrefix'] )media/$( $pageCfg['mediaNamePrefix'] )-image1.png){width=`"1in`" height=`"2in`"}" + $split[9] | Should -Be "[$( $pageCfg['insertedAttachments'][0]['nameCompat'] )]($( $pageCfg['levelsPrefix'] )media/$( $pageCfg['insertedAttachments'][0]['nameCompat'] ))" + } + It "Should honor config conversion" { $params['Config']['conversion']['value'] = 'gfm+pipe_tables' @@ -1760,14 +1867,15 @@ Describe 'Convert-OneNotePage' -Tag 'Unit' { Assert-MockCalled -CommandName Get-ChildItem -Times 1 -Scope It Assert-MockCalled -CommandName Move-Item -Times 1 -Scope It + Assert-MockCalled -CommandName Move-Item -ParameterFilter { (Split-Path $Destination -Leaf) -eq "$( $params['ConversionConfig']['mediaNamePrefix'] )-image.jpg" } -Times 1 -Scope It } - It "Does not halt conversion if renaming image(s) fails" { + It "Does not halt conversion if renaming image(s) fails, and reports the image's destination path" { Mock Move-Item { throw } $err = Convert-OneNotePage @params 6>$null 2>&1 - $err.Exception.Message | Select-Object -First 1 | Should -match 'Failed to rename image' + $err.Exception.Message | Select-Object -First 1 | Should -match "Failed to rename image .+ to $( [regex]::Escape([io.path]::combine( $params['ConversionConfig']['mediaPath'], "$( $params['ConversionConfig']['mediaNamePrefix'] )-image.jpg" )) )\." } It "Markdown Mutation: Rename page image references in markdown to unique names" { diff --git a/ConvertOneNote2MarkDown-v2.ps1 b/ConvertOneNote2MarkDown-v2.ps1 index 3668cba..a9034e6 100644 --- a/ConvertOneNote2MarkDown-v2.ps1 +++ b/ConvertOneNote2MarkDown-v2.ps1 @@ -117,6 +117,16 @@ Default: 32 Whether to store media in single or multiple folders 1: Images stored in single 'media' folder at Notebook-level - Default 2: Separate 'media' folder for each folder in the hierarchy +'@ + default = 1 + value = 1 + validateRange = 1,2 + } + mediaLinkFormat = @{ + description = @' +Whether to link media (e.g. images, attachments) with markdown links or wikilinks +1: Markdown links with relative paths, e.g. ![](../media/mysection-mypage-image1.png) - Default +2: Wikilinks with only the file name, e.g. ![[mynotebook-mysection-mypage-image1.png]]. For markdown viewers that find linked files anywhere by their name (e.g. Obsidian), so that links don't break when moving notes or media. Media file names include the notebook name, so that they are unique across notebooks '@ default = 1 value = 1 @@ -861,6 +871,32 @@ Function New-SectionGroupConversionConfig { $cfg['notesBaseDirectory'] } $pageCfg['mediaPath'] = [io.path]::combine( $pageCfg['mediaParentPath'], 'media' ) + $pageCfg['mediaNamePrefix'] = if ($config['mediaLinkFormat']['value'] -eq 2) { + # Wikilinks contain only the file name, so the prefix also includes the notebook, i.e. the .md file's path within the notes destination, so that media file names are unique across notebooks. '#' and '^' are replaced, since they have a special meaning in wikilinks + $prefix = ("$( $cfg['notebookName'] )$( [io.path]::DirectorySeparatorChar )$( $sectionCfg['pathFromRoot'] )$( [io.path]::DirectorySeparatorChar )$( $pageCfg['filePathRel'] )" | Remove-InvalidFileNameChars) -replace '[#^]', '-' + if ($prefix.Length -gt 100) { + # Shorten long prefixes, so that media paths stay well below the path limit of 260 characters on Windows, also after moving the notes to another folder. Keep the start (e.g. the notebook) and the end (e.g. the page), with a hash of the full prefix in between, so that it is still unique + $sha256 = [System.Security.Cryptography.SHA256]::Create() + $hash = [BitConverter]::ToString( $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($prefix)) ).Replace('-', '').Substring(0, 8).ToLower() + $sha256.Dispose() + $start = 40 + if ([char]::IsHighSurrogate($prefix[$start - 1])) { $start-- } # Don't split characters like emojis + $end = $prefix.Length - 50 + if ([char]::IsLowSurrogate($prefix[$end])) { $end++ } + # Cut at dashes (i.e. between words or names), if possible + $i = $prefix.LastIndexOf('-', $start) + if ($i -gt 0) { $start = $i } + $i = $prefix.IndexOf('-', $end - 1) + if ($i -ge 0 -and $i -lt $prefix.Length - 1) { $end = $i + 1 } + $prefix ="$( $prefix.Substring(0, $start).TrimEnd('-') )-$hash-$( $prefix.Substring($end).TrimStart('-') )" + } + $prefix + }elseif ($config['medialocation']['value'] -eq 2) { + $pageCfg['filePathRelUnderscore'] + }else { + # The media of all of a notebook's pages are saved in the same folder, so the prefix includes the page's parent pages and the postfix of identically-named pages, i.e. the .md file's path. Or else e.g. subpages with the same name would overwrite each other's media + "$( $sectionCfg['pathFromRoot'] )$( [io.path]::DirectorySeparatorChar )$( $pageCfg['filePathRel'] )" | Remove-InvalidFileNameChars + } $pageCfg['mediaParentPathPandoc'] = [io.path]::combine( $pageCfg['tmpPath'] ).Replace( [io.path]::DirectorySeparatorChar, '/' ) # Pandoc outputs paths in markdown with with front slahes after the supplied , e.g. '/media/image.png'. So let's use a front-slashed supplied mediaPath $pageCfg['mediaPathPandoc'] = [io.path]::combine( $pageCfg['tmpPath'], 'media').Replace( [io.path]::DirectorySeparatorChar, '/' ) # Pandoc outputs paths in markdown with with front slahes after the supplied , e.g. '/media/image.png'. So let's use a front-slashed supplied mediaPath $pageCfg['docxExportFilePath'] = if ($config['docxNamingConvention']['value'] -eq 1) { @@ -879,7 +915,24 @@ Function New-SectionGroupConversionConfig { foreach ($i in $insertedFiles) { $attachmentCfg = [ordered]@{} $attachmentCfg['object'] = $i - $attachmentCfg['nameCompat'] = $i.preferredName | Remove-InvalidFileNameCharsInsertedFiles + $attachmentCfg['nameCompat'] = & { + $name = $i.preferredName | Remove-InvalidFileNameCharsInsertedFiles + if ($config['mediaLinkFormat']['value'] -eq 2) { + # Wikilinks contain only the file name, so attachments are prefixed like images, so that their names are unique too + $name = "$( $pageCfg['mediaNamePrefix'] )-$name" + if ($name.Length -gt 150) { + # Shorten long names, so that media paths stay well below the path limit of 260 characters on Windows. Keep the extension, and append a hash of the full name, so that it is still unique + $sha256 = [System.Security.Cryptography.SHA256]::Create() + $hash = [BitConverter]::ToString( $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($name)) ).Replace('-', '').Substring(0, 8).ToLower() + $sha256.Dispose() + $extension = [io.path]::GetExtension($name) + $length = 150 - $extension.Length - $hash.Length - 1 + if ([char]::IsHighSurrogate($name[$length - 1])) { $length-- } # Don't split characters like emojis + $name = "$( $name.Substring(0, $length).TrimEnd('-') )-$hash$extension" + } + } + $name + } $attachmentCfg['markdownFileName'] = $attachmentCfg['nameCompat'] | Encode-Markdown -Uri $attachmentCfg['source'] = $i.pathCache $attachmentCfg['destination'] = [io.path]::combine( $pageCfg['mediaPath'], $attachmentCfg['nameCompat'] ) @@ -897,7 +950,23 @@ Function New-SectionGroupConversionConfig { replacements = @( @{ searchRegex = [regex]::Escape( $attachmentCfg['object'].preferredName ) - replacement = "[$( $attachmentCfg['markdownFileName'] )]($( $pageCfg['mediaPathPandoc'] )/$( $attachmentCfg['markdownFileName'] ))" + replacement = if ($config['mediaLinkFormat']['value'] -eq 2) { + "[[$( $attachmentCfg['nameCompat'] )]]".Replace('$', '$$') # Escape '$', since it denotes a substitution in the replacement + }else { + "[$( $attachmentCfg['markdownFileName'] )]($( $pageCfg['mediaPathPandoc'] )/$( $attachmentCfg['markdownFileName'] ))" + } + } + ) + } + } + if ($config['mediaLinkFormat']['value'] -eq 2) { + @{ + description = 'Use wikilinks for images' + replacements = @( + @{ + # E.g. '![some \[text\]](C:/temp/notes/mynotebook/media/mynotebook-mysection-mypage-image1.png){width="1in" height="2in"}' -> '![[mynotebook-mysection-mypage-image1.png]]'. The image names are matched by their prefix, since it may contain any parentheses, e.g. 'f(g(x))' or 'a)'. Wikilinks have no alternative text, and markdown viewers that support wikilinks don't support pandoc's link attributes + searchRegex = '!\[(?:\\.|[^\\\]])*\]\(\[\]]+)>?\)(?:\{[^{}]*\})?' + replacement = "![[$( $pageCfg['mediaNamePrefix'].Replace('$', '$$') )-`$1]]" # Escape '$', since it denotes a substitution in the replacement } ) } @@ -1241,18 +1310,14 @@ Function Convert-OneNotePage { foreach ($image in $images) { # Rename Image try { - $newimageName = if ($config['medialocation']['value'] -eq 2) { - "$( $pageCfg['filePathRelUnderscore'] )-$($image.BaseName)$($image.Extension)" - }else { - "$( $pageCfg['pathFromRootCompat'] )-$($image.BaseName)$($image.Extension)" - } + $newimageName = "$( $pageCfg['mediaNamePrefix'] )-$($image.BaseName)$($image.Extension)" $newimagePath = [io.path]::combine( $pageCfg['mediaPath'], $newimageName ) "Moving image: $( $image.FullName ) to $( $newimagePath )" | Write-Verbose if (!$config['dryRun']['value']) { $item = Move-Item -Path "$( $image.FullName )" -Destination $newimagePath -Force -ErrorAction Stop -PassThru } }catch { - Write-Error "Failed to rename image $( $image.FullName ) to $( $item.FullName ). Exception: $( $_.Exception.Message )" -ErrorAction Continue + Write-Error "Failed to rename image $( $image.FullName ) to $( $newimagePath ). Exception: $( $_.Exception.Message )" -ErrorAction Continue } # Mutate markdown content with new image references try { diff --git a/README.md b/README.md index c23006f..166a8fc 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The powershell script `ConvertOneNote2MarkDown-v2.ps1` will utilize the OneNote * Choose between putting all media (images, attachments) in a central `/media` folder for each notebook, or in a separate `/media` folder in each folder of the hierarchy * Symbols in media file names removed for link compatibility * Updates media references in the resulting `.md` files, generating **relative** references to the media files within the markdown document + * Choose whether to use markdown links with relative paths, or wikilinks with only the file name (e.g. `![[image.png]]`), which don't break when moving notes or media in markdown viewers like Obsidian. With wikilinks, media file names are unique across notebooks * Choose between **discarding or keeping intermediate Word files**. Intermediate Word files are stored in a central notebook folder. * Choose between converting from existing `.docx` (90% faster) and creating new ones - useful if just want to test differences in the various processing options without generating new `.docx`each time * Choose between naming `.docx` files using page ID and last modified epoch date e.g. `{somelongid}-1234567890.docx` or hierarchy e.g. `-
-.docx` diff --git a/config.example.ps1 b/config.example.ps1 index 6077bdf..ac2eafd 100644 --- a/config.example.ps1 +++ b/config.example.ps1 @@ -49,6 +49,11 @@ $mdFileNameAndFolderNameMaxLength = 32 # 2: Separate 'media' folder for each folder in the hierarchy $medialocation = 1 +# Whether to link media (e.g. images, attachments) with markdown links or wikilinks +# 1: Markdown links with relative paths, e.g. ![](../media/mysection-mypage-image1.png) - Default +# 2: Wikilinks with only the file name, e.g. ![[mynotebook-mysection-mypage-image1.png]]. For markdown viewers that find linked files anywhere by their name (e.g. Obsidian), so that links don't break when moving notes or media. Media file names include the notebook name, so that they are unique across notebooks +$mediaLinkFormat = 1 + # Specify Pandoc output format and optional extension(s) in the format: <+extension><-extension> # Use this to customize the markdown flavor # Extension(s) must be supported by your pandoc version. See: https://pandoc.org/MANUAL.html#options