From 6dbf3f18cebe6e9ec19c23fb0f37a3f48492d4d0 Mon Sep 17 00:00:00 2001 From: Matthew Fishman Date: Sat, 26 Sep 2026 15:24:29 -0400 Subject: [PATCH 1/5] Close HDF5 handles eagerly when reading and writing MPS and MPO The HDF5 readers and writers left group and attribute handles open until garbage collection, so repeated reads got slower and slower (https://itensor.discourse.group/t/2639). They are now closed on return. The leaks inside each site tensor are fixed in https://github.com/ITensor/ITensors.jl/pull/1781. Co-Authored-By: Claude Fable 5.1 --- Project.toml | 11 ++++++- ext/ITensorMPSHDF5Ext/ITensorMPSHDF5Ext.jl | 11 +++++++ ext/ITensorMPSHDF5Ext/mpo.jl | 38 ++++++++++++---------- ext/ITensorMPSHDF5Ext/mps.jl | 38 ++++++++++++---------- test/base/test_readwrite.jl | 19 +++++++++++ 5 files changed, 80 insertions(+), 37 deletions(-) diff --git a/Project.toml b/Project.toml index acaed477..dcfec592 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "ITensorMPS" uuid = "0d1a4710-d33b-49a5-8f18-73bdf49b47e2" -version = "0.4.1" +version = "0.4.2" authors = ["Matthew Fishman ", "Miles Stoudenmire "] [workspace] @@ -26,6 +26,15 @@ Observers = "338f10d5-c7f1-4033-a7d1-f9dec39bcaa0" PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444" +[sources.ITensors] +rev = "mf/hdf5-close-handles" +url = "https://github.com/ITensor/ITensors.jl" + +[sources.NDTensors] +rev = "mf/hdf5-close-handles" +subdir = "NDTensors" +url = "https://github.com/ITensor/ITensors.jl" + [extensions] ITensorMPSChainRulesCoreExt = "ChainRulesCore" ITensorMPSHDF5Ext = "HDF5" diff --git a/ext/ITensorMPSHDF5Ext/ITensorMPSHDF5Ext.jl b/ext/ITensorMPSHDF5Ext/ITensorMPSHDF5Ext.jl index db651361..a42ae8bc 100644 --- a/ext/ITensorMPSHDF5Ext/ITensorMPSHDF5Ext.jl +++ b/ext/ITensorMPSHDF5Ext/ITensorMPSHDF5Ext.jl @@ -1,4 +1,15 @@ module ITensorMPSHDF5Ext + +# HDF5.jl has no do-block form of `open_group` or `create_group`, so without this +# the handles opened below stay alive until their finalizers run. +function closing(f, obj) + try + return f(obj) + finally + close(obj) + end +end + include("mps.jl") include("mpo.jl") end diff --git a/ext/ITensorMPSHDF5Ext/mpo.jl b/ext/ITensorMPSHDF5Ext/mpo.jl index 5ab8fc55..8e0a4f9a 100644 --- a/ext/ITensorMPSHDF5Ext/mpo.jl +++ b/ext/ITensorMPSHDF5Ext/mpo.jl @@ -1,29 +1,31 @@ -using HDF5: HDF5, attributes, create_group, open_group, read, write +using HDF5: HDF5, attributes, create_group, open_group, read, read_attribute, write using ITensorMPS: MPO using ITensors: ITensor function HDF5.write(parent::Union{HDF5.File, HDF5.Group}, name::AbstractString, M::MPO) - g = create_group(parent, name) - attributes(g)["type"] = "MPO" - attributes(g)["version"] = 1 - N = length(M) - write(g, "rlim", M.rlim) - write(g, "llim", M.llim) - write(g, "length", N) - for n in 1:N - write(g, "MPO[$(n)]", M[n]) + closing(create_group(parent, name)) do g + attributes(g)["type"] = "MPO" + attributes(g)["version"] = 1 + N = length(M) + write(g, "rlim", M.rlim) + write(g, "llim", M.llim) + write(g, "length", N) + for n in 1:N + write(g, "MPO[$(n)]", M[n]) + end end return end function HDF5.read(parent::Union{HDF5.File, HDF5.Group}, name::AbstractString, ::Type{MPO}) - g = open_group(parent, name) - if read(attributes(g)["type"]) != "MPO" - error("HDF5 group or file does not contain MPO data") + return closing(open_group(parent, name)) do g + if read_attribute(g, "type") != "MPO" + error("HDF5 group or file does not contain MPO data") + end + N = read(g, "length") + rlim = read(g, "rlim") + llim = read(g, "llim") + v = [read(g, "MPO[$(i)]", ITensor) for i in 1:N] + return MPO(v, llim, rlim) end - N = read(g, "length") - rlim = read(g, "rlim") - llim = read(g, "llim") - v = [read(g, "MPO[$(i)]", ITensor) for i in 1:N] - return MPO(v, llim, rlim) end diff --git a/ext/ITensorMPSHDF5Ext/mps.jl b/ext/ITensorMPSHDF5Ext/mps.jl index 7e591ecc..f9c466c3 100644 --- a/ext/ITensorMPSHDF5Ext/mps.jl +++ b/ext/ITensorMPSHDF5Ext/mps.jl @@ -1,29 +1,31 @@ -using HDF5: HDF5, attributes, create_group, open_group, read, write +using HDF5: HDF5, attributes, create_group, open_group, read, read_attribute, write using ITensorMPS: MPS using ITensors: ITensor function HDF5.write(parent::Union{HDF5.File, HDF5.Group}, name::AbstractString, M::MPS) - g = create_group(parent, name) - attributes(g)["type"] = "MPS" - attributes(g)["version"] = 1 - N = length(M) - write(g, "length", N) - write(g, "rlim", M.rlim) - write(g, "llim", M.llim) - for n in 1:N - write(g, "MPS[$(n)]", M[n]) + closing(create_group(parent, name)) do g + attributes(g)["type"] = "MPS" + attributes(g)["version"] = 1 + N = length(M) + write(g, "length", N) + write(g, "rlim", M.rlim) + write(g, "llim", M.llim) + for n in 1:N + write(g, "MPS[$(n)]", M[n]) + end end return end function HDF5.read(parent::Union{HDF5.File, HDF5.Group}, name::AbstractString, ::Type{MPS}) - g = open_group(parent, name) - if read(attributes(g)["type"]) != "MPS" - error("HDF5 group or file does not contain MPS data") + return closing(open_group(parent, name)) do g + if read_attribute(g, "type") != "MPS" + error("HDF5 group or file does not contain MPS data") + end + N = read(g, "length") + rlim = read(g, "rlim") + llim = read(g, "llim") + v = [read(g, "MPS[$(i)]", ITensor) for i in 1:N] + return MPS(v, llim, rlim) end - N = read(g, "length") - rlim = read(g, "rlim") - llim = read(g, "llim") - v = [read(g, "MPS[$(i)]", ITensor) for i in 1:N] - return MPS(v, llim, rlim) end diff --git a/test/base/test_readwrite.jl b/test/base/test_readwrite.jl index 2e265c76..7361c612 100644 --- a/test/base/test_readwrite.jl +++ b/test/base/test_readwrite.jl @@ -35,6 +35,25 @@ include(joinpath(@__DIR__, "utils", "util.jl")) end end + @testset "No leaked HDF5 handles" begin + nopen(file) = HDF5.API.h5f_get_obj_count(file, HDF5.API.H5F_OBJ_ALL) + sites = siteinds("S=1/2", 20) + mps = makeRandomMPS(sites) + mpo = makeRandomMPO(sites) + GC.gc() + h5open("data.h5", "w") do fo + write(fo, "mps", mps) + write(fo, "mpo", mpo) + # `h5f_get_obj_count` includes the file handle itself. + @test nopen(fo) == 1 + end + h5open("data.h5", "r") do fi + read(fi, "mps", MPS) + read(fi, "mpo", MPO) + @test nopen(fi) == 1 + end + end + # # Clean up the test hdf5 file # From 7dda57c02ab0d76500071a640c5f48db220b5bdc Mon Sep 17 00:00:00 2001 From: Matthew Fishman Date: Sun, 27 Sep 2026 12:33:10 -0400 Subject: [PATCH 2/5] Rename the close-on-exit helper to closeafter Co-Authored-By: Claude Fable 5.1 --- ext/ITensorMPSHDF5Ext/ITensorMPSHDF5Ext.jl | 2 +- ext/ITensorMPSHDF5Ext/mpo.jl | 4 ++-- ext/ITensorMPSHDF5Ext/mps.jl | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/ITensorMPSHDF5Ext/ITensorMPSHDF5Ext.jl b/ext/ITensorMPSHDF5Ext/ITensorMPSHDF5Ext.jl index a42ae8bc..acb85657 100644 --- a/ext/ITensorMPSHDF5Ext/ITensorMPSHDF5Ext.jl +++ b/ext/ITensorMPSHDF5Ext/ITensorMPSHDF5Ext.jl @@ -2,7 +2,7 @@ module ITensorMPSHDF5Ext # HDF5.jl has no do-block form of `open_group` or `create_group`, so without this # the handles opened below stay alive until their finalizers run. -function closing(f, obj) +function closeafter(f, obj) try return f(obj) finally diff --git a/ext/ITensorMPSHDF5Ext/mpo.jl b/ext/ITensorMPSHDF5Ext/mpo.jl index 8e0a4f9a..36fe2d63 100644 --- a/ext/ITensorMPSHDF5Ext/mpo.jl +++ b/ext/ITensorMPSHDF5Ext/mpo.jl @@ -3,7 +3,7 @@ using ITensorMPS: MPO using ITensors: ITensor function HDF5.write(parent::Union{HDF5.File, HDF5.Group}, name::AbstractString, M::MPO) - closing(create_group(parent, name)) do g + closeafter(create_group(parent, name)) do g attributes(g)["type"] = "MPO" attributes(g)["version"] = 1 N = length(M) @@ -18,7 +18,7 @@ function HDF5.write(parent::Union{HDF5.File, HDF5.Group}, name::AbstractString, end function HDF5.read(parent::Union{HDF5.File, HDF5.Group}, name::AbstractString, ::Type{MPO}) - return closing(open_group(parent, name)) do g + return closeafter(open_group(parent, name)) do g if read_attribute(g, "type") != "MPO" error("HDF5 group or file does not contain MPO data") end diff --git a/ext/ITensorMPSHDF5Ext/mps.jl b/ext/ITensorMPSHDF5Ext/mps.jl index f9c466c3..1502d4e5 100644 --- a/ext/ITensorMPSHDF5Ext/mps.jl +++ b/ext/ITensorMPSHDF5Ext/mps.jl @@ -3,7 +3,7 @@ using ITensorMPS: MPS using ITensors: ITensor function HDF5.write(parent::Union{HDF5.File, HDF5.Group}, name::AbstractString, M::MPS) - closing(create_group(parent, name)) do g + closeafter(create_group(parent, name)) do g attributes(g)["type"] = "MPS" attributes(g)["version"] = 1 N = length(M) @@ -18,7 +18,7 @@ function HDF5.write(parent::Union{HDF5.File, HDF5.Group}, name::AbstractString, end function HDF5.read(parent::Union{HDF5.File, HDF5.Group}, name::AbstractString, ::Type{MPS}) - return closing(open_group(parent, name)) do g + return closeafter(open_group(parent, name)) do g if read_attribute(g, "type") != "MPS" error("HDF5 group or file does not contain MPS data") end From 4e69fd6007997acfe63390ee11b5f2695677e4c9 Mon Sep 17 00:00:00 2001 From: Matthew Fishman Date: Sun, 27 Sep 2026 13:00:26 -0400 Subject: [PATCH 3/5] Pin NDTensors to its own fix branch Co-Authored-By: Claude Fable 5.1 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index dcfec592..987ac92b 100644 --- a/Project.toml +++ b/Project.toml @@ -31,7 +31,7 @@ rev = "mf/hdf5-close-handles" url = "https://github.com/ITensor/ITensors.jl" [sources.NDTensors] -rev = "mf/hdf5-close-handles" +rev = "mf/ndtensors-hdf5-close-handles" subdir = "NDTensors" url = "https://github.com/ITensor/ITensors.jl" From fd47702f3c6c8e9ab155d1c95084b517191f93ba Mon Sep 17 00:00:00 2001 From: Matthew Fishman Date: Sun, 27 Sep 2026 13:29:27 -0400 Subject: [PATCH 4/5] Use mktempdir for the HDF5 test file Co-Authored-By: Claude Fable 5.1 --- test/base/test_readwrite.jl | 92 ++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/test/base/test_readwrite.jl b/test/base/test_readwrite.jl index 7361c612..2b214d92 100644 --- a/test/base/test_readwrite.jl +++ b/test/base/test_readwrite.jl @@ -7,56 +7,54 @@ using Test include(joinpath(@__DIR__, "utils", "util.jl")) @testset "HDF5 Read and Write" begin - @testset "MPO/MPS" begin - N = 6 - sites = siteinds("S=1/2", N) - - # MPO - mpo = makeRandomMPO(sites) - - h5open("data.h5", "w") do fo - return write(fo, "mpo", mpo) - end - - h5open("data.h5", "r") do fi - rmpo = read(fi, "mpo", MPO) - @test prod([norm(rmpo[i] - mpo[i]) / norm(mpo[i]) < 1.0e-10 for i in 1:N]) - end - - # MPS - mps = makeRandomMPS(sites) - h5open("data.h5", "w") do fo - return write(fo, "mps", mps) + mktempdir() do dir + fn = joinpath(dir, "data.h5") + @testset "MPO/MPS" begin + N = 6 + sites = siteinds("S=1/2", N) + + # MPO + mpo = makeRandomMPO(sites) + + h5open(fn, "w") do fo + return write(fo, "mpo", mpo) + end + + h5open(fn, "r") do fi + rmpo = read(fi, "mpo", MPO) + @test prod([norm(rmpo[i] - mpo[i]) / norm(mpo[i]) < 1.0e-10 for i in 1:N]) + end + + # MPS + mps = makeRandomMPS(sites) + h5open(fn, "w") do fo + return write(fo, "mps", mps) + end + + h5open(fn, "r") do fi + rmps = read(fi, "mps", MPS) + @test prod([norm(rmps[i] - mps[i]) / norm(mps[i]) < 1.0e-10 for i in 1:N]) + end end - h5open("data.h5", "r") do fi - rmps = read(fi, "mps", MPS) - @test prod([norm(rmps[i] - mps[i]) / norm(mps[i]) < 1.0e-10 for i in 1:N]) + @testset "No leaked HDF5 handles" begin + nopen(file) = HDF5.API.h5f_get_obj_count(file, HDF5.API.H5F_OBJ_ALL) + sites = siteinds("S=1/2", 20) + mps = makeRandomMPS(sites) + mpo = makeRandomMPO(sites) + GC.gc() + h5open(fn, "w") do fo + write(fo, "mps", mps) + write(fo, "mpo", mpo) + # `h5f_get_obj_count` includes the file handle itself. + @test nopen(fo) == 1 + end + h5open(fn, "r") do fi + read(fi, "mps", MPS) + read(fi, "mpo", MPO) + @test nopen(fi) == 1 + end end end - - @testset "No leaked HDF5 handles" begin - nopen(file) = HDF5.API.h5f_get_obj_count(file, HDF5.API.H5F_OBJ_ALL) - sites = siteinds("S=1/2", 20) - mps = makeRandomMPS(sites) - mpo = makeRandomMPO(sites) - GC.gc() - h5open("data.h5", "w") do fo - write(fo, "mps", mps) - write(fo, "mpo", mpo) - # `h5f_get_obj_count` includes the file handle itself. - @test nopen(fo) == 1 - end - h5open("data.h5", "r") do fi - read(fi, "mps", MPS) - read(fi, "mpo", MPO) - @test nopen(fi) == 1 - end - end - - # - # Clean up the test hdf5 file - # - rm("data.h5"; force = true) end end From 2210fb2cec8c45f6493a767334e31e095fa81bc5 Mon Sep 17 00:00:00 2001 From: Matthew Fishman Date: Sun, 27 Sep 2026 15:00:20 -0400 Subject: [PATCH 5/5] Require the ITensors and NDTensors releases with the HDF5 handle fix Co-Authored-By: Claude Fable 5.1 --- Project.toml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Project.toml b/Project.toml index 987ac92b..056cb4fa 100644 --- a/Project.toml +++ b/Project.toml @@ -26,15 +26,6 @@ Observers = "338f10d5-c7f1-4033-a7d1-f9dec39bcaa0" PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444" -[sources.ITensors] -rev = "mf/hdf5-close-handles" -url = "https://github.com/ITensor/ITensors.jl" - -[sources.NDTensors] -rev = "mf/ndtensors-hdf5-close-handles" -subdir = "NDTensors" -url = "https://github.com/ITensor/ITensors.jl" - [extensions] ITensorMPSChainRulesCoreExt = "ChainRulesCore" ITensorMPSHDF5Ext = "HDF5" @@ -47,11 +38,11 @@ Adapt = "4.1" ChainRulesCore = "1.10" Compat = "4.16" HDF5 = "0.14, 0.15, 0.16, 0.17" -ITensors = "0.8, 0.9" +ITensors = "0.9.32" IsApprox = "2" KrylovKit = "0.8.1, 0.9, 0.10" LinearAlgebra = "1.10" -NDTensors = "0.4" +NDTensors = "0.4.32" Observers = "0.2" PackageCompiler = "1, 2" Printf = "1.10"