From 94a6771ecabbe79f157bd3db2c410b39e55a6c40 Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Tue, 27 Oct 2026 09:39:14 -0400 Subject: [PATCH 1/2] =?UTF-8?q?-=20Phase=202=20fix=20(Fp4E2M1Value):=20fou?= =?UTF-8?q?nd=20and=20fixed=20two=20real=20bugs=20=E2=80=94=20the=20ADAM/A?= =?UTF-8?q?DAMW=20gradient=20path=20needed=20new=20MatMulAccum/SumRowsAccu?= =?UTF-8?q?m=20backend=20methods=20(the=20old=20MatMul/SumRows=20only=20wo?= =?UTF-8?q?rked=20by=20accident=20when=20T=3D=3DAccumT),=20and=20GetRequir?= =?UTF-8?q?edElements()'s=20arena-capacity=20math=20assumed=2016-element?= =?UTF-8?q?=20padding,=20which=20undercounts=20for=20any=20T=20smaller=20t?= =?UTF-8?q?han=204=20bytes.=20-=20Phase=203=20(hwy::float16=5Ft,=20correct?= =?UTF-8?q?ness-first=20wrapper=20per=20your=20choice):=20replaced=20~79?= =?UTF-8?q?=20generic=20cast=20sites=20in=20CPUBackendT.cpp=20with=20hwy::?= =?UTF-8?q?ConvertScalarTo=20for=20cross-compiler=20portability;=20fixed?= =?UTF-8?q?=20a=20CMakeLists.txt=20gap=20(hwy=20was=20linked=20PRIVATE,=20?= =?UTF-8?q?breaking=20any=20external=20consumer=20of=20the=20now-public=20?= =?UTF-8?q?hwy::float16=5Ft=20type).=20-=20Phase=204=20(3=20posit=20format?= =?UTF-8?q?s=20=E2=80=94=20Posit4=5F0Value,=20Posit4=5F1Value,=20Posit4=5F?= =?UTF-8?q?2Value):=20mechanically=20identical=20to=20Fp4E2M1Value's=20pro?= =?UTF-8?q?ven=20pattern,=20as=20the=20plan=20predicted=20=E2=80=94=20all?= =?UTF-8?q?=20three=20compiled=20and=20passed=20verification=20on=20the=20?= =?UTF-8?q?first=20try.=20-=20Verification=20harness=20(tests/tFullPCLayer?= =?UTF-8?q?TemplatingVerify.cpp)=20grew=20to=2015=20passing=20checks=20acr?= =?UTF-8?q?oss=20all=20phases;=20unregistered=20from=20CMakeLists.txt=20no?= =?UTF-8?q?w=20that=20every=20gate=20has=20passed,=20with=20the=20source?= =?UTF-8?q?=20file=20kept=20on=20disk=20per=20this=20project's=20existing?= =?UTF-8?q?=20convention=20for=20one-time=20verification=20tests.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 11 +- Doxyfile | 4 +- bindings/BindingHelpers.h | 6 + bindings/LayerBindings.cpp | 178 +++++ bindings/NetworkBindings.cpp | 159 ++++ deepity/FullPCN.py | 90 ++- include/deepity/backend/BackendT.h | 38 + include/deepity/backend/CPUBackend.h | 18 + include/deepity/backend/CPUBackendT.h | 274 +++++++ include/deepity/backend/CUDABackend.h | 16 + include/deepity/backend/IComputeBackend.h | 255 +++--- include/deepity/layers/FullPCLayerT.h | 351 +++++++++ include/deepity/layers/Layer.h | 12 + include/deepity/networks/FullPCNetworkT.h | 447 +++++++++++ include/deepity/utils/Activations.h | 80 ++ include/deepity/utils/DTypeTag.h | 75 ++ include/deepity/utils/DeviceMemoryArena.h | 76 +- include/deepity/utils/LayerBindHelpers.h | 53 ++ include/deepity/utils/MemoryArena.h | 88 ++- include/deepity/utils/ModelIO.h | 37 +- include/deepity/utils/QuantFormat.h | 289 +++++++ include/deepity/utils/RandomizeDistribution.h | 29 + pyproject.toml | 6 +- resources/customdoxygen.css | 346 ++++++--- resources/deepity-favicon.png | Bin 0 -> 4897 bytes src/backend/CPUBackendT.cpp | 620 +++++++++++++++ src/layers/FullPCLayerT.cpp | 645 ++++++++++++++++ tests/tFullPCLayerTemplatingVerify.cpp | 726 ++++++++++++++++++ 28 files changed, 4608 insertions(+), 321 deletions(-) create mode 100644 include/deepity/backend/BackendT.h create mode 100644 include/deepity/backend/CPUBackendT.h create mode 100644 include/deepity/layers/FullPCLayerT.h create mode 100644 include/deepity/networks/FullPCNetworkT.h create mode 100644 include/deepity/utils/DTypeTag.h create mode 100644 include/deepity/utils/QuantFormat.h create mode 100644 resources/deepity-favicon.png create mode 100644 src/backend/CPUBackendT.cpp create mode 100644 src/layers/FullPCLayerT.cpp create mode 100644 tests/tFullPCLayerTemplatingVerify.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 85120940..95fd2d2a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -686,7 +686,9 @@ set(DEEPITY_SOURCES src/utils/StreamAlignedBatcher.cpp src/backend/Backend.cpp src/backend/CPUBackend.cpp + src/backend/CPUBackendT.cpp src/backend/Tensor.cpp + src/layers/FullPCLayerT.cpp ) if(DEEPITY_CUDA_ENABLED) @@ -748,7 +750,14 @@ target_link_libraries(Deepity PUBLIC sleef ) -target_link_libraries(Deepity PRIVATE hwy DeepityHighwayKernels) +# hwy is PUBLIC (not PRIVATE like DeepityHighwayKernels below it): the +# quantization templating pass's hwy::float16_t storage type now +# appears in Deepity's own public headers (DTypeTag.h, FullPCLayerT.h), +# so any consumer of Deepity that touches FullPCLayerT +# -- including this project's own test binaries -- needs hwy/base.h on +# its include path too, not just inside Deepity's own translation units. +target_link_libraries(Deepity PUBLIC hwy) +target_link_libraries(Deepity PRIVATE DeepityHighwayKernels) if(DEEPITY_CUDA_ENABLED) target_link_libraries(Deepity PUBLIC CUDA::cublas CUDA::cudart) diff --git a/Doxyfile b/Doxyfile index b2e6ae05..54cfcbbe 100644 --- a/Doxyfile +++ b/Doxyfile @@ -67,7 +67,7 @@ PROJECT_LOGO = "resources/deepity-mark.png" # when the HTML document is shown. Doxygen will copy the logo to the output # directory. -PROJECT_ICON = "resources/chicken_100.png" +PROJECT_ICON = "resources/deepity-favicon.png" # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path # into which the generated documentation will be written. If a relative path is @@ -1514,7 +1514,7 @@ HTML_EXTRA_FILES = # The default value is: AUTO_LIGHT. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_COLORSTYLE = AUTO_LIGHT +HTML_COLORSTYLE = DARK # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to diff --git a/bindings/BindingHelpers.h b/bindings/BindingHelpers.h index 2db8b5e7..51ac7fbe 100644 --- a/bindings/BindingHelpers.h +++ b/bindings/BindingHelpers.h @@ -26,6 +26,12 @@ using FloatArray = nb::ndarray; /// @brief Same as FloatArray, for int32 label/index arrays. using IntArray = nb::ndarray; +/// @brief Generic counterpart to FloatArray/DoubleArray, for the +/// templated FullPCLayerT/FullPCNetworkT +/// bindings (BindFullPCLayerT/BindFullPCNetworkT), which bind once per +/// storage type T rather than being hardcoded to float. +template using TArray = nb::ndarray; + namespace { diff --git a/bindings/LayerBindings.cpp b/bindings/LayerBindings.cpp index 07ab566f..60de44a1 100644 --- a/bindings/LayerBindings.cpp +++ b/bindings/LayerBindings.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -188,6 +189,180 @@ void BindCommonPCLayer(nb::class_& cls, const char* classNa }); } +// Generic (T-templated) counterparts to ClampStateHelper/SetWeightsHelper/ +// SetBiasesHelper above, for BindFullPCLayerT -- those three +// are hardcoded to FloatArray since every OTHER bound layer is float-only; +// FullPCLayerT is the one type binding once per T. +template void ClampStateHelperT(LayerT& self, TArray input) +{ + std::vector values(input.data(), input.data() + input.size()); + self.ClampState(values); +} + +template +void SetWeightsHelperT(LayerT& self, TArray values, size_t expectedCount) +{ + if ((size_t)values.size() != expectedCount) + throw std::invalid_argument("set_weights: expected " + std::to_string(expectedCount) + + " values, got " + std::to_string(values.size())); + self.GetBackend()->CopyFromHost(self.GetWeights(), values.data(), expectedCount); +} + +template +void SetBiasesHelperT(LayerT& self, TArray values, size_t expectedCount) +{ + if ((size_t)values.size() != expectedCount) + throw std::invalid_argument("set_biases: expected " + std::to_string(expectedCount) + + " values, got " + std::to_string(values.size())); + self.GetBackend()->CopyFromHost(self.GetBiases(), values.data(), expectedCount); +} + +/// @brief Templated counterpart to BindCommonPCLayer()+FullPCLayer's own +/// extra .def() calls combined, for FullPCLayerT. Not built +/// on top of BindCommonPCLayer() itself: that helper's signature requires +/// `nb::class_&`, and FullPCLayerT +/// deliberately doesn't derive from Deep::Layer (see FullPCLayerT's own +/// class doc comment) -- so this takes a plain `nb::class_&` +/// instead. Bound once per supported (T, AccumT) pair from bind_layers() +/// below, each producing its own distinct Python class +/// ("FullPCLayer_f32", "FullPCLayer_f64", ...). +template +void BindFullPCLayerT(nb::module_& m, const char* className) +{ + using LayerT = Deep::FullPCLayerT; + + nb::class_( + m, + className, + "PC layer combining muPC scaling, optional residual connections, and DKP direct " + "feedback, templated over its storage type -- see FullPCLayer for the float32 default.") + .def( + "__init__", + [](LayerT* self, size_t size, size_t next_size, size_t terminal_size, size_t batch_size, + AccumT learning_rate, AccumT inference_rate, AccumT feedback_rate, AccumT lmbda, + const std::string& activation, const std::string& activation_deriv) + { + new (self) LayerT(size, next_size, terminal_size, batch_size, learning_rate, + inference_rate, feedback_rate, lmbda, resolveActEnum(activation), + resolveActEnum(activation_deriv)); + }, + nb::arg("size"), nb::arg("next_size"), nb::arg("terminal_size"), + nb::arg("batch_size") = 1, nb::arg("learning_rate") = AccumT(1e-6), + nb::arg("inference_rate") = AccumT(0.01), nb::arg("feedback_rate") = AccumT(1e-4), + nb::arg("lmbda") = AccumT(1e-2), nb::arg("activation") = "relu", + nb::arg("activation_deriv") = "drelu") + .def("calculate_state", static_cast(&LayerT::CalculateState)) + .def("update_state", &LayerT::UpdateState) + .def("update_weights", &LayerT::UpdateWeights) + .def("clamp_state", &ClampStateHelperT, nb::arg("input")) + .def("unclamp_state", &LayerT::UnclampState) + .def("randomize_weights", &RandomizeWeightsHelper, nb::arg("seed") = nb::none(), + nb::arg("distribution") = "normal") + .def("set_layer_above", &LayerT::SetLayerAbove, nb::rv_policy::reference) + .def("set_layer_below", &LayerT::SetLayerBelow, nb::rv_policy::reference) + .def("set_terminal_layer", &LayerT::SetTerminalLayer, nb::arg("layer")) + .def("set_learning_rate", &LayerT::SetLearningRate, nb::arg("lr")) + .def("set_inference_rate", &LayerT::SetInferenceRate, nb::arg("ir")) + .def("set_feedback_rate", &LayerT::SetFeedbackRate, nb::arg("fl")) + .def("set_lambda", &LayerT::SetLambda, nb::arg("l")) + .def("direct_feedback_update", &LayerT::DirectFeedbackUpdate, + "Perturbs W using the layer above's Psi and the terminal layer's error, " + "the DFA phase, run once per batch before settling begins.") + .def( + "set_optimizer", + [](LayerT& self, const std::string& opt) + { + if (opt == "ADAM") + self.SetOptimizer(Deep::OptimizerType::ADAM); + else if (opt == "ADAMW") + self.SetOptimizer(Deep::OptimizerType::ADAMW); + else + self.SetOptimizer(Deep::OptimizerType::SGD); + }, + nb::arg("optimizer")) + .def( + "set_psi_optimizer", + [](LayerT& self, const std::string& opt) + { + if (opt == "ADAM") + self.SetPsiOptimizer(Deep::OptimizerType::ADAM); + else if (opt == "ADAMW") + self.SetPsiOptimizer(Deep::OptimizerType::ADAMW); + else + self.SetPsiOptimizer(Deep::OptimizerType::SGD); + }, + nb::arg("optimizer")) + .def("set_mu_pc_scale", &LayerT::SetMuPCScale, nb::arg("a"), + "Set this layer's muPC forward-scaling factor directly. Usually set by " + "FullPCNetworkT::Compile() instead.") + .def("set_residual", &LayerT::SetResidual, nb::arg("enabled"), + "Enable/disable this layer's residual connection directly. Same caveat as " + "set_mu_pc_scale.") + .def( + "set_weights", + [](LayerT& self, TArray values) + { SetWeightsHelperT(self, values, (size_t)self.GetOutputSize() * (size_t)self.GetInputSize()); }, + nb::arg("values")) + .def( + "set_biases", + [](LayerT& self, TArray values) + { SetBiasesHelperT(self, values, (size_t)self.GetOutputSize()); }, + nb::arg("values")) + .def_prop_ro("beliefs", + [](LayerT& self) -> nb::object + { + auto shape = {(size_t)self.GetBatchSize(), (size_t)self.GetInputSize()}; + if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) + return nb::cast(ViewOwnedBy(self.GetBeliefs(), shape, self)); + return nb::cast(CopyFromBackend(self, self.GetBeliefs(), shape)); + }) + .def_prop_ro("errors", + [](LayerT& self) -> nb::object + { + auto shape = {(size_t)self.GetBatchSize(), (size_t)self.GetInputSize()}; + if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) + return nb::cast(ViewOwnedBy(self.GetErrors(), shape, self)); + return nb::cast(CopyFromBackend(self, self.GetErrors(), shape)); + }) + .def_prop_ro("weights", + [](LayerT& self) -> nb::object + { + auto shape = {(size_t)self.GetOutputSize(), (size_t)self.GetInputSize()}; + if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) + return nb::cast(ViewOwnedBy(self.GetWeights(), shape, self)); + return nb::cast(CopyFromBackend(self, self.GetWeights(), shape)); + }) + .def_prop_ro("biases", + [](LayerT& self) -> nb::object + { + auto shape = {(size_t)self.GetOutputSize()}; + if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) + return nb::cast(ViewOwnedBy(self.GetBiases(), shape, self)); + return nb::cast(CopyFromBackend(self, self.GetBiases(), shape)); + }) + .def_prop_ro("psi", + [](LayerT& self) -> nb::object + { + auto shape = {(size_t)self.GetInputSize(), (size_t)self.GetTerminalSize()}; + if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) + return nb::cast(ViewOwnedBy(self.GetDirectFeedbackWeights(), shape, self)); + return nb::cast(CopyFromBackend(self, self.GetDirectFeedbackWeights(), shape)); + }) + .def_prop_ro("mu_pc_scale", &LayerT::GetMuPCScale) + .def_prop_ro("residual", &LayerT::GetResidual) + .def_prop_ro("terminal_size", &LayerT::GetTerminalSize) + .def_prop_ro("batch_size", &LayerT::GetBatchSize) + .def_prop_ro("input_size", &LayerT::GetInputSize) + .def_prop_ro("output_size", &LayerT::GetOutputSize) + .def("__repr__", + [className](const LayerT& self) + { + return "<" + std::string(className) + " in=" + std::to_string(self.GetInputSize()) + + ", out=" + std::to_string(self.GetOutputSize()) + + ", batch=" + std::to_string(self.GetBatchSize()) + ">"; + }); +} + } // namespace // ============================================================================ @@ -1014,4 +1189,7 @@ void bind_layers(nb::module_& m) " in=" + std::to_string(self.GetInHeight()) + "x" + std::to_string(self.GetInWidth()) + ">"; }); + + BindFullPCLayerT(m, "FullPCLayer_f32"); + BindFullPCLayerT(m, "FullPCLayer_f64"); } diff --git a/bindings/NetworkBindings.cpp b/bindings/NetworkBindings.cpp index b163e5f8..93258013 100644 --- a/bindings/NetworkBindings.cpp +++ b/bindings/NetworkBindings.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -97,6 +98,161 @@ template void BindCommonPCNetwork(nb::class_& cls, const c }); } +/// @brief Templated counterpart to FullPCNetwork's own manual (not +/// BindCommonPCNetwork-based) binding block below, for +/// FullPCNetworkT. Not built on BindCommonPCNetwork() -- +/// like FullPCNetwork itself, FullPCNetworkT's API shape (Step()/ +/// EPCStep() settling, not CalculateState()/UpdateState()) differs from +/// the networks that helper actually fits. Bound once per supported +/// (T, AccumT) pair from bind_networks() below, each producing its own +/// distinct Python class ("FullPCNetwork_f32", "FullPCNetwork_f64", ...). +/// Requires BindFullPCLayerT() to have already registered +/// FullPCLayerT's Python type (bind_layers() runs before +/// bind_networks(), see pybinding.cpp) -- get_terminal_layer()/layers +/// below return that type. +template +void BindFullPCNetworkT(nb::module_& m, const char* className) +{ + using NetT = Deep::FullPCNetworkT; + using LayerT = Deep::FullPCLayerT; + + nb::class_(m, + className, + "Predictive Coding Network combining muPC scaling, optional residual " + "connections, and DKP direct feedback, templated over its storage type -- " + "see FullPCNetwork for the float32 default.") + .def( + "__init__", + [](NetT* self, int batch_size, const std::string& device) + { + Deep::DeviceType dt = (device == "cuda" || device == "gpu") + ? Deep::DeviceType::DEVICE_GPU + : Deep::DeviceType::DEVICE_CPU; + new (self) NetT(batch_size, dt); + }, + nb::arg("batch_size"), nb::arg("device") = "cpu") + .def( + "add_layer", + [](NetT& self, size_t size, size_t next_size, size_t terminal_size, AccumT lr, + AccumT ir, AccumT fl, AccumT lmbda, const std::string& activation, + const std::string& activation_deriv) + { + self.AddLayer(size, next_size, terminal_size, lr, ir, fl, lmbda, + resolveActEnum(activation), resolveActEnum(activation_deriv)); + }, + nb::arg("size"), nb::arg("next_size"), nb::arg("terminal_size"), + nb::arg("lr") = AccumT(1e-6), nb::arg("ir") = AccumT(0.1), nb::arg("fl") = AccumT(1e-4), + nb::arg("lmbda") = AccumT(1e-2), nb::arg("activation") = "relu", + nb::arg("activation_deriv") = "drelu") + .def("set_use_mu_pc_scaling", &NetT::SetUseMuPCScaling, nb::arg("enabled")) + .def("set_use_residual_connections", &NetT::SetUseResidualConnections, nb::arg("enabled")) + .def("set_use_ipc", &NetT::SetUseIPC, nb::arg("enabled")) + .def("set_use_epc", &NetT::SetUseEPC, nb::arg("enabled")) + .def("set_use_momentum", &NetT::SetUseMomentum, nb::arg("enabled"), + nb::arg("beta") = AccumT(0.9)) + .def("set_use_cross_entropy", &NetT::SetUseCrossEntropy, nb::arg("enabled")) + .def("compile", &NetT::Compile) + .def( + "randomize_weights", + [](NetT& self, std::optional seed, const std::string& distribution) + { + std::mt19937 rng = seed.has_value() ? std::mt19937(seed.value()) + : std::mt19937(std::random_device{}()); + self.RandomizeWeights(rng, distribution.c_str()); + }, + nb::arg("seed") = nb::none(), nb::arg("distribution") = "normal") + .def("reset_state", &NetT::ResetState) + .def( + "clamp_input", + [](NetT& self, TArray input) + { + std::vector values(input.data(), input.data() + input.size()); + self.Clamp(values); + }, + nb::arg("input")) + .def("project_forward", &NetT::ProjectForward) + .def("calculate_terminal_error", &NetT::CalculateTerminalError) + .def("direct_feedback_update", &NetT::DirectFeedbackUpdate) + .def("step", &NetT::Step) + .def("update_weights", &NetT::UpdateWeights) + .def( + "set_optimizer", + [](NetT& self, const std::string& opt) + { + if (opt == "ADAM") + self.SetOptimizer(Deep::OptimizerType::ADAM); + else if (opt == "ADAMW") + self.SetOptimizer(Deep::OptimizerType::ADAMW); + else + self.SetOptimizer(Deep::OptimizerType::SGD); + }, + nb::arg("optimizer")) + .def( + "set_psi_optimizer", + [](NetT& self, const std::string& opt) + { + if (opt == "ADAM") + self.SetPsiOptimizer(Deep::OptimizerType::ADAM); + else if (opt == "ADAMW") + self.SetPsiOptimizer(Deep::OptimizerType::ADAMW); + else + self.SetPsiOptimizer(Deep::OptimizerType::SGD); + }, + nb::arg("optimizer")) + .def("set_learning_rate", &NetT::SetLearningRate, nb::arg("lr")) + .def("set_feedback_rate", &NetT::SetFeedbackRate, nb::arg("fl")) + .def( + "train_step", + [](NetT& self, TArray x, TArray y, int inference_steps) + { + std::vector xvec(x.data(), x.data() + x.size()); + std::vector yvec(y.data(), y.data() + y.size()); + return self.TrainStep(xvec, yvec, inference_steps); + }, + nb::arg("x"), nb::arg("y"), nb::arg("inference_steps") = 1) + .def( + "predict", + [](NetT& self, TArray x, int inference_steps) + { + std::vector xvec(x.data(), x.data() + x.size()); + std::vector result = self.Predict(xvec, inference_steps); + LayerT* terminal = self.GetTerminalLayer(); + return CopyToNewArray( + result.data(), {(size_t)terminal->GetBatchSize(), (size_t)terminal->GetInputSize()}); + }, + nb::arg("x"), nb::arg("inference_steps")) + .def_prop_ro("batch_size", &NetT::GetBatchSize) + .def_prop_ro("layers", + [](NetT& self) + { + nb::list result; + for (auto& layer : self.GetLayers()) + result.append(nb::cast(layer.get(), nb::rv_policy::reference)); + return result; + }) + .def("get_terminal_layer", &NetT::GetTerminalLayer, nb::rv_policy::reference) + .def("__len__", [](const NetT& self) { return self.GetLayers().size(); }) + .def( + "__getitem__", + [](NetT& self, std::ptrdiff_t index) + { + auto& layers = self.GetLayers(); + if (index < 0) + index += static_cast(layers.size()); + if (index < 0 || index >= static_cast(layers.size())) + throw nb::index_error(); + return layers[index].get(); + }, + nb::rv_policy::reference_internal) + .def("__repr__", + [className](const NetT& self) + { + return "<" + std::string(className) + + " layers=" + std::to_string(self.GetLayers().size()) + + " batch_size=" + std::to_string(self.GetBatchSize()) + ">"; + }); +} + } // namespace // ============================================================================ @@ -1334,4 +1490,7 @@ void bind_networks(nb::module_& m) .def("train_step_with_projection", &Deep::SimpleConvPCNetwork::TrainStepWithProjection) .def("predict_with_projection", &Deep::SimpleConvPCNetwork::PredictWithProjection) .def("project_forward", &Deep::SimpleConvPCNetwork::ProjectForward); + + BindFullPCNetworkT(m, "FullPCNetwork_f32"); + BindFullPCNetworkT(m, "FullPCNetwork_f64"); } diff --git a/deepity/FullPCN.py b/deepity/FullPCN.py index d816152a..a9c4b41a 100644 --- a/deepity/FullPCN.py +++ b/deepity/FullPCN.py @@ -10,7 +10,13 @@ from .utils import _fit_with_progress -class FullPCN(dy.FullPCNetwork): +_IMPL_BY_DTYPE = { + "float32": (dy.FullPCNetwork_f32, np.float32), + "float64": (dy.FullPCNetwork_f64, np.float64), +} + + +class FullPCN: """ The "everything" Predictive Coding Network: every PC mechanism this library implements, bolted onto one class as independently toggleable @@ -63,6 +69,16 @@ class FullPCN(dy.FullPCNetwork): decisions: set them via configure(), not afterwards. The other four extras are read fresh every settling step and can be flipped at any point via their own set_use_*() method. + + dtype selects which templated C++ backend (FullPCNetworkT) this + instance wraps: "float32" (the default, every existing caller's + behavior unchanged) or "float64" for genuine double precision. + FullPCN itself no longer subclasses the bound C++ network type -- + there's no longer one single bound type to subclass once there's a + distinct one per dtype -- so it holds the chosen instance as + self._impl and forwards to it, either through an explicit wrapper + method below or (for anything not explicitly wrapped) via + __getattr__. """ def __init__( @@ -70,6 +86,7 @@ def __init__( *architecture: Layer | Activation, batch_size: Optional[int] = None, device: str = "cpu", + dtype: str = "float32", ) -> None: resolved_batch_size = ( dy.auto_batch_size() @@ -77,8 +94,15 @@ def __init__( else batch_size ) + if dtype not in _IMPL_BY_DTYPE: + raise ValueError( + f"Unsupported dtype {dtype!r}, expected one of {sorted(_IMPL_BY_DTYPE)}." + ) + self.architecture = architecture self.device = device + self.dtype = dtype + self._np_dtype = _IMPL_BY_DTYPE[dtype][1] self._configured = False self._learning_rate: float = 1e-6 @@ -90,7 +114,15 @@ def __init__( self._validate_architecture() - super().__init__(resolved_batch_size, device) + impl_cls = _IMPL_BY_DTYPE[dtype][0] + self._impl = impl_cls(resolved_batch_size, device) + + def __getattr__(self, name: str): + # Reached only for attributes not found on FullPCN itself (every + # explicitly wrapped method/property above takes precedence) -- + # forwards anything else (e.g. batch_size) straight to the + # underlying bound network. + return getattr(self._impl, name) def _validate_architecture(self) -> None: if not self.architecture: @@ -147,7 +179,7 @@ def _build_backend(self) -> None: activation_deriv = "d" + activation - super().add_layer( + self._impl.add_layer( layer.in_n, layer.out_n, terminal_size, @@ -159,7 +191,7 @@ def _build_backend(self) -> None: activation_deriv=activation_deriv, ) - super().add_layer( + self._impl.add_layer( terminal_size, 0, terminal_size, @@ -250,15 +282,15 @@ def configure( # Calling super() directly (not self.set_use_*()): those wrappers # guard on _require_configured(), which isn't true yet this early # in configure() itself. - super().set_use_mu_pc_scaling(use_mu_pc_scaling) - super().set_use_residual_connections(use_residual_connections) - super().set_use_ipc(use_ipc) - super().set_use_epc(use_epc) - super().set_use_momentum(use_momentum, momentum_beta) - super().set_use_cross_entropy(use_cross_entropy) + self._impl.set_use_mu_pc_scaling(use_mu_pc_scaling) + self._impl.set_use_residual_connections(use_residual_connections) + self._impl.set_use_ipc(use_ipc) + self._impl.set_use_epc(use_epc) + self._impl.set_use_momentum(use_momentum, momentum_beta) + self._impl.set_use_cross_entropy(use_cross_entropy) - super().set_optimizer(optimizer) - super().set_psi_optimizer(psi_optimizer) + self._impl.set_optimizer(optimizer) + self._impl.set_psi_optimizer(psi_optimizer) self.compile() self.randomize_weights(seed=seed, dist=dist) @@ -281,7 +313,7 @@ def set_use_mu_pc_scaling(self, enabled: bool) -> None: before a manual compile() call); calling this after the network is already configured has no effect. """ - super().set_use_mu_pc_scaling(enabled) + self._impl.set_use_mu_pc_scaling(enabled) def set_use_residual_connections(self, enabled: bool) -> None: """ @@ -289,22 +321,22 @@ def set_use_residual_connections(self, enabled: bool) -> None: takes effect if set before compile(), same caveat as set_use_mu_pc_scaling(). """ - super().set_use_residual_connections(enabled) + self._impl.set_use_residual_connections(enabled) def set_use_ipc(self, enabled: bool) -> None: """Enable/disable incremental PC. Safe to call at any time.""" self._require_configured() - super().set_use_ipc(enabled) + self._impl.set_use_ipc(enabled) def set_use_epc(self, enabled: bool) -> None: """Enable/disable ePC settling. Safe to call at any time.""" self._require_configured() - super().set_use_epc(enabled) + self._impl.set_use_epc(enabled) def set_use_momentum(self, enabled: bool, beta: float = 0.9) -> None: """Enable/disable momentum settling. Safe to call at any time.""" self._require_configured() - super().set_use_momentum(enabled, beta) + self._impl.set_use_momentum(enabled, beta) def set_use_cross_entropy(self, enabled: bool) -> None: """ @@ -312,26 +344,26 @@ def set_use_cross_entropy(self, enabled: bool) -> None: layer. Safe to call at any time. """ self._require_configured() - super().set_use_cross_entropy(enabled) + self._impl.set_use_cross_entropy(enabled) def set_optimizer(self, optimizer: str) -> None: """Set the optimizer used for the forward weights W.""" - super().set_optimizer(optimizer) + self._impl.set_optimizer(optimizer) self._optimizer = optimizer def set_psi_optimizer(self, optimizer: str) -> None: """Set the optimizer used for the feedback weights Psi.""" - super().set_psi_optimizer(optimizer) + self._impl.set_psi_optimizer(optimizer) self._psi_optimizer = optimizer def set_learning_rate(self, learning_rate: float) -> None: """Set the learning rate for the forward weights W.""" - super().set_learning_rate(learning_rate) + self._impl.set_learning_rate(learning_rate) self._learning_rate = learning_rate def set_feedback_rate(self, feedback_rate: float) -> None: """Set the learning rate for the feedback weights Psi.""" - super().set_feedback_rate(feedback_rate) + self._impl.set_feedback_rate(feedback_rate) self._feedback_rate = feedback_rate def set_inference_rate(self, inference_rate: float) -> None: @@ -344,14 +376,14 @@ def set_inference_rate(self, inference_rate: float) -> None: self._inference_rate = inference_rate def compile(self) -> None: - super().compile() + self._impl.compile() def randomize_weights(self, seed: Optional[int] = None, dist: str = "normal") -> None: """Randomize all network weights. `seed` gives a reproducible run; `dist` e.g. "normal(0,1)" or "uniform(-0.3,0.3)" (the default "normal" has no "(a,b)" and so deliberately falls back to each layer's own existing init).""" - super().randomize_weights(seed, dist) + self._impl.randomize_weights(seed, dist) def train_step( self, @@ -361,9 +393,9 @@ def train_step( ) -> float: self._require_configured() - return super().train_step( - X.flatten(), - Y.flatten(), + return self._impl.train_step( + X.astype(self._np_dtype, copy=False).flatten(), + Y.astype(self._np_dtype, copy=False).flatten(), inference_steps, ) @@ -375,8 +407,8 @@ def predict( self._require_configured() return np.asarray( - super().predict( - X.flatten(), + self._impl.predict( + X.astype(self._np_dtype, copy=False).flatten(), inference_steps, ) ) diff --git a/include/deepity/backend/BackendT.h b/include/deepity/backend/BackendT.h new file mode 100644 index 00000000..53b67977 --- /dev/null +++ b/include/deepity/backend/BackendT.h @@ -0,0 +1,38 @@ +#pragma once +#include +#include +#include +#include + +/** + * @file BackendT.h + * @brief Templated counterpart to Backend.h's CreateBackend(), for + * FullPCNetworkT. Header-only (not CreateBackend()'s + * declaration + CreateBackend.cpp's definition split): a function + * template's definition has to be visible at every instantiation site + * anyway, so splitting it into a .cpp would just mean explicitly + * instantiating it there for every T/AccumT pair FullPCNetworkT + * supports -- no simpler than defining it here once. + */ + +namespace Deep +{ +/// @brief Factory function to create the appropriate backend for a +/// given T/AccumT. No CUDABackendT exists yet (CUDA support +/// is still only on the non-templated CUDABackend) -- DEVICE_GPU always +/// throws here, not gated behind DEEPITY_USE_CUDA like CreateBackend()'s +/// own GPU branch, since the real gap is "not implemented yet," not +/// "not compiled in this build." +template +std::unique_ptr> CreateBackendT(DeviceType device) +{ + if (device == DeviceType::DEVICE_CPU) + return std::make_unique>(); + + if (device == DeviceType::DEVICE_GPU) + throw std::runtime_error("CreateBackendT: no CUDABackendT exists yet -- the " + "templated layer/network pass only has a CPU backend so far."); + + throw std::invalid_argument("Unknown DeviceType requested from CreateBackendT."); +} +} // namespace Deep diff --git a/include/deepity/backend/CPUBackend.h b/include/deepity/backend/CPUBackend.h index 4304a2d4..dbf033f2 100644 --- a/include/deepity/backend/CPUBackend.h +++ b/include/deepity/backend/CPUBackend.h @@ -73,10 +73,28 @@ class CPUBackend : public IComputeBackend /// @copydoc Deep::IComputeBackend::MatMul void MatMul(bool transA, bool transB, int M, int N, int K, float alpha, const float* A, int lda, const float* B, int ldb, float beta, float* C, int ldc) noexcept override; + /// @copydoc Deep::IComputeBackendT::MatMulAccum + /// CPUBackend's T and AccumT are always float, so this is MatMul() + /// itself -- added only because IComputeBackend (an alias for + /// IComputeBackendT) now declares it for the templated + /// FullPCLayerT path's sake; no non-templated layer calls + /// this overload. + void MatMulAccum(bool transA, bool transB, int M, int N, int K, float alpha, const float* A, + int lda, const float* B, int ldb, float beta, float* C, + int ldc) noexcept override + { + MatMul(transA, transB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + } /// @copydoc Deep::IComputeBackend::SetAllowTF32 void SetAllowTF32(bool) noexcept override {} /// @copydoc Deep::IComputeBackend::SumRows void SumRows(float* dst, const float* src, size_t batchSize, size_t width) noexcept override; + /// @copydoc Deep::IComputeBackendT::SumRowsAccum + /// Same "T==AccumT==float always" reasoning as MatMulAccum() above. + void SumRowsAccum(float* dst, const float* src, size_t batchSize, size_t width) noexcept override + { + SumRows(dst, src, batchSize, width); + } /// @copydoc Deep::IComputeBackend::Sum float Sum(const float* buf, size_t n) noexcept override; diff --git a/include/deepity/backend/CPUBackendT.h b/include/deepity/backend/CPUBackendT.h new file mode 100644 index 00000000..1c49e8ff --- /dev/null +++ b/include/deepity/backend/CPUBackendT.h @@ -0,0 +1,274 @@ +#pragma once +#include +#include +#include + +namespace Deep +{ + +/// @brief Templated CPU implementation of IComputeBackendT, +/// used exclusively by the new templated FullPCLayer/FullPCNetwork +/// path (see QuantFormat.h and the "Templating Deepity's layers/networks +/// over a numeric type" plan). Deliberately NOT used by, and NOT a +/// replacement for, the existing non-templated CPUBackend -- every +/// other layer (SimplePCLayer, DirectKPPCLayer, GaussSeidelPCLayer, the +/// conv family) keeps using CPUBackend exactly as before. +/// +/// Only implements the ~20 IComputeBackendT methods FullPCLayer/ +/// FullPCNetwork actually call (MatMul, the elementwise/activation +/// ops, Adam, graph-capture no-ops, ...) with real per-T behavior: for +/// T=float, every one of those reuses the EXACT SAME dispatch this +/// project's existing CPUBackend uses (cblas_sgemm, the Highway- +/// dispatched kernels in CPUBackendHighway.*), so FullPCLayer's +/// output is bit-exact against the pre-templating FullPCLayer -- see +/// this project's quantization plan's verification gate #1. For any +/// other T (double, and eventually the quantized formats), those same +/// methods fall back to a plain, correct, non-Highway-vectorized scalar +/// loop; genuinely vectorizing them (via Highway's own T-generic +/// `hn::ScalableTag` kernels) is flagged as a deliberate, documented +/// follow-up, not attempted in this pass. +/// +/// Every OTHER IComputeBackendT method -- the "Try*" fused paths +/// (oneDNN/libxsmm/Halide/cuDNN-flavored; FullPCLayer never calls any +/// of them, only SimplePCLayer/the conv family do) and the conv/ +/// precision-weighted-layer-only ops (Im2Col, Col2Im, MaxPool2D*, +/// RepackForBatchedGemm, AddBiasPerChannel*, ComputePrecisionWeighted*, +/// AxpyBroadcastInto, MultiplyBroadcastInto, UpdatePrecisionFromError) +/// -- is unreachable from FullPCLayer/FullPCNetwork. The "Try*" +/// ones return false, their own interface's well-defined "no fused path +/// here" contract; the conv/precision-weighted ones throw +/// std::logic_error naming this class's scope, rather than shipping an +/// unexercised, unverified implementation of logic nothing in this pass +/// actually calls or tests (the conv family and DiscriminativePCLayer/ +/// ConvPCLayer stay on the non-templated CPUBackend, matching the +/// plan's explicit non-goals). +template +class CPUBackendT : public IComputeBackendT +{ +public: + CPUBackendT() = default; + ~CPUBackendT() override = default; + + void BeginGraphCapture(size_t slot = 0) noexcept override + { + (void)slot; + } + bool EndGraphCapture(size_t slot = 0) noexcept override + { + (void)slot; + return true; + } + void ReplayGraph(size_t slot = 0) noexcept override + { + (void)slot; + } + void Synchronize() noexcept override {} + void PrepareForBatchSize(size_t) noexcept override {} + void SetAllowTF32(bool) noexcept override {} + + T* Allocate(size_t numElements) override; + void Free(T* ptr) noexcept override; + void Zero(T* ptr, size_t numElements) noexcept override; + void Copy(T* dst, const T* src, size_t numElements) noexcept override; + void CopyFromHost(T* deviceDst, const T* hostSrc, size_t numElements) noexcept override; + void CopyToHost(T* hostDst, const T* deviceSrc, size_t numElements) noexcept override; + void RandomizeNormal(T* buf, size_t n, AccumT mean, AccumT stddev, uint32_t seed) noexcept override; + void RandomizeUniform(T* buf, size_t n, AccumT min, AccumT max, uint32_t seed) noexcept override; + + void MatMul(bool transA, bool transB, int M, int N, int K, AccumT alpha, const T* A, int lda, + const T* B, int ldb, AccumT beta, T* C, int ldc) noexcept override; + void MatMulAccum(bool transA, bool transB, int M, int N, int K, AccumT alpha, const T* A, + int lda, const T* B, int ldb, AccumT beta, AccumT* C, + int ldc) noexcept override; + + void SumRows(T* dst, const T* src, size_t batchSize, size_t width) noexcept override; + void SumRowsAccum(AccumT* dst, const T* src, size_t batchSize, size_t width) noexcept override; + AccumT Sum(const T* buf, size_t n) noexcept override; + + void Scale(T* buf, size_t n, AccumT alpha) noexcept override; + void AxpyInto(T* y, const T* x, size_t n, AccumT alpha) noexcept override; + void AddBiasBroadcast(T* buf, const T* bias, size_t batchSize, size_t width) noexcept override; + + bool TryFusedMatMulBias(bool, bool, int, int, int, const T*, int, const T*, int, const T*, T*, + int) noexcept override + { + return false; + } + bool TryFusedMatMul(bool, bool, int, int, int, AccumT, const T*, int, const T*, int, T*, + int) noexcept override + { + return false; + } + bool TryFusedFeedbackUpdate(ActivationType, int, int, int, const T*, const T*, T*, const T*, + AccumT, T*, T*) noexcept override + { + return false; + } + bool TryFusedForwardPass(ActivationType, const T*, const T*, const T*, T*, int, int, + int) noexcept override + { + return false; + } + + void Activation(ActivationType type, T* buf, size_t n) noexcept override; + void ActivationInto(ActivationType type, T* dst, const T* src, size_t n) noexcept override; + void ActivationDerivative(ActivationType type, T* buf, size_t n, bool activated) noexcept override; + void ActivationDerivativeInto(ActivationType type, T* dst, const T* src, size_t n) noexcept override; + + void FusedStateUpdateMomentum(T*, T*, const T*, ActivationType, const T*, size_t, AccumT, + AccumT) noexcept override + { + throw std::logic_error( + "CPUBackendT::FusedStateUpdateMomentum: out of scope -- only SimplePCLayer uses this, " + "and SimplePCLayer stays on the non-templated CPUBackend in this templating pass."); + } + void FusedStateUpdate(T*, const T*, ActivationType, const T*, size_t, AccumT) noexcept override + { + throw std::logic_error( + "CPUBackendT::FusedStateUpdate: out of scope -- only SimplePCLayer uses this, and " + "SimplePCLayer stays on the non-templated CPUBackend in this templating pass."); + } + + AccumT ComputeErrorAndEnergy(T* e, const T* z, const T* mu, size_t n) noexcept override; + void ComputeError(T* e, const T* z, const T* mu, size_t n) noexcept override; + + bool TryComputeErrorAndActivation(ActivationType, T*, T*, const T*, const T*, + size_t) noexcept override + { + return false; + } + + AccumT ComputeSoftmaxCrossEntropyErrorAndEnergy(T* e, const T* z, const T* mu, size_t batchSize, + size_t nextSize, + AccumT* rowEnergies) noexcept override; + void ComputeSoftmaxCrossEntropyError(T* e, const T* z, const T* mu, size_t batchSize, + size_t nextSize) noexcept override; + + void Im2Col(const T*, int, int, int, int, int, int, int, int, int, int, T*) noexcept override + { + throw std::logic_error( + "CPUBackendT::Im2Col: out of scope -- the conv layer family stays on the " + "non-templated CPUBackend in this templating pass."); + } + void Col2Im(const T*, int, int, int, int, int, int, int, int, int, int, T*) noexcept override + { + throw std::logic_error("CPUBackendT::Col2Im: out of scope, see Im2Col()'s own message."); + } + void MaxPool2DForward(const T*, int, int, int, int, int, int, int, int, T*, + int*) noexcept override + { + throw std::logic_error( + "CPUBackendT::MaxPool2DForward: out of scope, see Im2Col()'s own message."); + } + void MaxPool2DBackward(const T*, const int*, int, int, int, int, int, int, T*) noexcept override + { + throw std::logic_error( + "CPUBackendT::MaxPool2DBackward: out of scope, see Im2Col()'s own message."); + } + void RepackForBatchedGemm(T*, const T*, size_t, size_t, size_t) noexcept override + { + throw std::logic_error( + "CPUBackendT::RepackForBatchedGemm: out of scope, see Im2Col()'s own message."); + } + + bool TryFusedConvForward(const T*, const T*, T*, AccumT, int, int, int, int, int, int, int, int, + int, int, int) noexcept override + { + return false; + } + bool TryFusedConvBackwardData(const T*, const T*, T*, AccumT, int, int, int, int, int, int, int, + int, int, int, int) noexcept override + { + return false; + } + bool TryFusedConvBackwardFilter(const T*, const T*, T*, AccumT, AccumT, int, int, int, int, int, + int, int, int, int, int, int) noexcept override + { + return false; + } + void PrepareConvWorkspace(int, int, int, int, int, int, int, int, int, int, int) noexcept override + { + } + + void IncrementCounter(int* counter) noexcept override; + void AdamStep(T* param, const AccumT* grad, AccumT* m, AccumT* v, size_t n, const int* t, + const AccumT* lr, AccumT beta1 = AccumT(0.9), AccumT beta2 = AccumT(0.999), + AccumT eps = AccumT(1e-8)) noexcept override; + void AdamWStep(T* param, const AccumT* grad, AccumT* m, AccumT* v, size_t n, const int* t, + const AccumT* lr, AccumT weightDecay, AccumT beta1 = AccumT(0.9), + AccumT beta2 = AccumT(0.999), AccumT eps = AccumT(1e-8)) noexcept override; + + DeviceType GetDeviceType() const noexcept override + { + return DeviceType::DEVICE_CPU; + } + + void MultiplyInto(T* dst, const T* a, const T* b, size_t n) noexcept override; + + void FusedActivationDerivativeMultiply(T*, const T*, T*, ActivationType, size_t) noexcept override + { + throw std::logic_error( + "CPUBackendT::FusedActivationDerivativeMultiply: out of scope -- only " + "SimpleConvPCLayer/FullConvPCLayer use this, and the conv family stays on the " + "non-templated CPUBackend in this templating pass."); + } + void FusedActivationDerivativeMultiplyFromPreact(T*, const T*, const T*, ActivationType, + size_t) noexcept override + { + throw std::logic_error( + "CPUBackendT::FusedActivationDerivativeMultiplyFromPreact: out of scope, see " + "FusedActivationDerivativeMultiply()'s own message."); + } + void Fill(T*, size_t, AccumT) noexcept override + { + throw std::logic_error( + "CPUBackendT::Fill: out of scope -- FullPCLayer/FullPCNetwork never call this " + "in this templating pass."); + } + void AddBiasPerChannel(T*, const T*, size_t, size_t) noexcept override + { + throw std::logic_error( + "CPUBackendT::AddBiasPerChannel: out of scope, see Im2Col()'s own message."); + } + void AddBiasPerChannelNCHW(T*, const T*, size_t, size_t, size_t) noexcept override + { + throw std::logic_error( + "CPUBackendT::AddBiasPerChannelNCHW: out of scope, see Im2Col()'s own message."); + } + + AccumT ComputePrecisionWeightedErrorAndEnergy(T*, const T*, const T*, const T*, size_t, + size_t) noexcept override + { + throw std::logic_error( + "CPUBackendT::ComputePrecisionWeightedErrorAndEnergy: out of scope -- only " + "DiscriminativePCLayer/ConvPCLayer use this, and they stay on the non-templated " + "CPUBackend in this templating pass."); + } + void ComputePrecisionWeightedError(T*, const T*, const T*, const T*, size_t, + size_t) noexcept override + { + throw std::logic_error( + "CPUBackendT::ComputePrecisionWeightedError: out of scope, see " + "ComputePrecisionWeightedErrorAndEnergy()'s own message."); + } + void AxpyBroadcastInto(T*, const T*, const T*, size_t, size_t, AccumT) noexcept override + { + throw std::logic_error( + "CPUBackendT::AxpyBroadcastInto: out of scope, see " + "ComputePrecisionWeightedErrorAndEnergy()'s own message."); + } + void MultiplyBroadcastInto(T*, const T*, const T*, const T*, size_t, size_t) noexcept override + { + throw std::logic_error( + "CPUBackendT::MultiplyBroadcastInto: out of scope, see " + "ComputePrecisionWeightedErrorAndEnergy()'s own message."); + } + void UpdatePrecisionFromError(T*, T*, const T*, size_t, size_t, AccumT) noexcept override + { + throw std::logic_error( + "CPUBackendT::UpdatePrecisionFromError: out of scope, see " + "ComputePrecisionWeightedErrorAndEnergy()'s own message."); + } +}; + +} // namespace Deep diff --git a/include/deepity/backend/CUDABackend.h b/include/deepity/backend/CUDABackend.h index 4993d462..b6c3d385 100644 --- a/include/deepity/backend/CUDABackend.h +++ b/include/deepity/backend/CUDABackend.h @@ -51,6 +51,14 @@ class CUDABackend : public IComputeBackend /// @copydoc Deep::IComputeBackend::SumRows void SumRows(float* dst, const float* src, size_t batchSize, size_t width) noexcept override; + /// @copydoc Deep::IComputeBackendT::SumRowsAccum + /// CUDABackend's T and AccumT are always float, same as CPUBackend's + /// own identical-override reasoning -- see CPUBackend.h's own comment + /// on MatMulAccum/SumRowsAccum for why these exist at all. + void SumRowsAccum(float* dst, const float* src, size_t batchSize, size_t width) noexcept override + { + SumRows(dst, src, batchSize, width); + } /// @copydoc Deep::IComputeBackend::Sum float Sum(const float* buf, size_t n) noexcept override; @@ -63,6 +71,14 @@ class CUDABackend : public IComputeBackend /// @copydoc Deep::IComputeBackend::MatMul void MatMul(bool transA, bool transB, int M, int N, int K, float alpha, const float* A, int lda, const float* B, int ldb, float beta, float* C, int ldc) noexcept override; + /// @copydoc Deep::IComputeBackendT::MatMulAccum + /// Same "T==AccumT==float always" reasoning as SumRowsAccum() above. + void MatMulAccum(bool transA, bool transB, int M, int N, int K, float alpha, const float* A, + int lda, const float* B, int ldb, float beta, float* C, + int ldc) noexcept override + { + MatMul(transA, transB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + } /// @copydoc Deep::IComputeBackend::SetAllowTF32 void SetAllowTF32(bool allow) noexcept override; diff --git a/include/deepity/backend/IComputeBackend.h b/include/deepity/backend/IComputeBackend.h index 74359e6a..9b8cc94e 100644 --- a/include/deepity/backend/IComputeBackend.h +++ b/include/deepity/backend/IComputeBackend.h @@ -10,10 +10,27 @@ namespace Deep /// primitive a layer/network needs, implemented once each by CPUBackend /// (SIMD/BLAS) and CUDABackend (kernels/cuBLAS/CUTLASS), so layer code /// never branches on device directly. -class IComputeBackend +/// +/// @tparam T Element type for this backend's data buffers (activations, +/// weights, errors, biases, ...). `IComputeBackend` (see the alias +/// below) is `IComputeBackendT`, bit-for-bit identical to +/// this interface before it was templated -- every existing caller +/// (SimplePCLayer, DirectKPPCLayer, GaussSeidelPCLayer, the conv +/// family) keeps using that exact spelling, unchanged. +/// @tparam AccumT Wider type used for optimizer accumulator state +/// (AdamStep/AdamWStep's grad/m/v/lr) and for scalar +/// hyperparameters/reductions (alpha, beta, eps, ir, Sum()'s return, +/// ...) -- "accumulate wider than you store/compute the forward pass +/// at," so a sub-8-bit T never gets Adam-stepped in its own +/// low-precision representation. Defaults to T for every existing +/// (non-templated) caller, where it's simply unused distinction +/// (AccumT=T=float there); FullPCLayer is the only +/// place this is ever spelled as something other than T. +template +class IComputeBackendT { public: - virtual ~IComputeBackend() = default; + virtual ~IComputeBackendT() = default; // Graphs @@ -56,38 +73,38 @@ class IComputeBackend // Memory - /// @brief Allocates `numFloats` floats on this backend's device. + /// @brief Allocates `numElements` elements of T on this backend's device. /// @return nullptr on allocation failure. - virtual float* Allocate(size_t numFloats) = 0; + virtual T* Allocate(size_t numElements) = 0; /// @brief Frees a buffer previously returned by Allocate(). No-op on nullptr. - virtual void Free(float* ptr) noexcept = 0; - /// @brief Zero-fills `numFloats` floats starting at `ptr`. - virtual void Zero(float* ptr, size_t numFloats) noexcept = 0; - /// @brief Copies `numFloats` floats from `src` to `dst`, both on this - /// backend's own device. - virtual void Copy(float* dst, const float* src, size_t numFloats) noexcept = 0; - /// @brief Copies `numFloats` floats from a host buffer into a device - /// buffer (a plain memcpy on CPUBackend). - virtual void CopyFromHost(float* deviceDst, const float* hostSrc, size_t numFloats) noexcept = 0; - /// @brief Copies `numFloats` floats from a device buffer into a host - /// buffer (a plain memcpy on CPUBackend). - virtual void CopyToHost(float* hostDst, const float* deviceSrc, size_t numFloats) noexcept = 0; - /// @brief Fills `n` floats with samples from a normal distribution. + virtual void Free(T* ptr) noexcept = 0; + /// @brief Zero-fills `numElements` elements of T starting at `ptr`. + virtual void Zero(T* ptr, size_t numElements) noexcept = 0; + /// @brief Copies `numElements` elements of T from `src` to `dst`, both on + /// this backend's own device. + virtual void Copy(T* dst, const T* src, size_t numElements) noexcept = 0; + /// @brief Copies `numElements` elements of T from a host buffer into a + /// device buffer (a plain memcpy on CPUBackend). + virtual void CopyFromHost(T* deviceDst, const T* hostSrc, size_t numElements) noexcept = 0; + /// @brief Copies `numElements` elements of T from a device buffer into a + /// host buffer (a plain memcpy on CPUBackend). + virtual void CopyToHost(T* hostDst, const T* deviceSrc, size_t numElements) noexcept = 0; + /// @brief Fills `n` elements with samples from a normal distribution. /// @param buf Buffer to fill. - /// @param n Number of floats to fill. + /// @param n Number of elements to fill. /// @param mean Distribution mean. /// @param stddev Distribution standard deviation. /// @param seed RNG seed. - virtual void RandomizeNormal(float* buf, size_t n, float mean, float stddev, + virtual void RandomizeNormal(T* buf, size_t n, AccumT mean, AccumT stddev, uint32_t seed) noexcept = 0; - /// @brief Fills `n` floats with samples from a uniform distribution + /// @brief Fills `n` elements with samples from a uniform distribution /// over [min, max). /// @param buf Buffer to fill. - /// @param n Number of floats to fill. + /// @param n Number of elements to fill. /// @param min Distribution lower bound (inclusive). /// @param max Distribution upper bound (exclusive). /// @param seed RNG seed. - virtual void RandomizeUniform(float* buf, size_t n, float min, float max, + virtual void RandomizeUniform(T* buf, size_t n, AccumT min, AccumT max, uint32_t seed) noexcept = 0; /// @brief Must be called once, before Compile()'s first @@ -111,8 +128,23 @@ class IComputeBackend /// @param B Right operand. /// @param beta Scalar multiplied into the existing C before accumulating. /// @param C Output; also read if beta != 0. - virtual void MatMul(bool transA, bool transB, int M, int N, int K, float alpha, const float* A, - int lda, const float* B, int ldb, float beta, float* C, int ldc) noexcept = 0; + virtual void MatMul(bool transA, bool transB, int M, int N, int K, AccumT alpha, const T* A, + int lda, const T* B, int ldb, AccumT beta, T* C, int ldc) noexcept = 0; + + /// @brief Same operation as MatMul(), but writes into an AccumT* + /// buffer instead of T* -- for a layer's own Adam/AdamW gradient + /// computation (grad_W, grad_Psi, ...), which must land in the wider + /// AccumT accumulator, not get rounded down to storage precision T + /// mid-computation (exactly the gap this method closes: MatMul() + /// itself can only ever write a T* C, so it alone can't produce a + /// loss-free gradient when T is a lossy storage format with + /// AccumT != T, e.g. Fp4E2M1Value). For T == AccumT (every float/ + /// double instantiation) this is numerically identical to MatMul() + /// with that same buffer, just spelled through the AccumT-typed + /// overload the optimizer path calls either way. + virtual void MatMulAccum(bool transA, bool transB, int M, int N, int K, AccumT alpha, + const T* A, int lda, const T* B, int ldb, AccumT beta, AccumT* C, + int ldc) noexcept = 0; /// @brief Whether MatMul() may use TF32 (10 mantissa bits, ~1e-3 /// relative error per GEMM) instead of full FP32 on tensor cores -- @@ -132,26 +164,33 @@ class IComputeBackend /// cublasSgemv call against a cached all-ones vector, reinterpreting /// src's row-major [batchSize,width] layout as column-major /// [width,batchSize] with no data movement (verified numerically). - virtual void SumRows(float* dst, const float* src, size_t batchSize, size_t width) noexcept = 0; + virtual void SumRows(T* dst, const T* src, size_t batchSize, size_t width) noexcept = 0; + + /// @brief Same operation as SumRows(), but writes into an AccumT* + /// buffer instead of T* -- see MatMulAccum()'s own doc comment for + /// why this exists (a layer's bias gradient, grad_b, needs the same + /// AccumT-accumulating treatment grad_W does). + virtual void SumRowsAccum(AccumT* dst, const T* src, size_t batchSize, + size_t width) noexcept = 0; /// @brief Returns the plain (signed) sum of every element in buf. /// Synchronous on CUDABackend (reads the reduction back to a host - /// float before returning) -- same constraint as ComputeErrorAndEnergy + /// value before returning) -- same constraint as ComputeErrorAndEnergy /// and friends: never call this from inside a captured CUDA graph /// region, only from code that runs outside any /// BeginGraphCapture()/EndGraphCapture() pair. - virtual float Sum(const float* buf, size_t n) noexcept = 0; + virtual AccumT Sum(const T* buf, size_t n) noexcept = 0; // Elementwise scalar ops /// @brief buf[i] *= alpha for all i in [0, n). - virtual void Scale(float* buf, size_t n, float alpha) noexcept = 0; + virtual void Scale(T* buf, size_t n, AccumT alpha) noexcept = 0; /// @brief y[i] += alpha * x[i] for all i in [0, n), the standard AXPY. - virtual void AxpyInto(float* y, const float* x, size_t n, float alpha) noexcept = 0; + virtual void AxpyInto(T* y, const T* x, size_t n, AccumT alpha) noexcept = 0; /// @brief buf[row*width + col] += bias[col] for every row in /// [0,batchSize) and col in [0,width), broadcasts a per-column bias /// across every row (the dense-layer convention). - virtual void AddBiasBroadcast(float* buf, const float* bias, size_t batchSize, + virtual void AddBiasBroadcast(T* buf, const T* bias, size_t batchSize, size_t width) noexcept = 0; /// @brief Attempts C = A @ B(^T) + bias in one fused kernel (CPU only, @@ -178,8 +217,8 @@ class IComputeBackend /// pattern, not a general GEMM). /// @param bias Length-N vector, broadcast across all M rows of C /// (same convention as AddBiasBroadcast). - virtual bool TryFusedMatMulBias(bool transA, bool transB, int M, int N, int K, const float* A, - int lda, const float* B, int ldb, const float* bias, float* C, + virtual bool TryFusedMatMulBias(bool transA, bool transB, int M, int N, int K, const T* A, + int lda, const T* B, int ldb, const T* bias, T* C, int ldc) noexcept = 0; /// @brief Attempts C = alpha * (A @ B(^T)) via oneDNN's matmul @@ -195,8 +234,8 @@ class IComputeBackend /// @param transA/transB/M/N/K/A/lda/B/ldb/C/ldc Same meaning and /// shapes as MatMul(). @param alpha Scalar multiplier on the product, /// same convention as MatMul(). - virtual bool TryFusedMatMul(bool transA, bool transB, int M, int N, int K, float alpha, - const float* A, int lda, const float* B, int ldb, float* C, + virtual bool TryFusedMatMul(bool transA, bool transB, int M, int N, int K, AccumT alpha, + const T* A, int lda, const T* B, int ldb, T* C, int ldc) noexcept = 0; /// @brief Fuses SimplePCLayer::UpdateState()'s feedback GEMM directly @@ -247,16 +286,16 @@ class IComputeBackend /// post-op chain directly, see CPUBackend.cpp's implementation) and /// bound as a post-op's runtime operand. Unused on the Halide path. virtual bool TryFusedFeedbackUpdate(ActivationType dType, int batchSize, int size, int nextSize, - const float* eAbove, const float* W, float* z, - const float* e, float ir, float* scratch, - float* derivScratch) noexcept = 0; + const T* eAbove, const T* W, T* z, + const T* e, AccumT ir, T* scratch, + T* derivScratch) noexcept = 0; // Activation /// @brief Applies `type` to `buf` in place. - virtual void Activation(ActivationType type, float* buf, size_t n) noexcept = 0; + virtual void Activation(ActivationType type, T* buf, size_t n) noexcept = 0; /// @brief Two-buffer variant of Activation(): reads src, writes into dst. - virtual void ActivationInto(ActivationType type, float* dst, const float* src, + virtual void ActivationInto(ActivationType type, T* dst, const T* src, size_t n) noexcept = 0; /// @brief Applies `type`'s derivative to `buf` in place. /// @param type Which activation's derivative to apply. @@ -264,11 +303,11 @@ class IComputeBackend /// @param n Length of buf. /// @param activated If true, buf already holds the activated value; /// if false, buf holds the pre-activation value. - virtual void ActivationDerivative(ActivationType type, float* buf, size_t n, + virtual void ActivationDerivative(ActivationType type, T* buf, size_t n, bool activated) noexcept = 0; /// @brief Two-buffer variant of ActivationDerivative(): reads src /// (the pre-activation value), writes the derivative into dst. - virtual void ActivationDerivativeInto(ActivationType type, float* dst, const float* src, + virtual void ActivationDerivativeInto(ActivationType type, T* dst, const T* src, size_t n) noexcept = 0; // Fused PC-specific ops @@ -284,9 +323,9 @@ class IComputeBackend /// @param dType Which activation's derivative to apply to z in /// place, deriv is computed inline from z, element by element, /// rather than read from a separate precomputed buffer. - virtual void FusedStateUpdateMomentum(float* z, float* v, const float* feedback, - ActivationType dType, const float* e, size_t n, float ir, - float beta) noexcept = 0; + virtual void FusedStateUpdateMomentum(T* z, T* v, const T* feedback, + ActivationType dType, const T* e, size_t n, AccumT ir, + AccumT beta) noexcept = 0; /// @brief One settling step's state update, fused into a single call: /// z += ir * ((feedback * deriv) - e) @@ -294,16 +333,16 @@ class IComputeBackend /// @param dType Which activation's derivative to apply to z in /// place, deriv is computed inline from z, element by element, /// rather than read from a separate precomputed buffer. - virtual void FusedStateUpdate(float* z, const float* feedback, ActivationType dType, - const float* e, size_t n, float ir) noexcept = 0; + virtual void FusedStateUpdate(T* z, const T* feedback, ActivationType dType, + const T* e, size_t n, AccumT ir) noexcept = 0; /// @brief Computes e = z - mu, then returns 0.5 * sum(e^2), the /// Gaussian error/energy used by every layer except a cross-entropy /// terminal. - virtual float ComputeErrorAndEnergy(float* e, const float* z, const float* mu, - size_t n) noexcept = 0; + virtual AccumT ComputeErrorAndEnergy(T* e, const T* z, const T* mu, + size_t n) noexcept = 0; /// @brief Same as ComputeErrorAndEnergy(), without computing (or /// returning) the energy, for callers that only need e. - virtual void ComputeError(float* e, const float* z, const float* mu, size_t n) noexcept = 0; + virtual void ComputeError(T* e, const T* z, const T* mu, size_t n) noexcept = 0; /// @brief Fuses ComputeError()'s e = z - mu with the forward /// activation zF = f(z) that SimplePCLayer::ComputeMuOnly() computes @@ -327,8 +366,8 @@ class IComputeBackend /// @return false if this backend has no fused implementation, in /// which case the caller falls back to separate ComputeError()/ /// ActivationInto() calls, same convention as TryFusedMatMul() above. - virtual bool TryComputeErrorAndActivation(ActivationType activationType, float* e, float* zF, - const float* z, const float* mu, + virtual bool TryComputeErrorAndActivation(ActivationType activationType, T* e, T* zF, + const T* z, const T* mu, size_t n) noexcept = 0; /// @brief Softmax cross-entropy variant of ComputeErrorAndEnergy, for @@ -342,21 +381,21 @@ class IComputeBackend /// energy = -sum(z * log(probs + eps)) (eps=1e-8, avoids log(0)) /// mu holds the RAW LOGITS (unchanged from the Gaussian case, still /// a*W\@phi(z)+b), softmax is applied here, not baked into mu itself. - /// `rowEnergies` is caller-provided scratch, >= batchSize floats, + /// `rowEnergies` is caller-provided scratch, >= batchSize elements, /// NOT allocated internally (this runs inside CalculateState(), which /// runs inside TrainStep()'s CUDA-graph-captured region; dynamic /// allocation there is unsafe under capture, same reasoning as the /// CUTLASS workspace). CPUBackend's own override ignores this /// parameter entirely, kept in the shared interface so callers pass /// the same arguments to either backend. - virtual float ComputeSoftmaxCrossEntropyErrorAndEnergy(float* e, const float* z, const float* mu, - size_t batchSize, size_t nextSize, - float* rowEnergies) noexcept = 0; + virtual AccumT ComputeSoftmaxCrossEntropyErrorAndEnergy(T* e, const T* z, const T* mu, + size_t batchSize, size_t nextSize, + AccumT* rowEnergies) noexcept = 0; /// @brief Same as ComputeSoftmaxCrossEntropyErrorAndEnergy(), without /// computing (or requiring `rowEnergies` for) the energy, for /// callers that only need e. - virtual void ComputeSoftmaxCrossEntropyError(float* e, const float* z, const float* mu, + virtual void ComputeSoftmaxCrossEntropyError(T* e, const T* z, const T* mu, size_t batchSize, size_t nextSize) noexcept = 0; /// @brief Attempts a fused forward pass (GEMM + bias + activation in @@ -388,8 +427,8 @@ class IComputeBackend /// @param batchSize Number of rows in zF/mu. /// @param size Input width (columns of zF, columns of W). /// @param nextSize Output width (rows of W, columns of mu, length of bias). - virtual bool TryFusedForwardPass(ActivationType actType, const float* zF, const float* W, - const float* bias, float* mu, int batchSize, int size, + virtual bool TryFusedForwardPass(ActivationType actType, const T* zF, const T* W, + const T* bias, T* mu, int batchSize, int size, int nextSize) noexcept = 0; // Convolution (im2col-based, ConvPCLayer family) @@ -408,9 +447,9 @@ class IComputeBackend /// single-image contract. Positions outside the input (due to /// padding) are written as zero. @p columns is fully overwritten, /// not accumulated into. - virtual void Im2Col(const float* input, int batchSize, int channels, int height, int width, + virtual void Im2Col(const T* input, int batchSize, int channels, int height, int width, int kernelH, int kernelW, int strideH, int strideW, int padH, int padW, - float* columns) noexcept = 0; + T* columns) noexcept = 0; /// @brief The adjoint of Im2Col(): scatters a whole batch of /// (channels*kH*kW, outH*outW) column-gradient buffers back into @@ -419,9 +458,9 @@ class IComputeBackend /// rationale as Im2Col() above. ACCUMULATES into @p outputImage /// (does not zero it first), caller must zero the destination if a /// fresh result is wanted, matching Deep::Col2Im's own contract. - virtual void Col2Im(const float* columns, int batchSize, int channels, int height, int width, + virtual void Col2Im(const T* columns, int batchSize, int channels, int height, int width, int kernelH, int kernelW, int strideH, int strideW, int padH, int padW, - float* outputImage) noexcept = 0; + T* outputImage) noexcept = 0; /// @brief Batched 2D max pooling, no padding. @p input is [batchSize, /// channels, height, width], @p output and @p argmax are [batchSize, @@ -431,18 +470,18 @@ class IComputeBackend /// within its channel plane -- MaxPool2DBackward()'s only input besides /// the incoming gradient, no pre-activation buffer needed (a hard max's /// backward is exact from the argmax alone). - virtual void MaxPool2DForward(const float* input, int batchSize, int channels, int height, + virtual void MaxPool2DForward(const T* input, int batchSize, int channels, int height, int width, int poolH, int poolW, int strideH, int strideW, - float* output, int* argmax) noexcept = 0; + T* output, int* argmax) noexcept = 0; /// @brief The adjoint of MaxPool2DForward(): scatters a batch of /// [batchSize, channels, outH, outW] gradients back into [batchSize, /// channels, height, width] using the recorded @p argmax. ACCUMULATES /// into @p inputGrad (does not zero it first), matching /// Deep::MaxPool2DBackward's own contract. - virtual void MaxPool2DBackward(const float* outputGrad, const int* argmax, int batchSize, + virtual void MaxPool2DBackward(const T* outputGrad, const int* argmax, int batchSize, int channels, int height, int width, int outH, int outW, - float* inputGrad) noexcept = 0; + T* inputGrad) noexcept = 0; /// @brief Repacks a [batchSize, rows, cols] tensor (batch-major) /// into [rows, batchSize, cols] (row-major, batch second), @@ -456,7 +495,7 @@ class IComputeBackend /// AddBiasBroadcast/SumRows: a future strided-batched-GEMM path /// could remove the need for this entirely, but this matches /// the existing, verified CPU repacking loops exactly for now. - virtual void RepackForBatchedGemm(float* dst, const float* src, size_t batchSize, size_t rows, + virtual void RepackForBatchedGemm(T* dst, const T* src, size_t batchSize, size_t rows, size_t cols) noexcept = 0; /// @brief Attempts a single fused convolution forward pass, skipping @@ -474,7 +513,7 @@ class IComputeBackend /// @param W [outChannels, inChannels, kernelH, kernelW], the exact /// same memory every ConvPCLayer-family layer already stores its flat /// [outChannels, inChannels*kernelH*kernelW] weights in. - virtual bool TryFusedConvForward(const float* input, const float* W, float* output, float alpha, + virtual bool TryFusedConvForward(const T* input, const T* W, T* output, AccumT alpha, int batchSize, int inChannels, int height, int width, int outChannels, int kernelH, int kernelW, int strideH, int strideW, int padH, int padW) noexcept = 0; @@ -485,8 +524,8 @@ class IComputeBackend /// @p W) -- unlike Col2Im()'s own accumulate contract, every known /// caller here Zero()s its destination immediately beforehand anyway. /// Same false-means-unsupported contract as TryFusedConvForward(). - virtual bool TryFusedConvBackwardData(const float* outputGrad, const float* W, float* inputGrad, - float alpha, int batchSize, int inChannels, int height, + virtual bool TryFusedConvBackwardData(const T* outputGrad, const T* W, T* inputGrad, + AccumT alpha, int batchSize, int inChannels, int height, int width, int outChannels, int kernelH, int kernelW, int strideH, int strideW, int padH, int padW) noexcept = 0; @@ -498,8 +537,8 @@ class IComputeBackend /// W itself) and one that wants a fresh gradient buffer (Adam: beta=0 /// into grad_W) both work unchanged. Same false-means-unsupported /// contract as TryFusedConvForward(). - virtual bool TryFusedConvBackwardFilter(const float* input, const float* outputGrad, float* dW, - float alpha, float beta, int batchSize, int inChannels, + virtual bool TryFusedConvBackwardFilter(const T* input, const T* outputGrad, T* dW, + AccumT alpha, AccumT beta, int batchSize, int inChannels, int height, int width, int outChannels, int kernelH, int kernelW, int strideH, int strideW, int padH, int padW) noexcept = 0; @@ -525,9 +564,14 @@ class IComputeBackend /// (e.g. Adam's step count `t`) without a host round-trip. virtual void IncrementCounter(int* counter) noexcept = 0; - /// @brief Adam update for `n` parameters in place. - /// @param param Parameters to update. - /// @param grad Gradient w.r.t. param. + /// @brief Adam update for `n` parameters in place. `param` stays in + /// this backend's storage type T; the optimizer's own accumulator + /// state (grad/m/v/lr) is always AccumT, regardless of T -- a + /// sub-8-bit T's weights are never Adam-stepped directly in their own + /// low-precision representation, only through this wider shadow + /// accumulator (see FullPCLayer's own doc comment). + /// @param param Parameters to update, in storage type T. + /// @param grad Gradient w.r.t. param, in accumulator type AccumT. /// @param m Adam's first-moment (momentum) buffer, persistent across calls. /// @param v Adam's second-moment (variance) buffer, persistent across calls. /// @param n Number of parameters. @@ -536,20 +580,21 @@ class IComputeBackend /// @param lr Device-resident learning rate. /// @param beta1,beta2 Momentum/variance EMA decay rates. /// @param eps Denominator stabilizer. - virtual void AdamStep(float* param, const float* grad, float* m, float* v, size_t n, const int* t, - const float* lr, float beta1 = 0.9f, float beta2 = 0.999f, - float eps = 1e-8f) noexcept = 0; + virtual void AdamStep(T* param, const AccumT* grad, AccumT* m, AccumT* v, size_t n, const int* t, + const AccumT* lr, AccumT beta1 = AccumT(0.9), AccumT beta2 = AccumT(0.999), + AccumT eps = AccumT(1e-8)) noexcept = 0; /// @brief Same as AdamStep(), plus decoupled weight decay (AdamW): /// param -= lr * weightDecay * param, applied before the Adam step itself. - virtual void AdamWStep(float* param, const float* grad, float* m, float* v, size_t n, - const int* t, const float* lr, float weightDecay, float beta1 = 0.9f, - float beta2 = 0.999f, float eps = 1e-8f) noexcept = 0; + virtual void AdamWStep(T* param, const AccumT* grad, AccumT* m, AccumT* v, size_t n, + const int* t, const AccumT* lr, AccumT weightDecay, + AccumT beta1 = AccumT(0.9), AccumT beta2 = AccumT(0.999), + AccumT eps = AccumT(1e-8)) noexcept = 0; /// @brief Which device this backend instance runs on. virtual DeviceType GetDeviceType() const noexcept = 0; /// @brief dst[i] = a[i] * b[i] for all i in [0, n), elementwise product. - virtual void MultiplyInto(float* dst, const float* a, const float* b, size_t n) noexcept = 0; + virtual void MultiplyInto(T* dst, const T* a, const T* b, size_t n) noexcept = 0; /// @brief Fuses an in-place "derivative from activated value" step with /// the elementwise multiply that immediately follows it in every known /// caller: dst[i] = a[i] * f'(activatedInOut[i]), then @@ -567,7 +612,7 @@ class IComputeBackend /// ActivationDerivative(..., activated=true) already silently gives /// today, rather than pretending to fix a deeper design issue. /// @param dType A derivative-flavored ActivationType. - virtual void FusedActivationDerivativeMultiply(float* dst, const float* a, float* activatedInOut, + virtual void FusedActivationDerivativeMultiply(T* dst, const T* a, T* activatedInOut, ActivationType dType, size_t n) noexcept = 0; /// @brief Sibling of FusedActivationDerivativeMultiply() above, for /// callers that hold the PRE-activation value instead of the @@ -583,11 +628,11 @@ class IComputeBackend /// FullConvPCLayer/SimpleConvPCLayer's UpdateWeights() already had to /// un-fuse their own call sites over. /// @param dType A derivative-flavored ActivationType. - virtual void FusedActivationDerivativeMultiplyFromPreact(float* dst, const float* a, - const float* preact, ActivationType dType, + virtual void FusedActivationDerivativeMultiplyFromPreact(T* dst, const T* a, + const T* preact, ActivationType dType, size_t n) noexcept = 0; /// @brief buf[i] = value for all i in [0, n). - virtual void Fill(float* buf, size_t n, float value) noexcept = 0; + virtual void Fill(T* buf, size_t n, AccumT value) noexcept = 0; /// @brief Convolutional bias-add: buf[c*spatialSize + s] += bias[c] for /// all c in [0,channels), s in [0,spatialSize). Per-CHANNEL broadcast /// across spatial positions, the transpose relationship to @@ -599,7 +644,7 @@ class IComputeBackend /// batch item (e.g. FullConvPCLayer::DirectFeedbackUpdate()'s /// per-image projChannel), which still has to call this once per item /// with @p buf offset to that item's slice. - virtual void AddBiasPerChannel(float* buf, const float* bias, size_t channels, + virtual void AddBiasPerChannel(T* buf, const T* bias, size_t channels, size_t spatialSize) noexcept = 0; /// @brief Convolutional bias-add for the batch-MAJOR NCHW layout: @@ -615,7 +660,7 @@ class IComputeBackend /// elements whenever batchSize > 1 (confirmed: every gradient-check /// test in this codebase uses batchSize=1, which collapses the two /// layouts into the same thing and could never have caught this). - virtual void AddBiasPerChannelNCHW(float* buf, const float* bias, size_t batchSize, + virtual void AddBiasPerChannelNCHW(T* buf, const T* bias, size_t batchSize, size_t channels, size_t spatialSize) noexcept = 0; // Precision-weighted PC ops (DiscriminativePCLayer/ConvPCLayer family, @@ -630,19 +675,19 @@ class IComputeBackend /// existing CPU code already applies, guarding against log(0)). `p` has /// length `width`, one precision per own-position, broadcast across /// every batch item, NOT length batchSize*width like e/z/mu. - virtual float ComputePrecisionWeightedErrorAndEnergy(float* e, const float* z, const float* mu, - const float* p, size_t batchSize, - size_t width) noexcept = 0; + virtual AccumT ComputePrecisionWeightedErrorAndEnergy(T* e, const T* z, const T* mu, + const T* p, size_t batchSize, + size_t width) noexcept = 0; /// @brief Same as ComputePrecisionWeightedErrorAndEnergy(), without /// computing (or returning) the energy, for callers that only need e. /// On CUDABackend this additionally means: no blocking /// cudaStreamSynchronize() -- the ...AndEnergy() variant must - /// synchronously read the energy reduction back to a host float, which + /// synchronously read the energy reduction back to a host value, which /// is illegal while a CUDA graph is being captured, exactly why this /// counterpart exists (mirrors ComputeError() alongside /// ComputeErrorAndEnergy() above). - virtual void ComputePrecisionWeightedError(float* e, const float* z, const float* mu, - const float* p, size_t batchSize, + virtual void ComputePrecisionWeightedError(T* e, const T* z, const T* mu, + const T* p, size_t batchSize, size_t width) noexcept = 0; /// @brief y[b*width+i] += alpha * x[b*width+i] * factor[i], for every b @@ -650,8 +695,8 @@ class IComputeBackend /// per-position factor broadcast across the batch dimension. Covers /// dz_dt[idx] -= p[i]*e[idx] (alpha=-1, factor=p) without a temporary /// buffer for the elementwise product. - virtual void AxpyBroadcastInto(float* y, const float* x, const float* factor, size_t batchSize, - size_t width, float alpha) noexcept = 0; + virtual void AxpyBroadcastInto(T* y, const T* x, const T* factor, size_t batchSize, + size_t width, AccumT alpha) noexcept = 0; /// @brief dst[b*width+i] = a[b*width+i] * factor[i] * b[b*width+i], for /// every batch item, a three-operand elementwise product with one @@ -659,8 +704,8 @@ class IComputeBackend /// bottom_up_cols[idx] = e_above[idx] * p_above[i] * mu[idx], the same /// shape needed in both UpdateState() and UpdateWeights() for every /// precision-weighted layer. - virtual void MultiplyBroadcastInto(float* dst, const float* a, const float* factor, - const float* b, size_t batchSize, size_t width) noexcept = 0; + virtual void MultiplyBroadcastInto(T* dst, const T* a, const T* factor, + const T* b, size_t batchSize, size_t width) noexcept = 0; /// @brief Precision update from accumulated squared error, fused: /// for each i in [0,width), grad = mean_{b in [0,batchSize)} of @@ -668,7 +713,17 @@ class IComputeBackend /// [-5,5]; p[i] = exp(log_p[i]). Matches /// DiscriminativePCLayer::UpdatePrecision()/ConvPCLayer::UpdatePrecision()'s /// existing CPU formula exactly, including the clamp bounds. - virtual void UpdatePrecisionFromError(float* p, float* log_p, const float* e, size_t batchSize, - size_t width, float pr) noexcept = 0; + virtual void UpdatePrecisionFromError(T* p, T* log_p, const T* e, size_t batchSize, + size_t width, AccumT pr) noexcept = 0; }; + +/// @brief Non-templated alias preserving every existing caller's +/// behavior exactly: `IComputeBackend` is `IComputeBackendT`, bit-for-bit identical to this interface before it was +/// templated. CPUBackend/CUDABackend still implement this exact +/// spelling; new templated code (FullPCLayer/FullPCNetwork) uses +/// IComputeBackendT directly, implemented by the new +/// CPUBackendT/CUDABackendT. +using IComputeBackend = IComputeBackendT; + } // namespace Deep diff --git a/include/deepity/layers/FullPCLayerT.h b/include/deepity/layers/FullPCLayerT.h new file mode 100644 index 00000000..68bc4646 --- /dev/null +++ b/include/deepity/layers/FullPCLayerT.h @@ -0,0 +1,351 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @file FullPCLayerT.h + * @brief Templated counterpart to FullPCLayer, over a storage type `T` + * (the layer's own beliefs/errors/weights/activations) and an + * accumulator type `AccumT` (the Adam/AdamW optimizer's gradient/ + * moment state, always at least as wide as T -- see + * IComputeBackendT's own doc comment). Produced by + * mechanically retyping FullPCLayer's own `float` members/parameters to + * `T`/`AccumT`, NOT a reimplementation -- every method here must stay + * line-for-line equivalent to FullPCLayer's own method bodies, so that + * FullPCLayerT reproduces FullPCLayer's output bit-exactly + * (the quantization plan's verification gate #1). + * + * Deliberately does NOT inherit from Layer: Layer's virtual interface + * (GetBeliefs() etc.) is fixed at `float*`, which only FullPCLayerT could ever satisfy. Nothing in this pass needs a FullPCLayerT + * to be used polymorphically as a Layer*, so this is a free-standing + * class template instead (the same position FullPCNetwork already has). + * Every OTHER layer (SimplePCLayer, DirectKPPCLayer, GaussSeidelPCLayer, + * the conv family) is unaffected and keeps deriving from Layer exactly + * as before. + */ + +namespace Deep +{ +/// @brief Templated counterpart to FullPCLayer, see this file's own +/// comment for the full rationale. +template +class FullPCLayerT +{ +protected: + size_t size; ///< This layer's own belief (z) size. + size_t nextSize; ///< Size of the layer above (0 marks a terminal layer). + size_t terminalSize; ///< Size of the network's final output layer. + size_t batchSize; ///< Batch size. + + AccumT lr; ///< Learning rate for W. + AccumT ir; ///< Inference rate (Euler integration step size). + AccumT fl; ///< Feedback rate for Psi. + AccumT lmbda; ///< Weight decay (L2 regularization) coefficient, shared between W and Psi. + + AccumT a = AccumT(1); ///< muPC forward-scaling factor, see SetMuPCScale(). + bool useResidual = false; ///< Whether the residual/skip connection is enabled, see SetResidual(). + + bool useMuPCInit = false; ///< Whether unit-variance W init is enabled, see SetMuPCInit(). + + bool isClamped = false; ///< Whether ClampState() is currently active. + bool muCacheValid = false; ///< Whether `cachedMu` holds a valid, up-to-date value. + + FullPCLayerT* layerAbove = nullptr; ///< The next layer up, or nullptr if this is terminal. + FullPCLayerT* layerBelow = + nullptr; ///< The next layer down, or nullptr if this is the input layer. + FullPCLayerT* terminalLayer = + nullptr; ///< The network's terminal layer, for DirectFeedbackUpdate(). + + ActivationType activationType; ///< This layer's activation type. + + OptimizerType opt = OptimizerType::SGD; ///< Optimizer for W/b. + OptimizerType optPsi = OptimizerType::SGD; ///< Optimizer for Psi. + + /// @brief Custom deleter type for `backend`, needed since + /// IComputeBackendT is an abstract base, see the member below. + using BackendDeleter = void (*)(IComputeBackendT*); + std::unique_ptr, BackendDeleter> backend; ///< This layer's compute backend. + + T* z = nullptr; ///< This layer's belief buffer, shape [batchSize, size]. + T* e = nullptr; ///< This layer's error buffer, shape [batchSize, size]. + + T* W = nullptr; ///< Weight matrix, shape [nextSize, size]. + T* b = nullptr; ///< Bias vector, length nextSize. + T* mu = nullptr; ///< Forward prediction buffer, shape [batchSize, nextSize]. + T* cachedMu = nullptr; ///< Cached mu, valid when muCacheValid is true. + T* preact = nullptr; ///< Pre-activation snapshot of mu, see FullPCLayer's own doc comment. + T* cachedPreact = nullptr; ///< Cached pre-activation, valid alongside cachedMu. + T* derivScratch = nullptr; ///< nextSize-shaped scratch holding f'(preact). + T* feedbackError = nullptr; ///< nextSize-shaped scratch, see FullPCLayer's own doc comment. + T* Psi = nullptr; ///< Direct feedback (DFA) weight matrix, shape [size, terminalSize]. + T* proj = nullptr; ///< Scratch buffer for DirectFeedbackUpdate()'s W update. + + // grad_*/m_*/v_* and lr_device/fl_device/t_device/tPsi_device are all + // AccumT-typed (or int, for the step counters) optimizer-accumulator + // state, owned directly rather than arena-allocated like every T-typed + // buffer above -- see BindAdamStateT()/BindLearningRateStateT()'s own + // doc comments in LayerBindHelpers.h for why (in short: a layer's + // MemoryArenaT::Allocate() returns T*, which isn't AccumT* unless + // T==AccumT). + std::vector grad_W; ///< W's gradient scratch buffer. + std::vector grad_b; ///< b's gradient scratch buffer. + std::vector m_W; ///< Adam/AdamW first-moment buffer for W. + std::vector v_W; ///< Adam/AdamW second-moment buffer for W. + std::vector m_b; ///< Adam/AdamW first-moment buffer for b. + std::vector v_b; ///< Adam/AdamW second-moment buffer for b. + + std::vector grad_Psi; ///< Psi's gradient scratch buffer. + std::vector m_Psi; ///< Adam/AdamW first-moment buffer for Psi. + std::vector v_Psi; ///< Adam/AdamW second-moment buffer for Psi. + + int t_device = 0; ///< Adam/AdamW step count for W/b. + AccumT lr_device = AccumT(0); ///< Mirror of `lr`, passed by address to the backend. + int tPsi_device = 0; ///< Adam/AdamW step count for Psi. + AccumT fl_device = AccumT(0); ///< Mirror of `fl`, passed by address to the backend. + + T* feedbackScratch = nullptr; ///< Scratch buffer for the feedback GEMM's output. + + T* v = nullptr; ///< Momentum buffer for the settling update, see SetMomentum(). + bool useMomentum = false; ///< Whether momentum settling is enabled, see SetMomentum(). + AccumT momentumBeta = AccumT(0.9); ///< Momentum EMA decay rate, see SetMomentum(). + + bool useCrossEntropy = false; ///< Whether softmax cross-entropy energy is enabled. + /// @brief Per-row energy scratch for the cross-entropy path. Owned + /// directly (not arena-allocated like every other buffer here): + /// it's AccumT-typed, and CPUBackendT has no AccumT-typed Zero()/ + /// allocation primitive of its own to hand it a slot in the T-typed + /// arena when T != AccumT (see IComputeBackendT's own doc comment on + /// why AccumT buffers are the optimizer's domain, not the backend's). + /// Harmless: it's unused on CPU (CPUBackendT's own + /// ComputeSoftmaxCrossEntropyErrorAndEnergy() ignores it, matching + /// the non-templated CPUBackend), kept only for CUDA-path parity. + std::vector rowEnergies; + + bool useEPC = false; ///< Whether ePC settling is enabled, see SetUseEPC(). + T* adjoint = nullptr; ///< Chained backward-sweep signal, shape [batchSize, size]. + + T* biasGradScratch = nullptr; ///< nextSize-length scratch for SumRows' output. + + std::unique_ptr> + localArena; ///< This layer's own arena, used when it isn't bound into a network-wide one. + +public: + /// @param size input size + /// @param nextSize output size (0 marks a terminal layer) + /// @param terminalSize the size of the network's final output layer + /// @param batchSize batch size + /// @param learningRate learning rate for W + /// @param inferenceRate inference rate (Euler integration step size) + /// @param feedback feedback rate for Psi + /// @param lmbda weight decay (L2 regularization) coefficient + /// @param aType activation type + /// @param dType activation derivative type (unused, kept for call-site + /// parity with FullPCLayer's own constructor; derivative type is + /// always derived from aType via ToDerivativeType() at each use site) + /// @param backend Compute backend to run on; defaults to a new + /// CPUBackendT if nullptr. + FullPCLayerT(size_t size, size_t nextSize, size_t terminalSize, size_t batchSize, + AccumT learningRate, AccumT inferenceRate, AccumT feedback, AccumT lmbda, + ActivationType aType, ActivationType dType, + IComputeBackendT* backend = nullptr); + + ~FullPCLayerT() = default; + + template void BindMemory(ArenaT& arena); + size_t GetRequiredElements() const noexcept; + void RandomizeWeights(std::mt19937& seedGenerator, const char* distribution = "normal") noexcept; + + std::map> GetStateDict() const; + + void SetLayerAbove(FullPCLayerT* l) noexcept + { + layerAbove = l; + } + void SetLayerBelow(FullPCLayerT* l) noexcept + { + layerBelow = l; + } + void SetTerminalLayer(FullPCLayerT* l) noexcept + { + terminalLayer = l; + } + + void SetMuPCScale(AccumT a) noexcept + { + this->a = a; + } + AccumT GetMuPCScale() const noexcept + { + return a; + } + + void SetMuPCInit(bool enabled) noexcept + { + useMuPCInit = enabled; + } + bool GetMuPCInit() const noexcept + { + return useMuPCInit; + } + + void SetResidual(bool enabled) noexcept + { + useResidual = enabled; + } + bool GetResidual() const noexcept + { + return useResidual; + } + + void SetCrossEntropy(bool enabled) noexcept + { + useCrossEntropy = enabled; + } + bool GetCrossEntropy() const noexcept + { + return useCrossEntropy; + } + + /// @brief Always computes real energy; not a Layer:: override (see + /// this file's own class comment on why FullPCLayerT no longer + /// derives from Layer). + AccumT CalculateState() noexcept + { + return CalculateState(true); + } + AccumT CalculateState(bool needEnergy) noexcept; + + void ComputeMuOnly() noexcept; + void UpdateState() noexcept; + void UpdateWeights() noexcept; + void DirectFeedbackUpdate() noexcept; + + void ReconstructBelief() noexcept; + void ComputeAdjoint(const T* adjointAbove, AccumT adjointAboveScale = AccumT(1)) noexcept; + void UpdateErrorEPC() noexcept; + const T* GetAdjoint() const noexcept + { + return adjoint; + } + + void ClampState(const std::vector& inputData) noexcept; + void UnclampState() noexcept; + void ResetState() noexcept; + + bool IsClamped() const noexcept + { + return isClamped; + } + + void InvalidateMuCache() noexcept + { + muCacheValid = false; + } + + void SetMomentum(bool enabled, AccumT beta = AccumT(0.9)) noexcept + { + useMomentum = enabled; + momentumBeta = beta; + } + bool GetMomentum() const noexcept + { + return useMomentum; + } + AccumT GetMomentumBeta() const noexcept + { + return momentumBeta; + } + + void SetUseEPC(bool enabled) noexcept + { + useEPC = enabled; + } + bool GetUseEPC() const noexcept + { + return useEPC; + } + + void SetOptimizer(OptimizerType o) noexcept + { + opt = o; + } + void SetPsiOptimizer(OptimizerType o) noexcept + { + optPsi = o; + } + void SetLearningRate(AccumT learningRate) noexcept; + void SetInferenceRate(AccumT inferenceRate) noexcept + { + ir = inferenceRate; + } + void SetFeedbackRate(AccumT feedbackRate) noexcept; + void SetLambda(AccumT lmbda) noexcept + { + this->lmbda = lmbda; + } + + T* GetBeliefs() noexcept + { + return z; + } + const T* GetErrors() const noexcept + { + return e; + } + const T* GetMu() const noexcept + { + return mu; + } + const T* GetWeights() const noexcept + { + return W; + } + T* GetWeights() noexcept + { + return W; + } + const T* GetDirectFeedbackWeights() const noexcept + { + return Psi; + } + const T* GetBiases() const noexcept + { + return b; + } + T* GetBiases() noexcept + { + return b; + } + IComputeBackendT* GetBackend() const noexcept + { + return backend.get(); + } + + size_t GetBatchSize() const noexcept + { + return batchSize; + } + size_t GetInputSize() const noexcept + { + return size; + } + size_t GetOutputSize() const noexcept + { + return nextSize; + } + size_t GetTerminalSize() const noexcept + { + return terminalSize; + } +}; +} // namespace Deep diff --git a/include/deepity/layers/Layer.h b/include/deepity/layers/Layer.h index 651df16d..8f0778c3 100644 --- a/include/deepity/layers/Layer.h +++ b/include/deepity/layers/Layer.h @@ -28,6 +28,18 @@ struct TensorDescriptor float* data; ///< Pointer to the tensor's first element. std::vector shape; ///< Tensor's dimensions, e.g. {out, in}. }; + +/// @brief Templated counterpart to TensorDescriptor, for the new +/// templated FullPCLayer/FullPCNetwork's own +/// GetStateDict() (a plain method there, not a Layer override -- see +/// FullPCLayer's own doc comment on why it no longer derives +/// from Layer). +template +struct TensorDescriptorT +{ + T* data; ///< Pointer to the tensor's first element. + std::vector shape; ///< Tensor's dimensions, e.g. {out, in}. +}; /// @brief A deepity layer virtual class. class Layer { diff --git a/include/deepity/networks/FullPCNetworkT.h b/include/deepity/networks/FullPCNetworkT.h new file mode 100644 index 00000000..41f31481 --- /dev/null +++ b/include/deepity/networks/FullPCNetworkT.h @@ -0,0 +1,447 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @file FullPCNetworkT.h + * @brief Templated counterpart to FullPCNetwork, over the same `T` + * (storage) / `AccumT` (optimizer accumulator) pair as FullPCLayerT -- + * see that class's own doc comment. Produced by mechanically retyping + * FullPCNetwork's own `float` members/parameters to `T`/`AccumT`, NOT a + * reimplementation: every method here must stay line-for-line + * equivalent to FullPCNetwork's own method bodies, so that + * FullPCNetworkT reproduces FullPCNetwork bit-exactly. + */ + +namespace Deep +{ +/// @brief Templated counterpart to FullPCNetwork, see this file's own +/// comment for the full rationale. +template +class FullPCNetworkT +{ +public: + /// @brief This network's storage element type, for ModelIO's dtype + /// derivation/checking (see DTypeTag). ModelIO itself actually + /// reads the type off each tensor's own GetStateDict() pointer + /// rather than this typedef, but it documents the network's own + /// element type at the type level, for anyone reading this class. + using ElementType = T; + + /// @brief Constructs an empty network; add layers via AddLayer(), + /// then Compile() before use. + /// @param batchSize Fixed batch size for every layer. + /// @param device Which device this network's layers run on. + explicit FullPCNetworkT(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept + : device(device) + , batchSize(batchSize) + { + backend = CreateBackendT(device); + } + ~FullPCNetworkT() = default; + + FullPCNetworkT(const FullPCNetworkT&) = delete; + FullPCNetworkT& operator=(const FullPCNetworkT&) = delete; + + /// @brief Adds a layer to the network. See FullPCLayerT's own + /// constructor doc comment for each parameter's meaning. + void AddLayer(size_t size, size_t nextSize, size_t terminalSize, AccumT lr, AccumT ir, AccumT fl, + AccumT lmbda, ActivationType aType, ActivationType dType) + { + auto l = std::make_unique>(size, nextSize, terminalSize, batchSize, lr, + ir, fl, lmbda, aType, dType, backend.get()); + + if (!layers.empty()) + { + layers.back()->SetLayerAbove(l.get()); + l->SetLayerBelow(layers.back().get()); + } + + layers.push_back(std::move(l)); + } + + /// @brief Enables muPC's full Table 1 parameterization, see + /// FullPCNetwork::SetUseMuPCScaling()'s own doc comment. + void SetUseMuPCScaling(bool enabled) noexcept + { + useMuPCScaling = enabled; + } + + /// @brief Enables residual/skip connections, see + /// FullPCNetwork::SetUseResidualConnections()'s own doc comment. + void SetUseResidualConnections(bool enabled) noexcept + { + useResidualConnections = enabled; + } + + /// @brief Randomizes the weights (W and Psi) of each layer. + void RandomizeWeights(std::mt19937& rng, const char* distribution = "normal") + { + for (auto& l : layers) + l->RandomizeWeights(rng, distribution); + } + + /// @brief Resets each layer's state without touching learned weights. + void ResetState() noexcept + { + for (auto& l : layers) + l->ResetState(); + } + + /// @brief Clamps the input to the first layer. + void Clamp(const std::vector& input) + { + layers.front()->ClampState(input); + } + + /// @brief Seeds hidden layers from a genuine forward pass through + /// current weights, see FullPCNetwork::ProjectForward()'s own doc comment. + void ProjectForward() noexcept + { + for (size_t i = 0; i + 1 < layers.size(); ++i) + { + layers[i]->ComputeMuOnly(); + + if (layers[i + 1]->IsClamped()) + continue; + + const T* mu = layers[i]->GetMu(); + T* nextZ = layers[i + 1]->GetBeliefs(); + size_t n = layers[i]->GetBatchSize() * layers[i]->GetOutputSize(); + + backend->Copy(nextZ, mu, n); + } + } + + /// @brief Runs the DFA phase on every non-terminal layer. + void DirectFeedbackUpdate() noexcept + { + for (size_t i = 0; i < layers.size() - 1; ++i) + layers[i]->DirectFeedbackUpdate(); + } + + /// @brief Computes the terminal layer's error/energy against its + /// clamped target. + AccumT CalculateTerminalError() noexcept + { + return GetTerminalLayer()->CalculateState(false); + } + + /// @brief Runs one settling step on every layer. + AccumT Step(bool needEnergy = true) noexcept + { + AccumT e(0); + for (auto& l : layers) + e += l->CalculateState(needEnergy); + for (auto& l : layers) + l->UpdateState(); + return needEnergy ? e : AccumT(0); + } + + /// @brief ePC's settling step, see FullPCNetwork::EPCStep()'s own doc comment. + void EPCStep() noexcept + { + layers[0]->ComputeMuOnly(); + for (size_t i = 1; i + 1 < layers.size(); ++i) + { + layers[i]->ReconstructBelief(); + layers[i]->ComputeMuOnly(); + } + GetTerminalLayer()->CalculateState(false); + + const T* adjointAbove = GetTerminalLayer()->GetErrors(); + bool firstHop = true; + for (size_t i = layers.size() - 2; i >= 1; --i) + { + layers[i]->ComputeAdjoint(adjointAbove, firstHop ? AccumT(-1) : AccumT(1)); + firstHop = false; + layers[i]->UpdateErrorEPC(); + adjointAbove = layers[i]->GetAdjoint(); + } + } + + /// @brief Applies weight updates to every layer. + void UpdateWeights() noexcept + { + for (size_t i = 0; i + 1 < layers.size(); i++) + layers[i]->UpdateWeights(); + } + + void SetOptimizer(OptimizerType o) noexcept + { + for (auto& layer : layers) + layer->SetOptimizer(o); + } + void SetPsiOptimizer(OptimizerType o) noexcept + { + for (auto& layer : layers) + layer->SetPsiOptimizer(o); + } + void SetLearningRate(AccumT lr) noexcept + { + for (auto& layer : layers) + layer->SetLearningRate(lr); + } + void SetFeedbackRate(AccumT fl) noexcept + { + for (auto& layer : layers) + layer->SetFeedbackRate(fl); + } + + FullPCLayerT* GetTerminalLayer() noexcept + { + return layers.back().get(); + } + std::vector>>& GetLayers() noexcept + { + return layers; + } + const std::vector>>& GetLayers() const noexcept + { + return layers; + } + int GetBatchSize() const noexcept + { + return batchSize; + } + DeviceType GetDevice() const noexcept + { + return device; + } + + /// @brief Full train step: reset, clamp input+target, run all four + /// DKP-PC phases in order, unclamp. See FullPCNetwork::TrainStep()'s + /// own doc comment. CUDA graph capture/replay is not reachable here: + /// device==DEVICE_GPU already throws inside CreateBackendT() at + /// construction (no CUDABackendT yet), so this just runs the + /// non-graph path unconditionally. + AccumT TrainStep(const std::vector& x, const std::vector& y, int inferenceSteps = 1) + { + ResetState(); + Clamp(x); + GetTerminalLayer()->ClampState(y); + + auto settleStep = [this]() + { + if (useEPC) + EPCStep(); + else + Step(false); + if (useIPC) + { + UpdateWeights(); + for (auto& l : layers) + l->InvalidateMuCache(); + } + }; + + ProjectForward(); + CalculateTerminalError(); + DirectFeedbackUpdate(); + for (int t = 0; t < inferenceSteps; t++) + settleStep(); + if (!useIPC) + UpdateWeights(); + + AccumT finalEnergy(0); + for (auto& l : layers) + finalEnergy += l->CalculateState(true); + + GetTerminalLayer()->UnclampState(); + return finalEnergy; + } + + /// @brief Forward prediction: clamp input, settle with no DFA + /// perturbation and no target clamped, read the terminal's beliefs. + /// See FullPCNetwork::Predict()'s own doc comment; same CUDA-graph + /// caveat as TrainStep() above. + std::vector Predict(const std::vector& x, int inferenceSteps) + { + ResetState(); + Clamp(x); + + auto settleStep = [this]() + { + if (useEPC) + EPCStep(); + else + Step(); + }; + + FullPCLayerT* terminal = GetTerminalLayer(); + + ProjectForward(); + for (int t = 0; t < inferenceSteps; t++) + settleStep(); + if (terminal->GetCrossEntropy()) + { + FullPCLayerT* below = layers[layers.size() - 2].get(); + below->ComputeMuOnly(); + size_t n = below->GetBatchSize() * below->GetOutputSize(); + backend->Copy(terminal->GetBeliefs(), below->GetMu(), n); + } + + const T* beliefs = terminal->GetBeliefs(); + size_t count = terminal->GetBatchSize() * terminal->GetInputSize(); + + std::vector result(count); + backend->CopyToHost(result.data(), beliefs, count); + return result; + } + + /// @brief Loads all layers into one contiguous block of memory, + /// see FullPCNetwork::Compile()'s own doc comment. + void Compile() + { + if (!layers.empty() && layers.back()->GetInputSize() != layers.back()->GetTerminalSize()) + { + throw std::invalid_argument( + "FullPCNetworkT::Compile(): terminalSize (" + + std::to_string(layers.back()->GetTerminalSize()) + + ") passed to AddLayer() doesn't match the actual terminal layer's size (" + + std::to_string(layers.back()->GetInputSize()) + ")."); + } + +#pragma omp parallel + { + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON); + } + size_t total_elements_needed = 0; + for (auto& layer : layers) + total_elements_needed += layer->GetRequiredElements(); + + if (device == DeviceType::DEVICE_CPU) + { + cpuArena = std::make_unique>(total_elements_needed, false); + for (auto& layer : layers) + { + layer->BindMemory(*cpuArena); + layer->SetTerminalLayer(layers.back().get()); + } + } +#if defined(DEEPITY_USE_CUDA) + else + { + backend->PrepareForBatchSize(batchSize); + gpuArena = std::make_unique>(backend.get(), + total_elements_needed); + for (auto& layer : layers) + { + layer->BindMemory(*gpuArena); + layer->SetTerminalLayer(layers.back().get()); + } + } +#endif + + if (!useMuPCScaling && !useResidualConnections) + return; + + const int L = (int)layers.size() - 1; + const int H = L - 1; + + if (L < 1) + return; + + if (useMuPCScaling) + { + const AccumT d_in = AccumT(layers[0]->GetInputSize()); + + AccumT N; + if (H > 0) + { + N = AccumT(0); + for (int i = 0; i < L - 1; ++i) + N = (std::max)(N, AccumT(layers[i]->GetOutputSize())); + } + else + { + N = AccumT(layers[L - 1]->GetOutputSize()); + } + + for (int l = 1; l <= L; ++l) + { + AccumT a; + if (l == 1) + a = std::pow(d_in, AccumT(-0.5)); + else if (l == L) + a = AccumT(1) / N; + else + a = std::pow(N * AccumT(L), AccumT(-0.5)); + + layers[l - 1]->SetMuPCScale(a); + layers[l - 1]->SetMuPCInit(true); + } + } + + if (useResidualConnections) + { + for (int l = 2; l <= H; ++l) + { + FullPCLayerT* layer = layers[l - 1].get(); + if (layer->GetInputSize() != layer->GetOutputSize()) + { + throw std::invalid_argument( + "FullPCNetworkT::Compile(): residual connections require " + "matching width, layer index " + + std::to_string(l - 1) + " has input size " + std::to_string(layer->GetInputSize()) + + " but output size " + std::to_string(layer->GetOutputSize()) + "."); + } + layer->SetResidual(true); + } + } + } + + void SetUseIPC(bool enabled) noexcept + { + useIPC = enabled; + } + + void SetUseMomentum(bool enabled, AccumT beta = AccumT(0.9)) noexcept + { + for (auto& layer : layers) + layer->SetMomentum(enabled, beta); + } + + void SetUseCrossEntropy(bool enabled) noexcept + { + GetTerminalLayer()->SetCrossEntropy(enabled); + } + + void SetUseEPC(bool enabled) noexcept + { + useEPC = enabled; + for (size_t i = 1; i + 1 < layers.size(); ++i) + layers[i]->SetUseEPC(enabled); + } + +private: + std::vector>> layers; + + std::unique_ptr> backend; + DeviceType device; + + std::unique_ptr> cpuArena; +#if defined(DEEPITY_USE_CUDA) + std::unique_ptr> gpuArena; +#endif + + int batchSize; + bool useIPC = false; + bool useEPC = false; + + bool useMuPCScaling = false; + bool useResidualConnections = false; +}; +} // namespace Deep diff --git a/include/deepity/utils/Activations.h b/include/deepity/utils/Activations.h index 21571dcb..50bca0f6 100644 --- a/include/deepity/utils/Activations.h +++ b/include/deepity/utils/Activations.h @@ -163,4 +163,84 @@ static inline float ActivationDerivativeFromActivatedScalar(ActivationType dType return 1.0f; } } + +/// @brief Generic (non-SIMD) counterpart to ActivationScalar(), over any +/// floating-point T -- std::tanh/std::exp instead of Sleef's float-only +/// intrinsics, so this is genuinely correct (not truncated through +/// float) for T=double. Used only by CPUBackendT's non-float +/// elementwise fallback path (see CPUBackendT.h); the original +/// float-only ActivationScalar() above is untouched and still what +/// CPUBackend's existing Highway-dispatched paths use for T=float. +template +static inline T ActivationScalarT(ActivationType type, T z) noexcept +{ + switch (type) + { + case ActivationType::RELU: + return z > T(0) ? z : T(0); + case ActivationType::GELU: + { + constexpr T c1 = T(0.7978845608028654), c2 = T(0.044715); + T zsq = z * z; + T inner = c1 * z + (c2 * c1) * zsq * z; + T t = std::tanh(inner); + return T(0.5) * z * (T(1) + t); + } + case ActivationType::SIGMOID: + return T(1) / (T(1) + std::exp(-z)); + case ActivationType::eSIGMOID: + return T(0.5) + T(0.5) * (z / (T(1) + std::fabs(z))); + case ActivationType::TANH: + return std::tanh(z); + case ActivationType::HARD_TANH: + return std::clamp(z, T(-1), T(1)); + case ActivationType::LINEAR: + case ActivationType::NONE: + default: + return z; + } +} + +/// @brief Generic (non-SIMD) counterpart to ActivationDerivativeScalar(), +/// see ActivationScalarT()'s own doc comment for why this exists +/// alongside (not instead of) the original float-only version. +template +static inline T ActivationDerivativeScalarT(ActivationType dType, T z) noexcept +{ + switch (dType) + { + case ActivationType::dRELU: + return (z > T(0)) ? T(1) : T(0); + case ActivationType::dGELU: + { + constexpr T c1 = T(0.7978845608028654), c2 = T(0.044715); + T zsq = z * z; + T inner = c1 * z + (c2 * c1) * zsq * z; + T t = std::tanh(inner); + T gprime = c1 + (T(3) * c2 * c1) * zsq; + return T(0.5) * (T(1) + t) + T(0.5) * z * gprime * (T(1) - t * t); + } + case ActivationType::dSIGMOID: + { + T sig = T(1) / (T(1) + std::exp(-z)); + return sig * (T(1) - sig); + } + case ActivationType::d_eSIGMOID: + { + T a = T(1) + std::fabs(z); + return T(0.5) / (a * a); + } + case ActivationType::dTANH: + { + T t = std::tanh(z); + return T(1) - t * t; + } + case ActivationType::dHARD_TANH: + return (z > T(-1) && z < T(1)) ? T(1) : T(0); + case ActivationType::dLINEAR: + case ActivationType::NONE: + default: + return T(1); + } +} } // namespace Deep diff --git a/include/deepity/utils/DTypeTag.h b/include/deepity/utils/DTypeTag.h new file mode 100644 index 00000000..ffed1c60 --- /dev/null +++ b/include/deepity/utils/DTypeTag.h @@ -0,0 +1,75 @@ +#pragma once +#include +#include + +/** + * @file DTypeTag.h + * @brief Maps a C++ storage type to its safetensors-style dtype string, + * for ModelIO's save/load -- see that class's own doc comment on why + * the dtype now gets authentically derived instead of hardcoded "F32". + * Add a new specialization here as each new templated T lands (half, + * the FP4/posit formats, ...); ModelIO itself needs no changes to pick + * it up, since it reads this trait off the type it already has in hand + * (a layer's GetStateDict() tensor's own element type). + */ + +namespace Deep +{ +template struct DTypeTag; + +template <> struct DTypeTag +{ + static constexpr const char* Name() noexcept + { + return "F32"; + } +}; + +template <> struct DTypeTag +{ + static constexpr const char* Name() noexcept + { + return "F64"; + } +}; + +template <> struct DTypeTag +{ + static constexpr const char* Name() noexcept + { + return "FP4_E2M1"; + } +}; + +template <> struct DTypeTag +{ + static constexpr const char* Name() noexcept + { + return "F16"; + } +}; + +template <> struct DTypeTag +{ + static constexpr const char* Name() noexcept + { + return "POSIT4_0"; + } +}; + +template <> struct DTypeTag +{ + static constexpr const char* Name() noexcept + { + return "POSIT4_1"; + } +}; + +template <> struct DTypeTag +{ + static constexpr const char* Name() noexcept + { + return "POSIT4_2"; + } +}; +} // namespace Deep diff --git a/include/deepity/utils/DeviceMemoryArena.h b/include/deepity/utils/DeviceMemoryArena.h index ace5a691..39ff9f17 100644 --- a/include/deepity/utils/DeviceMemoryArena.h +++ b/include/deepity/utils/DeviceMemoryArena.h @@ -36,43 +36,53 @@ namespace Deep { /// @brief A 64-byte-aligned bump-pointer allocator over a single /// contiguous block of device memory, obtained via a supplied -/// IComputeBackend (expected to be a CUDABackend in practice, though -/// this class itself doesn't hard-require that specific type). -class DeviceMemoryArena +/// IComputeBackendT (expected to be a CUDABackend/ +/// CUDABackendT in practice, though this class itself doesn't +/// hard-require that specific type). +/// +/// @tparam T Element type this arena allocates, matching the backend's +/// own T. `DeviceMemoryArena` (see the alias below) is +/// `DeviceMemoryArenaT`, bit-for-bit identical to this +/// class before it was templated. +/// @tparam AccumT Matches the backend's own AccumT (only used here to +/// name the exact IComputeBackendT this arena's backend +/// pointer must be); defaults to T, as in IComputeBackendT itself. +template +class DeviceMemoryArenaT { private: /// @brief Backend used for the underlying allocation and its /// eventual release. Non-owning, must outlive this arena. - IComputeBackend* backend; + IComputeBackendT* backend; /// @brief Base address of the underlying backend-allocated buffer. - float* base_ptr; + T* base_ptr; /// @brief Total capacity of the arena, in bytes, rounded up to a /// multiple of 64. size_t capacity_bytes; /// @brief Current allocation offset from base_ptr, in bytes. - /// Advances monotonically with each AllocateFloats() call and is + /// Advances monotonically with each Allocate() call and is /// never reset or reclaimed. size_t offset_bytes; public: /// @brief Allocates a single 64-byte-aligned device buffer large - /// enough to hold `total_floats` floats, rounded up to the + /// enough to hold `total_elements` elements of T, rounded up to the /// nearest 64-byte boundary. /// @param backend The backend to allocate from (and later free - /// through). Must outlive this DeviceMemoryArena. Passing a + /// through). Must outlive this DeviceMemoryArenaT. Passing a /// CPUBackend here would allocate host memory under a class /// named "device", that's a caller contract, not something /// this class checks at runtime, mirroring Tensor's own /// backend/device pairing contract. - /// @param total_floats Total number of floats this arena can - /// hand out across all future AllocateFloats() calls combined. + /// @param total_elements Total number of elements this arena can + /// hand out across all future Allocate() calls combined. /// @throws std::bad_alloc if the underlying allocation fails. - DeviceMemoryArena(IComputeBackend* backend, size_t total_floats) + DeviceMemoryArenaT(IComputeBackendT* backend, size_t total_elements) : backend(backend) { - capacity_bytes = (total_floats * sizeof(float) + 63) & ~(size_t)63; + capacity_bytes = (total_elements * sizeof(T) + 63) & ~(size_t)63; - base_ptr = backend->Allocate(capacity_bytes / sizeof(float)); + base_ptr = backend->Allocate(capacity_bytes / sizeof(T)); if (!base_ptr) { throw std::bad_alloc(); @@ -82,7 +92,7 @@ class DeviceMemoryArena /// @brief Frees the underlying device buffer via the same /// backend it was allocated from. - ~DeviceMemoryArena() + ~DeviceMemoryArenaT() { if (base_ptr) { @@ -92,39 +102,48 @@ class DeviceMemoryArena // Delete copy/move constructors to prevent double-free corruption. - /// @brief Deleted: DeviceMemoryArena owns a single device + /// @brief Deleted: DeviceMemoryArenaT owns a single device /// allocation, so copying would risk a double-free. - DeviceMemoryArena(const DeviceMemoryArena&) = delete; - /// @brief Deleted: DeviceMemoryArena owns a single device + DeviceMemoryArenaT(const DeviceMemoryArenaT&) = delete; + /// @brief Deleted: DeviceMemoryArenaT owns a single device /// allocation, so copy-assignment would risk a double-free. - DeviceMemoryArena& operator=(const DeviceMemoryArena&) = delete; + DeviceMemoryArenaT& operator=(const DeviceMemoryArenaT&) = delete; - /// @brief Allocates a 64-byte aligned chunk of floats from the + /// @brief Allocates a 64-byte aligned chunk of elements from the /// arena. - /// @param num_floats Number of floats to allocate from the + /// @param num_elements Number of elements to allocate from the /// arena. The actual reservation is rounded up to the nearest - /// 64-byte boundary, matching MemoryArena's own guarantee. + /// 64-byte boundary, matching MemoryArenaT's own guarantee. /// @return Pointer to the start of the allocated chunk, valid - /// for the lifetime of this DeviceMemoryArena. 64-byte aligned. + /// for the lifetime of this DeviceMemoryArenaT. 64-byte aligned. /// @throws std::runtime_error if the requested allocation would /// exceed the arena's total capacity. /// @warning Individual chunks are never freed independently; the /// entire arena is released at once in the destructor. - float* AllocateFloats(size_t num_floats) + T* Allocate(size_t num_elements) { - size_t allocation_size = (num_floats * sizeof(float) + 63) & ~(size_t)63; + size_t allocation_size = (num_elements * sizeof(T) + 63) & ~(size_t)63; if (offset_bytes + allocation_size > capacity_bytes) { - throw std::runtime_error("Fatal: DeviceMemoryArena capacity exceeded during allocation."); + throw std::runtime_error("Fatal: DeviceMemoryArenaT capacity exceeded during allocation."); } - float* chunk = reinterpret_cast(reinterpret_cast(base_ptr) + offset_bytes); + T* chunk = reinterpret_cast(reinterpret_cast(base_ptr) + offset_bytes); offset_bytes += allocation_size; return chunk; } + /// @brief Back-compat alias for Allocate(), kept for every existing + /// (non-templated) caller written against the old AllocateFloats() + /// name. Only meaningful when T=float; new templated code should + /// call Allocate() directly. + T* AllocateFloats(size_t num_elements) + { + return Allocate(num_elements); + } + /// @brief Returns how many bytes have been allocated from the /// arena so far. size_t GetUsedBytes() const @@ -137,5 +156,10 @@ class DeviceMemoryArena return capacity_bytes; } }; + +/// @brief Non-templated alias preserving every existing caller's +/// behavior exactly: `DeviceMemoryArena` is `DeviceMemoryArenaT`, bit-for-bit identical to this class before it was templated. +using DeviceMemoryArena = DeviceMemoryArenaT; } // namespace Deep #endif \ No newline at end of file diff --git a/include/deepity/utils/LayerBindHelpers.h b/include/deepity/utils/LayerBindHelpers.h index 4cdff398..50f39a79 100644 --- a/include/deepity/utils/LayerBindHelpers.h +++ b/include/deepity/utils/LayerBindHelpers.h @@ -1,7 +1,9 @@ #pragma once #include +#include #include #include +#include #include #include @@ -60,4 +62,55 @@ inline void ReleaseLocalArenaIfForeign(std::unique_ptr& localArena, localArena.reset(); } } + +/// @brief Templated counterpart to BindAdamState(), for FullPCLayer: `grad`/`m`/`v` are AccumT-typed (the optimizer's own wider +/// accumulator, see IComputeBackendT's own doc comment), not +/// T-typed like the layer's forward-path buffers -- so, unlike those, +/// they're never allocated from a layer's T-typed arena (a +/// MemoryArenaT's Allocate() returns T*, which only happens to equal +/// AccumT* when T==AccumT; for a genuinely different pair, e.g. +/// Fp4E2M1Value's storage vs. its float accumulator, it's a flat type +/// mismatch). Each one is instead a `std::vector` the layer +/// owns directly. Valid for a host-resident backend only (every +/// templated backend today); a future CUDABackendT would +/// need its own device-resident accumulator storage instead of a host +/// std::vector. +template +inline void BindAdamStateT(size_t paramSize, std::vector& grad, std::vector& m, + std::vector& v) +{ + grad.assign(paramSize, AccumT(0)); + m.assign(paramSize, AccumT(0)); + v.assign(paramSize, AccumT(0)); +} + +/// @brief Templated counterpart to BindLearningRateState(), for +/// FullPCLayer. Same host-only-AccumT-scratch reasoning as +/// BindAdamStateT() above -- `lr`/`timestep` are plain scalar members, +/// not arena-allocated pointers, with their address taken at each +/// backend call site that needs a `const AccumT*`/`const int*`. +template +inline void BindLearningRateStateT(int& timestep, AccumT& lrState, AccumT lr) +{ + timestep = 0; + lrState = lr; +} + +/// @brief Templated counterpart to ReleaseLocalArenaIfForeign(), for +/// FullPCLayer's own MemoryArenaT local arena. +template +inline void ReleaseLocalArenaIfForeignT(std::unique_ptr>& localArena, + ArenaT& arena) +{ + if constexpr (std::is_same_v>) + { + if (localArena && localArena.get() != &arena) + localArena.reset(); + } + else + { + localArena.reset(); + } +} } // namespace Deep diff --git a/include/deepity/utils/MemoryArena.h b/include/deepity/utils/MemoryArena.h index 59da48cf..bd287e0f 100644 --- a/include/deepity/utils/MemoryArena.h +++ b/include/deepity/utils/MemoryArena.h @@ -18,12 +18,12 @@ * (see the "Contiguous Arena Allocator" design note in the project * README). Host-side counterpart to DeviceMemoryArena.h's CUDA version. * - * @note Individual chunks handed out by AllocateFloats() cannot be freed + * @note Individual chunks handed out by Allocate() cannot be freed * independently, the entire arena is released at once in the * destructor, matching the bump-pointer/no-reclaim design used * throughout this codebase's memory arenas. - * @version 1.0 - * @date 2026-06-30 + * @version 1.1 + * @date 2026-10-26 * @author Jack Rose */ @@ -134,19 +134,26 @@ inline void HugePageFreePortable(void* ptr, size_t size) } // namespace detail /// @brief A 64-byte-aligned bump-pointer allocator over a single -/// contiguous block of host memory. Every chunk handed out via -/// AllocateFloats() is itself 64-byte aligned, satisfying this -/// codebase's strict alignment requirement for AVX2/AVX-512 loads. -class MemoryArena +/// contiguous block of host memory, generic over the element type `T` +/// it hands out. Every chunk handed out via Allocate() is itself +/// 64-byte aligned, satisfying this codebase's strict alignment +/// requirement for AVX2/AVX-512 loads. +/// +/// @tparam T Element type this arena allocates. `MemoryArena` (see the +/// alias below) is `MemoryArenaT`, preserving every existing +/// non-templated caller's behavior exactly; new templated code +/// (FullPCLayer/FullPCNetwork) uses MemoryArenaT directly. +template +class MemoryArenaT { private: /// @brief Base address of the underlying aligned allocation. - float* base_ptr; + T* base_ptr; /// @brief Total capacity of the arena, in bytes, rounded up to a /// multiple of 64. size_t capacity_bytes; /// @brief Current allocation offset from base_ptr, in bytes. - /// Advances monotonically with each AllocateFloats() call and is + /// Advances monotonically with each Allocate() call and is /// never reset or reclaimed. size_t offset_bytes; /// @brief True if base_ptr was obtained via HugePageAllocPortable() @@ -158,10 +165,10 @@ class MemoryArena public: /// @brief Allocates a single 64-byte-aligned buffer large enough - /// to hold `total_floats` floats, rounded up to the nearest - /// 64-byte boundary. - /// @param total_floats Total number of floats this arena can - /// hand out across all future AllocateFloats() calls combined. + /// to hold `total_elements` elements of type T, rounded up to the + /// nearest 64-byte boundary. + /// @param total_elements Total number of elements this arena can + /// hand out across all future Allocate() calls combined. /// @param use_huge_pages Opt-in, best-effort request for /// huge-page-backed memory (see HugePageAllocPortable() for the /// platform-specific caveats). Defaults to false, preserving @@ -172,16 +179,16 @@ class MemoryArena /// @throws std::bad_alloc if the underlying allocation fails /// (including the fallback path, if huge pages were requested /// but unavailable). - explicit MemoryArena(size_t total_floats, bool use_huge_pages = false) + explicit MemoryArenaT(size_t total_elements, bool use_huge_pages = false) : used_huge_pages(false) { // Guarantee total capacity is a multiple of 64 bytes - capacity_bytes = (total_floats * sizeof(float) + 63) & ~63; + capacity_bytes = (total_elements * sizeof(T) + 63) & ~size_t(63); if (use_huge_pages) { bool succeeded = false; - base_ptr = static_cast(detail::HugePageAllocPortable(capacity_bytes, succeeded)); + base_ptr = static_cast(detail::HugePageAllocPortable(capacity_bytes, succeeded)); used_huge_pages = succeeded; } @@ -192,7 +199,7 @@ class MemoryArena // system lacks the required configuration/privilege), // fall back to the standard aligned allocator rather // than treating this as a hard failure. - base_ptr = static_cast(detail::AlignedAllocPortable(64, capacity_bytes)); + base_ptr = static_cast(detail::AlignedAllocPortable(64, capacity_bytes)); } if (!base_ptr) @@ -205,7 +212,7 @@ class MemoryArena /// @brief Frees the underlying allocation, via whichever /// deallocator matches the allocator actually used (see /// used_huge_pages). - ~MemoryArena() + ~MemoryArenaT() { if (base_ptr) { @@ -218,40 +225,50 @@ class MemoryArena // Delete copy/move constructors to prevent double-free corruption - /// @brief Deleted: MemoryArena owns a single aligned allocation, + /// @brief Deleted: MemoryArenaT owns a single aligned allocation, /// so copying would risk a double-free. - MemoryArena(const MemoryArena&) = delete; - /// @brief Deleted: MemoryArena owns a single aligned allocation, + MemoryArenaT(const MemoryArenaT&) = delete; + /// @brief Deleted: MemoryArenaT owns a single aligned allocation, /// so copy-assignment would risk a double-free. - MemoryArena& operator=(const MemoryArena&) = delete; + MemoryArenaT& operator=(const MemoryArenaT&) = delete; - /// @brief Allocates a 64-byte aligned chunk of floats from the arena - /// @param num_floats Number of floats to allocate from the arena. - /// The actual reservation is rounded up to the nearest 64-byte - /// boundary, so the returned chunk may have more usable capacity - /// than requested. + /// @brief Allocates a 64-byte aligned chunk of `num_elements` + /// elements of type T from the arena. + /// @param num_elements Number of elements to allocate from the + /// arena. The actual reservation is rounded up to the nearest + /// 64-byte boundary, so the returned chunk may have more usable + /// capacity than requested. /// @return Pointer to the start of the allocated chunk, valid for - /// the lifetime of this MemoryArena. 64-byte aligned. + /// the lifetime of this MemoryArenaT. 64-byte aligned. /// @throws std::runtime_error if the requested allocation would /// exceed the arena's total capacity. /// @warning Individual chunks are never freed independently; the /// entire arena is released at once in the destructor. - float* AllocateFloats(size_t num_floats) + T* Allocate(size_t num_elements) { // Calculate how many bytes we need, padded to the nearest 64-byte boundary - size_t allocation_size = (num_floats * sizeof(float) + 63) & ~63; + size_t allocation_size = (num_elements * sizeof(T) + 63) & ~size_t(63); if (offset_bytes + allocation_size > capacity_bytes) { - throw std::runtime_error("Fatal: MemoryArena capacity exceeded during allocation."); + throw std::runtime_error("Fatal: MemoryArenaT capacity exceeded during allocation."); } - float* chunk = reinterpret_cast(reinterpret_cast(base_ptr) + offset_bytes); + T* chunk = reinterpret_cast(reinterpret_cast(base_ptr) + offset_bytes); offset_bytes += allocation_size; return chunk; } + /// @brief Back-compat alias for Allocate(), kept for every existing + /// (non-templated) caller written against the old AllocateFloats() + /// name. Only meaningful when T=float; new templated code should + /// call Allocate() directly. + T* AllocateFloats(size_t num_elements) + { + return Allocate(num_elements); + } + /// @brief Returns how many bytes have been allocated from the /// arena so far. /// @return The current offset, in bytes. @@ -269,7 +286,7 @@ class MemoryArena /// huge-page-backed. Since huge pages are requested on a /// best-effort basis, this can be false even if use_huge_pages /// was passed to the constructor, if the request wasn't honored - /// (system lacks configuration/privilege, or the allocation + /// (system lacks configuration/privilege), or the allocation /// call failed) and the arena silently fell back to the /// standard aligned allocator instead. bool UsedHugePages() const @@ -277,4 +294,9 @@ class MemoryArena return used_huge_pages; } }; + +/// @brief Non-templated alias preserving every existing caller's +/// behavior exactly: `MemoryArena` is `MemoryArenaT`, bit-for-bit +/// identical to this class before it was templated. +using MemoryArena = MemoryArenaT; } // namespace Deep diff --git a/include/deepity/utils/ModelIO.h b/include/deepity/utils/ModelIO.h index 0b59d723..fc5dfef8 100644 --- a/include/deepity/utils/ModelIO.h +++ b/include/deepity/utils/ModelIO.h @@ -1,13 +1,25 @@ #pragma once #include +#include #include #include +#include #include +#include /** * @file ModelIO.h * @brief Saves/loads a network's compiled weight arena to/from disk, as a - * JSON header (per-layer shapes/offsets) plus a raw binary float blob. + * JSON header (per-layer shapes/offsets/dtypes) plus a raw binary data + * blob. Templated on NetworkType the same as before (works across every + * PC network variant that exposes GetLayers()), but the element type + * backing each tensor is no longer assumed to be `float`: Save() derives + * it from each tensor's own `.data` pointer (via DTypeTag), so a + * FullPCNetworkT writes real "F64" tensors at their real + * 8-byte stride, not truncated through float. Load() checks that + * dtype against what the CALLING network's own tensors expect and + * throws std::runtime_error on a mismatch -- strictly new behavior: + * the previous version never checked "dtype" on read at all. */ namespace Deep @@ -29,6 +41,7 @@ class ModelIO nlohmann::json header; size_t current_offset = 0; int layer_idx = 0; + std::string fileDtype = "F32"; // stays "F32" for an empty network, matching old behavior. for (const auto& layerPtr : net.GetLayers()) { @@ -36,6 +49,9 @@ class ModelIO for (const auto& [name, tensor] : state_dict) { + using ElementType = std::remove_pointer_t; + fileDtype = DTypeTag::Name(); + std::string key = "layer_" + std::to_string(layer_idx) + "." + name; size_t num_elements = 1; @@ -43,9 +59,9 @@ class ModelIO { num_elements *= dim; } - size_t byte_size = num_elements * sizeof(float); + size_t byte_size = num_elements * sizeof(ElementType); - header[key] = {{"dtype", "F32"}, + header[key] = {{"dtype", fileDtype}, {"shape", tensor.shape}, {"data_offsets", {current_offset, current_offset + byte_size}}}; current_offset += byte_size; @@ -53,7 +69,7 @@ class ModelIO layer_idx++; } - header["__metadata__"] = {{"format", "deepity_safetensors"}}; + header["__metadata__"] = {{"format", "deepity_safetensors"}, {"dtype", fileDtype}}; std::string header_str = header.dump(); uint64_t header_size = header_str.size(); @@ -69,12 +85,13 @@ class ModelIO { for (const auto& [name, tensor] : layerPtr->GetStateDict()) { + using ElementType = std::remove_pointer_t; size_t num_elements = 1; for (size_t dim : tensor.shape) { num_elements *= dim; } - out.write(reinterpret_cast(tensor.data), num_elements * sizeof(float)); + out.write(reinterpret_cast(tensor.data), num_elements * sizeof(ElementType)); } } return true; @@ -106,10 +123,20 @@ class ModelIO auto state_dict = layerPtr->GetStateDict(); for (auto& [name, tensor] : state_dict) { + using ElementType = std::remove_pointer_t; std::string key = "layer_" + std::to_string(layer_idx) + "." + name; if (header.contains(key)) { + std::string fileDtype = header[key].value("dtype", std::string()); + const char* expectedDtype = DTypeTag::Name(); + if (!fileDtype.empty() && fileDtype != expectedDtype) + { + throw std::runtime_error("ModelIO::Load: dtype mismatch for '" + key + "' -- file " + "has " + + fileDtype + ", network expects " + expectedDtype + "."); + } + size_t start_offset = header[key]["data_offsets"][0]; size_t end_offset = header[key]["data_offsets"][1]; size_t byte_size = end_offset - start_offset; diff --git a/include/deepity/utils/QuantFormat.h b/include/deepity/utils/QuantFormat.h new file mode 100644 index 00000000..50af3b11 --- /dev/null +++ b/include/deepity/utils/QuantFormat.h @@ -0,0 +1,289 @@ +#pragma once +#include +#include +#include + +/** + * @file QuantFormat.h + * @brief Sub-8-bit storage types for the quantization plan's Phase 2+: + * each format is its own 1-byte struct (no bit-packing yet, see the + * plan's own "Known, accepted catch" section), with a free Decode/Encode + * pair using the paper's grid table. No native CPU/GPU ALU exists for + * these, so every use goes through a decode-to-AccumT/compute/encode- + * back-to-T wrap -- CPUBackendT's own generic (non-float/double) code + * paths already do exactly that (every op decodes via `AccumT(x)` + * before computing and re-encodes via `T(result)` afterward), so these + * structs only need to support that round-trip conversion themselves, + * not any arithmetic of their own. + * + * Four separate structs are planned here (Fp4E2M1Value first, then the + * three posit variants), not a `template` -- each + * format needs its own hand-tuned grid, and with only four callers a + * shared abstraction isn't worth it (see the plan's own reasoning). + */ + +namespace Deep +{ +namespace detail +{ +/// @brief FP4 E2M1's 8 non-negative magnitude levels (1 sign bit + 2 +/// exponent bits + 1 mantissa bit, exponent bias 1): index 0 is the +/// denormal/zero row (0, 0.5), indices 1-3 are exponent 01 (1, 1.5), +/// indices 4-5 are exponent 10 (2, 3), indices 6-7 are exponent 11 (4, +/// 6) -- i.e. the magnitude bits read directly as a 0-7 table index. +inline constexpr float kFp4E2M1Magnitudes[8] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f}; + +/// @brief Posit<4,0>'s 8 non-negative magnitude levels (es=0: no +/// exponent bits, so all 3 remaining bits go to the mantissa -- the +/// most uniform spacing near 1.0 of the three posit variants, at the +/// cost of the narrowest dynamic range). +inline constexpr float kPosit4_0Magnitudes[8] = {0.0f, 0.25f, 0.5f, 0.75f, 1.0f, 1.5f, 2.0f, 4.0f}; + +/// @brief Posit<4,1>'s 8 non-negative magnitude levels (es=1: one +/// exponent bit traded away from the mantissa for range -- every level +/// here is a power of two, unlike Posit4_0's). +inline constexpr float kPosit4_1Magnitudes[8] = {0.0f, 0.0625f, 0.25f, 0.5f, 1.0f, 2.0f, 4.0f, + 16.0f}; + +/// @brief Posit<4,2>'s 8 non-negative magnitude levels (es=2: the +/// widest dynamic range of the three, at the cost of almost no +/// resolution near 1.0 -- only {1, 2} fall between its two widest gaps). +inline constexpr float kPosit4_2Magnitudes[8] = {0.0f, 1.0f / 256.0f, 0.0625f, 0.125f, 1.0f, 2.0f, + 16.0f, 256.0f}; +} // namespace detail + +/// @brief Decodes an Fp4E2M1Value's raw 4-bit pattern (bit 3 = sign, +/// bits 2:0 = magnitude index into kFp4E2M1Magnitudes) to its real value. +inline float DecodeFp4E2M1(uint8_t bits) noexcept +{ + bool negative = (bits & 0x8) != 0; + float magnitude = detail::kFp4E2M1Magnitudes[bits & 0x7]; + return negative ? -magnitude : magnitude; +} + +/// @brief Encodes a float to the nearest Fp4E2M1Value grid point +/// (round-to-nearest over the 8 magnitude levels, saturating at ±6 -- +/// the format's max representable magnitude -- for anything beyond +/// it). A genuine zero always encodes as bits=0 (never the sign-bit-set +/// "negative zero" pattern), so every all-zero buffer -- e.g. from +/// MemoryArenaT's own Zero()/memset -- already is the +/// correct zero value with no separate init needed. +inline uint8_t EncodeFp4E2M1(float value) noexcept +{ + bool negative = value < 0.0f; + float magnitude = std::fabs(value); + + uint8_t bestIdx = 0; + float bestDist = std::fabs(magnitude - detail::kFp4E2M1Magnitudes[0]); + for (uint8_t i = 1; i < 8; ++i) + { + float dist = std::fabs(magnitude - detail::kFp4E2M1Magnitudes[i]); + if (dist < bestDist) + { + bestDist = dist; + bestIdx = i; + } + } + + return (negative && bestIdx != 0) ? uint8_t(bestIdx | 0x8) : bestIdx; +} + +/// @brief Storage type for the FP4 E2M1 format (see arXiv:2610.15933's +/// W4A4E4 QAT setup). Converts to/from any arithmetic AccumT via +/// Decode/EncodeFp4E2M1 (always through a `float` intermediate -- the +/// grid itself is only ever specified to float precision, so routing a +/// double AccumT through it loses nothing beyond what the 4-bit grid +/// already discards). The conversion constructor is explicit, since an +/// implicit float/double -> 4-bit-grid narrowing happening by accident +/// (rather than at a deliberate CPUBackendT decode/encode point) would +/// be exactly the kind of silent precision bug this whole templating +/// pass exists to make impossible to miss. +struct Fp4E2M1Value +{ + uint8_t bits = 0; + + Fp4E2M1Value() noexcept = default; + + template >> + explicit Fp4E2M1Value(U value) noexcept : bits(EncodeFp4E2M1(static_cast(value))) + { + } + + explicit operator float() const noexcept + { + return DecodeFp4E2M1(bits); + } + explicit operator double() const noexcept + { + return double(DecodeFp4E2M1(bits)); + } +}; + +/// @brief Decodes a Posit4_0Value's raw 4-bit pattern (bit 3 = sign, +/// bits 2:0 = magnitude index into kPosit4_0Magnitudes) to its real value. +inline float DecodePosit4_0(uint8_t bits) noexcept +{ + bool negative = (bits & 0x8) != 0; + float magnitude = detail::kPosit4_0Magnitudes[bits & 0x7]; + return negative ? -magnitude : magnitude; +} + +/// @brief Encodes a float to the nearest Posit4_0Value grid point -- +/// same round-to-nearest-with-clamping and canonical-zero rules as +/// EncodeFp4E2M1() above. +inline uint8_t EncodePosit4_0(float value) noexcept +{ + bool negative = value < 0.0f; + float magnitude = std::fabs(value); + + uint8_t bestIdx = 0; + float bestDist = std::fabs(magnitude - detail::kPosit4_0Magnitudes[0]); + for (uint8_t i = 1; i < 8; ++i) + { + float dist = std::fabs(magnitude - detail::kPosit4_0Magnitudes[i]); + if (dist < bestDist) + { + bestDist = dist; + bestIdx = i; + } + } + + return (negative && bestIdx != 0) ? uint8_t(bestIdx | 0x8) : bestIdx; +} + +/// @brief Storage type for the Posit<4,0> format (es=0). See +/// Fp4E2M1Value's own doc comment for the conversion-through-float, +/// explicit-constructor reasoning -- identical here. +struct Posit4_0Value +{ + uint8_t bits = 0; + + Posit4_0Value() noexcept = default; + + template >> + explicit Posit4_0Value(U value) noexcept : bits(EncodePosit4_0(static_cast(value))) + { + } + + explicit operator float() const noexcept + { + return DecodePosit4_0(bits); + } + explicit operator double() const noexcept + { + return double(DecodePosit4_0(bits)); + } +}; + +/// @brief Decodes a Posit4_1Value's raw 4-bit pattern (bit 3 = sign, +/// bits 2:0 = magnitude index into kPosit4_1Magnitudes) to its real value. +inline float DecodePosit4_1(uint8_t bits) noexcept +{ + bool negative = (bits & 0x8) != 0; + float magnitude = detail::kPosit4_1Magnitudes[bits & 0x7]; + return negative ? -magnitude : magnitude; +} + +/// @brief Encodes a float to the nearest Posit4_1Value grid point -- +/// same round-to-nearest-with-clamping and canonical-zero rules as +/// EncodeFp4E2M1() above. +inline uint8_t EncodePosit4_1(float value) noexcept +{ + bool negative = value < 0.0f; + float magnitude = std::fabs(value); + + uint8_t bestIdx = 0; + float bestDist = std::fabs(magnitude - detail::kPosit4_1Magnitudes[0]); + for (uint8_t i = 1; i < 8; ++i) + { + float dist = std::fabs(magnitude - detail::kPosit4_1Magnitudes[i]); + if (dist < bestDist) + { + bestDist = dist; + bestIdx = i; + } + } + + return (negative && bestIdx != 0) ? uint8_t(bestIdx | 0x8) : bestIdx; +} + +/// @brief Storage type for the Posit<4,1> format (es=1). See +/// Fp4E2M1Value's own doc comment for the conversion-through-float, +/// explicit-constructor reasoning -- identical here. +struct Posit4_1Value +{ + uint8_t bits = 0; + + Posit4_1Value() noexcept = default; + + template >> + explicit Posit4_1Value(U value) noexcept : bits(EncodePosit4_1(static_cast(value))) + { + } + + explicit operator float() const noexcept + { + return DecodePosit4_1(bits); + } + explicit operator double() const noexcept + { + return double(DecodePosit4_1(bits)); + } +}; + +/// @brief Decodes a Posit4_2Value's raw 4-bit pattern (bit 3 = sign, +/// bits 2:0 = magnitude index into kPosit4_2Magnitudes) to its real value. +inline float DecodePosit4_2(uint8_t bits) noexcept +{ + bool negative = (bits & 0x8) != 0; + float magnitude = detail::kPosit4_2Magnitudes[bits & 0x7]; + return negative ? -magnitude : magnitude; +} + +/// @brief Encodes a float to the nearest Posit4_2Value grid point -- +/// same round-to-nearest-with-clamping and canonical-zero rules as +/// EncodeFp4E2M1() above. +inline uint8_t EncodePosit4_2(float value) noexcept +{ + bool negative = value < 0.0f; + float magnitude = std::fabs(value); + + uint8_t bestIdx = 0; + float bestDist = std::fabs(magnitude - detail::kPosit4_2Magnitudes[0]); + for (uint8_t i = 1; i < 8; ++i) + { + float dist = std::fabs(magnitude - detail::kPosit4_2Magnitudes[i]); + if (dist < bestDist) + { + bestDist = dist; + bestIdx = i; + } + } + + return (negative && bestIdx != 0) ? uint8_t(bestIdx | 0x8) : bestIdx; +} + +/// @brief Storage type for the Posit<4,2> format (es=2). See +/// Fp4E2M1Value's own doc comment for the conversion-through-float, +/// explicit-constructor reasoning -- identical here. +struct Posit4_2Value +{ + uint8_t bits = 0; + + Posit4_2Value() noexcept = default; + + template >> + explicit Posit4_2Value(U value) noexcept : bits(EncodePosit4_2(static_cast(value))) + { + } + + explicit operator float() const noexcept + { + return DecodePosit4_2(bits); + } + explicit operator double() const noexcept + { + return double(DecodePosit4_2(bits)); + } +}; +} // namespace Deep diff --git a/include/deepity/utils/RandomizeDistribution.h b/include/deepity/utils/RandomizeDistribution.h index 85cdd264..955376d4 100644 --- a/include/deepity/utils/RandomizeDistribution.h +++ b/include/deepity/utils/RandomizeDistribution.h @@ -51,4 +51,33 @@ inline void RandomizeWithDistribution(IComputeBackend* backend, float* buf, size else backend->RandomizeNormal(buf, n, fallbackMean, fallbackStddev, seed); } + +/// @brief Templated counterpart to RandomizeWithDistribution(), for +/// FullPCLayer: the "name(a,b)" literals are parsed as plain +/// floats (parameter strings aren't expected to carry double-precision +/// detail), then widened to AccumT for the RandomizeNormal/ +/// RandomizeUniform call, matching IComputeBackendT's own +/// AccumT-typed distribution parameters. +template +inline void RandomizeWithDistributionT(IComputeBackendT* backend, T* buf, size_t n, + const char* distribution, uint32_t seed, + AccumT fallbackMean, AccumT fallbackStddev) noexcept +{ + char name[16] = {0}; + float a = 0.0f, b = 1.0f; + + if (!distribution || DEEPITY_SCANF_DISTRIBUTION(distribution, name, a, b) != 3) + { + std::memcpy(name, "normal", sizeof("normal")); + backend->RandomizeNormal(buf, n, fallbackMean, fallbackStddev, seed); + return; + } + + if (std::strcmp(name, "normal") == 0) + backend->RandomizeNormal(buf, n, AccumT(a), AccumT(b), seed); + else if (std::strcmp(name, "uniform") == 0) + backend->RandomizeUniform(buf, n, AccumT(a), AccumT(b), seed); + else + backend->RandomizeNormal(buf, n, fallbackMean, fallbackStddev, seed); +} } // namespace Deep diff --git a/pyproject.toml b/pyproject.toml index 0268edd4..75311e8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,9 +38,9 @@ classifiers = [ ] [project.urls] -Homepage = "https://github.com/Ra4ster/Deepity" -Repository = "https://github.com/Ra4ster/Deepity" -Issues = "https://github.com/Ra4ster/Deepity/issues" +Homepage = "https://github.com/Deepity-HQ/Deepity" +Repository = "https://github.com/Deepity-HQ/Deepity" +Issues = "https://github.com/Deepity-HQ/Deepity/issues" # Dev-only deps; install with `uv sync --group dev --no-install-project` # (use build.py to actually build the C++ extension, see README.md). diff --git a/resources/customdoxygen.css b/resources/customdoxygen.css index 422e6780..0c3fbc07 100644 --- a/resources/customdoxygen.css +++ b/resources/customdoxygen.css @@ -1,39 +1,40 @@ /* ========================================================================== customdoxygen.css - Light theme for Doxygen HTML output, matched to the Deepity site's own - light palette (see DeepitySite's src/index.css, HomePage.jsx, Navbar.jsx, - and CodeBlock.jsx) -- clicking "Docs" from the site should feel like the - same page continuing, not a jump to a different product. - - USAGE: - 1. Doxyfile already points HTML_EXTRA_STYLESHEET at this file's path - (resources/customdoxygen.css) and sets GENERATE_TREEVIEW = YES -- - the treeview-specific rules below assume it's on. - 2. Re-run `doxygen Doxyfile`. - - HOW THIS WORKS: Doxygen 1.10+ generates its own theme (light and dark - variants) entirely through CSS custom properties defined on :root -- - over 150 of them, covering everything from page background to tooltip - shadows to code syntax colors. Redeclaring the FULL set here, after - doxygen.css loads, is far more reliable than chasing individual - selectors piecemeal -- it's also how this file avoids leaving corners - (tooltips, search results, table-of-contents, admonition colors, code - syntax highlighting) silently mismatched. The values below start from - Doxygen's own light-mode defaults (doxygen.css's un-prefixed :root - block) and are remapped only where the site has an opinion; anything - not overridden keeps behaving exactly as stock Doxygen light mode does. - - Only a handful of hand-written selectors remain past the variable - overrides: the resizable per-page outline panel and smartmenus - dropdown (GENERATE_TREEVIEW extras with no corresponding variables), - inline single-backtick markdown code spans, and the header/footer - gradient bands that echo the site's own Navbar/MainFooter treatment. - - Built against Doxygen 1.16's generated markup/variable set. If a - future Doxygen version renames or drops a variable, the values below - should still carry over -- just diff against the new doxygen.css. + Dark theme for the Doxygen HTML output, matched to the Deepity website + (near-black surfaces, violet accents, Michroma headings, Jura body text, + JetBrains Mono code). Doxyfile points HTML_EXTRA_STYLESHEET here and sets + HTML_COLORSTYLE = DARK; GENERATE_TREEVIEW = YES is assumed. + + Doxygen themes its output through CSS custom properties on :root; this + file redeclares the full set after doxygen.css loads, then adds a few + hand-written rules for the parts that have no variables (the outline + panel, the smartmenus main menu and its dropdowns, the title bar). + + The font files are loaded from /fonts/ and /fonts/JetBrainsMono.woff2, + which exist when the docs are served from the Deepity site. Opened on + their own, the docs fall back to system fonts. ========================================================================== */ +@font-face { + font-family: "Michroma"; + src: url("/fonts/Michroma-Regular.ttf") format("truetype"); + font-display: swap; +} + +@font-face { + font-family: "Jura"; + src: url("/fonts/Jura-VariableFont_wght.ttf") format("truetype"); + font-weight: 300 700; + font-display: swap; +} + +@font-face { + font-family: "JetBrains Mono"; + src: url("/fonts/JetBrainsMono.woff2") format("woff2"); + font-weight: 100 800; + font-display: swap; +} + :root { /* -------------------------------------------------------------------- Deepity brand tokens. --dx-purple/-violet/-blue are the three-stop @@ -43,28 +44,28 @@ (HomePage.jsx's bg-white / bg-[#e4e6e7] / bg-[#e6e8e9], and Navbar.jsx's border-[#202d3b]). -------------------------------------------------------------------- */ - --dx-bg: #ffffff; - --dx-panel: #e6e8e9; - --dx-panel-raised: #eef0f1; - --dx-border: #d7dadc; - --dx-border-strong: #202d3b; - --dx-accent: var(--dx-blue); + color-scheme: dark; + + --dx-bg: #020205; + --dx-panel: #0c0c0d; + --dx-panel-raised: #141415; + --dx-border: rgba(255, 255, 255, 0.1); + --dx-border-strong: rgba(255, 255, 255, 0.18); + --dx-accent: var(--dx-violet); - --dx-purple: #9a53c3; - --dx-violet: #7556d6; - --dx-blue: #4169e1; + --dx-purple: #c4b5fd; + --dx-violet: #a99bff; + --dx-blue: #7dd3fc; - --dx-text: #111111; - --dx-text-dim: rgba(17, 17, 17, 0.75); - --dx-text-muted: rgba(17, 17, 17, 0.5); + --dx-text: #ffffff; + --dx-text-dim: rgba(255, 255, 255, 0.75); + --dx-text-muted: rgba(255, 255, 255, 0.5); - /* Matches CodeBlock.jsx / HomePage.jsx's syntax-highlighter background */ - --dx-code-bg: #fafafa; - --dx-code-fg: #24292e; + /* Matches the site's code blocks */ + --dx-code-bg: #0c0c0d; + --dx-code-fg: #f2f2f2; - /* A red distinct from the purple/violet/blue accent trio, used only for - warning/attention/bug admonitions so they still read as "danger" */ - --dx-red: #c62828; + --dx-red: #ff7b72; } /* -------------------------------------------------------------------- @@ -78,8 +79,8 @@ /* page base colors */ --page-background-color: var(--dx-bg); --page-foreground-color: var(--dx-text-dim); - --page-link-color: var(--dx-blue); - --page-visited-link-color: var(--dx-blue); + --page-link-color: var(--dx-violet); + --page-visited-link-color: var(--dx-violet); --page-external-link-color: var(--dx-purple); /* index (class/file/namespace listing tables) */ @@ -94,8 +95,8 @@ --group-header-separator-color: var(--dx-border); --group-header-color: var(--dx-text); --footer-foreground-color: var(--dx-text-muted); - --citation-label-color: var(--dx-blue); - --glow-color: var(--dx-blue); + --citation-label-color: var(--dx-violet); + --glow-color: var(--dx-violet); --title-background-color: var(--dx-bg); --title-separator-color: var(--dx-border); @@ -106,13 +107,13 @@ --scrollbar-thumb-color: var(--dx-border); --scrollbar-background-color: var(--dx-panel); - --icon-background-color: var(--dx-blue); - --icon-foreground-color: white; + --icon-background-color: var(--dx-violet); + --icon-foreground-color: #020205; --icon-folder-open-fill-color: var(--dx-border); --icon-folder-fill-color: var(--dx-panel); - --icon-folder-border-color: var(--dx-blue); + --icon-folder-border-color: var(--dx-violet); --icon-doc-fill-color: var(--dx-panel); - --icon-doc-border-color: var(--dx-blue); + --icon-doc-border-color: var(--dx-violet); /* brief member declaration list (the summary tables atop class/file pages) */ --memdecl-background-color: var(--dx-panel); @@ -125,7 +126,7 @@ --memdef-title-background-color: var(--dx-panel-raised); --memdef-proto-background-color: var(--dx-panel); --memdef-proto-text-color: var(--dx-text); - --memdef-param-name-color: var(--dx-blue); + --memdef-param-name-color: var(--dx-violet); --memdef-template-color: var(--dx-purple); /* tables */ @@ -135,7 +136,7 @@ /* labels (e.g. "deprecated", "static" badges) */ --label-background-color: var(--dx-panel-raised); - --label-left-top-border-color: var(--dx-blue); + --label-left-top-border-color: var(--dx-violet); --label-right-bottom-border-color: var(--dx-border); --label-foreground-color: var(--dx-text-dim); @@ -145,9 +146,9 @@ --nav-border-color: var(--dx-border); --nav-breadcrumb-separator-color: var(--dx-border); --nav-breadcrumb-active-bg: var(--dx-panel-raised); - --nav-breadcrumb-color: var(--dx-blue); + --nav-breadcrumb-color: var(--dx-violet); --nav-splitbar-bg-color: var(--dx-border); - --nav-splitbar-handle-color: var(--dx-blue); + --nav-splitbar-handle-color: var(--dx-violet); --nav-text-normal-color: var(--dx-text-dim); --nav-menu-button-color: var(--dx-text-dim); --nav-menu-background-color: var(--dx-bg); @@ -155,14 +156,14 @@ --nav-menu-active-bg: var(--dx-panel); --nav-menu-active-color: var(--dx-text); --nav-arrow-color: var(--dx-text-muted); - --nav-arrow-selected-color: var(--dx-blue); + --nav-arrow-selected-color: var(--dx-violet); /* sync icon (the "keep source/docs in sync" toggle in the nav tree) */ --sync-icon-border-color: var(--dx-border); --sync-icon-background-color: var(--dx-panel); --sync-icon-selected-background-color: var(--dx-panel-raised); --sync-icon-color: var(--dx-text-muted); - --sync-icon-selected-color: var(--dx-blue); + --sync-icon-selected-color: var(--dx-violet); /* table of contents */ --toc-background-color: var(--dx-panel); @@ -175,14 +176,14 @@ --search-active-color: var(--dx-text); --search-box-border-color: var(--dx-border); --search-close-icon-bg-color: var(--dx-text-muted); - --search-close-icon-fg-color: white; - --search-filter-background-color: rgba(255, 255, 255, 0.85); + --search-close-icon-fg-color: #020205; + --search-filter-background-color: rgba(12, 12, 13, 0.92); --search-filter-foreground-color: var(--dx-text-dim); --search-filter-backdrop-filter: blur(4px); --search-filter-border-color: var(--dx-border); - --search-filter-highlight-text-color: white; - --search-filter-highlight-bg-color: var(--dx-blue); - --search-results-background-color: rgba(255, 255, 255, 0.9); + --search-filter-highlight-text-color: #020205; + --search-filter-highlight-bg-color: var(--dx-violet); + --search-results-background-color: rgba(12, 12, 13, 0.96); --search-results-foreground-color: var(--dx-text-dim); --search-results-backdrop-filter: blur(4px); --search-results-border-color: var(--dx-border); @@ -191,13 +192,13 @@ syntax palette -- readability matters more than brand color here, same reasoning `oneLight`/`docco` on the site's own code blocks use an independent palette rather than the brand accent trio) */ - --code-keyword-color: var(--dx-violet); - --code-type-keyword-color: var(--dx-purple); - --code-flow-keyword-color: #b3661e; - --code-comment-color: #6a737d; - --code-preprocessor-color: #a0538c; - --code-string-literal-color: #032f62; - --code-char-literal-color: #005cc5; + --code-keyword-color: #569cd6; + --code-type-keyword-color: #4ec9b0; + --code-flow-keyword-color: #c586c0; + --code-comment-color: #6a9955; + --code-preprocessor-color: #c586c0; + --code-string-literal-color: #ce9178; + --code-char-literal-color: #b5cea8; --code-xml-cdata-color: var(--dx-code-fg); --code-vhdl-digit-color: #e91e63; --code-vhdl-char-color: var(--dx-text-muted); @@ -211,65 +212,65 @@ --fragment-lineno-border-color: var(--dx-border); --fragment-lineno-background-color: var(--dx-panel); --fragment-lineno-foreground-color: var(--dx-text-muted); - --fragment-lineno-link-fg-color: var(--dx-blue); + --fragment-lineno-link-fg-color: var(--dx-violet); --fragment-lineno-link-bg-color: var(--dx-border); - --fragment-lineno-link-hover-fg-color: var(--dx-blue); + --fragment-lineno-link-hover-fg-color: var(--dx-violet); --fragment-lineno-link-hover-bg-color: var(--dx-panel-raised); - --fragment-copy-ok-color: #2e7d32; + --fragment-copy-ok-color: #7ee787; --fold-line-color: var(--dx-text-muted); /* tooltips (the hover preview popups over symbol links) */ --tooltip-foreground-color: var(--dx-text-dim); - --tooltip-background-color: rgba(255, 255, 255, 0.95); - --tooltip-arrow-background-color: white; + --tooltip-background-color: rgba(20, 20, 21, 0.96); + --tooltip-arrow-background-color: #141415; --tooltip-backdrop-filter: blur(3px); --tooltip-border-color: var(--dx-border); --tooltip-doc-color: var(--dx-text-muted); - --tooltip-declaration-color: #1e7d32; - --tooltip-link-color: var(--dx-blue); - --tooltip-shadow: 0 4px 12px rgba(0, 0, 0, 0.12); + --tooltip-declaration-color: #7ee787; + --tooltip-link-color: var(--dx-violet); + --tooltip-shadow: 0 4px 16px rgba(0, 0, 0, 0.5); /* fonts -- Inter/Roboto match the main site's body copy; Fira Code matches the monospace font used in the site's own homepage code samples (see HomePage.jsx's syntaxCustomStyle) */ - --font-family-normal: 'Inter', 'Roboto', sans-serif; - --font-family-title: 'Inter', 'Roboto', sans-serif; - --font-family-monospace: 'Fira Code', Consolas, Monaco, monospace; - --font-family-nav: 'Inter', 'Roboto', sans-serif; - --font-family-toc: 'Inter', 'Roboto', sans-serif; - --font-family-search: 'Inter', 'Roboto', sans-serif; - --font-family-tooltip: 'Inter', 'Roboto', sans-serif; + --font-family-normal: "Jura", "Segoe UI", sans-serif; + --font-family-title: "Michroma", "Jura", sans-serif; + --font-family-monospace: "JetBrains Mono", "Fira Code", ui-monospace, Consolas, monospace; + --font-family-nav: "Michroma", "Jura", sans-serif; + --font-family-toc: "Jura", sans-serif; + --font-family-search: "Jura", sans-serif; + --font-family-tooltip: "Jura", sans-serif; /* special sections (@note/@warning/@deprecated/etc admonitions) -- kept close to Doxygen's own light values, which already read clearly as their intended severity; only bug/deprecated nudged toward the site's purple/muted-slate family instead of doxygen's stock hues */ - --warning-color-bg: #fdecea; + --warning-color-bg: #2a1214; --warning-color-hl: var(--dx-red); - --warning-color-text: #7a1f1f; - --note-color-bg: #fff8e1; - --note-color-hl: #f3a600; - --note-color-text: #5f4204; - --todo-color-bg: #e8f0fe; + --warning-color-text: #ffb4ab; + --note-color-bg: #261e0a; + --note-color-hl: #f5c542; + --note-color-text: #f5d77f; + --todo-color-bg: #0f1a2e; --todo-color-hl: var(--dx-blue); - --todo-color-text: #1a3c78; - --test-color-bg: #f1ecfb; + --todo-color-text: #a9c7ff; + --test-color-bg: #1a1530; --test-color-hl: var(--dx-violet); - --test-color-text: #3d2a70; + --test-color-text: #cfc6ff; --deprecated-color-bg: var(--dx-panel); --deprecated-color-hl: var(--dx-text-muted); --deprecated-color-text: var(--dx-text-dim); - --bug-color-bg: #f3e8fb; + --bug-color-bg: #22142e; --bug-color-hl: var(--dx-purple); - --bug-color-text: #5a2d78; - --invariant-color-bg: #e6f4ea; - --invariant-color-hl: #2e7d32; - --invariant-color-text: #1b4d22; + --bug-color-text: #e0c2f5; + --invariant-color-bg: #0f2316; + --invariant-color-hl: #7ee787; + --invariant-color-text: #a6e9b4; --satisfies-color-hl: var(--dx-red); - --satisfies-color-bg: #fdecea; + --satisfies-color-bg: #2a1214; --verifies-color-hl: var(--dx-red); - --verifies-color-bg: #fdecea; + --verifies-color-bg: #2a1214; } /* -------------------------------------------------------------------- @@ -286,7 +287,7 @@ html { a/a.el/a.code/a.line. */ a:hover, a:focus { - color: var(--dx-violet); + color: #ffffff; text-decoration: underline; } @@ -298,8 +299,8 @@ a { depending on Doxygen version -- not covered by the fragment variables above, since those target multi-line @code blocks specifically. */ tt, .tt, code.tt, span.tt, code { - color: #a0538c; - background: var(--dx-code-bg); + color: #d7bafd; + background: rgba(255, 255, 255, 0.07); border-radius: 3px; padding: 0.05em 0.3em; } @@ -410,24 +411,46 @@ ul.page-outline .arrowhead.opened { #top, #titlearea { - background: rgba(255, 255, 255, 0.85); + background: rgba(2, 2, 5, 0.9) !important; backdrop-filter: blur(8px); - border-bottom: 1px solid var(--dx-border-strong); + border-bottom: 1px solid var(--dx-border); } #projectname { color: var(--dx-text); - font-weight: 700; + font-family: "Michroma", "Jura", sans-serif; + font-weight: 400; + letter-spacing: -0.02em; +} + +#projectbrief { + color: var(--dx-text-muted); +} + +/* Page title bars, headings and body background */ +html, +body { + background: var(--dx-bg); + color: var(--dx-text-dim); +} + +h1, +h2, +h3, +div.title, +.contents h1, +.contents h2, +.contents h3 { + font-family: "Michroma", "Jura", sans-serif; + font-weight: 400; + color: var(--dx-text); } /* Top tabs (.tabs/.tabs2/.tabs3) -- the current-tab treatment reuses the site's own .magic-text gradient-clip trick from src/index.css. */ .tablist li.current a, .tablist a:hover { - background: linear-gradient(to right, var(--dx-purple), var(--dx-violet), var(--dx-blue)); - -webkit-background-clip: text; - background-clip: text; - -webkit-text-fill-color: transparent; + color: var(--dx-violet); } /* -------------------------------------------------------------------- @@ -461,12 +484,22 @@ ul.page-outline .arrowhead.opened { } .sm-dox a span.sub-arrow { - border-color: var(--dx-text-dim) transparent transparent transparent; + border-color: var(--dx-text-dim) transparent transparent transparent !important; } -.sm-dox ul { - background: var(--dx-bg) !important; +.sm-dox ul, +.sm-dox ul ul { + background: var(--dx-panel) !important; + background-image: none !important; border: 1px solid var(--dx-border) !important; + box-shadow: 0 12px 32px rgba(0, 0, 0, 0.55); +} + +.sm-dox ul li, +.sm-dox ul a, +.sm-dox ul a:visited { + background: transparent !important; + background-image: none !important; } .sm-dox ul a { @@ -474,8 +507,9 @@ ul.page-outline .arrowhead.opened { border-color: var(--dx-border) !important; } -.sm-dox ul a:hover { - background: var(--dx-panel) !important; +.sm-dox ul a:hover, +.sm-dox ul a.highlighted { + background: var(--dx-panel-raised) !important; color: var(--dx-text) !important; } @@ -487,3 +521,71 @@ ul.page-outline .arrowhead.opened { .directory tr:hover { background: var(--dx-panel); } + +/* Search box and its dropdown */ +#MSearchBox, +#MSearchBox .left, +#MSearchBox .right, +#MSearchField { + background: var(--dx-bg) !important; + color: var(--dx-text) !important; +} + +#MSearchBox { + border: 1px solid var(--dx-border); + border-radius: 8px; +} + +#MSearchSelectWindow, +#MSearchResultsWindow, +iframe#MSearchResults { + background: var(--dx-panel) !important; + border: 1px solid var(--dx-border) !important; +} + +#MSearchSelectWindow a, +.SelectItem { + color: var(--dx-text-dim) !important; +} + +/* The navigation tree, footer and any left-over light surfaces */ +#nav-tree, +#side-nav, +#nav-tree-contents, +div.header, +div.contents, +#doc-content, +.footer, +address.footer { + background: var(--dx-bg) !important; +} + +#nav-tree .label a, +#nav-tree a { + color: var(--dx-text-dim); +} + +#nav-tree .selected { + background-image: none; + background: var(--dx-panel-raised); + text-shadow: none; +} + +#nav-tree .selected a { + color: var(--dx-text); +} + +/* Doxygen draws diagrams and the logo on transparent backgrounds */ +img.formulaInl, +img.formulaDsp { + filter: invert(1) hue-rotate(180deg); +} + +/* Graphviz diagrams (inheritance, collaboration, call graphs) are drawn on + white; invert them to sit on the dark page. */ +div.center img, +div.dyncontent img, +div.image img, +.dynheader + .dyncontent img { + filter: invert(0.9) hue-rotate(180deg); +} diff --git a/resources/deepity-favicon.png b/resources/deepity-favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..9d83691df9d0d922286ae4b8f25954839c93efce GIT binary patch literal 4897 zcmV++6W;8JP) zdyE~|UB|z_Gc)(D*N;Fbp%ow`+CNHC{s18!q3YU5h*qcwsjTZhB&dKmRuxfE9&zQ( z^#volQ~^rVN=`v-2z6DjqS9I@JS1!*;17sYl*&#J6cB{87zGmV``$UfAAiiuopa8d zGmpD_*SnQ}(rEY2ojG&n_c_1EdCXkkuH2Qoa#!|^Q0R-xixPe7Ezq^ytCN&ZeZq&g7^_T(m_Sh3F{}V7 z&Qt&c;Qaq$Kg0Q)pDRTf&hb_`#c^&q`7t1~9QIz9sXV8sGmxvzDaX2|Q8)8mNhyKi zb%y$HD10?ea04T}IQ%57PINbrcsPXKuGLh?3PgS8yU=^x0sQWLeLVcC1}ImM6kC1+ zVkl2M+e8_G??id_Q?3->If3gmeLvY>)Vbk1@3r%~GJvrLH1HA;{zeGTj3PYy_~W#O z+-jDWN$(gCQ@(J&{N{X&Gi!`Z24Wu+nEyLBD@Nkn@KV zJbGIr6+&z&%5X)d@@kb}D$m(xfpN>5>=UoGdaq&BbK=$l^{cgDPk9Ux5LgI^Vim{}?@u%#Eqxka!1x*ie?gH-oaGk#Yg0PI)tg5wwgcVR-A@0K?tK5EcT* ziHf5P%<|TXI`xT9&?}3VC0ah--i4_eR&wk*^wkB47r#QMS2QjhN)UsPHY(4!xjg|x zg)L(m7;xM(6e)@*9%<%{O-_@(TI_uj{`TPyZ@p#tIO)Ma;d^ef3UC= zg&0BQIs13oP1RZ73@n3<7!!(Z<~3L(P3Bargzb@6mG&L=4~@B80Rqx8E}`lpjXLTHC@1Ss%(i3n_}W zMw}*dcoX;Wso{|L_;S6@I{?X`409;&|9d)pJ1yOpuk{7p6x{*K2<>y~)Eb7TJUHcX z1|{BD=d?eMcbz;d?^!-hdhxPEK@E2UQi#xC0nzX}rJs3F&L8UI(Uo!LzR&`OS9uLE zyujO9MnHK}xv2n*koD!Ri=zaTYW4D*#{Iv47Aph*9@o{;LJBD@<>&s6PTwBm!a>7Q zf^KRTV8{$%NCy~&b~7lXQVrcdh;Lty@h}m|;4HjWGNpwSQd-I%yiLv@?&pD?a!YBj zjH1Qure##zR2>XwRHAc|V4W+h5=E)zm*SCn^OHkt6ph=|=#&;xNNFj*@Yi&DCAXAG zpl@(f6Bw$5!Lx)x?P2gvz))p*P^8k7xpzQ)=ctFb10)NBs+DXLhy&%C0Yy21KIEp9 zv>s;)!SE#uZIu^-fhk~k`&8zXg@R-ozZ(GVdjORfvQ3v|O6k5X{&?0;Sxxh?uNlLb zMtQSa5m^OHDBHnNuO?s1Hb_-vR))?zg?v4g;~FeDKe6D1F3GNEs|l zECLZB5hnX@v2p&rKtz{~3djl&&>LZ| zod5u|9v1EnMf6}n;zypP(@(rpFyF(YYpDWRsjl7P$!$>tkfVU8gA1|F&-anX?U7NJ zLTTo8wS-5T>tluu!?CJw0|0kb&y0XLP_EGF-}@;o{a8J`slhB9O>Rc=3QGPtKU?5@ zmi)%Wa5<89)Xd{02hJW~Y`7TC%VR^aY%F>x%OW8h0TFeAL17sAHt*Bg41r(*MIc09 zqK`cB)AD#sX7dDA%)<|RN-+NxbacJ}xP7E;O^jSdm)Ba!|6Os8@ApAlTaZ-Y-WdE3 z6wy7*EJ8$;POqmk*#ILbGX??xB$UrDKQg3OW`Dh%m4|*^QcfT$U>T-@n_!q(AqSw$ zAgKFgI3$*QfMn*I?8v4>gfNH1m!(Yu!%U+?vTHy9fIzY<8*P6BvWcW*3sM=D*(MAt zfpBD3f$U2;QiZT>7?>`yUu=IbOP1j9+`5rDygDXO~`(n5Sa`)OdnI5#AF&_V_Z zxVB3T5ad-Qe4}2=xqQ!L#gNpqXNVnR`JNp zfe3^s2XXNsNlrdVWB-xY6aWU{9iZMR0NH@jjA{ky0hAs9aAk7s(9y!6i7ZXi@`wD+&}XtZ@q@!0`Q+Eq>2!)LdGd1O=noY6rdQyt1XT#gRw7;7%;2{ zg93w*T=difmzr7d96;3B0f;~{|5bcmDRnZA(5wIMF74#ayDKY&X>jhAKOm zGw+9(Fh2S|T$=0S#OjF0LP#b<4>d{eUY1D^HiteulTnQ@TBN-@Vj)EUpSYYp^w^K25r{zOET)9fqG!6-Wd%Uu%!SnT z&fVDQiz8F_)F(Y~STMW++&dZL8M@w&QfXY`jwbH;D>lZ&c|(eJ36 z1#APOD2KZQ_0};X@SqG}n0HaBso$}7=f!fn+lA54Fg-vdo5?m{tc-Y!mN1HEF2uRG zsQvp6PsyJvX$!-CSEN|)cx)O{w2vL<=1E<~vP2z%N@76b+2zU- zgA#zphgu!o1f#|>T-`AMRdZWjAZ+-{sbQEwy>?w-Ax9aw4hWuwj5cPzloW^{#Y3?9 zwS&j!$}|QUTg8kT-F05%VEO+kVZgBv_rXeAvbD9@T3cMHJ#@#Veg(WZ?l%O(^bEBx zIjiZZ`#Xk#S6rC;tN;~&VZBkOwQ}~};|Oks<%JQL|L`ROT%(Ur4Dk*T_+a?J0rr~< z%%SUM*`tpmDC3!W*G<(8BDy+)p+k@OV@Q!JyNV82O$dgyZy%3k6+QJ33~39)NOVHt zw-ArZs#u1nlZ3jLxa@PWmXlI7W_V#VbQOFO9_?*%Q=x3ifnh=2hq6*xEPJw?3)OvV5;c3dW2848v!V z)}uF`IxqjBHxVU$2yWX;gIcfKY8ZkE_IFw?Ls|il#-KQbAl|xHC?>#Bs?+&vvJ5l! z*0~0MmR0lf4wc{(fMLC`D*{AgEu!ALX@n06cs<>Mk~xs1<)EFC(2ltd`c-1Qd~#PJ)aTD!FS=j;Wh16R^|KqP3cnNo15=lqZhm1Le!j3a?P? z6CmqEIhHXM47V|O#O73tJo+r50}RWbmMw$uGVZxJq{r4*ajrK{5aF>FX3Z?tCguOk zEL!LLr2L+B{#||yJNvBjozJZE&pO+RbG6RP&@9YaSZiU{V%<+07_Q}-E}p{cdaFQ; zz0y!{8yHq=pse15fvAgTX=!tV&n5FJk?ePz2#N?q1Vtp~|CD{6?8~>wzWi@-oLNq& zJo7Ii=Uhc33f4YvHdzP5*jI_SMW)qhg39boy4F$?GrTa2$|rrua7g^nMM*yOShTdc z!RPw(DuzNT5P|Z?3~!PUh;dAIOLoiiDbH?nPCsKx7;bFyy;-$25UsQw?pVy zMj3E(dBe-lgMq|L-=Ikwptzh_EYSZtOgkZ=H!(iXJE{wnoYH;Yp z4eEd&P#p|0%P`On&q6>Ml;KBc=>r!udWY1R%?(Zgi3Jocuf{T_h2iaiH^8v0Ar#aD zG%(z3!xoa^5PD%C@uwe+mbTXUT>qfT-x)7E3q#!j7!-g}e)qN3 zKzu|yTZZA!3uRS1ek9j(C{O)bw6wX-=XwPcNM&ATYRf{yP@evk3KVh6>;gmDScXr5Zn9EiXX2o$g@M*UY$(f*L`z!*6or&?DDJ@Gv@n7V zZweURq{FBu16mz{g4Vxt^Vp%UFN#F%f2B>8!MyU*3mssXmFzaBZb12938m=KhcY5} z0Md4I+lL`FKjm84Oc)|s9g5=gHT}Y?NPZh4YB{iw!=Gir_NPrSLK#*iwsevb$}^`d zOVN`5riV{?m6udH&t_}~hSiT~C=VqJrXDHLyveD4?@JF-_UmkcpI;5e5O!}YWEmlA z@%Z@eQbPH{$t=+4F+~fcmGY*AQMl)6V3?LcJ*AK+NjLb!&A0adaq+T5Zf4a8q_vw0 zWmvv`30f|;u7UE!Uo4@Z=+VyzhSyp;1aKBf)uAYogx1sb^j$B!HTmMgRf(H^!aLj# zQDv(Rx~Whc@wl*X$FKvGs}f)N`DkfmW?%U*OAtog+=Nf@ZRX6q{VWkCVTPm&GZNs*VKzsRPHtH8Y1o>eIcXu);3gmt6dEhj2<4dQ zg@MFZK0r%9`IJU-Se+TI^H@Zv0;8yl7z!@3M*zC0FDCW0jspQKh8HMes(35~ASjB| z0Y&IT=}rFM@fx0Z?cdZ#Uph`3iz3l7HNyx30pW1}0AdXlBWmbdH0)ynR19pa((BUR zbdkh@GR&a->Ce*AJD%3)-J{NEwy&uSi}clvRep@mO&Km$c@l(5dHFHvx@I2p^CarB z8mT|}%7gS@Sp`HbQ@xuC0=aoJYa9ag7wel?+ZwS3A!3gUD~1uwCHK~QYs*P1p;!?V zt)*jZZVyNaW%05^%g1TyCqA40Uvwb){&bUBh}C4w$y1lfXJ;4$#!}Z}lmf*)!G8H# zc27zW0U|u2NPSHb^&%<#<>!B>f1@~VQ6yRhxGsQyx# zMwR=MZrhRj9)L7u7}Ln&Wr=Rx=->+79}e6hxqcL!_buwjQSZuKxhr?&8%O>h;kXgq Tu)-zA00000NkvXXu0mjfMJPh4 literal 0 HcmV?d00001 diff --git a/src/backend/CPUBackendT.cpp b/src/backend/CPUBackendT.cpp new file mode 100644 index 00000000..a2a3e0b7 --- /dev/null +++ b/src/backend/CPUBackendT.cpp @@ -0,0 +1,620 @@ +#include +#include +#include +#include +#include // Deep::detail::Aligned{Alloc,Free}Portable +#include +#include // hwy::ConvertScalarTo, see this file's generic decode/encode casts below +#include + +#include "CPUBackendHighway.h" + +#ifdef DEEPITY_USE_MKL +#include +#else +#include +#endif + +#include + +namespace Deep +{ + +namespace +{ +// Only dRELU/dSIGMOID/dTANH/dGELU/d_eSIGMOID/dHARD_TANH/dLINEAR/NONE +// exist; FusedStateUpdate{,Momentum} themselves are out of scope for +// CPUBackendT (see its header's own doc comment on why), so there's no +// NeedsTranscendentalDerivative counterpart needed here. +} // namespace + +template +T* CPUBackendT::Allocate(size_t numElements) +{ + return static_cast(detail::AlignedAllocPortable(16, numElements * sizeof(T))); +} + +template +void CPUBackendT::Free(T* ptr) noexcept +{ + detail::AlignedFreePortable(ptr); +} + +template +void CPUBackendT::Zero(T* ptr, size_t numElements) noexcept +{ + memset(ptr, 0, numElements * sizeof(T)); +} + +template +void CPUBackendT::Copy(T* dst, const T* src, size_t numElements) noexcept +{ + memcpy(dst, src, numElements * sizeof(T)); +} + +template +void CPUBackendT::CopyFromHost(T* deviceDst, const T* hostSrc, + size_t numElements) noexcept +{ + memcpy(deviceDst, hostSrc, numElements * sizeof(T)); +} + +template +void CPUBackendT::CopyToHost(T* hostDst, const T* deviceSrc, + size_t numElements) noexcept +{ + memcpy(hostDst, deviceSrc, numElements * sizeof(T)); +} + +template +void CPUBackendT::RandomizeNormal(T* buf, size_t n, AccumT mean, AccumT stddev, + uint32_t seed) noexcept +{ + std::mt19937 seedGenerator(seed); + std::uniform_int_distribution seedDist; + + std::vector seeds(omp_get_max_threads()); + for (auto& s : seeds) + s = seedDist(seedGenerator); + +#pragma omp parallel if (!omp_in_parallel()) + { + std::mt19937 rng(seeds[omp_get_thread_num()]); + std::normal_distribution dist(mean, stddev); + +#pragma omp for + for (ptrdiff_t i = 0; i < (ptrdiff_t)n; ++i) + buf[i] = static_cast(dist(rng)); + } +} + +template +void CPUBackendT::RandomizeUniform(T* buf, size_t n, AccumT min, AccumT max, + uint32_t seed) noexcept +{ + std::mt19937 seedGenerator(seed); + std::uniform_int_distribution seedDist; + + std::vector seeds(omp_get_max_threads()); + for (auto& s : seeds) + s = seedDist(seedGenerator); + +#pragma omp parallel if (!omp_in_parallel()) + { + std::mt19937 rng(seeds[omp_get_thread_num()]); + std::uniform_real_distribution dist(min, max); + +#pragma omp for + for (ptrdiff_t i = 0; i < (ptrdiff_t)n; ++i) + buf[i] = static_cast(dist(rng)); + } +} + +template +void CPUBackendT::MatMul(bool transA, bool transB, int M, int N, int K, AccumT alpha, + const T* A, int lda, const T* B, int ldb, AccumT beta, T* C, + int ldc) noexcept +{ + if constexpr (std::is_same_v) + { + cblas_sgemm(CblasRowMajor, transA ? CblasTrans : CblasNoTrans, + transB ? CblasTrans : CblasNoTrans, M, N, K, (float)alpha, A, lda, B, ldb, + (float)beta, C, ldc); + } + else if constexpr (std::is_same_v) + { + cblas_dgemm(CblasRowMajor, transA ? CblasTrans : CblasNoTrans, + transB ? CblasTrans : CblasNoTrans, M, N, K, (double)alpha, A, lda, B, ldb, + (double)beta, C, ldc); + } + else + { + // No native BLAS for this T (sub-8-bit storage formats): a plain, + // correct, non-vectorized triple loop. Decode/encode-wrap through a + // float scratch GEMM is the eventual real path for these formats + // (see the quantization plan's "Bucket C" design), not attempted + // here since no sub-8-bit T is instantiated yet. + for (int i = 0; i < M; ++i) + { + for (int j = 0; j < N; ++j) + { + AccumT acc(0); + for (int k = 0; k < K; ++k) + { + AccumT a = transA ? hwy::ConvertScalarTo(A[k * lda + i]) : hwy::ConvertScalarTo(A[i * lda + k]); + AccumT b = transB ? hwy::ConvertScalarTo(B[j * ldb + k]) : hwy::ConvertScalarTo(B[k * ldb + j]); + acc += a * b; + } + T& c = C[i * ldc + j]; + c = hwy::ConvertScalarTo(alpha * acc + beta * hwy::ConvertScalarTo(c)); + } + } + } +} + +template +void CPUBackendT::MatMulAccum(bool transA, bool transB, int M, int N, int K, + AccumT alpha, const T* A, int lda, const T* B, int ldb, + AccumT beta, AccumT* C, int ldc) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + cblas_sgemm(CblasRowMajor, transA ? CblasTrans : CblasNoTrans, + transB ? CblasTrans : CblasNoTrans, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + } + else if constexpr (std::is_same_v && std::is_same_v) + { + cblas_dgemm(CblasRowMajor, transA ? CblasTrans : CblasNoTrans, + transB ? CblasTrans : CblasNoTrans, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + } + else + { + // No decode/encode round trip here (unlike MatMul() above): C is + // already AccumT*, so the product accumulates directly at full + // AccumT precision with no intermediate rounding to T at all -- + // exactly the point of this method existing separately from + // MatMul(), see this method's own declaration doc comment. + for (int i = 0; i < M; ++i) + { + for (int j = 0; j < N; ++j) + { + AccumT acc(0); + for (int k = 0; k < K; ++k) + { + AccumT a = transA ? hwy::ConvertScalarTo(A[k * lda + i]) : hwy::ConvertScalarTo(A[i * lda + k]); + AccumT b = transB ? hwy::ConvertScalarTo(B[j * ldb + k]) : hwy::ConvertScalarTo(B[k * ldb + j]); + acc += a * b; + } + AccumT& c = C[i * ldc + j]; + c = alpha * acc + beta * c; + } + } + } +} + +template +void CPUBackendT::SumRowsAccum(AccumT* dst, const T* src, size_t batchSize, + size_t width) noexcept +{ + memset(dst, 0, width * sizeof(AccumT)); + + if constexpr (std::is_same_v && std::is_same_v) + { + for (size_t b = 0; b < batchSize; ++b) + cblas_saxpy((int)width, 1.0f, src + b * width, 1, dst, 1); + } + else if constexpr (std::is_same_v && std::is_same_v) + { + for (size_t b = 0; b < batchSize; ++b) + cblas_daxpy((int)width, 1.0, src + b * width, 1, dst, 1); + } + else + { + for (size_t b = 0; b < batchSize; ++b) + for (size_t i = 0; i < width; ++i) + dst[i] += hwy::ConvertScalarTo(src[b * width + i]); + } +} + +template +void CPUBackendT::SumRows(T* dst, const T* src, size_t batchSize, + size_t width) noexcept +{ + memset(dst, 0, width * sizeof(T)); + + // Bit-exactness against the non-templated CPUBackend for T=float + // requires reusing cblas_saxpy itself, not a mathematically-equivalent + // hand-rolled accumulation loop: BLAS's internal summation order + // (e.g. SIMD-lane-parallel partial sums, then a horizontal reduce) + // rounds differently than a straight sequential scalar accumulation, + // even though both compute "the same sum" in exact arithmetic. + if constexpr (std::is_same_v && std::is_same_v) + { + for (size_t b = 0; b < batchSize; ++b) + cblas_saxpy((int)width, 1.0f, src + b * width, 1, dst, 1); + } + else if constexpr (std::is_same_v && std::is_same_v) + { + for (size_t b = 0; b < batchSize; ++b) + cblas_daxpy((int)width, 1.0, src + b * width, 1, dst, 1); + } + else + { + for (size_t b = 0; b < batchSize; ++b) + for (size_t i = 0; i < width; ++i) + dst[i] = hwy::ConvertScalarTo(hwy::ConvertScalarTo(dst[i]) + hwy::ConvertScalarTo(src[b * width + i])); + } +} + +template +AccumT CPUBackendT::Sum(const T* buf, size_t n) noexcept +{ + AccumT total(0); +#pragma omp parallel for schedule(static) reduction(+ : total) if (n > 4096 && !omp_in_parallel()) + for (size_t i = 0; i < n; ++i) + total += hwy::ConvertScalarTo(buf[i]); + return total; +} + +// Scale/AxpyInto/AddBiasBroadcast dispatch to cblas_s/dscal/saxpy for +// T=float/double, matching the non-templated CPUBackend exactly, rather +// than a mathematically-equivalent hand-rolled loop -- see SumRows' +// own comment above on why that distinction matters for bit-exactness. +template +void CPUBackendT::Scale(T* buf, size_t n, AccumT alpha) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + cblas_sscal((int)n, alpha, buf, 1); + } + else if constexpr (std::is_same_v && std::is_same_v) + { + cblas_dscal((int)n, alpha, buf, 1); + } + else + { + for (size_t i = 0; i < n; ++i) + buf[i] = hwy::ConvertScalarTo(hwy::ConvertScalarTo(buf[i]) * alpha); + } +} + +template +void CPUBackendT::AxpyInto(T* y, const T* x, size_t n, AccumT alpha) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + cblas_saxpy((int)n, alpha, x, 1, y, 1); + } + else if constexpr (std::is_same_v && std::is_same_v) + { + cblas_daxpy((int)n, alpha, x, 1, y, 1); + } + else + { + for (size_t i = 0; i < n; ++i) + y[i] = hwy::ConvertScalarTo(hwy::ConvertScalarTo(y[i]) + alpha * hwy::ConvertScalarTo(x[i])); + } +} + +template +void CPUBackendT::AddBiasBroadcast(T* buf, const T* bias, size_t batchSize, + size_t width) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { +#pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) + for (size_t b = 0; b < batchSize; ++b) + cblas_saxpy((int)width, 1.0f, bias, 1, buf + b * width, 1); + } + else if constexpr (std::is_same_v && std::is_same_v) + { +#pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) + for (size_t b = 0; b < batchSize; ++b) + cblas_daxpy((int)width, 1.0, bias, 1, buf + b * width, 1); + } + else + { +#pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) + for (size_t b = 0; b < batchSize; ++b) + for (size_t i = 0; i < width; ++i) + buf[b * width + i] = hwy::ConvertScalarTo(hwy::ConvertScalarTo(buf[b * width + i]) + hwy::ConvertScalarTo(bias[i])); + } +} + +template +void CPUBackendT::Activation(ActivationType type, T* buf, size_t n) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + To_Fn(type)(buf, n); + } + else + { + // Decode each T to AccumT, run the existing generic scalar kernel + // there, re-encode back to T -- the "Bucket C" decode/compute/ + // encode wrap the quantization plan calls for, so a storage T with + // no arithmetic of its own (e.g. a sub-8-bit grid type) never needs + // operators of its own: it only has to convert to/from AccumT (see + // QuantFormat.h). Bit-identical to the old T-only path when T==AccumT + // (e.g. double), since hwy::ConvertScalarTo(x) and hwy::ConvertScalarTo(x) are then both the identity. + for (size_t i = 0; i < n; ++i) + buf[i] = hwy::ConvertScalarTo(ActivationScalarT(type, hwy::ConvertScalarTo(buf[i]))); + } +} + +template +void CPUBackendT::ActivationInto(ActivationType type, T* dst, const T* src, + size_t n) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + cblas_scopy((int)n, src, 1, dst, 1); + To_Fn(type)(dst, n); + } + else + { + for (size_t i = 0; i < n; ++i) + dst[i] = hwy::ConvertScalarTo(ActivationScalarT(type, hwy::ConvertScalarTo(src[i]))); + } +} + +template +void CPUBackendT::ActivationDerivative(ActivationType type, T* buf, size_t n, + bool activated) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + To_dFn(type)(buf, n, activated); + } + else + { + // Matches To_dFn's own contract: `activated` selects the + // closed-form-from-activated-value path where one exists. Same + // decode/compute/encode-via-AccumT wrap as Activation() above. + for (size_t i = 0; i < n; ++i) + buf[i] = activated + ? hwy::ConvertScalarTo(ActivationDerivativeFromActivatedScalar(type, hwy::ConvertScalarTo(buf[i]))) + : hwy::ConvertScalarTo(ActivationDerivativeScalarT(type, hwy::ConvertScalarTo(buf[i]))); + } +} + +template +void CPUBackendT::ActivationDerivativeInto(ActivationType type, T* dst, const T* src, + size_t n) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + To_dFn2(type)(dst, src, n); + } + else + { + for (size_t i = 0; i < n; ++i) + dst[i] = hwy::ConvertScalarTo(ActivationDerivativeScalarT(type, hwy::ConvertScalarTo(src[i]))); + } +} + +template +AccumT CPUBackendT::ComputeErrorAndEnergy(T* e, const T* z, const T* mu, + size_t n) noexcept +{ + AccumT energy(0); +#pragma omp parallel for reduction(+ : energy) schedule(static) if (n > 100000 && !omp_in_parallel()) + for (size_t i = 0; i < n; ++i) + { + AccumT err = hwy::ConvertScalarTo(z[i]) - hwy::ConvertScalarTo(mu[i]); + e[i] = hwy::ConvertScalarTo(err); + energy += err * err; + } + return hwy::ConvertScalarTo(0.5) * energy; +} + +template +void CPUBackendT::ComputeError(T* e, const T* z, const T* mu, size_t n) noexcept +{ +#pragma omp parallel for schedule(static) if (n > 100000 && !omp_in_parallel()) + for (size_t i = 0; i < n; ++i) + e[i] = hwy::ConvertScalarTo(hwy::ConvertScalarTo(z[i]) - hwy::ConvertScalarTo(mu[i])); +} + +template +AccumT CPUBackendT::ComputeSoftmaxCrossEntropyErrorAndEnergy( + T* e, const T* z, const T* mu, size_t batchSize, size_t nextSize, + AccumT* /*rowEnergies, unused on CPU*/) noexcept +{ + AccumT energy(0); + const AccumT eps(1e-8); + +#pragma omp parallel for reduction(+ : energy) \ + schedule(static) if (batchSize > 8 && !omp_in_parallel()) + for (size_t b = 0; b < batchSize; ++b) + { + size_t base = b * nextSize; + + AccumT max_val = hwy::ConvertScalarTo(mu[base]); + for (size_t j = 1; j < nextSize; ++j) + max_val = std::max(max_val, hwy::ConvertScalarTo(mu[base + j])); + + AccumT sum_exp(0); + for (size_t j = 0; j < nextSize; ++j) + { + AccumT ex = std::exp(hwy::ConvertScalarTo(mu[base + j]) - max_val); + e[base + j] = hwy::ConvertScalarTo(ex); // e temporarily holds unnormalized exp + sum_exp += ex; + } + + AccumT row_energy(0); + for (size_t j = 0; j < nextSize; ++j) + { + AccumT prob = hwy::ConvertScalarTo(e[base + j]) / sum_exp; + e[base + j] = hwy::ConvertScalarTo(prob); // e now holds probs + row_energy -= hwy::ConvertScalarTo(z[base + j]) * std::log(prob + eps); + } + energy += row_energy; + + for (size_t j = 0; j < nextSize; ++j) + e[base + j] = hwy::ConvertScalarTo(hwy::ConvertScalarTo(z[base + j]) - hwy::ConvertScalarTo(e[base + j])); // e now holds the final error + } + + return energy; +} + +template +void CPUBackendT::ComputeSoftmaxCrossEntropyError(T* e, const T* z, const T* mu, + size_t batchSize, + size_t nextSize) noexcept +{ + if constexpr (std::is_same_v) + { +#pragma omp parallel for schedule(static) if (batchSize > 8 && !omp_in_parallel()) + for (size_t b = 0; b < batchSize; ++b) + { + size_t base = b * nextSize; + ComputeSoftmaxCrossEntropyErrorRowDispatch(e + base, z + base, mu + base, nextSize); + } + } + else + { +#pragma omp parallel for schedule(static) if (batchSize > 8 && !omp_in_parallel()) + for (size_t b = 0; b < batchSize; ++b) + { + size_t base = b * nextSize; + + AccumT max_val = hwy::ConvertScalarTo(mu[base]); + for (size_t j = 1; j < nextSize; ++j) + max_val = std::max(max_val, hwy::ConvertScalarTo(mu[base + j])); + + AccumT sum_exp(0); + for (size_t j = 0; j < nextSize; ++j) + { + AccumT ex = std::exp(hwy::ConvertScalarTo(mu[base + j]) - max_val); + e[base + j] = hwy::ConvertScalarTo(ex); + sum_exp += ex; + } + for (size_t j = 0; j < nextSize; ++j) + e[base + j] = hwy::ConvertScalarTo(hwy::ConvertScalarTo(z[base + j]) - hwy::ConvertScalarTo(e[base + j]) / sum_exp); + } + } +} + +template +void CPUBackendT::IncrementCounter(int* counter) noexcept +{ + ++(*counter); +} + +template +void CPUBackendT::AdamStep(T* param, const AccumT* grad, AccumT* m, AccumT* v, size_t n, + const int* t, const AccumT* lr, AccumT beta1, AccumT beta2, + AccumT eps) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + // Bit-exact reuse of the existing Highway-vectorized kernel, same + // OMP chunking as the non-templated CPUBackend::AdamStep(). + float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(*t)); + float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(*t)); + float step_size = *lr * Sleef_sqrtf(beta2_t) / beta1_t; + + if (n > 100000 && !omp_in_parallel()) + { +#pragma omp parallel + { + int nt = omp_get_num_threads(), tid = omp_get_thread_num(); + size_t start = n * (size_t)tid / nt, end = n * (size_t)(tid + 1) / nt; + AdamStepChunkDispatch(param, grad, m, v, start, end, step_size, beta1, beta2, eps); + } + } + else + { + AdamStepChunkDispatch(param, grad, m, v, 0, n, step_size, beta1, beta2, eps); + } + return; + } + + AccumT beta1_t = hwy::ConvertScalarTo(1) - std::pow(beta1, hwy::ConvertScalarTo(*t)); + AccumT beta2_t = hwy::ConvertScalarTo(1) - std::pow(beta2, hwy::ConvertScalarTo(*t)); + AccumT step_size = *lr * std::sqrt(beta2_t) / beta1_t; + + for (size_t i = 0; i < n; ++i) + { + m[i] = beta1 * m[i] + (hwy::ConvertScalarTo(1) - beta1) * grad[i]; + v[i] = beta2 * v[i] + (hwy::ConvertScalarTo(1) - beta2) * grad[i] * grad[i]; + param[i] = hwy::ConvertScalarTo(hwy::ConvertScalarTo(param[i]) - step_size * m[i] / (std::sqrt(v[i]) + eps)); + } +} + +template +void CPUBackendT::AdamWStep(T* param, const AccumT* grad, AccumT* m, AccumT* v, + size_t n, const int* t, const AccumT* lr, + AccumT weightDecay, AccumT beta1, AccumT beta2, + AccumT eps) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(*t)); + float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(*t)); + float step_size = *lr * Sleef_sqrtf(beta2_t) / beta1_t; + float lrWeightDecay = *lr * weightDecay; + + if (n > 100000 && !omp_in_parallel()) + { +#pragma omp parallel + { + int nt = omp_get_num_threads(), tid = omp_get_thread_num(); + size_t start = n * (size_t)tid / nt, end = n * (size_t)(tid + 1) / nt; + AdamWStepChunkDispatch(param, grad, m, v, start, end, step_size, beta1, beta2, eps, + lrWeightDecay); + } + } + else + { + AdamWStepChunkDispatch(param, grad, m, v, 0, n, step_size, beta1, beta2, eps, lrWeightDecay); + } + return; + } + + AccumT beta1_t = hwy::ConvertScalarTo(1) - std::pow(beta1, hwy::ConvertScalarTo(*t)); + AccumT beta2_t = hwy::ConvertScalarTo(1) - std::pow(beta2, hwy::ConvertScalarTo(*t)); + AccumT step_size = *lr * std::sqrt(beta2_t) / beta1_t; + AccumT lrWeightDecay = *lr * weightDecay; + + for (size_t i = 0; i < n; ++i) + { + AccumT p = hwy::ConvertScalarTo(param[i]) - lrWeightDecay * hwy::ConvertScalarTo(param[i]); + m[i] = beta1 * m[i] + (hwy::ConvertScalarTo(1) - beta1) * grad[i]; + v[i] = beta2 * v[i] + (hwy::ConvertScalarTo(1) - beta2) * grad[i] * grad[i]; + param[i] = hwy::ConvertScalarTo(p - step_size * m[i] / (std::sqrt(v[i]) + eps)); + } +} + +template +void CPUBackendT::MultiplyInto(T* dst, const T* a, const T* b, size_t n) noexcept +{ + if constexpr (std::is_same_v) + { + MultiplyIntoDispatch(dst, a, b, n); + } + else + { + for (size_t i = 0; i < n; ++i) + dst[i] = hwy::ConvertScalarTo(hwy::ConvertScalarTo(a[i]) * hwy::ConvertScalarTo(b[i])); + } +} + +// --- Explicit instantiation: the (T, AccumT) pairs this templating +// pass proves the machinery with so far (see the plan's "Implementation +// order" section) -- every future T added there needs a matching line +// here. Fp4E2M1Value pairs with AccumT=float (NOT the default AccumT=T +// a FullPCLayerT caller would get by leaving AccumT +// unspecified): the whole point of the AccumT split is that the +// optimizer accumulator stays wide even when storage is 4-bit, so +// Fp4E2M1Value must always be spelled with an explicit, wider AccumT. +template class CPUBackendT; +template class CPUBackendT; +template class CPUBackendT; +template class CPUBackendT; +template class CPUBackendT; +template class CPUBackendT; +template class CPUBackendT; + +} // namespace Deep diff --git a/src/layers/FullPCLayerT.cpp b/src/layers/FullPCLayerT.cpp new file mode 100644 index 00000000..b07b27be --- /dev/null +++ b/src/layers/FullPCLayerT.cpp @@ -0,0 +1,645 @@ +#include "deepity/layers/FullPCLayerT.h" +#include "deepity/backend/CPUBackendT.h" +#include "deepity/utils/LayerBindHelpers.h" +#include "deepity/utils/Optimize.h" +#include "deepity/utils/QuantFormat.h" +#include "deepity/utils/RandomizeDistribution.h" +#include +#include + +namespace Deep +{ +namespace +{ +template void DeleteBackendT(IComputeBackendT* p) +{ + delete p; +} +template void NoOpDeleterT(IComputeBackendT*) {} +} // namespace + +template +FullPCLayerT::FullPCLayerT(size_t size, size_t nextSize, size_t terminalSize, + size_t batchSize, AccumT learningRate, AccumT inferenceRate, + AccumT feedback, AccumT lmbda, ActivationType aType, + ActivationType /*dType*/, + IComputeBackendT* backend) + : size(size) + , nextSize(nextSize) + , terminalSize(terminalSize) + , batchSize(batchSize) + , lr(learningRate) + , ir(inferenceRate) + , fl(feedback) + , lmbda(lmbda) + , activationType(aType) + , backend(backend ? backend : new CPUBackendT(), + backend ? &NoOpDeleterT : &DeleteBackendT) +{ + DynamicThread((int)batchSize); + + localArena = std::make_unique>(GetRequiredElements()); + BindMemory(*localArena); +} + +template +void FullPCLayerT::SetLearningRate(AccumT learningRate) noexcept +{ + lr = learningRate; + // Direct write, not backend->CopyFromHost(): lr_device is AccumT*, + // and CopyFromHost() takes T* -- see BindLearningRateStateT's own + // doc comment on why these host-resident AccumT scratch writes go + // around the backend rather than through it (valid today since + // every templated backend is host-resident; see that comment for + // the CUDA caveat). + lr_device = lr; +} + +template +void FullPCLayerT::SetFeedbackRate(AccumT feedbackRate) noexcept +{ + fl = feedbackRate; + fl_device = fl; +} + +template +template +void FullPCLayerT::BindMemory(ArenaT& arena) +{ + size_t own_state_size = batchSize * size; + size_t out_state_size = batchSize * nextSize; + size_t direct_size = size * terminalSize; + + z = arena.Allocate(own_state_size); + e = arena.Allocate(own_state_size); + rowEnergies.assign(batchSize, AccumT(0)); + + backend->Zero(z, own_state_size); + backend->Zero(e, own_state_size); + + if (nextSize > 0) + { + size_t w_size = size * nextSize; + + W = arena.Allocate(w_size); + b = arena.Allocate(nextSize); + mu = arena.Allocate(out_state_size); + cachedMu = arena.Allocate(out_state_size); + preact = arena.Allocate(out_state_size); + cachedPreact = arena.Allocate(out_state_size); + derivScratch = arena.Allocate(out_state_size); + feedbackError = arena.Allocate(out_state_size); + proj = arena.Allocate(out_state_size); + Psi = arena.Allocate(direct_size); + feedbackScratch = arena.Allocate(own_state_size); + v = arena.Allocate(own_state_size); + backend->Zero(v, own_state_size); + adjoint = arena.Allocate(own_state_size); + backend->Zero(adjoint, own_state_size); + biasGradScratch = arena.Allocate(nextSize); + + backend->Zero(b, nextSize); + backend->Zero(mu, out_state_size); + backend->Zero(cachedMu, out_state_size); + backend->Zero(preact, out_state_size); + backend->Zero(cachedPreact, out_state_size); + backend->Zero(derivScratch, out_state_size); + backend->Zero(feedbackError, out_state_size); + backend->Zero(proj, out_state_size); + backend->Zero(Psi, direct_size); + backend->Zero(feedbackScratch, own_state_size); + backend->Zero(biasGradScratch, nextSize); + + BindAdamStateT(w_size, grad_W, m_W, v_W); + BindAdamStateT(nextSize, grad_b, m_b, v_b); + BindLearningRateStateT(t_device, lr_device, lr); + + BindAdamStateT(direct_size, grad_Psi, m_Psi, v_Psi); + BindLearningRateStateT(tPsi_device, fl_device, fl); + } + + ReleaseLocalArenaIfForeignT(localArena, arena); +} + +template +size_t FullPCLayerT::GetRequiredElements() const noexcept +{ + // Padding granularity in *elements*, not a hardcoded 16: Allocate() + // below rounds every call up to a 64-BYTE boundary regardless of T, + // so the element-count padding this must match is 64/sizeof(T) -- + // 16 for float, 8 for double, 64 for a 1-byte T like Fp4E2M1Value. + // Getting this wrong undercounts the arena's real consumption for + // any T smaller than 4 bytes and throws "capacity exceeded" at + // runtime despite this function's own total looking sufficient. + constexpr size_t kAlignElems = (64 + sizeof(T) - 1) / sizeof(T); + auto pad16 = [](size_t n) { return (n + kAlignElems - 1) & ~(kAlignElems - 1); }; + + size_t total = 0; + size_t own_state_size = batchSize * size; + size_t direct_size = terminalSize * size; + + total += pad16(own_state_size) * 2; + + if (nextSize > 0) + { + size_t out_state_size = (size_t)batchSize * nextSize; + size_t w_size = (size_t)size * nextSize; + + total += pad16(w_size); + total += pad16(nextSize); + total += pad16(out_state_size) * 3; // mu, cachedMu, proj + total += pad16(out_state_size) * 4; // preact, cachedPreact, derivScratch, feedbackError + total += pad16(own_state_size); // feedbackScratch + total += pad16(own_state_size) * 2; // v, adjoint + total += pad16(direct_size); + total += pad16(nextSize); + + total += pad16(w_size) * 3; + total += pad16(nextSize) * 3; + total += pad16(1) * 2; + + total += pad16(direct_size) * 3; + total += pad16(1) * 2; + } + + return total; +} + +template +void FullPCLayerT::RandomizeWeights(std::mt19937& seedGenerator, + const char* distribution) noexcept +{ + if (nextSize == 0) + return; + + std::uniform_int_distribution seedDist; + size_t Wsz = size * nextSize; + size_t Psisz = terminalSize * size; + + AccumT limit = useMuPCInit ? AccumT(1) : std::sqrt(AccumT(2) / AccumT(size + nextSize)); + AccumT limPsi = std::sqrt(AccumT(2) / AccumT(size + terminalSize)); + + uint32_t seedW = seedDist(seedGenerator); + uint32_t seedPsi = seedDist(seedGenerator); + + RandomizeWithDistributionT(backend.get(), W, Wsz, distribution, seedW, AccumT(0), + limit); + backend->RandomizeNormal(Psi, Psisz, AccumT(0), limPsi, seedPsi); +} + +template +std::map> FullPCLayerT::GetStateDict() const +{ + return {{"W", {W, std::vector{nextSize, size}}}, + {"b", {b, std::vector{nextSize}}}, + {"Psi", {Psi, std::vector{nextSize, terminalSize}}}}; +} + +template +AccumT FullPCLayerT::CalculateState(bool needEnergy) noexcept +{ + const size_t N = batchSize * size; + + if (layerBelow == nullptr) + { + backend->Zero(e, N); + if (nextSize > 0) + ComputeMuOnly(); + return AccumT(0); + } + + AccumT totalEnergy(0); + if (useCrossEntropy) + { + if (needEnergy) + totalEnergy = backend->ComputeSoftmaxCrossEntropyErrorAndEnergy( + e, z, layerBelow->mu, batchSize, size, rowEnergies.data()); + else + backend->ComputeSoftmaxCrossEntropyError(e, z, layerBelow->mu, batchSize, size); + } + else if (needEnergy) + { + totalEnergy = backend->ComputeErrorAndEnergy(e, z, layerBelow->mu, N); + } + else + { + backend->ComputeError(e, z, layerBelow->mu, N); + } + + if (nextSize > 0) + ComputeMuOnly(); + + return needEnergy ? totalEnergy : AccumT(0); +} + +template +void FullPCLayerT::ComputeMuOnly() noexcept +{ + if (nextSize == 0) + return; + + size_t Nout = batchSize * nextSize; + + if (isClamped && muCacheValid) + { + backend->Copy(mu, cachedMu, Nout); + backend->Copy(preact, cachedPreact, Nout); + return; + } + + backend->MatMul( + /*transA=*/false, + /*transB=*/true, + (int)batchSize, + (int)nextSize, + (int)size, + a, + z, + (int)size, + W, + (int)size, + AccumT(0), + mu, + (int)nextSize); + + backend->AddBiasBroadcast(mu, b, batchSize, nextSize); + + backend->Copy(preact, mu, Nout); + backend->Activation(activationType, mu, Nout); + + if (useResidual) + backend->AxpyInto(mu, z, Nout, AccumT(1)); + + if (isClamped) + { + backend->Copy(cachedMu, mu, Nout); + backend->Copy(cachedPreact, preact, Nout); + muCacheValid = true; + } +} + +template void FullPCLayerT::UpdateState() noexcept +{ + size_t N = (size_t)batchSize * size; + + if (isClamped) + return; + + if (useCrossEntropy) + return; + + if (layerAbove != nullptr && nextSize > 0) + { + const T* e_above = layerAbove->GetErrors(); + + bool topDownMeaningless = layerAbove->GetCrossEntropy() && !layerAbove->IsClamped(); + + if (topDownMeaningless) + { + backend->AxpyInto(z, e, N, -ir); + } + else + { + size_t Nout = (size_t)batchSize * nextSize; + + ActivationType dType = ToDerivativeType(activationType); + backend->ActivationDerivativeInto(dType, derivScratch, preact, Nout); + backend->MultiplyInto(feedbackError, e_above, derivScratch, Nout); + + backend->MatMul( + /*transA=*/false, + /*transB=*/false, + (int)batchSize, + (int)size, + (int)nextSize, + a, + feedbackError, + (int)nextSize, + W, + (int)size, + AccumT(0), + feedbackScratch, + (int)size); + + if (useMomentum) + { + backend->AxpyInto(feedbackScratch, e, N, AccumT(-1)); + backend->Scale(v, N, momentumBeta); + backend->AxpyInto(v, feedbackScratch, N, AccumT(1) - momentumBeta); + backend->AxpyInto(z, v, N, ir); + } + else + { + backend->AxpyInto(feedbackScratch, e, N, AccumT(-1)); + backend->AxpyInto(z, feedbackScratch, N, ir); + } + + if (useResidual) + backend->AxpyInto(z, e_above, N, ir); + } + } + else + { + backend->AxpyInto(z, e, N, -ir); + } +} + +template void FullPCLayerT::UpdateWeights() noexcept +{ + if (layerAbove == nullptr || nextSize == 0) + return; + + size_t Nout = (size_t)batchSize * nextSize; + const T* e_above = layerAbove->GetErrors(); + + ActivationType dType = ToDerivativeType(activationType); + backend->ActivationDerivativeInto(dType, derivScratch, preact, Nout); + backend->MultiplyInto(feedbackError, e_above, derivScratch, Nout); + const T* local_grad = feedbackError; + + switch (opt) + { + case OptimizerType::SGD: + { + if (lmbda > AccumT(0)) + backend->Scale(W, (size_t)nextSize * size, AccumT(1) - lmbda); + + backend->MatMul( + /*transA=*/true, + /*transB=*/false, + (int)nextSize, + (int)size, + (int)batchSize, + a * lr / AccumT(batchSize), + local_grad, + (int)nextSize, + z, + (int)size, + AccumT(1), + W, + (int)size); + + AccumT lr_batch = lr / AccumT(batchSize); + backend->SumRows(biasGradScratch, local_grad, batchSize, nextSize); + backend->AxpyInto(b, biasGradScratch, nextSize, lr_batch); + + break; + } + case OptimizerType::ADAM: + case OptimizerType::ADAMW: + { + backend->IncrementCounter(&t_device); + + size_t num_weights = (size_t)nextSize * size; + AccumT adam_scale = AccumT(-1) * a; + + backend->MatMulAccum( + /*transA=*/true, + /*transB=*/false, + (int)nextSize, + (int)size, + (int)batchSize, + adam_scale, + local_grad, + (int)nextSize, + z, + (int)size, + AccumT(0), + grad_W.data(), + (int)size); + + backend->SumRowsAccum(grad_b.data(), local_grad, batchSize, nextSize); + // Direct negate, not backend->Scale(): grad_b is AccumT*, and + // Scale() takes T* -- same host-resident-AccumT-scratch caveat as + // SetLearningRate()'s own comment above. + for (size_t i = 0; i < nextSize; ++i) + grad_b[i] = -grad_b[i]; + + if (opt == OptimizerType::ADAMW) + backend->AdamWStep(W, grad_W.data(), m_W.data(), v_W.data(), num_weights, &t_device, + &lr_device, lmbda); + else + backend->AdamStep(W, grad_W.data(), m_W.data(), v_W.data(), num_weights, &t_device, + &lr_device); + + backend->AdamStep(b, grad_b.data(), m_b.data(), v_b.data(), nextSize, &t_device, &lr_device); + break; + } + } + + switch (optPsi) + { + case OptimizerType::SGD: + { + backend->MatMul( + /*transA=*/true, + /*transB=*/false, + (int)size, + (int)terminalSize, + (int)batchSize, + fl / AccumT(batchSize), + z, + (int)size, + terminalLayer->GetErrors(), + (int)terminalSize, + AccumT(1), + Psi, + (int)terminalSize); + break; + } + case OptimizerType::ADAMW: + case OptimizerType::ADAM: + { + backend->IncrementCounter(&tPsi_device); + + size_t num_weights_psi = size * terminalSize; + AccumT adam_scale = AccumT(-1); + + backend->MatMulAccum( + /*transA=*/true, + /*transB=*/false, + (int)size, + (int)terminalSize, + (int)batchSize, + adam_scale, + z, + (int)size, + terminalLayer->GetErrors(), + (int)terminalSize, + AccumT(0), + grad_Psi.data(), + (int)terminalSize); + + if (optPsi == OptimizerType::ADAMW) + backend->AdamWStep(Psi, grad_Psi.data(), m_Psi.data(), v_Psi.data(), num_weights_psi, + &tPsi_device, &fl_device, lmbda); + else + backend->AdamStep(Psi, grad_Psi.data(), m_Psi.data(), v_Psi.data(), num_weights_psi, + &tPsi_device, &fl_device); + + break; + } + } +} + +template void FullPCLayerT::DirectFeedbackUpdate() noexcept +{ + if (layerAbove == nullptr || layerAbove->GetDirectFeedbackWeights() == nullptr) + return; + + backend->MatMul( + /*transA=*/false, + /*transB=*/true, + (int)batchSize, + (int)nextSize, + (int)terminalSize, + AccumT(1), + terminalLayer->GetErrors(), + (int)terminalSize, + layerAbove->GetDirectFeedbackWeights(), + (int)terminalSize, + AccumT(0), + proj, + (int)nextSize); + + { + size_t Nout = (size_t)batchSize * nextSize; + ActivationType dType = ToDerivativeType(activationType); + backend->ActivationDerivativeInto(dType, derivScratch, preact, Nout); + backend->MultiplyInto(proj, proj, derivScratch, Nout); + } + + backend->MatMul( + /*transA=*/true, + /*transB=*/false, + (int)nextSize, + (int)size, + (int)batchSize, + fl / AccumT(batchSize), + proj, + (int)nextSize, + z, + (int)size, + AccumT(1), + W, + (int)size); +} + +template void FullPCLayerT::ReconstructBelief() noexcept +{ + if (layerBelow == nullptr) + return; + + const size_t N = batchSize * size; + backend->Copy(z, layerBelow->GetMu(), N); + backend->AxpyInto(z, e, N, AccumT(1)); +} + +template +void FullPCLayerT::ComputeAdjoint(const T* adjointAbove, + AccumT adjointAboveScale) noexcept +{ + if (layerAbove == nullptr || nextSize == 0) + return; + + const size_t N = batchSize * size; + const size_t Nout = batchSize * nextSize; + + ActivationType dType = ToDerivativeType(activationType); + backend->ActivationDerivativeInto(dType, derivScratch, preact, Nout); + backend->MultiplyInto(feedbackError, adjointAbove, derivScratch, Nout); + + backend->MatMul( + /*transA=*/false, + /*transB=*/false, + (int)batchSize, + (int)size, + (int)nextSize, + a * adjointAboveScale, + feedbackError, + (int)nextSize, + W, + (int)size, + AccumT(0), + adjoint, + (int)size); + + if (useResidual) + backend->AxpyInto(adjoint, adjointAbove, N, adjointAboveScale); +} + +template void FullPCLayerT::UpdateErrorEPC() noexcept +{ + if (isClamped) + return; + + const size_t N = batchSize * size; + backend->Scale(e, N, AccumT(1) - ir); + backend->AxpyInto(e, adjoint, N, -ir); +} + +template +void FullPCLayerT::ClampState(const std::vector& inputData) noexcept +{ + size_t copyElements = (std::min)(inputData.size(), (size_t)(batchSize * size)); + backend->CopyFromHost(z, inputData.data(), copyElements); + isClamped = true; + muCacheValid = false; +} + +template void FullPCLayerT::UnclampState() noexcept +{ + isClamped = false; +} + +template void FullPCLayerT::ResetState() noexcept +{ + size_t N = (size_t)batchSize * size; + backend->Zero(z, N); + backend->Zero(e, N); + if (v) + backend->Zero(v, N); +} + +template class FullPCLayerT; +template void FullPCLayerT::BindMemory>( + MemoryArenaT& arena); + +template class FullPCLayerT; +template void FullPCLayerT::BindMemory>( + MemoryArenaT& arena); + +// Fp4E2M1Value pairs with AccumT=float, never the default AccumT=T -- +// see CPUBackendT.cpp's own explicit-instantiation comment for why. +template class FullPCLayerT; +template void FullPCLayerT::BindMemory>( + MemoryArenaT& arena); + +// Same AccumT=float pairing for half precision -- see this plan's own +// "optimizer state is a second, fixed, wider type" design note; a +// float16 shadow accumulator would defeat the point of accumulating +// wider than storage. +template class FullPCLayerT; +template void FullPCLayerT::BindMemory>( + MemoryArenaT& arena); + +// The three posit formats, same AccumT=float pairing as Fp4E2M1Value. +template class FullPCLayerT; +template void FullPCLayerT::BindMemory>( + MemoryArenaT& arena); + +template class FullPCLayerT; +template void FullPCLayerT::BindMemory>( + MemoryArenaT& arena); + +template class FullPCLayerT; +template void FullPCLayerT::BindMemory>( + MemoryArenaT& arena); + +#if defined(DEEPITY_USE_CUDA) +template void FullPCLayerT::BindMemory>( + DeviceMemoryArenaT& arena); +template void FullPCLayerT::BindMemory>( + DeviceMemoryArenaT& arena); +#endif +} // namespace Deep diff --git a/tests/tFullPCLayerTemplatingVerify.cpp b/tests/tFullPCLayerTemplatingVerify.cpp new file mode 100644 index 00000000..a95f9530 --- /dev/null +++ b/tests/tFullPCLayerTemplatingVerify.cpp @@ -0,0 +1,726 @@ +/** + * @file tFullPCLayerTemplatingVerify.cpp + * @brief TEMP (Quantization branch templating pass): bit-exact + * comparison of FullPCLayer vs. FullPCLayerT, identically + * seeded/constructed, run through several real TrainStep-shaped + * iterations (CalculateState/UpdateState/UpdateWeights, two-layer + * network with a terminal), diffing every externally-visible buffer + * (z, e, mu, W, b, Psi) bit-for-bit (==, not tolerance-based) after + * each iteration. This is the plan's verification gate #1 exercised + * directly against the new class, not just "did anything else + * regress" (that's tFullPCLayerVerify.cpp and friends, unchanged). + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Deep; + +namespace +{ +bool ExactEqual(const float* a, const float* b, size_t n, const char* label) +{ + for (size_t i = 0; i < n; ++i) + { + if (a[i] != b[i]) + { + printf(" MISMATCH in %s at index %zu: old=%.9g new=%.9g\n", label, i, (double)a[i], + (double)b[i]); + return false; + } + } + return true; +} + +/// @brief Shared by Parts 7/8/9's posit checks (and could have covered +/// Part 5a's Fp4E2M1Value check too, left as-is to avoid touching an +/// already-passing block): grid round trip across all 8 magnitude +/// levels (both signs), clamping beyond the grid's max, and the 1.25- +/// style "exactly between two neighbors" non-throwing check. +template +bool VerifyFourBitGrid(const float* magnitudes, float maxMagnitude, const char* label) +{ + bool ok = true; + for (int i = 0; i < 8; ++i) + { + for (float signedVal : {magnitudes[i], -magnitudes[i]}) + { + float decoded = float(QuantT(signedVal)); + float expected = (signedVal == 0.0f) ? 0.0f : signedVal; + if (decoded != expected) + { + printf(" FAIL: %s grid point %.9g round-tripped to %.9g\n", label, (double)expected, + (double)decoded); + ok = false; + } + } + } + + for (float beyond : {maxMagnitude + 1.0f, maxMagnitude * 20.0f, -(maxMagnitude + 1.0f), + -(maxMagnitude * 20.0f)}) + { + float decoded = float(QuantT(beyond)); + float expectedClamp = beyond > 0.0f ? maxMagnitude : -maxMagnitude; + if (decoded != expectedClamp) + { + printf(" FAIL: %s(%.9g) clamped to %.9g, expected %.9g\n", label, (double)beyond, + (double)decoded, (double)expectedClamp); + ok = false; + } + } + return ok; +} + +/// @brief Shared by Parts 7/8/9's posit checks: FullPCNetworkT trains for 100 steps with no NaN/Inf/divergence -- no +/// accuracy bar at 4 bits, same scope as Fp4E2M1Value's Part 5b. +template +bool VerifyFourBitTrainingSmoke(const std::vector& input, const std::vector& target, + size_t inSize, size_t hidSize, size_t outSize, size_t batchSize, + float lr, float ir, float fl, float lmbda, int settlingSteps, + OptimizerType opt, unsigned seed, const char* label) +{ + FullPCNetworkT net(batchSize); + net.AddLayer(inSize, hidSize, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + net.AddLayer(hidSize, outSize, outSize, lr, ir, fl, lmbda, ActivationType::SIGMOID, + ActivationType::dSIGMOID); + net.AddLayer(outSize, 0, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + net.Compile(); + net.SetOptimizer(opt); + std::mt19937 netSeed(seed); + net.RandomizeWeights(netSeed); + + std::vector inputQ(input.begin(), input.end()); + std::vector targetQ(target.begin(), target.end()); + + for (int step = 0; step < 100; ++step) + { + float e = net.TrainStep(inputQ, targetQ, settlingSteps); + if (std::isnan(e) || std::isinf(e)) + { + printf(" FAIL: %s energy is NaN/Inf at step %d\n", label, step); + return false; + } + if (std::fabs(e) > 1e6f) + { + printf(" FAIL: %s energy diverged to %.9g at step %d\n", label, (double)e, step); + return false; + } + } + return true; +} +} // namespace + +int main() +{ + const size_t inSize = 8, hidSize = 8, outSize = 4, batchSize = 4; + const float lr = 0.01f, ir = 0.1f, fl = 0.001f, lmbda = 0.0f; + + std::vector input(batchSize * inSize), target(batchSize * outSize); + std::mt19937 dataRng(42); + std::uniform_real_distribution dist(-1.0f, 1.0f); + for (auto& v : input) + v = dist(dataRng); + for (auto& v : target) + v = dist(dataRng); + + // --- old, non-templated FullPCLayer, two layers (hidden + terminal) --- + FullPCLayer oldIn(inSize, hidSize, outSize, batchSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + FullPCLayer oldHid(hidSize, outSize, outSize, batchSize, lr, ir, fl, lmbda, + ActivationType::SIGMOID, ActivationType::dSIGMOID); + FullPCLayer oldOut(outSize, 0, outSize, batchSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + oldIn.SetLayerAbove(&oldHid); + oldHid.SetLayerBelow(&oldIn); + oldHid.SetLayerAbove(&oldOut); + oldOut.SetLayerBelow(&oldHid); + oldIn.SetTerminalLayer(&oldOut); + oldHid.SetTerminalLayer(&oldOut); + oldOut.SetTerminalLayer(&oldOut); + + std::mt19937 seedRng1(7); + oldIn.RandomizeWeights(seedRng1); + std::mt19937 seedRng1b(7); + oldHid.RandomizeWeights(seedRng1b); + + // --- new, templated FullPCLayerT, identically shaped --- + FullPCLayerT newIn(inSize, hidSize, outSize, batchSize, lr, ir, fl, lmbda, + ActivationType::LINEAR, ActivationType::dLINEAR); + FullPCLayerT newHid(hidSize, outSize, outSize, batchSize, lr, ir, fl, lmbda, + ActivationType::SIGMOID, ActivationType::dSIGMOID); + FullPCLayerT newOut(outSize, 0, outSize, batchSize, lr, ir, fl, lmbda, + ActivationType::LINEAR, ActivationType::dLINEAR); + newIn.SetLayerAbove(&newHid); + newHid.SetLayerBelow(&newIn); + newHid.SetLayerAbove(&newOut); + newOut.SetLayerBelow(&newHid); + newIn.SetTerminalLayer(&newOut); + newHid.SetTerminalLayer(&newOut); + newOut.SetTerminalLayer(&newOut); + + std::mt19937 seedRng2(7); + newIn.RandomizeWeights(seedRng2); + std::mt19937 seedRng2b(7); + newHid.RandomizeWeights(seedRng2b); + + oldIn.SetOptimizer(OptimizerType::ADAM); + oldHid.SetOptimizer(OptimizerType::ADAMW); + oldIn.SetMomentum(true); + newIn.SetOptimizer(OptimizerType::ADAM); + newHid.SetOptimizer(OptimizerType::ADAMW); + newIn.SetMomentum(true); + + // Identical seeding must produce identical initial weights before + // any training happens at all. + bool ok = true; + ok &= ExactEqual(oldIn.GetWeights(), newIn.GetWeights(), inSize * hidSize, "W (input layer, init)"); + ok &= ExactEqual(oldHid.GetWeights(), newHid.GetWeights(), hidSize * outSize, + "W (hidden layer, init)"); + + const int settlingSteps = 5; + const int trainSteps = 10; + + for (int step = 0; step < trainSteps; ++step) + { + oldIn.ClampState(input); + newIn.ClampState(input); + oldOut.ClampState(target); + newOut.ClampState(target); + + for (int s = 0; s < settlingSteps; ++s) + { + oldOut.CalculateState(); + oldHid.CalculateState(); + oldIn.CalculateState(); + newOut.CalculateState(); + newHid.CalculateState(); + newIn.CalculateState(); + + oldHid.UpdateState(); + oldIn.UpdateState(); + newHid.UpdateState(); + newIn.UpdateState(); + } + + oldHid.UpdateWeights(); + oldIn.UpdateWeights(); + newHid.UpdateWeights(); + newIn.UpdateWeights(); + + oldIn.UnclampState(); + newIn.UnclampState(); + oldOut.UnclampState(); + newOut.UnclampState(); + + char label[64]; + snprintf(label, sizeof(label), "z (input, step %d)", step); + ok &= ExactEqual(oldIn.GetBeliefs(), newIn.GetBeliefs(), batchSize * inSize, label); + snprintf(label, sizeof(label), "e (hidden, step %d)", step); + ok &= ExactEqual(oldHid.GetErrors(), newHid.GetErrors(), batchSize * hidSize, label); + snprintf(label, sizeof(label), "mu (hidden, step %d)", step); + ok &= ExactEqual(oldHid.GetMu(), newHid.GetMu(), batchSize * outSize, label); + snprintf(label, sizeof(label), "W (input layer, step %d)", step); + ok &= ExactEqual(oldIn.GetWeights(), newIn.GetWeights(), inSize * hidSize, label); + snprintf(label, sizeof(label), "W (hidden layer, step %d)", step); + ok &= ExactEqual(oldHid.GetWeights(), newHid.GetWeights(), hidSize * outSize, label); + snprintf(label, sizeof(label), "b (hidden layer, step %d)", step); + ok &= ExactEqual(oldHid.GetBiases(), newHid.GetBiases(), outSize, label); + } + + printf(ok ? "PASS: FullPCLayerT is bit-exact against FullPCLayer\n" + : "FAIL: see mismatches above\n"); + + // --- Part 1b: same as Part 1, but with cross-entropy enabled on the + // terminal layer -- ComputeSoftmaxCrossEntropyError(AndEnergy)'s + // generic (AccumT-casting) implementation in CPUBackendT was never + // otherwise exercised against the original CPUBackend's dedicated + // float path. --- + bool ok1b = true; + { + FullPCLayer oldIn2(inSize, outSize, outSize, batchSize, lr, ir, fl, lmbda, + ActivationType::LINEAR, ActivationType::dLINEAR); + FullPCLayer oldOut2(outSize, 0, outSize, batchSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + oldIn2.SetLayerAbove(&oldOut2); + oldOut2.SetLayerBelow(&oldIn2); + oldIn2.SetTerminalLayer(&oldOut2); + oldOut2.SetTerminalLayer(&oldOut2); + oldOut2.SetCrossEntropy(true); + std::mt19937 ceSeed1(13); + oldIn2.RandomizeWeights(ceSeed1); + + FullPCLayerT newIn2(inSize, outSize, outSize, batchSize, lr, ir, fl, lmbda, + ActivationType::LINEAR, ActivationType::dLINEAR); + FullPCLayerT newOut2(outSize, 0, outSize, batchSize, lr, ir, fl, lmbda, + ActivationType::LINEAR, ActivationType::dLINEAR); + newIn2.SetLayerAbove(&newOut2); + newOut2.SetLayerBelow(&newIn2); + newIn2.SetTerminalLayer(&newOut2); + newOut2.SetTerminalLayer(&newOut2); + newOut2.SetCrossEntropy(true); + std::mt19937 ceSeed2(13); + newIn2.RandomizeWeights(ceSeed2); + + std::vector onehot(batchSize * outSize, 0.0f); + for (size_t b = 0; b < batchSize; ++b) + onehot[b * outSize + (b % outSize)] = 1.0f; + + for (int step = 0; step < 5; ++step) + { + oldIn2.ClampState(input); + newIn2.ClampState(input); + oldOut2.ClampState(onehot); + newOut2.ClampState(onehot); + + for (int s = 0; s < settlingSteps; ++s) + { + float oldE = oldOut2.CalculateState(true); + float newE = newOut2.CalculateState(true); + oldIn2.CalculateState(); + newIn2.CalculateState(); + oldIn2.UpdateState(); + newIn2.UpdateState(); + + char label[64]; + snprintf(label, sizeof(label), "cross-entropy energy (step %d, settle %d)", step, s); + ok1b &= ExactEqual(&oldE, &newE, 1, label); + } + + oldIn2.UpdateWeights(); + newIn2.UpdateWeights(); + oldIn2.UnclampState(); + newIn2.UnclampState(); + oldOut2.UnclampState(); + newOut2.UnclampState(); + } + + ok1b &= ExactEqual(oldIn2.GetWeights(), newIn2.GetWeights(), inSize * outSize, + "cross-entropy final W"); + } + + printf(ok1b ? "PASS: FullPCLayerT cross-entropy path is bit-exact\n" + : "FAIL: see mismatches above\n"); + + // --- Part 2: FullPCNetwork vs. FullPCNetworkT, a real + // 3-layer network (input/hidden/terminal) built through AddLayer()/ + // Compile(), run through several TrainStep()s -- exercises the + // network-level orchestration (ProjectForward/DirectFeedbackUpdate/ + // Step/UpdateWeights/Compile's muPC+residual machinery), not just + // the per-layer mechanics Part 1 already covered. --- + bool ok2 = true; + { + FullPCNetwork oldNet(batchSize); + oldNet.AddLayer(inSize, hidSize, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + oldNet.AddLayer(hidSize, outSize, outSize, lr, ir, fl, lmbda, ActivationType::SIGMOID, + ActivationType::dSIGMOID); + oldNet.AddLayer(outSize, 0, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + oldNet.SetUseMuPCScaling(true); + oldNet.Compile(); + std::mt19937 netSeed1(99); + oldNet.RandomizeWeights(netSeed1); + + FullPCNetworkT newNet(batchSize); + newNet.AddLayer(inSize, hidSize, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + newNet.AddLayer(hidSize, outSize, outSize, lr, ir, fl, lmbda, ActivationType::SIGMOID, + ActivationType::dSIGMOID); + newNet.AddLayer(outSize, 0, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + newNet.SetUseMuPCScaling(true); + newNet.Compile(); + std::mt19937 netSeed2(99); + newNet.RandomizeWeights(netSeed2); + + for (int step = 0; step < trainSteps; ++step) + { + float oldEnergy = oldNet.TrainStep(input, target, settlingSteps); + float newEnergy = newNet.TrainStep(input, target, settlingSteps); + + char label[64]; + snprintf(label, sizeof(label), "network energy (step %d)", step); + ok2 &= ExactEqual(&oldEnergy, &newEnergy, 1, label); + + snprintf(label, sizeof(label), "network terminal beliefs (step %d)", step); + ok2 &= ExactEqual(oldNet.GetTerminalLayer()->GetBeliefs(), + newNet.GetTerminalLayer()->GetBeliefs(), batchSize * outSize, label); + + snprintf(label, sizeof(label), "network input-layer W (step %d)", step); + ok2 &= ExactEqual(oldNet.GetLayers()[0]->GetWeights(), newNet.GetLayers()[0]->GetWeights(), + inSize * hidSize, label); + } + + std::vector oldPred = oldNet.Predict(input, settlingSteps); + std::vector newPred = newNet.Predict(input, settlingSteps); + ok2 &= ExactEqual(oldPred.data(), newPred.data(), oldPred.size(), "Predict() output"); + } + + printf(ok2 ? "PASS: FullPCNetworkT is bit-exact against FullPCNetwork\n" + : "FAIL: see mismatches above\n"); + + // --- Part 3: FullPCNetworkT qualitative convergence + // smoke test (plan's gate #2) -- double has no bit-exact reference to + // diff against (FullPCNetwork is float-only), so this just confirms + // training actually runs and the energy trends down, plus that + // MatMul() really dispatches to cblas_dgemm for T=double (verified + // directly in CPUBackendT::MatMul's own source above, not re-derived + // at runtime here). + bool ok3 = true; + { + std::vector inputD(input.begin(), input.end()); + std::vector targetD(target.begin(), target.end()); + + FullPCNetworkT netD(batchSize); + netD.AddLayer(inSize, hidSize, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + netD.AddLayer(hidSize, outSize, outSize, lr, ir, fl, lmbda, ActivationType::SIGMOID, + ActivationType::dSIGMOID); + netD.AddLayer(outSize, 0, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + netD.Compile(); + std::mt19937 netSeedD(99); + netD.RandomizeWeights(netSeedD); + + double firstEnergy = 0.0, lastEnergy = 0.0; + for (int step = 0; step < 200; ++step) + { + double e = netD.TrainStep(inputD, targetD, settlingSteps); + if (std::isnan(e) || std::isinf(e)) + { + printf(" FAIL: FullPCNetworkT energy is NaN/Inf at step %d\n", step); + ok3 = false; + break; + } + if (step == 0) + firstEnergy = e; + lastEnergy = e; + } + + if (ok3 && !(lastEnergy < firstEnergy)) + { + printf(" FAIL: FullPCNetworkT energy did not decrease (first=%.9g " + "last=%.9g)\n", + firstEnergy, lastEnergy); + ok3 = false; + } + } + + printf(ok3 ? "PASS: FullPCNetworkT trains without NaN/Inf and converges\n" + : "FAIL: see mismatches above\n"); + + // --- Part 4: ModelIO round-trip (plan's gate #5) -- save a + // FullPCNetworkT checkpoint, confirm loading it back + // into another FullPCNetworkT reproduces its weights + // exactly, then confirm loading that SAME float32 file into a + // FullPCNetworkT raises on the dtype mismatch -- + // new behavior, ModelIO previously never checked "dtype" on read. --- + bool ok4 = true; + { + std::string path = + (std::filesystem::temp_directory_path() / "deepity_modelio_templating_test.bin").string(); + + FullPCNetworkT srcNet(batchSize); + srcNet.AddLayer(inSize, outSize, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + srcNet.AddLayer(outSize, 0, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + srcNet.Compile(); + std::mt19937 srcSeed(123); + srcNet.RandomizeWeights(srcSeed); + + if (!Deep::ModelIO::Save(srcNet, path)) + { + printf(" FAIL: ModelIO::Save() couldn't open %s for writing\n", path.c_str()); + ok4 = false; + } + + FullPCNetworkT dstNet(batchSize); + dstNet.AddLayer(inSize, outSize, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + dstNet.AddLayer(outSize, 0, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + dstNet.Compile(); + + if (ok4 && !Deep::ModelIO::Load(dstNet, path)) + { + printf(" FAIL: ModelIO::Load() couldn't open %s for reading\n", path.c_str()); + ok4 = false; + } + + if (ok4) + ok4 &= ExactEqual(srcNet.GetLayers()[0]->GetWeights(), dstNet.GetLayers()[0]->GetWeights(), + inSize * outSize, "ModelIO round-trip W (layer 0)"); + + if (ok4) + { + FullPCNetworkT mismatchedNet(batchSize); + mismatchedNet.AddLayer(inSize, outSize, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + mismatchedNet.AddLayer(outSize, 0, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + mismatchedNet.Compile(); + + bool threw = false; + try + { + Deep::ModelIO::Load(mismatchedNet, path); + } + catch (const std::runtime_error&) + { + threw = true; + } + + if (!threw) + { + printf(" FAIL: ModelIO::Load() didn't throw loading a float32 file into a " + "FullPCNetworkT\n"); + ok4 = false; + } + } + + std::filesystem::remove(path); + } + + printf(ok4 ? "PASS: ModelIO round-trips and rejects a dtype mismatch\n" + : "FAIL: see mismatches above\n"); + + // --- Part 5a: Fp4E2M1Value Decode(Encode(x)) round trip (plan's + // gate #3, first half) -- swept across and beyond the format's grid + // range ([-6, 6]), confirming every grid point round-trips exactly + // and anything beyond the range clamps to the nearest extreme (+-6) + // rather than wrapping/overflowing. --- + bool ok5a = true; + { + for (float g : detail::kFp4E2M1Magnitudes) + { + for (float signedVal : {g, -g}) + { + Fp4E2M1Value v(signedVal); + float decoded = float(v); + float expected = (signedVal == 0.0f) ? 0.0f : signedVal; + if (decoded != expected) + { + printf(" FAIL: Fp4E2M1Value grid point %.9g round-tripped to %.9g\n", + (double)expected, (double)decoded); + ok5a = false; + } + } + } + + for (float beyond : {7.0f, 100.0f, -7.0f, -100.0f}) + { + float decoded = float(Fp4E2M1Value(beyond)); + float expectedClamp = beyond > 0.0f ? 6.0f : -6.0f; + if (decoded != expectedClamp) + { + printf(" FAIL: Fp4E2M1Value(%.9g) clamped to %.9g, expected %.9g\n", (double)beyond, + (double)decoded, (double)expectedClamp); + ok5a = false; + } + } + + // A value exactly halfway between two grid points (1.25, between 1 + // and 1.5) should round to one side consistently without throwing + // or producing a third value. + float halfway = float(Fp4E2M1Value(1.25f)); + if (halfway != 1.0f && halfway != 1.5f) + { + printf(" FAIL: Fp4E2M1Value(1.25) decoded to %.9g, neither grid neighbor\n", + (double)halfway); + ok5a = false; + } + } + + printf(ok5a ? "PASS: Fp4E2M1Value Decode(Encode(x)) round-trips and clamps correctly\n" + : "FAIL: see mismatches above\n"); + + // --- Part 5b: FullPCNetworkT training smoke + // test (plan's gate #3, second half) -- no accuracy bar expected at + // 4 bits of storage, just confirming the whole TrainStep path (ADAM + // on the hidden layer, ADAMW on the input layer, exercising the + // MatMulAccum/SumRowsAccum gradient path this phase added) runs for + // many steps without NaN/Inf or blowing up. --- + bool ok5b = true; + { + FullPCNetworkT netQ(batchSize); + netQ.AddLayer(inSize, hidSize, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + netQ.AddLayer(hidSize, outSize, outSize, lr, ir, fl, lmbda, ActivationType::SIGMOID, + ActivationType::dSIGMOID); + netQ.AddLayer(outSize, 0, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + netQ.Compile(); + netQ.SetOptimizer(OptimizerType::ADAM); + std::mt19937 netSeedQ(7); + netQ.RandomizeWeights(netSeedQ); + + std::vector inputQ(input.begin(), input.end()); + std::vector targetQ(target.begin(), target.end()); + + for (int step = 0; step < 100; ++step) + { + float e = netQ.TrainStep(inputQ, targetQ, settlingSteps); + if (std::isnan(e) || std::isinf(e)) + { + printf(" FAIL: FullPCNetworkT energy is NaN/Inf at step %d\n", step); + ok5b = false; + break; + } + if (std::fabs(e) > 1e6f) + { + printf(" FAIL: FullPCNetworkT energy diverged to %.9g at step %d\n", + (double)e, step); + ok5b = false; + break; + } + } + } + + printf(ok5b ? "PASS: FullPCNetworkT trains without NaN/Inf or divergence\n" + : "FAIL: see mismatches above\n"); + + // --- Part 6a: hwy::float16_t round trip (plan's gate #3, applied to + // the correctness-first wrapper scope chosen for this format -- see + // CPUBackendT.cpp's own "hwy::ConvertScalarTo" comment). Unlike + // Fp4E2M1Value's fixed 8-point grid, float16 is IEEE-754 half: this + // checks representative values round-trip within half's own ~3 + // decimal digits of precision (not bit-exact, that's not what half + // precision promises), confirms a genuine zero stays zero, and + // confirms a magnitude beyond half's ~65504 max produces +-Inf + // rather than silently wrapping or clamping (half's real overflow + // behavior, deliberately different from Fp4E2M1Value's clamp-to-grid + // design). --- + bool ok6a = true; + { + for (float v : {0.0f, -0.0f, 1.0f, -1.0f, 0.5f, 100.0f, -100.0f, 3.14159f, 1e-3f}) + { + hwy::float16_t h = hwy::ConvertScalarTo(v); + float back = hwy::ConvertScalarTo(h); + float tol = std::max(1e-3f, std::fabs(v) * 5e-3f); + if (std::fabs(back - v) > tol) + { + printf(" FAIL: hwy::float16_t round trip of %.9g came back as %.9g (tol %.9g)\n", + (double)v, (double)back, (double)tol); + ok6a = false; + } + } + + float zeroBack = hwy::ConvertScalarTo(hwy::ConvertScalarTo(0.0f)); + if (zeroBack != 0.0f) + { + printf(" FAIL: hwy::float16_t(0.0f) round-tripped to %.9g, expected exactly 0\n", + (double)zeroBack); + ok6a = false; + } + + float overflowBack = + hwy::ConvertScalarTo(hwy::ConvertScalarTo(1e9f)); + if (!std::isinf(overflowBack)) + { + printf(" FAIL: hwy::float16_t(1e9f) round-tripped to %.9g, expected +Inf\n", + (double)overflowBack); + ok6a = false; + } + } + + printf(ok6a ? "PASS: hwy::float16_t round-trips within half precision and overflows to Inf\n" + : "FAIL: see mismatches above\n"); + + // --- Part 6b: FullPCNetworkT training smoke + // test, same structure and same "no accuracy bar" scope as Part 5b. --- + bool ok6b = true; + { + FullPCNetworkT netH(batchSize); + netH.AddLayer(inSize, hidSize, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + netH.AddLayer(hidSize, outSize, outSize, lr, ir, fl, lmbda, ActivationType::SIGMOID, + ActivationType::dSIGMOID); + netH.AddLayer(outSize, 0, outSize, lr, ir, fl, lmbda, ActivationType::LINEAR, + ActivationType::dLINEAR); + netH.Compile(); + netH.SetOptimizer(OptimizerType::ADAMW); + std::mt19937 netSeedH(11); + netH.RandomizeWeights(netSeedH); + + std::vector inputH(input.begin(), input.end()); + std::vector targetH(target.begin(), target.end()); + + for (int step = 0; step < 100; ++step) + { + float e = netH.TrainStep(inputH, targetH, settlingSteps); + if (std::isnan(e) || std::isinf(e)) + { + printf(" FAIL: FullPCNetworkT energy is NaN/Inf at step %d\n", + step); + ok6b = false; + break; + } + if (std::fabs(e) > 1e6f) + { + printf(" FAIL: FullPCNetworkT energy diverged to %.9g at step %d\n", + (double)e, step); + ok6b = false; + break; + } + } + } + + printf(ok6b ? "PASS: FullPCNetworkT trains without NaN/Inf or divergence\n" + : "FAIL: see mismatches above\n"); + + // --- Parts 7/8/9: the three posit formats, Phase 4 (plan's "just + // new grid tables" once the wrapper pattern -- proven in Phase 2 -- + // exists). Each is a grid round-trip/clamp check plus a training + // smoke test, same scope and gate as Fp4E2M1Value's Part 5a/5b. --- + bool ok7a = VerifyFourBitGrid(detail::kPosit4_0Magnitudes, 4.0f, "Posit4_0Value"); + printf(ok7a ? "PASS: Posit4_0Value Decode(Encode(x)) round-trips and clamps correctly\n" + : "FAIL: see mismatches above\n"); + + bool ok7b = VerifyFourBitTrainingSmoke(input, target, inSize, hidSize, outSize, + batchSize, lr, ir, fl, lmbda, + settlingSteps, OptimizerType::ADAM, 13, + "FullPCNetworkT"); + printf(ok7b ? "PASS: FullPCNetworkT trains without NaN/Inf or divergence\n" + : "FAIL: see mismatches above\n"); + + bool ok8a = VerifyFourBitGrid(detail::kPosit4_1Magnitudes, 16.0f, "Posit4_1Value"); + printf(ok8a ? "PASS: Posit4_1Value Decode(Encode(x)) round-trips and clamps correctly\n" + : "FAIL: see mismatches above\n"); + + bool ok8b = VerifyFourBitTrainingSmoke(input, target, inSize, hidSize, outSize, + batchSize, lr, ir, fl, lmbda, + settlingSteps, OptimizerType::ADAMW, 17, + "FullPCNetworkT"); + printf(ok8b ? "PASS: FullPCNetworkT trains without NaN/Inf or divergence\n" + : "FAIL: see mismatches above\n"); + + bool ok9a = + VerifyFourBitGrid(detail::kPosit4_2Magnitudes, 256.0f, "Posit4_2Value"); + printf(ok9a ? "PASS: Posit4_2Value Decode(Encode(x)) round-trips and clamps correctly\n" + : "FAIL: see mismatches above\n"); + + bool ok9b = VerifyFourBitTrainingSmoke(input, target, inSize, hidSize, outSize, + batchSize, lr, ir, fl, lmbda, + settlingSteps, OptimizerType::ADAM, 19, + "FullPCNetworkT"); + printf(ok9b ? "PASS: FullPCNetworkT trains without NaN/Inf or divergence\n" + : "FAIL: see mismatches above\n"); + + bool allOk = ok && ok1b && ok2 && ok3 && ok4 && ok5a && ok5b && ok6a && ok6b && ok7a && ok7b && + ok8a && ok8b && ok9a && ok9b; + return allOk ? 0 : 1; +} From e19d764039567fe24d2bd29a9fe5866048eeea1d Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Fri, 30 Oct 2026 13:11:32 -0400 Subject: [PATCH 2/2] quantization + ModelIO + safety work --- CMakeLists.txt | 1 + bindings/BindingHelpers.h | 22 ++ bindings/LayerBindings.cpp | 109 ++++-- bindings/NetworkBindings.cpp | 112 +++++- bindings/QuantFormatBindings.cpp | 82 +++++ bindings/QuantFormatBindings.h | 14 + bindings/pybinding.cpp | 2 + deepity/FullPCN.py | 6 + examples/mnist_fullpcn.py | 136 +++++++ include/deepity/backend/CPUBackend.h | 32 ++ include/deepity/backend/CPUBackendT.h | 8 + include/deepity/backend/CUDABackend.h | 32 ++ include/deepity/backend/IComputeBackend.h | 40 ++ include/deepity/layers/FullPCLayerT.h | 39 +- include/deepity/networks/ConvPCNetwork.h | 2 +- include/deepity/networks/DirectKPPCNetwork.h | 2 +- include/deepity/networks/FullConvPCNetwork.h | 2 +- include/deepity/networks/FullPCNetwork.h | 2 +- include/deepity/networks/FullPCNetworkT.h | 2 +- .../deepity/networks/GaussSeidelPCNetwork.h | 2 +- .../deepity/networks/SimpleConvPCNetwork.h | 2 +- include/deepity/networks/SimplePCNetwork.h | 2 +- include/deepity/utils/ModelIO.h | 36 ++ src/backend/CPUBackendT.cpp | 112 ++++++ src/layers/FullPCLayerT.cpp | 133 ++++--- src/networks/ConvPCNetwork.cpp | 2 +- src/networks/DirectKPPCNetwork.cpp | 2 +- src/networks/FullConvPCNetwork.cpp | 2 +- src/networks/FullPCNetwork.cpp | 2 +- src/networks/GaussSeidelPCNetwork.cpp | 2 +- src/networks/SimpleConvPCNetwork.cpp | 2 +- src/networks/SimplePCNetwork.cpp | 2 +- tests/tBitPackedGemmPrototype.cpp | 346 ++++++++++++++++++ 33 files changed, 1188 insertions(+), 104 deletions(-) create mode 100644 bindings/QuantFormatBindings.cpp create mode 100644 bindings/QuantFormatBindings.h create mode 100644 examples/mnist_fullpcn.py create mode 100644 tests/tBitPackedGemmPrototype.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 95fd2d2a..03351278 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1004,6 +1004,7 @@ if(DEEPITY_BUILD_PYTHON_BINDINGS AND NOT CMAKE_BUILD_TYPE STREQUAL "Debug") bindings/pybinding.cpp bindings/LayerBindings.cpp bindings/NetworkBindings.cpp + bindings/QuantFormatBindings.cpp bindings/UtilityBindings.cpp ) diff --git a/bindings/BindingHelpers.h b/bindings/BindingHelpers.h index 51ac7fbe..58a14ff9 100644 --- a/bindings/BindingHelpers.h +++ b/bindings/BindingHelpers.h @@ -50,6 +50,28 @@ nb::ndarray CopyToNewArray(const T* src, std::initializer_list(data, shape, owner); } +/// @brief Same as CopyToNewArray(), but for a layer/network whose +/// storage type T is lossy (e.g. Fp4E2M1Value) and whose Python-facing +/// wire type is AccumT instead: decodes each element via `AccumT(x)` +/// while copying, rather than a raw memcpy -- used for a quantized +/// FullPCLayerT/FullPCNetworkT's .beliefs/.errors/ +/// .weights/predict() results, which can't be a zero-copy view over T +/// (numpy has no dtype for these custom structs) and shouldn't expose +/// T's raw bits to Python even if it did. +template +nb::ndarray DecodeCopyToNewArray(const T* src, + std::initializer_list shape) +{ + size_t n = 1; + for (auto s : shape) + n *= s; + AccumT* data = new AccumT[n]; + for (size_t i = 0; i < n; ++i) + data[i] = AccumT(src[i]); + nb::capsule owner(data, [](void* p) noexcept { delete[] static_cast(p); }); + return nb::ndarray(data, shape, owner); +} + /// @brief Maps a Python-facing activation name (e.g. "relu", "dgelu") to /// its Deep::ActivationType enumerator, for layers/networks that /// dispatch on the enum rather than a raw function pointer. diff --git a/bindings/LayerBindings.cpp b/bindings/LayerBindings.cpp index 60de44a1..098c1c3e 100644 --- a/bindings/LayerBindings.cpp +++ b/bindings/LayerBindings.cpp @@ -15,8 +15,11 @@ #include #include #include +#include #include +#include + #include #include @@ -193,28 +196,40 @@ void BindCommonPCLayer(nb::class_& cls, const char* classNa // SetBiasesHelper above, for BindFullPCLayerT -- those three // are hardcoded to FloatArray since every OTHER bound layer is float-only; // FullPCLayerT is the one type binding once per T. -template void ClampStateHelperT(LayerT& self, TArray input) +// +// The Python-facing wire type here is AccumT, not T: for float/float and +// double/double this is T itself (no behavior change), and for every +// quantized T (Fp4E2M1Value, the posits, hwy::float16_t) Python always +// speaks AccumT (float) -- quantization happens transparently at this +// boundary via T's own explicit converting constructor, which +// std::vector's range constructor below invokes per element +// (direct-initialization, so it's allowed even though that constructor +// is `explicit`). +template +void ClampStateHelperT(LayerT& self, TArray input) { std::vector values(input.data(), input.data() + input.size()); self.ClampState(values); } -template -void SetWeightsHelperT(LayerT& self, TArray values, size_t expectedCount) +template +void SetWeightsHelperT(LayerT& self, TArray values, size_t expectedCount) { if ((size_t)values.size() != expectedCount) throw std::invalid_argument("set_weights: expected " + std::to_string(expectedCount) + " values, got " + std::to_string(values.size())); - self.GetBackend()->CopyFromHost(self.GetWeights(), values.data(), expectedCount); + std::vector encoded(values.data(), values.data() + values.size()); + self.GetBackend()->CopyFromHost(self.GetWeights(), encoded.data(), expectedCount); } -template -void SetBiasesHelperT(LayerT& self, TArray values, size_t expectedCount) +template +void SetBiasesHelperT(LayerT& self, TArray values, size_t expectedCount) { if ((size_t)values.size() != expectedCount) throw std::invalid_argument("set_biases: expected " + std::to_string(expectedCount) + " values, got " + std::to_string(values.size())); - self.GetBackend()->CopyFromHost(self.GetBiases(), values.data(), expectedCount); + std::vector encoded(values.data(), values.data() + values.size()); + self.GetBackend()->CopyFromHost(self.GetBiases(), encoded.data(), expectedCount); } /// @brief Templated counterpart to BindCommonPCLayer()+FullPCLayer's own @@ -254,7 +269,7 @@ void BindFullPCLayerT(nb::module_& m, const char* className) .def("calculate_state", static_cast(&LayerT::CalculateState)) .def("update_state", &LayerT::UpdateState) .def("update_weights", &LayerT::UpdateWeights) - .def("clamp_state", &ClampStateHelperT, nb::arg("input")) + .def("clamp_state", &ClampStateHelperT, nb::arg("input")) .def("unclamp_state", &LayerT::UnclampState) .def("randomize_weights", &RandomizeWeightsHelper, nb::arg("seed") = nb::none(), nb::arg("distribution") = "normal") @@ -300,53 +315,92 @@ void BindFullPCLayerT(nb::module_& m, const char* className) "set_mu_pc_scale.") .def( "set_weights", - [](LayerT& self, TArray values) - { SetWeightsHelperT(self, values, (size_t)self.GetOutputSize() * (size_t)self.GetInputSize()); }, + [](LayerT& self, TArray values) + { + SetWeightsHelperT( + self, values, (size_t)self.GetOutputSize() * (size_t)self.GetInputSize()); + }, nb::arg("values")) .def( "set_biases", - [](LayerT& self, TArray values) - { SetBiasesHelperT(self, values, (size_t)self.GetOutputSize()); }, + [](LayerT& self, TArray values) + { SetBiasesHelperT(self, values, (size_t)self.GetOutputSize()); }, nb::arg("values")) .def_prop_ro("beliefs", [](LayerT& self) -> nb::object { auto shape = {(size_t)self.GetBatchSize(), (size_t)self.GetInputSize()}; - if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) - return nb::cast(ViewOwnedBy(self.GetBeliefs(), shape, self)); - return nb::cast(CopyFromBackend(self, self.GetBeliefs(), shape)); + if constexpr (!std::is_same_v) + { + return nb::cast(DecodeCopyToNewArray(self.GetBeliefs(), shape)); + } + else + { + if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) + return nb::cast(ViewOwnedBy(self.GetBeliefs(), shape, self)); + return nb::cast(CopyFromBackend(self, self.GetBeliefs(), shape)); + } }) .def_prop_ro("errors", [](LayerT& self) -> nb::object { auto shape = {(size_t)self.GetBatchSize(), (size_t)self.GetInputSize()}; - if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) - return nb::cast(ViewOwnedBy(self.GetErrors(), shape, self)); - return nb::cast(CopyFromBackend(self, self.GetErrors(), shape)); + if constexpr (!std::is_same_v) + { + return nb::cast(DecodeCopyToNewArray(self.GetErrors(), shape)); + } + else + { + if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) + return nb::cast(ViewOwnedBy(self.GetErrors(), shape, self)); + return nb::cast(CopyFromBackend(self, self.GetErrors(), shape)); + } }) .def_prop_ro("weights", [](LayerT& self) -> nb::object { auto shape = {(size_t)self.GetOutputSize(), (size_t)self.GetInputSize()}; - if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) - return nb::cast(ViewOwnedBy(self.GetWeights(), shape, self)); - return nb::cast(CopyFromBackend(self, self.GetWeights(), shape)); + if constexpr (!std::is_same_v) + { + return nb::cast(DecodeCopyToNewArray(self.GetWeights(), shape)); + } + else + { + if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) + return nb::cast(ViewOwnedBy(self.GetWeights(), shape, self)); + return nb::cast(CopyFromBackend(self, self.GetWeights(), shape)); + } }) .def_prop_ro("biases", [](LayerT& self) -> nb::object { auto shape = {(size_t)self.GetOutputSize()}; - if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) - return nb::cast(ViewOwnedBy(self.GetBiases(), shape, self)); - return nb::cast(CopyFromBackend(self, self.GetBiases(), shape)); + if constexpr (!std::is_same_v) + { + return nb::cast(DecodeCopyToNewArray(self.GetBiases(), shape)); + } + else + { + if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) + return nb::cast(ViewOwnedBy(self.GetBiases(), shape, self)); + return nb::cast(CopyFromBackend(self, self.GetBiases(), shape)); + } }) .def_prop_ro("psi", [](LayerT& self) -> nb::object { auto shape = {(size_t)self.GetInputSize(), (size_t)self.GetTerminalSize()}; - if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) - return nb::cast(ViewOwnedBy(self.GetDirectFeedbackWeights(), shape, self)); - return nb::cast(CopyFromBackend(self, self.GetDirectFeedbackWeights(), shape)); + if constexpr (!std::is_same_v) + { + return nb::cast( + DecodeCopyToNewArray(self.GetDirectFeedbackWeights(), shape)); + } + else + { + if (self.GetBackend()->GetDeviceType() == Deep::DeviceType::DEVICE_CPU) + return nb::cast(ViewOwnedBy(self.GetDirectFeedbackWeights(), shape, self)); + return nb::cast(CopyFromBackend(self, self.GetDirectFeedbackWeights(), shape)); + } }) .def_prop_ro("mu_pc_scale", &LayerT::GetMuPCScale) .def_prop_ro("residual", &LayerT::GetResidual) @@ -1192,4 +1246,5 @@ void bind_layers(nb::module_& m) BindFullPCLayerT(m, "FullPCLayer_f32"); BindFullPCLayerT(m, "FullPCLayer_f64"); + BindFullPCLayerT(m, "FullPCLayer_fp4_e2m1"); } diff --git a/bindings/NetworkBindings.cpp b/bindings/NetworkBindings.cpp index 93258013..756dc6ba 100644 --- a/bindings/NetworkBindings.cpp +++ b/bindings/NetworkBindings.cpp @@ -14,8 +14,12 @@ #include #include #include +#include #include +#include +#include + #include #include #include @@ -76,6 +80,16 @@ template void BindCommonPCNetwork(nb::class_& cls, const c return result; }, "List of layer objects owned by the network.") + .def( + "save", + [](const NetT& self, const std::string& filepath) { return Deep::ModelIO::Save(self, filepath); }, + nb::arg("filepath"), "Save this network's weights to filepath.") + .def( + "load", + [](NetT& self, const std::string& filepath) { return Deep::ModelIO::Load(self, filepath); }, + nb::arg("filepath"), + "Load weights from filepath into this already-compiled network; raises on a dtype " + "mismatch.") .def("__len__", [](const NetT& self) { return self.GetLayers().size(); }) .def( "__getitem__", @@ -164,7 +178,7 @@ void BindFullPCNetworkT(nb::module_& m, const char* className) .def("reset_state", &NetT::ResetState) .def( "clamp_input", - [](NetT& self, TArray input) + [](NetT& self, TArray input) { std::vector values(input.data(), input.data() + input.size()); self.Clamp(values); @@ -203,7 +217,7 @@ void BindFullPCNetworkT(nb::module_& m, const char* className) .def("set_feedback_rate", &NetT::SetFeedbackRate, nb::arg("fl")) .def( "train_step", - [](NetT& self, TArray x, TArray y, int inference_steps) + [](NetT& self, TArray x, TArray y, int inference_steps) { std::vector xvec(x.data(), x.data() + x.size()); std::vector yvec(y.data(), y.data() + y.size()); @@ -212,13 +226,16 @@ void BindFullPCNetworkT(nb::module_& m, const char* className) nb::arg("x"), nb::arg("y"), nb::arg("inference_steps") = 1) .def( "predict", - [](NetT& self, TArray x, int inference_steps) + [](NetT& self, TArray x, int inference_steps) -> nb::object { std::vector xvec(x.data(), x.data() + x.size()); std::vector result = self.Predict(xvec, inference_steps); LayerT* terminal = self.GetTerminalLayer(); - return CopyToNewArray( - result.data(), {(size_t)terminal->GetBatchSize(), (size_t)terminal->GetInputSize()}); + auto shape = {(size_t)terminal->GetBatchSize(), (size_t)terminal->GetInputSize()}; + if constexpr (!std::is_same_v) + return nb::cast(DecodeCopyToNewArray(result.data(), shape)); + else + return nb::cast(CopyToNewArray(result.data(), shape)); }, nb::arg("x"), nb::arg("inference_steps")) .def_prop_ro("batch_size", &NetT::GetBatchSize) @@ -231,6 +248,16 @@ void BindFullPCNetworkT(nb::module_& m, const char* className) return result; }) .def("get_terminal_layer", &NetT::GetTerminalLayer, nb::rv_policy::reference) + .def( + "save", + [](const NetT& self, const std::string& filepath) { return Deep::ModelIO::Save(self, filepath); }, + nb::arg("filepath"), "Save this network's weights to filepath.") + .def( + "load", + [](NetT& self, const std::string& filepath) { return Deep::ModelIO::Load(self, filepath); }, + nb::arg("filepath"), + "Load weights from filepath into this already-compiled network; raises on a dtype " + "mismatch.") .def("__len__", [](const NetT& self) { return self.GetLayers().size(); }) .def( "__getitem__", @@ -577,6 +604,18 @@ void bind_networks(nb::module_& m) result.append(nb::cast(layer.get(), nb::rv_policy::reference)); return result; }) + .def( + "save", + [](const Deep::GaussSeidelPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Save(self, filepath); }, + nb::arg("filepath"), "Save this network's weights to filepath.") + .def( + "load", + [](Deep::GaussSeidelPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Load(self, filepath); }, + nb::arg("filepath"), + "Load weights from filepath into this already-compiled network; raises on a dtype " + "mismatch.") .def("__len__", [](const Deep::GaussSeidelPCNetwork& self) { return self.GetLayers().size(); }) .def( @@ -729,6 +768,18 @@ void bind_networks(nb::module_& m) .def("get_terminal_layer", &Deep::DirectKPPCNetwork::GetTerminalLayer, nb::rv_policy::reference) + .def( + "save", + [](const Deep::DirectKPPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Save(self, filepath); }, + nb::arg("filepath"), "Save this network's weights to filepath.") + .def( + "load", + [](Deep::DirectKPPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Load(self, filepath); }, + nb::arg("filepath"), + "Load weights from filepath into this already-compiled network; raises on a dtype " + "mismatch.") .def("__len__", [](const Deep::DirectKPPCNetwork& self) { return self.GetLayers().size(); }) .def( "__getitem__", @@ -921,6 +972,18 @@ void bind_networks(nb::module_& m) return result; }) .def("get_terminal_layer", &Deep::FullPCNetwork::GetTerminalLayer, nb::rv_policy::reference) + .def( + "save", + [](const Deep::FullPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Save(self, filepath); }, + nb::arg("filepath"), "Save this network's weights to filepath.") + .def( + "load", + [](Deep::FullPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Load(self, filepath); }, + nb::arg("filepath"), + "Load weights from filepath into this already-compiled network; raises on a dtype " + "mismatch.") .def("__len__", [](const Deep::FullPCNetwork& self) { return self.GetLayers().size(); }) .def( "__getitem__", @@ -1155,6 +1218,20 @@ void bind_networks(nb::module_& m) .def("get_terminal_layer", &Deep::FullConvPCNetwork::GetTerminalLayer, nb::rv_policy::reference) + .def( + "save", + [](const Deep::FullConvPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Save(self, filepath); }, + nb::arg("filepath"), + "Save this network's weights to filepath. Safe on a CUDA-device network -- syncs " + "device-resident parameters to host before writing.") + .def( + "load", + [](Deep::FullConvPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Load(self, filepath); }, + nb::arg("filepath"), + "Load weights from filepath into this already-compiled network; raises on a dtype " + "mismatch. Safe on a CUDA-device network -- pushes loaded weights back to device.") .def("__len__", [](const Deep::FullConvPCNetwork& self) { return self.GetLayers().size(); }) .def( "__getitem__", @@ -1302,6 +1379,18 @@ void bind_networks(nb::module_& m) result.append(nb::cast(layer.get(), nb::rv_policy::reference)); return result; }) + .def( + "save", + [](const Deep::ConvPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Save(self, filepath); }, + nb::arg("filepath"), "Save this network's weights to filepath.") + .def( + "load", + [](Deep::ConvPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Load(self, filepath); }, + nb::arg("filepath"), + "Load weights from filepath into this already-compiled network; raises on a dtype " + "mismatch.") .def("__len__", [](const Deep::ConvPCNetwork& self) { return self.GetLayers().size(); }) .def( "__getitem__", @@ -1468,6 +1557,18 @@ void bind_networks(nb::module_& m) result.append(nb::cast(layer.get(), nb::rv_policy::reference)); return result; }) + .def( + "save", + [](const Deep::SimpleConvPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Save(self, filepath); }, + nb::arg("filepath"), "Save this network's weights to filepath.") + .def( + "load", + [](Deep::SimpleConvPCNetwork& self, const std::string& filepath) + { return Deep::ModelIO::Load(self, filepath); }, + nb::arg("filepath"), + "Load weights from filepath into this already-compiled network; raises on a dtype " + "mismatch.") .def("__len__", [](const Deep::SimpleConvPCNetwork& self) { return self.GetLayers().size(); }) .def( "__getitem__", @@ -1493,4 +1594,5 @@ void bind_networks(nb::module_& m) BindFullPCNetworkT(m, "FullPCNetwork_f32"); BindFullPCNetworkT(m, "FullPCNetwork_f64"); + BindFullPCNetworkT(m, "FullPCNetwork_fp4_e2m1"); } diff --git a/bindings/QuantFormatBindings.cpp b/bindings/QuantFormatBindings.cpp new file mode 100644 index 00000000..318781f7 --- /dev/null +++ b/bindings/QuantFormatBindings.cpp @@ -0,0 +1,82 @@ +/** + * @file QuantFormatBindings.cpp + * @brief nanobind bindings for QuantFormat.h's sub-8-bit storage types + * and their free Decode/Encode functions -- previously only reachable + * transitively through a FullPCNetwork_fp4_e2m1/FullPCLayer_fp4_e2m1 + * instance's train_step()/predict(), never directly. Split out like + * LayerBindings.cpp/NetworkBindings.cpp/UtilityBindings.cpp. + */ +#include "QuantFormatBindings.h" +#include "BindingHelpers.h" + +#include + +#include +#include + +#include +#include + +namespace +{ +/// @brief Binds one of the four 1-byte quantization structs (all share +/// the same shape: a raw `bits` field, a from-float constructor that +/// rounds to the nearest grid point, and float/double conversion), so +/// the four calls below don't each repeat this by hand. +template +void BindQuantType(nb::module_& m, const char* className, const char* formatDoc) +{ + nb::class_(m, className, formatDoc) + .def( + "__init__", + [](QuantT* self, float value) { new (self) QuantT(value); }, + nb::arg("value"), + "Rounds value to the nearest representable grid point (saturating at the " + "format's max magnitude).") + .def_rw("bits", &QuantT::bits, "Raw packed bit pattern (bit 3 = sign, bits 2:0 = " + "magnitude index).") + .def("__float__", [](const QuantT& self) { return (float)self; }) + .def("__repr__", + [className](const QuantT& self) + { return std::string(className) + "(" + std::to_string((float)self) + ")"; }); +} +} // namespace + +void bind_quant_formats(nb::module_& m) +{ + BindQuantType( + m, "Fp4E2M1Value", + "FP4 E2M1 (1 sign + 2 exponent + 1 mantissa bit) quantization grid value."); + BindQuantType(m, "Posit4_0Value", "Posit<4,0> (es=0) quantization grid value."); + BindQuantType(m, "Posit4_1Value", "Posit<4,1> (es=1) quantization grid value."); + BindQuantType(m, "Posit4_2Value", "Posit<4,2> (es=2) quantization grid value."); + + m.def("decode_fp4_e2m1", &Deep::DecodeFp4E2M1, nb::arg("bits"), + "Decodes a raw Fp4E2M1Value bit pattern (0-15) to its real value."); + m.def("encode_fp4_e2m1", &Deep::EncodeFp4E2M1, nb::arg("value"), + "Encodes a float to its nearest Fp4E2M1Value bit pattern."); + m.def("decode_posit4_0", &Deep::DecodePosit4_0, nb::arg("bits"), + "Decodes a raw Posit4_0Value bit pattern (0-15) to its real value."); + m.def("encode_posit4_0", &Deep::EncodePosit4_0, nb::arg("value"), + "Encodes a float to its nearest Posit4_0Value bit pattern."); + m.def("decode_posit4_1", &Deep::DecodePosit4_1, nb::arg("bits"), + "Decodes a raw Posit4_1Value bit pattern (0-15) to its real value."); + m.def("encode_posit4_1", &Deep::EncodePosit4_1, nb::arg("value"), + "Encodes a float to its nearest Posit4_1Value bit pattern."); + m.def("decode_posit4_2", &Deep::DecodePosit4_2, nb::arg("bits"), + "Decodes a raw Posit4_2Value bit pattern (0-15) to its real value."); + m.def("encode_posit4_2", &Deep::EncodePosit4_2, nb::arg("value"), + "Encodes a float to its nearest Posit4_2Value bit pattern."); + + // The grids themselves, for introspection -- each format's 8 + // non-negative magnitude levels (index 0 is always 0), mirroring + // QuantFormat.h's own kFp4E2M1Magnitudes/kPosit4_*Magnitudes tables. + m.attr("FP4_E2M1_MAGNITUDES") = std::vector( + std::begin(Deep::detail::kFp4E2M1Magnitudes), std::end(Deep::detail::kFp4E2M1Magnitudes)); + m.attr("POSIT4_0_MAGNITUDES") = std::vector( + std::begin(Deep::detail::kPosit4_0Magnitudes), std::end(Deep::detail::kPosit4_0Magnitudes)); + m.attr("POSIT4_1_MAGNITUDES") = std::vector( + std::begin(Deep::detail::kPosit4_1Magnitudes), std::end(Deep::detail::kPosit4_1Magnitudes)); + m.attr("POSIT4_2_MAGNITUDES") = std::vector( + std::begin(Deep::detail::kPosit4_2Magnitudes), std::end(Deep::detail::kPosit4_2Magnitudes)); +} diff --git a/bindings/QuantFormatBindings.h b/bindings/QuantFormatBindings.h new file mode 100644 index 00000000..d0ab5a03 --- /dev/null +++ b/bindings/QuantFormatBindings.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +/** + * @file QuantFormatBindings.h + * @brief Entry point for the sub-8-bit quantization format bindings + * (Fp4E2M1Value, Posit4_0/1/2Value, and their free Decode/Encode + * functions). See QuantFormatBindings.cpp. + */ + +/// @brief Binds QuantFormat.h's storage types and Decode/Encode +/// functions into `m`. +void bind_quant_formats(nanobind::module_& m); diff --git a/bindings/pybinding.cpp b/bindings/pybinding.cpp index 3e8702b6..17f33ae3 100644 --- a/bindings/pybinding.cpp +++ b/bindings/pybinding.cpp @@ -11,6 +11,7 @@ #include "LayerBindings.h" #include "NetworkBindings.h" +#include "QuantFormatBindings.h" #include "UtilityBindings.h" NB_MODULE(deepity, m) @@ -20,4 +21,5 @@ NB_MODULE(deepity, m) bind_layers(m); bind_networks(m); bind_utilities(m); + bind_quant_formats(m); } diff --git a/deepity/FullPCN.py b/deepity/FullPCN.py index a9c4b41a..88e652c9 100644 --- a/deepity/FullPCN.py +++ b/deepity/FullPCN.py @@ -13,6 +13,12 @@ _IMPL_BY_DTYPE = { "float32": (dy.FullPCNetwork_f32, np.float32), "float64": (dy.FullPCNetwork_f64, np.float64), + # Quantized storage types: Python always speaks float32 here (the + # bound layer's wire type is its AccumT, not its lossy storage type + # T -- see BindFullPCLayerT/BindFullPCNetworkT's own comments in + # bindings/LayerBindings.cpp). The 4-bit encode/decode round trip + # happens transparently inside C++ on every array in/out. + "fp4_e2m1": (dy.FullPCNetwork_fp4_e2m1, np.float32), } diff --git a/examples/mnist_fullpcn.py b/examples/mnist_fullpcn.py new file mode 100644 index 00000000..f19c0e4a --- /dev/null +++ b/examples/mnist_fullpcn.py @@ -0,0 +1,136 @@ +""" +MNIST on FullPCN, across storage dtypes -- built to answer one question: +does a quantized FullPCN (today: float32 vs. fp4_e2m1) train to a useful +accuracy at all? Not tuned for peak accuracy the way examples/mnist.py's +SimplePCN setup is (that one carries a float-only, heavily-tuned +forward-projection-init/weight-decay recipe that doesn't apply here -- +FullPCLayer/FullPCNetwork is the one family of layers this project has +templated over storage type so far; SimplePCLayer has not been). + +Usage: + python examples/mnist_fullpcn.py --dtype float32 + python examples/mnist_fullpcn.py --dtype fp4_e2m1 --ir 4.0 + +Run both with the same --seed to get the apples-to-apples comparison: +identical architecture, hyperparameters, batch order -- the only +difference is what FullPCN quantizes W/b/Psi down to internally. NOTE: +fp4_e2m1 needs a much larger --ir than float32's default (0.1) to train +at all -- see --ir's own help text for why. Expect noisier, less stable +convergence than float32 at that larger --ir: this is a real property of +settling on a coarse 4-bit grid, not a leftover bug. +""" +import argparse +import os +import sys +from time import perf_counter + +os.environ.setdefault("OMP_WAIT_POLICY", "ACTIVE") +os.environ.setdefault("GOMP_SPINCOUNT", "300000000") + +import numpy as np + +sys.path.insert(0, os.path.dirname(__file__)) +from mnist import load_full_mnist # noqa: E402 + +from deepity import FullPCN # noqa: E402 +from deepity.layer import Linear, Sigmoid # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dtype", default="float32", choices=["float32", "float64", "fp4_e2m1"]) + parser.add_argument("--seed", type=int, default=7) + parser.add_argument("--epochs", type=int, default=15) + parser.add_argument("--steps", type=int, default=20, help="settling steps per batch") + parser.add_argument("--batch-size", type=int, default=250) + parser.add_argument("--hidden", type=int, default=256) + parser.add_argument("--lr", type=float, default=0.005) + parser.add_argument("--ir", type=float, default=0.1, help="settling/inference rate; " + "fp4_e2m1 needs this much larger than float32's default to ever cross " + "a grid boundary at all -- see examples/mnist_fullpcn.py's own history " + "for why (an activation derivative, and everything computed from it, is " + "never itself quantized, but the settling step still writes its result " + "into a quantized belief, so a per-step delta smaller than the grid's " + "local spacing is silently discarded, same as a too-small gradient would " + "be discarded updating a quantized weight)") + parser.add_argument("--acc-batches", type=int, default=10, + help="train-set batches sampled for the per-epoch accuracy readout") + args = parser.parse_args() + + X_train, Y_train, X_test, y_test_labels = load_full_mnist() + + print(f"\nBuilding FullPCN(dtype={args.dtype!r}) (784->{args.hidden}->10), " + f"seed={args.seed}...") + net = FullPCN( + Linear(784, args.hidden), + Sigmoid(), + Linear(args.hidden, 10), + batch_size=args.batch_size, + dtype=args.dtype, + ) + net.configure( + learning_rate=args.lr, + inference_rate=args.ir, + feedback_rate=1e-4, + lmbda=0.0, # see examples/mnist.py's own comment: ADAMW+lmbda>0 destabilizes this family + optimizer="ADAM", + use_cross_entropy=True, + seed=args.seed, + ) + + rng = np.random.default_rng(args.seed) + n_batches = len(X_train) // args.batch_size + start_time = perf_counter() + epoch_accs = [] + + for epoch in range(args.epochs): + indices = rng.permutation(len(X_train)) + X_shuf, Y_shuf = X_train[indices], Y_train[indices] + + epoch_energy = 0.0 + for b in range(n_batches): + X_batch = X_shuf[b * args.batch_size:(b + 1) * args.batch_size] + Y_batch = Y_shuf[b * args.batch_size:(b + 1) * args.batch_size] + epoch_energy += net.train_step(X_batch, Y_batch, args.steps) + + # Accuracy readout on a sampled subset of train batches (same + # tradeoff as examples/mnist.py: cheap per-epoch signal, not the + # final number -- that's the held-out test pass after the loop). + correct, total = 0, 0 + for b in range(min(args.acc_batches, n_batches)): + X_batch = X_shuf[b * args.batch_size:(b + 1) * args.batch_size] + Y_batch = Y_shuf[b * args.batch_size:(b + 1) * args.batch_size] + pred = net.predict(X_batch, args.steps) + correct += np.sum(np.argmax(pred, axis=1) == np.argmax(Y_batch, axis=1)) + total += args.batch_size + + epoch_acc = 100.0 * correct / total + epoch_accs.append(epoch_acc) + elapsed = perf_counter() - start_time + avg_energy = epoch_energy / n_batches + print(f"Epoch {epoch + 1}/{args.epochs} | Time: {elapsed:.1f}s | " + f"Acc: {epoch_acc:.2f}% | Avg energy: {avg_energy:.4f}") + + train_time = perf_counter() - start_time + print(f"\nTraining complete in {train_time:.1f}s.") + + print("\nRunning final test evaluation...") + correct, total = 0, 0 + for i in range(0, len(X_test), args.batch_size): + X_batch = X_test[i:i + args.batch_size] + y_labels_batch = y_test_labels[i:i + args.batch_size] + if len(X_batch) != args.batch_size: + continue + pred = net.predict(X_batch, args.steps) + correct += np.sum(np.argmax(pred, axis=1) == y_labels_batch) + total += args.batch_size + + test_acc = 100.0 * correct / total + print(f"\n=== Result (dtype={args.dtype}) ===") + print(f"Peak Test Accuracy: {test_acc:.2f}%") + print(f"Train time: {train_time:.1f}s") + print(f"Per-epoch: {[round(a, 2) for a in epoch_accs]}") + + +if __name__ == "__main__": + main() diff --git a/include/deepity/backend/CPUBackend.h b/include/deepity/backend/CPUBackend.h index dbf033f2..b71050b6 100644 --- a/include/deepity/backend/CPUBackend.h +++ b/include/deepity/backend/CPUBackend.h @@ -85,6 +85,14 @@ class CPUBackend : public IComputeBackend { MatMul(transA, transB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); } + /// @copydoc Deep::IComputeBackendT::MatMulAccumMixed + /// Same "T==AccumT==float always" reasoning as MatMulAccum() above. + void MatMulAccumMixed(bool transA, bool transB, int M, int N, int K, float alpha, + const float* A, int lda, const float* B, int ldb, float beta, float* C, + int ldc) noexcept override + { + MatMul(transA, transB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + } /// @copydoc Deep::IComputeBackend::SetAllowTF32 void SetAllowTF32(bool) noexcept override {} /// @copydoc Deep::IComputeBackend::SumRows @@ -102,6 +110,17 @@ class CPUBackend : public IComputeBackend void Scale(float* buf, size_t n, float alpha) noexcept override; /// @copydoc Deep::IComputeBackend::AxpyInto void AxpyInto(float* y, const float* x, size_t n, float alpha) noexcept override; + /// @copydoc Deep::IComputeBackendT::AxpyIntoAccum + /// Same "T==AccumT==float always" reasoning as MatMulAccum() above. + void AxpyIntoAccum(float* y, const float* x, size_t n, float alpha) noexcept override + { + AxpyInto(y, x, n, alpha); + } + /// @copydoc Deep::IComputeBackendT::AxpyIntoFromAccum + void AxpyIntoFromAccum(float* y, const float* x, size_t n, float alpha) noexcept override + { + AxpyInto(y, x, n, alpha); + } /// @copydoc Deep::IComputeBackend::AddBiasBroadcast void AddBiasBroadcast(float* buf, const float* bias, size_t batchSize, size_t width) noexcept override; @@ -139,6 +158,13 @@ class CPUBackend : public IComputeBackend /// @copydoc Deep::IComputeBackend::ActivationDerivativeInto void ActivationDerivativeInto(ActivationType type, float* dst, const float* src, size_t n) noexcept override; + /// @copydoc Deep::IComputeBackendT::ActivationDerivativeIntoAccum + /// Same "T==AccumT==float always" reasoning as MatMulAccum() above. + void ActivationDerivativeIntoAccum(ActivationType type, float* dst, const float* src, + size_t n) noexcept override + { + ActivationDerivativeInto(type, dst, src, n); + } /// @copydoc Deep::IComputeBackend::FusedStateUpdateMomentum void FusedStateUpdateMomentum(float* z, float* v, const float* feedback, ActivationType dType, @@ -236,6 +262,12 @@ class CPUBackend : public IComputeBackend /// @copydoc Deep::IComputeBackend::MultiplyInto void MultiplyInto(float* dst, const float* a, const float* b, size_t n) noexcept override; + /// @copydoc Deep::IComputeBackendT::MultiplyIntoAccum + /// Same "T==AccumT==float always" reasoning as MatMulAccum() above. + void MultiplyIntoAccum(float* dst, const float* a, const float* b, size_t n) noexcept override + { + MultiplyInto(dst, a, b, n); + } /// @copydoc Deep::IComputeBackend::FusedActivationDerivativeMultiply void FusedActivationDerivativeMultiply(float* dst, const float* a, float* activatedInOut, ActivationType dType, size_t n) noexcept override; diff --git a/include/deepity/backend/CPUBackendT.h b/include/deepity/backend/CPUBackendT.h index 1c49e8ff..8898f225 100644 --- a/include/deepity/backend/CPUBackendT.h +++ b/include/deepity/backend/CPUBackendT.h @@ -80,6 +80,9 @@ class CPUBackendT : public IComputeBackendT void MatMulAccum(bool transA, bool transB, int M, int N, int K, AccumT alpha, const T* A, int lda, const T* B, int ldb, AccumT beta, AccumT* C, int ldc) noexcept override; + void MatMulAccumMixed(bool transA, bool transB, int M, int N, int K, AccumT alpha, + const AccumT* A, int lda, const T* B, int ldb, AccumT beta, AccumT* C, + int ldc) noexcept override; void SumRows(T* dst, const T* src, size_t batchSize, size_t width) noexcept override; void SumRowsAccum(AccumT* dst, const T* src, size_t batchSize, size_t width) noexcept override; @@ -87,6 +90,8 @@ class CPUBackendT : public IComputeBackendT void Scale(T* buf, size_t n, AccumT alpha) noexcept override; void AxpyInto(T* y, const T* x, size_t n, AccumT alpha) noexcept override; + void AxpyIntoAccum(AccumT* y, const T* x, size_t n, AccumT alpha) noexcept override; + void AxpyIntoFromAccum(T* y, const AccumT* x, size_t n, AccumT alpha) noexcept override; void AddBiasBroadcast(T* buf, const T* bias, size_t batchSize, size_t width) noexcept override; bool TryFusedMatMulBias(bool, bool, int, int, int, const T*, int, const T*, int, const T*, T*, @@ -114,6 +119,8 @@ class CPUBackendT : public IComputeBackendT void ActivationInto(ActivationType type, T* dst, const T* src, size_t n) noexcept override; void ActivationDerivative(ActivationType type, T* buf, size_t n, bool activated) noexcept override; void ActivationDerivativeInto(ActivationType type, T* dst, const T* src, size_t n) noexcept override; + void ActivationDerivativeIntoAccum(ActivationType type, AccumT* dst, const T* src, + size_t n) noexcept override; void FusedStateUpdateMomentum(T*, T*, const T*, ActivationType, const T*, size_t, AccumT, AccumT) noexcept override @@ -204,6 +211,7 @@ class CPUBackendT : public IComputeBackendT } void MultiplyInto(T* dst, const T* a, const T* b, size_t n) noexcept override; + void MultiplyIntoAccum(AccumT* dst, const T* a, const AccumT* b, size_t n) noexcept override; void FusedActivationDerivativeMultiply(T*, const T*, T*, ActivationType, size_t) noexcept override { diff --git a/include/deepity/backend/CUDABackend.h b/include/deepity/backend/CUDABackend.h index b6c3d385..3f128aff 100644 --- a/include/deepity/backend/CUDABackend.h +++ b/include/deepity/backend/CUDABackend.h @@ -79,6 +79,14 @@ class CUDABackend : public IComputeBackend { MatMul(transA, transB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); } + /// @copydoc Deep::IComputeBackendT::MatMulAccumMixed + /// Same "T==AccumT==float always" reasoning as SumRowsAccum() above. + void MatMulAccumMixed(bool transA, bool transB, int M, int N, int K, float alpha, + const float* A, int lda, const float* B, int ldb, float beta, float* C, + int ldc) noexcept override + { + MatMul(transA, transB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + } /// @copydoc Deep::IComputeBackend::SetAllowTF32 void SetAllowTF32(bool allow) noexcept override; @@ -86,6 +94,17 @@ class CUDABackend : public IComputeBackend void Scale(float* buf, size_t n, float alpha) noexcept override; /// @copydoc Deep::IComputeBackend::AxpyInto void AxpyInto(float* y, const float* x, size_t n, float alpha) noexcept override; + /// @copydoc Deep::IComputeBackendT::AxpyIntoAccum + /// Same "T==AccumT==float always" reasoning as SumRowsAccum() above. + void AxpyIntoAccum(float* y, const float* x, size_t n, float alpha) noexcept override + { + AxpyInto(y, x, n, alpha); + } + /// @copydoc Deep::IComputeBackendT::AxpyIntoFromAccum + void AxpyIntoFromAccum(float* y, const float* x, size_t n, float alpha) noexcept override + { + AxpyInto(y, x, n, alpha); + } /// @copydoc Deep::IComputeBackend::AddBiasBroadcast void AddBiasBroadcast(float* buf, const float* bias, size_t batchSize, size_t width) noexcept override; @@ -139,6 +158,13 @@ class CUDABackend : public IComputeBackend /// @copydoc Deep::IComputeBackend::ActivationDerivativeInto void ActivationDerivativeInto(ActivationType type, float* dst, const float* src, size_t n) noexcept override; + /// @copydoc Deep::IComputeBackendT::ActivationDerivativeIntoAccum + /// Same "T==AccumT==float always" reasoning as SumRowsAccum() above. + void ActivationDerivativeIntoAccum(ActivationType type, float* dst, const float* src, + size_t n) noexcept override + { + ActivationDerivativeInto(type, dst, src, n); + } /// @copydoc Deep::IComputeBackend::FusedStateUpdate void FusedStateUpdate(float* z, const float* feedback, ActivationType dType, const float* e, @@ -186,6 +212,12 @@ class CUDABackend : public IComputeBackend /// @copydoc Deep::IComputeBackend::MultiplyInto void MultiplyInto(float* dst, const float* a, const float* b, size_t n) noexcept override; + /// @copydoc Deep::IComputeBackendT::MultiplyIntoAccum + /// Same "T==AccumT==float always" reasoning as SumRowsAccum() above. + void MultiplyIntoAccum(float* dst, const float* a, const float* b, size_t n) noexcept override + { + MultiplyInto(dst, a, b, n); + } /// @copydoc Deep::IComputeBackend::FusedActivationDerivativeMultiply void FusedActivationDerivativeMultiply(float* dst, const float* a, float* activatedInOut, ActivationType dType, size_t n) noexcept override; diff --git a/include/deepity/backend/IComputeBackend.h b/include/deepity/backend/IComputeBackend.h index 9b8cc94e..c0754704 100644 --- a/include/deepity/backend/IComputeBackend.h +++ b/include/deepity/backend/IComputeBackend.h @@ -146,6 +146,22 @@ class IComputeBackendT const T* A, int lda, const T* B, int ldb, AccumT beta, AccumT* C, int ldc) noexcept = 0; + /// @brief Same operation as MatMulAccum(), but A is already AccumT* -- + /// for a layer's transient settling/gradient scratch (e.g. the + /// activation-derivative-weighted error signal that feeds both + /// UpdateState()'s feedback GEMM and UpdateWeights()'s gradient GEMM), + /// which must never round-trip through storage precision T even + /// transiently: for a lossy T (e.g. Fp4E2M1Value), an activation + /// derivative's natural range commonly falls entirely below T's + /// coarsest resolution (sigmoid's derivative maxes out at 0.25, below + /// Fp4E2M1Value's smallest nonzero magnitude of 0.5), so writing it + /// into a T* buffer even once rounds it to exactly zero and kills the + /// gradient outright -- this is that fix. Only B (the real, stored + /// parameter, e.g. W) decodes from T; A is used as-is. + virtual void MatMulAccumMixed(bool transA, bool transB, int M, int N, int K, AccumT alpha, + const AccumT* A, int lda, const T* B, int ldb, AccumT beta, + AccumT* C, int ldc) noexcept = 0; + /// @brief Whether MatMul() may use TF32 (10 mantissa bits, ~1e-3 /// relative error per GEMM) instead of full FP32 on tensor cores -- /// CUDABackend only; a no-op on CPUBackend, which has no such @@ -187,6 +203,15 @@ class IComputeBackendT virtual void Scale(T* buf, size_t n, AccumT alpha) noexcept = 0; /// @brief y[i] += alpha * x[i] for all i in [0, n), the standard AXPY. virtual void AxpyInto(T* y, const T* x, size_t n, AccumT alpha) noexcept = 0; + /// @brief Same operation as AxpyInto(), but y is AccumT* and x stays + /// T* -- see MatMulAccumMixed()'s own doc comment for why a settling/ + /// gradient scratch accumulator must never round-trip through T. + virtual void AxpyIntoAccum(AccumT* y, const T* x, size_t n, AccumT alpha) noexcept = 0; + /// @brief Same operation as AxpyInto(), but x is AccumT* -- for + /// applying an AccumT-precision settling/gradient delta onto a real, + /// stored T-typed buffer (z, W, b, ...), which is the one place this + /// delta is *meant* to finally round-trip through storage precision. + virtual void AxpyIntoFromAccum(T* y, const AccumT* x, size_t n, AccumT alpha) noexcept = 0; /// @brief buf[row*width + col] += bias[col] for every row in /// [0,batchSize) and col in [0,width), broadcasts a per-column bias /// across every row (the dense-layer convention). @@ -309,6 +334,14 @@ class IComputeBackendT /// (the pre-activation value), writes the derivative into dst. virtual void ActivationDerivativeInto(ActivationType type, T* dst, const T* src, size_t n) noexcept = 0; + /// @brief Same operation as ActivationDerivativeInto(), but writes + /// into an AccumT* buffer instead of T* -- see MatMulAccumMixed()'s + /// own doc comment: an activation derivative's natural range commonly + /// undershoots a lossy T's coarsest resolution entirely, so this must + /// land in AccumT precision at the point of computation, never T, even + /// transiently. + virtual void ActivationDerivativeIntoAccum(ActivationType type, AccumT* dst, const T* src, + size_t n) noexcept = 0; // Fused PC-specific ops @@ -595,6 +628,13 @@ class IComputeBackendT /// @brief dst[i] = a[i] * b[i] for all i in [0, n), elementwise product. virtual void MultiplyInto(T* dst, const T* a, const T* b, size_t n) noexcept = 0; + /// @brief Same operation as MultiplyInto(), but dst and b are AccumT* + /// (a stays T*) -- see MatMulAccumMixed()'s own doc comment: b is + /// typically an activation derivative already computed in AccumT + /// precision via ActivationDerivativeIntoAccum(), and the product with + /// a (e.g. the layer-above error, still T*) must stay in that + /// precision too rather than round-trip through storage type T. + virtual void MultiplyIntoAccum(AccumT* dst, const T* a, const AccumT* b, size_t n) noexcept = 0; /// @brief Fuses an in-place "derivative from activated value" step with /// the elementwise multiply that immediately follows it in every known /// caller: dst[i] = a[i] * f'(activatedInOut[i]), then diff --git a/include/deepity/layers/FullPCLayerT.h b/include/deepity/layers/FullPCLayerT.h index 68bc4646..f29ed089 100644 --- a/include/deepity/layers/FullPCLayerT.h +++ b/include/deepity/layers/FullPCLayerT.h @@ -85,10 +85,25 @@ class FullPCLayerT T* cachedMu = nullptr; ///< Cached mu, valid when muCacheValid is true. T* preact = nullptr; ///< Pre-activation snapshot of mu, see FullPCLayer's own doc comment. T* cachedPreact = nullptr; ///< Cached pre-activation, valid alongside cachedMu. - T* derivScratch = nullptr; ///< nextSize-shaped scratch holding f'(preact). - T* feedbackError = nullptr; ///< nextSize-shaped scratch, see FullPCLayer's own doc comment. + // derivScratch/feedbackError/proj/feedbackScratch below are AccumT*, + // not T*, unlike every buffer above -- a real, deliberate asymmetry, + // not an inconsistency with the rest of this class. They're pure + // transient compute (never read back by a caller, never persisted), + // and their natural magnitude -- an activation derivative, and + // everything downstream of multiplying it into an error signal -- can + // fall entirely below a lossy T's coarsest resolution (sigmoid's + // derivative maxes out at 0.25, under Fp4E2M1Value's smallest nonzero + // magnitude of 0.5). Routing them through T even transiently rounds + // them to exactly zero and silently breaks every gradient that flows + // through them, for every activation whose derivative range + // undershoots the grid -- found via this plan's own MNIST validation + // pass, not anticipated at design time. See + // IComputeBackendT::MatMulAccumMixed()'s own doc comment for the + // backend-side half of this fix. + std::vector derivScratch; ///< nextSize-shaped scratch holding f'(preact). + std::vector feedbackError; ///< nextSize-shaped scratch, see FullPCLayer's own doc comment. T* Psi = nullptr; ///< Direct feedback (DFA) weight matrix, shape [size, terminalSize]. - T* proj = nullptr; ///< Scratch buffer for DirectFeedbackUpdate()'s W update. + std::vector proj; ///< Scratch buffer for DirectFeedbackUpdate()'s W update. // grad_*/m_*/v_* and lr_device/fl_device/t_device/tPsi_device are all // AccumT-typed (or int, for the step counters) optimizer-accumulator @@ -113,8 +128,20 @@ class FullPCLayerT int tPsi_device = 0; ///< Adam/AdamW step count for Psi. AccumT fl_device = AccumT(0); ///< Mirror of `fl`, passed by address to the backend. - T* feedbackScratch = nullptr; ///< Scratch buffer for the feedback GEMM's output. - + std::vector feedbackScratch; ///< Scratch buffer for the feedback GEMM's output. + + // v (momentum) and adjoint (ePC) stay T*, unlike derivScratch/ + // feedbackError/proj/feedbackScratch above -- a known, deliberately + // deferred gap, not an oversight: both are only reachable with + // useMomentum/useEPC explicitly enabled, which the plan's own + // verification pass already flagged as not separately validated at + // low precision ("note this as a known deferred-validation area ... + // not a silent gap" -- see the plan's FullPCLayer section). + // Converting them properly needs ComputeAdjoint()'s adjointAbove + // parameter templated on its own input type (the terminal layer's + // first hop is T* errors, every later hop is a previous layer's own + // adjoint), which is more machinery than this pass's actual + // (non-EPC, non-momentum) validation target needs. T* v = nullptr; ///< Momentum buffer for the settling update, see SetMomentum(). bool useMomentum = false; ///< Whether momentum settling is enabled, see SetMomentum(). AccumT momentumBeta = AccumT(0.9); ///< Momentum EMA decay rate, see SetMomentum(). @@ -134,8 +161,6 @@ class FullPCLayerT bool useEPC = false; ///< Whether ePC settling is enabled, see SetUseEPC(). T* adjoint = nullptr; ///< Chained backward-sweep signal, shape [batchSize, size]. - T* biasGradScratch = nullptr; ///< nextSize-length scratch for SumRows' output. - std::unique_ptr> localArena; ///< This layer's own arena, used when it isn't bound into a network-wide one. diff --git a/include/deepity/networks/ConvPCNetwork.h b/include/deepity/networks/ConvPCNetwork.h index 90891221..9ce6df56 100644 --- a/include/deepity/networks/ConvPCNetwork.h +++ b/include/deepity/networks/ConvPCNetwork.h @@ -26,7 +26,7 @@ class ConvPCNetwork /// then Compile() before use. /// @param batchSize Fixed batch size for every layer. /// @param device Which device this network's layers run on. - explicit ConvPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept; + explicit ConvPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU); ~ConvPCNetwork() = default; ConvPCNetwork(const ConvPCNetwork&) = delete; diff --git a/include/deepity/networks/DirectKPPCNetwork.h b/include/deepity/networks/DirectKPPCNetwork.h index 7c3a4e11..a3127254 100644 --- a/include/deepity/networks/DirectKPPCNetwork.h +++ b/include/deepity/networks/DirectKPPCNetwork.h @@ -60,7 +60,7 @@ class DirectKPPCNetwork /// on. Defaults to DEVICE_CPU, preserving existing /// behavior exactly for anyone not explicitly requesting /// DEVICE_GPU. - explicit DirectKPPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept; + explicit DirectKPPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU); /// @brief Default destructor. ~DirectKPPCNetwork() = default; diff --git a/include/deepity/networks/FullConvPCNetwork.h b/include/deepity/networks/FullConvPCNetwork.h index 48b7701f..bc84ff22 100644 --- a/include/deepity/networks/FullConvPCNetwork.h +++ b/include/deepity/networks/FullConvPCNetwork.h @@ -22,7 +22,7 @@ namespace Deep class FullConvPCNetwork { public: - explicit FullConvPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept; + explicit FullConvPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU); ~FullConvPCNetwork() = default; FullConvPCNetwork(const FullConvPCNetwork&) = delete; diff --git a/include/deepity/networks/FullPCNetwork.h b/include/deepity/networks/FullPCNetwork.h index a9519400..16adf1fa 100644 --- a/include/deepity/networks/FullPCNetwork.h +++ b/include/deepity/networks/FullPCNetwork.h @@ -28,7 +28,7 @@ class FullPCNetwork /// then Compile() before use. /// @param batchSize Fixed batch size for every layer. /// @param device Which device this network's layers run on. - explicit FullPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept; + explicit FullPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU); ~FullPCNetwork() = default; FullPCNetwork(const FullPCNetwork&) = delete; diff --git a/include/deepity/networks/FullPCNetworkT.h b/include/deepity/networks/FullPCNetworkT.h index 41f31481..ac3980c4 100644 --- a/include/deepity/networks/FullPCNetworkT.h +++ b/include/deepity/networks/FullPCNetworkT.h @@ -43,7 +43,7 @@ class FullPCNetworkT /// then Compile() before use. /// @param batchSize Fixed batch size for every layer. /// @param device Which device this network's layers run on. - explicit FullPCNetworkT(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept + explicit FullPCNetworkT(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) : device(device) , batchSize(batchSize) { diff --git a/include/deepity/networks/GaussSeidelPCNetwork.h b/include/deepity/networks/GaussSeidelPCNetwork.h index 1ae84db8..9cd71b4d 100644 --- a/include/deepity/networks/GaussSeidelPCNetwork.h +++ b/include/deepity/networks/GaussSeidelPCNetwork.h @@ -41,7 +41,7 @@ class GaussSeidelPCNetwork /// @brief Constructs an empty network with a predetermined batch size. /// @param batchSize Batch size /// @param device Which device this network's layers run on. - explicit GaussSeidelPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept; + explicit GaussSeidelPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU); /// @brief Default destructor. ~GaussSeidelPCNetwork() = default; diff --git a/include/deepity/networks/SimpleConvPCNetwork.h b/include/deepity/networks/SimpleConvPCNetwork.h index f881e3a0..51b78792 100644 --- a/include/deepity/networks/SimpleConvPCNetwork.h +++ b/include/deepity/networks/SimpleConvPCNetwork.h @@ -29,7 +29,7 @@ class SimpleConvPCNetwork /// then Compile() before use. /// @param batchSize Fixed batch size for every layer. /// @param device Which device this network's layers run on. - explicit SimpleConvPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept; + explicit SimpleConvPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU); SimpleConvPCNetwork(const SimpleConvPCNetwork&) = delete; SimpleConvPCNetwork& operator=(const SimpleConvPCNetwork&) = delete; diff --git a/include/deepity/networks/SimplePCNetwork.h b/include/deepity/networks/SimplePCNetwork.h index 9aae19ae..7636609b 100644 --- a/include/deepity/networks/SimplePCNetwork.h +++ b/include/deepity/networks/SimplePCNetwork.h @@ -41,7 +41,7 @@ class SimplePCNetwork /// on. Defaults to DEVICE_CPU, preserving existing /// behavior exactly for anyone not explicitly requesting /// DEVICE_GPU. - explicit SimplePCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept; + explicit SimplePCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU); SimplePCNetwork(const SimplePCNetwork&) = delete; SimplePCNetwork& operator=(const SimplePCNetwork&) = delete; diff --git a/include/deepity/utils/ModelIO.h b/include/deepity/utils/ModelIO.h index fc5dfef8..a227974b 100644 --- a/include/deepity/utils/ModelIO.h +++ b/include/deepity/utils/ModelIO.h @@ -24,6 +24,32 @@ namespace Deep { +namespace detail +{ +// FullPCLayerT is free-standing (doesn't inherit Layer, see +// FullPCLayerT.h's own doc comment on why), so it has no +// SyncStateDictFromDevice/ToDevice at all -- only Layer declares those. +// These two helpers let Save()/Load() call them where they exist +// (every Layer-derived network) and skip them where they don't, instead +// of forcing every NetworkType's layer to carry a method it has no use +// for. NOTE: this means a FullPCLayerT running on a CUDA device has no +// sync path here at all -- unlike FullConvPCLayer, it was never given +// host-staging buffers in the first place, so a GetStateDict() call on +// a device-resident FullPCLayerT is not proven safe. Not fixed here; +// flagging it is the point of this comment. +template void SyncFromDeviceIfSupported(LayerPtr& layerPtr) +{ + if constexpr (requires { layerPtr->SyncStateDictFromDevice(); }) + layerPtr->SyncStateDictFromDevice(); +} + +template void SyncToDeviceIfSupported(LayerPtr& layerPtr) +{ + if constexpr (requires { layerPtr->SyncStateDictToDevice(); }) + layerPtr->SyncStateDictToDevice(); +} +} // namespace detail + /// @brief Static save/load helpers for any network type exposing /// GetLayers(), templated so it works across every PC network /// variant without each one reimplementing serialization. @@ -45,6 +71,11 @@ class ModelIO for (const auto& layerPtr : net.GetLayers()) { + // A device-resident layer's GetStateDict() reads from host staging + // buffers it owns, not from the actual device memory -- those + // buffers only hold current data once this sync runs. No-op for a + // CPU layer, whose GetStateDict() already returns live pointers. + detail::SyncFromDeviceIfSupported(layerPtr); auto state_dict = layerPtr->GetStateDict(); for (const auto& [name, tensor] : state_dict) @@ -145,6 +176,11 @@ class ModelIO in.read(reinterpret_cast(tensor.data), byte_size); } } + // Mirror of Save()'s SyncStateDictFromDevice(): for a device-resident + // layer, the reads above only landed in its host staging buffers -- + // this pushes them on to the actual device-resident parameters. No-op + // for a CPU layer, whose GetStateDict() pointers are the real buffers. + detail::SyncToDeviceIfSupported(layerPtr); layer_idx++; } return true; diff --git a/src/backend/CPUBackendT.cpp b/src/backend/CPUBackendT.cpp index a2a3e0b7..4cb5e5ba 100644 --- a/src/backend/CPUBackendT.cpp +++ b/src/backend/CPUBackendT.cpp @@ -192,6 +192,46 @@ void CPUBackendT::MatMulAccum(bool transA, bool transB, int M, int N, } } +template +void CPUBackendT::MatMulAccumMixed(bool transA, bool transB, int M, int N, int K, + AccumT alpha, const AccumT* A, int lda, const T* B, + int ldb, AccumT beta, AccumT* C, int ldc) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + // Bit-exact with MatMul()'s own float/float path: A, B, and C are + // all float* here too (AccumT==T==float). + cblas_sgemm(CblasRowMajor, transA ? CblasTrans : CblasNoTrans, + transB ? CblasTrans : CblasNoTrans, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + return; + } + else if constexpr (std::is_same_v && std::is_same_v) + { + cblas_dgemm(CblasRowMajor, transA ? CblasTrans : CblasNoTrans, + transB ? CblasTrans : CblasNoTrans, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + return; + } + + // A is already AccumT (a settling/gradient scratch, see this method's + // own doc comment) -- only B, the real stored T-typed parameter (W), + // needs decoding, for every other (T, AccumT) pair. + for (int i = 0; i < M; ++i) + { + for (int j = 0; j < N; ++j) + { + AccumT acc(0); + for (int k = 0; k < K; ++k) + { + AccumT a = transA ? A[k * lda + i] : A[i * lda + k]; + AccumT b = transB ? hwy::ConvertScalarTo(B[j * ldb + k]) : hwy::ConvertScalarTo(B[k * ldb + j]); + acc += a * b; + } + AccumT& c = C[i * ldc + j]; + c = alpha * acc + beta * c; + } + } +} + template void CPUBackendT::SumRowsAccum(AccumT* dst, const T* src, size_t batchSize, size_t width) noexcept @@ -296,6 +336,43 @@ void CPUBackendT::AxpyInto(T* y, const T* x, size_t n, AccumT alpha) } } +template +void CPUBackendT::AxpyIntoAccum(AccumT* y, const T* x, size_t n, AccumT alpha) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + cblas_saxpy((int)n, alpha, x, 1, y, 1); + return; + } + else if constexpr (std::is_same_v && std::is_same_v) + { + cblas_daxpy((int)n, alpha, x, 1, y, 1); + return; + } + + for (size_t i = 0; i < n; ++i) + y[i] += alpha * hwy::ConvertScalarTo(x[i]); +} + +template +void CPUBackendT::AxpyIntoFromAccum(T* y, const AccumT* x, size_t n, + AccumT alpha) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + cblas_saxpy((int)n, alpha, x, 1, y, 1); + return; + } + else if constexpr (std::is_same_v && std::is_same_v) + { + cblas_daxpy((int)n, alpha, x, 1, y, 1); + return; + } + + for (size_t i = 0; i < n; ++i) + y[i] = hwy::ConvertScalarTo(hwy::ConvertScalarTo(y[i]) + alpha * x[i]); +} + template void CPUBackendT::AddBiasBroadcast(T* buf, const T* bias, size_t batchSize, size_t width) noexcept @@ -393,6 +470,25 @@ void CPUBackendT::ActivationDerivativeInto(ActivationType type, T* ds } } +template +void CPUBackendT::ActivationDerivativeIntoAccum(ActivationType type, AccumT* dst, + const T* src, size_t n) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + // Bit-exact with ActivationDerivativeInto()'s own float/float path: + // dst and src are both float* here too (AccumT==T==float), so + // there's no reason to fall back to the generic scalar formula and + // diverge from the SIMD kernel by a ULP or two. + To_dFn2(type)(dst, src, n); + } + else + { + for (size_t i = 0; i < n; ++i) + dst[i] = ActivationDerivativeScalarT(type, hwy::ConvertScalarTo(src[i])); + } +} + template AccumT CPUBackendT::ComputeErrorAndEnergy(T* e, const T* z, const T* mu, size_t n) noexcept @@ -601,6 +697,22 @@ void CPUBackendT::MultiplyInto(T* dst, const T* a, const T* b, size_t } } +template +void CPUBackendT::MultiplyIntoAccum(AccumT* dst, const T* a, const AccumT* b, + size_t n) noexcept +{ + if constexpr (std::is_same_v && std::is_same_v) + { + // Bit-exact with MultiplyInto()'s own float path: dst, a, and b are + // all float* here too (AccumT==T==float). + MultiplyIntoDispatch(dst, a, b, n); + return; + } + + for (size_t i = 0; i < n; ++i) + dst[i] = hwy::ConvertScalarTo(a[i]) * b[i]; +} + // --- Explicit instantiation: the (T, AccumT) pairs this templating // pass proves the machinery with so far (see the plan's "Implementation // order" section) -- every future T added there needs a matching line diff --git a/src/layers/FullPCLayerT.cpp b/src/layers/FullPCLayerT.cpp index b07b27be..73e1731b 100644 --- a/src/layers/FullPCLayerT.cpp +++ b/src/layers/FullPCLayerT.cpp @@ -87,28 +87,22 @@ void FullPCLayerT::BindMemory(ArenaT& arena) cachedMu = arena.Allocate(out_state_size); preact = arena.Allocate(out_state_size); cachedPreact = arena.Allocate(out_state_size); - derivScratch = arena.Allocate(out_state_size); - feedbackError = arena.Allocate(out_state_size); - proj = arena.Allocate(out_state_size); + derivScratch.assign(out_state_size, AccumT(0)); + feedbackError.assign(out_state_size, AccumT(0)); + proj.assign(out_state_size, AccumT(0)); Psi = arena.Allocate(direct_size); - feedbackScratch = arena.Allocate(own_state_size); + feedbackScratch.assign(own_state_size, AccumT(0)); v = arena.Allocate(own_state_size); backend->Zero(v, own_state_size); adjoint = arena.Allocate(own_state_size); backend->Zero(adjoint, own_state_size); - biasGradScratch = arena.Allocate(nextSize); backend->Zero(b, nextSize); backend->Zero(mu, out_state_size); backend->Zero(cachedMu, out_state_size); backend->Zero(preact, out_state_size); backend->Zero(cachedPreact, out_state_size); - backend->Zero(derivScratch, out_state_size); - backend->Zero(feedbackError, out_state_size); - backend->Zero(proj, out_state_size); backend->Zero(Psi, direct_size); - backend->Zero(feedbackScratch, own_state_size); - backend->Zero(biasGradScratch, nextSize); BindAdamStateT(w_size, grad_W, m_W, v_W); BindAdamStateT(nextSize, grad_b, m_b, v_b); @@ -147,12 +141,10 @@ size_t FullPCLayerT::GetRequiredElements() const noexcept total += pad16(w_size); total += pad16(nextSize); - total += pad16(out_state_size) * 3; // mu, cachedMu, proj - total += pad16(out_state_size) * 4; // preact, cachedPreact, derivScratch, feedbackError - total += pad16(own_state_size); // feedbackScratch + total += pad16(out_state_size) * 2; // mu, cachedMu + total += pad16(out_state_size) * 2; // preact, cachedPreact total += pad16(own_state_size) * 2; // v, adjoint total += pad16(direct_size); - total += pad16(nextSize); total += pad16(w_size) * 3; total += pad16(nextSize) * 3; @@ -303,35 +295,35 @@ template void FullPCLayerT::UpdateState size_t Nout = (size_t)batchSize * nextSize; ActivationType dType = ToDerivativeType(activationType); - backend->ActivationDerivativeInto(dType, derivScratch, preact, Nout); - backend->MultiplyInto(feedbackError, e_above, derivScratch, Nout); + backend->ActivationDerivativeIntoAccum(dType, derivScratch.data(), preact, Nout); + backend->MultiplyIntoAccum(feedbackError.data(), e_above, derivScratch.data(), Nout); - backend->MatMul( + backend->MatMulAccumMixed( /*transA=*/false, /*transB=*/false, (int)batchSize, (int)size, (int)nextSize, a, - feedbackError, + feedbackError.data(), (int)nextSize, W, (int)size, AccumT(0), - feedbackScratch, + feedbackScratch.data(), (int)size); if (useMomentum) { - backend->AxpyInto(feedbackScratch, e, N, AccumT(-1)); + backend->AxpyIntoAccum(feedbackScratch.data(), e, N, AccumT(-1)); backend->Scale(v, N, momentumBeta); - backend->AxpyInto(v, feedbackScratch, N, AccumT(1) - momentumBeta); + backend->AxpyIntoFromAccum(v, feedbackScratch.data(), N, AccumT(1) - momentumBeta); backend->AxpyInto(z, v, N, ir); } else { - backend->AxpyInto(feedbackScratch, e, N, AccumT(-1)); - backend->AxpyInto(z, feedbackScratch, N, ir); + backend->AxpyIntoAccum(feedbackScratch.data(), e, N, AccumT(-1)); + backend->AxpyIntoFromAccum(z, feedbackScratch.data(), N, ir); } if (useResidual) @@ -353,9 +345,21 @@ template void FullPCLayerT::UpdateWeigh const T* e_above = layerAbove->GetErrors(); ActivationType dType = ToDerivativeType(activationType); - backend->ActivationDerivativeInto(dType, derivScratch, preact, Nout); - backend->MultiplyInto(feedbackError, e_above, derivScratch, Nout); - const T* local_grad = feedbackError; + backend->ActivationDerivativeIntoAccum(dType, derivScratch.data(), preact, Nout); + backend->MultiplyIntoAccum(feedbackError.data(), e_above, derivScratch.data(), Nout); + const AccumT* local_grad = feedbackError.data(); + + // local_grad's own per-batch-element reduction into grad_b: pure + // AccumT->AccumT, no decode needed (unlike SumRows()/SumRowsAccum(), + // which both still expect a T* src for every other caller), so this + // is inlined here rather than adding a third backend method for it. + auto sumRowsIntoGradB = [&]() + { + std::fill(grad_b.begin(), grad_b.begin() + nextSize, AccumT(0)); + for (size_t bi = 0; bi < batchSize; ++bi) + for (size_t i = 0; i < nextSize; ++i) + grad_b[i] += local_grad[bi * nextSize + i]; + }; switch (opt) { @@ -364,24 +368,36 @@ template void FullPCLayerT::UpdateWeigh if (lmbda > AccumT(0)) backend->Scale(W, (size_t)nextSize * size, AccumT(1) - lmbda); - backend->MatMul( + // Computed into grad_W/grad_b (this layer's own Adam accumulator + // buffers, unused by SGD otherwise and free as scratch here) and + // then applied via AxpyIntoFromAccum, rather than a single blended + // MatMul() straight into W: local_grad is AccumT now (see this + // method's own comment above on derivScratch/feedbackError), and a + // GEMM that blends an AccumT operand directly into an existing T* + // accumulator (beta != 0) is a shape MatMulAccumMixed() doesn't + // support -- see its own doc comment, it only ever writes a fresh + // AccumT* C. This costs nothing extra: the subsequent + // AxpyIntoFromAccum() is the one and only re-quantization into W/b, + // same as ADAM's AdamStep() below. + backend->MatMulAccumMixed( /*transA=*/true, /*transB=*/false, (int)nextSize, (int)size, (int)batchSize, - a * lr / AccumT(batchSize), + AccumT(1), local_grad, (int)nextSize, z, (int)size, - AccumT(1), - W, + AccumT(0), + grad_W.data(), (int)size); + backend->AxpyIntoFromAccum(W, grad_W.data(), (size_t)nextSize * size, + a * lr / AccumT(batchSize)); - AccumT lr_batch = lr / AccumT(batchSize); - backend->SumRows(biasGradScratch, local_grad, batchSize, nextSize); - backend->AxpyInto(b, biasGradScratch, nextSize, lr_batch); + sumRowsIntoGradB(); + backend->AxpyIntoFromAccum(b, grad_b.data(), nextSize, lr / AccumT(batchSize)); break; } @@ -393,7 +409,7 @@ template void FullPCLayerT::UpdateWeigh size_t num_weights = (size_t)nextSize * size; AccumT adam_scale = AccumT(-1) * a; - backend->MatMulAccum( + backend->MatMulAccumMixed( /*transA=*/true, /*transB=*/false, (int)nextSize, @@ -408,7 +424,7 @@ template void FullPCLayerT::UpdateWeigh grad_W.data(), (int)size); - backend->SumRowsAccum(grad_b.data(), local_grad, batchSize, nextSize); + sumRowsIntoGradB(); // Direct negate, not backend->Scale(): grad_b is AccumT*, and // Scale() takes T* -- same host-resident-AccumT-scratch caveat as // SetLearningRate()'s own comment above. @@ -487,7 +503,7 @@ template void FullPCLayerT::DirectFeedb if (layerAbove == nullptr || layerAbove->GetDirectFeedbackWeights() == nullptr) return; - backend->MatMul( + backend->MatMulAccum( /*transA=*/false, /*transB=*/true, (int)batchSize, @@ -499,30 +515,39 @@ template void FullPCLayerT::DirectFeedb layerAbove->GetDirectFeedbackWeights(), (int)terminalSize, AccumT(0), - proj, + proj.data(), (int)nextSize); { size_t Nout = (size_t)batchSize * nextSize; ActivationType dType = ToDerivativeType(activationType); - backend->ActivationDerivativeInto(dType, derivScratch, preact, Nout); - backend->MultiplyInto(proj, proj, derivScratch, Nout); + backend->ActivationDerivativeIntoAccum(dType, derivScratch.data(), preact, Nout); + // Pure AccumT*AccumT elementwise, both operands already decoded by + // the calls above -- inlined rather than adding a third backend + // method for a product that never touches storage type T. + for (size_t i = 0; i < Nout; ++i) + proj[i] *= derivScratch[i]; } - backend->MatMul( + // Same grad_W-as-scratch + AxpyIntoFromAccum restructuring as + // UpdateWeights()'s SGD branch, and for the same reason: proj is + // AccumT now, and blending it directly into W via a single GEMM + // (beta != 0) isn't a shape MatMulAccumMixed() supports. + backend->MatMulAccumMixed( /*transA=*/true, /*transB=*/false, (int)nextSize, (int)size, (int)batchSize, - fl / AccumT(batchSize), - proj, + AccumT(1), + proj.data(), (int)nextSize, z, (int)size, - AccumT(1), - W, + AccumT(0), + grad_W.data(), (int)size); + backend->AxpyIntoFromAccum(W, grad_W.data(), (size_t)nextSize * size, fl / AccumT(batchSize)); } template void FullPCLayerT::ReconstructBelief() noexcept @@ -546,23 +571,31 @@ void FullPCLayerT::ComputeAdjoint(const T* adjointAbove, const size_t Nout = batchSize * nextSize; ActivationType dType = ToDerivativeType(activationType); - backend->ActivationDerivativeInto(dType, derivScratch, preact, Nout); - backend->MultiplyInto(feedbackError, adjointAbove, derivScratch, Nout); - - backend->MatMul( + backend->ActivationDerivativeIntoAccum(dType, derivScratch.data(), preact, Nout); + backend->MultiplyIntoAccum(feedbackError.data(), adjointAbove, derivScratch.data(), Nout); + + // adjoint stays T* (ePC's own precision is a known, deliberately + // deferred gap -- see this class's own comment on the `v`/`adjoint` + // members), so the AccumT result lands in feedbackScratch (free here: + // ComputeAdjoint() only runs from EPCStep(), never alongside + // UpdateState()'s own use of this same buffer within one settle + // step) and is re-encoded into adjoint afterward. + backend->MatMulAccumMixed( /*transA=*/false, /*transB=*/false, (int)batchSize, (int)size, (int)nextSize, a * adjointAboveScale, - feedbackError, + feedbackError.data(), (int)nextSize, W, (int)size, AccumT(0), - adjoint, + feedbackScratch.data(), (int)size); + for (size_t i = 0; i < N; ++i) + adjoint[i] = hwy::ConvertScalarTo(feedbackScratch[i]); if (useResidual) backend->AxpyInto(adjoint, adjointAbove, N, adjointAboveScale); diff --git a/src/networks/ConvPCNetwork.cpp b/src/networks/ConvPCNetwork.cpp index 2e2bed34..7734c94a 100644 --- a/src/networks/ConvPCNetwork.cpp +++ b/src/networks/ConvPCNetwork.cpp @@ -4,7 +4,7 @@ namespace Deep { -ConvPCNetwork::ConvPCNetwork(int batchSize, DeviceType device) noexcept +ConvPCNetwork::ConvPCNetwork(int batchSize, DeviceType device) : device(device) , batchSize(batchSize) { diff --git a/src/networks/DirectKPPCNetwork.cpp b/src/networks/DirectKPPCNetwork.cpp index b20a5cd6..1f4e82ce 100644 --- a/src/networks/DirectKPPCNetwork.cpp +++ b/src/networks/DirectKPPCNetwork.cpp @@ -6,7 +6,7 @@ namespace Deep { -DirectKPPCNetwork::DirectKPPCNetwork(int batchSize, DeviceType device) noexcept +DirectKPPCNetwork::DirectKPPCNetwork(int batchSize, DeviceType device) : device(device) , batchSize(batchSize) { diff --git a/src/networks/FullConvPCNetwork.cpp b/src/networks/FullConvPCNetwork.cpp index b16636ba..3d1d3bdc 100644 --- a/src/networks/FullConvPCNetwork.cpp +++ b/src/networks/FullConvPCNetwork.cpp @@ -8,7 +8,7 @@ namespace Deep { -FullConvPCNetwork::FullConvPCNetwork(int batchSize, DeviceType device) noexcept +FullConvPCNetwork::FullConvPCNetwork(int batchSize, DeviceType device) : device(device) , batchSize(batchSize) { diff --git a/src/networks/FullPCNetwork.cpp b/src/networks/FullPCNetwork.cpp index bff5a81f..f6951ba1 100644 --- a/src/networks/FullPCNetwork.cpp +++ b/src/networks/FullPCNetwork.cpp @@ -8,7 +8,7 @@ namespace Deep { -FullPCNetwork::FullPCNetwork(int batchSize, DeviceType device) noexcept +FullPCNetwork::FullPCNetwork(int batchSize, DeviceType device) : device(device) , batchSize(batchSize) { diff --git a/src/networks/GaussSeidelPCNetwork.cpp b/src/networks/GaussSeidelPCNetwork.cpp index 5c0045ca..5a77e70e 100644 --- a/src/networks/GaussSeidelPCNetwork.cpp +++ b/src/networks/GaussSeidelPCNetwork.cpp @@ -4,7 +4,7 @@ namespace Deep { -GaussSeidelPCNetwork::GaussSeidelPCNetwork(int batchSize, DeviceType device) noexcept +GaussSeidelPCNetwork::GaussSeidelPCNetwork(int batchSize, DeviceType device) : device(device) , batchSize(batchSize) { diff --git a/src/networks/SimpleConvPCNetwork.cpp b/src/networks/SimpleConvPCNetwork.cpp index a7a7f024..26c65512 100644 --- a/src/networks/SimpleConvPCNetwork.cpp +++ b/src/networks/SimpleConvPCNetwork.cpp @@ -4,7 +4,7 @@ namespace Deep { -SimpleConvPCNetwork::SimpleConvPCNetwork(int batchSize, DeviceType device) noexcept +SimpleConvPCNetwork::SimpleConvPCNetwork(int batchSize, DeviceType device) : device(device) , batchSize(batchSize) { diff --git a/src/networks/SimplePCNetwork.cpp b/src/networks/SimplePCNetwork.cpp index bdee78d6..31c00e63 100644 --- a/src/networks/SimplePCNetwork.cpp +++ b/src/networks/SimplePCNetwork.cpp @@ -8,7 +8,7 @@ namespace Deep { -SimplePCNetwork::SimplePCNetwork(int batchSize, DeviceType device) noexcept +SimplePCNetwork::SimplePCNetwork(int batchSize, DeviceType device) : device(device) , batchSize(batchSize) { diff --git a/tests/tBitPackedGemmPrototype.cpp b/tests/tBitPackedGemmPrototype.cpp new file mode 100644 index 00000000..23674647 --- /dev/null +++ b/tests/tBitPackedGemmPrototype.cpp @@ -0,0 +1,346 @@ +// Standalone proof-of-concept, deliberately outside FullPCLayerT/the +// backend interface: proves bit-offset / radix-offset addressing (true +// sub-byte packing, N values per byte instead of 1 byte per value) is +// viable for real compute, before touching any of the main layer/backend +// code. Not wired into CMakeLists.txt -- compile directly, e.g.: +// g++ -O2 -std=c++20 tests/tBitPackedGemmPrototype.cpp -o /tmp/bppoc && /tmp/bppoc +// +// Single generic mechanism, no per-width special cases: PackedRadix +// is the only storage class, for ANY RADIX (2 = binary, 3 = ternary, 8 = +// 3-bit, 16 = 4-bit, ...). Group size (how many elements share a packed +// group) and group byte count are both *derived* from RADIX via a +// constexpr search, never hand-picked per width (no "5 trits per byte" +// written down anywhere) -- RadixGrid builds its quantization +// levels the same way, via one closed-form exponential ladder, not a +// lookup table sized per bit width. Because RADIX is a compile-time +// template parameter, the compiler constant-folds the modulo/divide by +// RADIX on its own (strength-reduced to shifts when RADIX is a power of +// 2) -- performance differences across widths fall out of that generic +// code, not from hand-tuned per-width paths. + +#include +#include +#include +#include +#include +#include +#include + +// --- PackedRadix: N values, each one of RADIX distinct codes, +// packed into the fewest bytes a generic search can find. Elements are +// grouped so that RADIX^kGroupSize values fit in kGroupBytes bytes; +// both numbers come from ComputeGroupSize/ComputeGroupBytes below -- +// purely a function of RADIX, no per-width table anywhere in this +// class. Get/Set address a group's packed integer and do mixed-radix +// div/mod to reach one element inside it (the direct generalization of +// bit-shift addressing to a non-power-of-2 RADIX). +template +class PackedRadix +{ + static_assert(RADIX >= 2, "RADIX must be at least 2"); + + // Largest group size G with RADIX^G <= 65536: bounds every group to + // at most 2 bytes and at most ~16 sequential div/mod steps per access + // regardless of RADIX, so no width gets a pathologically large group. + static constexpr int ComputeGroupSize() + { + uint64_t span = 1; + int g = 0; + while (span * RADIX <= 65536) + { + span *= RADIX; + ++g; + } + return g > 0 ? g : 1; + } + + static constexpr uint64_t ComputeGroupSpan() + { + uint64_t span = 1; + for (int i = 0; i < ComputeGroupSize(); ++i) + span *= RADIX; + return span; + } + + static constexpr int ComputeGroupBytes() + { + uint64_t span = ComputeGroupSpan(); + uint64_t cap = 1; + int bytes = 0; + while (cap < span) + { + cap *= 256; + ++bytes; + } + return bytes > 0 ? bytes : 1; + } + +public: + static constexpr int kGroupSize = ComputeGroupSize(); + static constexpr int kGroupBytes = ComputeGroupBytes(); + + explicit PackedRadix(size_t n) + : n_(n), bytes_(((n + kGroupSize - 1) / kGroupSize) * (size_t)kGroupBytes, 0) + { + } + + size_t size() const noexcept { return n_; } + size_t byteSize() const noexcept { return bytes_.size(); } + + uint64_t Get(size_t i) const noexcept + { + size_t groupIdx = i / kGroupSize; + int slot = (int)(i % kGroupSize); + uint64_t packed = ReadGroup(groupIdx); + for (int s = 0; s < slot; ++s) + packed /= RADIX; + return packed % RADIX; + } + + void Set(size_t i, uint64_t code) noexcept + { + size_t groupIdx = i / kGroupSize; + int slot = (int)(i % kGroupSize); + code %= RADIX; + + uint64_t packed = ReadGroup(groupIdx); + uint64_t digits[kGroupSize]; + for (int s = 0; s < kGroupSize; ++s) + { + digits[s] = packed % RADIX; + packed /= RADIX; + } + digits[slot] = code; + + uint64_t rebuilt = 0, place = 1; + for (int s = 0; s < kGroupSize; ++s) + { + rebuilt += digits[s] * place; + place *= RADIX; + } + WriteGroup(groupIdx, rebuilt); + } + +private: + uint64_t ReadGroup(size_t groupIdx) const noexcept + { + uint64_t v = 0; + for (int b = 0; b < kGroupBytes; ++b) + v |= (uint64_t)bytes_[groupIdx * (size_t)kGroupBytes + b] << (8 * b); + return v; + } + + void WriteGroup(size_t groupIdx, uint64_t v) noexcept + { + for (int b = 0; b < kGroupBytes; ++b) + bytes_[groupIdx * (size_t)kGroupBytes + b] = (uint8_t)((v >> (8 * b)) & 0xFF); + } + + size_t n_; + std::vector bytes_; +}; + +// --- RadixGrid: maps RADIX codes to float values via one +// closed-form exponential ladder (code 0 -> 0, codes 1..kPos -> positive +// magnitudes 0.5, 1, 2, 4, ..., the mirror codes -> negatives), not a +// lookup table sized per bit width. Works for any RADIX >= 2; for even +// RADIX one code goes unused (documented, not hidden) since codes split +// evenly into a positive/negative mirror plus one zero. +template +struct RadixGrid +{ + static constexpr int kPosLevels = (int)((RADIX - 1) / 2); + + static float Magnitude(int level) noexcept + { + if (level == 0) + return 0.0f; + return std::exp2((float)(level - 1)) * 0.5f; // 0.5, 1, 2, 4, 8, ... + } + + static uint64_t Encode(float x) noexcept + { + bool neg = x < 0.0f; + float ax = std::fabs(x); + + int best = 0; + float bestDist = std::fabs(ax - Magnitude(0)); + for (int level = 1; level <= kPosLevels; ++level) + { + float d = std::fabs(ax - Magnitude(level)); + if (d < bestDist) + { + bestDist = d; + best = level; + } + } + if (best == 0) + return 0; + return neg ? (uint64_t)(kPosLevels + best) : (uint64_t)best; + } + + static float Decode(uint64_t code) noexcept + { + if (code == 0) + return 0.0f; + if (code <= (uint64_t)kPosLevels) + return Magnitude((int)code); + return -Magnitude((int)(code - (uint64_t)kPosLevels)); + } +}; + +template +float RadixDecodeAt(const PackedRadix& arr, size_t i) noexcept +{ + return RadixGrid::Decode(arr.Get(i)); +} + +template +void RadixEncodeInto(PackedRadix& arr, size_t i, float x) noexcept +{ + arr.Set(i, RadixGrid::Encode(x)); +} + +// --- The actual proof: a GEMM reading both operands directly from +// packed storage, decoding via radix div/mod inside the inner loop -- +// the same function body regardless of RADIX, no per-width overload. +template +void RadixGemm(int M, int N, int K, const PackedRadix& A, + const PackedRadix& B, float* C) noexcept +{ + for (int i = 0; i < M; ++i) + for (int j = 0; j < N; ++j) + { + float acc = 0.0f; + for (int k = 0; k < K; ++k) + acc += RadixDecodeAt(A, (size_t)i * K + k) * RadixDecodeAt(B, (size_t)k * N + j); + C[i * N + j] = acc; + } +} + +template +static void TestRoundTrip(const char* label) +{ + printf("[RADIX=%llu, %s] round-trip + packing density\n", (unsigned long long)RADIX, label); + const size_t n = 23; // deliberately not a multiple of any group size + PackedRadix arr(n); + + std::mt19937 rng(42); + std::uniform_real_distribution dist(-3.0f, 3.0f); + std::vector inputs(n); + for (size_t i = 0; i < n; ++i) + inputs[i] = dist(rng); + + for (size_t i = 0; i < n; ++i) + RadixEncodeInto(arr, i, inputs[i]); + + bool allRoundTrip = true; + for (size_t i = 0; i < n; ++i) + { + float decoded = RadixDecodeAt(arr, i); + float expected = RadixGrid::Decode(RadixGrid::Encode(inputs[i])); + allRoundTrip &= (decoded == expected); + } + double bitsPerElem = 8.0 * (double)arr.byteSize() / (double)n; + double theoreticalBits = std::log2((double)RADIX); + printf(" round-trip: %s | %zu elements -> %zu bytes -> %.2f bits/elem (ideal: %.2f)\n\n", + allRoundTrip ? "OK" : "MISMATCH", n, arr.byteSize(), bitsPerElem, theoreticalBits); +} + +template +static void TestGemmCorrectness(const char* label) +{ + printf("[RADIX=%llu, %s] GEMM read directly from packed storage vs. reference\n", + (unsigned long long)RADIX, label); + const int M = 4, K = 5, N = 3; + + PackedRadix A((size_t)(M * K)), B((size_t)(K * N)); + std::vector Aref(M * K), Bref(K * N); + + std::mt19937 rng(7); + std::uniform_real_distribution dist(-3.0f, 3.0f); + + for (int i = 0; i < M * K; ++i) + { + float x = dist(rng); + RadixEncodeInto(A, (size_t)i, x); + Aref[i] = RadixGrid::Decode(RadixGrid::Encode(x)); + } + for (int i = 0; i < K * N; ++i) + { + float x = dist(rng); + RadixEncodeInto(B, (size_t)i, x); + Bref[i] = RadixGrid::Decode(RadixGrid::Encode(x)); + } + + std::vector Cpacked(M * N), Cref(M * N, 0.0f); + RadixGemm(M, N, K, A, B, Cpacked.data()); + + for (int i = 0; i < M; ++i) + for (int j = 0; j < N; ++j) + { + float acc = 0.0f; + for (int k = 0; k < K; ++k) + acc += Aref[i * K + k] * Bref[k * N + j]; + Cref[i * N + j] = acc; + } + + bool allMatch = true; + for (int i = 0; i < M * N; ++i) + allMatch &= (Cpacked[i] == Cref[i]); + printf(" %s\n\n", allMatch ? "ALL MATCH (bit-exact)" : "FAILED"); +} + +template +static double TimeDecodeNsPerElement(size_t n, int reps) +{ + PackedRadix arr(n); + std::mt19937 rng(1); + std::uniform_real_distribution dist(-3.0f, 3.0f); + for (size_t i = 0; i < n; ++i) + RadixEncodeInto(arr, i, dist(rng)); + + volatile float sink = 0.0f; + auto start = std::chrono::steady_clock::now(); + for (int r = 0; r < reps; ++r) + for (size_t i = 0; i < n; ++i) + sink += RadixDecodeAt(arr, i); + auto end = std::chrono::steady_clock::now(); + + double ns = std::chrono::duration(end - start).count(); + (void)sink; + return ns / ((double)n * reps); +} + +static void TestPerformanceAcrossRadixes() +{ + printf("[Performance] ns/element decoded, same generic code, varying RADIX only\n"); + const size_t n = 10000; + const int reps = 200; + printf(" RADIX=2 (binary): %.2f ns/elem\n", TimeDecodeNsPerElement<2>(n, reps)); + printf(" RADIX=3 (ternary): %.2f ns/elem\n", TimeDecodeNsPerElement<3>(n, reps)); + printf(" RADIX=5 : %.2f ns/elem\n", TimeDecodeNsPerElement<5>(n, reps)); + printf(" RADIX=8 (3-bit): %.2f ns/elem\n", TimeDecodeNsPerElement<8>(n, reps)); + printf(" RADIX=16 (4-bit): %.2f ns/elem\n", TimeDecodeNsPerElement<16>(n, reps)); + printf(" RADIX=256 (1 byte): %.2f ns/elem\n", TimeDecodeNsPerElement<256>(n, reps)); + printf(" (no per-width code path above -- same template body for every RADIX)\n\n"); +} + +int main() +{ + TestRoundTrip<2>("binary"); + TestRoundTrip<3>("ternary"); + TestRoundTrip<5>("radix-5"); + TestRoundTrip<8>("3-bit"); + TestRoundTrip<16>("4-bit"); + + TestGemmCorrectness<3>("ternary"); + TestGemmCorrectness<8>("3-bit"); + TestGemmCorrectness<16>("4-bit"); + + TestPerformanceAcrossRadixes(); + + printf("Generic radix-packed addressing + direct-from-packed compute: " + "proven viable for arbitrary widths, no per-width hardcoding.\n"); + return 0; +}