diff --git a/Ubytec.Tests/PortableCliTests.cs b/Ubytec.Tests/PortableCliTests.cs index 0a44f985..538fcd6c 100644 --- a/Ubytec.Tests/PortableCliTests.cs +++ b/Ubytec.Tests/PortableCliTests.cs @@ -22,8 +22,20 @@ push 42 } """; + private const string LibrarySource = """ + export Echo as portable_echo + module (name:"portable-library", version:"0.1", author:"Ubytec") + { + func Echo(t_int64 value) -> t_int64 + { + load @value + return + } + } + """; + [Fact] - public void PortableX64CliWritesTheExactVerifiedModuleUsedForNativeCodegen() + public void DefaultCliWritesTheExactVerifiedModuleUsedForNativeCodegen() { using var fixture = new TempFixture(); string source = fixture.Write("program.ubc", Source); @@ -34,7 +46,6 @@ public void PortableX64CliWritesTheExactVerifiedModuleUsedForNativeCodegen() [ source, "-o", nasm, - "--portable-x64", "--portable-module", module, "--no-metadata" ]); @@ -57,7 +68,75 @@ public void PortableX64CliWritesTheExactVerifiedModuleUsedForNativeCodegen() } [Fact] - public void PortableX64CliDoesNotRequirePersistingTheIntermediateModule() + public void DefaultPortableLibraryBindsExportsFromTheVerifiedModule() + { + using var fixture = new TempFixture(); + string source = fixture.Write("library.ubc", LibrarySource); + string nasm = fixture.PathOf("library.nasm"); + string module = fixture.PathOf("library.ubcm"); + + int status = CommandLineDriver.Run( + [ + source, + "--library", + "-o", nasm, + "--portable-module", module, + "--no-metadata" + ]); + + Assert.Equal(0, status); + byte[] bytes = File.ReadAllBytes(module); + PortableProcessEntrypoint.Verify(bytes); + PortableBinaryModule decoded = PortableBinaryCodec.Decode(bytes); + PortableExecutableModule executable = Assert.IsType(decoded.Executable); + PortableExecutableExport export = Assert.Single(executable.Exports); + Assert.Equal("portable_echo", executable.Strings[checked((int)export.SymbolStringIndex)]); + Assert.Equal(0u, export.FunctionIndex); + + string emitted = File.ReadAllText(nasm).ReplaceLineEndings("\n"); + Assert.Contains("global ubytec_binary_export_0", emitted); + Assert.Contains("global portable_echo", emitted); + Assert.Contains("portable_echo:\n", emitted); + Assert.Contains("call ubytec_binary_export_0", emitted); + Assert.DoesNotContain("call Echo", emitted); + } + + [Fact] + public void PortableX64FlagRemainsACompatibilityAliasForTheDefaultPipeline() + { + using var fixture = new TempFixture(); + string source = fixture.Write("alias.ubc", Source); + string defaultNasm = fixture.PathOf("default.nasm"); + string defaultModule = fixture.PathOf("default.ubcm"); + string explicitNasm = fixture.PathOf("explicit.nasm"); + string explicitModule = fixture.PathOf("explicit.ubcm"); + + int defaultStatus = CommandLineDriver.Run( + [ + source, + "-o", defaultNasm, + "--portable-module", defaultModule, + "--no-metadata" + ]); + int explicitStatus = CommandLineDriver.Run( + [ + source, + "-o", explicitNasm, + "--portable-x64", + "--portable-module", explicitModule, + "--no-metadata" + ]); + + Assert.Equal(0, defaultStatus); + Assert.Equal(0, explicitStatus); + Assert.Equal(File.ReadAllBytes(defaultModule), File.ReadAllBytes(explicitModule)); + Assert.Equal( + File.ReadAllText(defaultNasm).ReplaceLineEndings("\n"), + File.ReadAllText(explicitNasm).ReplaceLineEndings("\n")); + } + + [Fact] + public void DefaultCliDoesNotRequirePersistingTheIntermediateModule() { using var fixture = new TempFixture(); string source = fixture.Write("memory-only.ubc", Source); @@ -67,7 +146,6 @@ public void PortableX64CliDoesNotRequirePersistingTheIntermediateModule() [ source, "-o", nasm, - "--portable-x64", "--no-metadata" ]); @@ -78,7 +156,7 @@ public void PortableX64CliDoesNotRequirePersistingTheIntermediateModule() } [Fact] - public void PortableX64CliRejectsLegacyBackendOptimizationFlags() + public void DefaultCliRejectsLegacyBackendOptimizationFlags() { using var fixture = new TempFixture(); string source = fixture.Write("reject.ubc", Source); @@ -88,8 +166,7 @@ public void PortableX64CliRejectsLegacyBackendOptimizationFlags() [ source, "-o", nasm, - "--portable-x64", - "--tail-calls", + "--stack-transfers", "--no-metadata" ]); @@ -98,28 +175,41 @@ public void PortableX64CliRejectsLegacyBackendOptimizationFlags() } [Fact] - public void PortableX64CliRejectsLegacyHostExportAdapters() + public void DefaultCliLowersHostExportThroughVerifiedPortableModule() { using var fixture = new TempFixture(); string source = fixture.Write("host-export.ubc", Source); string nasm = fixture.PathOf("host-export.nasm"); + string module = fixture.PathOf("host-export.ubcm"); int status = CommandLineDriver.Run( [ source, "-o", nasm, - "--portable-x64", "--library", "--host-export", "Main=ubytec_host_main", + "--portable-module", module, "--no-metadata" ]); - Assert.Equal(1, status); - Assert.False(File.Exists(nasm)); + Assert.Equal(0, status); + byte[] bytes = File.ReadAllBytes(module); + PortableProcessEntrypoint.Verify(bytes); + PortableExecutableModule executable = Assert.IsType( + PortableBinaryCodec.Decode(bytes).Executable); + PortableExecutableExport export = Assert.Single(executable.Exports); + Assert.Equal("ubytec_host_main", executable.Strings[checked((int)export.SymbolStringIndex)]); + Assert.Equal(1u, export.FunctionIndex); + + string emitted = File.ReadAllText(nasm).ReplaceLineEndings("\n"); + Assert.Contains("global ubytec_host_main", emitted); + Assert.Contains("ubytec_host_main:\n", emitted); + Assert.Contains("call ubytec_binary_export_0", emitted); + Assert.DoesNotContain("call Main", emitted); } [Fact] - public void LegacyCliPathRemainsAvailableDuringMigration() + public void LegacySourceX64CliPathRemainsAvailableDuringMigration() { using var fixture = new TempFixture(); string source = fixture.Write("legacy.ubc", Source); @@ -129,6 +219,7 @@ public void LegacyCliPathRemainsAvailableDuringMigration() [ source, "-o", nasm, + "--legacy-source-x64", "--no-metadata" ]); @@ -138,6 +229,48 @@ public void LegacyCliPathRemainsAvailableDuringMigration() Assert.DoesNotContain("ubytec_binary_f_", emitted); } + [Fact] + public void LegacySourceX64ExplicitlyUnlocksLegacyBackendOptions() + { + using var fixture = new TempFixture(); + string source = fixture.Write("legacy-options.ubc", Source); + string nasm = fixture.PathOf("legacy-options.nasm"); + + int status = CommandLineDriver.Run( + [ + source, + "-o", nasm, + "--legacy-source-x64", + "--stack-transfers", + "--no-metadata" + ]); + + Assert.Equal(0, status); + Assert.True(File.Exists(nasm)); + } + + [Theory] + [InlineData("--portable-x64", "--legacy-source-x64")] + [InlineData("--legacy-source-x64", "--portable-x64")] + public void CanonicalAndLegacyPipelineFlagsCannotBeCombined(string first, string second) + { + using var fixture = new TempFixture(); + string source = fixture.Write("conflict.ubc", Source); + string nasm = fixture.PathOf("conflict.nasm"); + + int status = CommandLineDriver.Run( + [ + source, + "-o", nasm, + first, + second, + "--no-metadata" + ]); + + Assert.Equal(1, status); + Assert.False(File.Exists(nasm)); + } + private sealed class TempFixture : IDisposable { private readonly string root = System.IO.Path.Combine( diff --git a/Ubytec.Tests/PortableModuleContractAbiTests.cs b/Ubytec.Tests/PortableModuleContractAbiTests.cs new file mode 100644 index 00000000..c17ea19b --- /dev/null +++ b/Ubytec.Tests/PortableModuleContractAbiTests.cs @@ -0,0 +1,68 @@ +using System.Text; +using Ubytec.Language.Portable; +using Xunit; + +namespace Ubytec.Tests; + +public class PortableModuleContractAbiTests +{ + private const string Source = """ + import ReadTool from tools.Read (t_uint64,t_uint64,t_uint64,t_uint64) -> t_int64 + export Main as Main + module (name:"portable-contract", version:"0.1", author:"Ubytec") + { + func Main() -> t_int64 + { + push 0 + return + } + } + """; + + [Fact] + public void LoaderAliasesAndAdaptersComeFromVerifiedUbcmContractManifest() + { + PortableSourceX64Result compilation = PortableSourceX64Pipeline.Compile( + Source, + processImage: false, + enableModuleCalls: true); + + PortableProcessEntrypoint.Verify(compilation.ModuleBytes); + PortableModuleImage image = PortableModuleFormat.Decode(compilation.ModuleBytes); + PortableModuleSection manifest = image.GetRequiredSection(PortableModuleSectionKind.ContractManifest); + string contract = Encoding.ASCII.GetString(manifest.Payload.Span); + + Assert.Contains("I|1|ReadTool|tools|Read|UInt64,UInt64,UInt64,UInt64:Int64\n", contract); + Assert.Contains("E|Main|ubytec_host_contract_Main|:Int64\n", contract); + + PortableExecutableModule executable = Assert.IsType( + PortableBinaryCodec.Decode(compilation.ModuleBytes).Executable); + Assert.Equal(2, executable.Imports.Count); + Assert.Single(executable.Exports); + + int sourceImportIndex = Enumerable.Range(0, executable.Imports.Count).Single(index => + { + PortableExecutableImport import = executable.Imports[index]; + return executable.Strings[checked((int)import.ModuleStringIndex)] == "tools" && + executable.Strings[checked((int)import.SymbolStringIndex)] == "Read"; + }); + int runtimeImportIndex = Enumerable.Range(0, executable.Imports.Count).Single(index => + { + PortableExecutableImport import = executable.Imports[index]; + return executable.Strings[checked((int)import.ModuleStringIndex)] == "ubytec.runtime" && + executable.Strings[checked((int)import.SymbolStringIndex)] == "module.invoke"; + }); + + string sourceImportLabel = compilation.NativeCompilation.GetImportLabel(sourceImportIndex); + string runtimeImportLabel = compilation.NativeCompilation.GetImportLabel(runtimeImportIndex); + string assembly = compilation.NativeCompilation.Assembly.ReplaceLineEndings("\n"); + Assert.Contains(sourceImportLabel + ":\n", assembly); + Assert.Contains(runtimeImportLabel + ":\n", assembly); + Assert.DoesNotContain("extern " + sourceImportLabel + "\n", assembly); + Assert.DoesNotContain("extern " + runtimeImportLabel + "\n", assembly); + Assert.Contains("extern ubytec_bridge_call\n", assembly); + Assert.Contains("global ubytec_host_contract_Main\n", assembly); + Assert.Contains("call ubytec_binary_export_0\n", assembly); + Assert.Contains("section .ubytec_contract alloc noexec nowrite progbits\n", assembly); + } +} diff --git a/Ubytec.Tests/PortableStructuredControlTests.cs b/Ubytec.Tests/PortableStructuredControlTests.cs new file mode 100644 index 00000000..99853cb3 --- /dev/null +++ b/Ubytec.Tests/PortableStructuredControlTests.cs @@ -0,0 +1,91 @@ +using Ubytec.Language.AST; +using Ubytec.Language.Portable; +using Xunit; +using static Ubytec.Language.Operations.CoreOperations; + +namespace Ubytec.Tests; + +public class PortableStructuredControlTests +{ + private const string Source = """ + export Main as Execute (t_int64) -> t_int64 + module (name:"portable-control", version:"0.1", author:"Ubytec") + { + func Main(t_int64 value) -> t_int64 + { + local + { + t_int64 total 0 + } + + while @value > 0 + load @total + load @value + add + store @total + load @value + dec + store @value + end + + load @total + push 3 + gt + if + load @total + push 10 + add + else + load @total + push 20 + add + end + return + } + } + """; + + [Theory] + [InlineData(3L, 16L)] + [InlineData(1L, 21L)] + public void StructuredSourceCrossesVerifiedUbcmAndMatchesLegacySemantics(long value, long expected) + { + SourceCompilation source = SourceCompiler.Compile(Source, emitProcessEntryPoint: false); + byte[] bytes = PortableModuleFormat.Encode( + source.Module, + contractManifest: null, + includeIndexedExecutable: true, + portableContracts: source.PortableContracts); + + PortableBinaryVerifier.Verify(bytes); + PortableBinaryModule binary = PortableBinaryCodec.Decode(bytes); + PortableExecutableExport export = Assert.Single(binary.Executable!.Exports); + PortableExecutableFunction function = binary.Executable.Functions[checked((int)export.FunctionIndex)]; + byte[] structured = function.Instructions + .OfType() + .Select(instruction => instruction.OpCode) + .ToArray(); + + Assert.Contains(LOOP.OP, structured); + Assert.Contains(IF.OP, structured); + Assert.Contains(BREAK.OP, structured); + Assert.Contains(ELSE.OP, structured); + Assert.Contains(END.OP, structured); + Assert.DoesNotContain(WHILE.OP, structured); + + PortableX64Compilation portable = PortableX64Backend.Compile(bytes); + string portableLabel = portable.GetFunctionLabel(checked((int)export.FunctionIndex)); + long sourceResult = RunOneArgument(source.Assembly, source.GetFunctionLabel("Main"), value); + long portableResult = RunOneArgument(portable.Assembly, portableLabel, value); + + Assert.Equal(expected, sourceResult); + Assert.Equal(sourceResult, portableResult); + } + + private static long RunOneArgument(string assembly, string label, long argument) + { + string body = $"mov rax, 0x{unchecked((ulong)argument):X16}\npush rax\n" + + $"call {label}\nadd rsp, 8\npush rax"; + return Assert.Single(NativeStack.Run(body, 1, assembly)); + } +} diff --git a/Ubytec.Tests/PortableSystemVLibraryAbiTests.cs b/Ubytec.Tests/PortableSystemVLibraryAbiTests.cs new file mode 100644 index 00000000..e0087623 --- /dev/null +++ b/Ubytec.Tests/PortableSystemVLibraryAbiTests.cs @@ -0,0 +1,55 @@ +using Xunit; + +namespace Ubytec.Tests; + +public sealed class PortableSystemVLibraryAbiTests +{ + private const string TwoArgumentLibrarySource = """ + export Pair as portable_pair + module (name:"portable-systemv-abi", version:"0.1", author:"Ubytec") + { + func Pair(t_uint64 first,t_int32 second) -> t_uint64 + { + load @first + return + } + } + """; + + [Fact] + public void AdapterPushesSystemVArgumentsInSourceOrderForTheUbytecStackAbi() + { + string directory = Path.Combine( + Path.GetTempPath(), + $"ubytec-systemv-abi-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + + try + { + string source = Path.Combine(directory, "library.ubc"); + string nasm = Path.Combine(directory, "library.nasm"); + File.WriteAllText(source, TwoArgumentLibrarySource); + + int status = CommandLineDriver.Run( + [ + source, + "--library", + "-o", nasm, + "--no-metadata" + ]); + + Assert.Equal(0, status); + string emitted = File.ReadAllText(nasm).ReplaceLineEndings("\n"); + Assert.Contains( + " sub rsp, 8\n" + + " push rdi\n" + + " push rsi\n" + + " call ubytec_binary_export_0\n", + emitted); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } +} diff --git a/Ubytec.Tests/PortableTailCallOptimizerTests.cs b/Ubytec.Tests/PortableTailCallOptimizerTests.cs new file mode 100644 index 00000000..a0421bcc --- /dev/null +++ b/Ubytec.Tests/PortableTailCallOptimizerTests.cs @@ -0,0 +1,57 @@ +using Ubytec.Language.Portable; +using Xunit; + +namespace Ubytec.Tests; + +public class PortableTailCallOptimizerTests +{ + private const string TailSource = """ + module (name:"portable-tails", version:"0.1", author:"Ubytec") + { + func Sum(t_int64 n, t_int64 acc) -> t_int64 + { + local + { + t_int64 step 7 + } + if @n == 0 + load @acc + return + end + load @n + dec + load @acc + load @step + add + call Sum + return + } + } + """; + + [Fact] + public void ExplicitPortableTailCallsReuseVerifiedIndexedFrame() + { + PortableSourceX64Result compilation = PortableSourceX64Pipeline.Compile( + TailSource, + processImage: false, + optimizeTailCalls: true); + + PortableProcessEntrypoint.Verify(compilation.ModuleBytes); + string assembly = compilation.NativeCompilation.Assembly.ReplaceLineEndings("\n"); + Assert.Contains("; tail call: reuse incoming argument cells", assembly); + Assert.Contains("jmp ubytec_binary_f_0", assembly); + Assert.DoesNotContain("call ubytec_binary_f_0\n add rsp, 16\n push rax\n pop rax", assembly); + } + + [Fact] + public void PortableTailCallsRemainOptIn() + { + PortableSourceX64Result compilation = PortableSourceX64Pipeline.Compile( + TailSource, + processImage: false); + + Assert.DoesNotContain("; tail call: reuse incoming argument cells", compilation.NativeCompilation.Assembly); + Assert.Contains("call ubytec_binary_f_0", compilation.NativeCompilation.Assembly); + } +} diff --git a/Ubytec/CommandLineDriver.cs b/Ubytec/CommandLineDriver.cs index 664ee079..4d6e531c 100644 --- a/Ubytec/CommandLineDriver.cs +++ b/Ubytec/CommandLineDriver.cs @@ -12,12 +12,9 @@ using Ubytec.Language.Portable; using Ubytec.Language.Tools; -[assembly: System.CLSCompliant(true)] -[assembly: System.Runtime.InteropServices.ComVisible(true)] - namespace Ubytec; -/// End-to-end command-line driver for legacy and portable-native compilation paths. +/// End-to-end command-line driver for canonical portable and legacy source-native compilation paths. internal static class CommandLineDriver { private static readonly string CanonicalLineEnding = ((char)10).ToString(); @@ -25,7 +22,16 @@ internal static class CommandLineDriver private static readonly IReadOnlyDictionary> FlagSetters = new Dictionary>(StringComparer.Ordinal) { - ["--portable-x64"] = options => options.PortableX64 = true, + ["--portable-x64"] = options => + { + options.PortableX64 = true; + options.PortableX64Requested = true; + }, + ["--legacy-source-x64"] = options => + { + options.PortableX64 = false; + options.LegacySourceX64Requested = true; + }, ["--indexed-executable"] = options => options.IndexedExecutable = true, ["--arena-imports"] = options => options.ArenaImports = true, ["--host-io"] = options => options.HostIo = true, @@ -162,12 +168,22 @@ private static void ApplyImplications(Options options) private static void Validate(Options options) { + ValidatePipelineSelection(options); ValidateOutputPaths(options); ValidateModeDependencies(options); if (options.PortableX64) ValidatePortableX64(options); } + private static void ValidatePipelineSelection(Options options) + { + if (options.PortableX64Requested && options.LegacySourceX64Requested) + { + throw new ArgumentException( + "--portable-x64 and --legacy-source-x64 select mutually exclusive native compilation pipelines."); + } + } + private static void ValidateOutputPaths(Options options) { if (options.GeneratedSourcePath is not null) @@ -179,7 +195,7 @@ private static void ValidateOutputPaths(Options options) private static void ValidateModeDependencies(Options options) { if (options.IndexedExecutable && options.PortableModulePath is null && !options.PortableX64) - throw new ArgumentException("--indexed-executable requires --portable-module unless --portable-x64 is selected."); + throw new ArgumentException("--indexed-executable requires --portable-module on the legacy source-to-x64 path."); if (options.ModuleCalls && !options.Library) throw new ArgumentException("--module-calls requires --library."); if (options.Exports.Count != 0 && !options.Library) @@ -188,12 +204,10 @@ private static void ValidateModeDependencies(Options options) private static void ValidatePortableX64(Options options) { - if (options.Exports.Count != 0) - throw new ArgumentException("--portable-x64 does not yet support --host-export; portable exports must come from the Ubytec module contract."); - + // Host exports and tail-call selection now have portable contracts. The remaining switches below + // still belong to the deprecated direct source backend and remain explicit legacy-only requests. (bool Enabled, string Name)[] legacyNativeFeatures = [ - (options.TailCalls, "--tail-calls"), (options.StackTransfers, "--stack-transfers"), (options.FrameValues, "--frame-values"), (options.FrameRegisters, "--frame-registers"), @@ -211,8 +225,9 @@ private static void ValidatePortableX64(Options options) if (unsupported.Length != 0) { throw new ArgumentException( - "--portable-x64 rejects legacy source-to-x64 optimization flags until they have a binary-backend contract: " + - string.Join(", ", unsupported) + "."); + "The canonical portable x64 pipeline rejects legacy source-to-x64 optimization flags until they have a binary-backend contract: " + + string.Join(", ", unsupported) + + ". Use --legacy-source-x64 to opt into deprecated direct source-to-x64 compilation."); } } @@ -278,12 +293,17 @@ private static void Execute(Options options) private static void ExecutePortable(string source, Options options, Encoding encoding) { + PortableRequestedExport[] requestedExports = options.Exports + .Select(ParsePortableRequestedExport) + .ToArray(); PortableSourceX64Result result = PortableSourceX64Pipeline.Compile( source, options.GrammarPath is null ? null : File.ReadAllText(options.GrammarPath), processImage: !options.Library, enableModuleCalls: options.ModuleCalls, - enableHostIO: options.HostIo); + enableHostIO: options.HostIo, + optimizeTailCalls: options.TailCalls, + requestedExports: requestedExports); string finalAssembly = result.NativeCompilation.Assembly.ReplaceLineEndings(CanonicalLineEnding); File.WriteAllText(options.OutputPath, finalAssembly, encoding); @@ -293,6 +313,19 @@ private static void ExecutePortable(string source, Options options, Encoding enc PrintOutputs(options); } + private static PortableRequestedExport ParsePortableRequestedExport(string value) + { + int separator = value.IndexOf('='); + if (separator <= 0 || separator == value.Length - 1 || + value.IndexOf('=', separator + 1) >= 0) + { + throw new ArgumentException( + "Invalid host export; expected Function=ubytec_host_symbol."); + } + + return new PortableRequestedExport(value[..separator], value[(separator + 1)..]); + } + private static void ExecuteLegacy(string source, Options options, Encoding encoding) { SourceCompilation compilation = CompileLegacy(source, options); @@ -418,6 +451,7 @@ private static string[] GetFeatures(Options options) (options.Library, "library"), (options.PortableX64, "portable-x64-pipeline-v0"), (options.PortableX64 && !options.Library, "process-entrypoint-v0"), + (options.LegacySourceX64Requested, "legacy-source-x64"), (options.TailCalls, "tail-calls"), (options.StackTransfers, "stack-transfers"), (options.FrameValues, "frame-values"), @@ -439,14 +473,18 @@ private static string[] GetFeatures(Options options) private static void PrintHelp() { - Console.WriteLine("Usage: ubytec [-o ] [--portable-x64] [--portable-module ] [--indexed-executable] [--grammar ] [--json ] [--metadata ] [--no-metadata] [--library] [--tail-calls] [--stack-transfers] [--frame-values] [--frame-registers] [--loop-reductions] [--block-reductions] [--block-reductions-native] [--record-reductions] [--lookup-reductions] [--effect-registers] [--module-calls] [--host-io] [--arena-imports] [--generated-source ] [--host-export Function=ubytec_host_symbol]"); + Console.WriteLine("Usage: ubytec [-o ] [--portable-x64] [--legacy-source-x64] [--portable-module ] [--indexed-executable] [--grammar ] [--json ] [--metadata ] [--no-metadata] [--library] [--tail-calls] [--stack-transfers] [--frame-values] [--frame-registers] [--loop-reductions] [--block-reductions] [--block-reductions-native] [--record-reductions] [--lookup-reductions] [--effect-registers] [--module-calls] [--host-io] [--arena-imports] [--generated-source ] [--host-export Function=ubytec_host_symbol]"); Console.WriteLine(" ubytec --verify-module "); Console.WriteLine(" ubytec --format-stdin < input.ubc > output.ubc"); - Console.WriteLine("--portable-x64 routes final NASM through indexed UBCM -> independent verifier -> typed decoder -> x64; legacy source-to-x64 assembly is discarded."); - Console.WriteLine("--portable-module emits canonical architecture-neutral Portable Module Format v0; with --portable-x64 it is the exact UBCM consumed by native codegen."); - Console.WriteLine("--indexed-executable adds the verified indexed executable section group to legacy --portable-module output; --portable-x64 implies it."); + Console.WriteLine("Portable x64 is the default native path: source -> semantic frontend -> indexed UBCM -> independent verifier -> typed decoder -> x64."); + Console.WriteLine("--portable-x64 is retained as a compatibility alias for the canonical default pipeline."); + Console.WriteLine("--legacy-source-x64 opts into the deprecated direct source -> x64 path during migration; it bypasses the canonical UBCM backend boundary."); + Console.WriteLine("--portable-module emits canonical architecture-neutral Portable Module Format v0; on the canonical path it is the exact UBCM consumed by native codegen."); + Console.WriteLine("--indexed-executable adds the indexed executable section group to legacy --portable-module output; the canonical portable path always includes it."); Console.WriteLine("--verify-module validates PMF framing, indexed executable tables/code and optional process-entry metadata without compiling source."); - Console.WriteLine("--portable-x64 currently rejects legacy backend optimization flags and --host-export rather than silently changing semantics."); + Console.WriteLine("--host-export adds a host-facing System V export contract before UBCM serialization; the canonical path verifies that export before emitting its adapter."); + Console.WriteLine("--tail-calls is supported by the canonical x64 path as an explicit post-verification UBCM optimization; it changes physical frame lifetime and remains opt-in."); + Console.WriteLine("The canonical portable path rejects remaining legacy backend optimization flags rather than silently selecting the legacy compiler."); Console.WriteLine("--format-stdin canonicalizes UBC from standard input to standard output without compiling it."); Console.WriteLine("--metadata writes build provenance separately from canonical NASM; by default .metadata.json is written."); Console.WriteLine("--no-metadata disables the provenance sidecar."); @@ -466,7 +504,9 @@ private sealed class Options(string sourcePath) internal string? GeneratedSourcePath { get; set; } internal string? PortableModulePath { get; set; } internal bool EmitMetadata { get; set; } = true; - internal bool PortableX64 { get; set; } + internal bool PortableX64 { get; set; } = true; + internal bool PortableX64Requested { get; set; } + internal bool LegacySourceX64Requested { get; set; } internal bool IndexedExecutable { get; set; } internal bool Library { get; set; } internal bool TailCalls { get; set; } diff --git a/Ubytec/Language/AST/ModuleContracts.cs b/Ubytec/Language/AST/ModuleContracts.cs index d03b39d4..9ca5fdff 100644 --- a/Ubytec/Language/AST/ModuleContracts.cs +++ b/Ubytec/Language/AST/ModuleContracts.cs @@ -60,13 +60,49 @@ private static void ValidateTyped(string signature) { halves[1] is not ("Int64" or "UInt64")) throw new ArgumentException("Typed calls support up to six 64-bit arguments and two loans."); } - internal string Emit(SourceCompilation compilation) + + /// + /// Builds the canonical loader contract independently of native code generation. Portable compilation + /// embeds this exact text in the critical UBCM ContractManifest section so aliases and authority-slot + /// names cross the verifier boundary instead of being recovered from source after code generation. + /// + internal string BuildManifest(Module module) { + if (Imports.Count == 0 && Exports.Count == 0) return string.Empty; var manifest=new StringBuilder("UBC1\n"); + foreach(var import in Imports) + manifest.AppendLine($"I|{import.Handle}|{import.Alias}|{import.Module}|{import.Export}|{import.Signature}"); + foreach(var export in Exports) + { + string signature=ExportSignature(module,export); + string symbol="ubytec_host_contract_"+export.Name; + if(symbol.Length>=64) throw new ArgumentException("Public export name exceeds the adapter symbol limit."); + manifest.AppendLine($"E|{export.Name}|{symbol}|{signature}"); + } + manifest.AppendLine("END"); + string text=manifest.ToString().Replace("\r\n","\n"); + if(Encoding.ASCII.GetByteCount(text)>4096) throw new ArgumentException("Contract too large."); + return text; + } + + private static string ExportSignature(Module module,(string Function,string Name,string? Signature) export) + { + var fn=module.Functions.FirstOrDefault(f=>f.Name==export.Function); + if(string.IsNullOrEmpty(fn.Name)) throw new ArgumentException("Unknown exported function."); + string signature=string.Join(",",fn.Arguments.Select(a=>a.Type.Type.ToString()))+":"+fn.ReturnType.Type; + if(export.Signature is { } declared) { + string actual=declared.Replace("LoanRead","UInt64").Replace("LoanWrite","UInt64"); + if(actual!=signature) throw new ArgumentException("Export loan annotations must match its scalar ABI."); + signature=declared; + } + return signature; + } + + internal string Emit(SourceCompilation compilation) + { var assembly=new StringBuilder(); foreach(var import in Imports) { - manifest.AppendLine($"I|{import.Handle}|{import.Alias}|{import.Module}|{import.Export}|{import.Signature}"); assembly.AppendLine($"section .text\nubytec_import_{import.Alias}:\n push rbp\n mov rbp,rsp\n push {import.Handle}"); if(import.Signature==BufferSignature) { foreach(int offset in new[]{48,40,32,24,16}) assembly.AppendLine($" push qword [rbp+{offset}]"); @@ -89,23 +125,12 @@ internal string Emit(SourceCompilation compilation) } foreach(var export in Exports) { - var fn=compilation.Module.Functions.FirstOrDefault(f=>f.Name==export.Function); - if(string.IsNullOrEmpty(fn.Name)) throw new ArgumentException("Unknown exported function."); - string signature=string.Join(",",fn.Arguments.Select(a=>a.Type.Type.ToString()))+":"+fn.ReturnType.Type; - if(export.Signature is { } declared) { - string actual=declared.Replace("LoanRead","UInt64").Replace("LoanWrite","UInt64"); - if(actual!=signature) throw new ArgumentException("Export loan annotations must match its scalar ABI."); - signature=declared; - } + _=ExportSignature(compilation.Module,export); string symbol="ubytec_host_contract_"+export.Name; - if(symbol.Length>=64) throw new ArgumentException("Public export name exceeds the adapter symbol limit."); - manifest.AppendLine($"E|{export.Name}|{symbol}|{signature}"); assembly.Append(compilation.GetLinuxSystemVAdapter(export.Function,symbol)); } - manifest.AppendLine("END"); - compilation.ContractManifest=manifest.ToString().Replace("\r\n","\n"); + compilation.ContractManifest=BuildManifest(compilation.Module); var bytes=Encoding.ASCII.GetBytes(compilation.ContractManifest); - if(bytes.Length>4096) throw new ArgumentException("Contract too large."); assembly.AppendLine("section .ubytec_contract alloc noexec nowrite progbits"); assembly.AppendLine("db "+string.Join(",",bytes)); return assembly.ToString(); diff --git a/Ubytec/Language/Portable/IndexedExecutableFormat.cs b/Ubytec/Language/Portable/IndexedExecutableFormat.cs index 1f863382..89dc3675 100644 --- a/Ubytec/Language/Portable/IndexedExecutableFormat.cs +++ b/Ubytec/Language/Portable/IndexedExecutableFormat.cs @@ -1,10 +1,12 @@ using System.Buffers.Binary; +using System.Globalization; using System.Reflection; using System.Text; using Ubytec.Language.HighLevel; using Ubytec.Language.Operations; using Ubytec.Language.Operations.Extended; using Ubytec.Language.Operations.Interfaces; +using Ubytec.Language.Syntax.ExpressionFragments; using Ubytec.Language.Syntax.Model; using static Ubytec.Language.Operations.CoreOperations; using static Ubytec.Language.Syntax.TypeSystem.Types; @@ -573,6 +575,64 @@ private static void EncodeInstruction( EncodeMemory(stream, MemoryOperations.STORE.OP, store.VariableName, store.AccessType, callable); return; + case BLOCK block: + RequireDefaultBlockType(block.BlockType, operation); + stream.WriteByte(BLOCK.OP); + return; + + case LOOP loopBlock: + RequireDefaultBlockType(loopBlock.BlockType, operation); + stream.WriteByte(LOOP.OP); + return; + + case IF conditional: + if (conditional.Condition is null) + { + RequireDefaultBlockType(conditional.BlockType, operation); + stream.WriteByte(IF.OP); + return; + } + EncodeCondition(stream, conditional.Condition, conditional.BlockType, callable); + stream.WriteByte(IF.OP); + return; + + case WHILE whileBlock: + if (whileBlock.LabelIDxs is { Length: > 0 }) + throw Unsupported(operation, "labeled WHILE has no indexed executable v0 wire contract"); + if (whileBlock.Condition is null) + { + RequireDefaultBlockType(whileBlock.BlockType, operation); + stream.WriteByte(WHILE.OP); + return; + } + // Normalize source WHILE into LOOP + an architecture-neutral guard. The guard lives + // inside the LOOP so CONTINUE/END reevaluate it on every iteration. + stream.WriteByte(LOOP.OP); + EncodeCondition(stream, whileBlock.Condition, whileBlock.BlockType, callable); + EncodePushCell(stream, 0); + stream.WriteByte(ComparisonOperations.EQ.OP); + stream.WriteByte(IF.OP); + stream.WriteByte(BREAK.OP); + stream.WriteByte(END.OP); + return; + + case ELSE alternate: + RequireDefaultBlockType(alternate.BlockType, operation); + stream.WriteByte(ELSE.OP); + return; + + case BREAK breakOperation when breakOperation.LabelIDx is null: + stream.WriteByte(BREAK.OP); + return; + + case CONTINUE continueOperation when continueOperation.LabelIDx is null: + stream.WriteByte(CONTINUE.OP); + return; + + case END _: + stream.WriteByte(END.OP); + return; + default: if (IsOperandFreeCore(operation)) { @@ -613,6 +673,110 @@ private static void EncodeMemory( stream.WriteByte((byte)accessType.Value); } + private static void EncodeCondition( + Stream stream, + SyntaxExpression expression, + UType? explicitComparisonType, + CallableInfo callable) + { + ConditionExpressionFragment[] conditions = expression.Syntaxes + .OfType() + .ToArray(); + if (conditions.Length == 0 || conditions.Length != expression.Syntaxes.Count) + throw new NotSupportedException("Portable structured conditions must contain only comparison fragments."); + + for (int index = 0; index < conditions.Length; index++) + { + ConditionExpressionFragment condition = conditions[index]; + EncodeConditionValue(stream, condition.Left, callable); + EncodeConditionValue(stream, condition.Right, callable); + stream.WriteByte(ComparisonOpcode(condition, explicitComparisonType, callable)); + if (index != 0) + stream.WriteByte(BitwiseOperations.AND.OP); + } + } + + private static void EncodeConditionValue(Stream stream, object? rawValue, CallableInfo callable) + { + string value = Convert.ToString(rawValue, CultureInfo.InvariantCulture) ?? string.Empty; + if (value.StartsWith('@')) + { + EncodeMemory( + stream, + MemoryOperations.LOAD.OP, + FunctionCode.Identifier(value), + null, + callable); + return; + } + + EncodePushCell(stream, unchecked((ulong)FunctionCode.InitialValue(new UType(PrimitiveType.Int64), rawValue))); + } + + private static void EncodePushCell(Stream stream, ulong value) + { + stream.WriteByte(StackOperations.PUSH.OP); + stream.WriteByte(8); + WriteUInt64(stream, value); + } + + private static byte ComparisonOpcode( + ConditionExpressionFragment condition, + UType? explicitComparisonType, + CallableInfo callable) + { + PrimitiveType? explicitType = explicitComparisonType?.Type == PrimitiveType.Default + ? null + : explicitComparisonType?.Type; + if (explicitType is { } selected && !FunctionCode.IsCellType(new UType(selected))) + throw new InvalidDataException($"Condition type {selected} is not supported by the portable cell backend."); + + PrimitiveType[] namedTypes = new[] { condition.Left, condition.Right } + .Select(value => Convert.ToString(value, CultureInfo.InvariantCulture) ?? string.Empty) + .Where(value => value.StartsWith('@')) + .Select(value => BindingType(callable, FunctionCode.Identifier(value))) + .Where(type => type.IsNumeric()) + .ToArray(); + + bool useUnsigned; + if (explicitType is { } explicitPrimitive) + { + useUnsigned = explicitPrimitive.IsUnsigned(); + } + else + { + bool anyUnsigned = namedTypes.Any(type => type.IsUnsigned()); + bool anySigned = namedTypes.Any(type => type.IsSigned()); + if (anyUnsigned && anySigned) + throw new InvalidDataException( + "Mixed signed/unsigned condition operands require an explicit comparison type."); + useUnsigned = anyUnsigned; + } + + return condition.Operand switch + { + "==" => ComparisonOperations.EQ.OP, + "!=" => ComparisonOperations.NEQ.OP, + "<" => useUnsigned ? ComparisonOperations.ULT.OP : ComparisonOperations.LT.OP, + "<=" => useUnsigned ? ComparisonOperations.ULE.OP : ComparisonOperations.LE.OP, + ">" => useUnsigned ? ComparisonOperations.UGT.OP : ComparisonOperations.GT.OP, + ">=" => useUnsigned ? ComparisonOperations.UGE.OP : ComparisonOperations.GE.OP, + _ => throw new InvalidDataException($"Unsupported portable condition operator '{condition.Operand}'.") + }; + } + + private static PrimitiveType BindingType(CallableInfo callable, string name) + { + uint index = BindingIndex(callable, name); + return callable.Bindings[checked((int)index)].Type.Type; + } + + private static void RequireDefaultBlockType(UType? type, IOpCode operation) + { + if (type is { } blockType && blockType.Type != PrimitiveType.Default) + throw Unsupported(operation, "typed structured block metadata has no indexed executable v0 wire contract"); + } + private static bool TryResolveCallable( CallableInfo current, string name, @@ -1113,6 +1277,7 @@ private static void VerifyCode( ReadOnlySpan body = code.Slice(checked((int)function.CodeOffset), checked((int)function.CodeLength)); var reader = new SpanReader(body); ulong bindingCount = (ulong)function.ArgumentCount + function.LocalCount; + var controls = new Stack(); while (!reader.AtEnd) { @@ -1236,6 +1401,32 @@ private static void VerifyCode( break; } + case BLOCK.OP: + case LOOP.OP: + case IF.OP: + case WHILE.OP: + controls.Push(opcode); + break; + + case ELSE.OP: + if (controls.Count == 0 || controls.Pop() != IF.OP) + throw new InvalidDataException("Portable ELSE requires a directly enclosing IF."); + controls.Push(ELSE.OP); + break; + + case END.OP: + if (controls.Count == 0) + throw new InvalidDataException("Portable END has no matching structured control opener."); + controls.Pop(); + break; + + case BREAK.OP: + case CONTINUE.OP: + if (!controls.Any(control => control == LOOP.OP || control == WHILE.OP)) + throw new InvalidDataException( + $"Portable opcode 0x{opcode:X2} requires an enclosing LOOP or WHILE."); + break; + default: if (!OperandFreeCoreOpcodes.Contains(opcode)) throw new InvalidDataException($"Core opcode 0x{opcode:X2} has no supported indexed executable v0 wire contract."); @@ -1244,6 +1435,9 @@ private static void VerifyCode( } reader.RequireEnd($"Function {functionIndex} code"); + if (controls.Count != 0) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} contains {controls.Count} unterminated structured control scope(s)."); } } diff --git a/Ubytec/Language/Portable/PortableBinaryVerifier.cs b/Ubytec/Language/Portable/PortableBinaryVerifier.cs index 9231282c..3679de76 100644 --- a/Ubytec/Language/Portable/PortableBinaryVerifier.cs +++ b/Ubytec/Language/Portable/PortableBinaryVerifier.cs @@ -513,6 +513,7 @@ private static void VerifyCode( var reader = new RawReader(body); ulong bindingCount = (ulong)function.ArgumentCount + function.LocalCount; var initializedLocals = new bool[checked((int)function.LocalCount)]; + var controls = new Stack(); int stackDepth = 0; bool reachable = true; @@ -531,6 +532,22 @@ void Apply(int required, int delta, byte opcode) stackDepth = checked(stackDepth + delta); } + void RequireDepth(int expected, byte opcode, string boundary) + { + if (reachable && stackDepth != expected) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} opcode 0x{opcode:X2} reaches {boundary} with evaluation " + + $"stack depth {stackDepth}; expected {expected}."); + } + + ControlFrame TransferTarget(byte opcode) + { + ControlFrame? target = controls.FirstOrDefault(frame => + frame.Kind is ControlKind.Loop or ControlKind.While); + return target ?? throw new InvalidDataException( + $"FunctionIndex {functionIndex} opcode 0x{opcode:X2} requires an enclosing LOOP or WHILE."); + } + while (!reader.AtEnd) { byte opcode = reader.ReadByte("Opcode"); @@ -716,6 +733,119 @@ void Apply(int required, int delta, byte opcode) break; } + case BLOCK.OP: + controls.Push(new ControlFrame(ControlKind.Block, stackDepth, reachable)); + break; + + case LOOP.OP: + controls.Push(new ControlFrame(ControlKind.Loop, stackDepth, reachable)); + break; + + case IF.OP: + Apply(1, -1, opcode); + controls.Push(new ControlFrame(ControlKind.If, stackDepth, reachable)); + break; + + case WHILE.OP: + Apply(1, -1, opcode); + controls.Push(new ControlFrame(ControlKind.While, stackDepth, reachable)); + break; + + case ELSE.OP: + { + if (controls.Count == 0 || controls.Peek().Kind != ControlKind.If) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} ELSE requires a directly enclosing IF."); + ControlFrame conditional = controls.Pop(); + controls.Push(new ControlFrame( + ControlKind.Else, + conditional.EntryDepth, + conditional.EntryReachable) + { + ThenReachable = reachable, + ThenDepth = stackDepth + }); + reachable = conditional.EntryReachable; + stackDepth = conditional.EntryDepth; + break; + } + + case END.OP: + { + if (controls.Count == 0) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} END has no matching structured control opener."); + ControlFrame frame = controls.Pop(); + switch (frame.Kind) + { + case ControlKind.Block: + RequireDepth(frame.EntryDepth, opcode, "BLOCK end"); + stackDepth = frame.EntryDepth; + break; + + case ControlKind.If: + RequireDepth(frame.EntryDepth, opcode, "IF end"); + reachable = frame.EntryReachable; + stackDepth = frame.EntryDepth; + break; + + case ControlKind.Else: + { + bool elseReachable = reachable; + int elseDepth = stackDepth; + if (frame.ThenReachable && elseReachable && frame.ThenDepth != elseDepth) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} IF/ELSE branches merge with stack depths " + + $"{frame.ThenDepth} and {elseDepth}."); + reachable = frame.ThenReachable || elseReachable; + stackDepth = frame.ThenReachable + ? frame.ThenDepth + : elseReachable ? elseDepth : frame.EntryDepth; + break; + } + + case ControlKind.Loop: + RequireDepth(frame.EntryDepth, opcode, "LOOP backedge"); + reachable = frame.HasReachableBreak; + stackDepth = frame.EntryDepth; + break; + + case ControlKind.While: + RequireDepth(frame.EntryDepth + 1, opcode, "WHILE backedge"); + reachable = frame.EntryReachable || frame.HasReachableBreak; + stackDepth = frame.EntryDepth; + break; + + default: + throw new InvalidDataException("Unknown portable structured control frame."); + } + break; + } + + case BREAK.OP: + { + ControlFrame target = TransferTarget(opcode); + RequireDepth(target.EntryDepth, opcode, "BREAK target"); + if (reachable) + { + target.HasReachableBreak = true; + reachable = false; + } + break; + } + + case CONTINUE.OP: + { + ControlFrame target = TransferTarget(opcode); + int expected = target.Kind == ControlKind.While + ? target.EntryDepth + 1 + : target.EntryDepth; + RequireDepth(expected, opcode, "CONTINUE target"); + if (reachable) + reachable = false; + break; + } + default: if (!OperandFreeCoreOpcodes.Contains(opcode)) throw new InvalidDataException( @@ -727,6 +857,10 @@ void Apply(int required, int delta, byte opcode) reader.RequireEnd($"Function {functionIndex} code"); + if (controls.Count != 0) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} contains {controls.Count} unterminated structured control scope(s)."); + for (int local = 0; local < initializedLocals.Length; local++) { if (!initializedLocals[local]) @@ -1139,6 +1273,25 @@ private readonly record struct TypeEntry( ulong FixedArraySize, uint NameStringIndex); + private enum ControlKind : byte + { + Block, + If, + Else, + Loop, + While + } + + private sealed class ControlFrame(ControlKind kind, int entryDepth, bool entryReachable) + { + internal ControlKind Kind { get; } = kind; + internal int EntryDepth { get; } = entryDepth; + internal bool EntryReachable { get; } = entryReachable; + internal bool ThenReachable { get; set; } + internal int ThenDepth { get; set; } + internal bool HasReachableBreak { get; set; } + } + private sealed class ByteArrayComparer : IComparer { internal static ByteArrayComparer Instance { get; } = new(); diff --git a/Ubytec/Language/Portable/PortableSourceX64Pipeline.cs b/Ubytec/Language/Portable/PortableSourceX64Pipeline.cs index aaf19635..13a2ea00 100644 --- a/Ubytec/Language/Portable/PortableSourceX64Pipeline.cs +++ b/Ubytec/Language/Portable/PortableSourceX64Pipeline.cs @@ -17,15 +17,18 @@ public static class PortableSourceX64Pipeline { /// /// Compiles source semantically to indexed PMF and then generates x64 only from the resulting verified binary. - /// The returned semantic module is exposed only for diagnostics and sidecars; native code generation - /// depends exclusively on . + /// Requested host exports, source loader contracts and reserved supervisor imports are serialized before the + /// verifier boundary. Tail-call optimization, runtime bindings and native ABI projections run only afterward + /// and are driven by the verified UBCM representation rather than source or AST identity. /// public static PortableSourceX64Result Compile( string source, string? grammarJson = null, bool processImage = true, bool enableModuleCalls = false, - bool enableHostIO = false) + bool enableHostIO = false, + bool optimizeTailCalls = false, + IReadOnlyList? requestedExports = null) { SourceSemanticCompilation frontend = SourceCompiler.CompileSemantic( source, @@ -33,12 +36,25 @@ public static PortableSourceX64Result Compile( processImage: processImage, enableModuleCalls: enableModuleCalls, enableHostIO: enableHostIO); + PortableModuleContracts portableContracts = + PortableSystemVLibraryAbi.AddRequestedExports(frontend, requestedExports); + portableContracts = PortableX64RuntimeImports.AddContracts( + portableContracts, + enableModuleCalls, + enableHostIO); + + // The source contract parser is deterministic and is reused here only before serialization. + // Its loader-facing alias/slot names are embedded in the critical UBCM ContractManifest; + // no source contract state is consulted after the verifier boundary. + var sourceContracts = new ModuleContracts(); + _ = sourceContracts.Read(source); + string contractManifest = sourceContracts.BuildManifest(frontend.Module); byte[] moduleBytes = PortableModuleFormat.Encode( frontend.Module, - contractManifest: null, + contractManifest: string.IsNullOrEmpty(contractManifest) ? null : contractManifest, includeIndexedExecutable: true, - portableContracts: frontend.PortableContracts); + portableContracts: portableContracts); if (processImage) { @@ -59,6 +75,17 @@ public static PortableSourceX64Result Compile( PortableProcessEntrypoint.Verify(moduleBytes); PortableX64Compilation native = PortableProcessX64Backend.Compile(moduleBytes); + if (optimizeTailCalls) + native = PortableX64TailCallOptimizer.Apply(moduleBytes, native); + native = PortableX64RuntimeImports.Attach(moduleBytes, native); + if (!processImage) + { + native = PortableX64ModuleContractAbi.Attach(moduleBytes, native); + native = native with + { + Assembly = PortableSystemVLibraryAbi.Attach(moduleBytes, native) + }; + } return new PortableSourceX64Result(moduleBytes, native, frontend.Module); } diff --git a/Ubytec/Language/Portable/PortableSystemVLibraryAbi.cs b/Ubytec/Language/Portable/PortableSystemVLibraryAbi.cs new file mode 100644 index 00000000..1b71746a --- /dev/null +++ b/Ubytec/Language/Portable/PortableSystemVLibraryAbi.cs @@ -0,0 +1,206 @@ +using System.Text; +using Ubytec.Language.AST; +using Ubytec.Language.HighLevel; +using static Ubytec.Language.Syntax.TypeSystem.Types; + +namespace Ubytec.Language.Portable; + +/// A CLI-requested host export that must cross the portable binary boundary before native binding. +/// Top-level Ubytec function to export. +/// Native System V symbol requested by the embedding host. +public sealed record PortableRequestedExport(string FunctionName, string Symbol); + +/// +/// Linux System V library ABI adapters generated exclusively from verified UBCM export metadata. +/// The portable x64 backend remains index-only; source names never participate in native code generation. +/// +public static class PortableSystemVLibraryAbi +{ + private static readonly string[] ArgumentRegisters = ["rdi", "rsi", "rdx", "rcx", "r8", "r9"]; + private static readonly string[] SavedRegisters = ["rbx", "r12", "r13", "r14", "r15"]; + + /// + /// Appends host-facing System V adapters for every export carried by a verified portable library. + /// Each adapter calls the backend's index-derived export label using Ubytec's stack ABI. + /// + public static string Attach( + ReadOnlySpan moduleBytes, + PortableX64Compilation nativeCompilation) + { + ArgumentNullException.ThrowIfNull(nativeCompilation); + + PortableProcessEntrypoint.Verify(moduleBytes); + PortableBinaryModule module = PortableBinaryCodec.Decode(moduleBytes); + if (PortableProcessEntrypoint.Read(module) is not null) + throw new InvalidDataException("System V export adapters are valid only for portable libraries."); + if (module.Executable is not { } executable) + throw new InvalidDataException("System V export adapters require the indexed executable profile."); + if (nativeCompilation.ExportLabels.Count != executable.Exports.Count) + throw new InvalidDataException("Native export labels do not match the verified UBCM export table."); + + if (executable.Exports.Count == 0) + return nativeCompilation.Assembly.ReplaceLineEndings("\n"); + + var symbols = new HashSet(StringComparer.Ordinal); + var output = new StringBuilder(nativeCompilation.Assembly.ReplaceLineEndings("\n")); + output.AppendLine("section .text"); + + for (int index = 0; index < executable.Exports.Count; index++) + { + PortableExecutableExport export = executable.Exports[index]; + string symbol = ResolveSymbol(executable, export); + ValidateNativeSymbol(symbol, symbols); + ValidateSignature(executable, export, symbol); + EmitAdapter(output, symbol, nativeCompilation.GetExportLabel(index), export.Arguments.Count); + } + + output.AppendLine("section .note.GNU-stack noalloc noexec nowrite progbits"); + return output.ToString().ReplaceLineEndings("\n"); + } + + /// + /// Adds command-line host exports to semantic portable contracts before UBCM serialization. + /// This makes --host-export syntactic sugar for an export contract rather than a post-verification escape hatch. + /// + public static PortableModuleContracts AddRequestedExports( + SourceSemanticCompilation frontend, + IReadOnlyList? requestedExports) + { + ArgumentNullException.ThrowIfNull(frontend); + if (requestedExports is null || requestedExports.Count == 0) + return frontend.PortableContracts; + + var exports = frontend.PortableContracts.Exports.ToList(); + var symbols = exports.Select(export => export.Symbol).ToHashSet(StringComparer.Ordinal); + + foreach (PortableRequestedExport requested in requestedExports) + { + ValidateHostRequestedSymbol(requested.Symbol); + if (!symbols.Add(requested.Symbol)) + throw new ArgumentException($"Duplicate portable export symbol '{requested.Symbol}'."); + + Func function = frontend.Module.Functions.FirstOrDefault( + item => string.Equals(item.Name, requested.FunctionName, StringComparison.Ordinal)); + if (string.IsNullOrEmpty(function.Name)) + throw new ArgumentException($"Unknown exported function '{requested.FunctionName}'."); + + PortableBoundaryParameter[] arguments = function.Arguments + .Select(argument => new PortableBoundaryParameter( + PortablePassingMode.Value, + argument.Type.Type)) + .ToArray(); + exports.Add(new PortableExportContract( + function.Name, + requested.Symbol, + function.ReturnType.Type, + arguments)); + } + + return frontend.PortableContracts with { Exports = exports }; + } + + private static string ResolveSymbol( + PortableExecutableModule executable, + PortableExecutableExport export) + { + if (export.SymbolStringIndex >= executable.Strings.Count) + throw new InvalidDataException("Export SymbolStringIndex is outside the verified string table."); + return executable.Strings[checked((int)export.SymbolStringIndex)]; + } + + private static void ValidateNativeSymbol(string symbol, HashSet symbols) + { + if (!IsNativeIdentifier(symbol)) + throw new InvalidDataException( + $"Portable export '{symbol}' cannot be represented as a Linux System V symbol."); + if (symbol.StartsWith("ubytec_binary_", StringComparison.Ordinal)) + throw new InvalidDataException( + $"Portable export '{symbol}' collides with the reserved binary-backend namespace."); + if (!symbols.Add(symbol)) + throw new InvalidDataException($"Duplicate portable export symbol '{symbol}'."); + } + + private static void ValidateHostRequestedSymbol(string symbol) + { + if (!IsNativeIdentifier(symbol) || !symbol.StartsWith("ubytec_host_", StringComparison.Ordinal)) + throw new ArgumentException( + "Host symbols must begin with ubytec_host_ and contain only ASCII identifier characters.", + nameof(symbol)); + } + + private static bool IsNativeIdentifier(string symbol) + { + if (string.IsNullOrEmpty(symbol) || !(IsAsciiLetter(symbol[0]) || symbol[0] == '_')) + return false; + for (int index = 1; index < symbol.Length; index++) + if (!(IsAsciiLetter(symbol[index]) || char.IsAsciiDigit(symbol[index]) || symbol[index] == '_')) + return false; + return true; + } + + private static bool IsAsciiLetter(char value) => + value is >= 'A' and <= 'Z' or >= 'a' and <= 'z'; + + private static void ValidateSignature( + PortableExecutableModule executable, + PortableExecutableExport export, + string symbol) + { + if (export.FunctionIndex >= executable.Functions.Count) + throw new InvalidDataException($"Export '{symbol}' FunctionIndex is outside the verified function table."); + if (export.Arguments.Count > ArgumentRegisters.Length) + throw new NotSupportedException( + $"System V export '{symbol}' has more than six arguments."); + + foreach (PortableBoundaryDescriptor argument in export.Arguments) + { + PrimitiveType type = ResolvePrimitive(executable, argument.TypeIndex); + if (!IsSystemVCell(type)) + throw new NotSupportedException( + $"System V export '{symbol}' contains unsupported argument carrier type '{type}'."); + } + + PrimitiveType returnType = ResolvePrimitive(executable, export.ReturnTypeIndex); + if (returnType != PrimitiveType.Void && !IsSystemVCell(returnType)) + throw new NotSupportedException( + $"System V export '{symbol}' contains unsupported return carrier type '{returnType}'."); + } + + private static PrimitiveType ResolvePrimitive(PortableExecutableModule executable, uint typeIndex) + { + if (typeIndex >= executable.Types.Count) + throw new InvalidDataException("Export TypeIndex is outside the verified type table."); + return executable.Types[checked((int)typeIndex)].Primitive; + } + + private static bool IsSystemVCell(PrimitiveType type) => type is + PrimitiveType.Byte or + PrimitiveType.UInt32 or + PrimitiveType.Int32 or + PrimitiveType.UInt64 or + PrimitiveType.Int64; + + private static void EmitAdapter( + StringBuilder output, + string symbol, + string target, + int argumentCount) + { + output.AppendLine($"global {symbol}"); + output.AppendLine($"{symbol}:"); + output.AppendLine(" push rbp"); + output.AppendLine(" mov rbp, rsp"); + foreach (string register in SavedRegisters) + output.AppendLine($" push {register}"); + if (argumentCount % 2 == 0) + output.AppendLine(" sub rsp, 8"); + for (int index = 0; index < argumentCount; index++) + output.AppendLine($" push {ArgumentRegisters[index]}"); + output.AppendLine($" call {target}"); + output.AppendLine(" lea rsp, [rbp-40]"); + for (int index = SavedRegisters.Length - 1; index >= 0; index--) + output.AppendLine($" pop {SavedRegisters[index]}"); + output.AppendLine(" pop rbp"); + output.AppendLine(" ret"); + } +} diff --git a/Ubytec/Language/Portable/PortableX64Backend.cs b/Ubytec/Language/Portable/PortableX64Backend.cs index a81ad2b2..e3ee4f18 100644 --- a/Ubytec/Language/Portable/PortableX64Backend.cs +++ b/Ubytec/Language/Portable/PortableX64Backend.cs @@ -205,6 +205,10 @@ private static void EmitFunction( AppendIndented(output, assembly); } + if (scopes.Count != 2) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} ended with {scopes.Count - 2} unterminated portable control scope(s)."); + if (returnType.Type == PrimitiveType.Void) output.AppendLine(" xor eax, eax"); output.AppendLine(frame.EndLabel + ":"); @@ -324,6 +328,9 @@ private static string MemoryOperation( private static string CompileOperandFree(byte opcode, CompilationScopes scopes) { + if (opcode is BLOCK.OP or LOOP.OP or IF.OP or ELSE.OP or END.OP or BREAK.OP or CONTINUE.OP or WHILE.OP) + return CompileStructuredOperandFree(opcode, scopes); + if (!OperandFreeOperations.TryGetValue(opcode, out Type? type) || Activator.CreateInstance(type) is not IOpCode operation) throw new NotSupportedException($"Opcode 0x{opcode:X2} has no operand-free x64 lowering contract."); @@ -331,8 +338,40 @@ private static string CompileOperandFree(byte opcode, CompilationScopes scopes) int depth = scopes.Count; string assembly = operation.Compile(scopes); if (scopes.Count != depth) - throw new NotSupportedException( - $"Opcode 0x{opcode:X2} requires structured scope state that indexed executable v0 does not encode."); + throw new InvalidDataException( + $"Opcode 0x{opcode:X2} unexpectedly changed portable structured scope state."); + return assembly; + } + + private static string CompileStructuredOperandFree(byte opcode, CompilationScopes scopes) + { + if (opcode == END.OP && scopes.Count <= 2) + throw new InvalidDataException("Portable END cannot close the function or module scope."); + + int depth = scopes.Count; + IOpCode operation = opcode switch + { + BLOCK.OP => new BLOCK(), + LOOP.OP => new LOOP(), + IF.OP => new IF(), + ELSE.OP => new ELSE(), + END.OP => new END(), + BREAK.OP => new BREAK(null), + CONTINUE.OP => new CONTINUE(null), + WHILE.OP => new WHILE(), + _ => throw new InvalidDataException($"Opcode 0x{opcode:X2} is not normalized structured control flow.") + }; + + string assembly = operation.Compile(scopes); + int expectedDepth = opcode switch + { + BLOCK.OP or LOOP.OP or IF.OP or WHILE.OP => depth + 1, + END.OP => depth - 1, + _ => depth + }; + if (scopes.Count != expectedDepth) + throw new InvalidDataException( + $"Opcode 0x{opcode:X2} produced an invalid portable scope transition {depth}->{scopes.Count}; expected {expectedDepth}."); return assembly; } diff --git a/Ubytec/Language/Portable/PortableX64ModuleContractAbi.cs b/Ubytec/Language/Portable/PortableX64ModuleContractAbi.cs new file mode 100644 index 00000000..c9e9bb92 --- /dev/null +++ b/Ubytec/Language/Portable/PortableX64ModuleContractAbi.cs @@ -0,0 +1,358 @@ +using System.Globalization; +using System.Text; +using Ubytec.Language.AST; +using static Ubytec.Language.Syntax.TypeSystem.Types; + +namespace Ubytec.Language.Portable; + +/// +/// Projects a verified UBCM ContractManifest into the Linux module-loader ABI. The manifest is not +/// an authority grant: it names import slots and exports. Runtime grants remain supervisor-owned. +/// Every emitted import/export adapter is cross-checked against the verified indexed tables first. +/// +public static class PortableX64ModuleContractAbi +{ + private const string RuntimeModule = "ubytec.runtime"; + private static readonly string[] ArgumentRegisters = ["rdi", "rsi", "rdx", "rcx", "r8", "r9"]; + private static readonly string[] SavedRegisters = ["rbx", "r12", "r13", "r14", "r15"]; + + /// + /// Emits source-contract import adapters, host-facing contract exports and the native + /// .ubytec_contract section exclusively from verified UBCM bytes. + /// + public static PortableX64Compilation Attach( + ReadOnlySpan moduleBytes, + PortableX64Compilation nativeCompilation) + { + ArgumentNullException.ThrowIfNull(nativeCompilation); + PortableProcessEntrypoint.Verify(moduleBytes); + PortableModuleImage image = PortableModuleFormat.Decode(moduleBytes); + PortableModuleSection? manifestSection = image.GetSection(PortableModuleSectionKind.ContractManifest); + if (manifestSection is null) + return nativeCompilation; + + byte[] manifestBytes = manifestSection.Payload.ToArray(); + if (manifestBytes.Length == 0 || manifestBytes.Length > 4096) + throw new InvalidDataException("Portable contract manifest must contain 1..4096 bytes."); + string manifest = StrictAscii(manifestBytes); + ContractModel contract = ParseManifest(manifest); + + PortableBinaryModule module = PortableBinaryCodec.Decode(moduleBytes); + PortableExecutableModule executable = module.Executable + ?? throw new InvalidDataException("Portable module contracts require the indexed executable profile."); + if (nativeCompilation.ImportLabels.Count != executable.Imports.Count || + nativeCompilation.ExportLabels.Count != executable.Exports.Count) + { + throw new InvalidDataException("Native labels do not match the verified UBCM boundary tables."); + } + + string assembly = nativeCompilation.Assembly.ReplaceLineEndings("\n"); + var output = new StringBuilder(); + var externs = new HashSet(StringComparer.Ordinal); + var claimedImports = new HashSet(); + var claimedExports = new HashSet(); + + foreach (ContractImport entry in contract.Imports) + { + int importIndex = FindImport(executable, entry, claimedImports); + PortableExecutableImport import = executable.Imports[importIndex]; + string label = nativeCompilation.GetImportLabel(importIndex); + assembly = RemoveExtern(assembly, label); + string signature = Signature(executable, import); + if (!string.Equals(signature, entry.Signature, StringComparison.Ordinal)) + throw new InvalidDataException($"Contract import '{entry.Alias}' signature disagrees with verified UBCM."); + + if (string.Equals(entry.Signature, ModuleContracts.BufferSignature, StringComparison.Ordinal)) + { + AddExtern(output, externs, "ubytec_bridge_call"); + EmitBufferImport(output, label, entry.Handle); + } + else + { + AddExtern(output, externs, "ubytec_bridge_many"); + EmitTypedImport(output, label, entry.Handle, executable, import); + } + } + + int nonRuntimeImports = Enumerable.Range(0, executable.Imports.Count) + .Count(index => !IsRuntimeImport(executable, executable.Imports[index])); + if (claimedImports.Count != nonRuntimeImports) + throw new InvalidDataException("Contract manifest does not account for every non-runtime UBCM import slot."); + + foreach (ContractExport entry in contract.Exports) + { + int exportIndex = FindExport(executable, entry, claimedExports); + PortableExecutableExport export = executable.Exports[exportIndex]; + string signature = Signature(executable, export); + if (!string.Equals(signature, entry.Signature, StringComparison.Ordinal)) + throw new InvalidDataException($"Contract export '{entry.Name}' signature disagrees with verified UBCM."); + EmitSystemVExport( + output, + entry.NativeSymbol, + nativeCompilation.GetExportLabel(exportIndex), + export.Arguments.Count); + } + + output.AppendLine("section .ubytec_contract alloc noexec nowrite progbits"); + output.Append("db ").AppendLine(string.Join(",", manifestBytes)); + + var combined = new StringBuilder(assembly); + combined.AppendLine("section .text"); + combined.Append(output); + combined.AppendLine("section .note.GNU-stack noalloc noexec nowrite progbits"); + return nativeCompilation with { Assembly = combined.ToString().ReplaceLineEndings("\n") }; + } + + private static int FindImport( + PortableExecutableModule executable, + ContractImport entry, + HashSet claimed) + { + for (int index = 0; index < executable.Imports.Count; index++) + { + if (claimed.Contains(index)) continue; + PortableExecutableImport candidate = executable.Imports[index]; + if (IsRuntimeImport(executable, candidate)) continue; + if (!string.Equals(ResolveString(executable, candidate.ModuleStringIndex), entry.Module, StringComparison.Ordinal) || + !string.Equals(ResolveString(executable, candidate.SymbolStringIndex), entry.Symbol, StringComparison.Ordinal) || + !string.Equals(Signature(executable, candidate), entry.Signature, StringComparison.Ordinal)) + continue; + claimed.Add(index); + return index; + } + throw new InvalidDataException($"Contract import '{entry.Alias}' has no matching verified UBCM ImportIndex."); + } + + private static int FindExport( + PortableExecutableModule executable, + ContractExport entry, + HashSet claimed) + { + for (int index = 0; index < executable.Exports.Count; index++) + { + if (claimed.Contains(index)) continue; + PortableExecutableExport candidate = executable.Exports[index]; + if (!string.Equals(ResolveString(executable, candidate.SymbolStringIndex), entry.Name, StringComparison.Ordinal) || + !string.Equals(Signature(executable, candidate), entry.Signature, StringComparison.Ordinal)) + continue; + claimed.Add(index); + return index; + } + throw new InvalidDataException($"Contract export '{entry.Name}' has no matching verified UBCM ExportIndex."); + } + + private static bool IsRuntimeImport(PortableExecutableModule executable, PortableExecutableImport import) => + string.Equals(ResolveString(executable, import.ModuleStringIndex), RuntimeModule, StringComparison.Ordinal); + + private static string Signature(PortableExecutableModule executable, PortableExecutableImport import) => + string.Join(",", import.Arguments.Select(argument => Parameter(executable, argument))) + + ":" + ResolvePrimitive(executable, import.ReturnTypeIndex); + + private static string Signature(PortableExecutableModule executable, PortableExecutableExport export) => + string.Join(",", export.Arguments.Select(argument => Parameter(executable, argument))) + + ":" + ResolvePrimitive(executable, export.ReturnTypeIndex); + + private static string Parameter(PortableExecutableModule executable, PortableBoundaryDescriptor argument) => + argument.PassingMode switch + { + PortablePassingMode.LoanRead => "LoanRead", + PortablePassingMode.LoanWrite => "LoanWrite", + PortablePassingMode.Value => ResolvePrimitive(executable, argument.TypeIndex).ToString(), + _ => throw new InvalidDataException("Verified UBCM contains an unknown boundary passing mode.") + }; + + private static PrimitiveType ResolvePrimitive(PortableExecutableModule executable, uint typeIndex) + { + if (typeIndex >= executable.Types.Count) + throw new InvalidDataException("Contract TypeIndex is outside the verified type table."); + return executable.Types[checked((int)typeIndex)].Primitive; + } + + private static string ResolveString(PortableExecutableModule executable, uint stringIndex) + { + if (stringIndex >= executable.Strings.Count) + throw new InvalidDataException("Contract StringIndex is outside the verified string table."); + return executable.Strings[checked((int)stringIndex)]; + } + + private static void EmitBufferImport(StringBuilder output, string label, int handle) + { + output.AppendLine($"{label}:"); + output.AppendLine(" push rbp"); + output.AppendLine(" mov rbp, rsp"); + output.AppendLine($" mov rdi, {handle}"); + output.AppendLine(" mov rsi, [rbp+48]"); + output.AppendLine(" mov rdx, [rbp+40]"); + output.AppendLine(" mov rcx, [rbp+32]"); + output.AppendLine(" mov r8, [rbp+24]"); + output.AppendLine(" mov r9, [rbp+16]"); + output.AppendLine(" and rsp, -16"); + output.AppendLine(" call ubytec_bridge_call"); + output.AppendLine(" mov rsp, rbp"); + output.AppendLine(" pop rbp"); + output.AppendLine(" ret"); + } + + private static void EmitTypedImport( + StringBuilder output, + string label, + int handle, + PortableExecutableModule executable, + PortableExecutableImport import) + { + int cells = 2; + foreach (PortableBoundaryDescriptor argument in import.Arguments) + cells = checked(cells + (argument.PassingMode is PortablePassingMode.LoanRead or PortablePassingMode.LoanWrite ? 2 : 1)); + int localBytes = Math.Max(24, import.Arguments.Count * 24); + int cell = 0; + int Offset(int index) => 16 + (cells - 1 - index) * 8; + + output.AppendLine($"{label}:"); + output.AppendLine(" push rbp"); + output.AppendLine(" mov rbp, rsp"); + output.AppendLine($" sub rsp, {localBytes}"); + for (int index = 0; index < import.Arguments.Count; index++) + { + PortableBoundaryDescriptor argument = import.Arguments[index]; + int record = -localBytes + index * 24; + int kind = argument.PassingMode switch + { + PortablePassingMode.LoanWrite => 1, + PortablePassingMode.LoanRead => 2, + PortablePassingMode.Value when ResolvePrimitive(executable, argument.TypeIndex) == PrimitiveType.UInt64 => 3, + PortablePassingMode.Value when ResolvePrimitive(executable, argument.TypeIndex) == PrimitiveType.Int64 => 4, + _ => throw new InvalidDataException("Typed source imports support only UInt64, Int64 and loan carriers.") + }; + output.AppendLine($" mov qword [rbp{Signed(record)}], {kind}"); + output.AppendLine($" mov rax, [rbp+{Offset(cell++)}]"); + output.AppendLine($" mov [rbp{Signed(record + 8)}], rax"); + if (kind <= 2) + { + output.AppendLine($" mov rax, [rbp+{Offset(cell++)}]"); + output.AppendLine($" mov [rbp{Signed(record + 16)}], rax"); + } + else + { + output.AppendLine($" mov qword [rbp{Signed(record + 16)}], 0"); + } + } + output.AppendLine($" mov rdi, {handle}"); + output.AppendLine($" lea rsi, [rbp-{localBytes}]"); + output.AppendLine($" mov rdx, {import.Arguments.Count}"); + output.AppendLine($" mov rcx, [rbp+{Offset(cell++)}]"); + output.AppendLine($" mov r8, [rbp+{Offset(cell)}]"); + output.AppendLine(" and rsp, -16"); + output.AppendLine(" call ubytec_bridge_many"); + output.AppendLine(" mov rsp, rbp"); + output.AppendLine(" pop rbp"); + output.AppendLine(" ret"); + } + + private static string Signed(int offset) => offset < 0 ? offset.ToString(CultureInfo.InvariantCulture) : "+" + offset.ToString(CultureInfo.InvariantCulture); + + private static void EmitSystemVExport( + StringBuilder output, + string symbol, + string target, + int argumentCount) + { + if (argumentCount > ArgumentRegisters.Length) + throw new NotSupportedException($"Contract export '{symbol}' has more than six arguments."); + output.AppendLine($"global {symbol}"); + output.AppendLine($"{symbol}:"); + output.AppendLine(" push rbp"); + output.AppendLine(" mov rbp, rsp"); + foreach (string register in SavedRegisters) + output.AppendLine($" push {register}"); + if (argumentCount % 2 == 0) + output.AppendLine(" sub rsp, 8"); + for (int index = 0; index < argumentCount; index++) + output.AppendLine($" push {ArgumentRegisters[index]}"); + output.AppendLine($" call {target}"); + output.AppendLine(" lea rsp, [rbp-40]"); + for (int index = SavedRegisters.Length - 1; index >= 0; index--) + output.AppendLine($" pop {SavedRegisters[index]}"); + output.AppendLine(" pop rbp"); + output.AppendLine(" ret"); + } + + private static void AddExtern(StringBuilder output, HashSet externs, string symbol) + { + if (externs.Add(symbol)) output.AppendLine($"extern {symbol}"); + } + + private static string RemoveExtern(string assembly, string label) + { + string directive = $"extern {label}\n"; + int position = assembly.IndexOf(directive, StringComparison.Ordinal); + if (position < 0) + throw new InvalidDataException($"Contract ImportIndex label '{label}' is not declared by the x64 backend."); + if (assembly.IndexOf(directive, position + directive.Length, StringComparison.Ordinal) >= 0) + throw new InvalidDataException($"Contract ImportIndex label '{label}' has multiple extern declarations."); + return assembly.Remove(position, directive.Length); + } + + private static ContractModel ParseManifest(string manifest) + { + string[] lines = manifest.Split('\n'); + if (lines.Length < 3 || lines[0] != "UBC1" || lines[^2] != "END" || lines[^1].Length != 0) + throw new InvalidDataException("Portable contract manifest has invalid UBC1 framing."); + + var imports = new List(); + var exports = new List(); + var handles = new HashSet(); + var aliases = new HashSet(StringComparer.Ordinal); + var exportNames = new HashSet(StringComparer.Ordinal); + var nativeSymbols = new HashSet(StringComparer.Ordinal); + + foreach (string line in lines.Skip(1).SkipLast(2)) + { + string[] fields = line.Split('|'); + if (fields.Length == 6 && fields[0] == "I") + { + if (!int.TryParse(fields[1], NumberStyles.None, CultureInfo.InvariantCulture, out int handle) || handle is < 1 or > 16 || + !handles.Add(handle) || !Identifier(fields[2]) || !aliases.Add(fields[2]) || + !Identifier(fields[3]) || !Identifier(fields[4]) || string.IsNullOrEmpty(fields[5])) + throw new InvalidDataException("Portable contract manifest contains an invalid import row."); + imports.Add(new ContractImport(handle, fields[2], fields[3], fields[4], fields[5])); + continue; + } + if (fields.Length == 4 && fields[0] == "E") + { + if (!Identifier(fields[1]) || !exportNames.Add(fields[1]) || + !NativeIdentifier(fields[2]) || !nativeSymbols.Add(fields[2]) || string.IsNullOrEmpty(fields[3])) + throw new InvalidDataException("Portable contract manifest contains an invalid export row."); + exports.Add(new ContractExport(fields[1], fields[2], fields[3])); + continue; + } + throw new InvalidDataException("Portable contract manifest contains an unknown row."); + } + return new ContractModel(imports, exports); + } + + private static bool Identifier(string value) => + value.Length is >= 1 and <= 63 && NativeIdentifier(value); + + private static bool NativeIdentifier(string value) + { + if (string.IsNullOrEmpty(value) || !(AsciiLetter(value[0]) || value[0] == '_')) return false; + for (int index = 1; index < value.Length; index++) + if (!(AsciiLetter(value[index]) || char.IsAsciiDigit(value[index]) || value[index] == '_')) return false; + return true; + } + + private static bool AsciiLetter(char value) => value is >= 'A' and <= 'Z' or >= 'a' and <= 'z'; + + private static string StrictAscii(byte[] bytes) + { + if (bytes.Any(value => value is > 0x7F or 0)) + throw new InvalidDataException("Portable contract manifest must be non-NUL ASCII."); + return Encoding.ASCII.GetString(bytes); + } + + private sealed record ContractModel( + IReadOnlyList Imports, + IReadOnlyList Exports); + private sealed record ContractImport(int Handle, string Alias, string Module, string Symbol, string Signature); + private sealed record ContractExport(string Name, string NativeSymbol, string Signature); +} diff --git a/Ubytec/Language/Portable/PortableX64RuntimeImports.cs b/Ubytec/Language/Portable/PortableX64RuntimeImports.cs new file mode 100644 index 00000000..05f6c94d --- /dev/null +++ b/Ubytec/Language/Portable/PortableX64RuntimeImports.cs @@ -0,0 +1,212 @@ +using System.Text; +using static Ubytec.Language.Syntax.TypeSystem.Types; + +namespace Ubytec.Language.Portable; + +/// +/// Reserved supervisor/runtime imports used by language features such as ModuleInvoke and host I/O. +/// The source aliases are lowered into ordinary indexed UBCM imports; x64 binding happens only after +/// the portable module has crossed the independent verifier boundary. +/// +public static class PortableX64RuntimeImports +{ + private const string RuntimeModule = "ubytec.runtime"; + private const string ModuleInvokeSymbol = "module.invoke"; + private const string HostReadSymbol = "host.read"; + private const string HostWriteSymbol = "host.write"; + + /// Adds the reserved runtime import contracts required by enabled language features. + public static PortableModuleContracts AddContracts( + PortableModuleContracts contracts, + bool enableModuleCalls, + bool enableHostIO) + { + ArgumentNullException.ThrowIfNull(contracts); + if (!enableModuleCalls) + return contracts; + + var imports = contracts.Imports.ToList(); + // ModuleInvoke's internal stack ABI is: + // module_handle, buffer, length, rights, budget_ms, result_address + // Indexed import calls already carry budget_ms + result_address as the two boundary-control + // cells, so only the first four logical arguments belong in the portable import signature. + // The handle is UInt32 to distinguish this runtime operation from the legacy four-UInt64 + // buffer contract, whose v0 compatibility calling convention consumes five cells. + Add(imports, new PortableImportContract( + "ModuleInvoke", + RuntimeModule, + ModuleInvokeSymbol, + PrimitiveType.Int64, + [ + new PortableBoundaryParameter(PortablePassingMode.Value, PrimitiveType.UInt32), + new PortableBoundaryParameter(PortablePassingMode.Value, PrimitiveType.UInt64), + new PortableBoundaryParameter(PortablePassingMode.Value, PrimitiveType.UInt64), + new PortableBoundaryParameter(PortablePassingMode.Value, PrimitiveType.UInt64) + ])); + + if (enableHostIO) + { + // HostRead/HostWrite use the established v0 buffer-call convention: + // resource_handle, byte_offset, local_buffer, logical_length, result_address + // The first four cells are the portable boundary arguments and the final result cell is + // the v0 implicit return-address cell, so the verified CALL consumes five cells. + Add(imports, new PortableImportContract( + "HostRead", + RuntimeModule, + HostReadSymbol, + PrimitiveType.Int64, + ValueCells(4))); + Add(imports, new PortableImportContract( + "HostWrite", + RuntimeModule, + HostWriteSymbol, + PrimitiveType.Int64, + ValueCells(4))); + } + + return contracts with { Imports = imports }; + } + + /// + /// Resolves reserved runtime ImportIndex labels to the native supervisor bridge without exposing + /// source aliases to the index-only backend. Non-runtime imports remain unresolved for the loader. + /// + public static PortableX64Compilation Attach( + ReadOnlySpan moduleBytes, + PortableX64Compilation nativeCompilation) + { + ArgumentNullException.ThrowIfNull(nativeCompilation); + PortableProcessEntrypoint.Verify(moduleBytes); + PortableBinaryModule module = PortableBinaryCodec.Decode(moduleBytes); + PortableExecutableModule executable = module.Executable + ?? throw new InvalidDataException("Runtime import binding requires the indexed executable profile."); + if (nativeCompilation.ImportLabels.Count != executable.Imports.Count) + throw new InvalidDataException("Native import labels do not match the verified UBCM import table."); + + string assembly = nativeCompilation.Assembly.ReplaceLineEndings("\n"); + var adapters = new StringBuilder(); + var externs = new HashSet(StringComparer.Ordinal); + + for (int index = 0; index < executable.Imports.Count; index++) + { + PortableExecutableImport import = executable.Imports[index]; + string provider = ResolveString(executable, import.ModuleStringIndex, "module"); + if (!string.Equals(provider, RuntimeModule, StringComparison.Ordinal)) + continue; + + string symbol = ResolveString(executable, import.SymbolStringIndex, "symbol"); + string bridge = symbol switch + { + ModuleInvokeSymbol => "ubytec_bridge_call", + HostReadSymbol => "ubytec_bridge_read", + HostWriteSymbol => "ubytec_bridge_write", + _ => throw new InvalidDataException($"Unknown reserved Ubytec runtime import '{symbol}'.") + }; + int stackCells = symbol == ModuleInvokeSymbol ? 6 : 5; + ValidateSignature(executable, import, symbol); + + string label = nativeCompilation.GetImportLabel(index); + assembly = RemoveExtern(assembly, label); + if (externs.Add(bridge)) + adapters.AppendLine($"extern {bridge}"); + EmitAdapter(adapters, label, bridge, stackCells); + } + + if (adapters.Length == 0) + return nativeCompilation; + + var output = new StringBuilder(assembly); + output.AppendLine("section .text"); + output.Append(adapters); + output.AppendLine("section .note.GNU-stack noalloc noexec nowrite progbits"); + return nativeCompilation with { Assembly = output.ToString().ReplaceLineEndings("\n") }; + } + + private static void Add(List imports, PortableImportContract contract) + { + if (imports.Any(import => string.Equals(import.SourceAlias, contract.SourceAlias, StringComparison.Ordinal))) + throw new InvalidDataException($"Portable runtime import alias '{contract.SourceAlias}' is already defined."); + imports.Add(contract); + } + + private static PortableBoundaryParameter[] ValueCells(int count) => + Enumerable.Range(0, count) + .Select(_ => new PortableBoundaryParameter(PortablePassingMode.Value, PrimitiveType.UInt64)) + .ToArray(); + + private static string ResolveString(PortableExecutableModule executable, uint index, string field) + { + if (index >= executable.Strings.Count) + throw new InvalidDataException($"Runtime import {field} StringIndex is outside the verified string table."); + return executable.Strings[checked((int)index)]; + } + + private static void ValidateSignature( + PortableExecutableModule executable, + PortableExecutableImport import, + string symbol) + { + if (ResolvePrimitive(executable, import.ReturnTypeIndex) != PrimitiveType.Int64) + throw new InvalidDataException($"Reserved runtime import '{symbol}' must return Int64."); + + PrimitiveType[] expected = symbol switch + { + ModuleInvokeSymbol => [PrimitiveType.UInt32, PrimitiveType.UInt64, PrimitiveType.UInt64, PrimitiveType.UInt64], + HostReadSymbol or HostWriteSymbol => + [PrimitiveType.UInt64, PrimitiveType.UInt64, PrimitiveType.UInt64, PrimitiveType.UInt64], + _ => throw new InvalidDataException($"Unknown reserved Ubytec runtime import '{symbol}'.") + }; + + if (import.Arguments.Count != expected.Length) + throw new InvalidDataException($"Reserved runtime import '{symbol}' has an invalid verified argument count."); + for (int index = 0; index < expected.Length; index++) + { + PortableBoundaryDescriptor argument = import.Arguments[index]; + if (argument.PassingMode != PortablePassingMode.Value || + ResolvePrimitive(executable, argument.TypeIndex) != expected[index]) + { + throw new InvalidDataException($"Reserved runtime import '{symbol}' has an invalid verified signature."); + } + } + } + + private static PrimitiveType ResolvePrimitive(PortableExecutableModule executable, uint typeIndex) + { + if (typeIndex >= executable.Types.Count) + throw new InvalidDataException("Runtime import TypeIndex is outside the verified type table."); + return executable.Types[checked((int)typeIndex)].Primitive; + } + + private static string RemoveExtern(string assembly, string label) + { + string directive = $"extern {label}\n"; + int position = assembly.IndexOf(directive, StringComparison.Ordinal); + if (position < 0) + throw new InvalidDataException($"Runtime ImportIndex label '{label}' is not declared by the x64 backend."); + if (assembly.IndexOf(directive, position + directive.Length, StringComparison.Ordinal) >= 0) + throw new InvalidDataException($"Runtime ImportIndex label '{label}' has multiple extern declarations."); + return assembly.Remove(position, directive.Length); + } + + private static void EmitAdapter( + StringBuilder output, + string label, + string bridge, + int argumentCount) + { + output.AppendLine($"{label}:"); + output.AppendLine(" push rbp"); + output.AppendLine(" mov rbp, rsp"); + string[] registers = ["rdi", "rsi", "rdx", "rcx", "r8", "r9"]; + for (int index = 0; index < argumentCount; index++) + { + int offset = 16 + (argumentCount - 1 - index) * 8; + output.AppendLine($" mov {registers[index]}, [rbp+{offset}]"); + } + output.AppendLine(" and rsp, -16"); + output.AppendLine($" call {bridge}"); + output.AppendLine(" mov rsp, rbp"); + output.AppendLine(" pop rbp"); + output.AppendLine(" ret"); + } +} diff --git a/Ubytec/Language/Portable/PortableX64TailCallOptimizer.cs b/Ubytec/Language/Portable/PortableX64TailCallOptimizer.cs new file mode 100644 index 00000000..40f75cf8 --- /dev/null +++ b/Ubytec/Language/Portable/PortableX64TailCallOptimizer.cs @@ -0,0 +1,143 @@ +using System.Text; +using Ubytec.Language.Operations; +using Ubytec.Language.Syntax.TypeSystem; +using static Ubytec.Language.Syntax.TypeSystem.Types; + +namespace Ubytec.Language.Portable; + +/// +/// Explicit x64 tail-call optimization over the verified indexed UBCM instruction stream. +/// The pass never consults source names or AST state: tail position, target identity and +/// signature compatibility are derived exclusively from the verified portable module. +/// +public static class PortableX64TailCallOptimizer +{ + /// + /// Rewrites compatible local CALL + RETURN pairs to frame-reusing jumps. + /// This changes physical frame lifetime and is therefore opt-in rather than a default optimization. + /// + public static PortableX64Compilation Apply( + ReadOnlySpan moduleBytes, + PortableX64Compilation nativeCompilation) + { + ArgumentNullException.ThrowIfNull(nativeCompilation); + PortableProcessEntrypoint.Verify(moduleBytes); + PortableBinaryModule module = PortableBinaryCodec.Decode(moduleBytes); + PortableExecutableModule executable = module.Executable + ?? throw new InvalidDataException("Tail-call optimization requires the indexed executable profile."); + + if (nativeCompilation.FunctionLabels.Count != executable.Functions.Count) + throw new InvalidDataException("Native function labels do not match the verified UBCM function table."); + + string assembly = nativeCompilation.Assembly.ReplaceLineEndings("\n"); + for (int callerIndex = 0; callerIndex < executable.Functions.Count; callerIndex++) + { + PortableExecutableFunction caller = executable.Functions[callerIndex]; + IReadOnlyList instructions = caller.Instructions; + for (int instructionIndex = 0; instructionIndex + 1 < instructions.Count; instructionIndex++) + { + if (instructions[instructionIndex] is not PortableCallInstruction call || + instructions[instructionIndex + 1] is not PortableReturnInstruction ret || + call.TargetKind != PortableCallTargetKind.LocalFunction) + { + continue; + } + + if (call.TargetIndex >= executable.Functions.Count) + throw new InvalidDataException("Tail CALL FunctionIndex is outside the verified function table."); + + PortableExecutableFunction target = executable.Functions[checked((int)call.TargetIndex)]; + if (!Compatible(caller, target)) + continue; + + if (ret.TypeIndex is uint explicitReturn && explicitReturn != caller.ReturnTypeIndex) + throw new InvalidDataException("Tail RETURN TypeIndex differs from the enclosing function result type."); + + string callerLabel = nativeCompilation.GetFunctionLabel(callerIndex); + string targetLabel = nativeCompilation.GetFunctionLabel(checked((int)call.TargetIndex)); + UType returnType = ResolveType(executable, caller.ReturnTypeIndex); + string ordinary = EmitIndented(NormalCall(targetLabel, target, executable)) + + EmitIndented(NormalReturn(callerLabel, returnType)); + int position = assembly.IndexOf(ordinary, StringComparison.Ordinal); + if (position < 0) + { + throw new InvalidDataException( + $"Verified tail pair FunctionIndex {callerIndex}->{call.TargetIndex} does not match the x64 backend output."); + } + + string tail = EmitIndented(TailCall(targetLabel, target.ArgumentTypeIndices.Count)); + assembly = string.Concat( + assembly.AsSpan(0, position), + tail, + assembly.AsSpan(position + ordinary.Length)); + instructionIndex++; + } + } + + return nativeCompilation with { Assembly = assembly }; + } + + private static bool Compatible( + PortableExecutableFunction caller, + PortableExecutableFunction target) => + caller.ArgumentTypeIndices.Count == target.ArgumentTypeIndices.Count && + caller.ReturnTypeIndex == target.ReturnTypeIndex; + + private static string NormalCall( + string targetLabel, + PortableExecutableFunction target, + PortableExecutableModule executable) + { + string output = $"call {targetLabel}"; + if (target.ArgumentTypeIndices.Count != 0) + output += $"\n add rsp, {target.ArgumentTypeIndices.Count * 8}"; + UType returnType = ResolveType(executable, target.ReturnTypeIndex); + if (returnType.Type != PrimitiveType.Void) + output += "\n push rax"; + return output; + } + + private static string NormalReturn(string callerLabel, UType returnType) => + (returnType.Type == PrimitiveType.Void + ? "xor eax, eax" + : "pop rax\n " + FunctionCode.Canonicalize(returnType)) + + $"\n jmp {callerLabel}__end"; + + private static string TailCall(string targetLabel, int argumentCount) + { + var output = new StringBuilder("; tail call: reuse incoming argument cells\n"); + for (int index = 0; index < argumentCount; index++) + output.AppendLine($"pop rax\n mov qword [rbp + {16 + index * 8}], rax"); + output.AppendLine("mov rsp, rbp\n pop rbp"); + output.AppendLine($"jmp {targetLabel}"); + return output.ToString(); + } + + private static string EmitIndented(string assembly) + { + var output = new StringBuilder(); + foreach (string line in assembly.ReplaceLineEndings("\n").Split('\n')) + if (line.Length != 0) + output.Append(" ").AppendLine(line); + return output.ToString(); + } + + private static UType ResolveType(PortableExecutableModule executable, uint typeIndex) + { + if (typeIndex >= executable.Types.Count) + throw new InvalidDataException("Tail-call TypeIndex is outside the verified type table."); + PortableExecutableType type = executable.Types[checked((int)typeIndex)]; + string? name = null; + if (type.NameStringIndex is uint nameIndex) + { + if (nameIndex >= executable.Strings.Count) + throw new InvalidDataException("Tail-call type name StringIndex is outside the verified string table."); + name = executable.Strings[checked((int)nameIndex)]; + } + return new UType( + type.Primitive, + type.Modifiers, + name: name, + fixedArraySize: checked((nuint)type.FixedArraySize)); + } +} diff --git a/Ubytec/Program.cs b/Ubytec/Program.cs index 0575fca1..fdc57180 100644 --- a/Ubytec/Program.cs +++ b/Ubytec/Program.cs @@ -1,3 +1,6 @@ +[assembly: System.CLSCompliant(true)] +[assembly: System.Runtime.InteropServices.ComVisible(true)] + namespace Ubytec; internal static class Program diff --git a/prototypes/isolation/run_ubytec.py b/prototypes/isolation/run_ubytec.py index 4972bd96..8de29afc 100644 --- a/prototypes/isolation/run_ubytec.py +++ b/prototypes/isolation/run_ubytec.py @@ -66,7 +66,7 @@ def main(): "-p:GenerateDocumentationFile=false"], timeout=120) dll = project.parent/"bin/Release/net9.0/Ubytec.dll" dll = dll.resolve() - compiler_args = ["dotnet", dll, ROOT/"ubytec/attacks.ubc", "--library", "--tail-calls", "-o", assembly] + compiler_args = ["dotnet", dll, ROOT/"ubytec/attacks.ubc", "--library", "-o", assembly] for function, symbol in EXPORTS.items(): compiler_args.extend(["--host-export", f"{function}=ubytec_host_{symbol}"]) command(compiler_args)