Skip to content

Make verified portable x64 the canonical CLI path #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Papishushi wants to merge 1 commit into from
Draft

Make verified portable x64 the canonical CLI path #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 145 additions & 12 deletions Ubytec.Tests/PortableCliTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,20 @@ push 42
}
""";

private const string LibrarySource = """
export Echo as portable_echo
module (name:"portable-library", version:"0.1", author:"Ubytec")
{
func Echo(t_int64 value) -> t_int64
{
load @value
return
}
}
""";

[Fact]
public void PortableX64CliWritesTheExactVerifiedModuleUsedForNativeCodegen()
public void DefaultCliWritesTheExactVerifiedModuleUsedForNativeCodegen()
{
using var fixture = new TempFixture();
string source = fixture.Write("program.ubc", Source);
Expand All @@ -34,7 +46,6 @@ public void PortableX64CliWritesTheExactVerifiedModuleUsedForNativeCodegen()
[
source,
"-o", nasm,
"--portable-x64",
"--portable-module", module,
"--no-metadata"
]);
Expand All @@ -57,7 +68,75 @@ public void PortableX64CliWritesTheExactVerifiedModuleUsedForNativeCodegen()
}

[Fact]
public void PortableX64CliDoesNotRequirePersistingTheIntermediateModule()
public void DefaultPortableLibraryBindsExportsFromTheVerifiedModule()
{
using var fixture = new TempFixture();
string source = fixture.Write("library.ubc", LibrarySource);
string nasm = fixture.PathOf("library.nasm");
string module = fixture.PathOf("library.ubcm");

int status = CommandLineDriver.Run(
[
source,
"--library",
"-o", nasm,
"--portable-module", module,
"--no-metadata"
]);

Assert.Equal(0, status);
byte[] bytes = File.ReadAllBytes(module);
PortableProcessEntrypoint.Verify(bytes);
PortableBinaryModule decoded = PortableBinaryCodec.Decode(bytes);
PortableExecutableModule executable = Assert.IsType<PortableExecutableModule>(decoded.Executable);
PortableExecutableExport export = Assert.Single(executable.Exports);
Assert.Equal("portable_echo", executable.Strings[checked((int)export.SymbolStringIndex)]);
Assert.Equal(0u, export.FunctionIndex);

string emitted = File.ReadAllText(nasm).ReplaceLineEndings("\n");
Assert.Contains("global ubytec_binary_export_0", emitted);
Assert.Contains("global portable_echo", emitted);
Assert.Contains("portable_echo:\n", emitted);
Assert.Contains("call ubytec_binary_export_0", emitted);
Assert.DoesNotContain("call Echo", emitted);
}

[Fact]
public void PortableX64FlagRemainsACompatibilityAliasForTheDefaultPipeline()
{
using var fixture = new TempFixture();
string source = fixture.Write("alias.ubc", Source);
string defaultNasm = fixture.PathOf("default.nasm");
string defaultModule = fixture.PathOf("default.ubcm");
string explicitNasm = fixture.PathOf("explicit.nasm");
string explicitModule = fixture.PathOf("explicit.ubcm");

int defaultStatus = CommandLineDriver.Run(
[
source,
"-o", defaultNasm,
"--portable-module", defaultModule,
"--no-metadata"
]);
int explicitStatus = CommandLineDriver.Run(
[
source,
"-o", explicitNasm,
"--portable-x64",
"--portable-module", explicitModule,
"--no-metadata"
]);

Assert.Equal(0, defaultStatus);
Assert.Equal(0, explicitStatus);
Assert.Equal(File.ReadAllBytes(defaultModule), File.ReadAllBytes(explicitModule));
Assert.Equal(
File.ReadAllText(defaultNasm).ReplaceLineEndings("\n"),
File.ReadAllText(explicitNasm).ReplaceLineEndings("\n"));
}

[Fact]
public void DefaultCliDoesNotRequirePersistingTheIntermediateModule()
{
using var fixture = new TempFixture();
string source = fixture.Write("memory-only.ubc", Source);
Expand All @@ -67,7 +146,6 @@ public void PortableX64CliDoesNotRequirePersistingTheIntermediateModule()
[
source,
"-o", nasm,
"--portable-x64",
"--no-metadata"
]);

Expand All @@ -78,7 +156,7 @@ public void PortableX64CliDoesNotRequirePersistingTheIntermediateModule()
}

[Fact]
public void PortableX64CliRejectsLegacyBackendOptimizationFlags()
public void DefaultCliRejectsLegacyBackendOptimizationFlags()
{
using var fixture = new TempFixture();
string source = fixture.Write("reject.ubc", Source);
Expand All @@ -88,8 +166,7 @@ public void PortableX64CliRejectsLegacyBackendOptimizationFlags()
[
source,
"-o", nasm,
"--portable-x64",
"--tail-calls",
"--stack-transfers",
"--no-metadata"
]);

Expand All @@ -98,28 +175,41 @@ public void PortableX64CliRejectsLegacyBackendOptimizationFlags()
}

[Fact]
public void PortableX64CliRejectsLegacyHostExportAdapters()
public void DefaultCliLowersHostExportThroughVerifiedPortableModule()
{
using var fixture = new TempFixture();
string source = fixture.Write("host-export.ubc", Source);
string nasm = fixture.PathOf("host-export.nasm");
string module = fixture.PathOf("host-export.ubcm");

int status = CommandLineDriver.Run(
[
source,
"-o", nasm,
"--portable-x64",
"--library",
"--host-export", "Main=ubytec_host_main",
"--portable-module", module,
"--no-metadata"
]);

Assert.Equal(1, status);
Assert.False(File.Exists(nasm));
Assert.Equal(0, status);
byte[] bytes = File.ReadAllBytes(module);
PortableProcessEntrypoint.Verify(bytes);
PortableExecutableModule executable = Assert.IsType<PortableExecutableModule>(
PortableBinaryCodec.Decode(bytes).Executable);
PortableExecutableExport export = Assert.Single(executable.Exports);
Assert.Equal("ubytec_host_main", executable.Strings[checked((int)export.SymbolStringIndex)]);
Assert.Equal(1u, export.FunctionIndex);

string emitted = File.ReadAllText(nasm).ReplaceLineEndings("\n");
Assert.Contains("global ubytec_host_main", emitted);
Assert.Contains("ubytec_host_main:\n", emitted);
Assert.Contains("call ubytec_binary_export_0", emitted);
Assert.DoesNotContain("call Main", emitted);
}

[Fact]
public void LegacyCliPathRemainsAvailableDuringMigration()
public void LegacySourceX64CliPathRemainsAvailableDuringMigration()
{
using var fixture = new TempFixture();
string source = fixture.Write("legacy.ubc", Source);
Expand All @@ -129,6 +219,7 @@ public void LegacyCliPathRemainsAvailableDuringMigration()
[
source,
"-o", nasm,
"--legacy-source-x64",
"--no-metadata"
]);

Expand All @@ -138,6 +229,48 @@ public void LegacyCliPathRemainsAvailableDuringMigration()
Assert.DoesNotContain("ubytec_binary_f_", emitted);
}

[Fact]
public void LegacySourceX64ExplicitlyUnlocksLegacyBackendOptions()
{
using var fixture = new TempFixture();
string source = fixture.Write("legacy-options.ubc", Source);
string nasm = fixture.PathOf("legacy-options.nasm");

int status = CommandLineDriver.Run(
[
source,
"-o", nasm,
"--legacy-source-x64",
"--stack-transfers",
"--no-metadata"
]);

Assert.Equal(0, status);
Assert.True(File.Exists(nasm));
}

[Theory]
[InlineData("--portable-x64", "--legacy-source-x64")]
[InlineData("--legacy-source-x64", "--portable-x64")]
public void CanonicalAndLegacyPipelineFlagsCannotBeCombined(string first, string second)
{
using var fixture = new TempFixture();
string source = fixture.Write("conflict.ubc", Source);
string nasm = fixture.PathOf("conflict.nasm");

int status = CommandLineDriver.Run(
[
source,
"-o", nasm,
first,
second,
"--no-metadata"
]);

Assert.Equal(1, status);
Assert.False(File.Exists(nasm));
}

private sealed class TempFixture : IDisposable
{
private readonly string root = System.IO.Path.Combine(
Expand Down
68 changes: 68 additions & 0 deletions Ubytec.Tests/PortableModuleContractAbiTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
using System.Text;
using Ubytec.Language.Portable;
using Xunit;

namespace Ubytec.Tests;

public class PortableModuleContractAbiTests
{
private const string Source = """
import ReadTool from tools.Read (t_uint64,t_uint64,t_uint64,t_uint64) -> t_int64
export Main as Main
module (name:"portable-contract", version:"0.1", author:"Ubytec")
{
func Main() -> t_int64
{
push 0
return
}
}
""";

[Fact]
public void LoaderAliasesAndAdaptersComeFromVerifiedUbcmContractManifest()
{
PortableSourceX64Result compilation = PortableSourceX64Pipeline.Compile(
Source,
processImage: false,
enableModuleCalls: true);

PortableProcessEntrypoint.Verify(compilation.ModuleBytes);
PortableModuleImage image = PortableModuleFormat.Decode(compilation.ModuleBytes);
PortableModuleSection manifest = image.GetRequiredSection(PortableModuleSectionKind.ContractManifest);
string contract = Encoding.ASCII.GetString(manifest.Payload.Span);

Assert.Contains("I|1|ReadTool|tools|Read|UInt64,UInt64,UInt64,UInt64:Int64\n", contract);
Assert.Contains("E|Main|ubytec_host_contract_Main|:Int64\n", contract);

PortableExecutableModule executable = Assert.IsType<PortableExecutableModule>(
PortableBinaryCodec.Decode(compilation.ModuleBytes).Executable);
Assert.Equal(2, executable.Imports.Count);
Assert.Single(executable.Exports);

int sourceImportIndex = Enumerable.Range(0, executable.Imports.Count).Single(index =>
{
PortableExecutableImport import = executable.Imports[index];
return executable.Strings[checked((int)import.ModuleStringIndex)] == "tools" &&
executable.Strings[checked((int)import.SymbolStringIndex)] == "Read";
});
int runtimeImportIndex = Enumerable.Range(0, executable.Imports.Count).Single(index =>
{
PortableExecutableImport import = executable.Imports[index];
return executable.Strings[checked((int)import.ModuleStringIndex)] == "ubytec.runtime" &&
executable.Strings[checked((int)import.SymbolStringIndex)] == "module.invoke";
});

string sourceImportLabel = compilation.NativeCompilation.GetImportLabel(sourceImportIndex);
string runtimeImportLabel = compilation.NativeCompilation.GetImportLabel(runtimeImportIndex);
string assembly = compilation.NativeCompilation.Assembly.ReplaceLineEndings("\n");
Assert.Contains(sourceImportLabel + ":\n", assembly);
Assert.Contains(runtimeImportLabel + ":\n", assembly);
Assert.DoesNotContain("extern " + sourceImportLabel + "\n", assembly);
Assert.DoesNotContain("extern " + runtimeImportLabel + "\n", assembly);
Assert.Contains("extern ubytec_bridge_call\n", assembly);
Assert.Contains("global ubytec_host_contract_Main\n", assembly);
Assert.Contains("call ubytec_binary_export_0\n", assembly);
Assert.Contains("section .ubytec_contract alloc noexec nowrite progbits\n", assembly);
}
}
91 changes: 91 additions & 0 deletions Ubytec.Tests/PortableStructuredControlTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
using Ubytec.Language.AST;
using Ubytec.Language.Portable;
using Xunit;
using static Ubytec.Language.Operations.CoreOperations;

namespace Ubytec.Tests;

public class PortableStructuredControlTests
{
private const string Source = """
export Main as Execute (t_int64) -> t_int64
module (name:"portable-control", version:"0.1", author:"Ubytec")
{
func Main(t_int64 value) -> t_int64
{
local
{
t_int64 total 0
}

while @value > 0
load @total
load @value
add
store @total
load @value
dec
store @value
end

load @total
push 3
gt
if
load @total
push 10
add
else
load @total
push 20
add
end
return
}
}
""";

[Theory]
[InlineData(3L, 16L)]
[InlineData(1L, 21L)]
public void StructuredSourceCrossesVerifiedUbcmAndMatchesLegacySemantics(long value, long expected)
{
SourceCompilation source = SourceCompiler.Compile(Source, emitProcessEntryPoint: false);
byte[] bytes = PortableModuleFormat.Encode(
source.Module,
contractManifest: null,
includeIndexedExecutable: true,
portableContracts: source.PortableContracts);

PortableBinaryVerifier.Verify(bytes);
PortableBinaryModule binary = PortableBinaryCodec.Decode(bytes);
PortableExecutableExport export = Assert.Single(binary.Executable!.Exports);
PortableExecutableFunction function = binary.Executable.Functions[checked((int)export.FunctionIndex)];
byte[] structured = function.Instructions
.OfType<PortableOperandFreeInstruction>()
.Select(instruction => instruction.OpCode)
.ToArray();

Assert.Contains(LOOP.OP, structured);
Assert.Contains(IF.OP, structured);
Assert.Contains(BREAK.OP, structured);
Assert.Contains(ELSE.OP, structured);
Assert.Contains(END.OP, structured);
Assert.DoesNotContain(WHILE.OP, structured);

PortableX64Compilation portable = PortableX64Backend.Compile(bytes);
string portableLabel = portable.GetFunctionLabel(checked((int)export.FunctionIndex));
long sourceResult = RunOneArgument(source.Assembly, source.GetFunctionLabel("Main"), value);
long portableResult = RunOneArgument(portable.Assembly, portableLabel, value);

Assert.Equal(expected, sourceResult);
Assert.Equal(sourceResult, portableResult);
}

private static long RunOneArgument(string assembly, string label, long argument)
{
string body = $"mov rax, 0x{unchecked((ulong)argument):X16}\npush rax\n" +
$"call {label}\nadd rsp, 8\npush rax";
return Assert.Single(NativeStack.Run(body, 1, assembly));
}
}
Loading
Loading