From 57dd1eedd1c1095a0316829b18bfaf0d4a7218e5 Mon Sep 17 00:00:00 2001 From: Daniel Molinero Lucas Date: Thu, 15 Oct 2026 23:42:29 +0200 Subject: [PATCH] Rebase semantic verifier onto semantic frontend --- ...HighLevelParserDeclarationDispatchTests.cs | 211 ++++++++++ Ubytec.Tests/LexicalParserContractTests.cs | 80 ++++ .../PortableBinaryVerifierFrameAttackTests.cs | 73 ++++ Ubytec.Tests/PortableBinaryVerifierTests.cs | 270 +++++++++++- Ubytec/Language/AST/HighLevelParser.cs | 392 ++++++++++++------ .../Language/Grammar/ubytec.tmLanguage.json | 154 ++++--- .../Portable/IndexedExecutableFormat.cs | 54 ++- .../Portable/PortableBinaryVerifier.cs | 331 +++++++++++++-- 8 files changed, 1306 insertions(+), 259 deletions(-) create mode 100644 Ubytec.Tests/HighLevelParserDeclarationDispatchTests.cs create mode 100644 Ubytec.Tests/LexicalParserContractTests.cs create mode 100644 Ubytec.Tests/PortableBinaryVerifierFrameAttackTests.cs diff --git a/Ubytec.Tests/HighLevelParserDeclarationDispatchTests.cs b/Ubytec.Tests/HighLevelParserDeclarationDispatchTests.cs new file mode 100644 index 00000000..2c6c9e97 --- /dev/null +++ b/Ubytec.Tests/HighLevelParserDeclarationDispatchTests.cs @@ -0,0 +1,211 @@ +using Ubytec.Language.AST; +using Xunit; +using static Ubytec.Language.Syntax.TypeSystem.Types; + +namespace Ubytec.Tests; + +public class HighLevelParserDeclarationDispatchTests +{ + [Fact] + public void FuncAndActionDeclarationsParseAcrossScopes() + { + const string source = """ +module (name:"parser-entry", version:"0.1", author:"Ubytec") +{ + func ModuleFunc() + { + return + } + + action ModuleAction() + { + return + } + + global + { + func GlobalFunc() + { + return + } + + action GlobalAction() + { + return + } + } + + local + { + func LocalFunc() + { + return + } + + action LocalAction() + { + return + } + } + + class Holder + { + func ClassFunc() + { + return + } + + action ClassAction() + { + return + } + } +} +"""; + + var module = SourceCompiler.CompileSemantic(source, processImage: false).Module; + + Assert.Equal("ModuleFunc", Assert.Single(module.Functions).Name); + Assert.Equal("ModuleAction", Assert.Single(module.Actions).Name); + + Assert.True(module.GlobalContext.HasValue); + Assert.Equal("GlobalFunc", Assert.Single(module.GlobalContext.Value.Functions).Name); + Assert.Equal("GlobalAction", Assert.Single(module.GlobalContext.Value.Actions).Name); + + Assert.True(module.LocalContext.HasValue); + Assert.Equal("LocalFunc", Assert.Single(module.LocalContext.Value.Functions).Name); + Assert.Equal("LocalAction", Assert.Single(module.LocalContext.Value.Actions).Name); + + var holder = Assert.Single(module.Classes); + Assert.Equal("Holder", holder.Name); + Assert.Equal("ClassFunc", Assert.Single(holder.Functions).Name); + Assert.Equal("ClassAction", Assert.Single(holder.Actions).Name); + } + + [Fact] + public void DispatcherConsumesKeywordsAndPreservesModifiers() + { + const string source = """ +module (name:"parser-modifiers", version:"0.1", author:"Ubytec") +{ + public func Visible() + { + return + } + + private action Hidden() + { + return + } + + public class Holder + { + protected func Member() + { + return + } + } +} +"""; + + var module = SourceCompiler.CompileSemantic(source, processImage: false).Module; + + var visible = Assert.Single(module.Functions); + Assert.Equal("Visible", visible.Name); + Assert.True(visible.Modifiers.HasFlag(TypeModifiers.Public)); + + var hidden = Assert.Single(module.Actions); + Assert.Equal("Hidden", hidden.Name); + Assert.True(hidden.Modifiers.HasFlag(TypeModifiers.Private)); + + var holder = Assert.Single(module.Classes); + Assert.True(holder.Modifiers.HasFlag(TypeModifiers.Public)); + var member = Assert.Single(holder.Functions); + Assert.True(member.Modifiers.HasFlag(TypeModifiers.Protected)); + } + + [Fact] + public void RecordFunctionUsesDeclarationKeywordRatherThanCallKeyword() + { + const string source = """ +module (name:"parser-record", version:"0.1", author:"Ubytec") +{ + record Packet + { + func Decode() + { + return + } + + action Mark() + { + return + } + } +} +"""; + + var module = SourceCompiler.CompileSemantic(source, processImage: false).Module; + + var record = Assert.Single(module.Records); + Assert.Equal("Packet", record.Name); + Assert.Equal("Decode", Assert.Single(record.Functions).Name); + Assert.Equal("Mark", Assert.Single(record.Actions).Name); + } + + [Fact] + public void KeywordDispatchParsesFieldStructEnumAndNestedTypeDeclarations() + { + const string source = """ +module (name:"parser-declarations", version:"0.1", author:"Ubytec") +{ + field t_uint64 Counter + + struct Pair + { + field t_uint64 Left + field t_uint64 Right + + func Touch() + { + return + } + } + + enum State + { + Idle, + Ready + } + + class Outer + { + class Inner + { + func Run() + { + return + } + } + } +} +"""; + + var module = SourceCompiler.CompileSemantic(source, processImage: false).Module; + + Assert.Equal("Counter", Assert.Single(module.Fields).Name); + + var pair = Assert.Single(module.Structs); + Assert.Equal("Pair", pair.Name); + Assert.Equal(2, pair.Fields.Length); + Assert.Equal("Touch", Assert.Single(pair.Functions).Name); + + var state = Assert.Single(module.Enums); + Assert.Equal("State", state.Name); + + var outer = Assert.Single(module.Classes); + var inner = Assert.Single(outer.Classes); + Assert.Equal("Inner", inner.Name); + Assert.Equal("Run", Assert.Single(inner.Functions).Name); + } +} diff --git a/Ubytec.Tests/LexicalParserContractTests.cs b/Ubytec.Tests/LexicalParserContractTests.cs new file mode 100644 index 00000000..eeba03f6 --- /dev/null +++ b/Ubytec.Tests/LexicalParserContractTests.cs @@ -0,0 +1,80 @@ +using Ubytec.Language.AST; +using Xunit; +using static Ubytec.Language.Syntax.TypeSystem.Types; + +namespace Ubytec.Tests; + +public class LexicalParserContractTests +{ + [Fact] + public void MalformedDeclarationFailsAtFirstRequiredToken() + { + const string source = """ +module (name:"strict-consume", version:"0.1", author:"Ubytec") +{ + func + { + } +} +"""; + + var error = Assert.Throws(() => + SourceCompiler.CompileSemantic(source, processImage: false)); + + Assert.Contains("Expected", error.Message, StringComparison.Ordinal); + Assert.Contains("found '{'", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void FieldNamesAreRecognizedAfterEverySupportedNullableArrayShape() + { + const string source = """ +module (name:"typed-names", version:"0.1", author:"Ubytec") +{ + field t_int32 Scalar + field t_int32? NullableScalar + field [t_int32] Array + field [t_int32]? NullableArray + field [t_int32?] NullableItems + field [t_int32?]? NullableArrayAndItems +} +"""; + + var module = SourceCompiler.CompileSemantic(source, processImage: false).Module; + + Assert.Equal( + ["Scalar", "NullableScalar", "Array", "NullableArray", "NullableItems", "NullableArrayAndItems"], + module.Fields.Select(field => field.Name).ToArray()); + } + + [Fact] + public void GlobalContextAndGlobalModifierRemainDistinctProductions() + { + const string source = """ +module (name:"global-dispatch", version:"0.1", author:"Ubytec") +{ + global func Flagged() + { + return + } + + global + { + func Contextual() + { + return + } + } +} +"""; + + var module = SourceCompiler.CompileSemantic(source, processImage: false).Module; + + var flagged = Assert.Single(module.Functions); + Assert.Equal("Flagged", flagged.Name); + Assert.True((flagged.Modifiers & TypeModifiers.Global) != 0); + + var globalContext = Assert.IsType(module.GlobalContext); + Assert.Equal("Contextual", Assert.Single(globalContext.Functions).Name); + } +} diff --git a/Ubytec.Tests/PortableBinaryVerifierFrameAttackTests.cs b/Ubytec.Tests/PortableBinaryVerifierFrameAttackTests.cs new file mode 100644 index 00000000..9be770f9 --- /dev/null +++ b/Ubytec.Tests/PortableBinaryVerifierFrameAttackTests.cs @@ -0,0 +1,73 @@ +using System.Buffers.Binary; +using Ubytec.Language.AST; +using Ubytec.Language.Portable; +using Xunit; + +namespace Ubytec.Tests; + +public class PortableBinaryVerifierFrameAttackTests +{ + [Fact] + public void IndependentVerifierRejectsVarRetargetedFromLocalToArgument() + { + const string source = """ + module (name:"binary-var-attack", version:"0.1", author:"Ubytec") + { + func Main(t_int64 value) -> t_int64 + { + local + { + t_int64 scratch 0 + } + load @value + return + } + } + """; + + SourceSemanticCompilation compilation = SourceCompiler.CompileSemantic( + source, + processImage: false); + byte[] bytes = PortableModuleFormat.Encode( + compilation.Module, + contractManifest: null, + includeIndexedExecutable: true, + portableContracts: compilation.PortableContracts); + + var (codeOffset, codeLength) = FindSection(bytes, PortableModuleSectionKind.IndexedCode); + Span code = bytes.AsSpan(codeOffset, codeLength); + + Assert.True(code.Length >= 13, "Expected canonical local VAR initialization."); + Assert.Equal(0x10, code[0]); + Assert.Equal(1u, BinaryPrimitives.ReadUInt32LittleEndian(code.Slice(1, 4))); + + // Retarget the canonical local initialization to BindingIndex 0, which is the argument. + // The source frontend never produced this program; only the raw UBCM bytes are malicious. + BinaryPrimitives.WriteUInt32LittleEndian(code.Slice(1, 4), 0); + + InvalidDataException error = Assert.Throws(() => + PortableBinaryVerifier.Verify(bytes)); + + Assert.Contains("VAR cannot initialize an argument", error.Message, StringComparison.Ordinal); + } + + private static (int Offset, int Length) FindSection( + byte[] bytes, + PortableModuleSectionKind target) + { + uint count = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(8, 4)); + int cursor = 12; + for (uint index = 0; index < count; index++) + { + ushort kind = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan(cursor, 2)); + ulong rawLength = BinaryPrimitives.ReadUInt64LittleEndian(bytes.AsSpan(cursor + 4, 8)); + cursor += 12; + int length = checked((int)rawLength); + if (kind == (ushort)target) + return (cursor, length); + cursor += length; + } + + throw new InvalidOperationException($"Section {target} was not found."); + } +} diff --git a/Ubytec.Tests/PortableBinaryVerifierTests.cs b/Ubytec.Tests/PortableBinaryVerifierTests.cs index 3279be1b..c0487260 100644 --- a/Ubytec.Tests/PortableBinaryVerifierTests.cs +++ b/Ubytec.Tests/PortableBinaryVerifierTests.cs @@ -117,12 +117,225 @@ public void IndependentVerifierRejectsReservedCallTargetKind() Assert.Throws(() => PortableBinaryVerifier.Verify(bytes)); } - private static byte[] BuildIndexed() + [Fact] + public void IndependentVerifierRejectsMutatedAluUnderflowWithoutFrontend() + { + const string source = """ + module (name:"binary-underflow", version:"0.1", author:"Ubytec") + { + func Main() -> t_int64 + { + push 1 + dup + add + return + } + } + """; + byte[] bytes = BuildIndexed(source); + Span code = Code(bytes); + int duplicate = FindOpcode(code, StackOperations.DUP.OP); + code[duplicate] = ArithmeticOperations.ADD.OP; + + InvalidDataException error = Assert.Throws(() => + PortableBinaryVerifier.Verify(bytes)); + + Assert.Contains("evaluation stack cell", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void IndependentVerifierUsesLocalFunctionArityForCallStackEffects() + { + const string source = """ + module (name:"binary-local-call", version:"0.1", author:"Ubytec") + { + func Pair(t_int64 left, t_int64 right) -> t_int64 + { + load @left + load @right + add + return + } + + func Main() -> t_int64 + { + push 1 + dup + call Pair + return + } + } + """; + byte[] bytes = BuildIndexed(source); + Span code = Code(bytes); + int duplicate = FindOpcode(code, StackOperations.DUP.OP); + code[duplicate] = CoreOperations.NOP.OP; + + InvalidDataException error = Assert.Throws(() => + PortableBinaryVerifier.Verify(bytes)); + + Assert.Contains("evaluation stack cell", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void IndependentVerifierUsesImportLoanCellCountForCallStackEffects() + { + const string source = """ + import Remote from math.Read (t_uint64) -> t_int64 + module (name:"binary-import-call", version:"0.1", author:"Ubytec") + { + func Main() -> t_int64 + { + push 4096 + push 1000 + push 16384 + call Remote + return + } + } + """; + byte[] bytes = BuildIndexed(source, enableModuleCalls: true); + var (importsOffset, _) = FindSection(bytes, PortableModuleSectionKind.IndexedImportTable); + int firstArgumentMode = importsOffset + 4 + 16; + Assert.Equal((byte)PortablePassingMode.Value, bytes[firstArgumentMode]); + bytes[firstArgumentMode] = (byte)PortablePassingMode.LoanRead; + + InvalidDataException error = Assert.Throws(() => + PortableBinaryVerifier.Verify(bytes)); + + Assert.Contains("evaluation stack cell", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void IndependentVerifierRejectsStoreToReadonlyBindingFromMutatedBytes() + { + const string source = """ + module (name:"binary-readonly", version:"0.1", author:"Ubytec") + { + func Main() -> t_int64 + { + local + { + readonly t_int64 value 1 + } + push 2 + load @value + add + return + } + } + """; + byte[] bytes = BuildIndexed(source); + Span code = Code(bytes); + int load = FindSequence(code, new byte[] { MemoryOperations.LOAD.OP, 2 }); + code[load] = MemoryOperations.STORE.OP; + + InvalidDataException error = Assert.Throws(() => + PortableBinaryVerifier.Verify(bytes)); + + Assert.Contains("readonly/const", error.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void IndependentVerifierRejectsNonVoidReachableFallthroughFromMutatedBytes() + { + const string source = """ + module (name:"binary-fallthrough", version:"0.1", author:"Ubytec") + { + func Main() -> t_int64 + { + push 42 + return + } + } + """; + byte[] bytes = BuildIndexed(source); + Span code = Code(bytes); + int ret = FindSequence(code, new byte[] { CoreOperations.RETURN.OP, 0 }); + code[ret] = CoreOperations.NOP.OP; + code[ret + 1] = CoreOperations.NOP.OP; + + InvalidDataException error = Assert.Throws(() => + PortableBinaryVerifier.Verify(bytes)); + + Assert.Contains("reachable path without RETURN", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void IndependentVerifierRejectsReturnUnderflowAfterSignatureMutation() + { + const string source = """ + module (name:"binary-return", version:"0.1", author:"Ubytec") + { + func Anchor() -> t_int64 + { + push 1 + return + } + + action Stop() + { + return + } + } + """; + byte[] bytes = BuildIndexed(source); + FunctionRow[] functions = ReadFunctionRows(bytes); + Assert.Equal(2, functions.Length); + BinaryPrimitives.WriteUInt32LittleEndian( + bytes.AsSpan(functions[1].Offset, 4), + functions[0].ReturnType); + + InvalidDataException error = Assert.Throws(() => + PortableBinaryVerifier.Verify(bytes)); + + Assert.Contains("evaluation stack cell", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void IndependentVerifierRejectsNonCellBindingTypeFromMutatedFunctionTable() + { + const string source = """ + module (name:"binary-binding-type", version:"0.1", author:"Ubytec") + { + func Main() -> t_int64 + { + local + { + t_int64 value 1 + } + load @value + return + } + + action Marker() + { + return + } + } + """; + byte[] bytes = BuildIndexed(source); + FunctionRow[] functions = ReadFunctionRows(bytes); + Assert.Equal(2, functions.Length); + uint voidType = functions[1].ReturnType; + BinaryPrimitives.WriteUInt32LittleEndian( + bytes.AsSpan(functions[0].BindingTypesOffset, 4), + voidType); + + InvalidDataException error = Assert.Throws(() => + PortableBinaryVerifier.Verify(bytes)); + + Assert.Contains("scalar cell type", error.Message, StringComparison.Ordinal); + } + + private static byte[] BuildIndexed() => BuildIndexed(BoundarySource, enableModuleCalls: true); + + private static byte[] BuildIndexed(string source, bool enableModuleCalls = false) { - SourceCompilation compilation = SourceCompiler.Compile( - BoundarySource, - emitProcessEntryPoint: false, - enableModuleCalls: true); + SourceSemanticCompilation compilation = SourceCompiler.CompileSemantic( + source, + processImage: false, + enableModuleCalls: enableModuleCalls); return PortableModuleFormat.Encode( compilation.Module, contractManifest: null, @@ -130,6 +343,48 @@ private static byte[] BuildIndexed() portableContracts: compilation.PortableContracts); } + private static Span Code(byte[] bytes) + { + var (offset, length) = FindSection(bytes, PortableModuleSectionKind.IndexedCode); + return bytes.AsSpan(offset, length); + } + + private static int FindOpcode(ReadOnlySpan code, byte opcode) + { + int index = code.IndexOf(opcode); + Assert.True(index >= 0, $"Expected opcode 0x{opcode:X2} in portable code."); + return index; + } + + private static int FindSequence(ReadOnlySpan code, ReadOnlySpan sequence) + { + int index = code.IndexOf(sequence); + Assert.True(index >= 0, "Expected instruction byte sequence in portable code."); + return index; + } + + private static FunctionRow[] ReadFunctionRows(byte[] bytes) + { + var (offset, length) = FindSection(bytes, PortableModuleSectionKind.IndexedFunctionTable); + ReadOnlySpan table = bytes.AsSpan(offset, length); + uint count = BinaryPrimitives.ReadUInt32LittleEndian(table[..4]); + var result = new FunctionRow[checked((int)count)]; + int cursor = 4; + for (int index = 0; index < result.Length; index++) + { + int rowOffset = offset + cursor; + uint returnType = BinaryPrimitives.ReadUInt32LittleEndian(table.Slice(cursor, 4)); + uint argumentCount = BinaryPrimitives.ReadUInt32LittleEndian(table.Slice(cursor + 4, 4)); + uint localCount = BinaryPrimitives.ReadUInt32LittleEndian(table.Slice(cursor + 8, 4)); + cursor += 24; + int bindingTypesOffset = offset + cursor; + cursor = checked(cursor + (int)(argumentCount + localCount) * 4); + result[index] = new FunctionRow(rowOffset, returnType, bindingTypesOffset); + } + Assert.Equal(table.Length, cursor); + return result; + } + private static (int Offset, int Length) FindSection( byte[] bytes, PortableModuleSectionKind target) @@ -148,4 +403,9 @@ private static (int Offset, int Length) FindSection( } throw new InvalidOperationException($"Section {target} was not found."); } + + private readonly record struct FunctionRow( + int Offset, + uint ReturnType, + int BindingTypesOffset); } diff --git a/Ubytec/Language/AST/HighLevelParser.cs b/Ubytec/Language/AST/HighLevelParser.cs index e25938f6..018e79af 100644 --- a/Ubytec/Language/AST/HighLevelParser.cs +++ b/Ubytec/Language/AST/HighLevelParser.cs @@ -72,7 +72,7 @@ private static Module ParseModuleInternal(SyntaxToken[] toks, ref int i) { if (toks.Length <= i) throw new FormatException("Module body is missing its closing brace."); /* global { … } (max 1) */ - if (toks[i].Scopes.Helper.IsControl("global")) + if (IsContextBlock(toks, i, "global")) { if (seenGlobal) _errors.Add(new ParseError(toks[i].Line, "global", @@ -83,7 +83,7 @@ private static Module ParseModuleInternal(SyntaxToken[] toks, ref int i) } /* local { … } (max 1) */ - if (toks[i].Scopes.Helper.IsControl("local")) + if (IsContextBlock(toks, i, "local")) { if (seenLocal) _errors.Add(new ParseError(toks[i].Line, "local", @@ -93,21 +93,32 @@ private static Module ParseModuleInternal(SyntaxToken[] toks, ref int i) continue; } - var tok = toks[i]; - if (tok.Source == "func") { i++; funcs.Add(ParseFunc(toks, ref i)); continue; } + if (TryConsumeDeclarationPrefix(toks, ref i, "func", out var funcMods)) + { funcs.Add(ParseFunc(toks, ref i, funcMods)); continue; } - /* regular members (unchanged) */ - if (tok.Scopes.Helper.IsClassLabel) { classes.Add(ParseClass(toks, ref i)); continue; } - if (tok.Scopes.Helper.IsStructLabel) { structs.Add(ParseStruct(toks, ref i)); continue; } - if (tok.Scopes.Helper.IsRecordLabel) { records.Add(ParseRecord(toks, ref i)); continue; } - if (tok.Scopes.Helper.IsInterfaceLabel) { ifaces.Add(ParseInterface(toks, ref i)); continue; } - if (tok.Scopes.Helper.IsEnumLabel) { enums.Add(ParseEnum(toks, ref i)); continue; } + if (TryConsumeDeclarationPrefix(toks, ref i, "action", out var actionMods)) + { actions.Add(ParseAction(toks, ref i, actionMods)); continue; } - if (tok.Scopes.Helper.IsFieldLabel) { fields.Add(ParseField(toks, ref i)); continue; } - if (tok.Scopes.Helper.IsFuncLabel) { funcs.Add(ParseFunc(toks, ref i)); continue; } - if (tok.Scopes.Helper.IsActionLabel) { actions.Add(ParseAction(toks, ref i)); continue; } + if (TryConsumeDeclarationPrefix(toks, ref i, "class", out var classMods)) + { classes.Add(ParseClass(toks, ref i, classMods)); continue; } - if (tok.Scopes.Helper.IsControl("module")) { subMods.Add(ParseModuleInternal(toks, ref i)); continue; } + if (TryConsumeDeclarationPrefix(toks, ref i, "struct", out var structMods)) + { structs.Add(ParseStruct(toks, ref i, structMods)); continue; } + + if (TryConsumeDeclarationPrefix(toks, ref i, "record", out var recordMods, allowTypePrefix: true)) + { records.Add(ParseRecord(toks, ref i, recordMods)); continue; } + + if (TryConsumeDeclarationPrefix(toks, ref i, "interface", out var interfaceMods)) + { ifaces.Add(ParseInterface(toks, ref i, interfaceMods)); continue; } + + if (TryConsumeDeclarationPrefix(toks, ref i, "enum", out var enumMods)) + { enums.Add(ParseEnum(toks, ref i, enumMods)); continue; } + + if (TryConsumeDeclarationPrefix(toks, ref i, "field", out var fieldMods)) + { fields.Add(ParseField(toks, ref i, fieldMods)); continue; } + + if (LookAhead(toks, i, KeywordControl, "module")) + { subMods.Add(ParseModuleInternal(toks, ref i)); continue; } i++; // skip comments / whitespace / unknown } @@ -142,10 +153,18 @@ private static GlobalContext ParseGlobal(SyntaxToken[] t, ref int i) while (!LookAhead(t, i, MetaBlock, "}")) { - if (t[i].Scopes.Helper.IsControl("field")) { fields.Add(ParseField(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsStorageType) { props.Add(ParseProperty(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsControl("func")) { funcs.Add(ParseFunc(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsControl("action")) { actions.Add(ParseAction(t, ref i)); continue; } + if (TryConsumeDeclarationPrefix(t, ref i, "field", out var fieldMods)) + { fields.Add(ParseField(t, ref i, fieldMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "func", out var funcMods)) + { funcs.Add(ParseFunc(t, ref i, funcMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "action", out var actionMods)) + { actions.Add(ParseAction(t, ref i, actionMods)); continue; } + + if (TryBeginTypedDeclaration(t, ref i, out var propertyMods)) + { props.Add(ParseProperty(t, ref i, propertyMods)); continue; } + i++; } Consume(t, ref i, MetaBlock, "}"); @@ -164,9 +183,16 @@ private static LocalContext ParseLocal(SyntaxToken[] t, ref int i) while (!LookAhead(t, i, MetaBlock, "}")) { if (i >= t.Length) throw new FormatException("Local context has no closing brace."); - if (t[i].Scopes.Helper.IsStorageType || t[i].Scopes.Helper.IsModifier) { vars.Add(ParseVariable(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsFuncLabel) { funcs.Add(ParseFunc(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsActionLabel) { acts.Add(ParseAction(t, ref i)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "func", out var funcMods)) + { funcs.Add(ParseFunc(t, ref i, funcMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "action", out var actionMods)) + { acts.Add(ParseAction(t, ref i, actionMods)); continue; } + + if (TryBeginTypedDeclaration(t, ref i, out var variableMods)) + { vars.Add(ParseVariable(t, ref i, variableMods)); continue; } + i++; } Consume(t, ref i, MetaBlock, "}"); @@ -175,13 +201,11 @@ private static LocalContext ParseLocal(SyntaxToken[] t, ref int i) /*────────────────────── SINGLE-LINE DECLS ───────────────────*/ /* ─────────── FIELD ───────────────────────────────────────────── */ - private static Field ParseField(SyntaxToken[] t, ref int i) + // Contract: the caller consumed modifiers + "field"; i points at the field type. + private static Field ParseField(SyntaxToken[] t, ref int i, TypeModifiers mods) { - var mods = ParseModifierFlags(t, ref i); - Consume(t, ref i, KeywordControl, "field"); - _ = ConsumeType(t, ref i, out var ut); - var nameTok = Consume(t, ref i, "entity.name.var.explicit.ubytec"); + var nameTok = Consume(t, ref i, EntityNameField); SkipTrivia(t, ref i); var valTok = i < t.Length && t[i].Scopes.DataSource.Any(s => s.StartsWith("constant.")) ? t[i++] : default; @@ -191,12 +215,12 @@ private static Field ParseField(SyntaxToken[] t, ref int i) } /* ─────────── VARIABLE (local) ───────────────────────────────── */ - private static Variable ParseVariable(SyntaxToken[] t, ref int i) + // Contract: the caller consumed modifiers; i points at the variable type. + private static Variable ParseVariable(SyntaxToken[] t, ref int i, TypeModifiers mods) { - var mods = ParseModifierFlags(t, ref i); _ = ConsumeType(t, ref i, out var ut); - var nameTok = Consume(t, ref i, "entity.name.var.explicit.ubytec"); + var nameTok = Consume(t, ref i, EntityNameVarExplicit); SkipTrivia(t, ref i); var valTok = i < t.Length && t[i].Scopes.DataSource.Any(s => s.StartsWith("constant.")) ? t[i++] : default; @@ -204,14 +228,12 @@ private static Variable ParseVariable(SyntaxToken[] t, ref int i) } /* ─────────── PROPERTY ─────────────────────────────────────── */ - private static Property ParseProperty(SyntaxToken[] t, ref int i) + // Contract: the caller consumed modifiers; i points at the property type. + private static Property ParseProperty(SyntaxToken[] t, ref int i, TypeModifiers mods) { - /* 0. leading modifiers (public, readonly, …) */ - var mods = ParseModifierFlags(t, ref i); - /* 1. type + identifier */ _ = ConsumeType(t, ref i, out var ut); - var nameTok = Consume(t, ref i, "entity.name.var.explicit.ubytec"); + var nameTok = Consume(t, ref i, EntityNameVarExplicit); /* 2. check for an accessor body “{ … }” */ var accessorFuncs = new List(); @@ -224,9 +246,9 @@ private static Property ParseProperty(SyntaxToken[] t, ref int i) { /* only ‘func …’ is allowed inside a property body and the function name must be get / set / init */ - if (t[i].Scopes.Helper.IsFuncLabel) + if (TryConsumeDeclarationPrefix(t, ref i, "func", out var accessorMods)) { - var fn = ParseFunc(t, ref i); + var fn = ParseFunc(t, ref i, accessorMods); var n = fn.Name.ToLowerInvariant(); if (n is not ("get" or "set" or "init")) @@ -262,9 +284,9 @@ and the function name must be get / set / init */ } /* ─────────── FUNC ──────────────────────────────────────────── */ - private static Func ParseFunc(SyntaxToken[] t, ref int i) + // Contract: the caller consumed modifiers + "func"; i points at the function name. + private static Func ParseFunc(SyntaxToken[] t, ref int i, TypeModifiers mods) { - var mods = ParseModifierFlags(t, ref i); var nameTok = Consume(t, ref i, EntityNameTypeFunc); ParseParamList(t, ref i, out var args); @@ -288,11 +310,9 @@ private static Func ParseFunc(SyntaxToken[] t, ref int i) } /* ─────────── ACTION ───────────────────────────────────────── */ - private static Action ParseAction(SyntaxToken[] t, ref int i) + // Contract: the caller consumed modifiers + "action"; i points at the action name. + private static Action ParseAction(SyntaxToken[] t, ref int i, TypeModifiers mods) { - var mods = ParseModifierFlags(t, ref i); - Consume(t, ref i, KeywordControl, "action"); - var nameTok = Consume(t, ref i, EntityNameTypeAction); ParseParamList(t, ref i, out var args); @@ -306,13 +326,9 @@ private static Action ParseAction(SyntaxToken[] t, ref int i) definitionSentence: body); } /*────────────────────── ParseClass ──────────────────────*/ - private static Class ParseClass(SyntaxToken[] t, ref int i) + // Contract: the caller consumed modifiers + "class"; i points at the class name. + private static Class ParseClass(SyntaxToken[] t, ref int i, TypeModifiers mods) { - /* 0. modifiers in front of ‘class’ */ - var mods = ParseModifierFlags(t, ref i); // may advance i - - /* 1. keyword + name */ - Consume(t, ref i, KeywordControl, "class"); var nameTok = Consume(t, ref i, EntityNameTypeClass); /* 2. body “{ … }” */ @@ -332,25 +348,40 @@ private static Class ParseClass(SyntaxToken[] t, ref int i) LocalContext? localCtx = null; GlobalContext? globalCtx = null; - if (LookAhead(t, i, KeywordControl, "local")) + if (IsContextBlock(t, i, "local")) localCtx = ParseLocal(t, ref i); - else if (LookAhead(t, i, KeywordControl, "global")) + else if (IsContextBlock(t, i, "global")) globalCtx = ParseGlobal(t, ref i); /* scan until ‘}’ */ while (!LookAhead(t, i, MetaBlock, "}")) { - if (t[i].Scopes.Helper.IsFieldLabel) { fields.Add(ParseField(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsStorageType) { props.Add(ParseProperty(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsFuncLabel) { funcs.Add(ParseFunc(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsActionLabel) { actions.Add(ParseAction(t, ref i)); continue; } - - /* nested types ------------------------------------------------------ */ - if (t[i].Scopes.Helper.IsClassLabel) { classes.Add(ParseClass(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsStructLabel) { structs.Add(ParseStruct(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsRecordLabel) { records.Add(ParseRecord(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsInterfaceLabel) { ifaces.Add(ParseInterface(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsEnumLabel) { enums.Add(ParseEnum(t, ref i)); continue; } + if (TryConsumeDeclarationPrefix(t, ref i, "field", out var fieldMods)) + { fields.Add(ParseField(t, ref i, fieldMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "func", out var funcMods)) + { funcs.Add(ParseFunc(t, ref i, funcMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "action", out var actionMods)) + { actions.Add(ParseAction(t, ref i, actionMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "class", out var classMods)) + { classes.Add(ParseClass(t, ref i, classMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "struct", out var structMods)) + { structs.Add(ParseStruct(t, ref i, structMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "record", out var recordMods, allowTypePrefix: true)) + { records.Add(ParseRecord(t, ref i, recordMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "interface", out var interfaceMods)) + { ifaces.Add(ParseInterface(t, ref i, interfaceMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "enum", out var enumMods)) + { enums.Add(ParseEnum(t, ref i, enumMods)); continue; } + + if (TryBeginTypedDeclaration(t, ref i, out var propertyMods)) + { props.Add(ParseProperty(t, ref i, propertyMods)); continue; } i++; // comment / whitespace / unknown → skip } @@ -368,20 +399,16 @@ private static Class ParseClass(SyntaxToken[] t, ref int i) modifiers: mods); } /*────────────────────── ParseEnum ──────────────────────*/ - private static Enum ParseEnum(SyntaxToken[] t, ref int i) + // Contract: the caller consumed modifiers + "enum"; i points at the enum name. + private static Enum ParseEnum(SyntaxToken[] t, ref int i, TypeModifiers mods) { - /* 0. leading modifiers (public, readonly, …) */ - var mods = ParseModifierFlags(t, ref i); - - /* 1. keyword “enum” + identifier */ - Consume(t, ref i, KeywordControl, "enum"); var nameTok = Consume(t, ref i, EntityNameTypeEnum); /* 2. optional underlying primitive ( :: t_uint16 ) */ PrimitiveType underlying = PrimitiveType.Byte; // language default if (LookAhead(t, i, PunctuationScope, "::")) { - i++; // eat “::” + Consume(t, ref i, PunctuationScope, "::"); ConsumeType(t, ref i, out var ut); // t_int32 / [t_int32]? underlying = ut.Type; // only primitive part } @@ -396,13 +423,15 @@ private static Enum ParseEnum(SyntaxToken[] t, ref int i) { /* identifier token – grammar labels each as entity.name… */ var idTok = Consume(t, ref i, - s => s.StartsWith("entity.name."), - "enum member"); + s => s.StartsWith("entity.name.")); long value; if (LookAhead(t, i, OperatorAssign, "=")) { - i++; // eat '=' + Consume(t, ref i, OperatorAssign, "="); + SkipTrivia(t, ref i); + if (i >= t.Length) + throw new FormatException($"Enum member '{idTok.Source}' is missing its assigned value."); var valTok = t[i++]; value = ParseEnumConstant(valTok); autoVal = value + 1; // next implicit value @@ -416,11 +445,9 @@ private static Enum ParseEnum(SyntaxToken[] t, ref int i) /* optional trailing comma */ if (LookAhead(t, i, PunctuationComma, ",")) - i++; + Consume(t, ref i, PunctuationComma, ","); - /* skip comments / whitespace tokens that might sneak in */ - while (i < t.Length && t[i].Scopes.Helper.IsCommentLineDoubleSlash) - i++; + SkipTrivia(t, ref i); } Consume(t, ref i, MetaBlock, "}"); // closing brace @@ -439,13 +466,9 @@ private static Enum ParseEnum(SyntaxToken[] t, ref int i) modifiers: mods); } /*────────────────────── ParseStruct ──────────────────────*/ - private static Struct ParseStruct(SyntaxToken[] t, ref int i) + // Contract: the caller consumed modifiers + "struct"; i points at the struct name. + private static Struct ParseStruct(SyntaxToken[] t, ref int i, TypeModifiers mods) { - /* 0. modifiers that precede “struct” */ - var mods = ParseModifierFlags(t, ref i); - - /* 1. keyword + identifier */ - Consume(t, ref i, KeywordControl, "struct"); var nameTok = Consume(t, ref i, EntityNameTypeStruct); /* 2. opening brace */ @@ -465,10 +488,10 @@ private static Struct ParseStruct(SyntaxToken[] t, ref int i) bool doneGlobals = false; while (true) { - if (!doneLocals && t[i].Scopes.Helper.IsControl("local")) + if (!doneLocals && IsContextBlock(t, i, "local")) { locals = ParseLocal(t, ref i); doneLocals = true; continue; } - if (!doneGlobals && t[i].Scopes.Helper.IsControl("global")) + if (!doneGlobals && IsContextBlock(t, i, "global")) { globals = ParseGlobal(t, ref i); doneGlobals = true; continue; } break; @@ -477,10 +500,17 @@ private static Struct ParseStruct(SyntaxToken[] t, ref int i) /* 4. scan members until ‘}’ */ while (!LookAhead(t, i, MetaBlock, "}")) { - if (t[i].Scopes.Helper.IsControl("field")) { fields.Add(ParseField(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsStorageType) { props.Add(ParseProperty(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsControl("func")) { funcs.Add(ParseFunc(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsControl("action")) { actions.Add(ParseAction(t, ref i)); continue; } + if (TryConsumeDeclarationPrefix(t, ref i, "field", out var fieldMods)) + { fields.Add(ParseField(t, ref i, fieldMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "func", out var funcMods)) + { funcs.Add(ParseFunc(t, ref i, funcMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "action", out var actionMods)) + { actions.Add(ParseAction(t, ref i, actionMods)); continue; } + + if (TryBeginTypedDeclaration(t, ref i, out var propertyMods)) + { props.Add(ParseProperty(t, ref i, propertyMods)); continue; } /* comments / whitespace / unknown → skip */ i++; @@ -498,13 +528,9 @@ private static Struct ParseStruct(SyntaxToken[] t, ref int i) mods); } /*────────────────────── ParseInterface ──────────────────────*/ - private static Interface ParseInterface(SyntaxToken[] t, ref int i) + // Contract: the caller consumed modifiers + "interface"; i points at the interface name. + private static Interface ParseInterface(SyntaxToken[] t, ref int i, TypeModifiers mods) { - /* 0. any modifiers in front of the keyword */ - var mods = ParseModifierFlags(t, ref i); - - /* 1. “interface” keyword + identifier */ - Consume(t, ref i, KeywordControl, "interface"); var nameTok = Consume(t, ref i, EntityNameTypeInterface); /* 2. body “{ … }” */ @@ -516,46 +542,46 @@ private static Interface ParseInterface(SyntaxToken[] t, ref int i) while (!LookAhead(t, i, MetaBlock, "}")) { - /* ── a property signature (explicit-type) ───────────────────────────── */ - if (t[i].Scopes.Helper.IsStorageType) - { - props.Add(ParseProperty(t, ref i)); - continue; - } - /* ── function signature: must NOT have a body ───────────────────────── */ - if (t[i].Scopes.Helper.IsControl("func")) + if (TryConsumeDeclarationPrefix(t, ref i, "func", out var funcMods)) { - var f = ParseFunc(t, ref i); + var f = ParseFunc(t, ref i, funcMods); if (f.Definition is not null) _errors.Add(new ParseError( - t[i].Line, "interface", + t[Math.Min(i, t.Length - 1)].Line, "interface", $"Interface function '{f.Name}' cannot have a body.")); methods.Add(f); continue; } /* ── action signature: same rule ────────────────────────────────────── */ - if (t[i].Scopes.Helper.IsControl("action")) + if (TryConsumeDeclarationPrefix(t, ref i, "action", out var actionMods)) { - var a = ParseAction(t, ref i); + var a = ParseAction(t, ref i, actionMods); if (a.Definition is not null) _errors.Add(new ParseError( - t[i].Line, "interface", + t[Math.Min(i, t.Length - 1)].Line, "interface", $"Interface action '{a.Name}' cannot have a body.")); actions.Add(a); continue; } + /* ── a property signature (explicit-type) ───────────────────────────── */ + if (TryBeginTypedDeclaration(t, ref i, out var propertyMods)) + { + props.Add(ParseProperty(t, ref i, propertyMods)); + continue; + } + /* interface must NOT contain fields or nested type definitions */ - if (t[i].Scopes.Helper.IsControl("field")) + if (LookAhead(t, i, KeywordControl, "field")) _errors.Add(new ParseError( t[i].Line, "interface", "Interfaces cannot declare fields. Use properties instead.")); - if (t[i].Scopes.Helper.IsControl("class") || t[i].Scopes.Helper.IsControl("struct") || - t[i].Scopes.Helper.IsControl("record") || t[i].Scopes.Helper.IsControl("interface") || - t[i].Scopes.Helper.IsControl("enum")) + if (LookAhead(t, i, KeywordControl, "class") || LookAhead(t, i, KeywordControl, "struct") || + LookAhead(t, i, KeywordControl, "record") || LookAhead(t, i, KeywordControl, "interface") || + LookAhead(t, i, KeywordControl, "enum")) _errors.Add(new ParseError( t[i].Line, "interface", "Interfaces cannot contain nested type declarations.")); @@ -574,17 +600,9 @@ private static Interface ParseInterface(SyntaxToken[] t, ref int i) modifiers: mods); } /*────────────────────── ParseRecord ──────────────────────*/ - private static Record ParseRecord(SyntaxToken[] t, ref int i) + // Contract: the caller consumed modifiers + optional "type" + "record"; i points at the record name. + private static Record ParseRecord(SyntaxToken[] t, ref int i, TypeModifiers mods) { - /* 0. modifiers */ - var mods = ParseModifierFlags(t, ref i); // may advance i - - /* 1. optional “type” keyword (module-level alias) */ - if (LookAhead(t, i, KeywordControl, "type")) - i++; // just consume it - - /* 2. “record” keyword + identifier */ - Consume(t, ref i, KeywordControl, "record"); var nameTok = Consume(t, ref i, EntityNameTypeRecord); /* 3. optional positional list “( t_int32 X , … )” → auto-properties */ @@ -619,9 +637,9 @@ private static Record ParseRecord(SyntaxToken[] t, ref int i) Consume(t, ref i, MetaBlock, "{"); /* optional leading local / global ctx inside the record */ - if (LookAhead(t, i, KeywordControl, "local")) + if (IsContextBlock(t, i, "local")) localCtx = ParseLocal(t, ref i); - else if (LookAhead(t, i, KeywordControl, "global")) + else if (IsContextBlock(t, i, "global")) globalCtx = ParseGlobal(t, ref i); var propsList = new List(); @@ -630,13 +648,24 @@ private static Record ParseRecord(SyntaxToken[] t, ref int i) while (!LookAhead(t, i, MetaBlock, "}")) { - if (t[i].Scopes.Helper.IsStorageType) { propsList.Add(ParseProperty(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsFuncCallKeyword) { funcsList.Add(ParseFunc(t, ref i)); continue; } - if (t[i].Scopes.Helper.IsActionLabel) { actsList.Add(ParseAction(t, ref i)); continue; } + if (TryConsumeDeclarationPrefix(t, ref i, "func", out var funcMods)) + { funcsList.Add(ParseFunc(t, ref i, funcMods)); continue; } - /* fields are NOT allowed in records */ - if (t[i].Scopes.Helper.IsFieldLabel) - _errors.Add(new ParseError(t[i].Line, "record", $"Fields are not permitted inside record '{nameTok.Source}'.")); + if (TryConsumeDeclarationPrefix(t, ref i, "action", out var actionMods)) + { actsList.Add(ParseAction(t, ref i, actionMods)); continue; } + + if (TryConsumeDeclarationPrefix(t, ref i, "field", out var fieldMods)) + { + _ = ParseField(t, ref i, fieldMods); + _errors.Add(new ParseError( + t[Math.Min(i, t.Length - 1)].Line, + "record", + $"Fields are not permitted inside record '{nameTok.Source}'.")); + continue; + } + + if (TryBeginTypedDeclaration(t, ref i, out var propertyMods)) + { propsList.Add(ParseProperty(t, ref i, propertyMods)); continue; } i++; // comment / whitespace } @@ -662,6 +691,83 @@ private static Record ParseRecord(SyntaxToken[] t, ref int i) modifiers: mods); } /*────────────────────────── HELPERS ─────────────────────────*/ + + /// + /// Returns true only for a local/global context introducer followed by a block. + /// This keeps global func ... and local field ... available as + /// declaration modifiers instead of misclassifying them as context blocks. + /// + private static bool IsContextBlock(SyntaxToken[] t, int i, string keyword) + { + var cursor = i; + SkipTrivia(t, ref cursor); + if (cursor >= t.Length || + !t[cursor].Scopes.DataSource.Contains(StorageModifier) || + !t[cursor].Source.Equals(keyword, StringComparison.OrdinalIgnoreCase)) + return false; + + cursor++; + return LookAhead(t, cursor, MetaBlock, "{"); + } + + /// + /// Tries to consume a declaration prefix without mutating the caller cursor on failure. + /// On success, modifiers and the declaration keyword are consumed and + /// points at the first token owned by the declaration-specific parser. + /// + private static bool TryConsumeDeclarationPrefix( + SyntaxToken[] t, + ref int i, + string keyword, + out TypeModifiers modifiers, + bool allowTypePrefix = false) + { + var cursor = i; + SkipTrivia(t, ref cursor); + modifiers = ParseModifierFlags(t, ref cursor); + SkipTrivia(t, ref cursor); + + if (allowTypePrefix && LookAhead(t, cursor, KeywordControl, "type")) + { + Consume(t, ref cursor, KeywordControl, "type"); + SkipTrivia(t, ref cursor); + } + + if (!LookAhead(t, cursor, KeywordControl, keyword)) + { + modifiers = TypeModifiers.None; + return false; + } + + Consume(t, ref cursor, KeywordControl, keyword); + i = cursor; + return true; + } + + /// + /// Tries to consume declaration modifiers for a declaration that begins directly with a type. + /// On success points at that type. + /// + private static bool TryBeginTypedDeclaration( + SyntaxToken[] t, + ref int i, + out TypeModifiers modifiers) + { + var cursor = i; + SkipTrivia(t, ref cursor); + modifiers = ParseModifierFlags(t, ref cursor); + SkipTrivia(t, ref cursor); + + if (cursor >= t.Length || !t[cursor].Scopes.Helper.IsStorageType) + { + modifiers = TypeModifiers.None; + return false; + } + + i = cursor; + return true; + } + /* numeric literal → long (supports decimal / 0xHEX / 0bBIN) */ private static long ParseEnumConstant(SyntaxToken tok) { @@ -694,13 +800,13 @@ private static long ParseEnumConstant(SyntaxToken tok) int bodyStart = i; /* ── optional local { … } right after the opening brace ───────── */ - if (i < t.Length && t[i].Source.Equals("local", StringComparison.OrdinalIgnoreCase)) + if (IsContextBlock(t, i, "local")) { extractedLocals = ParseLocal(t, ref i); /* optional semicolon after the local block */ if (LookAhead(t, i, PunctuationSemicolon, ";")) - i++; + Consume(t, ref i, PunctuationSemicolon, ";"); SkipTrivia(t, ref i); // skip EOL / spaces bodyStart = i; // real body starts here @@ -735,8 +841,10 @@ private static void SkipTrivia(SyntaxToken[] t, ref int i) while (i < t.Length && (string.IsNullOrWhiteSpace(t[i].Source) || t[i].Scopes.Length == 0 || - (t[i].Scopes.Length == 1 && t[i].Scopes.Helper.IsSource))) - i++; // step over whitespace / comments / blank + (t[i].Scopes.Length == 1 && t[i].Scopes.Helper.IsSource) || + t[i].Scopes.Helper.IsCommentLineDoubleSlash || + t[i].Scopes.Helper.IsCommentBlock)) + i++; } private static SyntaxToken ConsumeType(SyntaxToken[] t, ref int i, out UType ut) @@ -751,14 +859,18 @@ private static SyntaxToken Consume(SyntaxToken[] t, ref int i, string scope, str private static SyntaxToken Consume(SyntaxToken[] t, ref int i, Func scopePred, string? src = null) { - SkipTrivia(t, ref i); - if (i >= t.Length) + var cursor = i; + SkipTrivia(t, ref cursor); + + if (cursor >= t.Length) throw new FormatException($"Unexpected end of tokens while reading {src ?? ""}."); - var tk = t[i]; + + var tk = t[cursor]; if (!tk.Scopes.DataSource.Any(scopePred) || (src != null && tk.Source != src)) - _errors.Add(new(tk.Line, src ?? "unknown", $"Expected {src ?? ""} but found '{tk.Source}'")); + throw new FormatException($"Expected {src ?? ""} but found '{tk.Source}' at line {tk.Line + 1}."); - return t[i++]; + i = cursor + 1; + return tk; } /// @@ -890,8 +1002,12 @@ private static TypeModifiers ParseModifierFlags(SyntaxToken[] t, ref int i) { TypeModifiers mods = TypeModifiers.None; - while (i < t.Length && t[i].Scopes.Helper.IsModifier) + while (true) { + SkipTrivia(t, ref i); + if (i >= t.Length || !t[i].Scopes.Helper.IsModifier) + break; + if (_modMap.TryGetValue(t[i].Source, out var m)) mods |= m; i++; // consume the modifier token } diff --git a/Ubytec/Language/Grammar/ubytec.tmLanguage.json b/Ubytec/Language/Grammar/ubytec.tmLanguage.json index fc8c74e3..6f867c11 100644 --- a/Ubytec/Language/Grammar/ubytec.tmLanguage.json +++ b/Ubytec/Language/Grammar/ubytec.tmLanguage.json @@ -62,7 +62,7 @@ "patterns": [ { "name": "storage.modifier.ubytec", - "match": "\\b(public|private|protected|internal|const|abstract|virtual|override|sealed|readonly|local|global|secret)\\b" + "match": "\\b(public|private|protected|internal|const|abstract|virtual|override|sealed|readonly|secret|local|global)\\b" } ] }, @@ -80,10 +80,6 @@ "name": "constant.numeric.double.ubytec", "match": "\\b(?:(?:\\d+(?:_\\d+)*\\.\\d+(?:_\\d+)*(?:[eE][+-]?\\d+(?:_\\d+)*)?)|(?:\\d+(?:_\\d+)*[eE][+-]?\\d+(?:_\\d+)*))\\b" }, - { - "name": "constant.numeric.double.ubytec", - "match": "\\b(?:(?:\\d+(?:_\\d+)*\\.\\d+(?:_\\d+)*(?:[eE][+-]?\\d+(?:_\\d+)*)?)|(?:\\d+(?:_\\d+)*[eE][+-]?\\d+(?:_\\d+)*))\\b" - }, { "name": "constant.numeric.int.ubytec", "match": "\\b\\d+(?:_\\d+)*(?:[Uu][Ll]|[Ll][Uu]|[Uu]|[Ll])?\\b" @@ -206,7 +202,7 @@ }, { "name": "keyword.control.ubytec", - "match": "\\b(module|type|class|record|func|action|field|local|global)\\b" + "match": "\\b(module|type|class|struct|record|interface|enum|func|action|field)\\b" }, { "name": "keyword.control.flow.ubytec", @@ -288,34 +284,6 @@ }, "operators": { "patterns": [ - { - "name": "operator.equality.ubytec", - "match": "(?<=\\S\\s*)==(?=\\s*\\S)" - }, - { - "name": "operator.inequality.ubytec", - "match": "(?<=\\S\\s*)!=(?=\\s*\\S)" - }, - { - "name": "operator.less-than-equals.ubytec", - "match": "(?<=\\S\\s*)<=(?=\\s*\\S)" - }, - { - "name": "operator.greater-than-equals.ubytec", - "match": "(?<=\\S\\s*)>=(?=\\s*\\S)" - }, - { - "name": "operator.less-than.ubytec", - "match": "(?<=\\S\\s*)<(?![=])(?=\\s*\\S)" - }, - { - "name": "operator.greater-than.ubytec", - "match": "(?<=\\S\\s*)>(?![=])(?=\\s*\\S)" - }, - { - "name": "operator.negation.ubytec", - "match": "(?<=\\S\\s*)!(?!=)(?=\\s*\\S)" - }, { "name": "operator.unsigned-right-shift.ubytec", "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))>>>(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" @@ -333,36 +301,48 @@ "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))>>(?!>)(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" }, { - "name": "operator.addition.ubytec", - "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))\\+(?!\\+)(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + "name": "operator.pipe.ubytec", + "match": "(?<=\\S\\s*)<->(?=\\s*\\S)" }, { - "name": "operator.subtraction.ubytec", - "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))-(?!-)(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + "name": "operator.pipe-in.ubytec", + "match": "(?<=\\S\\s*)<-<(?=\\s*\\S)" }, { - "name": "operator.division.ubytec", - "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))/(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + "name": "operator.pipe-out.ubytec", + "match": "(?<=\\S\\s*)>->(?=\\s*\\S)" }, { - "name": "operator.multiplication.ubytec", - "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))\\*(?!\\*)(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + "name": "operator.inline-return.ubytec", + "match": "(?<=\\S\\s*)=>(?=\\s*\\S)" }, { - "name": "operator.exponentiation.ubytec", - "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))\\^(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + "name": "operator.equality.ubytec", + "match": "(?<=\\S\\s*)==(?=\\s*\\S)" }, { - "name": "operator.modulo.ubytec", - "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))%(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + "name": "operator.inequality.ubytec", + "match": "(?<=\\S\\s*)!=(?=\\s*\\S)" }, { - "name": "operator.bitwise-and.ubytec", - "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))&(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + "name": "operator.less-than-equals.ubytec", + "match": "(?<=\\S\\s*)<=(?=\\s*\\S)" }, { - "name": "operator.hash.ubytec", - "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))#(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + "name": "operator.greater-than-equals.ubytec", + "match": "(?<=\\S\\s*)>=(?=\\s*\\S)" + }, + { + "name": "operator.less-than.ubytec", + "match": "(?<=\\S\\s*)<(?![=<-])(?=\\s*\\S)" + }, + { + "name": "operator.greater-than.ubytec", + "match": "(?<=\\S\\s*)>(?![=>-])(?=\\s*\\S)" + }, + { + "name": "operator.negation.ubytec", + "match": "(?<=\\S\\s*)!(?!=)(?=\\s*\\S)" }, { "name": "operator.increment.ubytec", @@ -384,18 +364,6 @@ "name": "operator.optional-chaining.ubytec", "match": "(?<=\\S\\s*)\\?\\.(?=\\s*\\S)" }, - { - "name": "operator.pipe.ubytec", - "match": "(?<=\\S\\s*)<->(?=\\s*\\S)" - }, - { - "name": "operator.pipe-in.ubytec", - "match": "(?<=\\S\\s*)<-<(?=\\s*\\S)" - }, - { - "name": "operator.pipe-out.ubytec", - "match": "(?<=\\S\\s*)>->(?=\\s*\\S)" - }, { "name": "operator.nullable-coalescence.ubytec", "match": "(?<=\\S\\s*)\\?\\?(?=\\s*\\S)" @@ -409,12 +377,40 @@ "match": "(?<=\\S\\s*)~~(?=\\s*\\S)" }, { - "name": "operator.assign.ubytec", - "match": "(?<=\\S\\s*)=(?=\\s*\\S)" + "name": "operator.addition.ubytec", + "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))\\+(?!\\+)(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" }, { - "name": "operator.inline-return.ubytec", - "match": "(?<=\\S\\s*)=>(?=\\s*\\S)" + "name": "operator.subtraction.ubytec", + "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))-(?!-|>)(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + }, + { + "name": "operator.division.ubytec", + "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))/(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + }, + { + "name": "operator.multiplication.ubytec", + "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))\\*(?!\\*)(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + }, + { + "name": "operator.exponentiation.ubytec", + "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))\\^(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + }, + { + "name": "operator.modulo.ubytec", + "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))%(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + }, + { + "name": "operator.bitwise-and.ubytec", + "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))&(?!&)(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + }, + { + "name": "operator.hash.ubytec", + "match": "(?<=((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))\\s*))#(?=\\s*((?:[A-Za-z_][A-Za-z0-9_]*|(?:0x[0-9A-Fa-f]+)|(?:0b[01]+)|(?:\\d+(?:\\.\\d+)?))))" + }, + { + "name": "operator.assign.ubytec", + "match": "(?<=\\S\\s*)=(?!=|>)(?=\\s*\\S)" } ] }, @@ -424,17 +420,17 @@ "name": "punctuation.comma.ubytec", "match": "," }, - { - "name": "punctuation.separator.key-value.ubytec", - "match": ":" - }, { "name": "punctuation.scope.ubytec", "match": "::" }, + { + "name": "punctuation.separator.key-value.ubytec", + "match": ":(?!:)" + }, { "name": "punctuation.separator.parent-child.ubytec", - "match": "\\." + "match": "\\.(?!\\.)" }, { "name": "punctuation.semicolon.ubytec", @@ -446,17 +442,15 @@ } ] }, - "arguments": { + "identifiers": { "patterns": [ { "name": "entity.name.argument.ubytec", - "match": "\\b[A-Za-z_][A-Za-z0-9_]*\\b" + "match": "\\b[A-Za-z_][A-Za-z0-9_]*\\b(?=\\s*:)" }, { - "include": "#comments" - }, - { - "include": "#constants" + "name": "entity.name.identifier.ubytec", + "match": "\\b[A-Za-z_][A-Za-z0-9_]*\\b" } ] }, @@ -496,11 +490,11 @@ }, { "name": "entity.name.field.ubytec", - "match": "(?<=\\bfield\\s+)[A-Za-z_][A-Za-z0-9_]*\\b" + "match": "(?<=\\bfield\\s+(?:t_[A-Za-z0-9_]+\\??|\\[t_[A-Za-z0-9_]+\\??\\]\\??)\\s+)[A-Za-z_][A-Za-z0-9_]*\\b" }, { "name": "entity.name.var.explicit.ubytec", - "match": "(?<=\\bt_[A-Za-z0-9_]+\\s+)[A-Za-z_][A-Za-z0-9_]*\\b" + "match": "(?<=(?:\\bt_[A-Za-z0-9_]+\\??|\\[t_[A-Za-z0-9_]+\\??\\]\\??)\\s+)[A-Za-z_][A-Za-z0-9_]*\\b" } ] }, @@ -545,7 +539,7 @@ "include": "#codeStructures" }, { - "include": "#arguments" + "include": "#identifiers" }, { "include": "#invalid" diff --git a/Ubytec/Language/Portable/IndexedExecutableFormat.cs b/Ubytec/Language/Portable/IndexedExecutableFormat.cs index cbb1e209..1f863382 100644 --- a/Ubytec/Language/Portable/IndexedExecutableFormat.cs +++ b/Ubytec/Language/Portable/IndexedExecutableFormat.cs @@ -228,10 +228,18 @@ private static void BuildBindings(CallableInfo callable) { uint next = 0; foreach (Argument argument in callable.Arguments) - Add(argument.Name, argument.Type, isLocal: false, value: null); + Add( + argument.Name, + WithBindingModifiers(argument.Type, argument.Modifiers), + isLocal: false, + value: null); foreach (Variable variable in callable.Locals?.Variables ?? []) - Add(variable.Name, variable.Type, isLocal: true, variable.Value); + Add( + variable.Name, + WithBindingModifiers(variable.Type, variable.Modifiers), + isLocal: true, + variable.Value); foreach (VAR declaration in FunctionEmitter.Instructions(callable.Definition).OfType()) Add(declaration.Variable.Name, declaration.Variable.Type, isLocal: true, declaration.Variable.Value, fromBodyVar: true); @@ -245,6 +253,18 @@ void Add(string name, UType type, bool isLocal, object? value, bool fromBodyVar } } + private static UType WithBindingModifiers(UType type, TypeModifiers modifiers) + { + const TypeModifiers bindingModifiers = TypeModifiers.ReadOnly | TypeModifiers.Const; + TypeModifiers merged = type.Modifiers | (modifiers & bindingModifiers); + return new UType( + type.Type, + merged, + type.ID, + type.Type == PrimitiveType.CustomType ? type.Name : null, + checked((nuint)type.FixedArraySize)); + } + private static HashSet CollectTypes( IEnumerable callables, PortableModuleContracts contracts) @@ -253,9 +273,7 @@ private static HashSet CollectTypes( foreach (CallableInfo callable in callables) { result.Add(TypeKey.From(callable.ReturnType)); - foreach (Argument argument in callable.Arguments) - result.Add(TypeKey.From(argument.Type)); - foreach (BindingInfo binding in callable.Bindings.Where(binding => binding.IsLocal)) + foreach (BindingInfo binding in callable.Bindings) result.Add(TypeKey.From(binding.Type)); } @@ -404,6 +422,7 @@ private static byte[] EncodeExportEntry( if (export.Arguments.Count != callable.Arguments.Length) throw new InvalidDataException($"Portable export '{export.Symbol}' argument count does not match '{callable.Name}'."); + BindingInfo[] internalArguments = callable.Bindings.Where(binding => !binding.IsLocal).ToArray(); using var stream = new MemoryStream(); WriteUInt32(stream, StringIndex(export.Symbol, stringIndices, "export symbol")); WriteUInt32(stream, functionIndices[callable]); @@ -414,8 +433,7 @@ private static byte[] EncodeExportEntry( PortableBoundaryParameter argument = export.Arguments[index]; ValidateBoundaryParameter(argument, "export argument"); uint carrier = TypeIndex(argument.CarrierType, typeIndices); - uint internalType = TypeIndex(callable.Arguments[index].Type, typeIndices); - if (carrier != internalType) + if (!MatchesBoundaryCarrier(internalArguments[index].Type, argument.CarrierType)) throw new InvalidDataException( $"Portable export '{export.Symbol}' argument {index} does not match '{callable.Name}'."); stream.WriteByte((byte)argument.PassingMode); @@ -424,6 +442,11 @@ private static byte[] EncodeExportEntry( return stream.ToArray(); } + private static bool MatchesBoundaryCarrier(UType internalType, PrimitiveType carrier) => + internalType.Type == carrier && + internalType.FixedArraySize == 1 && + (internalType.Modifiers & ~(TypeModifiers.ReadOnly | TypeModifiers.Const)) == TypeModifiers.None; + private static byte[] EncodeFunction( CallableInfo callable, IReadOnlyDictionary typeIndices, @@ -710,14 +733,15 @@ private static byte[] EncodeFunctionTable( foreach (FunctionEncoding encoded in functions) { CallableInfo function = encoded.Callable; + BindingInfo[] arguments = function.Bindings.Where(binding => !binding.IsLocal).ToArray(); BindingInfo[] locals = function.Bindings.Where(binding => binding.IsLocal).ToArray(); WriteUInt32(stream, TypeIndex(function.ReturnType, typeIndices)); - WriteUInt32(stream, checked((uint)function.Arguments.Length)); + WriteUInt32(stream, checked((uint)arguments.Length)); WriteUInt32(stream, checked((uint)locals.Length)); WriteUInt32(stream, encoded.CodeOffset); WriteUInt32(stream, encoded.CodeLength); WriteUInt32(stream, 0); - foreach (Argument argument in function.Arguments) + foreach (BindingInfo argument in arguments) WriteUInt32(stream, TypeIndex(argument.Type, typeIndices)); foreach (BindingInfo local in locals) WriteUInt32(stream, TypeIndex(local.Type, typeIndices)); @@ -967,8 +991,10 @@ private static void ParseExportTable( for (int argument = 0; argument < checked((int)arguments); argument++) { BoundaryDescriptor descriptor = ReadBoundaryDescriptor(ref reader, types, "Export argument"); - if (descriptor.TypeIndex != function.ArgumentTypes[argument]) - throw new InvalidDataException("Export argument carrier TypeIndex must match its FunctionIndex signature."); + TypeEntry carrier = types[checked((int)descriptor.TypeIndex)]; + TypeEntry internalType = types[checked((int)function.ArgumentTypes[argument])]; + if (!MatchesBoundaryCarrier(internalType, carrier)) + throw new InvalidDataException("Export argument carrier type must match its FunctionIndex signature modulo binding mutability."); } byte[] name = strings[checked((int)symbol)]; @@ -979,6 +1005,12 @@ private static void ParseExportTable( reader.RequireEnd("Export table"); } + private static bool MatchesBoundaryCarrier(TypeEntry internalType, TypeEntry carrier) => + internalType.Primitive == carrier.Primitive && + internalType.FixedArraySize == carrier.FixedArraySize && + carrier.Modifiers == 0 && + (internalType.Modifiers & ~(uint)(TypeModifiers.ReadOnly | TypeModifiers.Const)) == 0; + private static BoundaryDescriptor ReadBoundaryDescriptor( ref SpanReader reader, IReadOnlyList types, diff --git a/Ubytec/Language/Portable/PortableBinaryVerifier.cs b/Ubytec/Language/Portable/PortableBinaryVerifier.cs index 7aca91d1..9231282c 100644 --- a/Ubytec/Language/Portable/PortableBinaryVerifier.cs +++ b/Ubytec/Language/Portable/PortableBinaryVerifier.cs @@ -173,7 +173,7 @@ private static void VerifyIndexedExecutable(IReadOnlyList sections) ParseExportTable(exportsPayload, strings, functions, types); ParseConstantTable(constantsPayload); VerifyFunctionRanges(functions, codePayload.Length); - VerifyCode(functions, imports.Count, types, codePayload); + VerifyCode(functions, imports, types, codePayload); } private static RawSection Required( @@ -383,9 +383,11 @@ private static void ParseExportTable( for (int argument = 0; argument < checked((int)argumentCount); argument++) { BoundaryDescriptor descriptor = ReadBoundaryDescriptor(ref reader, types, "Export argument"); - if (descriptor.TypeIndex != function.ArgumentTypes[argument]) + TypeEntry carrier = types[checked((int)descriptor.TypeIndex)]; + TypeEntry internalType = types[checked((int)function.ArgumentTypes[argument])]; + if (!MatchesBoundaryCarrier(internalType, carrier)) throw new InvalidDataException( - "Export argument carrier TypeIndex must match its FunctionIndex signature."); + "Export argument carrier type must match its FunctionIndex signature modulo binding mutability."); } byte[] name = strings[checked((int)symbol)]; @@ -396,6 +398,12 @@ private static void ParseExportTable( reader.RequireEnd("Export table"); } + private static bool MatchesBoundaryCarrier(TypeEntry internalType, TypeEntry carrier) => + internalType.Primitive == carrier.Primitive && + internalType.FixedArraySize == carrier.FixedArraySize && + carrier.Modifiers == 0 && + (internalType.Modifiers & ~(uint)(TypeModifiers.ReadOnly | TypeModifiers.Const)) == 0; + private static BoundaryDescriptor ReadBoundaryDescriptor( ref RawReader reader, IReadOnlyList types, @@ -491,10 +499,12 @@ private static void VerifyFunctionRanges(IReadOnlyList functions, private static void VerifyCode( IReadOnlyList functions, - int importCount, + IReadOnlyList imports, IReadOnlyList types, ReadOnlySpan code) { + VerifyFunctionTypes(functions, types); + for (int functionIndex = 0; functionIndex < functions.Count; functionIndex++) { FunctionEntry function = functions[functionIndex]; @@ -502,17 +512,35 @@ private static void VerifyCode( checked((int)function.CodeOffset), checked((int)function.CodeLength)); var reader = new RawReader(body); ulong bindingCount = (ulong)function.ArgumentCount + function.LocalCount; + var initializedLocals = new bool[checked((int)function.LocalCount)]; + int stackDepth = 0; + bool reachable = true; + + void Need(int cells, byte opcode) + { + if (reachable && stackDepth < cells) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} opcode 0x{opcode:X2} needs {cells} evaluation stack cell(s), " + + $"but only {stackDepth} are available."); + } + + void Apply(int required, int delta, byte opcode) + { + Need(required, opcode); + if (reachable) + stackDepth = checked(stackDepth + delta); + } while (!reader.AtEnd) { byte opcode = reader.ReadByte("Opcode"); if (opcode == 0xFF) { - int depth = 1; + int escapeDepth = 1; while (!reader.AtEnd && reader.PeekByte() == 0xFF) { reader.ReadByte("Opcode escape"); - depth++; + escapeDepth++; } byte group = reader.ReadByte("Extended opcode group"); byte extendedOpcode = reader.ReadByte("Extended opcode number"); @@ -521,7 +549,7 @@ private static void VerifyCode( "0xFF is reserved in extended opcode group/code positions."); throw new NotSupportedException( $"Portable binary verifier does not yet support extended opcode " + - $"depth={depth}, group=0x{group:X2}, opcode=0x{extendedOpcode:X2}."); + $"depth={escapeDepth}, group=0x{group:X2}, opcode=0x{extendedOpcode:X2}."); } switch (opcode) @@ -529,14 +557,23 @@ private static void VerifyCode( case RETURN.OP: { byte mode = reader.ReadByte("RETURN mode"); - if (mode == 0) break; - if (mode != 1) + if (mode == 1) + { + uint typeIndex = reader.ReadUInt32("RETURN TypeIndex"); + RequireIndex(typeIndex, types.Count, "RETURN TypeIndex"); + if (typeIndex != function.ReturnType) + throw new InvalidDataException( + "Explicit RETURN TypeIndex must match the enclosing function return type."); + } + else if (mode != 0) + { throw new InvalidDataException($"Unknown RETURN mode {mode}."); - uint typeIndex = reader.ReadUInt32("RETURN TypeIndex"); - RequireIndex(typeIndex, types.Count, "RETURN TypeIndex"); - if (typeIndex != function.ReturnType) - throw new InvalidDataException( - "Explicit RETURN TypeIndex must match the enclosing function return type."); + } + + int resultCells = IsVoid(types[checked((int)function.ReturnType)]) ? 0 : 1; + Need(resultCells, opcode); + if (reachable) + reachable = false; break; } @@ -545,6 +582,13 @@ private static void VerifyCode( uint binding = reader.ReadUInt32("VAR BindingIndex"); if (binding >= bindingCount) throw new InvalidDataException("VAR BindingIndex is outside the current function frame."); + if (binding < function.ArgumentCount) + throw new InvalidDataException("VAR cannot initialize an argument BindingIndex."); + int localIndex = checked((int)(binding - function.ArgumentCount)); + if (initializedLocals[localIndex]) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} local BindingIndex {binding} is initialized more than once."); + initializedLocals[localIndex] = true; _ = reader.ReadUInt64("VAR initial cell"); break; } @@ -555,14 +599,30 @@ private static void VerifyCode( if (length is < 1 or > 8) throw new InvalidDataException("PUSH portable immediate must contain 1..8 bytes."); _ = reader.ReadBytes(length, "PUSH immediate"); + Apply(0, 1, opcode); break; } case StackOperations.DROP.OP: + { + byte index = reader.ReadByte("Stack index"); + Apply(index + 1, -1, opcode); + break; + } + case StackOperations.PICK.OP: + { + byte index = reader.ReadByte("Stack index"); + Apply(index + 1, 1, opcode); + break; + } + case StackOperations.ROLL.OP: - _ = reader.ReadByte("Stack index"); + { + byte index = reader.ReadByte("Stack index"); + Apply(index + 1, 0, opcode); break; + } case FunctionOperations.CALL.OP: { @@ -571,11 +631,22 @@ private static void VerifyCode( switch (targetKind) { case 0: + { RequireIndex(targetIndex, functions.Count, "CALL FunctionIndex"); + FunctionEntry target = functions[checked((int)targetIndex)]; + int arguments = checked((int)target.ArgumentCount); + int results = IsVoid(types[checked((int)target.ReturnType)]) ? 0 : 1; + Apply(arguments, results - arguments, opcode); break; + } case 1: - RequireIndex(targetIndex, importCount, "CALL ImportIndex"); + { + RequireIndex(targetIndex, imports.Count, "CALL ImportIndex"); + int consumed = ImportCallCellCount( + imports[checked((int)targetIndex)], types); + Apply(consumed, 1 - consumed, opcode); break; + } default: throw new InvalidDataException( $"CALL uses reserved TargetKind 0x{targetKind:X2}."); @@ -585,19 +656,23 @@ private static void VerifyCode( case FunctionOperations.FNADDR.OP: RequireIndex(reader.ReadUInt32("FunctionIndex"), functions.Count, "FunctionIndex"); + Apply(0, 1, opcode); break; case FunctionOperations.CALLI.OP: { - _ = reader.ReadByte("CALLI argument count"); - byte returnType = reader.ReadByte("CALLI return primitive type"); - if (!Enum.IsDefined(typeof(PrimitiveType), (PrimitiveType)returnType)) + byte argumentCount = reader.ReadByte("CALLI argument count"); + byte returnTypeRaw = reader.ReadByte("CALLI return primitive type"); + if (!Enum.IsDefined(typeof(PrimitiveType), (PrimitiveType)returnTypeRaw)) throw new InvalidDataException( - $"CALLI uses undefined primitive return type 0x{returnType:X2}."); - var type = new UType((PrimitiveType)returnType); - if (type.Type != PrimitiveType.Void && !FunctionCode.IsCellType(type)) + $"CALLI uses undefined primitive return type 0x{returnTypeRaw:X2}."); + var returnType = (PrimitiveType)returnTypeRaw; + if (returnType != PrimitiveType.Void && !IsCellPrimitive(returnType)) throw new InvalidDataException( "CALLI return type is not void or a scalar cell type."); + int consumed = argumentCount + 1; + int results = returnType == PrimitiveType.Void ? 0 : 1; + Apply(consumed, results - consumed, opcode); break; } @@ -611,9 +686,9 @@ private static void VerifyCode( break; case 1: { - byte primitive = reader.ReadByte("LOAD/STORE primitive type"); - if (!Enum.IsDefined(typeof(PrimitiveType), (PrimitiveType)primitive) || - !FunctionCode.IsCellType(new UType((PrimitiveType)primitive))) + byte primitiveRaw = reader.ReadByte("LOAD/STORE primitive type"); + if (!Enum.IsDefined(typeof(PrimitiveType), (PrimitiveType)primitiveRaw) || + !IsCellPrimitive((PrimitiveType)primitiveRaw)) throw new InvalidDataException( "LOAD/STORE raw type is not a supported scalar cell type."); break; @@ -624,11 +699,20 @@ private static void VerifyCode( if (binding >= bindingCount) throw new InvalidDataException( "LOAD/STORE BindingIndex is outside the current function frame."); + if (opcode == MemoryOperations.STORE.OP && + IsReadOnly(BindingType(function, binding, types))) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} cannot STORE to readonly/const BindingIndex {binding}."); break; } default: throw new InvalidDataException($"Unknown LOAD/STORE mode {mode}."); } + + if (opcode == MemoryOperations.LOAD.OP) + Apply(mode == 2 ? 0 : 1, mode == 2 ? 1 : 0, opcode); + else + Apply(mode == 2 ? 1 : 2, mode == 2 ? -1 : -2, opcode); break; } @@ -636,14 +720,211 @@ private static void VerifyCode( if (!OperandFreeCoreOpcodes.Contains(opcode)) throw new InvalidDataException( $"Core opcode 0x{opcode:X2} has no supported indexed executable v0 wire contract."); + ApplyOperandFreeStackEffect(opcode, ref stackDepth, ref reachable, functionIndex); break; } } reader.RequireEnd($"Function {functionIndex} code"); + + for (int local = 0; local < initializedLocals.Length; local++) + { + if (!initializedLocals[local]) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} local BindingIndex {function.ArgumentCount + (uint)local} has no VAR initialization."); + } + + if (!IsVoid(types[checked((int)function.ReturnType)]) && reachable) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} has a reachable path without RETURN."); + } + } + + private static void VerifyFunctionTypes( + IReadOnlyList functions, + IReadOnlyList types) + { + for (int functionIndex = 0; functionIndex < functions.Count; functionIndex++) + { + FunctionEntry function = functions[functionIndex]; + TypeEntry returnType = types[checked((int)function.ReturnType)]; + if (IsVoid(returnType)) + { + if (returnType.Modifiers != 0 || returnType.FixedArraySize != 1) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} void return type must be an unmodified one-cell type entry."); + } + else if (!IsCellType(returnType)) + { + throw new InvalidDataException( + $"FunctionIndex {functionIndex} return type is not supported by the portable cell ABI."); + } + + for (int argument = 0; argument < function.ArgumentTypes.Length; argument++) + { + TypeEntry type = types[checked((int)function.ArgumentTypes[argument])]; + if (!IsCellType(type)) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} argument BindingIndex {argument} is not a scalar cell type."); + } + + for (int local = 0; local < function.LocalTypes.Length; local++) + { + TypeEntry type = types[checked((int)function.LocalTypes[local])]; + if (!IsCellType(type)) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} local BindingIndex {function.ArgumentCount + (uint)local} is not a scalar cell type."); + } } } + private static void ApplyOperandFreeStackEffect( + byte opcode, + ref int stackDepth, + ref bool reachable, + int functionIndex) + { + if (!reachable) + return; + + if (opcode == TRAP.OP) + { + reachable = false; + return; + } + + if (opcode == CLEAR.OP) + { + stackDepth = 0; + return; + } + + int required; + int delta; + if (opcode == NULL.OP || opcode == DEFAULT.OP) + { + required = 0; + delta = 1; + } + else if (opcode == StackOperations.POP.OP) + { + required = 1; + delta = -1; + } + else if (opcode == StackOperations.DUP.OP) + { + required = 1; + delta = 1; + } + else if (opcode == StackOperations.SWAP.OP) + { + required = 2; + delta = 0; + } + else if (opcode == StackOperations.ROT.OP) + { + required = 3; + delta = 0; + } + else if (opcode == StackOperations.OVER.OP) + { + required = 2; + delta = 1; + } + else if (opcode == StackOperations.NIP.OP) + { + required = 2; + delta = -1; + } + else if (opcode == StackOperations.TwoDUP.OP) + { + required = 2; + delta = 2; + } + else if (opcode == StackOperations.TwoSWAP.OP) + { + required = 4; + delta = 0; + } + else if (opcode == StackOperations.TwoROT.OP) + { + required = 6; + delta = 0; + } + else if (opcode == StackOperations.TwoOVER.OP) + { + required = 4; + delta = 2; + } + else if (opcode is >= 0x20 and <= 0x24 or >= 0x30 and <= 0x32 or 0x29 or 0x2A or + >= 0x34 and <= 0x36 or >= 0x40 and <= 0x49) + { + required = 2; + delta = -1; + } + else if (opcode is >= 0x25 and <= 0x28 or 0x33) + { + required = 1; + delta = 0; + } + else + { + required = 0; + delta = 0; + } + + if (stackDepth < required) + throw new InvalidDataException( + $"FunctionIndex {functionIndex} opcode 0x{opcode:X2} needs {required} evaluation stack cell(s), " + + $"but only {stackDepth} are available."); + stackDepth = checked(stackDepth + delta); + } + + private static int ImportCallCellCount( + ImportEntry import, + IReadOnlyList types) + { + bool legacyBuffer = import.Arguments.Length == 4 && + import.Arguments.All(argument => + argument.PassingMode == PortablePassingMode.Value && + types[checked((int)argument.TypeIndex)].Primitive == PrimitiveType.UInt64) && + types[checked((int)import.ReturnType)].Primitive == PrimitiveType.Int64; + if (legacyBuffer) + return 5; + + int cells = 2; // budget_ms + result_address + foreach (BoundaryDescriptor argument in import.Arguments) + cells = checked(cells + + (argument.PassingMode is PortablePassingMode.LoanRead or PortablePassingMode.LoanWrite ? 2 : 1)); + return cells; + } + + private static TypeEntry BindingType( + FunctionEntry function, + uint binding, + IReadOnlyList types) + { + uint typeIndex = binding < function.ArgumentCount + ? function.ArgumentTypes[checked((int)binding)] + : function.LocalTypes[checked((int)(binding - function.ArgumentCount))]; + return types[checked((int)typeIndex)]; + } + + private static bool IsVoid(TypeEntry type) => type.Primitive == PrimitiveType.Void; + + private static bool IsReadOnly(TypeEntry type) => + (type.Modifiers & (uint)(TypeModifiers.ReadOnly | TypeModifiers.Const)) != 0; + + private static bool IsCellType(TypeEntry type) => + type.FixedArraySize == 1 && + (type.Modifiers & ~(uint)(TypeModifiers.Const | TypeModifiers.ReadOnly)) == 0 && + IsCellPrimitive(type.Primitive); + + private static bool IsCellPrimitive(PrimitiveType primitive) => + primitive is PrimitiveType.Bool or PrimitiveType.Char8 or PrimitiveType.Char16 or PrimitiveType.Byte or + PrimitiveType.SByte or PrimitiveType.Int16 or PrimitiveType.UInt16 or PrimitiveType.Int32 or + PrimitiveType.UInt32 or PrimitiveType.Int64 or PrimitiveType.UInt64; + private static HashSet DiscoverOperandFreeCoreOpcodes() { var result = new HashSet();