From 6d2176a8cd09ae689e11c2265071bba9363587e9 Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Tue, 15 Sep 2026 21:35:25 -0400 Subject: [PATCH] Add FQL standard library migration support for legacy-to-modern function conversions --- README.md | 59 +++-- cmd/internal/migrate/check.go | 4 +- cmd/internal/migrate/command_test.go | 4 +- cmd/internal/migrate/run.go | 7 +- cmd/internal/migrate/stdlib_test.go | 24 +- internal/migration/compatibility_stdlib.go | 8 +- .../migration/compatibility_stdlib_test.go | 14 +- internal/migration/fql_stdlib.go | 219 ++++++++++-------- .../migration/fql_stdlib_equivalence_test.go | 179 ++++++++++++++ .../migration/fql_stdlib_integration_test.go | 28 +-- internal/migration/fql_stdlib_literals.go | 87 +++++++ internal/migration/fql_stdlib_modes.go | 170 ++++++++++++++ internal/migration/fql_stdlib_rewrite.go | 105 +++++++++ .../fql_stdlib_structural_guards_test.go | 76 ++++++ .../migration/fql_stdlib_structural_test.go | 196 ++++++++++++++++ internal/migration/fql_stdlib_test.go | 19 +- 16 files changed, 1040 insertions(+), 159 deletions(-) create mode 100644 internal/migration/fql_stdlib_equivalence_test.go create mode 100644 internal/migration/fql_stdlib_literals.go create mode 100644 internal/migration/fql_stdlib_modes.go create mode 100644 internal/migration/fql_stdlib_rewrite.go create mode 100644 internal/migration/fql_stdlib_structural_guards_test.go create mode 100644 internal/migration/fql_stdlib_structural_test.go diff --git a/README.md b/README.md index d966a92..3641367 100644 --- a/README.md +++ b/README.md @@ -163,32 +163,63 @@ and `abs(value)` becomes `math::abs(value)`. Supported mappings include: | `encoding::` | `json_parse`, `json_stringify`, `encode_uri_component` → `query_escape`, `decode_uri_component` → `query_unescape`, `to_base64` → `base64_encode`, `from_base64` → `base64_decode`, `escape_html` → `html_escape`, `unescape_html` → `html_unescape` | | `crypto::` | `md5`, `sha1`, `sha512`, `random_token` | | `path::` | `base`, `clean`, `dir`, `ext`, `is_abs`, `separate`, `match` | +| `arrays::` | `first`, `flatten`, `last`, `sorted`, `unique`, `slice`, `intersection`, `nth` → `at`, `remove_values` → `remove_any`, `minus` → `difference`, `union` → `concat`, `union_distinct` → `union`, `range` | +| `random::` | Zero-argument `rand()` → `float()` | | `object::` | `values`, `has` → `has_key`, `zip`, `keep_keys`, `merge`, `merge_recursive` → `merge_deep`, and one-argument `keys(obj)` | | `datetime::` | `now`, `date` → `parse`, `date_dayofweek` → `day_of_week`, `date_dayofyear` → `day_of_year`, `date_leapyear` → `is_leap_year`; `date_year`, `date_month`, `date_day`, `date_hour`, `date_minute`, `date_second`, `date_millisecond`, `date_quarter`, `date_days_in_month`, `date_format`, `date_add`, `date_subtract` lose their `date_` prefix | | `math::` | `pi`, `abs`, `acos`, `asin`, `atan`, `atan2`, `ceil`, `cos`, `degrees`, `exp`, `exp2`, `floor`, `log`, `log2`, `log10`, `pow`, `radians`, `round`, `sin`, `sqrt`, `tan` | -Only parsed call targets are replaced; arguments retain their meaning, and -strings, comments, object keys, and variable names are not matched. Nested calls -are migrated independently, and already-qualified targets are preserved. Object -replacements use immutable operations, never `object::mut`. +Argument-aware replacements also include: + +| Legacy call | Replacement | +| --- | --- | +| `position(a, v)` or `position(a, v, false)` | `arrays::contains(a, v)` | +| `position(a, v, true)` | `arrays::index_of(a, v)` | +| `append(a, v)` / `push(a, v)`, optionally with `false` | `arrays::append(a, v)` | +| `remove_value(a, v)`, optionally with a negative integer literal | `arrays::remove(a, v)` | +| `sorted_unique(a)` | `arrays::sorted(arrays::unique(a))` | +| `shift(a)` | `arrays::slice(a, 1)` | +| `outersection(a, b)` | `arrays::symmetric_difference(a, b)` | +| `keys(o, false)` | `object::keys(o)` | +| `keys(o, true)` | `arrays::sorted(object::keys(o))` | +| `date_diff(a, b, unit, true)` | `datetime::diff(a, b, unit)` | + +Boolean modes and negative integer limits may be parenthesized; arbitrary +constant expressions are not evaluated. Retained arguments preserve their +evaluation order and count. Strings, comments, object keys, and variable names +are not matched. Nested calls migrate independently, including inside composed +replacements and calls requiring manual review. Already-qualified targets are +preserved. Array and object replacements use immutable operations, never +`arrays::mut` or `object::mut`. Manual follow-up includes the original path and line and explains why a call was preserved: - `join` is ambiguous between legacy path joining and modern global string joining. -- `keys` with any arity other than one needs argument-aware review. +- Dynamic modes in `position`, `keys`, `append`, and `push` need manual review. +- Unique `append`/`push` suppresses only the incoming duplicate; applying + `arrays::unique` would also remove existing duplicates. +- Zero, positive, and dynamic `remove_value` limits have no canonical limit mode. +- `outersection` with three or more arrays uses exactly-one-input semantics; + canonical symmetric difference uses odd-number-of-inputs semantics. +- `pop` needs an evaluation-count-preserving replacement; `unshift` must retain + argument evaluation order; `remove_nth` retains host-list removal semantics. +- Parameterized `rand` uses historical rounded/floored calculations rather than + canonical continuous or integer bounds. +- Unsupported arities of argument-aware rules require manual review. - `date_compare` has component-range semantics that differ from `datetime::same`; - legacy `date_diff` integer/floating behavior differs from `datetime::diff`. + `date_diff` without a literal `true` floating flag may truncate toward zero, + whereas `datetime::diff` always returns a Float. - `average`, `sum`, `min`, `max`, `median`, `percentile`, `stddev_population`, `stddev_sample`, `variance_population`, and `variance_sample` have permissive legacy behavior that differs from strict canonical math. - A matching function declaration or function alias anywhere in the file may change call resolution. These checks are deliberately conservative, including case variants and declarations in nested scopes. A namespace alias blocks a - replacement only when it would redirect that canonical target. + replacement when it would redirect any introduced namespace, including both + `arrays` and `object` for sorted keys. -Other safe calls in the same file still migrate. Array migrations and -`rand`/`range` are deferred and receive no new diagnostics in this pass. +Other safe calls in the same file still migrate. Rerunning migration produces no further edits; unresolved manual actions remain. If the formatter cannot preserve comments, the entire file is left unchanged and reported for manual follow-up. @@ -235,12 +266,12 @@ files. ### Checking FQL compatibility `migrate check` reports final collecting `FOR` compatibility and the same legacy -stdlib findings as `migrate run`: encoding, crypto, path, immutable object, -datetime, and scalar math replacements, plus calls requiring manual review. +stdlib findings as `migrate run`: encoding, crypto, path, immutable arrays and +objects, datetime, scalar math, range, and zero-argument random replacements, +plus calls requiring manual review. It uses the same conservative declaration and alias guards described above. -Already-qualified calls are left alone; arrays and `rand`/`range` remain outside -this pass. A clean check covers these supported rules, not all v1 application -semantics. +Already-qualified calls are left alone. A clean check covers these supported +rules, not all v1 application semantics. Check a standalone FQL file or recursively inspect a directory without modifying source files: diff --git a/cmd/internal/migrate/check.go b/cmd/internal/migrate/check.go index ea11c4c..12eac00 100644 --- a/cmd/internal/migrate/check.go +++ b/cmd/internal/migrate/check.go @@ -16,9 +16,9 @@ func newCompatibilityCheckCommand(service Service) *cobra.Command { Short: "Check FQL source for Ferret version compatibility", Long: "Check a standalone FQL file or recursively inspect a directory for supported behavior changes " + "between Ferret versions. The check does not require a Go module and never modifies source files.\n\n" + - "Checks final collecting FOR and legacy stdlib calls in encoding, crypto, path, object, datetime, and math. " + + "Checks final collecting FOR and legacy stdlib calls in encoding, crypto, path, arrays, object, datetime, math, and random. " + "Findings suggest canonical replacements or manual review for ambiguous calls, semantic differences, " + - "and local function or use alias collisions. Arrays and rand/range are outside this pass. " + + "and local function or use alias collisions. Literal modes, composed replacements, range, and zero-argument rand are supported. " + "Preview automatic replacements with migrate run --print.\n\n" + "Directory scans include testdata, hidden and underscore-prefixed directories, and nested Go modules. " + "They skip .git, .hg, .svn, vendor, and node_modules and do not follow directory symlinks.\n\n" + diff --git a/cmd/internal/migrate/command_test.go b/cmd/internal/migrate/command_test.go index b1aa688..43b49db 100644 --- a/cmd/internal/migrate/command_test.go +++ b/cmd/internal/migrate/command_test.go @@ -422,10 +422,10 @@ func TestMigrateCheckHelpDocumentsReadOnlyScanBoundaries(t *testing.T) { for _, expected := range []string{ "migrate check [path]", "does not require a Go module and never modifies source files", - "final collecting FOR and legacy stdlib calls in encoding, crypto, path, object, datetime, and math", + "final collecting FOR and legacy stdlib calls in encoding, crypto, path, arrays, object, datetime, math, and random", "manual review", "local function or use alias collisions", - "Arrays and rand/range are outside this pass", + "Literal modes, composed replacements, range, and zero-argument rand are supported", "migrate run --print", "include testdata, hidden and underscore-prefixed directories, and nested Go modules", "skip .git, .hg, .svn, vendor, and node_modules", diff --git a/cmd/internal/migrate/run.go b/cmd/internal/migrate/run.go index 7143be8..de1f832 100644 --- a/cmd/internal/migrate/run.go +++ b/cmd/internal/migrate/run.go @@ -26,10 +26,11 @@ func newRunCommand(store *config.Store, service Service) *cobra.Command { "directories, and nested Go modules. The selected directory itself is scanned regardless of its name. " + "Directory symlinks are not followed.\n\n" + "FQL migration returns and canonically formats a structurally recognized final top-level FOR. " + - "It also migrates safe legacy stdlib calls to encoding, crypto, path, object, datetime, and math namespaces. " + - "Ambiguous join, non-unary keys, date_compare, date_diff, aggregate math, and calls affected by " + + "It also migrates safe legacy stdlib calls to encoding, crypto, path, arrays, object, datetime, math, and random namespaces. " + + "Literal modes and composed replacements are supported, including sorted_unique, shift, and sorted keys. " + + "Ambiguous join, unsafe array modes, nonfloating date_diff, date_compare, aggregate math, parameterized rand, and calls affected by " + "local function or use alias collisions are left for manual follow-up. " + - "Arrays and rand/range are outside this pass. Already-qualified calls are preserved. " + + "Range and zero-argument rand are migrated. Already-qualified calls are preserved. " + "Malformed FQL is left unchanged and reported for manual follow-up.\n\n" + "The command performs only documented mechanical import, dependency, and source changes. " + "It does not convert application logic or arbitrary APIs to native Ferret v2 equivalents.", diff --git a/cmd/internal/migrate/stdlib_test.go b/cmd/internal/migrate/stdlib_test.go index 80d7baa..7d9d395 100644 --- a/cmd/internal/migrate/stdlib_test.go +++ b/cmd/internal/migrate/stdlib_test.go @@ -11,8 +11,8 @@ import ( func TestMigrateRunStdlibPreviewAndApply(t *testing.T) { path := filepath.Join(t.TempDir(), "query.fql") - before := "return [abs(-2), average(xs)]" - want := "return [math::abs(-2), average(xs)]" + before := "return [shift(union(a, b)), append(a, v, true)]" + want := "return [arrays::slice(arrays::concat(a, b), 1), append(a, v, true)]" if err := os.WriteFile(path, []byte(before), 0o600); err != nil { t.Fatal(err) } @@ -29,7 +29,7 @@ func TestMigrateRunStdlibPreviewAndApply(t *testing.T) { t.Fatal(err) } - if !strings.Contains(stderr, "query.fql:1: average(...)") || !strings.Contains(stderr, "heterogeneous") { + if !strings.Contains(stderr, "query.fql:1: append(...)") || !strings.Contains(stderr, "existing duplicates") { t.Fatalf("missing manual warning in mode %s: %s", mode, stderr) } @@ -39,7 +39,7 @@ func TestMigrateRunStdlibPreviewAndApply(t *testing.T) { strings.Contains(stdout, "Scanned") || !strings.Contains(stderr, "Scanned 1 FQL file") { t.Fatalf("print mixed diff and diagnostics: stdout=%q stderr=%q", stdout, stderr) } - } else if !strings.Contains(stdout, "query.fql") || strings.Contains(stdout, "average(...)") { + } else if !strings.Contains(stdout, "query.fql") || strings.Contains(stdout, "append(...)") { t.Fatalf("unexpected human output: %s", stdout) } @@ -63,7 +63,7 @@ func TestMigrateRunStdlibPreviewAndApply(t *testing.T) { t.Fatal(err) } - if stdout != "" || !strings.Contains(stderr, "average(...)") || !strings.Contains(stderr, "No safe automatic changes available") { + if stdout != "" || !strings.Contains(stderr, "append(...)") || !strings.Contains(stderr, "No safe automatic changes available") { t.Fatalf("manual-only second run: stdout=%q stderr=%q", stdout, stderr) } } @@ -74,7 +74,7 @@ func TestMigrateRunStdlibHelp(t *testing.T) { t.Fatal(err) } - for _, text := range []string{"safe legacy stdlib", "encoding, crypto, path, object, datetime, and math", "local function or use alias", "Arrays and rand/range"} { + for _, text := range []string{"safe legacy stdlib", "encoding, crypto, path, arrays, object, datetime, math, and random", "local function or use alias", "Range and zero-argument rand"} { if !strings.Contains(stdout, text) { t.Fatalf("help does not contain %q: %s", text, stdout) } @@ -98,6 +98,18 @@ func TestMigrateCheckStdlibReportsFindingsWithoutWriting(t *testing.T) { " help: legacy aggregate behavior is permissive for heterogeneous collections; " + "the strict math API requires a separate semantic migration\n\n", }, + { + name: "composed replacement", + input: "return keys(obj, true)", + diagnostic: ":1:8: Legacy stdlib call `keys` should use `arrays::sorted(object::keys(...))`.\n" + + " help: Preview automatic replacements with `ferret migrate run --print`. The final literal mode argument is removed.\n\n", + }, + { + name: "parameterized rand remains manual", + input: "return rand(10)", + diagnostic: ":1:8: Stdlib call `rand` needs manual review.\n" + + " help: parameterized legacy rand uses a historical rounded/floored range calculation (max/2 to max*2 for one argument) and has no behavior-preserving canonical call\n\n", + }, { name: "canonical source", input: `return object::has_key({ foo: "bar" }, "baz")`, diff --git a/internal/migration/compatibility_stdlib.go b/internal/migration/compatibility_stdlib.go index 501c987..eca57c5 100644 --- a/internal/migration/compatibility_stdlib.go +++ b/internal/migration/compatibility_stdlib.go @@ -10,7 +10,7 @@ import ( func checkFQLStdlib(src source.Source, program *fql.ProgramContext) ([]CompatibilityDiagnostic, error) { var diagnostics []CompatibilityDiagnostic for _, finding := range analyzeFQLStdlib(program) { - name := finding.name + name := finding.call.FunctionName() spelling := name.GetText() span, ok := fqlByteSpan(src.Content(), source.Span{ @@ -28,12 +28,16 @@ func checkFQLStdlib(src source.Source, program *fql.ProgramContext) ([]Compatibi diagnostic := CompatibilityDiagnostic{ Path: src.Name(), - Message: fmt.Sprintf("Legacy stdlib call `%s` should use `%s`.", spelling, finding.target), + Message: fmt.Sprintf("Legacy stdlib call `%s` should use `%s`.", spelling, finding.description()), Help: "Preview automatic replacements with `ferret migrate run --print`.", Line: position.Line, Column: position.Column, Kind: CompatibilityDiagnosticIssue, } + if finding.dropLast { + diagnostic.Help += " The final literal mode argument is removed." + } + if finding.reason != "" { diagnostic.Message = fmt.Sprintf("Stdlib call `%s` needs manual review.", spelling) diagnostic.Help = finding.reason diff --git a/internal/migration/compatibility_stdlib_test.go b/internal/migration/compatibility_stdlib_test.go index 5350e4c..d2347ea 100644 --- a/internal/migration/compatibility_stdlib_test.go +++ b/internal/migration/compatibility_stdlib_test.go @@ -51,13 +51,11 @@ func TestCheckFQLStdlibManualReview(t *testing.T) { tests := []struct{ call, reason string }{ {`join("a", "b")`, "legacy path::join"}, {`join(["a", "b"], ",")`, "modern global string joining"}, - {"keys()", "only keys(obj)"}, - {"keys(obj, true)", "argument-aware"}, - {"keys(obj, false)", "argument-aware"}, - {"keys(obj, option)", "argument-aware"}, - {"keys(obj, true, extra)", "argument-aware"}, + {"keys()", "arity"}, + {"keys(obj, option)", "dynamic sort mode"}, + {"keys(obj, true, extra)", "arity"}, {`date_compare(a, b, "year")`, "component-range"}, - {`date_diff(a, b, "day")`, "integer/floating"}, + {`date_diff(a, b, "day")`, "truncates toward zero"}, {"average(xs)", "heterogeneous"}, {"sum(xs)", "heterogeneous"}, {"min(xs)", "heterogeneous"}, @@ -138,7 +136,7 @@ func TestCheckFQLStdlibIgnoresUnrelatedSource(t *testing.T) { let has = "json_parse()" return { has, abs: "date_diff()", values: [ math::abs(-1), CUSTOM::HAS(obj, "key"), object::keys(obj), - union(a, b), push(a, 1), nth(a, 0), rand(), range(1, 10) + arrays::concat(a, b), arrays::append(a, 1), arrays::at(a, 0), random::float(), arrays::range(1, 10) ] }` if diagnostics := checkStdlibTestSource(t, input); len(diagnostics) != 0 { t.Fatalf("unrelated source has findings: %#v", diagnostics) @@ -228,6 +226,8 @@ func BenchmarkCheckFQLStdlibCompatibility(b *testing.B) { name, input string findings int }{ + {"structural", "return [keys(json_parse(body), true), shift(union(a, b)), position(a, v, false)]", 500}, + {"structural_canonical", "return [arrays::sorted(object::keys(encoding::json_parse(body))), arrays::slice(arrays::concat(a, b), 1), arrays::contains(a, v)]", 0}, {"legacy", "return [json_parse(text), sha1(text), base(path), has(obj, key), date_year(dt), abs(-1), average(xs)]", 700}, {"canonical", "return [encoding::json_parse(text), crypto::sha1(text), path::base(path), object::has_key(obj, key), datetime::year(dt), math::abs(-1)]", 0}, } { diff --git a/internal/migration/fql_stdlib.go b/internal/migration/fql_stdlib.go index 43dfc18..4a3691d 100644 --- a/internal/migration/fql_stdlib.go +++ b/internal/migration/fql_stdlib.go @@ -12,9 +12,10 @@ import ( type ( fqlStdlibRule struct { - target string - reason string - special func(fql.IFunctionCallContext) (target, reason string) + target string + namespace string + reason string + special func(fql.IFunctionCallContext) fqlStdlibRewrite } fqlStdlibCalls struct { @@ -24,92 +25,116 @@ type ( } fqlStdlibFinding struct { - name fql.IFunctionNameContext - target string - reason string + call fql.IFunctionCallContext + fqlStdlibRewrite } ) const fqlAggregateReason = "legacy aggregate behavior is permissive for heterogeneous collections; " + "the strict math API requires a separate semantic migration" -// Only behavior-preserving call-target replacements belong here. Arrays and -// rand/range are intentionally outside this pass, including manual diagnostics. +// Rules declare canonical namespaces separately from generated source text. +// Only behavior-preserving rewrites are automatic; semantic differences stay manual. var fqlStdlibRules = map[string]fqlStdlibRule{ - "json_parse": {target: "encoding::json_parse"}, - "json_stringify": {target: "encoding::json_stringify"}, - "encode_uri_component": {target: "encoding::query_escape"}, - "decode_uri_component": {target: "encoding::query_unescape"}, - "to_base64": {target: "encoding::base64_encode"}, - "from_base64": {target: "encoding::base64_decode"}, - "escape_html": {target: "encoding::html_escape"}, - "unescape_html": {target: "encoding::html_unescape"}, - - "md5": {target: "crypto::md5"}, - "sha1": {target: "crypto::sha1"}, - "sha512": {target: "crypto::sha512"}, - "random_token": {target: "crypto::random_token"}, - - "base": {target: "path::base"}, - "clean": {target: "path::clean"}, - "dir": {target: "path::dir"}, - "ext": {target: "path::ext"}, - "is_abs": {target: "path::is_abs"}, - "separate": {target: "path::separate"}, - "match": {target: "path::match"}, + "json_parse": {target: "encoding::json_parse", namespace: "encoding"}, + "json_stringify": {target: "encoding::json_stringify", namespace: "encoding"}, + "encode_uri_component": {target: "encoding::query_escape", namespace: "encoding"}, + "decode_uri_component": {target: "encoding::query_unescape", namespace: "encoding"}, + "to_base64": {target: "encoding::base64_encode", namespace: "encoding"}, + "from_base64": {target: "encoding::base64_decode", namespace: "encoding"}, + "escape_html": {target: "encoding::html_escape", namespace: "encoding"}, + "unescape_html": {target: "encoding::html_unescape", namespace: "encoding"}, + + "md5": {target: "crypto::md5", namespace: "crypto"}, + "sha1": {target: "crypto::sha1", namespace: "crypto"}, + "sha512": {target: "crypto::sha512", namespace: "crypto"}, + "random_token": {target: "crypto::random_token", namespace: "crypto"}, + + "base": {target: "path::base", namespace: "path"}, + "clean": {target: "path::clean", namespace: "path"}, + "dir": {target: "path::dir", namespace: "path"}, + "ext": {target: "path::ext", namespace: "path"}, + "is_abs": {target: "path::is_abs", namespace: "path"}, + "separate": {target: "path::separate", namespace: "path"}, + "match": {target: "path::match", namespace: "path"}, // There is no source-version marker: rerunning migration must preserve modern JOIN. "join": {reason: "join may mean legacy path::join or modern global string joining; the source version is ambiguous"}, - "values": {target: "object::values"}, - "has": {target: "object::has_key"}, - "zip": {target: "object::zip"}, - "keep_keys": {target: "object::keep_keys"}, - "merge": {target: "object::merge"}, - "merge_recursive": {target: "object::merge_deep"}, + "first": {target: "arrays::first", namespace: "arrays"}, + "flatten": {target: "arrays::flatten", namespace: "arrays"}, + "last": {target: "arrays::last", namespace: "arrays"}, + "sorted": {target: "arrays::sorted", namespace: "arrays"}, + "unique": {target: "arrays::unique", namespace: "arrays"}, + "nth": {target: "arrays::at", namespace: "arrays"}, + "remove_values": {target: "arrays::remove_any", namespace: "arrays"}, + "slice": {target: "arrays::slice", namespace: "arrays"}, + "intersection": {target: "arrays::intersection", namespace: "arrays"}, + "minus": {target: "arrays::difference", namespace: "arrays"}, + "union": {target: "arrays::concat", namespace: "arrays"}, + "union_distinct": {target: "arrays::union", namespace: "arrays"}, + "append": {special: migrateFQLAppend}, + "push": {special: migrateFQLAppend}, + "position": {special: migrateFQLPosition}, + "remove_value": {special: migrateFQLRemoveValue}, + "sorted_unique": {special: migrateFQLSortedUnique}, + "shift": {special: migrateFQLShift}, + "outersection": {special: migrateFQLOutersection}, + "range": {special: migrateFQLRange}, + "rand": {special: migrateFQLRand}, + "pop": {reason: "legacy pop returns an immutable copy without the last element; a length-dependent replacement must preserve argument evaluation count"}, + "unshift": {reason: "legacy unshift evaluates the array before the value; arrays::concat([value], array) reverses evaluation order, and unique mode has additional semantics"}, + "remove_nth": {reason: "legacy remove_nth delegates removal to the copied host list; arrays::remove_at returns an unchanged copy for negative or out-of-range indexes"}, + + "values": {target: "object::values", namespace: "object"}, + "has": {target: "object::has_key", namespace: "object"}, + "zip": {target: "object::zip", namespace: "object"}, + "keep_keys": {target: "object::keep_keys", namespace: "object"}, + "merge": {target: "object::merge", namespace: "object"}, + "merge_recursive": {target: "object::merge_deep", namespace: "object"}, "keys": {special: migrateFQLKeys}, - "now": {target: "datetime::now"}, - "date": {target: "datetime::parse"}, - "date_dayofweek": {target: "datetime::day_of_week"}, - "date_year": {target: "datetime::year"}, - "date_month": {target: "datetime::month"}, - "date_day": {target: "datetime::day"}, - "date_hour": {target: "datetime::hour"}, - "date_minute": {target: "datetime::minute"}, - "date_second": {target: "datetime::second"}, - "date_millisecond": {target: "datetime::millisecond"}, - "date_dayofyear": {target: "datetime::day_of_year"}, - "date_leapyear": {target: "datetime::is_leap_year"}, - "date_quarter": {target: "datetime::quarter"}, - "date_days_in_month": {target: "datetime::days_in_month"}, - "date_format": {target: "datetime::format"}, - "date_add": {target: "datetime::add"}, - "date_subtract": {target: "datetime::subtract"}, + "now": {target: "datetime::now", namespace: "datetime"}, + "date": {target: "datetime::parse", namespace: "datetime"}, + "date_dayofweek": {target: "datetime::day_of_week", namespace: "datetime"}, + "date_year": {target: "datetime::year", namespace: "datetime"}, + "date_month": {target: "datetime::month", namespace: "datetime"}, + "date_day": {target: "datetime::day", namespace: "datetime"}, + "date_hour": {target: "datetime::hour", namespace: "datetime"}, + "date_minute": {target: "datetime::minute", namespace: "datetime"}, + "date_second": {target: "datetime::second", namespace: "datetime"}, + "date_millisecond": {target: "datetime::millisecond", namespace: "datetime"}, + "date_dayofyear": {target: "datetime::day_of_year", namespace: "datetime"}, + "date_leapyear": {target: "datetime::is_leap_year", namespace: "datetime"}, + "date_quarter": {target: "datetime::quarter", namespace: "datetime"}, + "date_days_in_month": {target: "datetime::days_in_month", namespace: "datetime"}, + "date_format": {target: "datetime::format", namespace: "datetime"}, + "date_add": {target: "datetime::add", namespace: "datetime"}, + "date_subtract": {target: "datetime::subtract", namespace: "datetime"}, "date_compare": {reason: "legacy component-range comparison semantics differ from datetime::same; " + "it is not a drop-in replacement"}, - "date_diff": {reason: "legacy integer/floating behavior differs from the canonical datetime::diff contract"}, - - "pi": {target: "math::pi"}, - "abs": {target: "math::abs"}, - "acos": {target: "math::acos"}, - "asin": {target: "math::asin"}, - "atan": {target: "math::atan"}, - "atan2": {target: "math::atan2"}, - "ceil": {target: "math::ceil"}, - "cos": {target: "math::cos"}, - "degrees": {target: "math::degrees"}, - "exp": {target: "math::exp"}, - "exp2": {target: "math::exp2"}, - "floor": {target: "math::floor"}, - "log": {target: "math::log"}, - "log2": {target: "math::log2"}, - "log10": {target: "math::log10"}, - "pow": {target: "math::pow"}, - "radians": {target: "math::radians"}, - "round": {target: "math::round"}, - "sin": {target: "math::sin"}, - "sqrt": {target: "math::sqrt"}, - "tan": {target: "math::tan"}, + "date_diff": {special: migrateFQLDateDiff}, + + "pi": {target: "math::pi", namespace: "math"}, + "abs": {target: "math::abs", namespace: "math"}, + "acos": {target: "math::acos", namespace: "math"}, + "asin": {target: "math::asin", namespace: "math"}, + "atan": {target: "math::atan", namespace: "math"}, + "atan2": {target: "math::atan2", namespace: "math"}, + "ceil": {target: "math::ceil", namespace: "math"}, + "cos": {target: "math::cos", namespace: "math"}, + "degrees": {target: "math::degrees", namespace: "math"}, + "exp": {target: "math::exp", namespace: "math"}, + "exp2": {target: "math::exp2", namespace: "math"}, + "floor": {target: "math::floor", namespace: "math"}, + "log": {target: "math::log", namespace: "math"}, + "log2": {target: "math::log2", namespace: "math"}, + "log10": {target: "math::log10", namespace: "math"}, + "pow": {target: "math::pow", namespace: "math"}, + "radians": {target: "math::radians", namespace: "math"}, + "round": {target: "math::round", namespace: "math"}, + "sin": {target: "math::sin", namespace: "math"}, + "sqrt": {target: "math::sqrt", namespace: "math"}, + "tan": {target: "math::tan", namespace: "math"}, "average": {reason: fqlAggregateReason}, "sum": {reason: fqlAggregateReason}, @@ -123,20 +148,11 @@ var fqlStdlibRules = map[string]fqlStdlibRule{ "variance_sample": {reason: fqlAggregateReason}, } -func migrateFQLKeys(call fql.IFunctionCallContext) (string, string) { - args := call.ArgumentList() - if args != nil && len(args.AllExpression()) == 1 { - return "object::keys", "" - } - - return "", "only keys(obj) has a mechanical replacement; other arities require separate argument-aware review" -} - func planFQLStdlib(src source.Source, program *fql.ProgramContext) ([]fqlSourceEdit, []ManualAction, error) { var edits []fqlSourceEdit var actions []ManualAction for _, finding := range analyzeFQLStdlib(program) { - name := finding.name + name := finding.call.FunctionName() spelling := name.GetText() if finding.reason != "" { actions = append(actions, ManualAction{ @@ -149,15 +165,12 @@ func planFQLStdlib(src source.Source, program *fql.ProgramContext) ([]fqlSourceE continue } - span, ok := fqlByteSpan(src.Content(), source.Span{ - Start: name.GetStart().GetStart(), - End: name.GetStop().GetStop() + 1, - }) - if !ok || span.End <= span.Start { - return nil, nil, fmt.Errorf("locate stdlib call %s in Ferret source", spelling) + callEdits, err := finding.edits(src.Content(), finding.call) + if err != nil { + return nil, actions, err } - edits = append(edits, fqlSourceEdit{start: span.Start, end: span.End, text: finding.target}) + edits = append(edits, callEdits...) } return edits, actions, nil @@ -182,20 +195,28 @@ func analyzeFQLStdlib(program *fql.ProgramContext) []fqlStdlibFinding { continue } - target, reason := rule.target, rule.reason + rewrite := fqlStdlibRewrite{target: rule.target, reason: rule.reason} + if rule.namespace != "" { + rewrite.namespaces = []string{rule.namespace} + } + if rule.special != nil { - target, reason = rule.special(call) + rewrite = rule.special(call) } if info.locals[strings.ToLower(spelling)] { - reason = "a file-local function declaration or use function alias may resolve this call; the stdlib target is ambiguous" - } else if namespace, _, _ := strings.Cut(target, "::"); namespace != "" { - if alias, exists := info.aliases[namespace]; exists && alias != namespace { - reason = fmt.Sprintf("use alias %q would redirect the replacement %s; the canonical target is ambiguous", namespace, target) + rewrite.reason = "a file-local function declaration or use function alias may resolve this call; the stdlib target is ambiguous" + } else { + for _, namespace := range rewrite.namespaces { + if alias, exists := info.aliases[namespace]; exists && alias != namespace { + rewrite.reason = fmt.Sprintf("use alias %q would redirect the replacement %s; the canonical target is ambiguous", namespace, rewrite.description()) + + break + } } } - findings = append(findings, fqlStdlibFinding{name: name, target: target, reason: reason}) + findings = append(findings, fqlStdlibFinding{call: call, fqlStdlibRewrite: rewrite}) } return findings diff --git a/internal/migration/fql_stdlib_equivalence_test.go b/internal/migration/fql_stdlib_equivalence_test.go new file mode 100644 index 0000000..6cb0842 --- /dev/null +++ b/internal/migration/fql_stdlib_equivalence_test.go @@ -0,0 +1,179 @@ +package migration + +import ( + "context" + "encoding/json" + "reflect" + "slices" + "testing" + + ferret "github.com/MontFerret/ferret/v2" + "github.com/MontFerret/ferret/v2/pkg/runtime" + "github.com/MontFerret/ferret/v2/pkg/source" + "github.com/MontFerret/ferret/v2/pkg/stdlib" +) + +func TestFQLStdlibRuntimeEquivalence(t *testing.T) { + tests := []struct { + expression, want, check string + calls []string + }{ + {`position(observe("array", a), observe("value", 3))`, "true", "is_bool(result)", []string{"array", "value"}}, + {`position(a, 9, false)`, "false", "is_bool(result)", nil}, + {`position(a, 3, true)`, "0", "is_int(result)", nil}, + {`position(a, 9, true)`, "-1", "is_int(result)", nil}, + {`sorted_unique(observe("array", a))`, "[1,2,3]", "is_array(result)", []string{"array"}}, + {`sorted_unique([])`, "[]", "is_array(result)", nil}, + {`shift(observe("array", a))`, "[1,3,2]", "is_array(result)", []string{"array"}}, + {`shift([])`, "[]", "is_array(result)", nil}, + {`shift([1])`, "[]", "is_array(result)", nil}, + {`append(observe("array", a), observe("value", 3), false)`, "[3,1,3,2,3]", "is_array(result)", []string{"array", "value"}}, + {`push(a, 5, false)`, "[3,1,3,2,5]", "is_array(result)", nil}, + {`remove_value(observe("array", a), observe("value", 3), -17)`, "[1,2]", "is_array(result)", []string{"array", "value"}}, + {`remove_value([], 3, -1)`, "[]", "is_array(result)", nil}, + {`outersection(observe("left", a), observe("right", b))`, "[1,2,4]", "is_array(result)", []string{"left", "right"}}, + {`outersection(a, a)`, "[]", "is_array(result)", nil}, + {`union(a, b)`, "[3,1,3,2,3,4,4]", "is_array(result)", nil}, + {`union_distinct(a, b)`, "[3,1,2,4]", "is_array(result)", nil}, + {`date_diff(left, right, "seconds", true)`, "1.5", "is_float(result)", nil}, + {`date_diff(right, left, "second", (true))`, "-1.5", "is_float(result)", nil}, + {`date_diff(left, left, "hour", true)`, "0", "is_float(result)", nil}, + {`range(1, 3)`, "[1,2,3]", "is_array(result) AND is_float(arrays::first(result))", nil}, + {`range(3, 1, -1)`, "[3,2,1]", "is_array(result) AND is_float(arrays::first(result))", nil}, + {`range(0, 1, 0.5)`, "[0,0.5,1]", "is_array(result) AND is_float(arrays::first(result))", nil}, + {`rand()`, "", "is_float(result) AND result >= 0 AND result < 1", nil}, + {`sorted_unique(shift(a))`, "[1,2,3]", "is_array(result)", nil}, + {`sorted_unique(observe("array", null)) ON ERROR RETRY 1 OR RETURN []`, "[]", "is_array(result)", []string{"array", "array"}}, + {`shift(observe("array", null)) ON ERROR RETURN []`, "[]", "is_array(result)", []string{"array"}}, + {`sorted_unique(null)?`, "null", "is_none(result)", nil}, + } + + for _, test := range tests { + t.Run(test.expression, func(t *testing.T) { + var calls []string + engine, err := ferret.New(ferret.WithStdlib(stdlib.Full()), ferret.WithFunctionsRegistrar(func(ns runtime.Namespace) { + ns.Function().A2().Add("observe", func(_ context.Context, label, value runtime.Value) (runtime.Value, error) { + calls = append(calls, string(label.(runtime.String))) + + return value, nil + }) + })) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + if err := engine.Close(); err != nil { + t.Errorf("close engine: %v", err) + } + }) + + input := `let a = [3, 1, 3, 2] +let b = [3, 4, 4] +let left = datetime::parse("2026-09-15T00:00:00Z") +let right = datetime::parse("2026-09-15T00:00:01.500Z") +let result = ` + test.expression + ` +return [result, ` + test.check + `, a, b, random::float()]` + migrated, err := migrateFQLSource(source.New("query.fql", input)) + if err != nil || !migrated.Changed || len(migrated.ManualActions) != 0 { + t.Fatalf("migration failed: %#v, %v", migrated, err) + } + + var previous string + for _, query := range []string{input, string(migrated.Data)} { + calls = nil + + output, err := engine.Run(t.Context(), source.New("query.fql", query), ferret.WithSessionRandomSeed(42)) + if err != nil { + t.Fatal(err) + } + + var parts []json.RawMessage + if err := json.Unmarshal(output.Content, &parts); err != nil { + t.Fatal(err) + } + + if len(parts) != 5 || string(parts[1]) != "true" || string(parts[2]) != "[3,1,3,2]" || string(parts[3]) != "[3,4,4]" { + t.Fatalf("type or source immutability changed: %s", output.Content) + } + + if test.want != "" && string(parts[0]) != test.want { + t.Fatalf("result = %s, want %s", parts[0], test.want) + } + + if !reflect.DeepEqual(calls, test.calls) { + t.Fatalf("evaluation order/count = %v, want %v", calls, test.calls) + } + + if previous != "" && previous != string(output.Content) { + t.Fatalf("runtime result or RNG consumption changed: old=%s new=%s", previous, output.Content) + } + + previous = string(output.Content) + } + }) + } +} + +func TestFQLStdlibHistoricalKeysModes(t *testing.T) { + // Core alpha.52 objects/keys_test.go requires ascending keys for true and + // unordered membership for false. Alpha.54 no longer registers keys/2. + for _, sorted := range []bool{false, true} { + for _, object := range []string{`{ z: 1, a: 2, m: 3 }`, `{}`} { + mode := "false" + if sorted { + mode = "true" + } + + t.Run(object+"/"+mode, func(t *testing.T) { + calls := 0 + engine, err := ferret.New(ferret.WithStdlib(stdlib.Full()), ferret.WithFunctionsRegistrar(func(ns runtime.Namespace) { + ns.Function().A1().Add("observe", func(_ context.Context, value runtime.Value) (runtime.Value, error) { + calls++ + + return value, nil + }) + })) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + if err := engine.Close(); err != nil { + t.Errorf("close engine: %v", err) + } + }) + + input := "let obj = " + object + "\nreturn keys(observe(obj), " + mode + ")" + result, err := migrateFQLSource(source.New("query.fql", input)) + if err != nil || !result.Changed { + t.Fatalf("migrate keys: %#v, %v", result, err) + } + + output, err := engine.Run(t.Context(), source.New("query.fql", string(result.Data))) + if err != nil { + t.Fatal(err) + } + + var keys []string + if err := json.Unmarshal(output.Content, &keys); err != nil { + t.Fatal(err) + } + + if sorted && !slices.IsSorted(keys) { + t.Fatalf("keys are not ascending: %v", keys) + } + + slices.Sort(keys) + var want []string + if object != "{}" { + want = []string{"a", "m", "z"} + } + + if !slices.Equal(keys, want) || calls != 1 { + t.Fatalf("keys=%v, want=%v; evaluations=%d", keys, want, calls) + } + }) + } + } +} diff --git a/internal/migration/fql_stdlib_integration_test.go b/internal/migration/fql_stdlib_integration_test.go index 888fb85..1b9858a 100644 --- a/internal/migration/fql_stdlib_integration_test.go +++ b/internal/migration/fql_stdlib_integration_test.go @@ -14,8 +14,8 @@ import ( func TestMigratorStdlibStandaloneModes(t *testing.T) { root := t.TempDir() path := filepath.Join(root, "query.fql") - before := "// keep location\nreturn [abs(-2), keys(obj, true)]" - want := "// keep location\nreturn [math::abs(-2), keys(obj, true)]" + before := "// keep location\nreturn [abs(-2), keys(obj, mode)]" + want := "// keep location\nreturn [math::abs(-2), keys(obj, mode)]" if err := os.WriteFile(path, []byte(before), 0o600); err != nil { t.Fatal(err) } @@ -144,7 +144,7 @@ func TestMigratorPreservesManualActionsOnFormattingFailure(t *testing.T) { before += "let comparison = date_compare(left, right, \"day\")\n" } - before += "return sha1 /* preserve me */ (value)" + before += "return sorted_unique /* preserve me */ (value)" writeMigrationTargetFile(t, root, "query.fql", before) migrator, _ := newFixtureMigrator() var previous []ManualAction @@ -228,20 +228,22 @@ func TestMigratorStdlibRollsBackOnCommitFailure(t *testing.T) { } func BenchmarkPlanFQLStdlibSourceChanges(b *testing.B) { - for _, canonical := range []bool{false, true} { - name := "legacy" - content := "let obj = json_parse(body)\nlet digest = sha1(json_stringify(obj))\nreturn [abs(-2), keys(obj), base(path), date_year(date(text))]" - if canonical { - name = "canonical" - content = "let obj = encoding::json_parse(body)\nlet digest = crypto::sha1(encoding::json_stringify(obj))\nreturn [math::abs(-2), object::keys(obj), path::base(path), datetime::year(datetime::parse(text))]" - } + for _, test := range []struct { + name, content string + canonical bool + }{ + {"legacy", "let obj = json_parse(body)\nlet digest = sha1(json_stringify(obj))\nreturn [abs(-2), keys(obj), base(path), date_year(date(text))]", false}, + {"canonical", "let obj = encoding::json_parse(body)\nlet digest = crypto::sha1(encoding::json_stringify(obj))\nreturn [math::abs(-2), object::keys(obj), path::base(path), datetime::year(datetime::parse(text))]", true}, + {"structural", "return [keys(json_parse(body), true), shift(union(a, b)), position(a, v, false)]", false}, + {"structural_canonical", "return [arrays::sorted(object::keys(encoding::json_parse(body))), arrays::slice(arrays::concat(a, b), 1), arrays::contains(a, v)]", true}, + } { - b.Run(name, func(b *testing.B) { + b.Run(test.name, func(b *testing.B) { root := b.TempDir() files := make([]string, 100) for i := range files { files[i] = filepath.Join(root, fmt.Sprintf("query_%03d.fql", i)) - if err := os.WriteFile(files[i], []byte(content), 0o644); err != nil { + if err := os.WriteFile(files[i], []byte(test.content), 0o644); err != nil { b.Fatal(err) } } @@ -257,7 +259,7 @@ func BenchmarkPlanFQLStdlibSourceChanges(b *testing.B) { } want := len(files) - if canonical { + if test.canonical { want = 0 } diff --git a/internal/migration/fql_stdlib_literals.go b/internal/migration/fql_stdlib_literals.go new file mode 100644 index 0000000..f86a004 --- /dev/null +++ b/internal/migration/fql_stdlib_literals.go @@ -0,0 +1,87 @@ +package migration + +import ( + "strconv" + "strings" + + "github.com/MontFerret/ferret/v2/pkg/parser/fql" +) + +func fqlCallArguments(call fql.IFunctionCallContext) []fql.IExpressionContext { + if args := call.ArgumentList(); args != nil { + return args.AllExpression() + } + + return nil +} + +// Only descend through grouping and the grammar's single-child expression +// layers. Operators, recovery tails, and other expressions are never evaluated. +func fqlLiteralAtom(expression fql.IExpressionContext) fql.IExpressionAtomContext { + for expression != nil && expression.GetChildCount() == 1 { + predicate := expression.Predicate() + if predicate == nil || predicate.GetChildCount() != 1 { + return nil + } + + atom := predicate.ExpressionAtom() + if atom == nil { + return nil + } + + if atom.OpenParen() != nil && atom.GetChildCount() == 3 && atom.Expression() != nil { + expression = atom.Expression() + + continue + } + + if atom.GetChildCount() == 1 && atom.Literal() != nil { + return atom + } + + return nil + } + + return nil +} + +func fqlBooleanLiteral(expression fql.IExpressionContext) (bool, bool) { + atom := fqlLiteralAtom(expression) + if atom == nil || atom.Literal().BooleanLiteral() == nil { + return false, false + } + + return strings.EqualFold(atom.Literal().BooleanLiteral().GetText(), "true"), true +} + +func fqlNegativeIntegerLiteral(expression fql.IExpressionContext) bool { + // Parentheses may surround the entire signed literal as well as its operand. + for expression != nil && expression.GetChildCount() == 1 { + predicate := expression.Predicate() + if predicate == nil || predicate.GetChildCount() != 1 { + return false + } + + atom := predicate.ExpressionAtom() + if atom == nil || atom.OpenParen() == nil || atom.GetChildCount() != 3 || atom.Expression() == nil { + return false + } + + expression = atom.Expression() + } + + if expression == nil || expression.UnaryOperator() == nil || expression.UnaryOperator().Minus() == nil { + return false + } + + atom := fqlLiteralAtom(expression.GetRight()) + if atom == nil || atom.Literal().IntegerLiteral() == nil { + return false + } + + // Core compiles the unsigned operand with Atoi before applying unary minus. + // Reject overflow rather than removing a literal that would fail compilation. + value, err := strconv.Atoi(atom.Literal().IntegerLiteral().GetText()) + + return err == nil && value > 0 +} diff --git a/internal/migration/fql_stdlib_modes.go b/internal/migration/fql_stdlib_modes.go new file mode 100644 index 0000000..9f082d6 --- /dev/null +++ b/internal/migration/fql_stdlib_modes.go @@ -0,0 +1,170 @@ +package migration + +import "github.com/MontFerret/ferret/v2/pkg/parser/fql" + +func migrateFQLKeys(call fql.IFunctionCallContext) fqlStdlibRewrite { + rewrite := fqlStdlibRewrite{target: "object::keys", namespaces: []string{"object"}} + + args := fqlCallArguments(call) + if len(args) == 1 { + return rewrite + } + + if len(args) != 2 { + return fqlStdlibRewrite{reason: "keys supports one object and an optional literal Boolean sort flag; this arity requires manual review"} + } + + mode, ok := fqlBooleanLiteral(args[1]) + if !ok { + return fqlStdlibRewrite{reason: "legacy keys selects sorted or unordered keys using its Boolean flag; a dynamic sort mode cannot select a fixed canonical replacement"} + } + + rewrite.dropLast = true + + if mode { + rewrite.wrapper = "arrays::sorted" + rewrite.namespaces = []string{"object", "arrays"} + } + + return rewrite +} + +func migrateFQLPosition(call fql.IFunctionCallContext) fqlStdlibRewrite { + rewrite := fqlStdlibRewrite{target: "arrays::contains", namespaces: []string{"arrays"}} + + args := fqlCallArguments(call) + if len(args) == 2 { + return rewrite + } + + if len(args) != 3 { + return fqlStdlibRewrite{reason: "position supports two arguments and an optional literal Boolean index mode; this arity requires manual review"} + } + + mode, ok := fqlBooleanLiteral(args[2]) + if !ok { + return fqlStdlibRewrite{reason: "legacy position returns a Boolean or an index depending on its flag; a dynamic mode cannot select arrays::contains or arrays::index_of"} + } + + rewrite.dropLast = true + + if mode { + rewrite.target = "arrays::index_of" + } + + return rewrite +} + +func migrateFQLAppend(call fql.IFunctionCallContext) fqlStdlibRewrite { + rewrite := fqlStdlibRewrite{target: "arrays::append", namespaces: []string{"arrays"}} + + args := fqlCallArguments(call) + if len(args) == 2 { + return rewrite + } + + if len(args) != 3 { + return fqlStdlibRewrite{reason: "append/push supports two arguments and an optional Boolean unique mode; this arity requires manual review"} + } + + mode, ok := fqlBooleanLiteral(args[2]) + if !ok { + return fqlStdlibRewrite{reason: "legacy append/push has a conditional unique mode; a dynamic flag cannot select a behavior-preserving canonical call"} + } + + if mode { + return fqlStdlibRewrite{reason: "legacy unique append/push only suppresses the new value when already present; arrays::unique(arrays::append(...)) would also remove existing duplicates"} + } + + rewrite.dropLast = true + + return rewrite +} + +func migrateFQLRemoveValue(call fql.IFunctionCallContext) fqlStdlibRewrite { + rewrite := fqlStdlibRewrite{target: "arrays::remove", namespaces: []string{"arrays"}} + + args := fqlCallArguments(call) + if len(args) == 2 { + return rewrite + } + + if len(args) != 3 { + return fqlStdlibRewrite{reason: "remove_value supports two arguments and an optional integer removal limit; this arity requires manual review"} + } + + if !fqlNegativeIntegerLiteral(args[2]) { + return fqlStdlibRewrite{reason: "legacy remove_value uses negative integer limits for unlimited removal, zero for no removal, and positive limits for bounded removal; arrays::remove has no limit mode, so only a valid negative integer literal can be dropped"} + } + + rewrite.dropLast = true + + return rewrite +} + +func migrateFQLSortedUnique(call fql.IFunctionCallContext) fqlStdlibRewrite { + if len(fqlCallArguments(call)) != 1 { + return fqlStdlibRewrite{reason: "sorted_unique requires exactly one array to compose arrays::sorted(arrays::unique(...))"} + } + + return fqlStdlibRewrite{target: "arrays::unique", wrapper: "arrays::sorted", namespaces: []string{"arrays"}} +} + +func migrateFQLShift(call fql.IFunctionCallContext) fqlStdlibRewrite { + if len(fqlCallArguments(call)) != 1 { + return fqlStdlibRewrite{reason: "immutable shift requires exactly one array to replace it with arrays::slice(array, 1)"} + } + + return fqlStdlibRewrite{target: "arrays::slice", addLast: "1", namespaces: []string{"arrays"}} +} + +func migrateFQLOutersection(call fql.IFunctionCallContext) fqlStdlibRewrite { + count := len(fqlCallArguments(call)) + if count == 2 { + return fqlStdlibRewrite{target: "arrays::symmetric_difference", namespaces: []string{"arrays"}} + } + + if count < 2 { + return fqlStdlibRewrite{reason: "outersection requires at least two arrays; only exactly two inputs have a behavior-preserving canonical replacement"} + } + + return fqlStdlibRewrite{reason: "legacy outersection keeps values present in exactly one input; arrays::symmetric_difference uses odd-number-of-inputs semantics for three or more arrays"} +} + +func migrateFQLDateDiff(call fql.IFunctionCallContext) fqlStdlibRewrite { + args := fqlCallArguments(call) + if len(args) != 3 && len(args) != 4 { + return fqlStdlibRewrite{reason: "date_diff supports three arguments and an optional Boolean floating mode; this arity requires manual review"} + } + + if len(args) == 4 { + mode, ok := fqlBooleanLiteral(args[3]) + if ok && mode { + return fqlStdlibRewrite{target: "datetime::diff", dropLast: true, namespaces: []string{"datetime"}} + } + } + + return fqlStdlibRewrite{reason: "legacy date_diff truncates toward zero when its floating flag is false or omitted, while datetime::diff always returns a Float; only a literal true floating mode has a safe replacement"} +} + +func migrateFQLRange(call fql.IFunctionCallContext) fqlStdlibRewrite { + count := len(fqlCallArguments(call)) + if count != 2 && count != 3 { + return fqlStdlibRewrite{reason: "range requires start and end with an optional step; this arity requires manual review"} + } + + return fqlStdlibRewrite{target: "arrays::range", namespaces: []string{"arrays"}} +} + +func migrateFQLRand(call fql.IFunctionCallContext) fqlStdlibRewrite { + count := len(fqlCallArguments(call)) + if count == 0 { + return fqlStdlibRewrite{target: "random::float", namespaces: []string{"random"}} + } + + if count > 2 { + return fqlStdlibRewrite{reason: "legacy rand supports zero, one, or two arguments; this arity requires manual review"} + } + + return fqlStdlibRewrite{reason: "parameterized legacy rand uses a historical rounded/floored range calculation (max/2 to max*2 for one argument) and has no behavior-preserving canonical call"} +} diff --git a/internal/migration/fql_stdlib_rewrite.go b/internal/migration/fql_stdlib_rewrite.go new file mode 100644 index 0000000..0859c66 --- /dev/null +++ b/internal/migration/fql_stdlib_rewrite.go @@ -0,0 +1,105 @@ +package migration + +import ( + "fmt" + + "github.com/antlr4-go/antlr/v4" + + "github.com/MontFerret/ferret/v2/pkg/parser/fql" + "github.com/MontFerret/ferret/v2/pkg/source" +) + +// A rewrite changes only call boundaries and an optional trailing literal. +// Retained arguments remain in the original source, available for descendant edits. +type fqlStdlibRewrite struct { + target string + wrapper string + dropLast bool + addLast string + namespaces []string + reason string +} + +func (r fqlStdlibRewrite) description() string { + if r.wrapper != "" { + return r.wrapper + "(" + r.target + "(...))" + } + + if r.addLast != "" { + return r.target + "(..., " + r.addLast + ")" + } + + return r.target +} + +func (r fqlStdlibRewrite) edits(content string, call fql.IFunctionCallContext) ([]fqlSourceEdit, error) { + var edits []fqlSourceEdit + + add := func(start, stop antlr.Token, text string) error { + span, ok := fqlByteSpan(content, source.Span{Start: start.GetStart(), End: stop.GetStop() + 1}) + if !ok || span.End <= span.Start { + return fmt.Errorf("locate stdlib call %s in Ferret source", call.FunctionName().GetText()) + } + + edits = append(edits, fqlSourceEdit{start: span.Start, end: span.End, text: text}) + + return nil + } + + target := r.target + closing := ")" + if r.wrapper != "" { + target = r.wrapper + "(" + target + closing += ")" + } + + name := call.FunctionName() + if err := add(name.GetStart(), name.GetStop(), target); err != nil { + return nil, err + } + + args := call.ArgumentList() + + if r.dropLast { + expressions := args.AllExpression() + comma := args.Comma(len(expressions) - 2).GetSymbol() + if err := add(comma, comma, ""); err != nil { + return nil, err + } + + // Delete syntax tokens only: comments inside grouping parentheses or + // between a unary minus and its integer must survive argument removal. + last := expressions[len(expressions)-1] + stream := call.GetParser().GetTokenStream() + for i := last.GetStart().GetTokenIndex(); i <= last.GetStop().GetTokenIndex(); i++ { + token := stream.Get(i) + if token.GetChannel() != antlr.TokenDefaultChannel { + continue + } + + if err := add(token, token, ""); err != nil { + return nil, err + } + } + } + + if r.addLast != "" { + separator := ", " + if len(args.AllComma()) == len(args.AllExpression()) { + separator = "" + } + + closing = separator + r.addLast + closing + } + + if closing != ")" { + // Replace the delimiter instead of inserting after it: final-FOR edits + // may already insert at the end of this call, including in loop headers. + token := call.CloseParen().GetSymbol() + if err := add(token, token, closing); err != nil { + return nil, err + } + } + + return edits, nil +} diff --git a/internal/migration/fql_stdlib_structural_guards_test.go b/internal/migration/fql_stdlib_structural_guards_test.go new file mode 100644 index 0000000..99b30db --- /dev/null +++ b/internal/migration/fql_stdlib_structural_guards_test.go @@ -0,0 +1,76 @@ +package migration + +import ( + "strings" + "testing" + + "github.com/MontFerret/ferret/v2/pkg/source" +) + +func TestFQLStdlibStructuralResolutionGuards(t *testing.T) { + tests := []struct{ header, call, reason string }{ + {"func SHIFT(a) { return a }", "shift(a)", "file-local"}, + {"use custom::calculate as position", "POSITION(a, v, true)", "function alias"}, + {"use custom as arrays", "keys(o, true)", "arrays"}, + {"use custom as object", "keys(o, true)", "object"}, + {"use custom as arrays", "sorted_unique(a)", "arrays"}, + {"use custom as arrays", "shift(a)", "arrays"}, + {"use custom as arrays", "position(a, v, true)", "arrays"}, + {"use custom as arrays", "range(1, 2)", "arrays"}, + {"use custom as random", "rand()", "random"}, + {"use custom::calculate as random", "rand()", "random"}, + {"use custom as datetime", "date_diff(a, b, unit, true)", "datetime"}, + {"use custom as arrays\nuse arrays as arrays", "keys(o, true)", "arrays"}, + } + + for _, test := range tests { + t.Run(test.header+"/"+test.call, func(t *testing.T) { + input := test.header + "\nreturn " + test.call + result := assertFQLStdlibMigration(t, input, input, 1) + diagnostics := checkStdlibTestSource(t, input) + if len(diagnostics) != 1 || diagnostics[0].Help != result.ManualActions[0].Reason || + !strings.Contains(diagnostics[0].Help, test.reason) { + t.Fatalf("collision not shared by check/run: %#v, %#v", diagnostics, result.ManualActions) + } + }) + } + + for _, header := range []string{"use arrays as arrays", "use custom as ARRAYS", "use custom as unrelated"} { + assertFQLStdlibMigration(t, header+"\nreturn keys(o, true)", header+"\nreturn arrays::sorted(object::keys(o))", 0) + } + + assertFQLStdlibMigration(t, "use custom as arrays\nreturn keys(o, false)", "use custom as arrays\nreturn object::keys(o)", 0) + assertFQLStdlibMigration(t, "use custom as random\nreturn random::float()", "use custom as random\nreturn random::float()", 0) + assertFQLStdlibMigration(t, "use custom as arrays\nreturn keys(json_parse(body), true)", "use custom as arrays\nreturn keys(encoding::json_parse(body), true)", 1) +} + +func TestFQLStdlibStructuralDiagnosticDescriptions(t *testing.T) { + for _, test := range []struct{ call, description string }{ + {"sorted_unique(a)", "arrays::sorted(arrays::unique(...))"}, + {"keys(o, true)", "arrays::sorted(object::keys(...))"}, + {"shift(a)", "arrays::slice(..., 1)"}, + } { + diagnostics := checkStdlibTestSource(t, "return "+test.call) + if len(diagnostics) != 1 || !strings.Contains(diagnostics[0].Message, "`"+test.description+"`") { + t.Fatalf("missing composed replacement: %#v", diagnostics) + } + } + + diagnostics := checkStdlibTestSource(t, "return position(a, v, true)") + if len(diagnostics) != 1 || !strings.Contains(diagnostics[0].Help, "final literal mode argument is removed") { + t.Fatalf("missing argument-removal guidance: %#v", diagnostics) + } +} + +func TestFQLStdlibStructuralFailurePreservesManualFindings(t *testing.T) { + input := "let a = append(items, v, true)\nlet b = rand(10)\nreturn keys /* preserve me */ (obj, true)" + result, err := migrateFQLSource(source.New("query.fql", input)) + if err == nil || !strings.Contains(err.Error(), "preserve comments") || result.Changed || result.Data != nil { + t.Fatalf("unsafe formatting accepted: %#v, %v", result, err) + } + + if len(result.ManualActions) != 2 || result.ManualActions[0].Detail != "append(...)" || + result.ManualActions[0].Line != 1 || result.ManualActions[1].Detail != "rand(...)" || result.ManualActions[1].Line != 2 { + t.Fatalf("lost semantic findings: %#v", result.ManualActions) + } +} diff --git a/internal/migration/fql_stdlib_structural_test.go b/internal/migration/fql_stdlib_structural_test.go new file mode 100644 index 0000000..447de08 --- /dev/null +++ b/internal/migration/fql_stdlib_structural_test.go @@ -0,0 +1,196 @@ +package migration + +import ( + "strconv" + "strings" + "testing" + + "github.com/MontFerret/ferret/v2/pkg/source" +) + +func TestMigrateFQLStdlibArgumentRules(t *testing.T) { + tests := []struct{ before, after string }{ + {"first(a)", "arrays::first(a)"}, + {"flatten(a)", "arrays::flatten(a)"}, + {"flatten(a, 2)", "arrays::flatten(a, 2)"}, + {"last(a)", "arrays::last(a)"}, + {"sorted(a)", "arrays::sorted(a)"}, + {"unique(a)", "arrays::unique(a)"}, + {"append(a, v)", "arrays::append(a, v)"}, + {"push(a, v)", "arrays::append(a, v)"}, + {"nth(a, 1)", "arrays::at(a, 1)"}, + {"remove_value(a, v)", "arrays::remove(a, v)"}, + {"remove_values(a, b)", "arrays::remove_any(a, b)"}, + {"slice(a, 1)", "arrays::slice(a, 1)"}, + {"slice(a, 1, 2)", "arrays::slice(a, 1, 2)"}, + {"intersection(a, b)", "arrays::intersection(a, b)"}, + {"minus(a, b)", "arrays::difference(a, b)"}, + {"union(a, b)", "arrays::concat(a, b)"}, + {"union_distinct(a, b)", "arrays::union(a, b)"}, + {"position(a, v)", "arrays::contains(a, v)"}, + {"position(a, v, false)", "arrays::contains(a, v)"}, + {"position(a, v, true)", "arrays::index_of(a, v)"}, + {"position(a, v, ((TRUE)))", "arrays::index_of(a, v)"}, + {"position(a, v, FaLsE)", "arrays::contains(a, v)"}, + {"append(a, v, false)", "arrays::append(a, v)"}, + {"push(a, v, (FALSE))", "arrays::append(a, v)"}, + {"remove_value(a, v, -1)", "arrays::remove(a, v)"}, + {"remove_value(a, v, -27)", "arrays::remove(a, v)"}, + {"remove_value(a, v, ((-(2))))", "arrays::remove(a, v)"}, + {"remove_value(a, v, -" + strconv.Itoa(int(^uint(0)>>1)) + ")", "arrays::remove(a, v)"}, + {"outersection(a, b)", "arrays::symmetric_difference(a, b)"}, + {"sorted_unique(a)", "arrays::sorted(arrays::unique(a))"}, + {"shift(a)", "arrays::slice(a, 1)"}, + {"keys(o, false)", "object::keys(o)"}, + {"keys(o, ((true)))", "arrays::sorted(object::keys(o))"}, + {"date_diff(a, b, unit, true)", "datetime::diff(a, b, unit)"}, + {"date_diff(a, b, unit, (TrUe))", "datetime::diff(a, b, unit)"}, + {"range(a, b)", "arrays::range(a, b)"}, + {"range(a, b, step)", "arrays::range(a, b, step)"}, + {"rand()", "random::float()"}, + {"position(a, v, false,)", "arrays::contains(a, v)"}, + {"keys(o, true,)", "arrays::sorted(object::keys(o))"}, + {"shift(a,)", "arrays::slice(a, 1)"}, + {"sorted_unique(a,)", "arrays::sorted(arrays::unique(a))"}, + } + + for _, test := range tests { + name, rest, _ := strings.Cut(test.before, "(") + for _, spelling := range []string{name, strings.ToUpper(name), strings.ToUpper(name[:1]) + name[1:]} { + t.Run(spelling+"("+rest, func(t *testing.T) { + input := "return " + spelling + "(" + rest + assertFQLStdlibMigration(t, input, "return "+test.after, 0) + + diagnostics := checkStdlibTestSource(t, input) + if len(diagnostics) != 1 || !strings.Contains(diagnostics[0].Message, "should use") || + diagnostics[0].Line != 1 || diagnostics[0].Column != 8 { + t.Fatalf("unexpected automatic check finding: %#v", diagnostics) + } + + if got := checkStdlibTestSource(t, "return "+test.after); len(got) != 0 { + t.Fatalf("canonical expression has findings: %#v", got) + } + }) + } + } +} + +func TestMigrateFQLStdlibManualModes(t *testing.T) { + tests := []struct{ call, reason string }{ + {"position(a, v, mode)", "Boolean or an index"}, + {"append(a, v, true)", "existing duplicates"}, + {"push(a, v, true)", "existing duplicates"}, + {"append(a, v, mode)", "dynamic flag"}, + {"push(a, v, mode)", "dynamic flag"}, + {"remove_value(a, v, 0)", "no limit mode"}, + {"remove_value(a, v, 2)", "no limit mode"}, + {"remove_value(a, v, limit)", "negative integer literal"}, + {"outersection(a, b, c)", "exactly one input"}, + {"outersection(a, b, c, d)", "odd-number-of-inputs"}, + {"keys(o, mode)", "dynamic sort mode"}, + {"date_diff(a, b, unit)", "truncates toward zero"}, + {"date_diff(a, b, unit, false)", "always returns a Float"}, + {"date_diff(a, b, unit, mode)", "literal true"}, + {"rand(10)", "max/2 to max*2"}, + {"rand(10, 2)", "rounded/floored"}, + {"pop(a)", "evaluation count"}, + {"unshift(a, v)", "evaluation order"}, + {"remove_nth(a, 1)", "copied host list"}, + } + + for _, call := range []string{"position(a)", "position(a, v, true, extra)", "append()", "push(a)", "append(a, v, false, extra)", "remove_value(a)", "remove_value(a, v, -1, extra)", "keys()", "keys(o, true, extra)", "date_diff(a)", "date_diff(a, b, unit, true, extra)", "range(a)", "range(a, b, step, extra)", "rand(1, 2, 3)"} { + tests = append(tests, struct{ call, reason string }{call, "arity"}) + } + + for _, call := range []string{"sorted_unique()", "sorted_unique(a, b)", "shift()", "shift(a, b)"} { + tests = append(tests, struct{ call, reason string }{call, "exactly one"}) + } + + for _, call := range []string{"outersection()", "outersection(a)"} { + tests = append(tests, struct{ call, reason string }{call, "at least two"}) + } + + for _, mode := range []string{"-0", "-0.0", "-1.0", "-1e0", "-(1 + 1)", "-limit", "-(-2)", "1 - 2", "(-2)?", "-999999999999999999999999999999"} { + tests = append(tests, struct{ call, reason string }{"remove_value(a, v, " + mode + ")", "negative integer literal"}) + } + + for _, mode := range []string{"true == true", "!false", "flag", "1", "\"true\"", "(true)?", "(true) ON ERROR RETURN false", "true ? true : false"} { + tests = append(tests, struct{ call, reason string }{"position(a, v, " + mode + ")", "dynamic mode"}) + } + + for _, test := range tests { + t.Run(test.call, func(t *testing.T) { + input := "// original location\nRETURN " + test.call + result := assertFQLStdlibMigration(t, input, input, 1) + diagnostics := checkStdlibTestSource(t, input) + if len(diagnostics) != 1 || diagnostics[0].Help != result.ManualActions[0].Reason || + !strings.Contains(diagnostics[0].Help, test.reason) || diagnostics[0].Line != 2 || diagnostics[0].Column != 10 { + t.Fatalf("check/run manual findings disagree: %#v, %#v", diagnostics, result.ManualActions) + } + }) + } +} + +func TestMigrateFQLStdlibNestedStructuralRules(t *testing.T) { + tests := []struct{ input, want string }{ + {"return sorted_unique(nth(values, 1))", "return arrays::sorted(arrays::unique(arrays::at(values, 1)))"}, + {"return keys(json_parse(body), true)", "return arrays::sorted(object::keys(encoding::json_parse(body)))"}, + {"return shift(union(a, b))", "return arrays::slice(arrays::concat(a, b), 1)"}, + {"return sorted_unique(shift(sorted_unique(a)))", "return arrays::sorted(\n arrays::unique(arrays::slice(arrays::sorted(arrays::unique(a)), 1))\n)"}, + {"return [shift(a), keys(o, true), position(a, v, false)]", "return [\n arrays::slice(a, 1),\n arrays::sorted(object::keys(o)),\n arrays::contains(a, v)\n]"}, + {"for x in shift(a) return sorted_unique(x)", "return for x in arrays::slice(a, 1) {\n return arrays::sorted(arrays::unique(x))\n}"}, + {"for x in sorted_unique(a) { return shift(x) }", "return for x in arrays::sorted(arrays::unique(a)) {\n return arrays::slice(x, 1)\n}"}, + {"return sorted_unique(nth(a, 1)?)?", "return arrays::sorted(arrays::unique(arrays::at(a, 1)?))?"}, + {"return sorted_unique(a) ON ERROR RETURN []", "return arrays::sorted(arrays::unique(a)) on error return []"}, + {"return shift(\n union(a, b),\n)", "return arrays::slice(arrays::concat(a, b), 1)"}, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + assertFQLStdlibMigration(t, test.input, test.want, 0) + }) + } + + assertFQLStdlibMigration(t, "return append(shift(a), v, true)", "return append(arrays::slice(a, 1), v, true)", 1) +} + +func TestFQLStdlibStructuralEditsPreserveTrivia(t *testing.T) { + tests := []struct{ input, want string }{ + {"return keys(\n obj /* important */,\n (/* mode */ true /* end */),\n)", "return arrays::sorted(object::keys(\n obj /* important */\n /* mode */ /* end */,\n))"}, + {"return remove_value(a, v, (/* group */ - /* sign */ 2 /* end */))", "return arrays::remove(a, v /* group */ /* sign */ /* end */)"}, + {"return shift(\n union(a, b), // keep λ\n)", "return arrays::slice(\n arrays::concat(a, b), // keep λ\n1)"}, + {"return sorted_unique /* call */ (nth(values, 1))?", "return arrays::sorted(arrays::unique /* call */ (arrays::at(values, 1)))?"}, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + src := source.New("query.fql", test.input) + program, err := parseFQLSource(src) + if err != nil { + t.Fatal(err) + } + + edits, actions, err := planFQLStdlib(src, program) + if err != nil || len(actions) != 0 { + t.Fatalf("plan edits: %v, %#v", err, actions) + } + + got, err := applyFQLEdits(test.input, edits) + if err != nil || got != test.want { + t.Fatalf("raw edits = %q, want %q; err=%v", got, test.want, err) + } + + result, err := migrateFQLSource(src) + if err != nil { + if !strings.Contains(err.Error(), "preserve comments") || result.Changed || result.Data != nil { + t.Fatalf("unsafe formatter result: %#v, %v", result, err) + } + } else { + formatted, err := parseFQLSource(source.New("query.fql", string(result.Data))) + if err != nil || strings.Join(fqlComments(formatted), "\n") != strings.Join(fqlComments(program), "\n") { + t.Fatalf("formatted comments changed: %s, %v", result.Data, err) + } + } + }) + } +} diff --git a/internal/migration/fql_stdlib_test.go b/internal/migration/fql_stdlib_test.go index 5001e38..9b639a3 100644 --- a/internal/migration/fql_stdlib_test.go +++ b/internal/migration/fql_stdlib_test.go @@ -167,15 +167,12 @@ func TestMigrateFQLStdlibDeferredCalls(t *testing.T) { tests := []struct{ call, reason string }{ {`join("a", "b")`, "legacy path::join"}, {`join(["a", "b"], ",")`, "modern global string joining"}, - {"keys(obj, true)", "argument-aware"}, - {"keys(obj, false)", "argument-aware"}, - {"keys(obj, option)", "argument-aware"}, - {"keys()", "only keys(obj)"}, - {"keys(obj, true, extra)", "argument-aware"}, + {"keys(obj, option)", "dynamic sort mode"}, + {"keys()", "arity"}, + {"keys(obj, true, extra)", "arity"}, {`date_compare(a, b, "year")`, "component-range"}, {`date_compare(a, b, "year", "day")`, "not a drop-in"}, - {`date_diff(a, b, "day")`, "integer/floating"}, - {`date_diff(a, b, "day", true)`, "datetime::diff"}, + {`date_diff(a, b, "day")`, "truncates toward zero"}, {"average(xs)", "heterogeneous"}, {"sum(xs)", "heterogeneous"}, {"min(xs)", "heterogeneous"}, @@ -203,10 +200,10 @@ func TestMigrateFQLStdlibDeferredCalls(t *testing.T) { func TestMigrateFQLStdlibMixedManualActions(t *testing.T) { input := `let text = "λ" FOR item IN values(obj) - RETURN [keys(item, true), average([abs(-2)]), average(item)]` + RETURN [keys(item, mode), average([abs(-2)]), average(item)]` want := `let text = "λ" return for item in object::values(obj) { - return [keys(item, true), average([math::abs(-2)]), average(item)] + return [keys(item, mode), average([math::abs(-2)]), average(item)] }` result := assertFQLStdlibMigration(t, input, want, 3) for _, action := range result.ManualActions { @@ -220,8 +217,8 @@ func TestMigrateFQLStdlibLeavesUnrelatedSourceUnchanged(t *testing.T) { inputs := []string{ `RETURN [encoding::json_parse(body), crypto::sha1(text), path::base(path), object::merge(a, b), datetime::year(dt), math::sqrt(x)]`, `RETURN [ENCODING::JSON_PARSE(body), custom::ABS(x), path::join("a", "b"), object::keys(obj)]`, - `RETURN [union(a, b), union_distinct(a, b), nth(a, 0), minus(a, b), push(a, 1), pop(a), shift(a), unshift(a, 1), position(a, 1), remove_nth(a, 0), outersection(a, b)]`, - `RETURN [rand(), rand(10, 1), range(1, 10), random::float(), arrays::at(xs, 0)]`, + `RETURN [arrays::concat(a, b), arrays::union(a, b), arrays::at(a, 0), arrays::difference(a, b), arrays::append(a, 1)]`, + `RETURN [random::float(), arrays::range(1, 10), arrays::at(xs, 0)]`, `let json_parse = "abs(1)" // json_parse(body) RETURN { json_parse, abs: "date_diff()" }`, }