Add FQL standard library migration support for legacy-to-modern function conversions

README.md

@@ -163,32 +163,63 @@ and `abs(value)` becomes `math::abs(value)`. Supported mappings include:
 | `encoding::` | `json_parse`, `json_stringify`, `encode_uri_component` → `query_escape`, `decode_uri_component` → `query_unescape`, `to_base64` → `base64_encode`, `from_base64` → `base64_decode`, `escape_html` → `html_escape`, `unescape_html` → `html_unescape` |
 | `crypto::` | `md5`, `sha1`, `sha512`, `random_token` |
 | `path::` | `base`, `clean`, `dir`, `ext`, `is_abs`, `separate`, `match` |
+| `arrays::` | `first`, `flatten`, `last`, `sorted`, `unique`, `slice`, `intersection`, `nth` → `at`, `remove_values` → `remove_any`, `minus` → `difference`, `union` → `concat`, `union_distinct` → `union`, `range` |
+| `random::` | Zero-argument `rand()` → `float()` |
 | `object::` | `values`, `has` → `has_key`, `zip`, `keep_keys`, `merge`, `merge_recursive` → `merge_deep`, and one-argument `keys(obj)` |
 | `datetime::` | `now`, `date` → `parse`, `date_dayofweek` → `day_of_week`, `date_dayofyear` → `day_of_year`, `date_leapyear` → `is_leap_year`; `date_year`, `date_month`, `date_day`, `date_hour`, `date_minute`, `date_second`, `date_millisecond`, `date_quarter`, `date_days_in_month`, `date_format`, `date_add`, `date_subtract` lose their `date_` prefix |
 | `math::` | `pi`, `abs`, `acos`, `asin`, `atan`, `atan2`, `ceil`, `cos`, `degrees`, `exp`, `exp2`, `floor`, `log`, `log2`, `log10`, `pow`, `radians`, `round`, `sin`, `sqrt`, `tan` |
 
-Only parsed call targets are replaced; arguments retain their meaning, and
-strings, comments, object keys, and variable names are not matched. Nested calls
-are migrated independently, and already-qualified targets are preserved. Object
-replacements use immutable operations, never `object::mut`.
+Argument-aware replacements also include:
+
+| Legacy call | Replacement |
+| --- | --- |
+| `position(a, v)` or `position(a, v, false)` | `arrays::contains(a, v)` |
+| `position(a, v, true)` | `arrays::index_of(a, v)` |
+| `append(a, v)` / `push(a, v)`, optionally with `false` | `arrays::append(a, v)` |
+| `remove_value(a, v)`, optionally with a negative integer literal | `arrays::remove(a, v)` |
+| `sorted_unique(a)` | `arrays::sorted(arrays::unique(a))` |
+| `shift(a)` | `arrays::slice(a, 1)` |
+| `outersection(a, b)` | `arrays::symmetric_difference(a, b)` |
+| `keys(o, false)` | `object::keys(o)` |
+| `keys(o, true)` | `arrays::sorted(object::keys(o))` |
+| `date_diff(a, b, unit, true)` | `datetime::diff(a, b, unit)` |
+
+Boolean modes and negative integer limits may be parenthesized; arbitrary
+constant expressions are not evaluated. Retained arguments preserve their
+evaluation order and count. Strings, comments, object keys, and variable names
+are not matched. Nested calls migrate independently, including inside composed
+replacements and calls requiring manual review. Already-qualified targets are
+preserved. Array and object replacements use immutable operations, never
+`arrays::mut` or `object::mut`.
 
 Manual follow-up includes the original path and line and explains why a call
 was preserved:
 
 - `join` is ambiguous between legacy path joining and modern global string joining.
-- `keys` with any arity other than one needs argument-aware review.
+- Dynamic modes in `position`, `keys`, `append`, and `push` need manual review.
+- Unique `append`/`push` suppresses only the incoming duplicate; applying
+  `arrays::unique` would also remove existing duplicates.
+- Zero, positive, and dynamic `remove_value` limits have no canonical limit mode.
+- `outersection` with three or more arrays uses exactly-one-input semantics;
+  canonical symmetric difference uses odd-number-of-inputs semantics.
+- `pop` needs an evaluation-count-preserving replacement; `unshift` must retain
+  argument evaluation order; `remove_nth` retains host-list removal semantics.
+- Parameterized `rand` uses historical rounded/floored calculations rather than
+  canonical continuous or integer bounds.
+- Unsupported arities of argument-aware rules require manual review.
 - `date_compare` has component-range semantics that differ from `datetime::same`;
-  legacy `date_diff` integer/floating behavior differs from `datetime::diff`.
+  `date_diff` without a literal `true` floating flag may truncate toward zero,
+  whereas `datetime::diff` always returns a Float.
 - `average`, `sum`, `min`, `max`, `median`, `percentile`, `stddev_population`,
   `stddev_sample`, `variance_population`, and `variance_sample` have permissive
   legacy behavior that differs from strict canonical math.
 - A matching function declaration or function alias anywhere in the file may
   change call resolution. These checks are deliberately conservative, including
   case variants and declarations in nested scopes. A namespace alias blocks a
-  replacement only when it would redirect that canonical target.
+  replacement when it would redirect any introduced namespace, including both
+  `arrays` and `object` for sorted keys.
 
-Other safe calls in the same file still migrate. Array migrations and
-`rand`/`range` are deferred and receive no new diagnostics in this pass.
+Other safe calls in the same file still migrate.
 Rerunning migration produces no further edits; unresolved manual actions remain.
 If the formatter cannot preserve comments, the entire file is left unchanged
 and reported for manual follow-up.
@@ -235,12 +266,12 @@ files.
 ### Checking FQL compatibility
 
 `migrate check` reports final collecting `FOR` compatibility and the same legacy
-stdlib findings as `migrate run`: encoding, crypto, path, immutable object,
-datetime, and scalar math replacements, plus calls requiring manual review.
+stdlib findings as `migrate run`: encoding, crypto, path, immutable arrays and
+objects, datetime, scalar math, range, and zero-argument random replacements,
+plus calls requiring manual review.
 It uses the same conservative declaration and alias guards described above.
-Already-qualified calls are left alone; arrays and `rand`/`range` remain outside
-this pass. A clean check covers these supported rules, not all v1 application
-semantics.
+Already-qualified calls are left alone. A clean check covers these supported
+rules, not all v1 application semantics.
 
 Check a standalone FQL file or recursively inspect a directory without
 modifying source files:

cmd/internal/migrate/check.go

@@ -16,9 +16,9 @@ func newCompatibilityCheckCommand(service Service) *cobra.Command {
 		Short: "Check FQL source for Ferret version compatibility",
 		Long: "Check a standalone FQL file or recursively inspect a directory for supported behavior changes " +
 			"between Ferret versions. The check does not require a Go module and never modifies source files.\n\n" +
-			"Checks final collecting FOR and legacy stdlib calls in encoding, crypto, path, object, datetime, and math. " +
+			"Checks final collecting FOR and legacy stdlib calls in encoding, crypto, path, arrays, object, datetime, math, and random. " +
 			"Findings suggest canonical replacements or manual review for ambiguous calls, semantic differences, " +
-			"and local function or use alias collisions. Arrays and rand/range are outside this pass. " +
+			"and local function or use alias collisions. Literal modes, composed replacements, range, and zero-argument rand are supported. " +
 			"Preview automatic replacements with migrate run --print.\n\n" +
 			"Directory scans include testdata, hidden and underscore-prefixed directories, and nested Go modules. " +
 			"They skip .git, .hg, .svn, vendor, and node_modules and do not follow directory symlinks.\n\n" +

cmd/internal/migrate/command_test.go

@@ -422,10 +422,10 @@ func TestMigrateCheckHelpDocumentsReadOnlyScanBoundaries(t *testing.T) {
 	for _, expected := range []string{
 		"migrate check [path]",
 		"does not require a Go module and never modifies source files",
-		"final collecting FOR and legacy stdlib calls in encoding, crypto, path, object, datetime, and math",
+		"final collecting FOR and legacy stdlib calls in encoding, crypto, path, arrays, object, datetime, math, and random",
 		"manual review",
 		"local function or use alias collisions",
-		"Arrays and rand/range are outside this pass",
+		"Literal modes, composed replacements, range, and zero-argument rand are supported",
 		"migrate run --print",
 		"include testdata, hidden and underscore-prefixed directories, and nested Go modules",
 		"skip .git, .hg, .svn, vendor, and node_modules",

cmd/internal/migrate/run.go

@@ -26,10 +26,11 @@ func newRunCommand(store *config.Store, service Service) *cobra.Command {
 			"directories, and nested Go modules. The selected directory itself is scanned regardless of its name. " +
 			"Directory symlinks are not followed.\n\n" +
 			"FQL migration returns and canonically formats a structurally recognized final top-level FOR. " +
-			"It also migrates safe legacy stdlib calls to encoding, crypto, path, object, datetime, and math namespaces. " +
-			"Ambiguous join, non-unary keys, date_compare, date_diff, aggregate math, and calls affected by " +
+			"It also migrates safe legacy stdlib calls to encoding, crypto, path, arrays, object, datetime, math, and random namespaces. " +
+			"Literal modes and composed replacements are supported, including sorted_unique, shift, and sorted keys. " +
+			"Ambiguous join, unsafe array modes, nonfloating date_diff, date_compare, aggregate math, parameterized rand, and calls affected by " +
 			"local function or use alias collisions are left for manual follow-up. " +
-			"Arrays and rand/range are outside this pass. Already-qualified calls are preserved. " +
+			"Range and zero-argument rand are migrated. Already-qualified calls are preserved. " +
 			"Malformed FQL is left unchanged and reported for manual follow-up.\n\n" +
 			"The command performs only documented mechanical import, dependency, and source changes. " +
 			"It does not convert application logic or arbitrary APIs to native Ferret v2 equivalents.",

cmd/internal/migrate/stdlib_test.go

@@ -11,8 +11,8 @@ import (
 
 func TestMigrateRunStdlibPreviewAndApply(t *testing.T) {
 	path := filepath.Join(t.TempDir(), "query.fql")
-	before := "return [abs(-2), average(xs)]"
-	want := "return [math::abs(-2), average(xs)]"
+	before := "return [shift(union(a, b)), append(a, v, true)]"
+	want := "return [arrays::slice(arrays::concat(a, b), 1), append(a, v, true)]"
 	if err := os.WriteFile(path, []byte(before), 0o600); err != nil {
 		t.Fatal(err)
 	}
@@ -29,7 +29,7 @@ func TestMigrateRunStdlibPreviewAndApply(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if !strings.Contains(stderr, "query.fql:1: average(...)") || !strings.Contains(stderr, "heterogeneous") {
+		if !strings.Contains(stderr, "query.fql:1: append(...)") || !strings.Contains(stderr, "existing duplicates") {
 			t.Fatalf("missing manual warning in mode %s: %s", mode, stderr)
 		}
 
@@ -39,7 +39,7 @@ func TestMigrateRunStdlibPreviewAndApply(t *testing.T) {
 				strings.Contains(stdout, "Scanned") || !strings.Contains(stderr, "Scanned 1 FQL file") {
 				t.Fatalf("print mixed diff and diagnostics: stdout=%q stderr=%q", stdout, stderr)
 			}
-		} else if !strings.Contains(stdout, "query.fql") || strings.Contains(stdout, "average(...)") {
+		} else if !strings.Contains(stdout, "query.fql") || strings.Contains(stdout, "append(...)") {
 			t.Fatalf("unexpected human output: %s", stdout)
 		}
 
@@ -63,7 +63,7 @@ func TestMigrateRunStdlibPreviewAndApply(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if stdout != "" || !strings.Contains(stderr, "average(...)") || !strings.Contains(stderr, "No safe automatic changes available") {
+	if stdout != "" || !strings.Contains(stderr, "append(...)") || !strings.Contains(stderr, "No safe automatic changes available") {
 		t.Fatalf("manual-only second run: stdout=%q stderr=%q", stdout, stderr)
 	}
 }
@@ -74,7 +74,7 @@ func TestMigrateRunStdlibHelp(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, text := range []string{"safe legacy stdlib", "encoding, crypto, path, object, datetime, and math", "local function or use alias", "Arrays and rand/range"} {
+	for _, text := range []string{"safe legacy stdlib", "encoding, crypto, path, arrays, object, datetime, math, and random", "local function or use alias", "Range and zero-argument rand"} {
 		if !strings.Contains(stdout, text) {
 			t.Fatalf("help does not contain %q: %s", text, stdout)
 		}
@@ -98,6 +98,18 @@ func TestMigrateCheckStdlibReportsFindingsWithoutWriting(t *testing.T) {
 				"  help: legacy aggregate behavior is permissive for heterogeneous collections; " +
 				"the strict math API requires a separate semantic migration\n\n",
 		},
+		{
+			name:  "composed replacement",
+			input: "return keys(obj, true)",
+			diagnostic: ":1:8: Legacy stdlib call `keys` should use `arrays::sorted(object::keys(...))`.\n" +
+				"  help: Preview automatic replacements with `ferret migrate run --print`. The final literal mode argument is removed.\n\n",
+		},
+		{
+			name:  "parameterized rand remains manual",
+			input: "return rand(10)",
+			diagnostic: ":1:8: Stdlib call `rand` needs manual review.\n" +
+				"  help: parameterized legacy rand uses a historical rounded/floored range calculation (max/2 to max*2 for one argument) and has no behavior-preserving canonical call\n\n",
+		},
 		{
 			name:  "canonical source",
 			input: `return object::has_key({ foo: "bar" }, "baz")`,

internal/migration/compatibility_stdlib.go

@@ -10,7 +10,7 @@ import (
 func checkFQLStdlib(src source.Source, program *fql.ProgramContext) ([]CompatibilityDiagnostic, error) {
 	var diagnostics []CompatibilityDiagnostic
 	for _, finding := range analyzeFQLStdlib(program) {
-		name := finding.name
+		name := finding.call.FunctionName()
 		spelling := name.GetText()
 
 		span, ok := fqlByteSpan(src.Content(), source.Span{
@@ -28,12 +28,16 @@ func checkFQLStdlib(src source.Source, program *fql.ProgramContext) ([]Compatibi
 
 		diagnostic := CompatibilityDiagnostic{
 			Path:    src.Name(),
-			Message: fmt.Sprintf("Legacy stdlib call `%s` should use `%s`.", spelling, finding.target),
+			Message: fmt.Sprintf("Legacy stdlib call `%s` should use `%s`.", spelling, finding.description()),
 			Help:    "Preview automatic replacements with `ferret migrate run --print`.",
 			Line:    position.Line,
 			Column:  position.Column,
 			Kind:    CompatibilityDiagnosticIssue,
 		}
+		if finding.dropLast {
+			diagnostic.Help += " The final literal mode argument is removed."
+		}
+
 		if finding.reason != "" {
 			diagnostic.Message = fmt.Sprintf("Stdlib call `%s` needs manual review.", spelling)
 			diagnostic.Help = finding.reason

internal/migration/compatibility_stdlib_test.go

@@ -51,13 +51,11 @@ func TestCheckFQLStdlibManualReview(t *testing.T) {
 	tests := []struct{ call, reason string }{
 		{`join("a", "b")`, "legacy path::join"},
 		{`join(["a", "b"], ",")`, "modern global string joining"},
-		{"keys()", "only keys(obj)"},
-		{"keys(obj, true)", "argument-aware"},
-		{"keys(obj, false)", "argument-aware"},
-		{"keys(obj, option)", "argument-aware"},
-		{"keys(obj, true, extra)", "argument-aware"},
+		{"keys()", "arity"},
+		{"keys(obj, option)", "dynamic sort mode"},
+		{"keys(obj, true, extra)", "arity"},
 		{`date_compare(a, b, "year")`, "component-range"},
-		{`date_diff(a, b, "day")`, "integer/floating"},
+		{`date_diff(a, b, "day")`, "truncates toward zero"},
 		{"average(xs)", "heterogeneous"},
 		{"sum(xs)", "heterogeneous"},
 		{"min(xs)", "heterogeneous"},
@@ -138,7 +136,7 @@ func TestCheckFQLStdlibIgnoresUnrelatedSource(t *testing.T) {
 let has = "json_parse()"
 return { has, abs: "date_diff()", values: [
     math::abs(-1), CUSTOM::HAS(obj, "key"), object::keys(obj),
-    union(a, b), push(a, 1), nth(a, 0), rand(), range(1, 10)
+    arrays::concat(a, b), arrays::append(a, 1), arrays::at(a, 0), random::float(), arrays::range(1, 10)
 ] }`
 	if diagnostics := checkStdlibTestSource(t, input); len(diagnostics) != 0 {
 		t.Fatalf("unrelated source has findings: %#v", diagnostics)
@@ -228,6 +226,8 @@ func BenchmarkCheckFQLStdlibCompatibility(b *testing.B) {
 		name, input string
 		findings    int
 	}{
+		{"structural", "return [keys(json_parse(body), true), shift(union(a, b)), position(a, v, false)]", 500},
+		{"structural_canonical", "return [arrays::sorted(object::keys(encoding::json_parse(body))), arrays::slice(arrays::concat(a, b), 1), arrays::contains(a, v)]", 0},
 		{"legacy", "return [json_parse(text), sha1(text), base(path), has(obj, key), date_year(dt), abs(-1), average(xs)]", 700},
 		{"canonical", "return [encoding::json_parse(text), crypto::sha1(text), path::base(path), object::has_key(obj, key), datetime::year(dt), math::abs(-1)]", 0},
 	} {