diff --git a/docs/docs/guides/analytics.md b/docs/docs/guides/analytics.md index 0c609024b..2f891f218 100644 --- a/docs/docs/guides/analytics.md +++ b/docs/docs/guides/analytics.md @@ -231,24 +231,28 @@ The key list `[Sym Ts]` specifies equality columns followed by the time column ( [12:00:00 12:00:01 12:00:02 12:00:03 12:00:04 12:00:05 12:00:06 12:00:07 12:00:08 12:00:09] [928 528 648 914 918 626 577 817 620 698]))) -; Simple window join: match on [Sym], join by Time -(window-join trades quotes [Sym] 'Time) +; TIME offsets are milliseconds: aggregate one second before through one after. +(set intervals (map-left + [-1000 1000] (at trades 'Time))) +(window-join [Sym Time] intervals trades quotes + {total_size: (sum Size)}) ``` ```text -┌──────┬──────────────┬───────┬───────┐ -│ Sym │ Time │ Price │ Size │ -│ SYM │ TIME │ F64 │ I64 │ -├──────┼──────────────┼───────┼───────┤ -│ AAPL │ 12:00:01.000 │ 89.17 │ 528 │ -│ AAPL │ 12:00:04.000 │ 70.5 │ 918 │ -│ AAPL │ 12:00:06.000 │ 80.54 │ 577 │ -├──────┴──────────────┴───────┴───────┤ -│ 3 rows (3 shown) 4 columns (4 shown)│ -└─────────────────────────────────────┘ +┌──────┬──────────────┬───────┬────────────┐ +│ Sym │ Time │ Price │ total_size │ +│ SYM │ TIME │ F64 │ I64 │ +├──────┼──────────────┼───────┼────────────┤ +│ AAPL │ 12:00:01.000 │ 89.17 │ 2104 │ +│ AAPL │ 12:00:04.000 │ 70.5 │ 2458 │ +│ AAPL │ 12:00:06.000 │ 80.54 │ 2020 │ +├──────┴──────────────┴───────┴────────────┤ +│ 3 rows (3 shown) 4 columns (4 shown) │ +└──────────────────────────────────────────┘ ``` -The `window-join` function matches rows from the right table within a time window around each left row. The equality keys (`[Sym]`) filter candidates, and the time column (`'Time`) determines the window range. +The last key in `[Sym Time]` is temporal; preceding keys are equality +partitions. `intervals` supplies a lower and upper offset for every left row, +and the final dictionary names the aggregations applied to matching right rows. ## 7. Working with CSV { #csv-workflow } diff --git a/docs/docs/guides/ipc.md b/docs/docs/guides/ipc.md index 8a73d9da3..0a300b20d 100644 --- a/docs/docs/guides/ipc.md +++ b/docs/docs/guides/ipc.md @@ -102,14 +102,14 @@ The server evaluates any valid Rayfall expression sent as a string. This means y (.ipc.send h "(select {from: trades total: (sum qty)})") ;; Volume-weighted average price by symbol -(.ipc.send h "(select {from: trades by: sym vwap: (% (sum (* price qty)) (sum qty))})") +(.ipc.send h "(select {from: trades by: sym vwap: (/ (sum (* price qty)) (sum qty))})") ``` ### Joins ```text ;; Join trades with quotes on sym -(.ipc.send h "(select {from: (left-join trades quotes 'sym)})") +(.ipc.send h "(select {from: (left-join [sym] trades quotes)})") ``` !!! note "Tip" diff --git a/docs/docs/language/control-flow.md b/docs/docs/language/control-flow.md index 1dc489467..23628d5ec 100644 --- a/docs/docs/language/control-flow.md +++ b/docs/docs/language/control-flow.md @@ -81,7 +81,7 @@ If no error is raised, `try` returns the result of the body expression normally. ### Fallback value -If the second argument is **not** a function, it is returned as-is as the fallback value on error (evaluated only when the body fails). Because lambdas do not capture closures, this is the only way to surface an outer binding from the failure branch: +If the second argument is **not** a function, it is returned as-is as the fallback value on error (evaluated only when the body fails): ```lisp ‣ (try (raise "boom") 0) @@ -93,6 +93,15 @@ If the second argument is **not** a function, it is returned as-is as the fallba A handler must accept the single error argument, so only a lambda or a unary builtin is *called* with the error; any other value (including a multi-argument builtin) is treated as a fallback value. +Nested lambdas capture visible lexical bindings when they are created: + +```lisp +‣ (set make-adder (fn [x] (fn [y] (+ x y)))) +‣ (set add7 (make-adder 7)) +‣ (add7 5) +12 +``` + ## Early Return: return `return` exits the innermost enclosing compiled lambda early with the given value: diff --git a/docs/docs/language/functions.md b/docs/docs/language/functions.md index f41add196..ef0d1aafb 100644 --- a/docs/docs/language/functions.md +++ b/docs/docs/language/functions.md @@ -83,8 +83,8 @@ Aggregation functions are marked **aggr** and reduce vectors to scalar values. U | `any` | unary, aggr | True if any non-null numeric element is truthy; empty/all-null returns `false` | `(any [0 0 3])` → `true` | | `count` | unary, aggr | Count of elements | `(count [1 2 3])` → `3` | | `avg` | unary, aggr | Arithmetic mean | `(avg [1 2 3])` → `2.0` | -| `min` | unary, aggr | Minimum value | `(min [3 1 2])` → `1` | -| `max` | unary, aggr | Maximum value | `(max [3 1 2])` → `3` | +| `min` | unary aggr, or binary atomic | Reduce one vector, or choose the element-wise minimum of two values | `(min [3 9 1] [2 10 4])` → `[2 9 1]` | +| `max` | unary aggr, or binary atomic | Reduce one vector, or choose the element-wise maximum of two values | `(max [3 9 1] 5)` → `[5 9 5]` | | `med` | unary, aggr | Median value (returns f64) | `(med [1 3 2])` → `2.0` | | `mode` | unary, aggr | Most frequent non-null value; ties keep the first encountered value | `(mode [1 2 2 3])` → `2` | | `dev` | unary, aggr | Population standard deviation | `(dev [1 2 3])` → `0.816...` | @@ -157,8 +157,9 @@ Operations on vectors as collections. | `rotate` | binary | Rotate left by N positions; negative rotates right | `(rotate [1 2 3 4] 1)` → `[2 3 4 1]` | | `cut` | binary | Split at sorted 0-based indices | `(cut [1 2 3 4] [2])` → `([1 2] [3 4])` | | `cross` | binary | Cartesian product as pairs | `(cross [1 2] ['a 'b])` → `((1 'a) (1 'b) (2 'a) (2 'b))` | -| `at` | binary | Index into vector | `(at [10 20 30] 1)` → `20` | +| `at` | binary | Index a collection; vector keys batch dictionary lookups | `(at [10 20 30] 1)` → `20` | | `find` | binary | Find index of value | `(find [10 20 30] 20)` → `1` | +| `fill` | binary, atomic | Replace null values; the replacement is the first argument | `(fill 0 [1 0Nl 3])` → `[1 0 3]` | | `reverse` | unary | Reverse order | `(reverse [1 2 3])` → `[3 2 1]` | | `til` | unary | Range [0..n) | `(til 5)` → `[0 1 2 3 4]` | | `lag` | unary | Shift values one row back; first row is null/sentinel | `(lag [10 20 30])` → `[0Nl 10 20]` | @@ -214,7 +215,7 @@ The time-series vector functions above are lazy-aware DAG operations for vector | `key` | unary | Get column names (table) or keys (dict) | `(key trades)` → `[sym price size]` | | `cols` | unary | Get table column names | `(cols trades)` → `[sym price size]` | | `value` | unary | Get column data (table) or values (dict) | `(value trades)` | -| `dict` | binary | Create dictionary from keys and values | `(dict [a b] [1 2])` | +| `dict` | binary | Create a dictionary from evaluated keys and values | `(dict [a b] [1 2])` | | `get` | binary | Lookup key in dict/table | `(get d 'a)` → `1` | | `remove` | binary | Remove key from dict | `(remove d 'a)` | | `row` | binary | Extract single row from table as dict | `(row trades 0)` | @@ -228,7 +229,11 @@ The time-series vector functions above are lazy-aware DAG operations for vector | `del` | variadic, special | Delete columns or rows from table | `(del trades 'temp_col)` | | `modify` | variadic | Functional table update (returns new table) | `(modify trades 'price (fn [p] (* p 1.1)))` | -`xkey` returns a dictionary keyed by unique key column value(s). Duplicate keys are a domain error; use `xgroup` to get a dictionary whose values are grouped table slices. +`xkey` returns a dictionary keyed by unique key column value(s). Duplicate keys are a domain error; use `xgroup` to get a dictionary whose values are grouped table slices. Passing a typed vector of keys to `at` performs a batch lookup and returns a list of the corresponding values. + +Dictionary literals are literals: their entries are not evaluated, just as the +elements of other literals are not evaluated. Use `(dict keys values)` to +construct a dictionary from evaluated expressions. ## Query Operations @@ -322,10 +327,10 @@ Rayforce supports equi-joins, outer joins, anti-joins, and time-series-aware joi (set products (table [product_id name] (list [10 20] [widget gadget]))) ; Left join two tables on the sym column (join keys are a symbol list) -(left-join trades_j quotes [sym]) +(left-join [sym] trades_j quotes) ; Inner join -(inner-join orders products [product_id]) +(inner-join [product_id] orders products) ; Full outer join keeps rows from both sides (full-join [sym] trades_j quotes) @@ -430,7 +435,9 @@ Cross-temporal comparisons are supported: dates, times, and timestamps are all c | `.csv.read` | variadic | Load CSV file into table | `(.csv.read "data.csv")` | | `.csv.write` | variadic | Write table to CSV file | `(.csv.write trades "out.csv")` | | `read` | unary | Read file contents as string | `(read "file.txt")` | -| `write` | binary | Write string to file | `(write "file.txt" "content")` | +| `read-bytes` | unary | Read file contents as a `U8` byte vector | `(read-bytes "file.bin")` | +| `write` | binary | Write a string to a file | `(write "file.txt" "content")` | +| `write-bytes` | binary | Write a `U8` byte vector to a file | `(write-bytes "file.bin" bytes)` | | `load` | unary | Load and evaluate a Rayfall script | `(load "lib.rfl")` | ## Control Flow @@ -509,7 +516,7 @@ Built-in support for triple stores. The EAV table has three columns: `e` (entity |---|---|---|---| | `union-all` | binary | Concatenate two tables (all rows) | `(union-all t1 t2)` | | `distinct` | unary | Remove duplicate rows from a table | `(distinct t)` | -| `anti-join` | variadic | Anti-semi-join: rows in left not in right | `(anti-join t1 t2 [x])` | +| `anti-join` | variadic | Anti-semi-join: rows in left not in right | `(anti-join [x] t1 t2)` | ```lisp ; Table concatenation and deduplication @@ -517,7 +524,7 @@ Built-in support for triple stores. The EAV table has three columns: `e` (entity (set t2 (table [x] (list [2 3]))) (union-all t1 t2) ; 4 rows: 1 2 2 3 (distinct (union-all t1 t2)) ; 3 rows: 1 2 3 -(anti-join t1 t2 [x]) ; 1 row: 1 +(anti-join [x] t1 t2) ; 1 row: 1 ``` ## Datalog diff --git a/docs/docs/queries/joins.md b/docs/docs/queries/joins.md index 488b31654..ec89f08eb 100644 --- a/docs/docs/queries/joins.md +++ b/docs/docs/queries/joins.md @@ -33,8 +33,7 @@ Join two tables on shared key columns. Returns a table with all columns from bot Join on multiple keys: ```lisp -; Key columns must be symbol (or numeric) — string key columns are not -; supported for joins. +; Key columns may be strings, symbols, or numeric values. (set x (table [a b c] (list (take [aa bb cc] 10) (take [I J K] 10) @@ -49,6 +48,10 @@ Join on multiple keys: (inner-join [a b] x y) ``` +Every named key must exist in both input tables. A misspelled or missing key is +a `domain` error instead of an empty result. A string key must be paired with a +string key on the other side. + ## Left Join The `left-join` function keeps all rows from the left table. Unmatched rows have null in the right-side columns. diff --git a/docs/docs/reference/all-functions.md b/docs/docs/reference/all-functions.md index de933e63b..f37d04b71 100644 --- a/docs/docs/reference/all-functions.md +++ b/docs/docs/reference/all-functions.md @@ -44,7 +44,7 @@ Generated from `src/lang/eval.c` in this checkout. The categorized reference bel `signum`, `sum`, `prod`, `all`, `any`, `count`, `avg`, `min`, `max`, `first`, `last`, `med`, `mode`, `dev`, `stddev`, `stddev_pop`, `dev_pop`, `var`, `var_pop`, `raise`, `distinct`, `reverse`, `til`, `lag`, `lead`, `deltas`, `ratios`, `fills`, `sums`, `avgs`, `mins`, `maxs`, `prds`, -`differ`, `asc`, `desc`, `iasc`, `idesc`, `rank`, `key`, `cols`, `value`, `type`, `read`, `load`, `exit`, +`differ`, `asc`, `desc`, `iasc`, `idesc`, `rank`, `key`, `cols`, `value`, `type`, `read`, `read-bytes`, `load`, `exit`, `nil?`, `where`, `group`, `raze`, `ungroup`, `ser`, `de`, `guid`, `date`, `time`, `timestamp`, `ss`, `hh`, `minute`, `yyyy`, `mm`, `dd`, `dow`, `doy`, `eval`, `parse`, `meta`, `fkeys`, `.sys.exec`, `.sys.cmd`, `.sys.listen`, `.os.getenv`, `.fs.size`, `.fs.list`, `.ipc.close`, `.repl.connect`, `.log.write`, `.log.replay`, @@ -56,8 +56,8 @@ Generated from `src/lang/eval.c` in this checkout. The categorized reference bel ### Binary `+`, `-`, `*`, `/`, `%`, `>`, `<`, `>=`, `<=`, `==`, `!=`, `pow`, `top`, `bot`, `pearson_corr`, `cov`, `scov`, `wsum`, `wavg`, `quantile`, `percentile`, `set`, `let`, -`try`, `filter`, `in`, `except`, `union`, `sect`, `take`, `drop`, `rotate`, `cut`, `cross`, `at`, `find`, `msum`, `mavg`, `mmin`, `mmax`, -`mcount`, `mvar`, `mdev`, `xasc`, `xdesc`, `table`, `union-all`, `xbar`, `as`, `write`, `dict`, `concat`, `within`, `div`, +`try`, `filter`, `in`, `except`, `union`, `sect`, `take`, `drop`, `rotate`, `cut`, `cross`, `at`, `find`, `fill`, `min`, `max`, `msum`, `mavg`, `mmin`, `mmax`, +`mcount`, `mvar`, `mdev`, `xasc`, `xdesc`, `table`, `union-all`, `xbar`, `as`, `write`, `write-bytes`, `dict`, `concat`, `within`, `div`, `rand`, `bin`, `binr`, `split`, `str-find`, `str-join`, `like`, `.os.setenv`, `.ipc.send`, `.ipc.post`, `get`, `remove`, `row`, `unify`, `xcol`, `xcols`, `xkey`, `xgroup`, `xrank`, `dl-query`, `dl-provenance`, `cos-dist`, `inner-prod`, `l2-dist`, `hnsw-save`, `.attr.set`, `.col.link` @@ -175,8 +175,8 @@ Aggregation functions reduce vectors to scalar values. Functions marked **aggr** | `any` | unary | aggr | True if any non-null numeric element is truthy; empty/all-null returns `false` | `(any [0 0 3])` → `true` | | `count` | unary | aggr | Count of elements | `(count [1 2 3])` → `3` | | `avg` | unary | aggr | Arithmetic mean | `(avg [1 2 3])` → `2.0` | -| `min` | unary | aggr | Minimum value | `(min [3 1 2])` → `1` | -| `max` | unary | aggr | Maximum value | `(max [3 1 2])` → `3` | +| `min` | unary aggr, or binary | aggr/atomic | Reduce one vector, or choose the element-wise minimum of two values | `(min [3 9 1] [2 10 4])` → `[2 9 1]` | +| `max` | unary aggr, or binary | aggr/atomic | Reduce one vector, or choose the element-wise maximum of two values | `(max [3 9 1] 5)` → `[5 9 5]` | | `first` | unary | — | First element of a vector | `(first [10 20 30])` → `10` | | `last` | unary | — | Last element of a vector | `(last [10 20 30])` → `30` | | `med` | unary | aggr | Median value (returns f64) | `(med [1 3 2])` → `2.0` | @@ -258,8 +258,9 @@ Operations on vectors and lists as collections — set operations, indexing, sea | `rotate` | binary | — | Rotate left by N positions (negative rotates right) | `(rotate [1 2 3 4] 1)` → `[2 3 4 1]` | | `cut` | binary | — | Split a collection at sorted 0-based indices | `(cut [1 2 3 4] [2])` → `([1 2] [3 4])` | | `cross` | binary | — | Cartesian product as pairs | `(cross [1 2] ['a 'b])` → `((1 'a) (1 'b) (2 'a) (2 'b))` | -| `at` | binary | — | Index into vector (0-based) | `(at [10 20 30] 1)` → `20` | +| `at` | binary | — | Index a collection; vector keys batch dictionary lookups | `(at [10 20 30] 1)` → `20` | | `find` | binary | — | Find index of first occurrence | `(find [10 20 30] 20)` → `1` | +| `fill` | binary | atomic | Replace null values; the replacement is the first argument | `(fill 0 [1 0Nl 3])` → `[1 0 3]` | | `reverse` | unary | — | Reverse element order | `(reverse [1 2 3])` → `[3 2 1]` | | `til` | unary | — | Generate range [0..n) | `(til 5)` → `[0 1 2 3 4]` | | `lag` | unary | lazy/DAG | Shift values one row back; first row is null/sentinel | `(lag [10 20 30])` → `[0Nl 10 20]` | @@ -388,7 +389,7 @@ Create and manipulate tables, dictionaries, and their metadata. |---|---|---|---|---| | `list` | variadic | — | Create a list from vectors (column data for tables) | `(list [1 2] ['a 'b])` | | `table` | binary | — | Create table from column names and list of vectors | `(table [x y] (list [1 2] ['a 'b]))` | -| `dict` | binary | — | Create dictionary from keys and values vectors | `(dict ['a 'b] [1 2])` | +| `dict` | binary | — | Create dictionary from evaluated keys and values vectors | `(dict ['a 'b] [1 2])` | | `key` | unary | — | Get column names (table) or keys (dict) | `(key trades)` → `[sym price size]` | | `cols` | unary | — | Get table column names | `(cols trades)` → `[sym price size time date]` | | `value` | unary | — | Get column data (table) or values (dict) | `(value d)` | @@ -407,7 +408,11 @@ Create and manipulate tables, dictionaries, and their metadata. | `ungroup` | unary | — | Flatten a grouped table's nested list columns into one row per element | `(ungroup gt)` | | `pivot` | variadic | — | Pivot table — reshape long to wide format | `(pivot trades 'sym 'date 'price sum)` | -`xcol` requires exactly one new name for each existing column. `xcols`, `xkey`, and `xgroup` accept a symbol atom, a symbol vector, or a list of symbol atoms. `xkey` is a dictionary projection, not a native keyed-table type: key columns must be unique, and each value is a row dictionary containing the non-key columns. Use `xgroup` when keys may repeat; each dictionary value is a table slice for that group. +`xcol` requires exactly one new name for each existing column. `xcols`, `xkey`, and `xgroup` accept a symbol atom, a symbol vector, or a list of symbol atoms. `xkey` is a dictionary projection, not a native keyed-table type: key columns must be unique, and each value is a row dictionary containing the non-key columns. Use `xgroup` when keys may repeat; each dictionary value is a table slice for that group. Passing a typed vector of keys to `at` performs a batch lookup and returns a list. + +Dictionary literals are literals: their entries are not evaluated, just as the +elements of other literals are not evaluated. Use `(dict keys values)` to +construct a dictionary from evaluated expressions. ```lisp ; Create a table @@ -470,10 +475,10 @@ Rayforce supports seven join types, including time-series-aware as-of and window | Function | Type | Flags | Description | Example | |---|---|---|---|---| -| `left-join` | variadic | — | Left join — all left rows, unmatched filled with null. Keys are a symbol list. | `(left-join trades quotes [sym])` | -| `inner-join` | variadic | — | Inner join — only matching rows from both sides | `(inner-join orders products [product_id])` | +| `left-join` | variadic | — | Left join — all left rows, unmatched filled with null. Keys are a symbol list. | `(left-join [sym] trades quotes)` | +| `inner-join` | variadic | — | Inner join — only matching rows from both sides | `(inner-join [product_id] orders products)` | | `full-join` | variadic | — | Full outer join — all rows from both sides, unmatched columns filled with null | `(full-join [sym] trades quotes)` | -| `anti-join` | variadic | — | Anti-semi-join — left rows with no right match | `(anti-join t1 t2 [key])` | +| `anti-join` | variadic | — | Anti-semi-join — left rows with no right match | `(anti-join [key] t1 t2)` | | `window-join` | variadic | special | Window join — `[eq-keys... time-key]`, intervals, left, right, agg dict | `(window-join [sym time] iv t1 t2 {avg_bid: (avg bid)})` | | `window-join1` | variadic | special | Window join variant (strict window, no prevailing quote) | `(window-join1 [sym time] iv t1 t2 {avg_bid: (avg bid)})` | | `asof-join` | variadic | — | As-of join — match most recent preceding value. Keys come first, last key is the time key. | `(asof-join [sym time] trades quotes)` | @@ -491,7 +496,7 @@ Rayforce supports seven join types, including time-series-aware as-of and window [149.5 2799.5 150.5]))) ; Left join on sym column (join keys are a symbol list) -(left-join trades quotes [sym]) +(left-join [sym] trades quotes) ; Window join: keys are [equality-keys... time-key]; intervals is ; a two-vector list with one [lo hi] window bound per left row. @@ -608,7 +613,9 @@ Printing, file I/O, CSV loading, and script execution. | `.csv.read` | variadic | restricted | Load CSV file into table (mmap, parallel parse) | `(.csv.read "data.csv")` | | `.csv.write` | variadic | restricted | Write table to CSV file | `(.csv.write trades "out.csv")` | | `read` | unary | restricted | Read file contents as string | `(read "file.txt")` | -| `write` | binary | restricted | Write string to file | `(write "file.txt" "content")` | +| `read-bytes` | unary | restricted | Read file contents as a `U8` byte vector | `(read-bytes "file.bin")` | +| `write` | binary | restricted | Write a string to a file | `(write "file.txt" "content")` | +| `write-bytes` | binary | restricted | Write a `U8` byte vector to a file | `(write-bytes "file.bin" bytes)` | | `load` | unary | restricted | Load and evaluate a Rayfall script file | `(load "lib.rfl")` | | `exit` | unary | restricted | Exit the process with status code | `(exit 0)` | | `resolve` | variadic | special | Resolve a symbol in the current scope | `(resolve 'x)` | diff --git a/docs/docs/storage/ipc.md b/docs/docs/storage/ipc.md index e04cea496..3eadb6439 100644 --- a/docs/docs/storage/ipc.md +++ b/docs/docs/storage/ipc.md @@ -65,8 +65,8 @@ In restricted mode, specific builtins that write files, mutate state, or control | Category | Blocked Builtins | |---|---| | Mutation | `set`, `del`, `update`, `insert`, `upsert`, `modify` | -| File writes | `write`, `.csv.write`, `load`, `.db.splayed.set` | -| File reads | `read`, `.csv.read` | +| File writes | `write`, `write-bytes`, `.csv.write`, `load`, `.db.splayed.set` | +| File reads | `read`, `read-bytes`, `.csv.read` | | System | `.sys.exec`, `.os.getenv`, `.os.setenv`, `exit` | | IPC chaining | `.ipc.open`, `.ipc.close`, `.ipc.send`, `.ipc.post` | diff --git a/docs/docs/tutorials/analytics.md b/docs/docs/tutorials/analytics.md index a02fbf6f5..3b9d4d560 100644 --- a/docs/docs/tutorials/analytics.md +++ b/docs/docs/tutorials/analytics.md @@ -205,7 +205,9 @@ Each trade row now carries the `Bid` and `Ask` from the most recent quote for th ## 6. Window Joins -A window join is similar to an ASOF join but allows you to match each left row against right rows within a time window partitioned by equality keys. Use `window-join` with the form `(window-join leftTable rightTable [eqKeys] 'timeCol)`: +A window join aggregates right rows within a time interval for each left row, +partitioned by optional equality keys. Its form is +`(window-join [eqKeys... timeKey] intervals left right {aggregations})`: ```lisp (set trades (table [Time Symbol Price] @@ -219,24 +221,14 @@ A window join is similar to an ASOF join but allows you to match each left row a [149.0 149.5 278.0 279.0 150.5 279.5 278.5 151.0] [150.0 150.5 279.0 280.0 151.5 280.5 279.5 152.0]))) -; Join each trade to the latest quote for the same symbol -(window-join trades quotes ['Symbol] 'Time) +; Average quotes from ten time units before each trade through the trade time. +(set intervals (map-left + [-10 0] (at trades 'Time))) +(window-join [Symbol Time] intervals trades quotes + {avg_bid: (avg Bid) avg_ask: (avg Ask)}) ``` -```text -┌──────┬────────┬───────┬───────┬───────┐ -│ Time │ Symbol │ Price │ Bid │ Ask │ -│ I64 │ SYM │ F64 │ F64 │ F64 │ -├──────┼────────┼───────┼───────┼───────┤ -│ 10 │ AAPL │ 150.0 │ 149.5 │ 150.5 │ -│ 20 │ AAPL │ 151.0 │ 149.5 │ 150.5 │ -│ 30 │ GOOG │ 280.0 │ 279.5 │ 280.5 │ -│ 40 │ GOOG │ 279.0 │ 278.5 │ 279.5 │ -│ 50 │ AAPL │ 152.0 │ 151.0 │ 152.0 │ -└──────┴────────┴───────┴───────┴───────┘ -``` - -The `window-join` matches each trade to the closest preceding quote for the same `Symbol`, partitioned by the equality keys in the vector. The `'Time` argument specifies the temporal ordering column. +The last key, `Time`, is the temporal key; `Symbol` is the equality key. The +two vectors in `intervals` provide the lower and upper offsets for each trade. ## 7. Multi-Step Pipeline diff --git a/examples/rfl/join.rfl b/examples/rfl/join.rfl index 4ace2981f..df40def6b 100644 --- a/examples/rfl/join.rfl +++ b/examples/rfl/join.rfl @@ -1,10 +1,9 @@ -(set n 10) +(set orders (table [customer qty] + (list (list "alice" "bob" "carol") [10 20 30]))) -(set x (table [a b c d] - (list (take (list "aaaaaa" "bbbbbbbbbb" "cc" "dd") n) (take [I J K] n) (til n) (til n)))) +(set accounts (table [customer region] + (list (list "alice" "carol" "dave") [east west north]))) -(set y (table [a b c d e] - (list (take (list "aa" "bb" "cc" "dd" "ee") n) (take [I J K] n) (til n) (til n) (til n)))) - -;; (left-join [a] x y) -;; (left-join [a b] x y) +;; String columns are valid join keys. +(show (inner-join [customer] orders accounts)) +(show (left-join [customer] orders accounts)) diff --git a/src/core/runtime.h b/src/core/runtime.h index 90fc23689..d23655916 100644 --- a/src/core/runtime.h +++ b/src/core/runtime.h @@ -35,11 +35,13 @@ typedef struct { #define RAY_SCOPE_CAP 64 #define RAY_FRAME_CAP 64 +#define RAY_SCOPE_LEXICAL 0 +#define RAY_SCOPE_QUERY 1 -/* One lexical scope frame. keys/vals start out pointing at the inline - * arrays and move to heap blocks if the frame grows past RAY_FRAME_CAP - * (see env.c). Self-referential, so a frame must not be relocated - * while live. */ +/* One lexical or synthetic query scope frame. keys/vals start out pointing + * at the inline arrays and move to heap blocks if the frame grows past + * RAY_FRAME_CAP (see env.c). Self-referential, so a frame must not be + * relocated while live. */ typedef struct { int64_t keys_inline[RAY_FRAME_CAP]; ray_t* vals_inline[RAY_FRAME_CAP]; @@ -47,6 +49,7 @@ typedef struct { ray_t** vals; /* -> vals_inline, or heap once grown */ int32_t cap; int32_t count; + uint8_t kind; } ray_scope_frame_t; /* ===== Per-thread VM ===== diff --git a/src/io/csv.c b/src/io/csv.c index 18f49a461..ab7e16411 100644 --- a/src/io/csv.c +++ b/src/io/csv.c @@ -1017,6 +1017,7 @@ static bool csv_fill_str_cols(csv_strref_t** str_refs, int n_cols, memcpy(dst[r].prefix, p, 4); dst[r].pool_off = pool_off; memcpy(pool_base + pool_off, p, l); + ray_str_t_cache_hash(&dst[r], pool_base); pool_off += l; /* cannot wrap: pool_bytes <= UINT32_MAX */ } } diff --git a/src/lang/compile.c b/src/lang/compile.c index bd83adc38..5384ff31f 100644 --- a/src/lang/compile.c +++ b/src/lang/compile.c @@ -371,6 +371,27 @@ static void compile_list(compiler_t *c, ray_t *ast) { /* (fn [params] body...) — nested lambda via dynamic eval */ if (sym_id == sf_fn && n >= 3) { + if (ast_refs_locals(c, ast)) { + ray_t *syms = ray_alloc((size_t)c->n_locals * sizeof(int64_t)); + if (!syms || RAY_IS_ERR(syms)) { c->error = true; return; } + syms->type = RAY_I64; + syms->len = c->n_locals; + int64_t *ids = (int64_t*)ray_data(syms); + for (int32_t i = 0; i < c->n_locals; i++) + ids[i] = c->locals[i].sym_id; + int32_t syms_idx = add_constant(c, syms); + ray_release(syms); + if (c->error) return; + emit_const(c, syms_idx); + emit(c, OP_SCOPE_BEGIN); + int32_t idx = add_constant(c, ast); + emit_const(c, idx); + emit(c, OP_CALLD); + emit(c, 0); + emit_const(c, syms_idx); + emit(c, OP_SCOPE_END); + return; + } int32_t idx = add_constant(c, ast); emit_const(c, idx); emit(c, OP_CALLD); diff --git a/src/lang/env.c b/src/lang/env.c index e04cea250..53dce1f8a 100644 --- a/src/lang/env.c +++ b/src/lang/env.c @@ -114,6 +114,99 @@ static struct { int32_t ray_env_scope_depth(void) { return __VM ? __VM->scope_depth : 0; } int32_t ray_env_global_count(void) { return g_env.count; } +/* Query scopes are synthetic and must not eclipse a caller's lexical + * parameter/let binding. Other query frames are deliberately ignored so + * an inner query can bind a same-named column over an outer query column. + * Query-local callers push a query frame before probing, so scanning every + * frame still excludes the binding currently being installed. */ +ray_t* ray_env_get_lexical_local(int64_t sym_id) { + if (!__VM) return NULL; + for (int32_t d = __VM->scope_depth - 1; d >= 0; d--) { + ray_scope_frame_t* f = &__VM->scope_stack[d]; + if (f->kind == RAY_SCOPE_QUERY) continue; + for (int32_t i = 0; i < f->count; i++) + if (f->keys[i] == sym_id) return f->vals[i]; + } + return NULL; +} + +bool ray_env_has_lexical_local(int64_t sym_id) { + return ray_env_get_lexical_local(sym_id) != NULL; +} + +ray_err_t ray_env_set_query_local(int64_t sym_id, ray_t* val) { + return ray_env_has_lexical_local(sym_id) ? RAY_OK + : ray_env_set_local(sym_id, val); +} + +ray_t* ray_env_capture_locals(void) { + if (!__VM || __VM->scope_depth == 0) return NULL; + int64_t capacity = 0; + for (int32_t d = 0; d < __VM->scope_depth; d++) + capacity += __VM->scope_stack[d].count; + if (capacity == 0) return NULL; + + ray_t* keys = ray_sym_vec_new(RAY_SYM_W64, capacity); + ray_t* vals = ray_list_new(capacity); + if (!keys || RAY_IS_ERR(keys) || !vals || RAY_IS_ERR(vals)) { + if (keys && !RAY_IS_ERR(keys)) ray_release(keys); + if (vals && !RAY_IS_ERR(vals)) ray_release(vals); + return ray_error("oom", NULL); + } + + int64_t* key_ids = (int64_t*)ray_data(keys); + /* Top-to-bottom flattening preserves ordinary lexical lookup: the first + * occurrence of a name is the value visible at closure creation. */ + for (int32_t d = __VM->scope_depth - 1; d >= 0; d--) { + ray_scope_frame_t* f = &__VM->scope_stack[d]; + for (int32_t i = 0; i < f->count; i++) { + if (!f->vals[i]) continue; + bool seen = false; + for (int64_t k = 0; k < keys->len; k++) + if (key_ids[k] == f->keys[i]) { seen = true; break; } + if (seen) continue; + key_ids[keys->len++] = f->keys[i]; + vals = ray_list_append(vals, f->vals[i]); + if (!vals || RAY_IS_ERR(vals)) { + ray_release(keys); + return vals ? vals : ray_error("oom", NULL); + } + } + } + if (keys->len == 0) { + ray_release(keys); + ray_release(vals); + return NULL; + } + return ray_dict_new(keys, vals); +} + +ray_err_t ray_env_push_capture(ray_t* capture) { + if (ray_env_push_scope() != RAY_OK) return RAY_ERR_OOM; + if (!capture) return RAY_OK; + if (capture->type != RAY_DICT) { + ray_env_pop_scope(); + return RAY_ERR_TYPE; + } + ray_t* keys = ray_dict_keys(capture); + ray_t* vals = ray_dict_vals(capture); + if (!keys || keys->type != RAY_SYM || !vals || vals->type != RAY_LIST || + keys->len != vals->len) { + ray_env_pop_scope(); + return RAY_ERR_TYPE; + } + ray_t** value_items = (ray_t**)ray_data(vals); + for (int64_t i = 0; i < keys->len; i++) { + int64_t sym = ray_read_sym(ray_data(keys), i, RAY_SYM, keys->attrs); + ray_err_t err = ray_env_set_local(sym, value_items[i]); + if (err != RAY_OK) { + ray_env_pop_scope(); + return err; + } + } + return RAY_OK; +} + /* The five connection-hook sym ids carved out of the reserved-name reject. * Populated lazily on first probe; idempotent — ray_sym_intern is content- * keyed, so repeated calls return the same id. The carve-out applies ONLY @@ -199,6 +292,16 @@ static ray_t* env_lookup_flat(int64_t sym_id) { return NULL; } +ray_t* ray_env_get_local(int64_t sym_id) { + if (!__VM) return NULL; + for (int32_t d = __VM->scope_depth - 1; d >= 0; d--) { + ray_scope_frame_t* f = &__VM->scope_stack[d]; + for (int32_t i = 0; i < f->count; i++) + if (f->keys[i] == sym_id) return f->vals[i]; + } + return NULL; +} + ray_t* ray_env_get(int64_t sym_id) { /* Flat lookup first — covers every non-dotted name AND every * reserved builtin like `.sys.gc` which is bound both flat (for @@ -606,17 +709,26 @@ ray_err_t ray_env_set(int64_t sym_id, ray_t* val) { return env_bind_global_user(sym_id, val); } -ray_err_t ray_env_push_scope(void) { +static ray_err_t env_push_scope(uint8_t kind) { if (__VM->scope_depth >= RAY_SCOPE_CAP) return RAY_ERR_OOM; ray_scope_frame_t* f = &__VM->scope_stack[__VM->scope_depth]; f->keys = f->keys_inline; f->vals = f->vals_inline; f->cap = RAY_FRAME_CAP; f->count = 0; + f->kind = kind; __VM->scope_depth++; return RAY_OK; } +ray_err_t ray_env_push_scope(void) { + return env_push_scope(RAY_SCOPE_LEXICAL); +} + +ray_err_t ray_env_push_query_scope(void) { + return env_push_scope(RAY_SCOPE_QUERY); +} + void ray_env_pop_scope(void) { if (__VM->scope_depth <= 0) return; __VM->scope_depth--; @@ -630,6 +742,7 @@ void ray_env_pop_scope(void) { f->vals = f->vals_inline; f->cap = RAY_FRAME_CAP; f->count = 0; + f->kind = RAY_SCOPE_LEXICAL; } /* Materialize compiled-lambda locals into a fresh scope frame — the diff --git a/src/lang/env.h b/src/lang/env.h index 391193ef9..6af49f573 100644 --- a/src/lang/env.h +++ b/src/lang/env.h @@ -43,6 +43,7 @@ static inline const char* ray_fn_name(const ray_t* fn) { ray_err_t ray_env_init(void); void ray_env_destroy(void); ray_t* ray_env_get(int64_t sym_id); +ray_t* ray_env_get_local(int64_t sym_id); /* User-facing binder. Refuses any name starting with `.` — that root is * reserved for system namespaces (.sys, .os, .io, .ipc, …) populated by @@ -133,9 +134,15 @@ int32_t ray_env_global_count(void); /* Local scope stack for lexical binding (let, do, lambda) */ ray_err_t ray_env_push_scope(void); +ray_err_t ray_env_push_query_scope(void); void ray_env_pop_scope(void); int32_t ray_env_scope_depth(void); ray_err_t ray_env_set_local(int64_t sym_id, ray_t* val); +ray_t* ray_env_get_lexical_local(int64_t sym_id); +bool ray_env_has_lexical_local(int64_t sym_id); +ray_err_t ray_env_set_query_local(int64_t sym_id, ray_t* val); +ray_t* ray_env_capture_locals(void); +ray_err_t ray_env_push_capture(ray_t* capture); /* Compiled-lambda local materialization (OP_SCOPE_BEGIN / OP_SCOPE_END). * bind: push a fresh frame and bind syms[i] -> slots[i] (NULL slots — diff --git a/src/lang/eval.c b/src/lang/eval.c index 58c791a47..72f8196c0 100644 --- a/src/lang/eval.c +++ b/src/lang/eval.c @@ -124,6 +124,12 @@ static inline bool fn_is_restricted(ray_t* fn_obj) { return __VM->restricted && (fn_obj->attrs & RAY_FN_RESTRICTED); } +static inline ray_binary_fn unary_binary_overload(ray_unary_fn fn) { + if (fn == ray_min_fn) return ray_min2_fn; + if (fn == ray_max_fn) return ray_max2_fn; + return NULL; +} + static ray_t* materialize_owned_args(ray_t** args, int64_t n) { for (int64_t i = 0; i < n; i++) { if (!args[i] || !ray_is_lazy(args[i])) continue; @@ -1420,8 +1426,34 @@ ray_t* call_fn2(ray_t* fn, ray_t* a, ray_t* b) { return call_lambda(fn, args, 2); } if (fn->type == RAY_UNARY) { - /* Partial application not supported, just call with first arg */ ray_unary_fn f = (ray_unary_fn)(uintptr_t)fn->i64; + ray_binary_fn f2 = unary_binary_overload(f); + if (f2) { + ray_t* la = a; + ray_t* lb = b; + bool owned_a = false, owned_b = false; + if (ray_is_lazy(la)) { + ray_retain(la); + la = ray_lazy_materialize(la); + if (!la || RAY_IS_ERR(la)) return la ? la : ray_error("type", NULL); + owned_a = true; + } + if (ray_is_lazy(lb)) { + ray_retain(lb); + lb = ray_lazy_materialize(lb); + if (!lb || RAY_IS_ERR(lb)) { + if (owned_a) ray_release(la); + return lb ? lb : ray_error("type", NULL); + } + owned_b = true; + } + ray_t* out = (is_collection(la) || is_collection(lb)) + ? atomic_map_binary(f2, la, lb) : f2(la, lb); + if (owned_a) ray_release(la); + if (owned_b) ray_release(lb); + return out; + } + /* Legacy helper behavior for unary functions without an overload. */ return f(a); } return ray_error("type", "call: expected a callable function, got %s", ray_type_name(fn->type)); @@ -1939,14 +1971,18 @@ ray_t* ray_fn(ray_t** args, int64_t n) { } } - /* Create lambda object with space for 7 slots: + ray_t* closure = ray_env_capture_locals(); + if (closure && RAY_IS_ERR(closure)) return closure; + + /* Create lambda object with space for 8 slots: * [0] params, [1] body, [2] bytecode, [3] constants, [4] n_locals, - * [5] nfo (source location), [6] dbg (debug metadata) */ - ray_t* lambda = ray_alloc(7 * sizeof(ray_t*)); - if (!lambda) return ray_error("oom", NULL); + * [5] nfo (source location), [6] dbg (debug metadata), [7] closure. */ + ray_t* lambda = ray_alloc(8 * sizeof(ray_t*)); + if (!lambda) { ray_release(closure); return ray_error("oom", NULL); } lambda->type = RAY_LAMBDA; lambda->attrs = 0; lambda->len = 0; + memset(ray_data(lambda), 0, 8 * sizeof(ray_t*)); /* Store params list */ ray_retain(params_list); @@ -1956,8 +1992,8 @@ ray_t* ray_fn(ray_t** args, int64_t n) { int64_t body_count = n - 1; ray_t* body = ray_alloc(body_count * sizeof(ray_t*)); if (!body) { - ray_release(params_list); ray_release(lambda); + ray_release(closure); return ray_error("oom", NULL); } body->type = RAY_LIST; @@ -1982,6 +2018,7 @@ ray_t* ray_fn(ray_t** args, int64_t n) { LAMBDA_NFO(lambda) = NULL; } LAMBDA_DBG(lambda) = NULL; + LAMBDA_CLOSURE(lambda) = closure; return lambda; } @@ -2049,7 +2086,7 @@ static ray_t* vm_exec(ray_t* lambda, ray_t** call_args, int64_t argc); /* Call a lambda: compile on first call, then execute bytecode. */ ray_t* call_lambda(ray_t* lambda, ray_t** call_args, int64_t argc) { /* Lazy compilation on first call */ - if (!LAMBDA_IS_COMPILED(lambda)) { + if (!LAMBDA_CLOSURE(lambda) && !LAMBDA_IS_COMPILED(lambda)) { ray_compile(lambda); } @@ -2067,7 +2104,13 @@ ray_t* call_lambda(ray_t* lambda, ray_t** call_args, int64_t argc) { if (argc != param_count) return ray_error("arity", "expected %" PRId64 " args, got %" PRId64, param_count, argc); - if (ray_env_push_scope() != RAY_OK) return ray_error("oom", NULL); + bool has_closure = LAMBDA_CLOSURE(lambda) != NULL; + if (has_closure && ray_env_push_capture(LAMBDA_CLOSURE(lambda)) != RAY_OK) + return ray_error("oom", NULL); + if (ray_env_push_scope() != RAY_OK) { + if (has_closure) ray_env_pop_scope(); + return ray_error("oom", NULL); + } /* Bind 'self' to the current lambda for recursion */ { @@ -2089,11 +2132,13 @@ ray_t* call_lambda(ray_t* lambda, ray_t** call_args, int64_t argc) { result = ray_eval(body_exprs[i]); if (RAY_IS_ERR(result)) { ray_env_pop_scope(); + if (has_closure) ray_env_pop_scope(); return result; } } ray_env_pop_scope(); + if (has_closure) ray_env_pop_scope(); return result; } @@ -2518,6 +2563,28 @@ op_callf: { switch (fn_obj->type) { case RAY_UNARY: if (fn_is_restricted(fn_obj)) { for (int32_t i = 0; i < n; i++) ray_release(fn_args[i]); result = ray_error("access", "restricted"); break; } + { + ray_unary_fn unary = (ray_unary_fn)(uintptr_t)fn_obj->i64; + ray_binary_fn binary = unary_binary_overload(unary); + if (n == 2 && binary) { + for (int32_t i = 0; i < 2; i++) { + if (fn_args[i] && ray_is_lazy(fn_args[i])) { + fn_args[i] = ray_lazy_materialize(fn_args[i]); + if (!fn_args[i] || RAY_IS_ERR(fn_args[i])) { + result = fn_args[i] ? fn_args[i] : ray_error("type", NULL); + fn_args[i] = NULL; + ray_release(fn_args[1 - i]); + goto unary_done; + } + } + } + result = (is_collection(fn_args[0]) || is_collection(fn_args[1])) + ? atomic_map_binary(binary, fn_args[0], fn_args[1]) + : binary(fn_args[0], fn_args[1]); + ray_release(fn_args[0]); + ray_release(fn_args[1]); + goto unary_done; + } if (n != 1) { for (int32_t i = 0; i < n; i++) ray_release(fn_args[i]); result = ray_error("arity", "expected 1 arg, got %d", n); break; } /* fn_args[0] is an owned ref (POPped from VM stack); materialise * consumes it — no extra ray_release after the call. */ @@ -2525,9 +2592,11 @@ op_callf: { fn_args[0] = ray_lazy_materialize(fn_args[0]); /* consumes owned ref */ if (!fn_args[0] || RAY_IS_ERR(fn_args[0])) { result = fn_args[0] ? fn_args[0] : ray_error("type", NULL); fn_args[0] = NULL; break; } } - result = ((ray_unary_fn)(uintptr_t)fn_obj->i64)(fn_args[0]); + result = unary(fn_args[0]); ray_release(fn_args[0]); +unary_done: break; + } case RAY_BINARY: if (fn_is_restricted(fn_obj)) { for (int32_t i = 0; i < n; i++) ray_release(fn_args[i]); result = ray_error("access", "restricted"); break; } if (n != 2) { for (int32_t i = 0; i < n; i++) ray_release(fn_args[i]); result = ray_error("arity", "expected 2 args, got %d", n); break; } @@ -3130,6 +3199,7 @@ static void ray_register_builtins(void) { register_binary("cross", RAY_FN_NONE, ray_cross_fn); register_binary("at", RAY_FN_NONE, ray_at_fn); register_binary("find", RAY_FN_NONE, ray_find_fn); + register_binary("fill", RAY_FN_ATOMIC, ray_fill_fn); register_unary("reverse", RAY_FN_NONE | RAY_FN_LAZY_AWARE, ray_reverse_fn); register_unary("til", RAY_FN_NONE, ray_til_fn); register_unary_op("lag", RAY_FN_NONE | RAY_FN_LAZY_AWARE, ray_lag_fn, OP_LAG); @@ -3212,7 +3282,9 @@ static void ray_register_builtins(void) { register_binary("as", RAY_FN_NONE, ray_cast_fn); register_unary("type", RAY_FN_NONE, ray_type_fn); register_unary("read", RAY_FN_RESTRICTED, ray_read_file_fn); + register_unary("read-bytes", RAY_FN_RESTRICTED, ray_read_bytes_fn); register_binary("write", RAY_FN_RESTRICTED, ray_write_file_fn); + register_binary("write-bytes", RAY_FN_RESTRICTED, ray_write_bytes_fn); register_unary("load", RAY_FN_RESTRICTED, ray_load_file_fn); register_unary("exit", RAY_FN_RESTRICTED, ray_exit_fn); register_vary("resolve", RAY_FN_SPECIAL_FORM, ray_resolve_fn); @@ -3718,9 +3790,30 @@ ray_t* ray_eval(ray_t* obj) { switch (head->type) { case RAY_UNARY: { - if (n != 2) { ray_release(head); ret = ray_error("arity", "expected 1 arg, got %d", (int)(n-1)); goto out; } if (fn_is_restricted(head)) { ray_release(head); ret = ray_error("access", "restricted"); goto out; } ray_unary_fn fn = (ray_unary_fn)(uintptr_t)head->i64; + ray_binary_fn fn2 = unary_binary_overload(fn); + if (n == 3 && fn2) { + ray_t* left = ray_eval(elems[1]); + if (!left || RAY_IS_ERR(left)) { ray_release(head); ret = left ? left : ray_error("type", NULL); goto out; } + ray_t* right = ray_eval(elems[2]); + ray_release(head); + if (!right || RAY_IS_ERR(right)) { ray_release(left); ret = right ? right : ray_error("type", NULL); goto out; } + if (ray_is_lazy(left)) { + left = ray_lazy_materialize(left); + if (!left || RAY_IS_ERR(left)) { ray_release(right); ret = left ? left : ray_error("type", NULL); goto out; } + } + if (ray_is_lazy(right)) { + right = ray_lazy_materialize(right); + if (!right || RAY_IS_ERR(right)) { ray_release(left); ret = right ? right : ray_error("type", NULL); goto out; } + } + ret = (is_collection(left) || is_collection(right)) + ? atomic_map_binary(fn2, left, right) : fn2(left, right); + ray_release(left); + ray_release(right); + goto out; + } + if (n != 2) { ray_release(head); ret = ray_error("arity", "expected 1 arg, got %d", (int)(n-1)); goto out; } uint8_t fn_attrs = head->attrs; if (fn == (ray_unary_fn)ray_sum_fn) { int handled = 0; diff --git a/src/lang/eval.h b/src/lang/eval.h index e111d6931..d52e67b3e 100644 --- a/src/lang/eval.h +++ b/src/lang/eval.h @@ -111,6 +111,7 @@ enum { * data[4] = int32_t n_locals (number of local slots needed) * data[5] = ray_t* nfo (source location info, NULL if absent) * data[6] = ray_t* dbg (debug metadata, NULL if absent) + * data[7] = ray_t* closure (captured lexical locals, NULL if empty) */ #define RAY_FN_COMPILED 0x40 /* lambda has been compiled to bytecode */ @@ -122,6 +123,7 @@ enum { #define LAMBDA_NLOCALS(lam) (*((int32_t*)&((ray_t**)ray_data(lam))[4])) #define LAMBDA_NFO(lam) (((ray_t**)ray_data(lam))[5]) #define LAMBDA_DBG(lam) (((ray_t**)ray_data(lam))[6]) +#define LAMBDA_CLOSURE(lam) (((ray_t**)ray_data(lam))[7]) #define LAMBDA_IS_COMPILED(lam) ((lam)->attrs & RAY_FN_COMPILED) @@ -225,6 +227,8 @@ ray_t* ray_gt_fn(ray_t* a, ray_t* b); ray_t* ray_lt_fn(ray_t* a, ray_t* b); ray_t* ray_gte_fn(ray_t* a, ray_t* b); ray_t* ray_lte_fn(ray_t* a, ray_t* b); +ray_t* ray_min2_fn(ray_t* a, ray_t* b); +ray_t* ray_max2_fn(ray_t* a, ray_t* b); ray_t* ray_eq_fn(ray_t* a, ray_t* b); ray_t* ray_neq_fn(ray_t* a, ray_t* b); @@ -283,6 +287,7 @@ ray_t* ray_cut_fn(ray_t* vec, ray_t* idxs); ray_t* ray_cross_fn(ray_t* a, ray_t* b); ray_t* ray_at_fn(ray_t* vec, ray_t* idx); ray_t* ray_find_fn(ray_t* vec, ray_t* val); +ray_t* ray_fill_fn(ray_t* replacement, ray_t* value); ray_t* ray_til_fn(ray_t* x); ray_t* ray_reverse_fn(ray_t* x); ray_t* ray_lag_fn(ray_t* x); @@ -330,7 +335,9 @@ ray_t* ray_println_fn(ray_t** args, int64_t n); ray_t* ray_read_csv_fn(ray_t** args, int64_t n); ray_t* ray_write_csv_fn(ray_t** args, int64_t n); ray_t* ray_read_file_fn(ray_t* path_obj); +ray_t* ray_read_bytes_fn(ray_t* path_obj); ray_t* ray_write_file_fn(ray_t* path_obj, ray_t* content); +ray_t* ray_write_bytes_fn(ray_t* path_obj, ray_t* content); /* Vector similarity / embeddings / HNSW. * cos-dist and l2-dist return distance (lower = closer); inner-prod is diff --git a/src/lang/internal.h b/src/lang/internal.h index 6c030de3f..f5e19b6c2 100644 --- a/src/lang/internal.h +++ b/src/lang/internal.h @@ -454,6 +454,8 @@ ray_t* ray_cov_fn(ray_t* x, ray_t* y); ray_t* ray_scov_fn(ray_t* x, ray_t* y); ray_t* ray_wsum_fn(ray_t* x, ray_t* y); ray_t* ray_wavg_fn(ray_t* x, ray_t* y); +ray_t* ray_min2_fn(ray_t* a, ray_t* b); +ray_t* ray_max2_fn(ray_t* a, ray_t* b); /* In-place median (quickselect). Caller owns the buffer; we permute * elements. Returns NaN if n <= 0. Used by aggr_med_per_group_buf in @@ -496,6 +498,7 @@ ray_t* ray_cut_fn(ray_t* vec, ray_t* idxs); ray_t* ray_cross_fn(ray_t* a, ray_t* b); ray_t* ray_at_fn(ray_t* vec, ray_t* idx); ray_t* ray_find_fn(ray_t* vec, ray_t* val); +ray_t* ray_fill_fn(ray_t* replacement, ray_t* value); ray_t* ray_til_fn(ray_t* x); ray_t* ray_reverse_fn(ray_t* x); ray_t* ray_lag_fn(ray_t* x); @@ -526,6 +529,7 @@ ray_t* ray_fold_right_fn(ray_t** args, int64_t n); ray_t* ray_scan_left_fn(ray_t** args, int64_t n); ray_t* ray_scan_right_fn(ray_t** args, int64_t n); ray_t* ray_enlist_fn(ray_t** args, int64_t n); +uint64_t ray_atom_hash(ray_t* x); /* String builtins (formerly static in eval.c, now in str_builtin.c) */ ray_t* ray_split_fn(ray_t* str, ray_t* delim); @@ -678,8 +682,10 @@ ray_t* ray_write_csv_fn(ray_t** args, int64_t n); ray_t* ray_cast_fn(ray_t* type_sym, ray_t* val); ray_t* ray_type_fn(ray_t* val); ray_t* ray_read_file_fn(ray_t* path_obj); +ray_t* ray_read_bytes_fn(ray_t* path_obj); ray_t* ray_load_file_fn(ray_t* path_obj); ray_t* ray_write_file_fn(ray_t* path_obj, ray_t* content); +ray_t* ray_write_bytes_fn(ray_t* path_obj, ray_t* content); /* Misc builtins (formerly in eval.c, now in ops/builtins.c) */ ray_t* ray_dict_fn(ray_t* keys, ray_t* vals); diff --git a/src/mem/heap.c b/src/mem/heap.c index eb888bb1a..351a4a809 100644 --- a/src/mem/heap.c +++ b/src/mem/heap.c @@ -817,7 +817,8 @@ static void ray_release_owned_refs(ray_t* v) { if (ray_is_atom(v)) { if (v->type == RAY_LAMBDA) { - /* Lambda stores [params, body, bytecode, constants, n_locals, nfo, dbg] in ray_data */ + /* Lambda stores params/body/bytecode/constants plus optional + * nfo/debug/closure owned references in ray_data. */ ray_t** slots = (ray_t**)ray_data(v); for (int i = 0; i < 4; i++) { if (slots[i] && !RAY_IS_ERR(slots[i])) @@ -826,6 +827,7 @@ static void ray_release_owned_refs(ray_t* v) { /* Release optional debug info slots */ if (LAMBDA_NFO(v)) ray_release(LAMBDA_NFO(v)); if (LAMBDA_DBG(v)) ray_release(LAMBDA_DBG(v)); + if (LAMBDA_CLOSURE(v)) ray_release(LAMBDA_CLOSURE(v)); return; } if (v->type == RAY_LAZY) { @@ -946,6 +948,7 @@ bool ray_retain_owned_refs(ray_t* v) { } if (LAMBDA_NFO(v)) ray_retain(LAMBDA_NFO(v)); if (LAMBDA_DBG(v)) ray_retain(LAMBDA_DBG(v)); + if (LAMBDA_CLOSURE(v)) ray_retain(LAMBDA_CLOSURE(v)); return true; } /* Lazy handles own their graph uniquely — no retain on copy */ @@ -1058,6 +1061,7 @@ static void ray_detach_owned_refs(ray_t* v) { for (int i = 0; i < 4; i++) slots[i] = NULL; LAMBDA_NFO(v) = NULL; LAMBDA_DBG(v) = NULL; + LAMBDA_CLOSURE(v) = NULL; return; } if (v->type == RAY_LAZY) { diff --git a/src/ops/builtins.c b/src/ops/builtins.c index c8672f6fd..c2d4f82d2 100644 --- a/src/ops/builtins.c +++ b/src/ops/builtins.c @@ -24,11 +24,13 @@ /** I/O builtins, type casting, and misc builtins extracted from eval.c. */ +#include #include "lang/eval.h" #include "lang/internal.h" #include "lang/env.h" #include "core/platform.h" /* ray_vm_map_fd_ro / ray_vm_unmap_file (tracked) */ #include "vec/vec.h" +#include "vec/str.h" #include "lang/nfo.h" #include "lang/parse.h" #include "core/pool.h" @@ -1341,8 +1343,13 @@ ray_t* ray_cast_fn(ray_t* type_sym, ray_t* val) { const char* sp = ray_str_ptr(val); if (!sp) return ray_error("domain", "as: cannot parse empty str as i64"); char* end; + errno = 0; int64_t v = strtoll(sp, &end, 10); if (end == sp) return ray_error("domain", "as: cannot parse str as i64"); + if (*end != '\0') + return ray_error("domain", "as: cannot parse str as i64, unexpected trailing characters"); + if (errno == ERANGE) + return ray_error("domain", "as: cannot parse str as i64, value out of int64 range"); return make_i64(v); } /* Vector/list cast */ @@ -1363,8 +1370,13 @@ ray_t* ray_cast_fn(ray_t* type_sym, ray_t* val) { if (val->type == -RAY_TIMESTAMP) return ray_i32((int32_t)val->i64); if (val->type == -RAY_STR) { const char* sp = ray_str_ptr(val); char* end; + errno = 0; long v = strtol(sp, &end, 10); if (end == sp) return ray_error("domain", "as: cannot parse str as i32"); + if (*end != '\0') + return ray_error("domain", "as: cannot parse str as i32, unexpected trailing characters"); + if (errno == ERANGE) + return ray_error("domain", "as: cannot parse str as i32, value out of int64 range"); return ray_i32((int32_t)v); } /* Vector cast */ @@ -1385,8 +1397,13 @@ ray_t* ray_cast_fn(ray_t* type_sym, ray_t* val) { if (val->type == -RAY_TIMESTAMP) return ray_i16((int16_t)val->i64); if (val->type == -RAY_STR) { const char* sp = ray_str_ptr(val); char* end; + errno = 0; long v = strtol(sp, &end, 10); if (end == sp) return ray_error("domain", "as: cannot parse str as i16"); + if (*end != '\0') + return ray_error("domain", "as: cannot parse str as i16, unexpected trailing characters"); + if (errno == ERANGE) + return ray_error("domain", "as: cannot parse str as i16, value out of int64 range"); return ray_i16((int16_t)v); } /* Vector cast */ @@ -1409,8 +1426,11 @@ ray_t* ray_cast_fn(ray_t* type_sym, ray_t* val) { const char* sp = ray_str_ptr(val); if (!sp) return ray_error("domain", "as: cannot parse empty str as f64"); char* end; + errno = 0; double v = strtod(sp, &end); if (end == sp) return ray_error("domain", "as: cannot parse str as f64"); + if (*end != '\0') + return ray_error("domain", "as: cannot parse str as f64, unexpected trailing characters"); /* STAGE 2 (ingest/cast STR→F64): canonicalize at the ingest entry * point. strtod("inf")/strtod("1e400")/strtod("nan") would yield a * non-finite F64; make_f64 maps every non-finite to NULL_F64 (0Nf), @@ -1830,19 +1850,37 @@ ray_t* ray_cast_fn(ray_t* type_sym, ray_t* val) { ray_release(s); if (val->type == -RAY_GUID) { ray_retain(val); return val; } if (val->type == -RAY_STR) { - /* Parse UUID string: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" */ + /* Parse UUID string: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx". + * Require the canonical hyphenated form exactly: 32 hex + * nibbles with the four dashes at offsets 8/13/18/23. The old + * parser skipped '-' anywhere and decoded any character as a + * nibble, so non-hex garbage (e.g. all-'z') silently produced a + * wrong-but-valid GUID. */ const char* sp = ray_str_ptr(val); size_t sl = ray_str_len(val); - if (sl < 36) return ray_error("domain", "as: cannot parse str as guid, expected 36 chars, got %lld", (long long)sl); + if (sl != 36) return ray_error("domain", "as: cannot parse str as guid, expected 36 chars, got %lld", (long long)sl); uint8_t bytes[16]; - const char* p = sp; - for (int bi = 0; bi < 16; bi++) { - if (*p == '-') p++; - char hi = *p++; - char lo = *p++; - int h = (hi >= 'a') ? hi - 'a' + 10 : (hi >= 'A') ? hi - 'A' + 10 : hi - '0'; - int l = (lo >= 'a') ? lo - 'a' + 10 : (lo >= 'A') ? lo - 'A' + 10 : lo - '0'; - bytes[bi] = (uint8_t)((h << 4) | l); + int nib = 0; + for (size_t j = 0; j < sl; j++) { + char c = sp[j]; + /* Offsets 8/13/18/23 MUST be the dash; every other position + * MUST be a hex digit. This is position-driven (rather than + * "reject a misplaced dash") so the 32 non-dash positions are + * exactly 32 nibbles — nib never exceeds 31 and bytes[nib>>1] + * cannot run past bytes[16]. */ + if (j == 8 || j == 13 || j == 18 || j == 23) { + if (c != '-') + return ray_error("domain", "as: cannot parse str as guid, expected '-' at offset %lld", (long long)j); + continue; + } + int v; + if (c >= '0' && c <= '9') v = c - '0'; + else if (c >= 'a' && c <= 'f') v = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') v = c - 'A' + 10; + else return ray_error("domain", "as: cannot parse str as guid, non-hex character '%c'", c); + if ((nib & 1) == 0) bytes[nib >> 1] = (uint8_t)(v << 4); + else bytes[nib >> 1] |= (uint8_t)v; + nib++; } return ray_guid(bytes); } @@ -1881,8 +1919,13 @@ ray_t* ray_cast_fn(ray_t* type_sym, ray_t* val) { if (val->type == -RAY_F64) return ray_u8(ray_cast_f64_to_u8_null(val->f64)); if (val->type == -RAY_STR) { const char* sp = ray_str_ptr(val); - char* end; long v = strtol(sp, &end, 10); + char* end; errno = 0; + long v = strtol(sp, &end, 10); if (end == sp) return ray_error("domain", "as: cannot parse str as u8"); + if (*end != '\0') + return ray_error("domain", "as: cannot parse str as u8, unexpected trailing characters"); + if (errno == ERANGE) + return ray_error("domain", "as: cannot parse str as u8, value out of int64 range"); return ray_u8((uint8_t)v); } /* Vector cast */ @@ -1954,29 +1997,51 @@ ray_t* ray_type_fn(ray_t* val) { return ray_sym(id); } -/* (read path) — read a file's contents as a string */ -ray_t* ray_read_file_fn(ray_t* path_obj) { - if (path_obj->type != -RAY_STR) return ray_error("type", "read: path must be str, got %s", ray_type_name(path_obj->type)); +static ray_t* read_file_bytes(ray_t* path_obj, const char* op) { + if (path_obj->type != -RAY_STR) + return ray_error("type", "%s: path must be str, got %s", op, ray_type_name(path_obj->type)); const char* path = ray_str_ptr(path_obj); - if (!path) return ray_error("domain", "read: empty path"); + if (!path || ray_str_len(path_obj) == 0) + return ray_error("domain", "%s: empty path", op); + FILE* fp = fopen(path, "rb"); if (!fp) return ray_error("io", NULL); - fseek(fp, 0, SEEK_END); + if (fseek(fp, 0, SEEK_END) != 0) { fclose(fp); return ray_error("io", NULL); } long sz = ftell(fp); - fseek(fp, 0, SEEK_SET); if (sz < 0) { fclose(fp); return ray_error("io", NULL); } - /* Use ray_alloc for the buffer */ - ray_t* buf = ray_alloc((size_t)sz + 1); - if (!buf || RAY_IS_ERR(buf)) { fclose(fp); return ray_error("oom", NULL); } - char* data = (char*)ray_data(buf); - size_t rd = fread(data, 1, (size_t)sz, fp); - fclose(fp); - data[rd] = '\0'; - ray_t* result = ray_str(data, rd); - ray_release(buf); + if (fseek(fp, 0, SEEK_SET) != 0) { fclose(fp); return ray_error("io", NULL); } + + ray_t* result = ray_vec_new(RAY_U8, (int64_t)sz); + if (!result || RAY_IS_ERR(result)) { + fclose(fp); + return result ? result : ray_error("oom", NULL); + } + result->len = (int64_t)sz; + + size_t rd = sz > 0 ? fread(ray_data(result), 1, (size_t)sz, fp) : 0; + int close_rc = fclose(fp); + if (rd != (size_t)sz || close_rc != 0) { + ray_release(result); + return ray_error("io", NULL); + } + return result; +} + +/* (read path) — read a file's contents as a string */ +ray_t* ray_read_file_fn(ray_t* path_obj) { + ray_t* bytes = read_file_bytes(path_obj, "read"); + if (RAY_IS_ERR(bytes)) return bytes; + + ray_t* result = ray_str((const char*)ray_data(bytes), (size_t)bytes->len); + ray_release(bytes); return result; } +/* (read-bytes path) — read a file's contents as a U8 byte vector */ +ray_t* ray_read_bytes_fn(ray_t* path_obj) { + return read_file_bytes(path_obj, "read-bytes"); +} + /* (load path) — read and evaluate a Rayfall script file via mmap */ ray_t* ray_load_file_fn(ray_t* path_obj) { if (path_obj->type != -RAY_STR) return ray_error("type", "load: path must be str, got %s", ray_type_name(path_obj->type)); @@ -2045,22 +2110,39 @@ ray_t* ray_load_file_fn(ray_t* path_obj) { #endif } -/* (write path content) — write string to a file */ -ray_t* ray_write_file_fn(ray_t* path_obj, ray_t* content) { - if (path_obj->type != -RAY_STR) return ray_error("type", "write: path must be str, got %s", ray_type_name(path_obj->type)); - if (content->type != -RAY_STR) return ray_error("type", "write: content must be str, got %s", ray_type_name(content->type)); +static ray_t* write_file_data(ray_t* path_obj, const void* data, size_t len, + const char* op) { + if (path_obj->type != -RAY_STR) + return ray_error("type", "%s: path must be str, got %s", op, ray_type_name(path_obj->type)); const char* path = ray_str_ptr(path_obj); - const char* data = ray_str_ptr(content); - size_t len = ray_str_len(content); - if (!path || !data) return ray_error("domain", "write: empty path or content"); + if (!path || ray_str_len(path_obj) == 0) + return ray_error("domain", "%s: empty path", op); + if (len > 0 && !data) return ray_error("domain", "%s: invalid content", op); + FILE* fp = fopen(path, "wb"); if (!fp) return ray_error("io", NULL); - size_t written = fwrite(data, 1, len, fp); - fclose(fp); - if (written != len) return ray_error("io", NULL); + size_t written = len > 0 ? fwrite(data, 1, len, fp) : 0; + int close_rc = fclose(fp); + if (written != len || close_rc != 0) return ray_error("io", NULL); return make_i64(0); } +/* (write path content) — write a string to a file */ +ray_t* ray_write_file_fn(ray_t* path_obj, ray_t* content) { + if (content->type != -RAY_STR) + return ray_error("type", "write: content must be str, got %s", ray_type_name(content->type)); + return write_file_data(path_obj, ray_str_ptr(content), ray_str_len(content), + "write"); +} + +/* (write-bytes path content) — write a U8 byte vector to a file */ +ray_t* ray_write_bytes_fn(ray_t* path_obj, ray_t* content) { + if (content->type != RAY_U8) + return ray_error("type", "write-bytes: content must be U8, got %s", ray_type_name(content->type)); + return write_file_data(path_obj, ray_data(content), (size_t)content->len, + "write-bytes"); +} + /* ══════════════════════════════════════════ * Additional builtins (ported from rayforce) * ══════════════════════════════════════════ */ @@ -2446,7 +2528,7 @@ static inline uint64_t hash_i64(int64_t v) { * the existing path is degenerate for composite multi-key composites. * Uses the canonical wyhash helpers from ops/hash.h, same as the * pivot / datalog / join hashers. */ -static uint64_t atom_hash(ray_t* a) { +uint64_t ray_atom_hash(ray_t* a) { /* List-element position: elements may be a bare C NULL (ray_list_set stores * NULL unretained; ray_list_get is out-of-range), mirroring atom_eq's * C-NULL-tolerant element handling — so keep the !a guard, not RAY_ASSERT_VALUE. */ @@ -2471,7 +2553,7 @@ static uint64_t atom_hash(ray_t* a) { /* Seed with len so [] and a list of zeros differ. */ uint64_t h = ray_hash_i64(n); for (int64_t i = 0; i < n; i++) - h = ray_hash_combine(h, atom_hash(elems[i])); + h = ray_hash_combine(h, ray_atom_hash(elems[i])); return h; } default: @@ -2501,12 +2583,23 @@ static uint64_t ght_i64_hash_gi(uint32_t gi, void* ctx) { return hash_i64(c->gvals[gi]); } +typedef struct { + const ray_str_t* desc; + const char* pool; + const int64_t* gvals; +} ght_str_ctx_t; + +static uint64_t ght_str_hash_gi(uint32_t gi, void* ctx) { + ght_str_ctx_t* c = (ght_str_ctx_t*)ctx; + return ray_str_t_hash32(&c->desc[c->gvals[gi]], c->pool); +} + /* Context for the LIST-path rehash: gkeys holds atom pointers for each * unique group (one slot per gi), recomputed on grow via atom_hash. */ typedef struct { ray_t** gkeys; } ght_list_ctx_t; static uint64_t ght_list_hash_gi(uint32_t gi, void* ctx) { ght_list_ctx_t* c = (ght_list_ctx_t*)ctx; - return atom_hash(c->gkeys[gi]); + return ray_atom_hash(c->gkeys[gi]); } /* Grow the per-group bookkeeping arrays used by ray_group_indices_fn. @@ -2576,12 +2669,8 @@ ray_t* ray_group_indices_fn(ray_t* x) { return ray_dict_new(keys, vals); } - /* Collect unique values; the scalar / RAY_GUID / RAY_LIST paths - * grow these arrays on demand (group_grow / group_grow_listkeys). - * The RAY_STR path below still caps at this initial size — its - * side buffer isn't yet wired into a grow helper, but the cap is - * unreachable in practice (RAY_STR is char-vector, ≤256 distinct - * 1-byte chars). Starting at 1024 keeps the initial alloc cheap. */ + /* Collect unique values. Every path grows these arrays on demand; + * starting at 1024 keeps the initial allocation cheap. */ int64_t max_groups = n < 1024 ? n : 1024; ray_t* val_block = ray_alloc((size_t)(max_groups * sizeof(int64_t))); if (RAY_IS_ERR(val_block)) return val_block; @@ -2614,7 +2703,7 @@ ray_t* ray_group_indices_fn(ray_t* x) { for (int64_t i = 0; i < n; i++) { ray_t* elem = elems[i]; - uint64_t h = atom_hash(elem); + uint64_t h = ray_atom_hash(elem); uint32_t slot = (uint32_t)(h & ht.mask); uint32_t gi_found = GHT_EMPTY; while (ht.slots[slot] != GHT_EMPTY) { @@ -2759,66 +2848,97 @@ ray_t* ray_group_indices_fn(ray_t* x) { return ray_dict_new(keys_vec, vals_lst); } - /* RAY_STR: string-based grouping using ray_str_vec_get */ + /* RAY_STR: descriptor hashing plus collision-safe content equality. + * Pooled descriptors reuse their cached hash, so grouping digest keys + * does not reread 32/64-byte payloads merely to locate the hash slot. */ if (x->type == RAY_STR) { - /* Store group keys as (ptr, len) pairs -- use a scratch block for strings */ - ray_t* skblock = ray_alloc((size_t)(max_groups * sizeof(ray_t*))); - if (RAY_IS_ERR(skblock)) { ray_free(val_block); ray_free(ivblock); return skblock; } - ray_t** str_keys = (ray_t**)ray_data(skblock); + ray_t* owner = x; + int64_t off = 0; + if (x->attrs & RAY_ATTR_SLICE) { + owner = x->slice_parent; + off = x->slice_offset; + } + const ray_str_t* desc = (const ray_str_t*)ray_data(owner) + off; + const char* pool = (owner->str_pool && !RAY_IS_ERR(owner->str_pool)) + ? (const char*)ray_data(owner->str_pool) : NULL; + + group_ht_t ht; + uint32_t seed_cap = (uint32_t)(n < 64 ? 64 : (n < 1048576 ? (n * 2) : 2097152)); + if (!group_ht_init(&ht, seed_cap)) { + ray_free(val_block); ray_free(ivblock); + return ray_error("oom", NULL); + } + ght_str_ctx_t sctx = { .desc = desc, .pool = pool, .gvals = gvals }; for (int64_t i = 0; i < n; i++) { - size_t slen = 0; - const char* sp = ray_str_vec_get(x, i, &slen); - - int64_t gi = -1; - for (int64_t g = 0; g < ngroups; g++) { - size_t gsl = ray_str_len(str_keys[g]); - const char* gsp = ray_str_ptr(str_keys[g]); - if (gsl == slen && (slen == 0 || memcmp(gsp, sp, slen) == 0)) { - gi = g; break; + const ray_str_t* cur = &desc[i]; + uint64_t h = ray_str_t_hash32(cur, pool); + uint32_t slot = (uint32_t)(h & ht.mask); + uint32_t gi_found = GHT_EMPTY; + while (ht.slots[slot] != GHT_EMPTY) { + uint32_t gi = ht.slots[slot]; + if (ray_str_t_eq(&desc[gvals[gi]], pool, cur, pool)) { + gi_found = gi; + break; } + slot = (slot + 1) & ht.mask; } - if (gi < 0) { + + int64_t gi; + if (gi_found != GHT_EMPTY) { + gi = gi_found; + } else { if (ngroups >= max_groups) { - for (int64_t g = 0; g < ngroups; g++) { - ray_release(str_keys[g]); - ray_release(idx_vecs[g]); + if (!group_grow(&val_block, &ivblock, &gvals, &idx_vecs, + ngroups, &max_groups)) { + for (int64_t g = 0; g < ngroups; g++) ray_release(idx_vecs[g]); + group_ht_free(&ht); + ray_free(val_block); ray_free(ivblock); + return ray_error("oom", NULL); } - ray_free(val_block); ray_free(ivblock); ray_free(skblock); - return ray_error("limit", NULL); + sctx.gvals = gvals; } gi = ngroups++; - str_keys[gi] = ray_str(sp ? sp : "", slen); + gvals[gi] = i; idx_vecs[gi] = ray_vec_new(RAY_I64, 0); + ht.slots[slot] = (uint32_t)gi; + ht.count++; + if (ht.count * 2 > ht.cap) { + if (!group_ht_grow(&ht, ght_str_hash_gi, &sctx)) { + for (int64_t g = 0; g < ngroups; g++) ray_release(idx_vecs[g]); + group_ht_free(&ht); + ray_free(val_block); ray_free(ivblock); + return ray_error("oom", NULL); + } + } } idx_vecs[gi] = ray_vec_append(idx_vecs[gi], &i); } + group_ht_free(&ht); - /* Build dict: keys as RAY_STR vec from str_keys, vals as LIST of idx vecs. */ + /* Build keys from each group's first source row, preserving encounter order. */ ray_t* keys_vec = ray_vec_new(RAY_STR, ngroups); if (RAY_IS_ERR(keys_vec)) { - for (int64_t g = 0; g < ngroups; g++) { - ray_release(str_keys[g]); - ray_release(idx_vecs[g]); - } - ray_free(val_block); ray_free(ivblock); ray_free(skblock); + for (int64_t g = 0; g < ngroups; g++) ray_release(idx_vecs[g]); + ray_free(val_block); ray_free(ivblock); return ray_error("oom", NULL); } for (int64_t g = 0; g < ngroups; g++) { - keys_vec = ray_str_vec_append(keys_vec, ray_str_ptr(str_keys[g]), ray_str_len(str_keys[g])); - ray_release(str_keys[g]); + const ray_str_t* key = &desc[gvals[g]]; + keys_vec = ray_str_vec_append(keys_vec, + ray_str_t_ptr(key, pool), key->len); } ray_t* vals_lst = ray_list_new(ngroups); if (RAY_IS_ERR(vals_lst)) { - ray_release(keys_vec); ray_free(skblock); goto gfail; + ray_release(keys_vec); goto gfail; } for (int64_t g = 0; g < ngroups; g++) { vals_lst = ray_list_append(vals_lst, idx_vecs[g]); ray_release(idx_vecs[g]); idx_vecs[g] = NULL; - if (RAY_IS_ERR(vals_lst)) { ray_release(keys_vec); ray_free(skblock); goto gfail; } + if (RAY_IS_ERR(vals_lst)) { ray_release(keys_vec); goto gfail; } } - ray_free(val_block); ray_free(ivblock); ray_free(skblock); + ray_free(val_block); ray_free(ivblock); return ray_dict_new(keys_vec, vals_lst); } diff --git a/src/ops/cmp.c b/src/ops/cmp.c index 67ed6a44c..831ab9968 100644 --- a/src/ops/cmp.c +++ b/src/ops/cmp.c @@ -158,6 +158,26 @@ ray_t* ray_lte_fn(ray_t* a, ray_t* b) { return make_bool(as_f64(a) <= as_f64(b) ? 1 : 0); } +/* Elementwise dyadic overloads for the aggregate names `min` and `max`. + * Their vectorization is supplied by the evaluator, just like comparisons. */ +ray_t* ray_min2_fn(ray_t* a, ray_t* b) { + ray_t* cmp = ray_lte_fn(a, b); + if (!cmp || RAY_IS_ERR(cmp)) return cmp ? cmp : ray_error("oom", NULL); + ray_t* out = cmp->b8 ? a : b; + ray_retain(out); + ray_release(cmp); + return out; +} + +ray_t* ray_max2_fn(ray_t* a, ray_t* b) { + ray_t* cmp = ray_gte_fn(a, b); + if (!cmp || RAY_IS_ERR(cmp)) return cmp ? cmp : ray_error("oom", NULL); + ray_t* out = cmp->b8 ? a : b; + ray_retain(out); + ray_release(cmp); + return out; +} + ray_t* ray_eq_fn(ray_t* a, ray_t* b) { /* Handle null forms (RAY_NULL_OBJ, typed null atoms) */ diff --git a/src/ops/collection.c b/src/ops/collection.c index 8380545d1..798096d72 100644 --- a/src/ops/collection.c +++ b/src/ops/collection.c @@ -2260,6 +2260,33 @@ ray_t* ray_at_fn(ray_t* vec, ray_t* idx) { /* Dict key access: (at dict key) → value or 0Nl if missing */ if (vec->type == RAY_DICT) { + /* Vectorized lookup keeps LIST available as a composite scalar key + * (xkey uses lists for multi-column keys), while every typed vector + * is interpreted as a batch of independent keys. */ + if (is_collection(idx) && idx->type != RAY_LIST) { + int64_t n = ray_len(idx); + ray_t* out = ray_list_new(n); + if (!out || RAY_IS_ERR(out)) return out ? out : ray_error("oom", NULL); + for (int64_t i = 0; i < n; i++) { + int allocated = 0; + ray_t* key = collection_elem(idx, i, &allocated); + if (!key || RAY_IS_ERR(key)) { + ray_release(out); + return key ? key : ray_error("oom", NULL); + } + ray_t* value = ray_at_fn(vec, key); + if (allocated) ray_release(key); + if (!value || RAY_IS_ERR(value)) { + ray_release(out); + return value ? value : ray_error("oom", NULL); + } + out = ray_list_append(out, value); + ray_release(value); + if (!out || RAY_IS_ERR(out)) + return out ? out : ray_error("oom", NULL); + } + return out; + } ray_t* v = ray_dict_get(vec, idx); if (v) return v; return ray_typed_null(-RAY_I64); /* 0Nl for missing key */ @@ -2364,6 +2391,14 @@ ray_t* ray_at_fn(ray_t* vec, ray_t* idx) { return elem; } +/* (fill replacement values) — replace null cells in values. Registered as + * atomic, so scalar replacement broadcasts and vector replacements zip. */ +ray_t* ray_fill_fn(ray_t* replacement, ray_t* value) { + ray_t* out = RAY_ATOM_IS_NULL(value) ? replacement : value; + ray_retain(out); + return out; +} + /* (find vec val) — index of first occurrence, or -1 */ ray_t* ray_find_fn(ray_t* vec, ray_t* val) { if (ray_is_lazy(vec)) vec = ray_lazy_materialize(vec); diff --git a/src/ops/join.c b/src/ops/join.c index bb5138b0f..bf818fa2a 100644 --- a/src/ops/join.c +++ b/src/ops/join.c @@ -71,6 +71,37 @@ static int join_store_key_cell(ray_t* dst, int64_t dst_row, /* ── Hash helper (shared by radix and chained HT join paths) ──────────── */ +/* Resolve a STR cell directly from its 16-byte descriptor. Join hashing and + * equality execute once per input/matched row; calling ray_str_vec_get there + * repeatedly re-runs type/range checks and hides the descriptor prefix fast + * path from the compiler. The join has already validated both the type and + * row bounds, so retain only the slice adjustment and pool resolution here. */ +static inline const ray_str_t* join_str_cell(ray_t* col, int64_t row, + const char** pool_out) { + ray_t* owner = col; + if (col->attrs & RAY_ATTR_SLICE) { + owner = col->slice_parent; + row += col->slice_offset; + } + *pool_out = (owner->str_pool && !RAY_IS_ERR(owner->str_pool)) + ? (const char*)ray_data(owner->str_pool) : NULL; + return &((const ray_str_t*)ray_data(owner))[row]; +} + +/* The radix STR specialization calls this only after the complete cached + * 32-bit hashes match. Shared-pool offsets are exact identities; otherwise + * one content comparison preserves collision-safe equality. */ +static inline bool join_str_eq_hashed(const ray_str_t* a, const char* pool_a, + const ray_str_t* b, const char* pool_b) { + if (a->len != b->len) return false; + if (a->len == 0) return true; + if (!ray_str_is_inline(a) && pool_a && pool_a == pool_b && + a->pool_off == b->pool_off) return true; + const char* pa = ray_str_t_ptr(a, pool_a); + const char* pb = ray_str_t_ptr(b, pool_b); + return memcmp(pa, pb, a->len) == 0; +} + static uint64_t hash_row_keys(ray_t** key_vecs, uint32_t n_keys, int64_t row) { uint64_t h = 0; for (uint32_t k = 0; k < n_keys; k++) { @@ -80,7 +111,13 @@ static uint64_t hash_row_keys(ray_t** key_vecs, uint32_t n_keys, int64_t row) { if (ray_vec_is_null(col, row)) return h ^ ((uint64_t)row * 0x9E3779B97F4A7C15ULL); uint64_t kh; - if (col->type == RAY_F64) { + if (col->type == RAY_STR) { + const char* pool = NULL; + const ray_str_t* str = join_str_cell(col, row, &pool); + kh = ray_str_is_inline(str) + ? ray_hash_bytes(str->data, str->len) + : ray_str_t_hash32(str, pool); + } else if (col->type == RAY_F64) { kh = ray_hash_f64(((double*)ray_data(col))[row]); } else { int64_t kv = read_col_i64(ray_data(col), row, col->type, col->attrs); @@ -170,15 +207,49 @@ typedef struct { ray_t** key_vecs; uint32_t n_keys; uint32_t* hashes; /* output: hash[row] */ + const ray_str_t* str_desc; /* one-key STR specialization, else NULL */ + const char* str_pool; } join_radix_hash_ctx_t; static void join_radix_hash_fn(void* raw, uint32_t wid, int64_t start, int64_t end) { (void)wid; join_radix_hash_ctx_t* c = (join_radix_hash_ctx_t*)raw; + if (c->str_desc) { + const ray_str_t* desc = c->str_desc; + const char* pool = c->str_pool; + for (int64_t r = start; r < end; r++) { + const ray_str_t* str = &desc[r]; + c->hashes[r] = ray_str_is_inline(str) + ? (uint32_t)ray_hash_bytes(str->data, str->len) + : ray_str_t_hash32(str, pool); + } + return; + } for (int64_t r = start; r < end; r++) c->hashes[r] = (uint32_t)hash_row_keys(c->key_vecs, c->n_keys, r); } +static join_radix_hash_ctx_t join_radix_hash_ctx(ray_t** keys, uint32_t n_keys, + uint32_t* hashes) { + join_radix_hash_ctx_t c = { + .key_vecs = keys, .n_keys = n_keys, .hashes = hashes, + .str_desc = NULL, .str_pool = NULL, + }; + if (n_keys == 1 && keys[0] && keys[0]->type == RAY_STR) { + ray_t* col = keys[0]; + ray_t* owner = col; + int64_t off = 0; + if (col->attrs & RAY_ATTR_SLICE) { + owner = col->slice_parent; + off = col->slice_offset; + } + c.str_desc = (const ray_str_t*)ray_data(owner) + off; + c.str_pool = (owner->str_pool && !RAY_IS_ERR(owner->str_pool)) + ? (const char*)ray_data(owner->str_pool) : NULL; + } + return c; +} + /* Context for parallel partition histogram + scatter (pre-computed hashes). * Uses fixed row assignment: task i processes rows [i*chunk, (i+1)*chunk). * This ensures histogram and scatter see the same row ranges per task, @@ -505,7 +576,14 @@ static inline bool join_keys_eq(ray_t* const* l_vecs, ray_t* const* r_vecs, uint if (!lc || !rc) return false; /* NULL != NULL in join predicates */ if (ray_vec_is_null(lc, l) || ray_vec_is_null(rc, r)) return false; - if (lc->type == RAY_F64) { + if (lc->type == RAY_STR || rc->type == RAY_STR) { + if (lc->type != RAY_STR || rc->type != RAY_STR) return false; + const char* lpool = NULL; + const char* rpool = NULL; + const ray_str_t* ls = join_str_cell(lc, l, &lpool); + const ray_str_t* rs = join_str_cell(rc, r, &rpool); + if (!ray_str_t_eq(ls, lpool, rs, rpool)) return false; + } else if (lc->type == RAY_F64) { if (((double*)ray_data(lc))[l] != ((double*)ray_data(rc))[r]) return false; } else { int64_t lv = read_col_i64(ray_data(lc), l, lc->type, lc->attrs); @@ -546,6 +624,10 @@ typedef struct { ray_t** l_key_vecs; ray_t** r_key_vecs; uint32_t n_keys; + const ray_str_t* l_str_desc; /* one-key STR specialization */ + const ray_str_t* r_str_desc; + const char* l_str_pool; + const char* r_str_pool; uint8_t join_type; /* Per-partition output: pp_l[p], pp_r[p] are local buffers */ int32_t** pp_l; /* per-partition left indices (int32_t) */ @@ -599,6 +681,7 @@ static void join_radix_build_probe_fn(void* raw, uint32_t wid, int64_t task_star join_radix_part_t* rp = &c->r_parts[p]; join_radix_part_t* lp = &c->l_parts[p]; + bool str_specialized = c->l_str_desc && c->r_str_desc; /* Test knob: force the chained-path fallback. Bail before allocating * anything (pp headers are still NULL → cleanup-safe). */ @@ -709,14 +792,36 @@ static void join_radix_build_probe_fn(void* raw, uint32_t wid, int64_t task_star uint32_t h = lp->entries[i].hash; uint32_t lr = lp->entries[i].row_idx; uint32_t slot = h & ht_mask; - if (i + 4 < lp->count) - __builtin_prefetch(&ht[(lp->entries[i + 4].hash & ht_mask) * 2], 0, 1); + if (i + 4 < lp->count) { + join_radix_entry_t future = lp->entries[i + 4]; + uint32_t future_slot = future.hash & ht_mask; + __builtin_prefetch(&ht[future_slot * 2], 0, 1); + if (str_specialized) { + const ray_str_t* ls = &c->l_str_desc[future.row_idx]; + if (!ray_str_is_inline(ls) && c->l_str_pool) + __builtin_prefetch(c->l_str_pool + ls->pool_off, 0, 1); + + /* At a 0.5 HT load factor the initial slot is usually the + * matching digest. Pull its pool payload forward too; a + * collision merely makes this a harmless speculative read. */ + uint32_t rr = ht[future_slot * 2 + 1]; + if (rr != RADIX_HT_EMPTY && ht[future_slot * 2] == future.hash) { + const ray_str_t* rs = &c->r_str_desc[rr]; + if (!ray_str_is_inline(rs) && c->r_str_pool) + __builtin_prefetch(c->r_str_pool + rs->pool_off, 0, 1); + } + } + } bool matched = false; while (ht[slot * 2 + 1] != RADIX_HT_EMPTY) { if (ht[slot * 2] == h) { uint32_t rr = ht[slot * 2 + 1]; - if (join_keys_eq(c->l_key_vecs, c->r_key_vecs, c->n_keys, - (int64_t)lr, (int64_t)rr)) { + bool keys_equal = str_specialized + ? join_str_eq_hashed(&c->l_str_desc[lr], c->l_str_pool, + &c->r_str_desc[rr], c->r_str_pool) + : join_keys_eq(c->l_key_vecs, c->r_key_vecs, c->n_keys, + (int64_t)lr, (int64_t)rr); + if (keys_equal) { if (!bp_grow_bufs(c, p, &pl, &pr, &cap, cnt)) goto done; pl[cnt] = (int32_t)lr; @@ -1007,15 +1112,6 @@ static ray_t* exec_join_flat(ray_graph_t* g, ray_op_t* op, ray_t* left_table, ra r_key_vecs[k] = rk->literal; } - /* RAY_STR keys not yet supported (16-byte elements vs 8-byte hash/eq slots) */ - for (uint32_t k = 0; k < n_keys; k++) { - if ((l_key_vecs[k] && l_key_vecs[k]->type == RAY_STR) || - (r_key_vecs[k] && r_key_vecs[k]->type == RAY_STR)) { - scratch_free(key_vecs_hdr); - return ray_error("nyi", NULL); - } - } - /* Sequential LUT warm-up BEFORE any dispatch (see join_warm_sym_luts). */ if (!join_warm_sym_luts(l_key_vecs, r_key_vecs, n_keys)) { scratch_free(key_vecs_hdr); @@ -1070,8 +1166,8 @@ static ray_t* exec_join_flat(ray_graph_t* g, ray_op_t* op, ray_t* left_table, ra if (l_hash_hdr) scratch_free(l_hash_hdr); goto chained_ht_fallback; } - join_radix_hash_ctx_t rhctx = { .key_vecs = build_keys, .n_keys = n_keys, .hashes = r_hashes }; - join_radix_hash_ctx_t lhctx = { .key_vecs = probe_keys, .n_keys = n_keys, .hashes = l_hashes }; + join_radix_hash_ctx_t rhctx = join_radix_hash_ctx(build_keys, n_keys, r_hashes); + join_radix_hash_ctx_t lhctx = join_radix_hash_ctx(probe_keys, n_keys, l_hashes); if (pool) { ray_pool_dispatch(pool, join_radix_hash_fn, &rhctx, build_rows); ray_pool_dispatch(pool, join_radix_hash_fn, &lhctx, probe_rows); @@ -1166,6 +1262,8 @@ static ray_t* exec_join_flat(ray_graph_t* g, ray_op_t* op, ray_t* left_table, ra .l_parts = l_parts, .r_parts = r_parts, .l_key_vecs = probe_keys, .r_key_vecs = build_keys, .n_keys = n_keys, .join_type = join_type, + .l_str_desc = lhctx.str_desc, .r_str_desc = rhctx.str_desc, + .l_str_pool = lhctx.str_pool, .r_str_pool = rhctx.str_pool, .pp_l = pp_l, .pp_r = pp_r, .pp_l_hdr = pp_l_hdr, .pp_r_hdr = pp_r_hdr, .part_counts = part_counts, .pp_cap = pp_cap, @@ -1762,10 +1860,6 @@ int64_t ray_join_perpart_runs(void) { * - ANTI: eligible only if the parted side is LEFT, same reasoning. * - FULL (join_type 2): INELIGIBLE always — unmatched rows from BOTH * sides can span segments. - * STR keys are `nyi` in the flat kernel regardless — declining here just - * avoids the wasted per-segment calls (the flat kernel would return the - * same nyi on the first segment either way). - * * `op->opcode` distinguishes OP_JOIN (join_type 0/1/2 from ext->join) from * OP_ANTIJOIN (always anti semantics, no join_type field consulted). * Returns NULL if the shape is ineligible (caller falls through to the @@ -1929,15 +2023,6 @@ static ray_t* exec_antijoin_flat(ray_graph_t* g, ray_op_t* op, r_key_vecs[k] = rk->literal; } - /* RAY_STR keys not yet supported */ - for (uint32_t k = 0; k < n_keys; k++) { - if ((l_key_vecs[k] && l_key_vecs[k]->type == RAY_STR) || - (r_key_vecs[k] && r_key_vecs[k]->type == RAY_STR)) { - scratch_free(key_vecs_hdr); - return ray_error("nyi", NULL); - } - } - /* Sequential LUT warm-up BEFORE the parallel build dispatch. */ if (!join_warm_sym_luts(l_key_vecs, r_key_vecs, n_keys)) { scratch_free(key_vecs_hdr); diff --git a/src/ops/query.c b/src/ops/query.c index a0b5b2d52..cab918021 100644 --- a/src/ops/query.c +++ b/src/ops/query.c @@ -985,6 +985,12 @@ ray_op_t* compile_expr_dag(ray_graph_t* g, ray_t* expr) { if (expr->type == -RAY_SYM && !(expr->attrs & ATTR_QUOTED)) { ray_op_t* bound = cexpr_env_lookup(g, expr->i64); if (bound) return bound; + ray_t* local = ray_env_get_lexical_local(expr->i64); + if (local) { + if (ray_is_atom(local)) return ray_const_atom(g, local); + if (ray_is_vec(local)) return ray_const_vec(g, local); + return NULL; + } ray_t* s = ray_sym_str(expr->i64); if (!s) return NULL; @@ -1669,7 +1675,7 @@ static ray_t* bind_all_columns(ray_t* tbl) { for (int64_t c = 0; c < ncols; c++) { int64_t cn = ray_table_col_name(tbl, c); ray_t* cv = ray_table_get_col_idx(tbl, c); - if (cv) ray_env_set_local(cn, cv); + if (cv) ray_env_set_query_local(cn, cv); } return prev; } @@ -1706,7 +1712,7 @@ static bool query_atom_truthy(ray_t* v, bool* out) { } static ray_t* eval_where_mask(ray_t* where_expr, ray_t* tbl, const char* label) { - if (ray_env_push_scope() != RAY_OK) return ray_error("oom", NULL); + if (ray_env_push_query_scope() != RAY_OK) return ray_error("oom", NULL); ray_t* _aqt = bind_all_columns(tbl); ray_t* mask = ray_eval(where_expr); if (mask && !RAY_IS_ERR(mask)) @@ -1794,6 +1800,7 @@ static int is_agg_expr(ray_t* expr); /* defined below */ static int expr_refs_row_column(ray_t* expr, ray_t* tbl) { if (!expr) return 0; if (expr->type == -RAY_SYM && !(expr->attrs & ATTR_QUOTED)) { + if (ray_env_has_lexical_local(expr->i64)) return 0; if (ray_table_get_col(tbl, expr->i64)) return 1; /* Dotted name whose head is a column is a row-aligned ref — * `Timestamp.ss` flows through row-by-row the same as plain @@ -2703,7 +2710,7 @@ static ray_t* bind_col_slice(int64_t sym, ray_t* col, ray_t* idx_list) { if (!slice || RAY_IS_ERR(slice)) { return slice ? slice : ray_error("oom", NULL); } - ray_env_set_local(sym, slice); + ray_env_set_query_local(sym, slice); ray_release(slice); return NULL; } @@ -2759,7 +2766,7 @@ static ray_t* nonagg_eval_per_group_core(ray_t* expr, ray_t* tbl, for (int i = 0; i < n_cols; i++) cols[i] = ray_table_get_col(tbl, col_syms[i]); - if (ray_env_push_scope() != RAY_OK) { scratch_free(refs_hdr); return ray_error("oom", NULL); } + if (ray_env_push_query_scope() != RAY_OK) { scratch_free(refs_hdr); return ray_error("oom", NULL); } /* B3 Part 2: publish `tbl` as the active query table so a literal * column-name symbol inside `expr` resolves to its column during the @@ -2953,7 +2960,7 @@ static ray_t* eval_expr_per_row(ray_t* expr, ray_t* tbl, int64_t nrows) { for (int i = 0; i < n_cols; i++) cols[i] = ray_table_get_col(tbl, col_syms[i]); - if (ray_env_push_scope() != RAY_OK) { scratch_free(refs_hdr); return ray_error("oom", NULL); } + if (ray_env_push_query_scope() != RAY_OK) { scratch_free(refs_hdr); return ray_error("oom", NULL); } /* B3 Part 2: publish `tbl` as the active query table so a literal * column-name symbol inside `expr` resolves to its column during the @@ -2978,7 +2985,7 @@ static ray_t* eval_expr_per_row(ray_t* expr, ray_t* tbl, int64_t nrows) { scratch_free(refs_hdr); return arg ? arg : ray_error("domain", "select: failed to read column cell for per-row eval"); } - ray_env_set_local(col_syms[i], arg); + ray_env_set_query_local(col_syms[i], arg); if (allocated) ray_release(arg); } @@ -3075,7 +3082,7 @@ static ray_t* eval_expr_per_row(ray_t* expr, ray_t* tbl, int64_t nrows) { * count — the caller enforces cross-column length agreement. Mirrors * eval_expr_per_row's scope save/restore, including on every error exit. */ static ray_t* eval_expr_whole_column(ray_t* expr, ray_t* tbl) { - if (ray_env_push_scope() != RAY_OK) return ray_error("oom", NULL); + if (ray_env_push_query_scope() != RAY_OK) return ray_error("oom", NULL); ray_t* _aqt = bind_all_columns(tbl); ray_t* result = ray_eval(expr); /* distinct/asc/desc/reverse return a lazy DAG chain (RAY_LAZY) — force it @@ -3120,7 +3127,7 @@ static ray_t* aggr_unary_per_group_buf(ray_t* expr, ray_t* tbl, } if (!src) { /* Bind table cols and eval — same pattern as the existing path. */ - if (ray_env_push_scope() != RAY_OK) return ray_error("oom", NULL); + if (ray_env_push_query_scope() != RAY_OK) return ray_error("oom", NULL); ray_t* _aqt = bind_all_columns(tbl); src = ray_eval(col_expr); g_active_query_table = _aqt; @@ -3238,7 +3245,7 @@ static ray_t* aggr_med_per_group_buf(ray_t* expr, ray_t* tbl, if (src) ray_retain(src); } if (!src) { - if (ray_env_push_scope() != RAY_OK) return ray_error("oom", NULL); + if (ray_env_push_query_scope() != RAY_OK) return ray_error("oom", NULL); ray_t* _aqt = bind_all_columns(tbl); src = ray_eval(col_expr); g_active_query_table = _aqt; @@ -3796,7 +3803,7 @@ static ray_t* count_distinct_per_group_buf(ray_t* inner_expr, ray_t* tbl, if (src) ray_retain(src); } if (!src) { - if (ray_env_push_scope() != RAY_OK) return ray_error("oom", NULL); + if (ray_env_push_query_scope() != RAY_OK) return ray_error("oom", NULL); ray_t* _aqt = bind_all_columns(tbl); src = ray_eval(inner_expr); g_active_query_table = _aqt; @@ -3971,7 +3978,7 @@ static ray_t* count_distinct_per_group_groups(ray_t* inner_expr, ray_t* tbl, if (src) ray_retain(src); } if (!src) { - if (ray_env_push_scope() != RAY_OK) return ray_error("oom", NULL); + if (ray_env_push_query_scope() != RAY_OK) return ray_error("oom", NULL); ray_t* _aqt = bind_all_columns(tbl); src = ray_eval(inner_expr); g_active_query_table = _aqt; @@ -6173,12 +6180,12 @@ ray_t* ray_select(ray_t** args, int64_t n) { where_expr = NULL; } - ray_env_push_scope(); + ray_env_push_query_scope(); int64_t in_ncols = ray_table_ncols(tbl); for (int64_t c = 0; c < in_ncols; c++) { int64_t cn = ray_table_col_name(tbl, c); ray_t* cv = ray_table_get_col_idx(tbl, c); - if (cv) ray_env_set_local(cn, cv); + if (cv) ray_env_set_query_local(cn, cv); } by_sym_vec_owned = ray_vec_new(RAY_SYM, nk); @@ -6257,14 +6264,14 @@ ray_t* ray_select(ray_t** args, int64_t n) { failed = true; break; } materialized_refs[ri] = flat; - ray_env_set_local(ref_syms[ri], flat); + ray_env_set_query_local(ref_syms[ri], flat); } } if (failed) { for (int ri = 0; ri < n_refs; ri++) { if (materialized_refs[ri]) { ray_t* ref_col = ray_table_get_col(tbl, ref_syms[ri]); - if (ref_col) ray_env_set_local(ref_syms[ri], ref_col); + if (ref_col) ray_env_set_query_local(ref_syms[ri], ref_col); ray_release(materialized_refs[ri]); } } @@ -6277,7 +6284,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { for (int ri = 0; ri < n_refs; ri++) { if (materialized_refs[ri]) { ray_t* ref_col = ray_table_get_col(tbl, ref_syms[ri]); - if (ref_col) ray_env_set_local(ref_syms[ri], ref_col); + if (ref_col) ray_env_set_query_local(ref_syms[ri], ref_col); ray_release(materialized_refs[ri]); } } @@ -6300,7 +6307,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { tbl = new_tbl; /* Re-bind the newly added column under its dict key so * later dict vals can reference earlier keys. */ - ray_env_set_local(k->i64, col_vec); + ray_env_set_query_local(k->i64, col_vec); sv_data[i] = k->i64; } ray_env_pop_scope(); @@ -8111,7 +8118,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { /* Non-aggregation expression: evaluate on full table, * then gather per-group subsets into a LIST column * (non-agg produces list-of-vectors). */ - if (ray_env_push_scope() != RAY_OK) { + if (ray_env_push_query_scope() != RAY_OK) { for (int ai = 0; ai < n_agg_out; ai++) { if (agg_results[ai]) ray_release(agg_results[ai]); } scratch_free(aggnames_hdr); scratch_free(aggres_hdr); ray_release(groups); if (eval_tbl != tbl) ray_release(eval_tbl); ray_release(tbl); @@ -8930,10 +8937,10 @@ ray_t* ray_select(ray_t** args, int64_t n) { /* Evaluate by_expr against the (empty) filtered table * to get a length-0 key vector typed like the * non-empty path would produce it. */ - ray_env_push_scope(); + ray_env_push_query_scope(); for (int64_t c = 0; c < nc0; c++) { - ray_env_set_local(ray_table_col_name(filtered_tbl, c), - ray_table_get_col_idx(filtered_tbl, c)); + ray_env_set_query_local(ray_table_col_name(filtered_tbl, c), + ray_table_get_col_idx(filtered_tbl, c)); } ray_t* ck_vec = ray_eval(by_expr); ray_env_pop_scope(); @@ -8973,11 +8980,11 @@ ray_t* ray_select(ray_t** args, int64_t n) { /* Computed group key (e.g., xbar) — fall back to eval-level groupby */ ray_release(grouped); int64_t tbl_ncols = ray_table_ncols(filtered_tbl); - ray_env_push_scope(); + ray_env_push_query_scope(); for (int64_t c = 0; c < tbl_ncols; c++) { int64_t cn = ray_table_col_name(filtered_tbl, c); ray_t* cv = ray_table_get_col_idx(filtered_tbl, c); - ray_env_set_local(cn, cv); + ray_env_set_query_local(cn, cv); } ray_t* computed_key = ray_eval(by_expr); ray_env_pop_scope(); @@ -9960,7 +9967,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { ray_t* cerr = NULL; bool layout_ok = hbase + n_hidden_aggs <= rnc; if (layout_ok) { - if (ray_env_push_scope() != RAY_OK) { + if (ray_env_push_query_scope() != RAY_OK) { cerr = ray_error("oom", NULL); } else { for (int hi = 0; hi < n_hidden_aggs; hi++) @@ -10762,7 +10769,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { continue; } - if (ray_env_push_scope() != RAY_OK) { + if (ray_env_push_query_scope() != RAY_OK) { scatter_err = ray_error("oom", NULL); break; } ray_t* _aqt = bind_all_columns(tbl); @@ -11500,11 +11507,11 @@ ray_t* ray_update(ray_t** args, int64_t n) { if (!mask_vec || RAY_IS_ERR(mask_vec)) { /* Bind column names to column vectors in env, then eval */ int64_t ncols2 = ray_table_ncols(tbl); - ray_env_push_scope(); + ray_env_push_query_scope(); for (int64_t c = 0; c < ncols2; c++) { int64_t cn = ray_table_col_name(tbl, c); ray_t* col = ray_table_get_col_idx(tbl, c); - ray_env_set(cn, col); + ray_env_set_query_local(cn, col); } mask_vec = ray_eval(where_expr); ray_env_pop_scope(); @@ -11574,11 +11581,11 @@ ray_t* ray_update(ray_t** args, int64_t n) { if (!expr_vec || RAY_IS_ERR(expr_vec)) { /* Fallback: eval with column bindings */ int64_t ncols_e = ray_table_ncols(tbl); - ray_env_push_scope(); + ray_env_push_query_scope(); for (int64_t c2 = 0; c2 < ncols_e; c2++) { int64_t cn = ray_table_col_name(tbl, c2); ray_t* col2 = ray_table_get_col_idx(tbl, c2); - ray_env_set(cn, col2); + ray_env_set_query_local(cn, col2); } expr_vec = ray_eval(update_expr); ray_env_pop_scope(); @@ -11879,11 +11886,11 @@ ray_t* ray_update(ray_t** args, int64_t n) { if (!expr_vec || RAY_IS_ERR(expr_vec)) { /* Fallback: eval with column bindings */ int64_t ncols_f = ray_table_ncols(tbl); - ray_env_push_scope(); + ray_env_push_query_scope(); for (int64_t cf = 0; cf < ncols_f; cf++) { int64_t cn = ray_table_col_name(tbl, cf); ray_t* colf = ray_table_get_col_idx(tbl, cf); - ray_env_set(cn, colf); + ray_env_set_query_local(cn, colf); } expr_vec = ray_eval(update_expr); ray_env_pop_scope(); @@ -14251,6 +14258,30 @@ static ray_t* join_impl(ray_t** args, int64_t n, uint8_t join_type) { ray_graph_free(g); if (_bxk) ray_release(_bxk); return ray_error("type", "join: key must be a symbol name, got %s", ray_type_name(ke_t)); } + int64_t key_name = key_elems[i]->i64; + ray_t* left_key_col = ray_table_get_col(left_tbl, key_name); + ray_t* right_key_col = ray_table_get_col(right_tbl, key_name); + if (!left_key_col || !right_key_col) { + ray_t* key_name_str = ray_sym_str(key_name); + const char* side = !left_key_col ? "left" : "right"; + scratch_free(keyops_hdr); + ray_graph_free(g); if (_bxk) ray_release(_bxk); + if (key_name_str) { + ray_t* err = ray_error("domain", "join: key column '%.*s' not found in %s table", + (int)ray_str_len(key_name_str), ray_str_ptr(key_name_str), side); + return err; + } + return ray_error("domain", "join: key column not found in %s table", side); + } + int8_t left_key_type = left_key_col->type; + int8_t right_key_type = right_key_col->type; + if (RAY_IS_PARTED(left_key_type)) left_key_type = (int8_t)RAY_PARTED_BASETYPE(left_key_type); + if (RAY_IS_PARTED(right_key_type)) right_key_type = (int8_t)RAY_PARTED_BASETYPE(right_key_type); + if ((left_key_type == RAY_STR) != (right_key_type == RAY_STR)) { + scratch_free(keyops_hdr); + ray_graph_free(g); if (_bxk) ray_release(_bxk); + return ray_error("type", "join: string key columns must have string type on both sides"); + } ray_t* name_str = ray_sym_str(key_elems[i]->i64); if (!name_str) { scratch_free(keyops_hdr); ray_graph_free(g); if (_bxk) ray_release(_bxk); return ray_error("domain", "join: unknown key symbol"); } lk[i] = ray_scan(g, ray_str_ptr(name_str)); @@ -14325,6 +14356,28 @@ static ray_t* antijoin_impl(ray_t** args, int64_t n) { ray_graph_free(g); if (_bxk) ray_release(_bxk); return ray_error("type", "antijoin: key must be a symbol name, got %s", ray_type_name(ke_t)); } + int64_t key_name = key_elems[i]->i64; + ray_t* left_key_col = ray_table_get_col(left_tbl, key_name); + ray_t* right_key_col = ray_table_get_col(right_tbl, key_name); + if (!left_key_col || !right_key_col) { + ray_t* key_name_str = ray_sym_str(key_name); + const char* side = !left_key_col ? "left" : "right"; + scratch_free(keyops_hdr); + ray_graph_free(g); if (_bxk) ray_release(_bxk); + if (key_name_str) + return ray_error("domain", "antijoin: key column '%.*s' not found in %s table", + (int)ray_str_len(key_name_str), ray_str_ptr(key_name_str), side); + return ray_error("domain", "antijoin: key column not found in %s table", side); + } + int8_t left_key_type = left_key_col->type; + int8_t right_key_type = right_key_col->type; + if (RAY_IS_PARTED(left_key_type)) left_key_type = (int8_t)RAY_PARTED_BASETYPE(left_key_type); + if (RAY_IS_PARTED(right_key_type)) right_key_type = (int8_t)RAY_PARTED_BASETYPE(right_key_type); + if ((left_key_type == RAY_STR) != (right_key_type == RAY_STR)) { + scratch_free(keyops_hdr); + ray_graph_free(g); if (_bxk) ray_release(_bxk); + return ray_error("type", "antijoin: string key columns must have string type on both sides"); + } ray_t* name_str = ray_sym_str(key_elems[i]->i64); if (!name_str) { scratch_free(keyops_hdr); ray_graph_free(g); if (_bxk) ray_release(_bxk); return ray_error("domain", "antijoin: unknown key symbol"); } lk[i] = ray_scan(g, ray_str_ptr(name_str)); diff --git a/src/ops/string.c b/src/ops/string.c index f7c1e16cb..481d28b32 100644 --- a/src/ops/string.c +++ b/src/ops/string.c @@ -1145,6 +1145,7 @@ static ray_t* substr_str_scalar_view(ray_t* input, int64_t start, int64_t length } memcpy(d->prefix, sp, 4); d->pool_off = s->pool_off + (uint32_t)st; + ray_str_t_cache_hash(d, pool); } else { ray_release(result); return NULL; @@ -1238,6 +1239,7 @@ static ray_t* substr_str_scalar_start_len_view(ray_t* input, } memcpy(d->prefix, sp, 4); d->pool_off = s->pool_off + (uint32_t)st0; + ray_str_t_cache_hash(d, pool); } else { ray_release(result); return NULL; diff --git a/src/ops/tblop.c b/src/ops/tblop.c index 884580aac..d6c869858 100644 --- a/src/ops/tblop.c +++ b/src/ops/tblop.c @@ -905,26 +905,54 @@ ray_t* ray_xkey_fn(ray_t* tbl, ray_t* keys_arg) { return keys && RAY_IS_ERR(keys) ? keys : (vals && RAY_IS_ERR(vals) ? vals : ray_error("oom", NULL)); } + /* Duplicate detection is an open-addressed set of indices into `keys`. + * The old all-prior-keys scan made construction quadratic. */ + size_t ht_cap = 16; + if ((uint64_t)nrows > SIZE_MAX / 2) { + ray_release(keys); ray_release(vals); + ray_free_raw(key_cols); ray_free_raw(key_syms); + return ray_error("limit", "xkey: table is too large"); + } + while (nrows > 0 && ht_cap < (size_t)nrows * 2) { + if (ht_cap > SIZE_MAX / 2) { + ray_release(keys); ray_release(vals); + ray_free_raw(key_cols); ray_free_raw(key_syms); + return ray_error("limit", "xkey: table is too large"); + } + ht_cap *= 2; + } + int64_t* ht = (int64_t*)ray_alloc_raw(ht_cap * sizeof(int64_t)); + if (!ht) { + ray_release(keys); ray_release(vals); + ray_free_raw(key_cols); ray_free_raw(key_syms); + return ray_error("oom", NULL); + } + for (size_t i = 0; i < ht_cap; i++) ht[i] = -1; + for (int64_t r = 0; r < nrows; r++) { ray_t* k = table_row_key(key_cols, nkeys, r); - if (!k || RAY_IS_ERR(k)) { ray_release(keys); ray_release(vals); ray_free_raw(key_cols); ray_free_raw(key_syms); return k ? k : ray_error("oom", NULL); } + if (!k || RAY_IS_ERR(k)) { ray_free_raw(ht); ray_release(keys); ray_release(vals); ray_free_raw(key_cols); ray_free_raw(key_syms); return k ? k : ray_error("oom", NULL); } ray_t** key_items = (ray_t**)ray_data(keys); - for (int64_t i = 0; i < keys->len; i++) { - if (atom_eq(key_items[i], k)) { + size_t slot = (size_t)ray_atom_hash(k) & (ht_cap - 1); + while (ht[slot] >= 0) { + if (atom_eq(key_items[ht[slot]], k)) { ray_release(k); + ray_free_raw(ht); ray_release(keys); ray_release(vals); ray_free_raw(key_cols); ray_free_raw(key_syms); return ray_error("domain", "xkey: duplicate key at row %lld", (long long)r); } + slot = (slot + 1) & (ht_cap - 1); } ray_t* v = table_row_value_dict(tbl, key_syms, nkeys, r); - if (!v || RAY_IS_ERR(v)) { ray_release(k); ray_release(keys); ray_release(vals); ray_free_raw(key_cols); ray_free_raw(key_syms); return v ? v : ray_error("oom", NULL); } + if (!v || RAY_IS_ERR(v)) { ray_release(k); ray_free_raw(ht); ray_release(keys); ray_release(vals); ray_free_raw(key_cols); ray_free_raw(key_syms); return v ? v : ray_error("oom", NULL); } ray_t* nkeys_obj = ray_list_append(keys, k); if (!nkeys_obj || RAY_IS_ERR(nkeys_obj)) { ray_release(k); ray_release(v); + ray_free_raw(ht); ray_release(keys); ray_release(vals); ray_free_raw(key_cols); @@ -932,11 +960,13 @@ ray_t* ray_xkey_fn(ray_t* tbl, ray_t* keys_arg) { return nkeys_obj ? nkeys_obj : ray_error("oom", NULL); } keys = nkeys_obj; + ht[slot] = keys->len - 1; ray_t* nvals_obj = ray_list_append(vals, v); if (!nvals_obj || RAY_IS_ERR(nvals_obj)) { ray_release(k); ray_release(v); + ray_free_raw(ht); ray_release(keys); ray_release(vals); ray_free_raw(key_cols); @@ -948,6 +978,7 @@ ray_t* ray_xkey_fn(ray_t* tbl, ray_t* keys_arg) { ray_release(v); } + ray_free_raw(ht); ray_free_raw(key_cols); ray_free_raw(key_syms); return ray_dict_new(keys, vals); diff --git a/src/store/col.c b/src/store/col.c index 7f0c17dbf..d36855d59 100644 --- a/src/store/col.c +++ b/src/store/col.c @@ -1212,7 +1212,8 @@ static ray_err_t col_validate_str_region(ray_t* hdr, const void* ptr, if (pool_size > mapped_size - offset - 32) return RAY_ERR_CORRUPT; - const ray_str_t* elems = (const ray_str_t*)((const char*)ptr + 32); + ray_str_t* elems = (ray_str_t*)((char*)ptr + 32); + const char* pool_base = (const char*)ptr + offset + 32; for (int64_t i = 0; i < hdr->len; i++) { uint32_t len = elems[i].len; if (len <= RAY_STR_INLINE_MAX) continue; @@ -1220,12 +1221,27 @@ static ray_err_t col_validate_str_region(ray_t* hdr, const void* ptr, len > pool_size - elems[i].pool_off) return RAY_ERR_CORRUPT; if (len >= 4) { - const char* p = (const char*)ptr + offset + 32 + elems[i].pool_off; + const char* p = pool_base + elems[i].pool_off; if (memcmp(elems[i].prefix, p, 4) != 0) return RAY_ERR_CORRUPT; } } + /* Pre-hash-cache column files persisted the descriptor's final four + * bytes as uninitialized padding. A nonzero legacy value must never be + * trusted as a content hash: equal strings could otherwise probe + * different join/group slots and silently produce wrong results. + * + * ray_vm_map_file is MAP_PRIVATE / copy-on-write, so refreshing the + * validated descriptors repairs both callers: ray_col_load copies these + * values into its buddy block, while ray_col_mmap retains the private + * repaired mapping without modifying the file. */ + for (int64_t i = 0; i < hdr->len; i++) { + if (ray_str_is_inline(&elems[i])) continue; + uint32_t h = (uint32_t)ray_str_t_hash(&elems[i], pool_base); + elems[i].hash32 = h != 0 ? h : 1u; + } + out->has_str_pool = true; out->str_pool_offset = offset; out->str_pool_size = pool_size; diff --git a/src/store/serde.c b/src/store/serde.c index 8698d2d45..bad939fa0 100644 --- a/src/store/serde.c +++ b/src/store/serde.c @@ -89,7 +89,8 @@ /* Lambdas reuse the attr byte for function-internal flags: 0x40 is * RAY_FN_COMPILED, not "has nulls". Only params/body cross the wire, so * compiled metadata must be stripped and rebuilt by the receiving runtime. */ -#define RAY_SERDE_LAMBDA_ATTR_WIRE_MASK ((uint8_t)0) +#define RAY_SERDE_LAMBDA_HAS_CLOSURE ((uint8_t)0x01) +#define RAY_SERDE_LAMBDA_ATTR_WIRE_MASK RAY_SERDE_LAMBDA_HAS_CLOSURE /* Helper: strlen with bounds */ static size_t safe_strlen(const uint8_t* buf, int64_t max) { @@ -231,7 +232,9 @@ int64_t ray_serde_size(ray_t* obj) { } case RAY_LAMBDA: { ray_t** slots = (ray_t**)ray_data(obj); - return 1 + 1 + ray_serde_size(slots[0]) + ray_serde_size(slots[1]); + int64_t size = 1 + 1 + ray_serde_size(slots[0]) + ray_serde_size(slots[1]); + if (LAMBDA_CLOSURE(obj)) size += ray_serde_size(LAMBDA_CLOSURE(obj)); + return size; } case RAY_UNARY: case RAY_BINARY: @@ -471,11 +474,13 @@ int64_t ray_ser_raw(uint8_t* buf, ray_t* obj) { } case RAY_LAMBDA: { - buf[0] = obj->attrs & RAY_SERDE_LAMBDA_ATTR_WIRE_MASK; + buf[0] = LAMBDA_CLOSURE(obj) ? RAY_SERDE_LAMBDA_HAS_CLOSURE : 0; buf++; ray_t** slots = (ray_t**)ray_data(obj); c = ray_ser_raw(buf, slots[0]); /* params */ c += ray_ser_raw(buf + c, slots[1]); /* body */ + if (LAMBDA_CLOSURE(obj)) + c += ray_ser_raw(buf + c, LAMBDA_CLOSURE(obj)); return 1 + 1 + c; } @@ -872,19 +877,37 @@ static ray_t* de_raw_inner(uint8_t* buf, int64_t* len) { return body; } - /* Build lambda: allocate with 7 slots (same as eval.c) */ - ray_t* lambda = ray_alloc(7 * sizeof(ray_t*)); + ray_t* closure = NULL; + if (lam_attrs & RAY_SERDE_LAMBDA_HAS_CLOSURE) { + closure = ray_de_raw(buf + (saved - *len), len); + if (!closure || RAY_IS_ERR(closure)) { + ray_release(params); + ray_release(body); + return closure; + } + if (closure->type != RAY_DICT) { + ray_release(params); + ray_release(body); + ray_release(closure); + return ray_error("type", "deserialize lambda: closure must be a dict"); + } + } + + /* Build lambda: allocate with 8 slots (same as eval.c). */ + ray_t* lambda = ray_alloc(8 * sizeof(ray_t*)); if (!lambda || RAY_IS_ERR(lambda)) { ray_release(params); ray_release(body); + ray_release(closure); return lambda; } lambda->type = RAY_LAMBDA; - lambda->attrs = lam_attrs & RAY_SERDE_LAMBDA_ATTR_WIRE_MASK; + lambda->attrs = 0; lambda->len = 0; - memset(ray_data(lambda), 0, 7 * sizeof(ray_t*)); + memset(ray_data(lambda), 0, 8 * sizeof(ray_t*)); ((ray_t**)ray_data(lambda))[0] = params; ((ray_t**)ray_data(lambda))[1] = body; + LAMBDA_CLOSURE(lambda) = closure; return lambda; } diff --git a/src/vec/str.h b/src/vec/str.h index 6afe2e6a5..f6ce5d47b 100644 --- a/src/vec/str.h +++ b/src/vec/str.h @@ -39,7 +39,7 @@ typedef union { struct { uint32_t len; char data[12]; }; /* inline: len <= 12 */ struct { uint32_t len_; char prefix[4]; /* pooled: len > 12 */ - uint32_t pool_off; uint32_t _pad; }; + uint32_t pool_off; uint32_t hash32; }; } ray_str_t; #define RAY_STR_INLINE_MAX 12 @@ -66,6 +66,9 @@ static inline bool ray_str_t_eq(const ray_str_t* a, const char* pool_a, if (ray_str_is_inline(a)) { return memcmp(a->data, b->data, a->len) == 0; } + /* Slices, clones, and separately assembled tables can share one pool. + * Equal offsets in the same immutable pool identify the same bytes. */ + if (pool_a && pool_a == pool_b && a->pool_off == b->pool_off) return true; /* Both pooled: check prefix first */ if (memcmp(a->prefix, b->prefix, 4) != 0) return false; return memcmp(pool_a + a->pool_off, pool_b + b->pool_off, a->len) == 0; @@ -115,4 +118,21 @@ static inline uint64_t ray_str_t_hash(const ray_str_t* s, const char* pool_base) return h; } +/* Pooled descriptors have four bytes that are not needed for addressing. + * Cache a non-zero 32-bit content hash there so joins can hash long STR keys + * without rereading the pool on every execution. A zero field denotes an + * older/on-disk descriptor without a cache; computing it remains fully + * backward compatible. Inline strings use all 12 payload bytes and are + * cheap enough to hash directly. */ +static inline uint32_t ray_str_t_hash32(const ray_str_t* s, + const char* pool_base) { + if (!ray_str_is_inline(s) && s->hash32 != 0) return s->hash32; + uint32_t h = (uint32_t)ray_str_t_hash(s, pool_base); + return h != 0 ? h : 1u; +} + +static inline void ray_str_t_cache_hash(ray_str_t* s, const char* pool_base) { + if (!ray_str_is_inline(s)) s->hash32 = ray_str_t_hash32(s, pool_base); +} + #endif /* RAY_STR_H */ diff --git a/src/vec/vec.c b/src/vec/vec.c index 68679928a..5ff53e1c2 100644 --- a/src/vec/vec.c +++ b/src/vec/vec.c @@ -1133,6 +1133,7 @@ ray_t* ray_str_vec_append(ray_t* vec, const char* s, size_t len) { memcpy(elem->prefix, s, 4); elem->pool_off = (uint32_t)pool_off; + ray_str_t_cache_hash(elem, pool_base); vec->str_pool->len = pool_off + (int64_t)len; } @@ -1212,6 +1213,7 @@ ray_t* ray_str_vec_from_parts(const char* const* ptrs, const uint32_t* lens, d->len = lens[i]; d->pool_off = (uint32_t)pool_used; memcpy(d->prefix, ptrs[i], 4); + ray_str_t_cache_hash(d, pool_base); pool_used += (int64_t)lens[i]; } } @@ -1323,6 +1325,7 @@ ray_t* ray_str_vec_set(ray_t* vec, int64_t idx, const char* s, size_t len) { elem->len = (uint32_t)len; memcpy(elem->prefix, s, 4); elem->pool_off = (uint32_t)pool_used; + ray_str_t_cache_hash(elem, pool_base); vec->str_pool->len = pool_used + (int64_t)len; } diff --git a/test/rfl/ops/join_branch_cov.rfl b/test/rfl/ops/join_branch_cov.rfl index 972a47209..c6f26031f 100644 --- a/test/rfl/ops/join_branch_cov.rfl +++ b/test/rfl/ops/join_branch_cov.rfl @@ -352,26 +352,20 @@ (count (inner-join [price] bigFL bigFR)) -- 70000 ;; ────────────────────────────────────────────────────────────────── -;; L33: hash_row_keys — NULL key vector (key column missing from table) -;; When the join key doesn't exist in one table, the key_vec is NULL -;; and hash_row_keys skips it via `if (!col) continue;`. +;; Missing join keys are schema errors. They must not silently look like an +;; empty match set (or, for left/anti joins, a plausible all-unmatched result). ;; ────────────────────────────────────────────────────────────────── -;; inner-join: key exists only in left table → no right key vec → 0 matches (set mL (table [a val] (list [1 2] [10 20]))) (set mR (table [b val2] (list [1 2] [100 200]))) -(count (inner-join [a] mL mR)) -- 0 - -;; left-join: key exists only in left → all left rows unmatched -(count (left-join [a] mL mR)) -- 2 - -;; anti-join: key missing from right → all left rows pass (no matches possible) -(count (anti-join [a] mL mR)) -- 2 +(inner-join [a] mL mR) !- domain +(left-join [a] mL mR) !- domain +(anti-join [a] mL mR) !- domain ;; multi-key: one key exists in both, other only in left (set mkML (table [k1 k2 val] (list [1 2] [a b] [10 20]))) (set mkMR (table [k1 val2] (list [1 2] [100 200]))) -(count (inner-join [k1 k2] mkML mkMR)) -- 0 +(inner-join [k1 k2] mkML mkMR) !- domain ;; ────────────────────────────────────────────────────────────────── ;; Large table left-join with NULL keys — exercises L35 on radix path diff --git a/test/rfl/regress/issue_394.rfl b/test/rfl/regress/issue_394.rfl new file mode 100644 index 000000000..9c8b4a390 --- /dev/null +++ b/test/rfl/regress/issue_394.rfl @@ -0,0 +1,98 @@ +;; Issue #394 — verified runtime gaps and deliberate Rayfall semantics. + +;; STR columns are valid equi-join keys. +(set I394L (table [k lv] (list (list "alpha" "beta" "gamma") [1 2 3]))) +(set I394R (table [k rv] (list (list "gamma" "alpha" "delta") [30 10 40]))) +(count (inner-join [k] I394L I394R)) -- 2 +(sum (at (inner-join [k] I394L I394R) 'rv)) -- 40 +(count (anti-join [k] I394L I394R)) -- 1 + +;; Pooled (>12-byte) and sliced STR descriptors use the same fast path while +;; retaining content equality across independent string pools. +(set I394LongLKeys (as 'STR ["customer-account-alpha" "customer-account-beta" "customer-account-gamma"])) +(set I394LongRKeys (as 'STR ["customer-account-gamma" "customer-account-alpha" "customer-account-delta"])) +(set I394LongL (table [k lv] (list I394LongLKeys [1 2 3]))) +(set I394LongR (table [k rv] (list I394LongRKeys [30 10 40]))) +(sum (at (inner-join [k] I394LongL I394LongR) 'rv)) -- 40 +(set I394SliceL (table [k lv] (list (drop (as 'STR ["unused-long-prefix" "customer-account-alpha" "customer-account-beta"]) 1) [1 2]))) +(set I394SliceR (table [k rv] (list (drop (as 'STR ["unused-long-prefix" "customer-account-beta" "customer-account-alpha"]) 1) [20 10]))) +(sum (at (inner-join [k] I394SliceL I394SliceR) 'rv)) -- 30 + +;; STR remains a first-class component of composite join keys. +(set I394MultiL (table [tenant hk lv] (list [1 1 2 2] (as 'STR ["0123456789abcdef0123456789abcdef" "fedcba9876543210fedcba9876543210" "0123456789abcdef0123456789abcdef" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]) [10 20 30 40]))) +(set I394MultiR (table [tenant hk rv] (list [2 1 1 3] (as 'STR ["0123456789abcdef0123456789abcdef" "0123456789abcdef0123456789abcdef" "fedcba9876543210fedcba9876543210" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]) [300 100 200 400]))) +(count (inner-join [tenant hk] I394MultiL I394MultiR)) -- 3 +(sum (at (inner-join [tenant hk] I394MultiL I394MultiR) 'rv)) -- 600 +(count (anti-join [tenant hk] I394MultiL I394MultiR)) -- 1 + +;; STR grouping is hashed for both single and composite keys. More than 1024 +;; unique pooled strings exercises dynamic group growth and cached hashes. +(set I394GroupKeys (map (fn [i] (concat "0123456789abcdef0123456789abcdef" (as 'str i))) (til 1500))) +(count (key (group I394GroupKeys))) -- 1500 +(sum (map count (value (group I394GroupKeys)))) -- 1500 +(set I394GroupT (table [tenant hk amount] (list [1 1 1 2 2] (as 'STR ["0123456789abcdef0123456789abcdef" "0123456789abcdef0123456789abcdef" "fedcba9876543210fedcba9876543210" "0123456789abcdef0123456789abcdef" "0123456789abcdef0123456789abcdef"]) [10 20 30 40 50]))) +(count (select {from:I394GroupT by:[tenant hk] total:(sum amount)})) -- 3 +(sum (at (select {from:I394GroupT by:[tenant hk] total:(sum amount)}) 'total)) -- 150 + +;; A misspelled/missing join key is a schema error, not an empty result. +(inner-join [missing] I394L I394R) !- domain +(anti-join [missing] I394L I394R) !- domain + +;; Dyadic fill, min, and max are atomic and broadcast scalars. +(fill 0 [1 0Nl 3]) -- [1 0 3] +(min [3 9 1] [2 10 4]) -- [2 9 1] +(max [3 9 1] 5) -- [5 9 5] + +;; String find uses the typed hash-set path and preserves first-index/miss rules. +(find (list "alpha" "beta" "gamma") "beta") -- 1 +(find (list "alpha" "beta" "gamma") "absent") -- 0Nl + +;; Hashing is an internal implementation detail, not a language builtin. +(hash 42) !- name +(wyhash 42) !- name + +;; xkey construction is hash-based and keyed dictionaries accept vector probes. +(set I394T (table [k v] (list (til 5000) (+ 10 (til 5000))))) +(set I394D (xkey I394T 'k)) +(count I394D) -- 5000 +(at (at (at I394D [1 4999]) 0) 'v) -- 11 +(at (at (at I394D [1 4999]) 1) 'v) -- 5009 + +;; Nested functions capture visible lexical values. +(set I394MakeAdder (fn [x] (fn [y] (+ x y)))) +(set I394Add7 (I394MakeAdder 7)) +(I394Add7 5) -- 12 +(set I394Add7Roundtrip (de (ser I394Add7))) +(I394Add7Roundtrip 6) -- 13 + +;; Lambda parameters and let bindings take precedence over query columns. +(set I394Q (table [x v] (list [1 2] [10 20]))) +(set I394SelectParam (fn [x] (select {y:x from:I394Q}))) +(at (I394SelectParam 99) 'y) -- [99 99] +(set I394UpdateParam (fn [x] (update {v:x from:I394Q}))) +(at (I394UpdateParam 77) 'v) -- [77 77] + +;; Query scopes do not count as lexical shadowing. While the outer where: +;; expression has its `x` column mounted, the called lambda's inner query must +;; still mount its own same-named `x` column. `eval` forces the tree-walk path +;; that originally exposed the collision. +(set I394NestedInner (table [x] (list [10 20]))) +(set I394NestedOuter (table [x] (list [1 2]))) +(set I394NestedQuery (fn [] (first (at (select {v:(eval (list + x 0)) from:I394NestedInner}) 'v)))) +(count (select {from:I394NestedOuter where:(== (I394NestedQuery) 10)})) -- 2 +(set I394NestedDagQuery (fn [] (first (at (select {v:x from:I394NestedInner}) 'v)))) +(count (select {from:I394NestedOuter where:(== (I394NestedDagQuery) 10)})) -- 2 + +;; The same distinction must drive grouped-expression shape classification. +;; Otherwise the outer query's `x` makes the inner `(I394First x)` look +;; constant and silently broadcasts 10 into both inner groups. +(set I394NestedGroupedInner (table [g x] (list [1 2] [10 20]))) +(set I394First (fn [v] (first v))) +(set I394NestedGrouped (fn [] (at (select {picked:(I394First x) from:I394NestedGroupedInner by:g}) 'picked))) +(count (select {from:I394NestedOuter where:(== (last (I394NestedGrouped)) 20)})) -- 2 + +;; These issue claims conflict with established Rayfall semantics: `%` is +;; modulo, dict literals preserve query ASTs, and key(table) returns its schema. +(% 7 3) -- 1 +(first (at {a:(list 1 2)} 'a)) -- 'list +(key (select {s:(sum v) by:x from:I394Q})) -- [x s] diff --git a/test/rfl/type/as.rfl b/test/rfl/type/as.rfl index 316e24c85..15c98ad15 100644 --- a/test/rfl/type/as.rfl +++ b/test/rfl/type/as.rfl @@ -451,6 +451,24 @@ (type (as 'guid "d49f18a4-1969-49e8-9b8a-6bb9a4832eea")) -- 'guid ;; String → GUID → STR round-trip preserves the canonical 36-char form. (as 'STR (as 'guid "d49f18a4-1969-49e8-9b8a-6bb9a4832eea")) -- "d49f18a4-1969-49e8-9b8a-6bb9a4832eea" +;; Malformed GUID strings must be REJECTED, not silently decoded into a +;; wrong-but-valid GUID. The old parser skipped '-' anywhere and decoded +;; any character as a nibble, so "zzzz…" became a plausible GUID. +(as 'guid "zzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz") !- domain +(as 'guid "gggggggg-gggg-gggg-gggg-gggggggggggg") !- domain +(as 'guid "d49f18a4:1969-49e8-9b8a-6bb9a4832eea") !- domain +(as 'guid "d49f18a4_1969-49e8-9b8a-6bb9a4832eea") !- domain +(as 'guid "d49f18a4-1969-49e8-9b8a6bb9a4832eea") !- domain +(as 'guid "d49f18a4-1969-49e8-9b8a-6bb9a4832eea-") !- domain +(as 'guid "d49f18a4-1969-49e8-9b8a-6bb9a4832ee") !- domain +(as 'guid "d49f18a4-1969-49e8-9b8a-6bb9a4832eeax") !- domain +;; A 36-char all-hex string (a dash position holding a hex digit) must be +;; rejected, not decoded — the byte index would otherwise run past +;; bytes[16] (stack overflow). +(as 'guid "0123456789abcdef0123456789abcdef0123") !- domain +(as 'guid "d49f18a4a1969-49e8-9b8a-6bb9a4832eea") !- domain +;; Uppercase hex is accepted and normalized on the round-trip. +(as 'STR (as 'guid "ABCDEF01-2345-6789-ABCD-EF0123456789")) -- "abcdef01-2345-6789-abcd-ef0123456789" ;; ========== IDENTITY CASTS (same type) ========== (as 'b8 true) -- true (as 'u8 0xFF) -- 0xFF @@ -509,6 +527,36 @@ (as 'i64 (as 'i32 "-2147483648")) -- 0Nl (as 'i64 (as 'i32 "2147483647")) -- 2147483647 +;; ========== STRING → NUMERIC: FULL CONSUMPTION + OVERFLOW ========== +;; The old strtoll/strtol/strtod path stopped at the first non-digit and +;; never checked the end pointer or ERANGE, so trailing garbage and true +;; overflow were silently accepted ("1e19" → 1, "9999...9" → INT64_MAX). +;; All of these must now REJECT with a domain error, matching the strict +;; DATE/TIME/TIMESTAMP string casts. Narrow-int truncation itself still +;; wraps (documented narrow-int rule); only unparseable/overflowing input +;; errors. +(as 'i64 "123abc") !- domain +(as 'i64 "1e19") !- domain +(as 'i64 "12.5") !- domain +(as 'i64 "999999999999999999999999") !- domain +(as 'i64 "-999999999999999999999999") !- domain +(as 'i32 "123x") !- domain +(as 'i32 "1e9") !- domain +(as 'i16 "40000junk") !- domain +(as 'u8 "12zz") !- domain +(as 'u8 "300abc") !- domain +(as 'f64 "1.5x") !- domain +(as 'f64 "1.5e3junk") !- domain +;; Clean values still parse. +(as 'i64 " 123") -- 123 +(as 'i64 "-5") -- -5 +(as 'f64 "0x10") -- 16.0 +;; The string-VECTOR cast routes through the same per-element parser: a +;; malformed element rejects the whole cast, clean elements parse. +(as 'I64 (list "42" "-7" "100")) -- [42 -7 100] +(as 'I64 (list "42" "123abc")) !- domain +(as 'F64 (list "3.14" "1.5x")) !- domain + ;; ========== NULL PRESERVATION ACROSS CASTS ========== ;; Casting any null returns null of target type, never sentinel/INT_MIN. (nil? (as 'i64 0Nh)) -- true diff --git a/test/test_join_buildside.c b/test/test_join_buildside.c index f24f9a385..0cc93a3c0 100644 --- a/test/test_join_buildside.c +++ b/test/test_join_buildside.c @@ -45,8 +45,24 @@ static ray_t* jb_table1(const char* name, const int64_t* vals, int64_t n) { return tbl; } +static ray_t* jb_str_table1(const char* name, const char* value) { + ray_t* col = ray_vec_new(RAY_STR, 1); + if (!col || RAY_IS_ERR(col)) return col; + ray_t* next = ray_str_vec_append(col, value, strlen(value)); + if (!next || RAY_IS_ERR(next)) { + ray_release(col); + return next; + } + col = next; + ray_t* tbl = ray_table_new(1); + int64_t sym = ray_sym_intern(name, strlen(name)); + tbl = ray_table_add_col(tbl, sym, col); + ray_release(col); + return tbl; +} + /* ── Join helper ─────────────────────────────────────────────────────────── - * jb_inner_join: build and execute a single-key I64 inner join. + * jb_inner_join: build and execute a single-key inner join. * * Graph shape (mirrors query.c join_impl): * g = ray_graph_new(lt) — g->table = lt (used for type inference @@ -965,6 +981,41 @@ static test_result_t test_jb_not_sticky(void) { return (test_result_t){ TEST_PASS, NULL }; } +/* A direct graph can pair unlike key types even though the Rayfall query + * frontend rejects them. The radix executor must use generic equality unless + * both key vectors are STR. */ +static test_result_t test_jb_mixed_type_radix(void) { + ray_heap_init(); + (void)ray_sym_init(); + + static const char str_key[] = "mixed-key-24497"; + + int64_t n_r = RAY_PARALLEL_THRESHOLD + 5000; + int64_t* rv = malloc((size_t)n_r * sizeof(*rv)); + TEST_ASSERT_NOT_NULL(rv); + for (int64_t i = 0; i < n_r; i++) rv[i] = i; + + ray_t* lt = jb_str_table1("lk", str_key); + ray_t* rt = jb_table1("rk", rv, n_r); + free(rv); + TEST_ASSERT(lt && !RAY_IS_ERR(lt), "STR table allocation"); + TEST_ASSERT(rt && !RAY_IS_ERR(rt), "I64 table allocation"); + + ray_join_no_build_swap = true; + ray_t* got = jb_inner_join(lt, "lk", rt, "rk"); + ray_join_no_build_swap = false; + + TEST_ASSERT(got && !RAY_IS_ERR(got), "mixed-type direct join execution"); + TEST_ASSERT_EQ_I(ray_table_nrows(got), 0); + + ray_release(got); + ray_release(lt); + ray_release(rt); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + /* ── Entry table ─────────────────────────────────────────────────────────── */ const test_entry_t join_buildside_entries[] = { @@ -985,5 +1036,6 @@ const test_entry_t join_buildside_entries[] = { { "join_buildside/no_trip_low_dup", test_jb_no_trip_low_dup, NULL, NULL }, { "join_buildside/trip_boundary", test_jb_trip_boundary, NULL, NULL }, { "join_buildside/not_sticky", test_jb_not_sticky, NULL, NULL }, + { "join_buildside/mixed_type_radix", test_jb_mixed_type_radix, NULL, NULL }, { NULL, NULL, NULL, NULL }, }; diff --git a/test/test_lang.c b/test/test_lang.c index cec3ba4a9..960591ebe 100644 --- a/test/test_lang.c +++ b/test/test_lang.c @@ -7680,7 +7680,73 @@ static test_result_t test_builtin_load_file_fn(void) { PASS(); } -/* (write path content) — write a string to a file. */ +/* (read-bytes path) — read a file as a U8 byte vector. */ +static test_result_t test_builtin_read_bytes_fn(void) { + char path[64]; + snprintf(path, sizeof(path), "/tmp/ray_test_read_bytes_%d.bin", (int)getpid()); + static const uint8_t expected[] = { 0x00, 0x01, 0x7f, 0x80, 0xff }; + + FILE* fp = fopen(path, "wb"); + TEST_ASSERT_NOT_NULL(fp); + TEST_ASSERT_EQ_U(fwrite(expected, 1, sizeof(expected), fp), sizeof(expected)); + TEST_ASSERT_EQ_I(fclose(fp), 0); + + ray_t* p = ray_str(path, strlen(path)); + ray_t* bytes = ray_read_bytes_fn(p); + TEST_ASSERT_NOT_NULL(bytes); + TEST_ASSERT_FALSE(RAY_IS_ERR(bytes)); + TEST_ASSERT_EQ_I(bytes->type, RAY_U8); + TEST_ASSERT_EQ_I(bytes->len, (int64_t)sizeof(expected)); + TEST_ASSERT_MEM_EQ(sizeof(expected), ray_data(bytes), expected); + + /* The existing text reader keeps its string contract after sharing the + * exact-read path, including explicit length across embedded NULs. */ + ray_t* text = ray_read_file_fn(p); + TEST_ASSERT_NOT_NULL(text); + TEST_ASSERT_FALSE(RAY_IS_ERR(text)); + TEST_ASSERT_EQ_I(text->type, -RAY_STR); + TEST_ASSERT_EQ_U(ray_str_len(text), sizeof(expected)); + TEST_ASSERT_MEM_EQ(sizeof(expected), ray_str_ptr(text), expected); + + /* Exercise public builtin registration, not only the C entry point. */ + char expr[128]; + snprintf(expr, sizeof(expr), "(read-bytes \"%s\")", path); + ray_t* eval_bytes = ray_eval_str(expr); + TEST_ASSERT_NOT_NULL(eval_bytes); + TEST_ASSERT_FALSE(RAY_IS_ERR(eval_bytes)); + TEST_ASSERT_EQ_I(eval_bytes->type, RAY_U8); + TEST_ASSERT_EQ_I(eval_bytes->len, (int64_t)sizeof(expected)); + TEST_ASSERT_MEM_EQ(sizeof(expected), ray_data(eval_bytes), expected); + + /* Empty files produce an empty U8 vector. */ + fp = fopen(path, "wb"); + TEST_ASSERT_NOT_NULL(fp); + TEST_ASSERT_EQ_I(fclose(fp), 0); + ray_t* empty = ray_read_bytes_fn(p); + TEST_ASSERT_NOT_NULL(empty); + TEST_ASSERT_FALSE(RAY_IS_ERR(empty)); + TEST_ASSERT_EQ_I(empty->type, RAY_U8); + TEST_ASSERT_EQ_I(empty->len, 0); + + ray_t* bad_path = ray_i64(0); + ray_t* type_err = ray_read_bytes_fn(bad_path); + TEST_ASSERT_TRUE(RAY_IS_ERR(type_err)); + unlink(path); + ray_t* io_err = ray_read_bytes_fn(p); + TEST_ASSERT_TRUE(RAY_IS_ERR(io_err)); + + ray_release(io_err); + ray_release(type_err); + ray_release(bad_path); + ray_release(empty); + ray_release(eval_bytes); + ray_release(text); + ray_release(bytes); + ray_release(p); + PASS(); +} + +/* Text and byte file writers have distinct public builtins. */ static test_result_t test_builtin_write_file_fn(void) { char path[64]; snprintf(path, sizeof(path), "/tmp/ray_test_write_%d.txt", (int)getpid()); @@ -7701,6 +7767,47 @@ static test_result_t test_builtin_write_file_fn(void) { TEST_ASSERT_EQ_U(rd, 11); TEST_ASSERT_TRUE(memcmp(buf, "hello world", 11) == 0); + /* write-bytes writes U8 content verbatim, including embedded NULs and + * bytes that are not valid text. */ + static const uint8_t expected[] = { 0x00, 0x01, 0x7f, 0x80, 0xff }; + ray_t* bytes = ray_vec_from_raw(RAY_U8, expected, (int64_t)sizeof(expected)); + TEST_ASSERT_NOT_NULL(bytes); + TEST_ASSERT_FALSE(RAY_IS_ERR(bytes)); + + ray_t* rbb = ray_write_bytes_fn(p, bytes); + TEST_ASSERT_NOT_NULL(rbb); + TEST_ASSERT_FALSE(RAY_IS_ERR(rbb)); + char expr[192]; + snprintf(expr, sizeof(expr), + "(write-bytes \"%s\" (as 'U8 [0 1 127 128 255]))", path); + ray_t* eval_write = ray_eval_str(expr); + TEST_ASSERT_NOT_NULL(eval_write); + TEST_ASSERT_FALSE(RAY_IS_ERR(eval_write)); + fp = fopen(path, "rb"); + TEST_ASSERT_NOT_NULL(fp); + uint8_t byte_buf[sizeof(expected)] = {0}; + rd = fread(byte_buf, 1, sizeof(byte_buf), fp); + fclose(fp); + TEST_ASSERT_EQ_U(rd, sizeof(expected)); + TEST_ASSERT_MEM_EQ(sizeof(expected), byte_buf, expected); + + ray_t* string_err = ray_write_bytes_fn(p, c); + TEST_ASSERT_TRUE(RAY_IS_ERR(string_err)); + ray_t* bytes_err = ray_write_file_fn(p, bytes); + TEST_ASSERT_TRUE(RAY_IS_ERR(bytes_err)); + + /* Empty byte vectors truncate/create an empty file. */ + ray_t* empty = ray_vec_new(RAY_U8, 0); + TEST_ASSERT_NOT_NULL(empty); + TEST_ASSERT_FALSE(RAY_IS_ERR(empty)); + ray_t* re = ray_write_bytes_fn(p, empty); + TEST_ASSERT_NOT_NULL(re); + TEST_ASSERT_FALSE(RAY_IS_ERR(re)); + fp = fopen(path, "rb"); + TEST_ASSERT_NOT_NULL(fp); + TEST_ASSERT_EQ_I(fgetc(fp), EOF); + fclose(fp); + /* Wrong-type paths. */ ray_t* bad_path = ray_i64(0); ray_t* re1 = ray_write_file_fn(bad_path, c); @@ -7710,6 +7817,13 @@ static test_result_t test_builtin_write_file_fn(void) { TEST_ASSERT_TRUE(RAY_IS_ERR(re2)); unlink(path); + ray_release(bytes_err); + ray_release(string_err); + ray_release(eval_write); + ray_release(rbb); + ray_release(re); + ray_release(empty); + ray_release(bytes); ray_release(re2); ray_release(bad_content); ray_release(re1); @@ -9074,6 +9188,7 @@ const test_entry_t lang_entries[] = { { "lang/builtin/show", test_builtin_show_fn, lang_setup, lang_teardown }, { "lang/builtin/timeit", test_builtin_timeit_fn, lang_setup, lang_teardown }, { "lang/builtin/load_file", test_builtin_load_file_fn, lang_setup, lang_teardown }, + { "lang/builtin/read_bytes", test_builtin_read_bytes_fn, lang_setup, lang_teardown }, { "lang/builtin/write_file", test_builtin_write_file_fn, lang_setup, lang_teardown }, { "lang/builtin/group_ht_grow_i64", test_builtin_group_ht_grow_i64, lang_setup, lang_teardown }, { "lang/builtin/group_ht_grow_guid", test_builtin_group_ht_grow_guid, lang_setup, lang_teardown }, diff --git a/test/test_store.c b/test/test_store.c index 6001e6b51..03e37ad16 100644 --- a/test/test_store.c +++ b/test/test_store.c @@ -41,8 +41,10 @@ #include "core/platform.h" #include "core/runtime.h" #include "lang/eval.h" +#include "lang/internal.h" #include "mem/sys.h" #include "table/sym.h" +#include "vec/str.h" #ifndef RAY_OS_WINDOWS #include @@ -4363,6 +4365,87 @@ static test_result_t test_col_str_pool_roundtrip(void) { PASS(); } +/* ---- test_col_str_legacy_hash_repair ----------------------------------- */ +/* Older column writers persisted the pooled descriptor's final four bytes + * as uninitialized padding. Forge distinct nonzero values there for equal + * strings and verify both load modes recompute content hashes before the + * grouping fast path consumes them. */ +static test_result_t test_col_str_legacy_hash_repair(void) { + const char* same = "legacy-pooled-string-value"; + const char* other = "different-pooled-string"; + ray_t* vec = ray_vec_new(RAY_STR, 3); + TEST_ASSERT_FALSE(RAY_IS_ERR(vec)); + vec = ray_str_vec_append(vec, same, strlen(same)); + TEST_ASSERT_FALSE(RAY_IS_ERR(vec)); + vec = ray_str_vec_append(vec, same, strlen(same)); + TEST_ASSERT_FALSE(RAY_IS_ERR(vec)); + vec = ray_str_vec_append(vec, other, strlen(other)); + TEST_ASSERT_FALSE(RAY_IS_ERR(vec)); + TEST_ASSERT_EQ_I(ray_col_save(vec, TMP_COL_PATH), RAY_OK); + + /* [32-byte column header][16-byte descriptors]; hash32 is descriptor + * bytes 12..15. These model arbitrary nonzero legacy padding. */ + static const uint32_t garbage[] = { + 0x11111111u, 0x22222222u, 0x33333333u + }; + FILE* f = fopen(TMP_COL_PATH, "r+b"); + TEST_ASSERT_NOT_NULL(f); + for (int i = 0; i < 3; i++) { + TEST_ASSERT_EQ_I(fseek(f, 32L + (long)i * 16L + 12L, SEEK_SET), 0); + TEST_ASSERT_EQ_U(fwrite(&garbage[i], 1, sizeof(garbage[i]), f), + sizeof(garbage[i])); + } + TEST_ASSERT_EQ_I(fclose(f), 0); + + ray_t* loaded = ray_col_load(TMP_COL_PATH); + TEST_ASSERT_NOT_NULL(loaded); + TEST_ASSERT_FALSE(RAY_IS_ERR(loaded)); + ray_str_t* ld = (ray_str_t*)ray_data(loaded); + const char* lpool = (const char*)ray_data(loaded->str_pool); + for (int i = 0; i < 3; i++) { + uint32_t h = (uint32_t)ray_str_t_hash(&ld[i], lpool); + TEST_ASSERT_EQ_U(ld[i].hash32, h != 0 ? h : 1u); + } + TEST_ASSERT_EQ_U(ld[0].hash32, ld[1].hash32); + ray_t* groups = ray_group_indices_fn(loaded); + TEST_ASSERT_NOT_NULL(groups); + TEST_ASSERT_FALSE(RAY_IS_ERR(groups)); + TEST_ASSERT_EQ_I(ray_dict_keys(groups)->len, 2); + ray_release(groups); + ray_release(loaded); + + ray_t* mapped = ray_col_mmap(TMP_COL_PATH); + TEST_ASSERT_NOT_NULL(mapped); + TEST_ASSERT_FALSE(RAY_IS_ERR(mapped)); + ray_str_t* md = (ray_str_t*)ray_data(mapped); + const char* mpool = (const char*)ray_data(mapped->str_pool); + for (int i = 0; i < 3; i++) { + uint32_t h = (uint32_t)ray_str_t_hash(&md[i], mpool); + TEST_ASSERT_EQ_U(md[i].hash32, h != 0 ? h : 1u); + } + TEST_ASSERT_EQ_U(md[0].hash32, md[1].hash32); + groups = ray_group_indices_fn(mapped); + TEST_ASSERT_NOT_NULL(groups); + TEST_ASSERT_FALSE(RAY_IS_ERR(groups)); + TEST_ASSERT_EQ_I(ray_dict_keys(groups)->len, 2); + ray_release(groups); + ray_release(mapped); + + /* MAP_PRIVATE repair must not rewrite the persisted legacy bytes. */ + f = fopen(TMP_COL_PATH, "rb"); + TEST_ASSERT_NOT_NULL(f); + TEST_ASSERT_EQ_I(fseek(f, 32L + 12L, SEEK_SET), 0); + uint32_t persisted = 0; + TEST_ASSERT_EQ_U(fread(&persisted, 1, sizeof(persisted), f), + sizeof(persisted)); + TEST_ASSERT_EQ_I(fclose(f), 0); + TEST_ASSERT_EQ_U(persisted, garbage[0]); + + ray_release(vec); + unlink(TMP_COL_PATH); + PASS(); +} + /* ---- test_col_format_version_roundtrip ---------------------------------- */ /* A saved column carries the format generation in the 32-byte header's * `order` byte (offset 17), with aux (bytes 0-15) ZERO on disk (no magic — @@ -5103,6 +5186,7 @@ const test_entry_t store_entries[] = { { "store/col_recursive_sym_in_list", test_col_recursive_sym_in_list, store_setup, store_teardown }, { "store/col_sym_w64_neg_index", test_col_sym_w64_negative_index, store_setup, store_teardown }, { "store/col_str_pool_roundtrip", test_col_str_pool_roundtrip, store_setup, store_teardown }, + { "store/col_str_legacy_hash_repair", test_col_str_legacy_hash_repair, store_setup, store_teardown }, { "store/col_format_version_roundtrip", test_col_format_version_roundtrip, store_setup, store_teardown }, { "store/col_format_bad_version", test_col_format_bad_version, store_setup, store_teardown }, { "store/col_str_empty_roundtrip", test_col_str_empty_roundtrip, store_setup, store_teardown }, diff --git a/test/test_str.c b/test/test_str.c index 77d6b4dd1..b0e901c39 100644 --- a/test/test_str.c +++ b/test/test_str.c @@ -635,6 +635,8 @@ static test_result_t test_str_t_eq_pooled(void) { ray_str_t* elems = (ray_str_t*)ray_data(v); const char* pool = (const char*)ray_data(v->str_pool); + /* Same immutable pool + same offset is an exact identity fast path. */ + TEST_ASSERT_TRUE(ray_str_t_eq(&elems[0], pool, &elems[0], pool)); TEST_ASSERT_TRUE(ray_str_t_eq(&elems[0], pool, &elems[1], pool)); /* Same prefix "a]lo" but different content */ TEST_ASSERT_FALSE(ray_str_t_eq(&elems[0], pool, &elems[2], pool)); @@ -1655,6 +1657,30 @@ static test_result_t test_str_t_hash_pooled(void) { PASS(); } +static test_result_t test_str_t_hash32_cache(void) { + ray_t* v = ray_vec_new(RAY_STR, 3); + v = ray_str_vec_append(v, "this is a cached pooled string", 30); + v = ray_str_vec_append(v, "this is a cached pooled string", 30); + v = ray_str_vec_append(v, "this is another pooled string", 29); + + ray_str_t* elems = (ray_str_t*)ray_data(v); + const char* pool = (const char*)ray_data(v->str_pool); + TEST_ASSERT_TRUE(elems[0].hash32 != 0); + TEST_ASSERT_TRUE(elems[1].hash32 != 0); + TEST_ASSERT_TRUE(elems[2].hash32 != 0); + + uint32_t cached = ray_str_t_hash32(&elems[0], pool); + TEST_ASSERT_EQ_U(cached, ray_str_t_hash32(&elems[1], pool)); + TEST_ASSERT_TRUE(cached != ray_str_t_hash32(&elems[2], pool)); + + /* A zero cache models columns written by older Rayforce versions. */ + elems[1].hash32 = 0; + TEST_ASSERT_EQ_U(cached, ray_str_t_hash32(&elems[1], pool)); + + ray_release(v); + PASS(); +} + static test_result_t test_str_t_hash_empty(void) { ray_t* v = ray_vec_new(RAY_STR, 2); v = ray_str_vec_append(v, "", 0); @@ -1928,6 +1954,7 @@ const test_entry_t str_entries[] = { { "str/vec_concat", test_str_vec_concat_vecs, str_setup, str_teardown }, { "str/t_hash_inline", test_str_t_hash_inline, str_setup, str_teardown }, { "str/t_hash_pooled", test_str_t_hash_pooled, str_setup, str_teardown }, + { "str/t_hash32_cache", test_str_t_hash32_cache, str_setup, str_teardown }, { "str/t_hash_empty", test_str_t_hash_empty, str_setup, str_teardown }, { "str/vec_concat_pooled_rebase", test_str_vec_concat_pooled_rebase, str_setup, str_teardown }, { "str/vec_concat_nulls", test_str_vec_concat_nulls, str_setup, str_teardown },