diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml index 778c06f..f389611 100644 --- a/.github/workflows/TagBot.yml +++ b/.github/workflows/TagBot.yml @@ -4,6 +4,11 @@ on: types: - created workflow_dispatch: + inputs: + lookback: + default: 3 +permissions: + contents: write jobs: TagBot: if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' @@ -12,3 +17,4 @@ jobs: - uses: JuliaRegistries/TagBot@v1 with: token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb2b0ee..3fa0eb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,5 @@ name: CI + on: pull_request: branches: @@ -7,15 +8,27 @@ on: branches: - master tags: '*' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} + timeout-minutes: 15 + continue-on-error: ${{ matrix.version == 'nightly' }} + env: + JULIA_NUM_THREADS: 2 strategy: fail-fast: false matrix: version: - - '1.0' + - '1.9' - '1' # automatically expands to the latest stable 1.x release of Julia - 'nightly' os: @@ -26,35 +39,40 @@ jobs: - os: windows-latest version: '1' arch: x86 + - os: windows-latest + version: '1' + arch: x64 + - os: macos-latest + version: '1' + arch: aarch64 steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: actions/cache@v1 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- + - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v5 with: - file: lcov.info + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} Documenter: name: Documentation runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write steps: - - uses: actions/checkout@v2 - - uses: julia-actions/julia-buildpkg@latest - - uses: julia-actions/julia-docdeploy@latest + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v2 + with: + version: '1' + - uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-docdeploy@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/Project.toml b/Project.toml index dee17d8..54284ac 100644 --- a/Project.toml +++ b/Project.toml @@ -1,10 +1,11 @@ name = "DBInterface" uuid = "a10d1c49-ce27-4219-8d33-6db1a4562965" authors = ["Jacob Quinn "] -version = "2.6.1" +version = "2.7.0" [compat] -julia = "1" +julia = "1.9" +Test = "1.9" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/README.md b/README.md index 2fb6c15..8e77757 100644 --- a/README.md +++ b/README.md @@ -4,59 +4,63 @@ [![version](https://juliahub.com/docs/DBInterface/version.svg)](https://juliahub.com/ui/Packages/DBInterface/bSj9k) [![pkgeval](https://juliahub.com/docs/DBInterface/pkgeval.svg)](https://juliahub.com/ui/Packages/DBInterface/bSj9k) -### Purpose -DBInterface.jl provides interface definitions to allow common database operations to be implemented consistently -across various database packages. +## Purpose -### For Users -To use DBInterface.jl, select an implementing database package, then utilize the consistent DBInterface.jl interface methods: -```julia -conn = DBInterface.connect(T, args...; kw...) # create a connection to a specific database T; required parameters are database-specific +DBInterface.jl defines a small, common interface for Julia database drivers. Select a driver package, then use the `DBInterface` methods against that driver's connection, statement, and result types. -stmt = DBInterface.prepare(conn, sql) # prepare a sql statement against the connection; returns a statement object +## Basic Use -results = DBInterface.execute(stmt) # execute a prepared statement; returns an iterator of rows (property-accessible & indexable) +Use the do-block forms when a connection, statement, or result should be closed at the end of an operation: -rowid = DBInterface.lastrowid(results) # get the last row id of an INSERT statement, as supported by the database +```julia +using DBInterface + +DBInterface.connect(Driver.Connection, args...; kwargs...) do conn + DBInterface.execute(conn, "SELECT id, name FROM users WHERE id = ?", (42,)) do cursor + for row in cursor + @show row.id + @show row[2] + end + end -# example of using a query resultset -for row in results - @show propertynames(row) # see possible column names of row results - row.col1 # access the value of a column named `col1` - row[1] # access the first column in the row results + DBInterface.prepare(conn, "INSERT INTO users (id, name) VALUES (?, ?)") do stmt + DBInterface.execute(_ -> nothing, stmt, (43, "Ada")) + end end +``` -# results also implicitly satisfy the Tables.jl `Tables.rows` interface, so any compatible sink can ingest results -df = DataFrame(results) -CSV.write("results.csv", results) +Rows must support property access by column name and indexing by column position. Result cursors should also satisfy the Tables.jl row-table interface, so Tables.jl-compatible sinks can consume them: -results = DBInterface.execute(conn, sql) # convenience method if statement preparation/re-use isn't needed +```julia +df = DBInterface.execute(DataFrame, conn, "SELECT * FROM users") +DBInterface.execute(cursor -> CSV.write("users.csv", cursor), conn, "SELECT * FROM users") +``` -stmt = DBInterface.prepare(conn, "INSERT INTO test_table VALUES(?, ?)") # prepare a statement with positional parameters +Use `executemany` for column-oriented bulk parameters. Each parameter collection must have the same length: -DBInterface.execute(stmt, [1, 3.14]) # execute the prepared INSERT statement, passing 1 and 3.14 as positional parameters +```julia +DBInterface.executemany( + conn, + "INSERT INTO users (id, name) VALUES (?, ?)", + ([1, 2, 3], ["Ada", "Grace", "Katherine"]), +) +``` -stmt = DBInterface.prepare(conn, "INSERT INTO test_table VALUES(:col1, :col2)") # prepare a statement with named parameters +Named parameters can be passed as a `NamedTuple`, an `AbstractDict`, or keywords when the database and driver support named placeholders: -DBInterface.execute(stmt, (col1=1, col2=3.14)) # execute the prepared INSERT statement, with 1 and 3.14 as named parameters +```julia +DBInterface.execute(conn, "SELECT * FROM users WHERE id = :id", (id=42,)) +DBInterface.execute(conn, "SELECT * FROM users WHERE id = :id"; id=42) +``` -DBInterface.executemany(stmt, (col1=[1,2,3,4,5], col2=[3.14, 1.23, 2.34 3.45, 4.56])) # execute the prepared statement multiple times for each set of named parameters; each named parameter must be an indexable collection +Placeholder syntax is driver-specific. For example, a driver may require `?`, `:name`, `$1`, or another form. One placeholder normally binds one scalar value. A collection does not normally expand into an SQL `IN` list. -results = DBInterface.executemultiple(conn, sql) # where sql is a query that returns multiple resultsets +## SQL Safety -# first iterate through resultsets -for result in results - # for each resultset, we can iterate through resultset rows - for row in result - @show propertynames(row) - row.col1 - row[1] - end -end +DBInterface passes SQL text to the driver unchanged. Use bound parameters for untrusted values. Do not interpolate untrusted data into SQL strings. Bound parameters do not quote table names, column names, SQL keywords, or other SQL fragments. Use the driver's identifier-quoting API when an identifier must be dynamic. -DBInterface.close!(stmt) # close the prepared statement -DBInterface.close!(conn) # close connection -``` +The `sql"..."` string macro does not parse, escape, validate, or sanitize SQL. + +## Driver Authors -### For Database Package Developers -See the [documentation](https://juliadatabases.org/DBInterface.jl/dev) for expanded details on required interface methods. +See the [documentation](https://juliadatabases.org/DBInterface.jl/dev) for the required methods, result contract, resource ownership rules, and optional extensions. diff --git a/docs/Project.toml b/docs/Project.toml index dfa65cd..c1d4492 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,2 +1,6 @@ [deps] +DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" + +[compat] +Documenter = "1" diff --git a/docs/make.jl b/docs/make.jl index 7d6a89f..0006399 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -2,14 +2,13 @@ using Documenter, DBInterface makedocs(; modules=[DBInterface], - format=Documenter.HTML(), + format=Documenter.HTML(repolink="https://github.com/JuliaDatabases/DBInterface.jl"), pages=[ "Home" => "index.md", ], repo="https://github.com/JuliaDatabases/DBInterface.jl/blob/{commit}{path}#L{line}", sitename="DBInterface.jl", authors="Jacob Quinn", - assets=String[], ) deploydocs(; diff --git a/docs/src/index.md b/docs/src/index.md index e18411b..bd3ca68 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -5,9 +5,34 @@ *DBInterface.jl* provides interface definitions to allow common database operations to be implemented consistently across various database packages. +## User Contract + +A database driver defines the concrete connection, statement, and result types. Prefer the do-block forms of [`DBInterface.connect`](@ref), [`DBInterface.prepare`](@ref), and [`DBInterface.execute`](@ref) when the resource should be closed immediately after an operation. + +SQL placeholder syntax is database- and driver-specific. Positional placeholders may use `?`, `$1`, or another form. Named placeholders may not be supported. One placeholder normally binds one scalar value, so a collection does not normally expand into an SQL `IN` list. + +DBInterface does not parse, escape, validate, or sanitize SQL. Bind untrusted values as parameters. Do not interpolate them into SQL text. Parameters cannot safely replace identifiers, keywords, or SQL fragments; use a driver-specific identifier-quoting API for dynamic identifiers. + +## Driver Contract + +A driver should implement these core methods: + + * [`DBInterface.connect`](@ref) for its database or connection selector. + * [`DBInterface.prepare`](@ref) for its [`DBInterface.Connection`](@ref) subtype. + * [`DBInterface.execute`](@ref) for its [`DBInterface.Statement`](@ref) subtype. + * [`DBInterface.getconnection`](@ref) for its statement subtype. + * [`DBInterface.close!`](@ref) for its connections, statements, and result cursors. + +The generic connection form of `execute` prepares a statement and returns the driver's cursor. If that cursor depends on the statement, the driver must keep the statement alive for the cursor's lifetime and release it when the cursor is closed or collected. + +Each cursor row must support `propertynames`, `getproperty`, `length`, and positional `getindex`. A cursor should implement the Tables.jl row-table interface. Statements that return no rows must still return an empty cursor or iterator. The scoped `execute(f, ...)` forms call `DBInterface.close!` on the cursor. + +Drivers can override [`DBInterface.transaction`](@ref), [`DBInterface.executemany`](@ref), [`DBInterface.executemultiple`](@ref), and [`DBInterface.lastrowid`](@ref) when the generic behavior does not match the database. + ## Functions ```@docs DBInterface.connect +DBInterface.@sql_str DBInterface.getconnection DBInterface.prepare DBInterface.@prepare @@ -17,6 +42,9 @@ DBInterface.executemany DBInterface.executemultiple DBInterface.close! DBInterface.lastrowid +DBInterface.ParameterError +DBInterface.Error +DBInterface.Warning ``` ## Types @@ -25,4 +53,7 @@ DBInterface.lastrowid DBInterface.Connection DBInterface.Statement DBInterface.Cursor +DBInterface.PositionalStatementParams +DBInterface.NamedStatementParams +DBInterface.StatementParams ``` diff --git a/src/DBInterface.jl b/src/DBInterface.jl index ffe40ea..004a292 100644 --- a/src/DBInterface.jl +++ b/src/DBInterface.jl @@ -5,7 +5,7 @@ export @sql_str; """ Declare the string as written in SQL. -The macro doesn't do any processing of the string. +The macro does not parse, escape, validate, or sanitize the string. """ macro sql_str(cmd) cmd @@ -16,12 +16,24 @@ abstract type Connection end """ DBInterface.connect(DB, args...; kw...) => DBInterface.Connection + DBInterface.connect(f::Callable, DB, args...; kw...) Database packages should overload `DBInterface.connect` for a specific `DB` `DBInterface.Connection` subtype that returns a valid, live database connection that can be queried against. + +When `f` is provided, the connection is passed to `f`, closed upon exit, and the result of `f` is returned. """ function connect end +function connect(f::Base.Callable, DB, args...; kwargs...) + conn = connect(DB, args...; kwargs...) + try + return f(conn) + finally + close!(conn) + end +end + # Different `close!` signatures have their own docstrings. function close! end @@ -47,6 +59,7 @@ function getconnection end """ DBInterface.prepare(conn::DBInterface.Connection, sql::AbstractString) => DBInterface.Statement DBInterface.prepare(f::Function, sql::AbstractString) => DBInterface.Statement + DBInterface.prepare(f::Callable, conn::DBInterface.Connection, sql::AbstractString; kw...) Database packages should overload `DBInterface.prepare` for a specific `DBInterface.Connection` subtype, that validates and prepares a SQL statement given as an `AbstractString` `sql` argument, and returns a `DBInterface.Statement` subtype. It is expected @@ -54,26 +67,67 @@ that `DBInterface.Statement`s are only valid for the lifetime of the `DBInterfac For convenience, users may call `DBInterface.prepare(f::Function, sql)` which first calls `f()` to retrieve a valid `DBInterface.Connection` before calling `DBInterface.prepare(conn, sql)`; this allows deferring connection retrieval and thus statement preparation until runtime, which is often convenient when building applications. + +When both `f` and `conn` are provided, the prepared statement is passed to `f`, closed upon exit, and the result of `f` is returned. """ function prepare end prepare(f::Function, sql::AbstractString) = prepare(f(), sql) -const PREPARED_STMTS = Dict{Symbol, Statement}() +function prepare(f::Base.Callable, conn::Connection, sql::AbstractString; kwargs...) + stmt = prepare(conn, sql; kwargs...) + try + return f(stmt) + finally + close!(stmt) + end +end + +struct _PreparedStatementCacheEntry + sql::String + statement::Statement +end + +const PREPARED_STMTS = Dict{Tuple{Module, Symbol}, IdDict{Connection, _PreparedStatementCacheEntry}}() +const PREPARED_STMTS_LOCK = ReentrantLock() + +function _cached_prepare(getDB, sql::AbstractString, caller::Module, key::Symbol) + connection = getDB() + sql_string = String(sql) + cache_key = (caller, key) + lock(PREPARED_STMTS_LOCK) + try + entries = get!(PREPARED_STMTS, cache_key) do + IdDict{Connection, _PreparedStatementCacheEntry}() + end + entry = get(entries, connection, nothing) + if entry !== nothing && entry.sql == sql_string + return entry.statement + end + if entry !== nothing + delete!(entries, connection) + close!(entry.statement) + end + statement = prepare(connection, sql_string) + entries[connection] = _PreparedStatementCacheEntry(sql_string, statement) + return statement + finally + unlock(PREPARED_STMTS_LOCK) + end +end """ DBInterface.@prepare f sql -Takes a `DBInterface.Connection`-retrieval function `f` and SQL statement `sql` and will return a prepared statement, via usage of `DBInterface.prepare`. -If the statement has already been prepared, it will be re-used (prepared statements are cached). +Takes a zero-argument `DBInterface.Connection`-retrieval function `f` and SQL statement `sql` and returns a prepared statement via `DBInterface.prepare`. +Each call site caches one statement per connection object. A statement is reused while its SQL text remains unchanged. If the SQL changes, +the old statement for that connection is closed and replaced. +Cached entries are retained, so use this macro with a bounded set of long-lived connection objects. +The cache is synchronized, but it does not make a connection or statement safe for concurrent use. """ macro prepare(getDB, sql) key = gensym() - return quote - get!(DBInterface.PREPARED_STMTS, $(QuoteNode(key))) do - DBInterface.prepare($(esc(getDB)), $sql) - end - end + return :(DBInterface._cached_prepare($(esc(getDB)), $(esc(sql)), $(QuoteNode(__module__)), $(QuoteNode(key)))) end """ @@ -108,22 +162,25 @@ const StatementParams = Union{PositionalStatementParams, NamedStatementParams} DBInterface.execute(f::Callable, conn::DBInterface.Connection, sql::AbstractString, [params]) DBInterface.execute(f::Callable, stmt::DBInterface.Statement, [params]) -Database packages should overload `DBInterface.execute` for a valid, prepared `DBInterface.Statement` subtype (the first method -signature is defined in DBInterface.jl using `DBInterface.prepare`), which takes an optional `params` argument, which should be -an indexable collection (`Vector` or `Tuple`) for positional parameters, or a `NamedTuple` for named parameters. +Database packages should overload `DBInterface.execute` for a valid, prepared `DBInterface.Statement` subtype (the connection +signature is defined in DBInterface.jl using `DBInterface.prepare`), which takes an optional `params` argument. Parameters should be +an indexable collection (`AbstractVector` or `Tuple`) for positional parameters, or a `NamedTuple` or `AbstractDict` for named parameters. Alternatively, the parameters could be specified as keyword arguments of `DBInterface.execute`. +Placeholder syntax and named-parameter support are driver-specific. Each placeholder normally binds one scalar value. DBInterface +does not parse or sanitize SQL, and bound parameters cannot replace identifiers, keywords, or other SQL fragments. + `DBInterface.execute` should return a valid `DBInterface.Cursor` object, which is any iterator of "rows", which themselves must be property-accessible (i.e. implement `propertynames` and `getproperty` for value access by name), and indexable (i.e. implement `length` and `getindex` for value access by index). These "result" objects do not need -to subtype `DBInterface.Cursor` explicitly as long as they satisfy the interface. For DDL/DML SQL statements, which typically -do not return results, an iterator is still expected to be returned that just iterates `nothing`, i.e. an "empty" iterator. +to subtype `DBInterface.Cursor` explicitly as long as they satisfy the interface and implement `DBInterface.close!`. For DDL/DML +SQL statements, which typically do not return results, an empty iterator is still expected. Note that `DBInterface.execute` returns **a single** `DBInterface.Cursor`, which represents a single resultset from the database. For use-cases involving multiple result-sets from a single query, see `DBInterface.executemultiple`. -If function `f` is provided, `DBInterface.execute` will return the result of applying `f` to the `DBInterface.Cursor` object -and close the prepared statement upon exit. +If function `f` is provided, `DBInterface.execute` returns the result of applying `f` to the cursor and closes the cursor upon exit. +The connection form also closes the statement that it prepares internally. """ function execute end @@ -157,21 +214,31 @@ execute(f::Base.Callable, stmt::Statement; kwargs...) = execute(f, stmt, values( DBInterface.transaction(f, conn::DBInterface.Connection) Open a transaction against a database connection `conn`, execute a closure `f`, -then "commit" the transaction after executing the closure function. The default -definition in DBInterface.jl is a no-op in that it just executes the closure -function with no transaction. Used in `DBInterface.executemany` to wrap the -individual execute calls in a transaction since this often leads to much better -performance in database systems. +then commit the transaction after executing the closure. The default definition +executes `BEGIN TRANSACTION`, `COMMIT`, and, after an error, `ROLLBACK`. Database +packages should overload this method when those commands do not match the database's +transaction behavior. `DBInterface.executemany` uses this method because a transaction +often makes repeated statements much faster. If both the transaction and its rollback +fail, a `CompositeException` reports both errors, with the original error first. """ function transaction(f, conn::Connection) - execute(conn, "BEGIN TRANSACTION;") + _execute_and_close(conn, "BEGIN TRANSACTION;") try ret = f() - execute(conn, "COMMIT;") + _execute_and_close(conn, "COMMIT;") return ret - catch e - execute(conn, "ROLLBACK;") - rethrow(e) + catch transaction_error + transaction_backtrace = catch_backtrace() + try + _execute_and_close(conn, "ROLLBACK;") + catch rollback_error + rollback_backtrace = catch_backtrace() + throw(CompositeException([ + CapturedException(transaction_error, transaction_backtrace), + CapturedException(rollback_error, rollback_backtrace), + ])) + end + rethrow() end end @@ -185,35 +252,94 @@ Base.IteratorSize(::Type{<:LazyIndex}) = Base.HasLength() Base.size(x::LazyIndex) = (length(x.x),) Base.getindex(x::LazyIndex, i::Int) = x.x[i][x.i] +struct LazyNamedIndex{T, K, V} <: AbstractDict{K, V} + x::T + i::Int +end + +LazyNamedIndex(x::T, i::Int) where {T <: AbstractDict} = + LazyNamedIndex{T, Base.keytype(T), eltype(Base.valtype(T))}(x, i) + +Base.length(x::LazyNamedIndex) = length(x.x) +Base.getindex(x::LazyNamedIndex, key) = x.x[key][x.i] +Base.haskey(x::LazyNamedIndex, key) = haskey(x.x, key) +Base.get(x::LazyNamedIndex, key, default) = haskey(x.x, key) ? x.x[key][x.i] : default +Base.get(f::Base.Callable, x::LazyNamedIndex, key) = haskey(x.x, key) ? x.x[key][x.i] : f() + +function Base.iterate(x::LazyNamedIndex, state...) + result = iterate(x.x, state...) + result === nothing && return nothing + pair, next_state = result + return (pair.first => pair.second[x.i], next_state) +end + +_parameter_collections(params::PositionalStatementParams) = params +_parameter_collections(params::NamedStatementParams) = values(params) + +_parameter_row(params::PositionalStatementParams, i::Int) = LazyIndex(params, i) +_parameter_row(params::NamedTuple, i::Int) = LazyIndex(values(params), i) +_parameter_row(params::AbstractDict, i::Int) = LazyNamedIndex(params, i) + +function _execute_and_close(stmt::Statement, params) + cursor = execute(stmt, params) + applicable(close!, cursor) && close!(cursor) + return +end + +function _execute_and_close(stmt::Statement) + cursor = execute(stmt) + applicable(close!, cursor) && close!(cursor) + return +end + +function _execute_and_close(conn::Connection, sql::AbstractString) + stmt = prepare(conn, sql) + try + return _execute_and_close(stmt) + finally + close!(stmt) + end +end + """ DBInterface.executemany(conn::DBInterface.Connection, sql::AbstractString, [params]) => Nothing DBInterface.executemany(stmt::DBInterface.Statement, [params]) => Nothing Similar in usage to `DBInterface.execute`, but allows passing multiple sets of parameters to be executed in sequence. -`params`, like for `DBInterface.execute`, should be an indexable collection (`Vector` or `Tuple`) or `NamedTuple`, but instead +`params`, like for `DBInterface.execute`, should be an `AbstractVector`, `Tuple`, `NamedTuple`, or `AbstractDict`, but instead of a single scalar value per parameter, an indexable collection should be passed for each parameter. By default, each set of parameters will be looped over and `DBInterface.execute` will be called for each. Note that no result sets or cursors are returned -for any execution, so the usage is mainly intended for bulk INSERT statements. +for any execution, so the usage is mainly intended for bulk INSERT statements. For compatibility, a `NamedTuple` or keyword batch +is passed to each execution positionally in field order. Use an `AbstractDict` batch when each execution must retain parameter names. """ function executemany(stmt::Statement, params) - if !isempty(params) - param = params[1] + param_collections = _parameter_collections(params) + if !isempty(param_collections) + param = first(param_collections) len = length(param) - all(x -> length(x) == len, params) || throw(ParameterError("parameters provided to `DBInterface.executemany!` do not all have the same number of parameters")) + all(x -> length(x) == len, param_collections) || throw(ParameterError("parameter collections provided to `DBInterface.executemany` must have equal lengths")) + len == 0 && return transaction(getconnection(stmt)) do for i = 1:len - xargs = LazyIndex(params, i) - execute(stmt, xargs) + _execute_and_close(stmt, _parameter_row(params, i)) end end else - execute(stmt) + _execute_and_close(stmt, params) end return end # keyarg version -executemany(conn::Connection, sql::AbstractString, params) = executemany(prepare(conn, sql), params) +function executemany(conn::Connection, sql::AbstractString, params) + stmt = prepare(conn, sql) + try + return executemany(stmt, params) + finally + close!(stmt) + end +end + executemany(conn::Connection, sql::AbstractString; kwargs...) = executemany(conn, sql, values(kwargs)) """ @@ -263,4 +389,7 @@ struct Error <: Exception msg::String end +Base.showerror(io::IO, error::ParameterError) = print(io, error.msg) +Base.showerror(io::IO, error::Error) = print(io, error.msg) + end # module diff --git a/test/runtests.jl b/test/runtests.jl index 0d4fd75..c23ed48 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,3 +4,332 @@ using DBInterface, Test # test @sql_str macro (does nothing) @test sql"SELECT * FROM MyTable" == "SELECT * FROM MyTable" + +mutable struct MockConnection <: DBInterface.Connection + id::Int +end + +mutable struct MockStatement <: DBInterface.Statement + connection::MockConnection + sql::String + closed::Bool +end + +const prepare_count = Ref(0) + +function DBInterface.prepare(connection::MockConnection, sql::AbstractString) + prepare_count[] += 1 + return MockStatement(connection, String(sql), false) +end + +DBInterface.getconnection(statement::MockStatement) = statement.connection +DBInterface.close!(statement::MockStatement) = statement.closed = true + +cached_statement(connection, sql) = DBInterface.@prepare(() -> connection, sql) +other_cached_statement(connection, sql) = DBInterface.@prepare(() -> connection, sql) + +@testset "@prepare" begin + prepare_count[] = 0 + first_connection = MockConnection(1) + second_connection = MockConnection(2) + + first_statement = cached_statement(first_connection, "SELECT 1") + @test first_statement === cached_statement(first_connection, "SELECT 1") + @test prepare_count[] == 1 + + second_statement = cached_statement(second_connection, "SELECT 1") + @test second_statement.connection === second_connection + @test second_statement !== first_statement + @test !first_statement.closed + + changed_sql_statement = cached_statement(second_connection, "SELECT 2") + @test changed_sql_statement.sql == "SELECT 2" + @test changed_sql_statement !== second_statement + @test second_statement.closed + + @test cached_statement(first_connection, "SELECT 1") === first_statement + + @test other_cached_statement(second_connection, "SELECT 2") !== changed_sql_statement + + connections = [MockConnection(i) for i in 1:100] + failures = fill(false, length(connections)) + Threads.@threads for i in eachindex(connections) + sql = "SELECT $i" + statement = cached_statement(connections[i], sql) + failures[i] = statement.connection !== connections[i] || statement.sql != sql + end + @test !any(failures) + + pooled_prepare_count = prepare_count[] + Threads.@threads for i in eachindex(connections) + statement = cached_statement(connections[i], "SELECT $i") + failures[i] = statement.connection !== connections[i] || statement.sql != "SELECT $i" + end + @test !any(failures) + @test prepare_count[] == pooled_prepare_count +end + +mutable struct ExecutionConnection <: DBInterface.Connection + statements::Vector{Any} +end + +ExecutionConnection() = ExecutionConnection(Any[]) + +const execution_transaction_count = Ref(0) + +mutable struct ExecutionStatement <: DBInterface.Statement + connection::ExecutionConnection + sql::String + executions::Vector{Any} + cursors::Vector{Any} + closed::Bool +end + +mutable struct ExecutionCursor <: DBInterface.Cursor + closed::Bool +end + +function DBInterface.prepare(connection::ExecutionConnection, sql::AbstractString) + statement = ExecutionStatement(connection, String(sql), Any[], Any[], false) + push!(connection.statements, statement) + return statement +end + +DBInterface.getconnection(statement::ExecutionStatement) = statement.connection +function DBInterface.transaction(f, ::ExecutionConnection) + execution_transaction_count[] += 1 + return f() +end +DBInterface.close!(statement::ExecutionStatement) = statement.closed = true +DBInterface.close!(cursor::ExecutionCursor) = cursor.closed = true + +function DBInterface.execute(statement::ExecutionStatement, params) + push!(statement.executions, params) + statement.sql == "fail" && length(statement.executions) == 2 && error("execution failed") + statement.sql == "nothing" && return nothing + cursor = ExecutionCursor(false) + push!(statement.cursors, cursor) + return cursor +end + +@testset "executemany" begin + connection = ExecutionConnection() + + positional_statement = DBInterface.prepare(connection, "positional") + DBInterface.executemany(positional_statement, ([1, 2], [3.0, 4.0])) + @test collect.(positional_statement.executions) == [[1, 3.0], [2, 4.0]] + @test all(cursor -> cursor.closed, positional_statement.cursors) + + named_statement = DBInterface.prepare(connection, "named") + DBInterface.executemany(named_statement, (id=[1, 2], name=["one", "two"])) + @test collect.(named_statement.executions) == [[1, "one"], [2, "two"]] + @test all(params -> params isa AbstractVector, named_statement.executions) + @test all(cursor -> cursor.closed, named_statement.cursors) + + dictionary_statement = DBInterface.prepare(connection, "dictionary") + DBInterface.executemany(dictionary_statement, Dict(:id => [1, 2], :name => ["one", "two"])) + @test Dict.(dictionary_statement.executions) == [ + Dict(:id => 1, :name => "one"), + Dict(:id => 2, :name => "two"), + ] + @test all(params -> params isa AbstractDict, dictionary_statement.executions) + dictionary_row = dictionary_statement.executions[1] + @test haskey(dictionary_row, :id) + @test !haskey(dictionary_row, :missing) + @test get(dictionary_row, :name, "unbound") == "one" + @test get(dictionary_row, :missing, "unbound") == "unbound" + @test get(() -> "unbound", dictionary_row, :name) == "one" + @test get(() -> "unbound", dictionary_row, :missing) == "unbound" + @test :id in keys(dictionary_row) + @test eltype(keys(dictionary_row)) === Symbol + + invalid_statement = DBInterface.prepare(connection, "invalid") + @test_throws DBInterface.ParameterError DBInterface.executemany( + invalid_statement, + (id=[1, 2], name=["one"]), + ) + @test isempty(invalid_statement.executions) + + DBInterface.executemany(connection, "managed", (id=[1, 2],)) + managed_statement = connection.statements[end] + @test managed_statement.closed + @test collect.(managed_statement.executions) == [[1], [2]] + + @test_throws ErrorException DBInterface.executemany(connection, "fail", (id=[1, 2],)) + failed_statement = connection.statements[end] + @test failed_statement.closed + @test failed_statement.cursors[1].closed + + nothing_statement = DBInterface.prepare(connection, "nothing") + DBInterface.executemany(nothing_statement, (id=[1, 2],)) + @test collect.(nothing_statement.executions) == [[1], [2]] + + execution_transaction_count[] = 0 + empty_batch_statement = DBInterface.prepare(connection, "empty batch") + DBInterface.executemany(empty_batch_statement, (id=Int[],)) + @test isempty(empty_batch_statement.executions) + @test execution_transaction_count[] == 0 + + empty_statement = DBInterface.prepare(connection, "empty") + DBInterface.executemany(empty_statement, ()) + @test empty_statement.executions == [()] + @test empty_statement.cursors[1].closed +end + +mutable struct TransactionConnection <: DBInterface.Connection + commands::Vector{String} + statements::Vector{Any} + fail_on::Union{Nothing, String} +end + +TransactionConnection(; fail_on=nothing) = TransactionConnection(String[], Any[], fail_on) + +mutable struct TransactionStatement <: DBInterface.Statement + connection::TransactionConnection + sql::String + closed::Bool +end + +mutable struct TransactionCursor <: DBInterface.Cursor + closed::Bool +end + +function DBInterface.prepare(connection::TransactionConnection, sql::AbstractString) + statement = TransactionStatement(connection, String(sql), false) + push!(connection.statements, statement) + return statement +end + +DBInterface.getconnection(statement::TransactionStatement) = statement.connection +DBInterface.close!(statement::TransactionStatement) = statement.closed = true +DBInterface.close!(cursor::TransactionCursor) = cursor.closed = true + +function DBInterface.execute(statement::TransactionStatement, params) + connection = statement.connection + push!(connection.commands, statement.sql) + connection.fail_on == statement.sql && error("$(statement.sql) failed") + return TransactionCursor(false) +end + +@testset "transaction" begin + connection = TransactionConnection() + @test DBInterface.transaction(() -> 42, connection) == 42 + @test connection.commands == ["BEGIN TRANSACTION;", "COMMIT;"] + @test all(statement -> statement.closed, connection.statements) + + body_connection = TransactionConnection() + body_error = ErrorException("body failed") + caught_error = try + DBInterface.transaction(body_connection) do + throw(body_error) + end + catch error + error + end + @test caught_error === body_error + @test body_connection.commands == ["BEGIN TRANSACTION;", "ROLLBACK;"] + @test all(statement -> statement.closed, body_connection.statements) + + rollback_connection = TransactionConnection(fail_on="ROLLBACK;") + rollback_body_error = ErrorException("body failed before rollback") + rollback_caught_error = try + DBInterface.transaction(rollback_connection) do + throw(rollback_body_error) + end + catch error + error + end + @test rollback_caught_error isa CompositeException + @test rollback_caught_error.exceptions[1].ex === rollback_body_error + @test occursin("ROLLBACK; failed", sprint(showerror, rollback_caught_error.exceptions[2].ex)) + @test rollback_connection.commands == ["BEGIN TRANSACTION;", "ROLLBACK;"] + @test all(statement -> statement.closed, rollback_connection.statements) + + commit_connection = TransactionConnection(fail_on="COMMIT;") + commit_error = try + DBInterface.transaction(() -> nothing, commit_connection) + catch error + error + end + @test occursin("COMMIT; failed", sprint(showerror, commit_error)) + @test commit_connection.commands == ["BEGIN TRANSACTION;", "COMMIT;", "ROLLBACK;"] +end + +struct MockDatabase end + +mutable struct ScopedConnection <: DBInterface.Connection + closed::Bool +end + +const scoped_connections = ScopedConnection[] + +function DBInterface.connect(::Type{MockDatabase}; option=false) + @test option + connection = ScopedConnection(false) + push!(scoped_connections, connection) + return connection +end + +DBInterface.close!(connection::ScopedConnection) = connection.closed = true + +mutable struct ScopedStatement <: DBInterface.Statement + connection::ScopedConnection + sql::String + option::Bool + closed::Bool +end + +function DBInterface.prepare(connection::ScopedConnection, sql::AbstractString; option=false) + return ScopedStatement(connection, String(sql), option, false) +end + +DBInterface.close!(statement::ScopedStatement) = statement.closed = true + +@testset "scoped resources" begin + empty!(scoped_connections) + result = DBInterface.connect(MockDatabase; option=true) do connection + @test !connection.closed + return 42 + end + @test result == 42 + @test scoped_connections[1].closed + + connection_error = ErrorException("connection body failed") + caught_connection_error = try + DBInterface.connect(MockDatabase; option=true) do connection + throw(connection_error) + end + catch error + error + end + @test caught_connection_error === connection_error + @test scoped_connections[2].closed + + connection = ScopedConnection(false) + statement = Ref{ScopedStatement}() + statement_result = DBInterface.prepare(connection, "SELECT 1"; option=true) do prepared + statement[] = prepared + @test !prepared.closed + @test prepared.option + return prepared.sql + end + @test statement_result == "SELECT 1" + @test statement[].closed + + statement_error = ErrorException("statement body failed") + caught_statement_error = try + DBInterface.prepare(connection, "SELECT 2"; option=true) do prepared + statement[] = prepared + throw(statement_error) + end + catch error + error + end + @test caught_statement_error === statement_error + @test statement[].closed +end + +@testset "error display" begin + @test sprint(showerror, DBInterface.ParameterError("invalid parameters")) == "invalid parameters" + @test sprint(showerror, DBInterface.Error("database error")) == "database error" +end