From 7fab77ed6c39330d63d0c912b84e8add1f5115d7 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Fri, 21 Aug 2026 13:54:52 +0500 Subject: [PATCH] Add in-core attach_node/detach_node alongside the SQL scripts. attach_node and detach_node join or remove a node entirely from C, reaching the other nodes over libpq, so dblink is no longer required. The SQL/dblink procedures in samples/Z0DAN stay, and either orchestration may be used. --- README.md | 2 +- docs/modify/zodan/index.md | 77 +- docs/modify/zodan/zodan_readme.md | 189 +- docs/modify/zodan/zodan_tutorial.md | 69 +- mkdocs.yml | 2 +- samples/Z0DAN/zodan.sql | 86 +- sql/spock--5.0.11--6.0.0.sql | 46 + sql/spock--6.0.0.sql | 46 + src/spock.c | 14 +- src/spock_apply.c | 6 +- src/spock_zodan.c | 2463 +++++++++++++++++ tests/docker/entrypoint.sh | 6 +- tests/tap/schedule | 5 + tests/tap/schedule-nightly | 3 + tests/tap/t/033_zodan_lolor_add_node.pl | 38 +- tests/tap/t/034_attach_detach_node_incore.pl | 350 +++ .../t/035_attach_node_sync_third_incore.pl | 450 +++ tests/tap/t/036_attach_node_basics_incore.pl | 119 + .../t/037_attach_node_3n_timeout_incore.pl | 197 ++ 19 files changed, 3935 insertions(+), 233 deletions(-) create mode 100644 src/spock_zodan.c create mode 100755 tests/tap/t/034_attach_detach_node_incore.pl create mode 100755 tests/tap/t/035_attach_node_sync_third_incore.pl create mode 100755 tests/tap/t/036_attach_node_basics_incore.pl create mode 100755 tests/tap/t/037_attach_node_3n_timeout_incore.pl diff --git a/README.md b/README.md index be9690df..ab3f6038 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ - [Modifying a Cluster](docs/modify/index.md) - Using Zodan - [Modifying your Cluster with Zodan](docs/modify/zodan/index.md) - - [Using Zodan Scripts and Workflows](docs/modify/zodan/zodan_readme.md) + - [Using Zodan (attach_node/detach_node and the SQL scripts)](docs/modify/zodan/zodan_readme.md) - [Adding a Node with Zero Downtime](docs/modify/zodan/zodan_tutorial.md) - [Adding a Node with Minimal Downtime with pgBackRest](docs/modify/add_node_pgbackrest.md) - Monitoring a Cluster diff --git a/docs/modify/zodan/index.md b/docs/modify/zodan/index.md index f0dfd3eb..01e4e823 100644 --- a/docs/modify/zodan/index.md +++ b/docs/modify/zodan/index.md @@ -1,52 +1,62 @@ # Modifying a Cluster with Zodan -Zodan provides tools to add or remove a node with zero downtime. The -scripts are located in the -[samples/Z0DAN](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) -directory of the [Spock GitHub](https://github.com/pgEdge/spock) repository. +Zodan (Zero Downtime Add/Remove Node) adds or removes a node with zero +downtime for the existing nodes. During node addition it manages creation of +the new node, subscription management (both to and from the node), replication +slot creation, data synchronization, replication slot advancement, and final +activation of subscriptions. During node removal it drops the node's +subscriptions, replication sets, slots, and origins in the correct order, +without deleting any Postgres artifacts (the database, data directory, log +files, and so on). + +Spock offers two ways to run this workflow. Both perform the same steps and +you can pick whichever fits your environment: + +- **In-core procedures (recommended).** `spock.attach_node` and + `spock.detach_node` are built into the Spock extension. A single + `CREATE EXTENSION spock` makes them available: there is no script to load + and no `dblink` dependency. All orchestration runs inside Spock and reaches + the other nodes over libpq. + +- **SQL scripts.** `spock.add_node` and `spock.remove_node` are loaded from + the SQL scripts in the + [samples/Z0DAN](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) + directory and reach the other nodes through the `dblink` extension. This + method is useful where you prefer to keep the orchestration in a script you + can read and modify. -During node addition, Zodan seamlessly manages creation of the new node, -subscription management (both to and from the node), replication slot -creation, data synchronization, replication slot advancement, and final -activation of subscriptions. +!!! note -During node removal, Zodan simplifies removing fully-functional or failed -nodes from a cluster. When you remove a node from a cluster, the removal -does not delete Postgres artifacts (the database, data directory, log -files, etc.). + Whichever method you use, the add procedure (`attach_node` or `add_node`) + must be run on the new node being added, and the remove procedure + (`detach_node` or `remove_node`) must be run on the node being removed. !!! hint - Zodan simplifies removing partially added nodes created during failed - node add operations. Additional cleanup steps may be required before - attempting another node deployment on the target host. - -!!! note - - Each script must be run from the target node being added or removed. + Zodan simplifies removing partially added nodes created during failed node + add operations. Additional cleanup steps may be required before attempting + another node deployment on the target host. ## Key Differences Between using Zodan and the Manual Process -The following differences highlight how Zodan automates and simplifies -node addition: +The following differences highlight how Zodan automates and simplifies node +addition: -- Zodan stores sync LSNs and uses them later to ensure subscriptions - start from the correct point even if hours pass between steps. +- Zodan stores sync LSNs and uses them later to ensure subscriptions start + from the correct point even if time passes between steps. -- Zodan automatically detects existing schemas on the new node and populates the - `skip_schema` parameter, preventing conflicts during structure sync. +- Zodan verifies all nodes run a compatible Spock version before starting. -- Zodan verifies all nodes run the same Spock version before starting. - -- Zodan includes the `verify_subscription_replicating()` function after - enabling subscriptions to ensure they reach replicating status. +- Zodan waits for each new subscription to reach the replicating state before + proceeding. - When adding to a single-node cluster, Zodan handles the process - differently — no disabled subscriptions are needed. - -- Zodan shows final status of all nodes and subscriptions across the - entire cluster, not just the new node. + differently, since no disabled subscriptions are needed. +- With the in-core procedures, every internal wait is bounded by the + `timeout_sec` argument (default 180 seconds), so a join that cannot make + progress fails quickly instead of blocking. Pass a larger `timeout_sec` if + your environment needs more headroom. For more information, review the following resources: @@ -54,4 +64,3 @@ For more information, review the following resources: - [Zodan Tutorial](zodan_tutorial.md) - [Zodan Scripts and Workflows](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) - [Spock Documentation](https://docs.pgedge.com/spock-v5/) - diff --git a/docs/modify/zodan/zodan_readme.md b/docs/modify/zodan/zodan_readme.md index cf0423c7..b4a213c0 100644 --- a/docs/modify/zodan/zodan_readme.md +++ b/docs/modify/zodan/zodan_readme.md @@ -1,110 +1,149 @@ -# Zodan: Zero-Downtime Node Addition for Spock +# Using Zodan: Zero-Downtime Node Addition and Removal -Zodan provides tools to add or remove a node with zero downtime. The -scripts are located in the -[samples/Z0DAN](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) -directory of the [Spock GitHub](https://github.com/pgEdge/spock) -repository. +Zodan adds or removes a node from a Spock cluster with zero downtime for the +existing nodes. Spock ships two implementations of the same workflow: the +in-core procedures `spock.attach_node` / `spock.detach_node`, and the SQL +scripts that provide `spock.add_node` / `spock.remove_node`. Both are covered +below. -Zodan's workflows and scripts streamline the process of adding a node to -or removing a node from a Spock cluster. Zodan features the following -scripts and workflows: +## In-core procedures: attach_node and detach_node -- The [zodan.sql](#using-the-zodansql-sql-workflow) workflow is a complete - SQL-based workflow that uses `dblink` to perform the same add node - operations from within Postgres. -- The [zodremove.sql](#the-zodremovesql-workflow) workflow is a complete - SQL-based workflow that uses `dblink` to perform the same removal - operations from within Postgres. +The in-core implementation is built into the Spock extension. A single +`CREATE EXTENSION spock` makes both procedures available. There is nothing to +install beyond the extension itself: the `dblink` extension is not required, +and there are no SQL scripts to load. All local work runs over SPI and all +cross-node work runs over libpq. -## Components +- `spock.attach_node` must be run on the new node being added. +- `spock.detach_node` must be run on the node being removed. -The following scripts and workflows are available via Zodan. +### Adding a node -### Using the zodan.sql SQL Workflow +`spock.attach_node` orchestrates the full join: it validates prerequisites, +creates the node, sets up replication slots and subscriptions in both +directions, coordinates synchronization events, advances replication slots and +origins to a consistent point, and enables replication. It supports both a two +node cluster and the general multi node case. -The SQL-based implementation utilizes the Postgres `dblink` extension to -handle node addition directly from within the database. This method is -ideal for environments where you may not have access to a shell or Python. +```sql +CALL spock.attach_node( + src_node_name => 'source_node_name', + src_dsn => 'src_dsn', + new_node_name => 'new_node_name', + new_node_dsn => 'new_node_dsn', + verb => false, -- verbose progress output, optional + new_node_location => 'NY', -- optional + new_node_country => 'USA', -- optional + new_node_info => '{}'::jsonb, -- optional metadata + timeout_sec => 180 -- bound on each wait, optional +); +``` -Within the workflow, SQL commands orchestrate the following operations: +In the following example, the command adds node `n4` to the cluster. Run it +while connected to `n4`: -- `add_node` - The main procedure to orchestrate the full workflow. -- `create_node` - Register the new node via `spock.node_create`. -- `get_spock_nodes` - Fetch current node metadata from a remote node. -- `create_sub` and `enable_sub` - Manage subscription creation and - activation. -- `create_replication_slot` - Create and configure logical replication - slots. -- `sync_event` and `wait_for_sync_event` - Coordinate data synchronization - events. -- `get_commit_timestamp` and `advance_replication_slot` - Align - replication states. +```sql +CALL spock.attach_node( + 'n1', + 'host=127.0.0.1 dbname=pgedge port=5431 user=pgedge password=', + 'n4', + 'host=127.0.0.1 dbname=pgedge port=5434 user=pgedge password=' +); +``` + +The `timeout_sec` argument bounds every internal wait loop, so a join that +cannot make progress fails quickly with a clear error rather than blocking for +a long fixed period. It defaults to 180 seconds; pass a larger value if your +environment needs more headroom. -To use the workflow, execute the following command in your Postgres -session. In the following example, the `spock.add_node` procedure adds a -new node to the cluster: +### Removing a node + +`spock.detach_node` removes a node in the correct order: it drops the inbound +subscriptions on each surviving node, drops the subscriptions local to the +node being removed, drops the node's replication sets, and finally drops the +node from the catalog. Replication slot and origin cleanup is handled as part +of dropping the subscriptions. A surviving node that cannot be reached during +teardown is skipped with a warning rather than aborting the removal. ```sql -CALL spock.add_node( - 'source_node_name', - 'src_dsn', - 'new_node_name', - 'new_node_dsn', - true|false, -- verbose? optional - 'new_node_location', -- optional - 'new_node_country', -- optional - '{}'::jsonb -- optional info +CALL spock.detach_node( + target_node_name => 'target_node_name', + target_node_dsn => 'target_dsn', -- DSN of the node being removed + verbose_mode => true -- verbose progress output, optional ); ``` -In the following example, the command adds node `n4` to the cluster: +`detach_node` runs on the node being removed and does all of its work locally +and against the surviving nodes it already knows about. `target_node_dsn` must +be the DSN of that same node: `detach_node` connects to it and compares the +system identifier and database name against the local ones, refusing to run if +they differ. This is the mirror of the `new_node_dsn` check in `attach_node`, +and it is what stops `detach_node` from tearing down the replication of +whichever node it was accidentally run on. + +In the following example, the command removes node `n4` from the cluster. Run +it while connected to `n4`: ```sql -CALL spock.add_node( - 'n1', - 'host=127.0.0.1 dbname=pgedge port=5431 user=pgedge password=', +CALL spock.detach_node( 'n4', 'host=127.0.0.1 dbname=pgedge port=5434 user=pgedge password=' ); ``` -### The zodremove.sql Workflow +## SQL scripts: add_node and remove_node -The SQL-based implementation utilizes the Postgres `dblink` extension to -handle node removal directly from within the database. This method is -ideal for environments where you may not have access to a shell or Python. +The SQL-based implementation uses the Postgres `dblink` extension to run the +same node add and remove operations from within the database. The scripts are +located in the +[samples/Z0DAN](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) +directory of the [Spock GitHub](https://github.com/pgEdge/spock) repository. +This method is useful where you prefer to keep the orchestration in a script +you can read and modify. Load `zodan.sql` and `zodremove.sql` on the node you +run the procedures from, and make sure `dblink` is installed there. -Within the workflow, SQL commands orchestrate the following operations: +### Adding a node with zodan.sql -- `spock.remove_node` - Main procedure to orchestrate the full workflow. -- `spock.remove_node_subscriptions` - Manages removing subscriptions. Also - removes the replication slot if there are no remaining subscriptions. -- `spock.remove_node_replication_sets` - Removes published repsets on the - node being removed. -- `spock.remove_node_from_cluster_registry` - Removes the node from the - cluster. +The `zodan.sql` workflow orchestrates the following operations: -The workflow is located in the -[samples/Z0DAN](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) -directory of the [Spock GitHub](https://github.com/pgEdge/spock) -repository. +- `add_node` - the main procedure to orchestrate the full workflow. +- `create_node` - register the new node via `spock.node_create`. +- `get_spock_nodes` - fetch current node metadata from a remote node. +- `create_sub` and `enable_sub` - manage subscription creation and activation. +- `create_replication_slot` - create and configure logical replication slots. +- `sync_event` and `wait_for_sync_event` - coordinate data synchronization + events. +- `get_commit_timestamp` and `advance_replication_slot` - align replication + states. -To use the workflow, call a command from your Postgres session. In the -following example, the Z0DAN `spock.remove_node` procedure removes a node -from the cluster. Note that `spock.remove_node` is a Z0DAN utility -procedure provided by `zodremove.sql`; it is not a built-in function of -the Spock extension. +To use the workflow, load `zodan.sql` and call `spock.add_node`. In the +following example, the command adds node `n4` to the cluster: ```sql -CALL spock.remove_node( - 'target_node_name', - 'target_dsn', - true -- verbose_mode, optional boolean +CALL spock.add_node( + 'n1', + 'host=127.0.0.1 dbname=pgedge port=5431 user=pgedge password=', + 'n4', + 'host=127.0.0.1 dbname=pgedge port=5434 user=pgedge password=' ); ``` -In the following example, the command removes node `n4` from the cluster: +### Removing a node with zodremove.sql + +The `zodremove.sql` workflow orchestrates the following operations: + +- `spock.remove_node` - the main procedure to orchestrate the full workflow. +- `spock.remove_node_subscriptions` - removes subscriptions, and the + replication slot once no subscriptions remain. +- `spock.remove_node_replication_sets` - removes published repsets on the node + being removed. +- `spock.remove_node_from_cluster_registry` - removes the node from the + cluster. + +To use the workflow, load `zodremove.sql` and call `spock.remove_node`. Note +that `spock.remove_node` is a Zodan utility procedure provided by +`zodremove.sql`; it is not a built-in function of the Spock extension. In the +following example, the command removes node `n4` from the cluster: ```sql CALL spock.remove_node( diff --git a/docs/modify/zodan/zodan_tutorial.md b/docs/modify/zodan/zodan_tutorial.md index c1a4a803..d7b9065e 100644 --- a/docs/modify/zodan/zodan_tutorial.md +++ b/docs/modify/zodan/zodan_tutorial.md @@ -23,8 +23,10 @@ node), `n2`, `n3`, and the new node is `n4`. - All nodes in your cluster must be available to Spock for the duration of the node addition. - The procedure should be performed on the new node being added. - - The `dblink` extension must be installed on the node from which - commands like `SELECT spock.add_node()` are being run. + - `spock.attach_node` and `spock.detach_node` are built into the Spock + extension. They reach the other nodes over libpq, so the `dblink` + extension is not required. (`dblink` is only needed if you choose to + follow the optional manual walkthrough later in this tutorial.) - Prepare the new node to meet all of the prerequisites described here. - If the process fails, do not immediately retry a command until you ensure that all artifacts created by the workflow have been removed. @@ -85,36 +87,27 @@ psql -c "CREATE DATABASE inventory;" psql -c "CREATE USER pgedge WITH PASSWORD '1safepassword';" psql -c "GRANT ALL ON DATABASE inventory TO pgedge;" -# Install Spock extension +# Install Spock extension (provides attach_node and detach_node) psql -d inventory -c "CREATE EXTENSION spock;" psql -d inventory -c "CREATE EXTENSION dblink;" ``` ## Using the Zodan Procedure to Add a Node -After creating the node, you can use [Zodan scripts](zodan_readme.md) to -simplify adding a node to a cluster. - -To use the SQL script, connect to the new node that you wish to add to the -pgEdge cluster. In the following example, the `psql` command connects to the -new node: +`spock.attach_node` is built into the extension, so there is nothing to load. +Connect to the new node that you wish to add to the pgEdge cluster. In the +following example, the `psql` command connects to the new node: ```bash psql -h 127.0.0.1 -p 5435 -d inventory -U pgedge ``` -Load the Zodan procedures with the following command: - -```sql -\i /path/to/zodan.sql -``` - -Then, use `spock.add_node()` from the new node to create the node -definition. In the following example, the `spock.add_node` procedure adds node `n4` to -the cluster: +Then, use `spock.attach_node()` from the new node to add it to the cluster. In +the following example, the `spock.attach_node` procedure adds node `n4` to the +cluster: ```sql -CALL spock.add_node( +CALL spock.attach_node( src_node_name := 'n1', src_dsn := 'host=127.0.0.1 dbname=inventory port=5432 user=pgedge password=1safepassword', new_node_name := 'n4', @@ -126,13 +119,41 @@ CALL spock.add_node( ); ``` -The `spock.add_node` function executes the steps required to add a node to +The `spock.attach_node` function executes the steps required to add a node to the cluster; a detailed explanation of the steps performed follows below. -Should a problem occur during this process, you can source the -`zodremove.sql` script and call the `spock.remove_node` procedure to -remove the node or reverse partially completed steps. The -`spock.remove_node` procedure should be called on the node being removed. +Should a problem occur during this process, you can call the +`spock.detach_node` procedure to remove the node or reverse partially +completed steps. The `spock.detach_node` procedure should be called on the +node being removed. + +### Setting the Synchronisation Timeout + +`spock.add_node()` waits at several points for the nodes to synchronise - +for a sync event to arrive on a peer, for the new node to catch up, and +for a subscription to start replicating. Each of those waits is allowed +180 seconds (3 minutes) by default, which is sized for a small cluster +where a node joins in seconds. On a large database, catchup can +legitimately take minutes or hours, and the procedure fails with a timeout +long before the work is done. + +To raise the budget, set +[`spock.sync_timeout`](../../configuring.md#spock-sync_timeout) in the +same session, before calling the add_node procedure: + +```sql +SET spock.sync_timeout = '2h'; +``` + +The new value applies to every synchronisation wait the procedure +performs, and it reverts when the session ends, so a long-running node +addition does not need a cluster-wide configuration change. To raise the +limit for every session, set the parameter in `postgresql.conf` instead. + +The setting bounds a whole wait, not an individual probe of a remote node; +steps that also bound each remote probe - so that one unresponsive peer +cannot consume the entire budget - keep their own, much smaller limit for +that purpose. ### Setting the Synchronisation Timeout diff --git a/mkdocs.yml b/mkdocs.yml index fc1181ea..2305651a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -71,7 +71,7 @@ nav: - Modifying a Cluster: modify/index.md - Using Z0DAN: - Modifying your Cluster with Zodan: modify/zodan/index.md - - Using Zodan Scripts and Workflows: modify/zodan/zodan_readme.md + - Using Zodan (attach_node/detach_node and SQL scripts): modify/zodan/zodan_readme.md - Adding a Node with Zero Downtime: modify/zodan/zodan_tutorial.md - Adding a Node with Minimal Downtime with pgBackRest: modify/add_node_pgbackrest.md - Recovering from Catastrophic Node Failure: recovery/catastrophic_node_failure.md diff --git a/samples/Z0DAN/zodan.sql b/samples/Z0DAN/zodan.sql index d0a444b2..276acbfc 100644 --- a/samples/Z0DAN/zodan.sql +++ b/samples/Z0DAN/zodan.sql @@ -302,13 +302,10 @@ BEGIN IF synchronize_structure THEN BEGIN - -- Query existing schemas on the new node (excluding system schemas). - -- lolor is excluded here: Spock already keeps it out of the - -- structure dump globally, and putting it into skip_schema would - -- also exclude the lolor tables from the initial data sync. + -- Query existing schemas on the new node (excluding system schemas) remotesql_schema := 'SELECT string_agg(schema_name, '','') as schemas FROM information_schema.schemata - WHERE schema_name NOT IN (''information_schema'', ''pg_catalog'', ''pg_toast'', ''spock'', ''public'', ''lolor'') + WHERE schema_name NOT IN (''information_schema'', ''pg_catalog'', ''pg_toast'', ''spock'', ''public'') AND schema_name NOT LIKE ''pg_temp_%'' AND schema_name NOT LIKE ''pg_toast_temp_%'''; @@ -1500,41 +1497,20 @@ BEGIN RAISE EXCEPTION 'Exiting add_node: Database % does not exist on new node. Please create it first.', new_db_name; END; - -- Check lolor state on the destination. Having the lolor extension - -- installed is fine (and required when the source replicates lolor - -- tables), but pre-existing large object data would conflict with the - -- data synchronized from the source. + -- Check if they previously installed lolor on the destination. + -- They should not have run CREATE EXTENSION yet DECLARE - new_lolor_installed boolean; - lolor_row_count bigint; - src_lolor_repset_tables integer; + user_table_count integer; remotesql text; BEGIN - remotesql := 'SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_extension WHERE extname = ''lolor'')'; - SELECT * FROM dblink(new_node_dsn, remotesql) AS t(installed boolean) INTO new_lolor_installed; - - IF new_lolor_installed THEN - remotesql := 'SELECT (SELECT count(*) FROM lolor.pg_largeobject) + (SELECT count(*) FROM lolor.pg_largeobject_metadata)'; - SELECT * FROM dblink(new_node_dsn, remotesql) AS t(row_count bigint) INTO lolor_row_count; - - IF lolor_row_count > 0 THEN - RAISE NOTICE ' [FAILED] %', rpad('Database ' || new_db_name || ' has pre-existing large object data in the lolor tables', 120, ' '); - RAISE EXCEPTION 'Exiting add_node: Database % on new node has pre-existing large object data in the lolor tables. The lolor tables must be empty so the large object data from the source node can be synchronized.', new_db_name; - ELSE - RAISE NOTICE ' OK: %', rpad('Checking database ' || new_db_name || ' lolor tables are empty', 120, ' '); - END IF; - END IF; - - -- If the source node replicates lolor tables, the new node must have - -- lolor installed or the initial data synchronization will fail. - remotesql := 'SELECT count(*) FROM spock.tables WHERE nspname = ''lolor'' AND set_name IS NOT NULL'; - SELECT * FROM dblink(src_dsn, remotesql) AS t(count integer) INTO src_lolor_repset_tables; + remotesql := 'SELECT count(*) FROM pg_tables WHERE schemaname = ''lolor'''; + SELECT * FROM dblink(new_node_dsn, remotesql) AS t(count integer) INTO user_table_count; - IF src_lolor_repset_tables > 0 AND NOT new_lolor_installed THEN - RAISE NOTICE ' [FAILED] %', rpad('Source node replicates lolor tables but database ' || new_db_name || ' does not have lolor installed', 120, ' '); - RAISE EXCEPTION 'Exiting add_node: Source node replicates lolor tables but database % on new node does not have the lolor extension installed. Please run CREATE EXTENSION lolor on the new node first.', new_db_name; + IF user_table_count > 0 THEN + RAISE NOTICE ' [FAILED] %', rpad('Database ' || new_db_name || ' has the lolor extension installed or remaining lolor data.', 120, ' '); + RAISE EXCEPTION 'Exiting add_node: Database % has the lolor extension installed or remaining lolor user data.', new_db_name; ELSE - RAISE NOTICE ' OK: %', rpad('Checking lolor requirements for database ' || new_db_name, 120, ' '); + RAISE NOTICE ' OK: %', rpad('Checking database ' || new_db_name || ' to ensure lolor is not installed', 120, ' '); END IF; END; @@ -1543,9 +1519,7 @@ BEGIN user_table_count integer; remotesql text; BEGIN - -- lolor tables are excluded: they belong to the lolor extension and - -- their emptiness is enforced by the lolor check above. - remotesql := 'SELECT count(*) FROM pg_tables WHERE schemaname NOT IN (''information_schema'', ''pg_catalog'', ''pg_toast'', ''spock'', ''lolor'') AND schemaname NOT LIKE ''pg_temp_%'' AND schemaname NOT LIKE ''pg_toast_temp_%'''; + remotesql := 'SELECT count(*) FROM pg_tables WHERE schemaname NOT IN (''information_schema'', ''pg_catalog'', ''pg_toast'', ''spock'') AND schemaname NOT LIKE ''pg_temp_%'' AND schemaname NOT LIKE ''pg_toast_temp_%'''; SELECT * FROM dblink(new_node_dsn, remotesql) AS t(count integer) INTO user_table_count; IF user_table_count > 0 THEN @@ -3001,9 +2975,6 @@ DECLARE new_version text; sub_count text; user_table_count text; - new_lolor_installed boolean; - lolor_row_count bigint; - src_lolor_repset_tables integer; BEGIN RAISE NOTICE ''; RAISE NOTICE '================================================================================'; @@ -3133,35 +3104,20 @@ BEGIN -- Check 8: Database prerequisites (pre-check only) -- ======================================================================== IF check_type = 'pre' AND new_node_dsn IS NOT NULL THEN - -- Check 8a: lolor state. Having lolor installed is OK, but the lolor - -- tables must be empty, and if the source replicates lolor tables the - -- new node must have lolor installed. + -- Check 8a: Verify lolor extension is not installed BEGIN - remotesql := 'SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_extension WHERE extname = ''lolor'')'; - SELECT * FROM dblink(new_node_dsn, remotesql) AS t(installed boolean) INTO new_lolor_installed; + remotesql := 'SELECT count(*) FROM pg_tables WHERE schemaname = ''lolor'''; + SELECT * FROM dblink(new_node_dsn, remotesql) AS t(table_count text) INTO user_table_count; - IF new_lolor_installed THEN - remotesql := 'SELECT (SELECT count(*) FROM lolor.pg_largeobject) + (SELECT count(*) FROM lolor.pg_largeobject_metadata)'; - SELECT * FROM dblink(new_node_dsn, remotesql) AS t(row_count bigint) INTO lolor_row_count; + IF user_table_count IS NOT NULL AND user_table_count::integer = 0 THEN + RAISE NOTICE 'PASS: Destination database does not have signs of lolor being installed'; + checks_passed := checks_passed + 1; ELSE - lolor_row_count := 0; - END IF; - - remotesql := 'SELECT count(*) FROM spock.tables WHERE nspname = ''lolor'' AND set_name IS NOT NULL'; - SELECT * FROM dblink(src_dsn, remotesql) AS t(count integer) INTO src_lolor_repset_tables; - - IF lolor_row_count > 0 THEN - RAISE NOTICE 'FAIL: Destination database has pre-existing large object data in the lolor tables'; - checks_failed := checks_failed + 1; - ELSIF src_lolor_repset_tables > 0 AND NOT new_lolor_installed THEN - RAISE NOTICE 'FAIL: Source node replicates lolor tables but the destination database does not have the lolor extension installed'; + RAISE NOTICE 'FAIL: Destination database has the lolor extension installed or remaining lolor user data in the lolor schema'; checks_failed := checks_failed + 1; - ELSE - RAISE NOTICE 'PASS: Destination database lolor state is compatible with add_node'; - checks_passed := checks_passed + 1; END IF; EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'FAIL: lolor check - %', SQLERRM; + RAISE NOTICE 'FAIL: lolor extension check - %', SQLERRM; checks_failed := checks_failed + 1; END; @@ -3169,7 +3125,7 @@ BEGIN BEGIN remotesql := $pg_tables$ SELECT count(*) FROM pg_tables - WHERE schemaname NOT IN ('information_schema', 'pg_catalog', 'pg_toast', 'spock', 'lolor') + WHERE schemaname NOT IN ('information_schema', 'pg_catalog', 'pg_toast', 'spock') AND schemaname NOT LIKE 'pg_temp_%' AND schemaname NOT LIKE 'pg_toast_temp_%' $pg_tables$; diff --git a/sql/spock--5.0.11--6.0.0.sql b/sql/spock--5.0.11--6.0.0.sql index 131bf114..bb1abf92 100644 --- a/sql/spock--5.0.11--6.0.0.sql +++ b/sql/spock--5.0.11--6.0.0.sql @@ -325,6 +325,52 @@ RETURNS boolean AS 'MODULE_PATHNAME', 'spock_alter_subscription_options' LANGUAGE C STRICT VOLATILE; +-- ---------------------------------------------------------------------------- +-- Zero Downtime Add/Remove Node (ZODAN), in-core C orchestration +-- +-- attach_node/detach_node orchestrate joining or removing a node entirely in +-- C, reaching the other cluster nodes over libpq (no dblink dependency). +-- +-- These are the in-core equivalents of the SQL/dblink procedures shipped in +-- samples/Z0DAN/zodan.sql (spock.add_node) and zodremove.sql +-- (spock.remove_node); users may choose either orchestration. +-- +-- attach_node must be run on the new node being added. +-- detach_node must be run on the node being removed. +-- +-- Neither procedure is atomic across the cluster. Both commit local state at +-- phase boundaries, and the subscriptions they create or drop on the other nodes +-- are committed one node at a time by the remote server. A failure part-way +-- through therefore leaves the cluster in an intermediate state that is not +-- rolled back: recovery from a failed attach_node is spock.detach_node() on the +-- new node, then a fresh attach_node(). See the header of src/spock_zodan.c. +-- +-- Both take an arbitrary connection string and reach out to it, and detach_node +-- drops every subscription, replication set and the node itself, so EXECUTE is +-- revoked from PUBLIC below. +-- ---------------------------------------------------------------------------- +CREATE PROCEDURE spock.attach_node( + src_node_name text, + src_dsn text, + new_node_name text, + new_node_dsn text, + verb boolean DEFAULT false, + new_node_location text DEFAULT 'NY', + new_node_country text DEFAULT 'USA', + new_node_info jsonb DEFAULT '{}'::jsonb, + timeout_sec int DEFAULT 180 +) LANGUAGE c AS 'MODULE_PATHNAME', 'spock_attach_node'; + +CREATE PROCEDURE spock.detach_node( + target_node_name text, + target_node_dsn text, + verbose_mode boolean DEFAULT true +) LANGUAGE c AS 'MODULE_PATHNAME', 'spock_detach_node'; + +REVOKE ALL ON PROCEDURE spock.attach_node(text, text, text, text, boolean, + text, text, jsonb, int) FROM PUBLIC; +REVOKE ALL ON PROCEDURE spock.detach_node(text, text, boolean) FROM PUBLIC; + -- ---- -- Enable the "failover" flag on spock's existing logical replication slots. -- diff --git a/sql/spock--6.0.0.sql b/sql/spock--6.0.0.sql index b35815d5..ca39b456 100644 --- a/sql/spock--6.0.0.sql +++ b/sql/spock--6.0.0.sql @@ -932,3 +932,49 @@ CREATE FUNCTION spock.reset_subscription_stats(subid oid DEFAULT NULL) RETURNS void AS 'MODULE_PATHNAME', 'spock_reset_subscription_stats' LANGUAGE C CALLED ON NULL INPUT VOLATILE; + +-- ---------------------------------------------------------------------------- +-- Zero Downtime Add/Remove Node (ZODAN), in-core C orchestration +-- +-- attach_node/detach_node orchestrate joining or removing a node entirely in +-- C, reaching the other cluster nodes over libpq (no dblink dependency). +-- +-- These are the in-core equivalents of the SQL/dblink procedures shipped in +-- samples/Z0DAN/zodan.sql (spock.add_node) and zodremove.sql +-- (spock.remove_node); users may choose either orchestration. +-- +-- attach_node must be run on the new node being added. +-- detach_node must be run on the node being removed. +-- +-- Neither procedure is atomic across the cluster. Both commit local state at +-- phase boundaries, and the subscriptions they create or drop on the other nodes +-- are committed one node at a time by the remote server. A failure part-way +-- through therefore leaves the cluster in an intermediate state that is not +-- rolled back: recovery from a failed attach_node is spock.detach_node() on the +-- new node, then a fresh attach_node(). See the header of src/spock_zodan.c. +-- +-- Both take an arbitrary connection string and reach out to it, and detach_node +-- drops every subscription, replication set and the node itself, so EXECUTE is +-- revoked from PUBLIC below. +-- ---------------------------------------------------------------------------- +CREATE PROCEDURE spock.attach_node( + src_node_name text, + src_dsn text, + new_node_name text, + new_node_dsn text, + verb boolean DEFAULT false, + new_node_location text DEFAULT 'NY', + new_node_country text DEFAULT 'USA', + new_node_info jsonb DEFAULT '{}'::jsonb, + timeout_sec int DEFAULT 180 +) LANGUAGE c AS 'MODULE_PATHNAME', 'spock_attach_node'; + +CREATE PROCEDURE spock.detach_node( + target_node_name text, + target_node_dsn text, + verbose_mode boolean DEFAULT true +) LANGUAGE c AS 'MODULE_PATHNAME', 'spock_detach_node'; + +REVOKE ALL ON PROCEDURE spock.attach_node(text, text, text, text, boolean, + text, text, jsonb, int) FROM PUBLIC; +REVOKE ALL ON PROCEDURE spock.detach_node(text, text, boolean) FROM PUBLIC; diff --git a/src/spock.c b/src/spock.c index 117c3e25..fd2c10bf 100644 --- a/src/spock.c +++ b/src/spock.c @@ -414,10 +414,20 @@ spock_connect_base(const char *connstr, const char *appname, conn = PQconnectdbParams(keys, vals, /* expand_dbname = */ true); if (PQstatus(conn) != CONNECTION_OK) { + char msg[1024]; + + /* + * A failed PGconn still owns memory and possibly a socket, and nothing + * here is registered with a resource owner, so it must be closed before + * we longjmp out. Copy the message out first: PQerrorMessage() points + * into the PGconn. + */ + snprintf(msg, sizeof(msg), "%s", PQerrorMessage(conn)); + PQfinish(conn); + ereport(ERROR, (errmsg("could not connect to the postgresql server%s: %s", - replication ? " in replication mode" : "", - PQerrorMessage(conn)), + replication ? " in replication mode" : "", msg), errdetail("dsn was: %s", s.data))); } diff --git a/src/spock_apply.c b/src/spock_apply.c index c3331db7..83da3baa 100644 --- a/src/spock_apply.c +++ b/src/spock_apply.c @@ -557,7 +557,7 @@ action_error_callback(void *arg) */ /* - * If the pause flag is set (slot creation in progress for add_node), + * If the pause flag is set (slot creation in progress for attach_node), * sleep on the ConditionVariable until resumed or timed out. */ static void @@ -601,8 +601,8 @@ begin_replication_step(void) if (!IsTransactionState()) { /* - * Check if slot creation (add_node) needs us to pause. This only - * fires during add_node (a rare operation). The fast path is a + * Check if slot creation (attach_node) needs us to pause. This only + * fires during attach_node (a rare operation). The fast path is a * single atomic read that almost always sees 0. * * Runs before StartTransactionCommand so the worker has no xid diff --git a/src/spock_zodan.c b/src/spock_zodan.c new file mode 100644 index 00000000..15b1ce1c --- /dev/null +++ b/src/spock_zodan.c @@ -0,0 +1,2463 @@ +/*------------------------------------------------------------------------- + * + * spock_zodan.c + * Zero Downtime Add/Remove Node (ZODAN) orchestration. + * + * Implements spock.attach_node() and spock.detach_node() entirely in C. These + * are the in-core counterparts of the PL/pgSQL procedures still shipped in + * samples/Z0DAN/zodan.sql (spock.add_node) and zodremove.sql + * (spock.remove_node), which reach every other node in the cluster through the + * dblink extension. Users may choose either orchestration. The in-core + * orchestration is driven from C: local work is done over SPI (calling the + * same spock.* SQL functions), and all cross-node work is done over libpq via + * spock_connect(), so the dblink dependency is gone. + * + * Both procedures are LANGUAGE c PROCEDUREs: they run non-atomically (invoked + * via top-level CALL) and use SPI_commit() at the phase boundaries where the + * original scripts relied on COMMIT so that apply/sync workers pick up newly + * created or enabled subscriptions. + * + * attach_node must be run on the new node being added. + * detach_node must be run on the node being removed. + * + * NEITHER PROCEDURE IS ATOMIC ACROSS THE CLUSTER. Local state is committed at + * the phase boundaries listed above, and the subscriptions created on the other + * nodes are committed one node at a time by the remote server (libpq + * autocommit). A failure part-way through therefore leaves the cluster in an + * intermediate state -- for example, attach_node failing on the third of five + * nodes leaves the first two subscribed to the new node -- and nothing is rolled + * back or compensated. Recovery is spock.detach_node() on the new node, + * followed by a fresh attach_node() once the cause has been addressed. This + * matches the behavior of the SQL/dblink spock.add_node(). + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, The Regents of the University of California + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "libpq-fe.h" + +#include "access/xact.h" +#include "executor/spi.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "storage/latch.h" +#include "utils/builtins.h" +#include "utils/memutils.h" +#include "utils/pg_lsn.h" +#include "utils/resowner.h" +#include "utils/timestamp.h" + +#include "spock.h" +#include "spock_node.h" + +PG_FUNCTION_INFO_V1(spock_attach_node); +PG_FUNCTION_INFO_V1(spock_detach_node); + +/* + * Replication sets used for a ZODAN-managed subscription when its provider + * reports none at all, which should not happen: spock.node_create() always + * creates these three. The real list is read from the provider, so a cluster + * with user-created sets does not silently lose them on the new node. + */ +#define ZODAN_DEFAULT_REPSETS "ARRAY['default', 'default_insert_only', 'ddl_sql']" + +/* Minimum Spock version required on every node participating in attach_node. */ +#define ZODAN_MIN_VERSION "5.0.9" + +/* + * An order-independent digest of a node's whole replication set configuration: + * the set definitions, the table memberships with their column lists and row + * filters, and the sequence memberships. Run on the source and on the new node + * to confirm the mirror took, and that the source did not move underneath it. + */ +#define ZODAN_REPSET_DIGEST_SQL \ + "SELECT md5(coalesce(string_agg(entry, E'\\n' ORDER BY entry), '')) FROM (" \ + " SELECT rs.set_name || ' ' || rs.replicate_insert || rs.replicate_update ||" \ + " rs.replicate_delete || rs.replicate_truncate AS entry" \ + " FROM spock.replication_set rs" \ + " JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id" \ + " UNION ALL" \ + " SELECT rs.set_name || ' ' ||" \ + " quote_ident(n.nspname) || '.' || quote_ident(c.relname) || ' ' ||" \ + " coalesce(rst.set_att_list::text, '') || ' ' ||" \ + " coalesce(pg_get_expr(rst.set_row_filter, rst.set_reloid), '')" \ + " FROM spock.replication_set_table rst" \ + " JOIN spock.replication_set rs ON rs.set_id = rst.set_id" \ + " JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id" \ + " JOIN pg_catalog.pg_class c ON c.oid = rst.set_reloid" \ + " JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace" \ + " UNION ALL" \ + " SELECT rs.set_name || ' seq ' ||" \ + " quote_ident(n.nspname) || '.' || quote_ident(c.relname)" \ + " FROM spock.replication_set_seq rss" \ + " JOIN spock.replication_set rs ON rs.set_id = rss.set_id" \ + " JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id" \ + " JOIN pg_catalog.pg_class c ON c.oid = rss.set_seqoid" \ + " JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace" \ + ") s" + +/* + * A remote node and the DSN we reach it on. Built by connecting to the source + * node and reading spock.node JOIN spock.node_interface. + */ +typedef struct ZNode +{ + char *name; + char *dsn; + char *location; + char *country; + char *info; +} ZNode; + +/* + * Everything the attach_node phases need, so we don't thread a dozen arguments + * through each helper. All strings live in the dedicated memory context + * (see spock_attach_node) so they survive the SPI_commit() calls between phases. + */ +typedef struct ZodanAddCtx +{ + char *src_node_name; + char *src_dsn; + char *new_node_name; + char *new_node_dsn; + bool verb; + char *new_node_location; + char *new_node_country; + char *new_node_info; /* jsonb rendered as text */ + int timeout_sec; + + /* Cluster snapshot fetched from the source node (all existing nodes). */ + ZNode *nodes; + int nnodes; + + MemoryContext mcxt; /* survives SPI_commit() */ +} ZodanAddCtx; + +/* A two-column row snapshotted out of SPI_tuptable; see zodan_local_pairs(). */ +typedef struct ZPair +{ + char *a; + char *b; +} ZPair; + +/* + * Progress reporting. All progress output is gated on verbose mode so a + * normal run is quiet; only the final confirmation is emitted unconditionally + * by the entry points. zodan_verbose is set once per top-level call, from the + * procedure's verbose argument, so the macro needs no per-call flag. + */ +static bool zodan_verbose = false; + +#define ZNOTICE(...) \ + do { if (zodan_verbose) ereport(NOTICE, (errmsg(__VA_ARGS__))); } while (0) + +/* + * A boolean column of a PGresult as a SQL literal. libpq renders booleans as + * "t"/"f", which parses as a column reference rather than a value when it is + * interpolated into a query. + */ +#define ZBOOL(res, row, col) \ + (PQgetvalue((res), (row), (col))[0] == 't' ? "true" : "false") + +/* ------------------------------------------------------------------------ + * Small utilities + * ------------------------------------------------------------------------ */ + +/* + * Sleep for the given number of milliseconds, honoring query cancel and + * postmaster death. Used by the various catch-up/sync wait loops. + */ +static void +zodan_sleep_ms(long ms) +{ + int rc; + + CHECK_FOR_INTERRUPTS(); + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + ms, PG_WAIT_EXTENSION); + ResetLatch(MyLatch); + if (rc & WL_LATCH_SET) + CHECK_FOR_INTERRUPTS(); +} + +/* + * Extract the database name from a libpq connection string. Errors out if the + * DSN does not name a database, matching the old extract_dbname_from_dsn(). + */ +static char * +zodan_dbname_from_dsn(const char *dsn) +{ + char *parse_err = NULL; + PQconninfoOption *opts; + PQconninfoOption *o; + char *dbname = NULL; + + opts = PQconninfoParse(dsn, &parse_err); + if (opts == NULL) + { + char buf[1024]; + + snprintf(buf, sizeof(buf), "%s", parse_err ? parse_err : "unknown error"); + if (parse_err) + PQfreemem(parse_err); + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("could not parse DSN \"%s\": %s", dsn, buf))); + } + + for (o = opts; o->keyword != NULL; o++) + { + if (strcmp(o->keyword, "dbname") == 0 && o->val != NULL && o->val[0] != '\0') + { + dbname = pstrdup(o->val); + break; + } + } + PQconninfoFree(opts); + + if (dbname == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("database name must be included in the DSN string: %s", + dsn))); + return dbname; +} + +/* Build the subscription name sub__. */ +static char * +zodan_gen_sub_name(const char *provider, const char *subscriber) +{ + return psprintf("sub_%s_%s", provider, subscriber); +} + +/* + * Build the replication slot name for a subscription, matching + * spock.spock_gen_slot_name(dbname, provider_node, sub_name). + */ +static char * +zodan_gen_slot_name(const char *dsn, const char *provider_node, + const char *sub_name) +{ + NameData slot; + char *dbname = zodan_dbname_from_dsn(dsn); + + gen_slot_name(&slot, dbname, provider_node, sub_name); + pfree(dbname); + return pstrdup(NameStr(slot)); +} + +/* + * Parse "5.0.10-devel" into {major, minor, patch}; ignores any suffix. label + * names the node the version came from, for the error message: silently + * treating an unparseable version as 0.0.0 would surface as a bogus "has + * version 0.0.0" mismatch instead of the real problem. + */ +static void +zodan_parse_version(const char *v, const char *label, + int *major, int *minor, int *patch) +{ + *major = *minor = *patch = 0; + if (v == NULL || sscanf(v, "%d.%d.%d", major, minor, patch) != 3) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("could not parse Spock version \"%s\" on %s", + v ? v : "(null)", label))); +} + +/* Convert a pg_lsn text value to XLogRecPtr, ERROR if malformed. */ +static XLogRecPtr +zodan_parse_lsn(const char *lsn) +{ + bool have_error = false; + XLogRecPtr ret; + + if (lsn == NULL) + ereport(ERROR, (errmsg("expected an LSN value, got NULL"))); + + ret = pg_lsn_in_internal(lsn, &have_error); + if (have_error) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid LSN value \"%s\"", lsn))); + return ret; +} + +/* Return <0, 0, >0 comparing the left major.minor.patch to the right one. */ +static int +zodan_version_cmp(int left_major, int left_minor, int left_patch, + int right_major, int right_minor, int right_patch) +{ + if (left_major != right_major) + return left_major - right_major; + if (left_minor != right_minor) + return left_minor - right_minor; + return left_patch - right_patch; +} + +/* ------------------------------------------------------------------------ + * libpq (remote node) helpers -- these replace dblink() + * ------------------------------------------------------------------------ */ + +/* + * Open libpq connections and PGresults are tracked here so that a phase helper + * which ereport(ERROR)s mid-orchestration does not strand its socket, or leak + * the result's malloc'd memory, for the rest of the backend's life. + * spock_connect() does not register the PGconn with any resource owner, PGresult + * has no resource-owner integration at all, and the phase helpers keep both in + * local variables -- so on error the entry point drains these registries (see + * zodan_release_all). A handful of slots is far more than the two or three + * connections and single result ever live at once. + */ +#define ZODAN_MAX_CONNS 16 +static PGconn *zodan_conns[ZODAN_MAX_CONNS]; +static int zodan_nconns = 0; + +#define ZODAN_MAX_RESULTS 16 +static PGresult *zodan_results[ZODAN_MAX_RESULTS]; +static int zodan_nresults = 0; + +/* Connect to a remote node, ERROR on failure. Tracks the connection. */ +static PGconn * +zodan_connect(const char *dsn, const char *purpose) +{ + PGconn *conn; + + /* + * Refuse rather than hand back an untracked connection: overflow would mean + * a leak on the error path, which is exactly what the registry exists to + * prevent. Reserve the slot before connecting so nothing can be dropped. + */ + if (zodan_nconns >= ZODAN_MAX_CONNS) + elog(ERROR, "too many concurrent zodan connections (max %d)", + ZODAN_MAX_CONNS); + + conn = spock_connect(dsn, "spock_zodan", purpose); + if (conn != NULL) + zodan_conns[zodan_nconns++] = conn; + return conn; +} + +/* Close a tracked connection (no-op on NULL) and forget it. */ +static void +zodan_disconnect(PGconn *conn) +{ + int i; + + if (conn == NULL) + return; + for (i = 0; i < zodan_nconns; i++) + { + if (zodan_conns[i] == conn) + { + zodan_conns[i] = zodan_conns[--zodan_nconns]; + break; + } + } + PQfinish(conn); +} + +/* Clear a tracked PGresult (no-op on NULL) and forget it. */ +static void +zodan_clear(PGresult *res) +{ + int i; + + if (res == NULL) + return; + for (i = 0; i < zodan_nresults; i++) + { + if (zodan_results[i] == res) + { + zodan_results[i] = zodan_results[--zodan_nresults]; + break; + } + } + PQclear(res); +} + +/* + * Release every still-live tracked result and connection. Used for cleanup on + * error, and to assert a clean slate on entry. + */ +static void +zodan_release_all(void) +{ + while (zodan_nresults > 0) + PQclear(zodan_results[--zodan_nresults]); + while (zodan_nconns > 0) + PQfinish(zodan_conns[--zodan_nconns]); +} + +/* + * Run a query on a remote node that is expected to return tuples. The result is + * tracked; the caller owns it and must release it with zodan_clear(). ERROR on + * failure. + */ +static PGresult * +zodan_remote_query(PGconn *conn, const char *sql) +{ + PGresult *res; + + if (zodan_nresults >= ZODAN_MAX_RESULTS) + elog(ERROR, "too many concurrent zodan results (max %d)", + ZODAN_MAX_RESULTS); + + res = PQexec(conn, sql); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + char msg[1024]; + + snprintf(msg, sizeof(msg), "%s", PQerrorMessage(conn)); + PQclear(res); + ereport(ERROR, + (errmsg("remote query failed on node"), + errdetail("query: %s", sql), + errdetail("error: %s", msg))); + } + zodan_results[zodan_nresults++] = res; + return res; +} + +/* + * Run a command on a remote node (COMMAND_OK or TUPLES_OK both accepted). + * ERROR on failure. + */ +static void +zodan_remote_command(PGconn *conn, const char *sql) +{ + PGresult *res = PQexec(conn, sql); + ExecStatusType st = PQresultStatus(res); + + if (st != PGRES_COMMAND_OK && st != PGRES_TUPLES_OK) + { + char msg[1024]; + + snprintf(msg, sizeof(msg), "%s", PQerrorMessage(conn)); + PQclear(res); + ereport(ERROR, + (errmsg("remote command failed on node"), + errdetail("command: %s", sql), + errdetail("error: %s", msg))); + } + PQclear(res); +} + +/* + * Run a query on a remote node and return the first column of the first row as + * a palloc'd string (NULL if no rows or a SQL NULL). + */ +static char * +zodan_remote_scalar(PGconn *conn, const char *sql) +{ + PGresult *res = zodan_remote_query(conn, sql); + char *ret = NULL; + + if (PQntuples(res) > 0 && !PQgetisnull(res, 0, 0)) + ret = pstrdup(PQgetvalue(res, 0, 0)); + zodan_clear(res); + return ret; +} + +/* ------------------------------------------------------------------------ + * SPI (local node) helpers + * ------------------------------------------------------------------------ */ + +/* Execute a local command over SPI, ERROR on failure. */ +static void +zodan_local_command(const char *sql) +{ + int rc = SPI_execute(sql, false, 0); + + if (rc < 0) + elog(ERROR, "SPI_execute failed for: %s", sql); +} + +/* + * Execute a local query over SPI and return the first column of the first row + * as a palloc'd string in the caller's context (NULL if no rows / SQL NULL). + * + * read_only is false so every call takes a fresh snapshot; the wait/poll loops + * depend on observing rows committed by the apply and sync workers while the + * loop is running. + */ +static char * +zodan_local_scalar(const char *sql) +{ + int rc; + char *ret = NULL; + + rc = SPI_execute(sql, false, 0); + if (rc != SPI_OK_SELECT) + elog(ERROR, "SPI_execute (select) failed for: %s", sql); + + /* SPI_getvalue() returns NULL for a SQL NULL, which is what we want. */ + if (SPI_processed > 0) + ret = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1); + + return ret; +} + +/* + * Snapshot a one- or two-column local query into palloc'd strings in mcxt. + * + * Callers act on the rows by issuing further SPI_execute() calls, and each of + * those replaces SPI_tuptable, so the rows have to be copied out first. A SQL + * NULL, and a missing second column, come back as NULL. + */ +static ZPair * +zodan_local_pairs(const char *sql, MemoryContext mcxt, int *nrows) +{ + ZPair *rows; + int n; + int i; + int natts; + MemoryContext old; + + if (SPI_execute(sql, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute (select) failed for: %s", sql); + + n = SPI_processed; + natts = SPI_tuptable->tupdesc->natts; + + /* SPI_getvalue() allocates in the current context, so switch first. */ + old = MemoryContextSwitchTo(mcxt); + rows = (ZPair *) palloc0(sizeof(ZPair) * Max(n, 1)); + for (i = 0; i < n; i++) + { + rows[i].a = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1); + if (natts >= 2) + rows[i].b = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 2); + } + MemoryContextSwitchTo(old); + + *nrows = n; + return rows; +} + +/* Convenience: local scalar as int64 (0 if NULL). */ +static int64 +zodan_local_count(const char *sql) +{ + char *s = zodan_local_scalar(sql); + int64 v = 0; + + if (s != NULL) + { + v = pg_strtoint64(s); + pfree(s); + } + return v; +} + +/* ------------------------------------------------------------------------ + * Cluster inspection + * ------------------------------------------------------------------------ */ + +/* + * Fetch the list of all nodes known to the source cluster (from the source + * node), storing them in ctx->nodes/ctx->nnodes in ctx->mcxt. + */ +static void +zodan_fetch_cluster_nodes(ZodanAddCtx *ctx) +{ + PGconn *conn; + PGresult *res; + int n; + int i; + MemoryContext old; + + conn = zodan_connect(ctx->src_dsn, "nodes"); + res = zodan_remote_query(conn, + "SELECT n.node_name, i.if_dsn, " + "COALESCE(n.location,''), COALESCE(n.country,''), " + "COALESCE(n.info::text,'') " + "FROM spock.node n " + "JOIN spock.node_interface i ON n.node_id = i.if_nodeid " + "ORDER BY n.node_name"); + n = PQntuples(res); + + old = MemoryContextSwitchTo(ctx->mcxt); + ctx->nodes = (ZNode *) palloc0(sizeof(ZNode) * Max(n, 1)); + for (i = 0; i < n; i++) + { + ctx->nodes[i].name = pstrdup(PQgetvalue(res, i, 0)); + ctx->nodes[i].dsn = pstrdup(PQgetvalue(res, i, 1)); + ctx->nodes[i].location = pstrdup(PQgetvalue(res, i, 2)); + ctx->nodes[i].country = pstrdup(PQgetvalue(res, i, 3)); + ctx->nodes[i].info = pstrdup(PQgetvalue(res, i, 4)); + } + ctx->nnodes = n; + MemoryContextSwitchTo(old); + + zodan_clear(res); + zodan_disconnect(conn); + + /* + * src_node_name is only ever used to compare against these names -- to pick + * the source out of the cluster, to count the "other" nodes (which selects + * the 2-node vs multi-node path in several phases), and to build + * subscription names. A name that does not match any node therefore does + * not fail here; it silently reclassifies the real source as an "other" + * node and fails much later, in a confusing place. Reject it up front. + */ + { + StringInfoData known; + + for (i = 0; i < n; i++) + { + if (strcmp(ctx->nodes[i].name, ctx->src_node_name) == 0) + return; + } + + initStringInfo(&known); + for (i = 0; i < n; i++) + appendStringInfo(&known, "%s%s", i > 0 ? ", " : "", + ctx->nodes[i].name); + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("source node \"%s\" is not a node in the cluster reached by src_dsn", + ctx->src_node_name), + errdetail("Nodes known to that cluster: %s", + known.len > 0 ? known.data : "(none)"))); + } +} + +/* + * Render a PGresult's first column as the SQL array literal ARRAY['a', 'b'], + * or ZODAN_DEFAULT_REPSETS when it has no rows. + */ +static char * +zodan_repset_array(PGresult *res) +{ + StringInfoData buf; + int n = PQntuples(res); + int i; + + if (n == 0) + { + /* + * A Spock node always has the built-ins, so no rows means the DSN does + * not point at one. Fall back rather than build an empty array that + * would replicate nothing. + */ + return pstrdup(ZODAN_DEFAULT_REPSETS); + } + + initStringInfo(&buf); + appendStringInfoString(&buf, "ARRAY["); + for (i = 0; i < n; i++) + appendStringInfo(&buf, "%s%s", i > 0 ? ", " : "", + quote_literal_cstr(PQgetvalue(res, i, 0))); + appendStringInfoChar(&buf, ']'); + return buf.data; +} + +/* + * A node's replication sets, as the SQL array literal for the replication_sets + * argument of sub_create(). + * + * Subscriptions used to be pinned to the three built-in sets, which silently + * dropped every table that only lives in a user-created set: the new node + * neither received those tables nor sent them, with no error anywhere. + * + * This must always be called with the subscription's PROVIDER dsn. A + * subscription's set names are resolved on the provider, by + * get_replication_sets() in spock_repset.c, and a name the provider does not + * have is an ERROR there ("replication set %s not found"), not something it + * ignores. The walsender then refuses START_REPLICATION and the apply worker + * retries forever. Replication sets are per-node local state, so two existing + * nodes can legitimately have different ones. + */ +static char * +zodan_node_repset_list(const char *node_dsn) +{ + PGconn *conn; + PGresult *res; + char *arr; + + conn = zodan_connect(node_dsn, "repsets"); + res = zodan_remote_query(conn, + "SELECT rs.set_name " + "FROM spock.replication_set rs " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id " + "ORDER BY rs.set_name"); + arr = zodan_repset_array(res); + zodan_clear(res); + zodan_disconnect(conn); + + return arr; +} + +/* Number of nodes that are neither the source nor the new node. */ +static int +zodan_num_other_nodes(ZodanAddCtx *ctx) +{ + int i; + int cnt = 0; + + for (i = 0; i < ctx->nnodes; i++) + { + if (strcmp(ctx->nodes[i].name, ctx->src_node_name) != 0 && + strcmp(ctx->nodes[i].name, ctx->new_node_name) != 0) + cnt++; + } + return cnt; +} + +/* + * Check the Spock extension version on a node reachable via conn, returning + * major/minor/patch. ERROR if the extension is missing. + */ +static void +zodan_remote_spock_version(PGconn *conn, const char *label, + int *major, int *minor, int *patch) +{ + char *v = zodan_remote_scalar(conn, + "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + + if (v == NULL) + ereport(ERROR, + (errmsg("Spock extension not found on %s", label))); + zodan_parse_version(v, label, major, minor, patch); + pfree(v); +} + +/* ------------------------------------------------------------------------ + * attach_node phases + * ------------------------------------------------------------------------ */ + +/* + * Phase 0: verify the Spock version on the source node, the new node and every + * existing cluster node. All nodes must be >= ZODAN_MIN_VERSION and share the + * same major.minor (patch differences are allowed for rolling upgrades). + */ +static void +zodan_check_versions(ZodanAddCtx *ctx) +{ + int min_major, + min_minor, + min_patch; + int src_major, + src_minor, + src_patch; + int new_major, + new_minor, + new_patch; + PGconn *conn; + int i; + + zodan_parse_version(ZODAN_MIN_VERSION, "ZODAN_MIN_VERSION", + &min_major, &min_minor, &min_patch); + + ZNOTICE("Checking Spock version on source node"); + conn = zodan_connect(ctx->src_dsn, "ver"); + zodan_remote_spock_version(conn, "source node", + &src_major, &src_minor, &src_patch); + zodan_disconnect(conn); + + if (zodan_version_cmp(src_major, src_minor, src_patch, + min_major, min_minor, min_patch) < 0) + ereport(ERROR, + (errmsg("Spock version mismatch: source node has version %d.%d.%d, " + "but minimum required version is %s", + src_major, src_minor, src_patch, ZODAN_MIN_VERSION))); + + ZNOTICE("Checking Spock version on new node"); + conn = zodan_connect(ctx->new_node_dsn, "ver"); + zodan_remote_spock_version(conn, "new node", + &new_major, &new_minor, &new_patch); + zodan_disconnect(conn); + + if (zodan_version_cmp(new_major, new_minor, new_patch, + min_major, min_minor, min_patch) < 0) + ereport(ERROR, + (errmsg("Spock version mismatch: new node has version %d.%d.%d, " + "but minimum required version is %s", + new_major, new_minor, new_patch, ZODAN_MIN_VERSION))); + + if (new_major != src_major || new_minor != src_minor) + ereport(ERROR, + (errmsg("Spock version mismatch: new node has version %d.%d.%d, " + "but source version is %d.%d.%d; major.minor versions must match", + new_major, new_minor, new_patch, + src_major, src_minor, src_patch))); + + /* Every existing cluster node must match too. */ + for (i = 0; i < ctx->nnodes; i++) + { + int node_major, + node_minor, + node_patch; + + conn = zodan_connect(ctx->nodes[i].dsn, "ver"); + zodan_remote_spock_version(conn, ctx->nodes[i].name, + &node_major, &node_minor, &node_patch); + zodan_disconnect(conn); + + if (zodan_version_cmp(node_major, node_minor, node_patch, + min_major, min_minor, min_patch) < 0) + ereport(ERROR, + (errmsg("Spock version mismatch: node %s has version %d.%d.%d, " + "but required version is at least %s", + ctx->nodes[i].name, + node_major, node_minor, node_patch, + ZODAN_MIN_VERSION))); + if (node_major != new_major || node_minor != new_minor) + ereport(ERROR, + (errmsg("Spock version mismatch: new node has version %d.%d.%d, " + "but found node %s version %d.%d.%d; major.minor must match", + new_major, new_minor, new_patch, ctx->nodes[i].name, + node_major, node_minor, node_patch))); + } + + ZNOTICE("Version check passed: source %d.%d.%d, new node %d.%d.%d", + src_major, src_minor, src_patch, new_major, new_minor, new_patch); +} + +/* + * Phase 1: verify prerequisites. Most importantly this enforces that attach_node + * is being run on the new node (by comparing the local system identifier and + * database name against what new_node_dsn points at). + */ +static void +zodan_verify_prerequisites(ZodanAddCtx *ctx) +{ + char *local_sysid; + char *local_dbname; + char *remote_sysid; + char *remote_dbname; + char *new_dbname; + PGconn *conn; + int64 cnt; + + ZNOTICE("Phase 1: Validating source and new node prerequisites"); + + /* attach_node must be run on the new node. */ + local_sysid = zodan_local_scalar("SELECT system_identifier::text FROM pg_control_system()"); + local_dbname = zodan_local_scalar("SELECT current_database()"); + + conn = zodan_connect(ctx->new_node_dsn, "prereq"); + remote_sysid = zodan_remote_scalar(conn, + "SELECT system_identifier::text FROM pg_control_system()"); + remote_dbname = zodan_remote_scalar(conn, "SELECT current_database()"); + zodan_disconnect(conn); + + if (local_sysid == NULL || remote_sysid == NULL || + strcmp(local_sysid, remote_sysid) != 0 || + local_dbname == NULL || remote_dbname == NULL || + strcmp(local_dbname, remote_dbname) != 0) + ereport(ERROR, + (errmsg("attach_node must be run on the new node being added"), + errdetail("new_node_dsn (%s) does not match the current database connection", + ctx->new_node_dsn), + errhint("Connect to the new node and re-run attach_node."))); + + ZNOTICE(" OK: attach_node is running on the new node"); + + /* Sanity-check the database named by the new node DSN. */ + new_dbname = zodan_dbname_from_dsn(ctx->new_node_dsn); + + /* + * lolor state on the new node. Having the lolor extension installed is + * fine (and required when the source replicates lolor tables), but any + * pre-existing large object data would conflict with the data synchronized + * from the source, so the lolor tables must be empty. The lolor structure + * itself is excluded from the structure dump (see skip_schema[]) and comes + * from CREATE EXTENSION lolor, so the extension must already exist on the + * new node when the source replicates lolor tables. + */ + { + bool new_lolor_installed; + int64 src_lolor_repset_tables; + + new_lolor_installed = zodan_local_count( + "SELECT count(*) FROM pg_catalog.pg_extension WHERE extname = 'lolor'") > 0; + + if (new_lolor_installed) + { + int64 lolor_rows = zodan_local_count( + "SELECT (SELECT count(*) FROM lolor.pg_largeobject) + " + "(SELECT count(*) FROM lolor.pg_largeobject_metadata)"); + + if (lolor_rows > 0) + ereport(ERROR, + (errmsg("database %s on the new node has pre-existing large object data in the lolor tables", + new_dbname), + errhint("The lolor tables must be empty so the large object data from the source node can be synchronized."))); + ZNOTICE(" OK: database %s lolor tables are empty", new_dbname); + } + + /* + * If the source node replicates lolor tables, the new node must have + * lolor installed or the initial data synchronization will fail. + */ + { + PGconn *src_conn = zodan_connect(ctx->src_dsn, "prereq"); + char *s; + + s = zodan_remote_scalar(src_conn, + "SELECT count(*) FROM spock.tables " + "WHERE nspname = 'lolor' AND set_name IS NOT NULL"); + zodan_disconnect(src_conn); + + src_lolor_repset_tables = (s != NULL) ? pg_strtoint64(s) : 0; + if (s != NULL) + pfree(s); + } + + if (src_lolor_repset_tables > 0 && !new_lolor_installed) + ereport(ERROR, + (errmsg("source node replicates lolor tables but database %s on the new node does not have the lolor extension installed", + new_dbname), + errhint("Run CREATE EXTENSION lolor on the new node first."))); + ZNOTICE(" OK: lolor requirements satisfied for database %s", new_dbname); + } + + /* + * No user tables on the new node. 'lolor' is excluded: its tables are + * created by CREATE EXTENSION lolor, which is permitted (and required) when + * the source replicates lolor data, and were already validated as empty + * above. + */ + cnt = zodan_local_count( + "SELECT count(*) FROM pg_tables " + "WHERE schemaname NOT IN ('information_schema','pg_catalog','pg_toast','spock','lolor') " + "AND schemaname NOT LIKE 'pg_temp_%' " + "AND schemaname NOT LIKE 'pg_toast_temp_%'"); + if (cnt > 0) + ereport(ERROR, + (errmsg("database %s on the new node has " INT64_FORMAT " user-created tables", + new_dbname, cnt), + errhint("The new node must be a freshly created database with no user tables."))); + ZNOTICE(" OK: database %s has no user tables", new_dbname); + + /* Every source login role must exist on the new node. */ + { + PGconn *src_conn = zodan_connect(ctx->src_dsn, "prereq"); + PGresult *res; + int i; + StringInfoData missing; + + initStringInfo(&missing); + res = zodan_remote_query(src_conn, + "SELECT rolname FROM pg_roles " + "WHERE rolcanlogin = true " + "AND rolname NOT IN ('postgres','rdsadmin','rdsrepladmin','rds_superuser') " + "ORDER BY rolname"); + for (i = 0; i < PQntuples(res); i++) + { + char *rolname = PQgetvalue(res, i, 0); + char *sql = psprintf( + "SELECT count(*) FROM pg_roles WHERE rolname = %s AND rolcanlogin = true", + quote_literal_cstr(rolname)); + int64 exists = zodan_local_count(sql); + + pfree(sql); + if (exists == 0) + { + if (missing.len > 0) + appendStringInfoString(&missing, ", "); + appendStringInfoString(&missing, rolname); + } + } + zodan_clear(res); + zodan_disconnect(src_conn); + + if (missing.len > 0) + ereport(ERROR, + (errmsg("new node is missing roles that exist on the source node: %s", + missing.data), + errhint("Create these roles on the new node before adding it to the cluster."))); + pfree(missing.data); + ZNOTICE(" OK: new node has all source-node login roles"); + } + + /* Every existing cluster node must have only enabled subscriptions. */ + { + int i; + + for (i = 0; i < ctx->nnodes; i++) + { + PGconn *nconn; + PGresult *res; + int j; + + if (strcmp(ctx->nodes[i].name, ctx->new_node_name) == 0) + continue; + + nconn = zodan_connect(ctx->nodes[i].dsn, "prereq"); + res = zodan_remote_query(nconn, + "SELECT sub_name, sub_enabled FROM spock.subscription"); + for (j = 0; j < PQntuples(res); j++) + { + char *en = PQgetvalue(res, j, 1); + + if (en[0] != 't') + { + /* + * Copy the name out of the PGresult before releasing it: + * PQgetvalue() points into the result's own storage. + */ + char *sub = pstrdup(PQgetvalue(res, j, 0)); + + zodan_clear(res); + zodan_disconnect(nconn); + ereport(ERROR, + (errmsg("node %s has disabled subscription %s", + ctx->nodes[i].name, sub), + errhint("All subscriptions must be enabled before adding a node."))); + } + } + zodan_clear(res); + zodan_disconnect(nconn); + } + ZNOTICE(" OK: every cluster node has only enabled subscriptions"); + } + + /* The new node must not already exist locally with subs/repsets. */ + cnt = zodan_local_count(psprintf( + "SELECT count(*) FROM spock.node WHERE node_name = %s", + quote_literal_cstr(ctx->new_node_name))); + if (cnt > 0) + ereport(ERROR, (errmsg("new node %s already exists", ctx->new_node_name))); + + cnt = zodan_local_count( + "SELECT count(*) FROM spock.subscription"); + if (cnt > 0) + ereport(ERROR, + (errmsg("new node already has subscriptions; it must be a clean node"))); + + ZNOTICE(" OK: new node %s is clean", ctx->new_node_name); + + pfree(new_dbname); +} + +/* + * Phase 2: create the new node in the local (new node's) database. The source + * node already exists in its own database, and the local representation of the + * source node is created automatically by spock.sub_create() when the + * source->new subscription is created, so we only create the new node here. + */ +static void +zodan_create_nodes(ZodanAddCtx *ctx) +{ + StringInfoData sql; + + ZNOTICE("Phase 2: Creating nodes"); + + /* Create the new (local) node. */ + initStringInfo(&sql); + appendStringInfo(&sql, + "SELECT spock.node_create(node_name := %s, dsn := %s, " + "location := %s, country := %s, info := %s::jsonb)", + quote_literal_cstr(ctx->new_node_name), + quote_literal_cstr(ctx->new_node_dsn), + quote_literal_cstr(ctx->new_node_location), + quote_literal_cstr(ctx->new_node_country), + quote_literal_cstr(ctx->new_node_info)); + zodan_local_command(sql.data); + pfree(sql.data); + ZNOTICE(" OK: created new node %s", ctx->new_node_name); + + /* Commit so the node rows are visible cluster-wide before we continue. */ + SPI_commit(); +} + +/* + * Create a logical replication slot on a remote node if it does not already + * exist. On PG17+ the slot is created with failover = true, matching Spock's + * own slot creation path. Returns the slot's LSN as a palloc'd string, or NULL + * if the slot already existed. + */ +static char * +zodan_remote_create_slot(PGconn *conn, const char *slot_name) +{ + char *exists; + char *server_ver; + int vernum; + StringInfoData sql; + char *lsn; + + exists = zodan_remote_scalar(conn, psprintf( + "SELECT count(*) FROM pg_replication_slots WHERE slot_name = %s", + quote_literal_cstr(slot_name))); + if (exists != NULL) + { + bool already = strcmp(exists, "0") != 0; + + pfree(exists); + if (already) + return NULL; + } + + server_ver = zodan_remote_scalar(conn, "SHOW server_version_num"); + vernum = server_ver ? atoi(server_ver) : 0; + if (server_ver != NULL) + pfree(server_ver); + + initStringInfo(&sql); + if (vernum >= 170000) + appendStringInfo(&sql, + "SELECT slot_name, lsn FROM pg_create_logical_replication_slot(%s, 'spock_output', false, false, true)", + quote_literal_cstr(slot_name)); + else + appendStringInfo(&sql, + "SELECT slot_name, lsn FROM pg_create_logical_replication_slot(%s, 'spock_output')", + quote_literal_cstr(slot_name)); + + lsn = NULL; + { + PGresult *res = zodan_remote_query(conn, sql.data); + + if (PQntuples(res) > 0 && !PQgetisnull(res, 0, 1)) + lsn = pstrdup(PQgetvalue(res, 0, 1)); + zodan_clear(res); + } + pfree(sql.data); + return lsn; +} + +/* + * Wait until the source node has applied changes from origin_node up to + * target_lsn (observed through spock.progress.remote_commit_lsn on the source). + * Bounded by ctx->timeout_sec. + */ +static void +zodan_wait_source_caughtup(ZodanAddCtx *ctx, const char *origin_node, + const char *target_lsn) +{ + PGconn *conn = zodan_connect(ctx->src_dsn, "catchup"); + char *progress_sql; + XLogRecPtr target; + TimestampTz start = GetCurrentTimestamp(); + + progress_sql = psprintf( + "SELECT p.remote_commit_lsn " + "FROM spock.progress p " + "JOIN spock.node n ON n.node_id = p.remote_node_id " + "WHERE p.node_id = (SELECT node_id FROM spock.node_info()) " + "AND n.node_name = %s", + quote_literal_cstr(origin_node)); + + ZNOTICE(" - Waiting for source node %s to apply %s changes up to %s", + ctx->src_node_name, origin_node, target_lsn); + + target = zodan_parse_lsn(target_lsn); + + for (;;) + { + char *cur = zodan_remote_scalar(conn, progress_sql); + + if (cur != NULL) + { + bool reached = zodan_parse_lsn(cur) >= target; + + pfree(cur); + if (reached) + break; + } + + if (TimestampDifferenceExceeds(start, GetCurrentTimestamp(), + ctx->timeout_sec * 1000)) + { + zodan_disconnect(conn); + ereport(ERROR, + (errmsg("timed out waiting for source node %s to apply %s changes through %s", + ctx->src_node_name, origin_node, target_lsn))); + } + zodan_sleep_ms(500); + } + + zodan_disconnect(conn); + pfree(progress_sql); +} + +/* + * Trigger a sync event on a remote node (conn) and return its LSN as a palloc'd + * string. transactional selects spock.sync_event(true) vs spock.sync_event(). + */ +static char * +zodan_remote_sync_event(PGconn *conn, bool transactional) +{ + return zodan_remote_scalar(conn, + transactional + ? "SELECT spock.sync_event(true)" + : "SELECT spock.sync_event()"); +} + +/* + * Wait for a sync event (origin_node / lsn) to be applied on the node reached + * by conn, by calling spock.wait_for_sync_event() there. ERROR on timeout. + */ +static void +zodan_remote_wait_for_sync_event(ZodanAddCtx *ctx, PGconn *conn, + const char *origin_node, const char *lsn) +{ + StringInfoData sql; + char *ok; + + initStringInfo(&sql); + appendStringInfo(&sql, + "CALL spock.wait_for_sync_event(true, %s, %s::pg_lsn, %d, true)", + quote_literal_cstr(origin_node), + quote_literal_cstr(lsn), + ctx->timeout_sec * 1000); + ok = zodan_remote_scalar(conn, sql.data); + pfree(sql.data); + + if (ok == NULL || ok[0] != 't') + ereport(ERROR, + (errmsg("wait_for_sync_event timed out for %s (lsn %s)", + origin_node, lsn))); + pfree(ok); +} + +/* + * Wait for a sync event locally (we are the new node) by polling + * spock.progress until we have applied origin_node's changes up to lsn. + */ +static void +zodan_local_wait_for_sync_event(ZodanAddCtx *ctx, const char *origin_node, + const char *lsn) +{ + char *progress_sql; + XLogRecPtr target = zodan_parse_lsn(lsn); + TimestampTz start = GetCurrentTimestamp(); + + progress_sql = psprintf( + "SELECT p.remote_commit_lsn " + "FROM spock.progress p " + "JOIN spock.node n ON n.node_id = p.remote_node_id " + "WHERE p.node_id = (SELECT node_id FROM spock.node_info()) " + "AND n.node_name = %s", + quote_literal_cstr(origin_node)); + + for (;;) + { + char *cur = zodan_local_scalar(progress_sql); + + if (cur != NULL) + { + bool reached = zodan_parse_lsn(cur) >= target; + + pfree(cur); + if (reached) + break; + } + + if (TimestampDifferenceExceeds(start, GetCurrentTimestamp(), + ctx->timeout_sec * 1000)) + ereport(ERROR, + (errmsg("timed out waiting for sync event from %s (lsn %s) on new node", + origin_node, lsn))); + zodan_sleep_ms(500); + } + pfree(progress_sql); +} + +/* + * Create a subscription. If on_dsn is NULL the subscription is created locally + * (on the new node) over SPI; otherwise it is created on the remote node over + * libpq. This mirrors the create_sub() helper in the old zodan.sql. + * + * The replication set list is read from provider_dsn rather than taken from a + * single cluster-wide list: see zodan_node_repset_list() for why it has to be + * the provider's. + */ +static void +zodan_create_sub(ZodanAddCtx *ctx, const char *on_dsn, const char *sub_name, + const char *provider_dsn, bool sync_structure, + bool sync_data, bool enabled) +{ + StringInfoData sql; + char *repsets = zodan_node_repset_list(provider_dsn); + + initStringInfo(&sql); + appendStringInfo(&sql, + "SELECT spock.sub_create(" + "subscription_name := %s, " + "provider_dsn := %s, " + "replication_sets := %s, " + "synchronize_structure := %s, " + "synchronize_data := %s, " + "forward_origins := '{}', " + "apply_delay := '0'::interval, " + "force_text_transfer := false, " + "enabled := %s)", + quote_literal_cstr(sub_name), + quote_literal_cstr(provider_dsn), + repsets, + sync_structure ? "true" : "false", + sync_data ? "true" : "false", + enabled ? "true" : "false"); + + if (on_dsn == NULL) + { + zodan_local_command(sql.data); + } + else + { + PGconn *conn = zodan_connect(on_dsn, "subcreate"); + + zodan_remote_command(conn, sql.data); + zodan_disconnect(conn); + } + pfree(sql.data); + pfree(repsets); +} + +/* + * Phase 3: create disabled subscriptions and replication slots for every + * "other" node (all cluster nodes except source and new). For a 2-node + * cluster this only records a sync event on the source node. + * + * The stored sync LSNs (for later enabling) are kept in a temp table on the + * new node, exactly as the old zodan.sql did, so the later phases can find + * them regardless of intervening commits. + */ +static void +zodan_create_disabled_subs_and_slots(ZodanAddCtx *ctx) +{ + int i; + + ZNOTICE("Phase 3: Creating disabled subscriptions and slots"); + + /* + * The temp table lives for the whole session, so a retry after a failed + * attach_node in the same session finds the previous attempt's rows. Start + * from an empty table rather than relying on every later read being for an + * origin this attempt has just refreshed. + */ + zodan_local_command( + "CREATE TEMP TABLE IF NOT EXISTS temp_sync_lsns (" + "origin_node text PRIMARY KEY, sync_lsn text NOT NULL, slot_lsn pg_lsn)"); + zodan_local_command("TRUNCATE temp_sync_lsns"); + + if (zodan_num_other_nodes(ctx) == 0) + { + /* 2-node scenario: capture a source sync event for later. */ + PGconn *conn = zodan_connect(ctx->src_dsn, "sync"); + char *lsn = zodan_remote_sync_event(conn, false); + + zodan_disconnect(conn); + if (lsn == NULL) + ereport(ERROR, + (errmsg("could not trigger sync event on source node %s", + ctx->src_node_name))); + + zodan_local_command(psprintf( + "INSERT INTO temp_sync_lsns (origin_node, sync_lsn) VALUES (%s, %s) " + "ON CONFLICT (origin_node) DO UPDATE SET sync_lsn = EXCLUDED.sync_lsn", + quote_literal_cstr(ctx->src_node_name), + quote_literal_cstr(lsn))); + ZNOTICE(" - 2-node scenario: stored source sync event %s", lsn); + pfree(lsn); + + /* Same durability point as the multi-node path below. */ + SPI_commit(); + return; + } + + for (i = 0; i < ctx->nnodes; i++) + { + ZNode *rec = &ctx->nodes[i]; + char *slot_name; + char *sub_name; + char *slot_lsn; + char *catchup_lsn; + PGconn *conn; + + if (strcmp(rec->name, ctx->src_node_name) == 0 || + strcmp(rec->name, ctx->new_node_name) == 0) + continue; + + sub_name = zodan_gen_sub_name(rec->name, ctx->new_node_name); + slot_name = zodan_gen_slot_name(rec->dsn, rec->name, sub_name); + + /* Create the slot on the other node. */ + conn = zodan_connect(rec->dsn, "slot"); + slot_lsn = zodan_remote_create_slot(conn, slot_name); + if (slot_lsn != NULL) + ZNOTICE(" OK: created replication slot %s on node %s", + slot_name, rec->name); + else + ZNOTICE(" - replication slot %s already exists on node %s", + slot_name, rec->name); + + /* Anchor a real commit past the slot so the catch-up target exists. */ + catchup_lsn = zodan_remote_sync_event(conn, true); + zodan_disconnect(conn); + if (catchup_lsn == NULL) + ereport(ERROR, + (errmsg("could not trigger sync event on node %s", rec->name))); + + zodan_local_command(psprintf( + "INSERT INTO temp_sync_lsns (origin_node, sync_lsn, slot_lsn) " + "VALUES (%s, %s, %s) " + "ON CONFLICT (origin_node) DO UPDATE " + "SET sync_lsn = EXCLUDED.sync_lsn, slot_lsn = EXCLUDED.slot_lsn", + quote_literal_cstr(rec->name), + quote_literal_cstr(catchup_lsn), + slot_lsn ? quote_literal_cstr(slot_lsn) : "NULL")); + + /* Wait until the source has applied rec's changes up to catchup_lsn. */ + zodan_wait_source_caughtup(ctx, rec->name, catchup_lsn); + + /* Drop any stale origin so the sub starts clean, then create disabled. */ + { + PGconn *nconn = zodan_connect(ctx->new_node_dsn, "origin"); + + zodan_remote_command(nconn, psprintf( + "DO $x$ BEGIN " + "IF EXISTS (SELECT 1 FROM pg_replication_origin WHERE roname = %s) THEN " + "PERFORM pg_replication_origin_drop(%s); END IF; END $x$", + quote_literal_cstr(slot_name), quote_literal_cstr(slot_name))); + zodan_disconnect(nconn); + } + + zodan_create_sub(ctx, NULL, sub_name, rec->dsn, false, false, false); + ZNOTICE(" OK: created disabled subscription %s (provider %s)", + sub_name, rec->name); + + if (catchup_lsn) + pfree(catchup_lsn); + if (slot_lsn) + pfree(slot_lsn); + } + + /* Make the disabled subs and stored LSNs durable before continuing. */ + SPI_commit(); +} + +/* + * Phase 4: for each other node, trigger a sync event on that node and wait for + * the source node to apply it. + */ +static void +zodan_sync_other_nodes_wait_source(ZodanAddCtx *ctx) +{ + int i; + + ZNOTICE("Phase 4: Triggering sync events on other nodes and waiting on source"); + + if (zodan_num_other_nodes(ctx) == 0) + { + ZNOTICE(" - No other nodes, skipping"); + return; + } + + for (i = 0; i < ctx->nnodes; i++) + { + ZNode *rec = &ctx->nodes[i]; + PGconn *conn; + PGconn *src_conn; + char *lsn; + + if (strcmp(rec->name, ctx->src_node_name) == 0 || + strcmp(rec->name, ctx->new_node_name) == 0) + continue; + + conn = zodan_connect(rec->dsn, "sync"); + lsn = zodan_remote_sync_event(conn, false); + zodan_disconnect(conn); + if (lsn == NULL) + ereport(ERROR, (errmsg("could not trigger sync event on node %s", rec->name))); + + src_conn = zodan_connect(ctx->src_dsn, "sync"); + zodan_remote_wait_for_sync_event(ctx, src_conn, rec->name, lsn); + zodan_disconnect(src_conn); + ZNOTICE(" OK: sync event from %s confirmed on source", rec->name); + pfree(lsn); + } +} + +/* + * Phase 5: create the enabled source->new subscription (with structure and + * data synchronization). + */ +static void +zodan_create_source_to_new_sub(ZodanAddCtx *ctx) +{ + char *sub_name = zodan_gen_sub_name(ctx->src_node_name, ctx->new_node_name); + + ZNOTICE("Phase 5: Creating source to new node subscription"); + zodan_create_sub(ctx, NULL, sub_name, ctx->src_dsn, true, true, true); + ZNOTICE(" OK: created subscription %s", sub_name); + SPI_commit(); +} + +/* + * Phase 6: trigger a sync event on the source node and wait for the new node + * (this node) to apply it. + */ +static void +zodan_source_sync_wait_new(ZodanAddCtx *ctx) +{ + PGconn *conn; + char *lsn; + + ZNOTICE("Phase 6: Triggering sync on source node and waiting on new node"); + + conn = zodan_connect(ctx->src_dsn, "sync"); + lsn = zodan_remote_sync_event(conn, false); + zodan_disconnect(conn); + if (lsn == NULL) + ereport(ERROR, + (errmsg("could not trigger sync event on source node %s", + ctx->src_node_name))); + + zodan_local_wait_for_sync_event(ctx, ctx->src_node_name, lsn); + ZNOTICE(" OK: sync event from source confirmed on new node"); + pfree(lsn); +} + +/* + * Wait until the source->new subscription has finished its initial COPY and is + * READY (no pending table sync). Bounded by ctx->timeout_sec. + */ +static void +zodan_wait_sub_ready(ZodanAddCtx *ctx, const char *sub_name) +{ + char *subid; + TimestampTz start = GetCurrentTimestamp(); + + subid = zodan_local_scalar(psprintf( + "SELECT sub_id::text FROM spock.subscription WHERE sub_name = %s", + quote_literal_cstr(sub_name))); + + /* + * The subscription was created and committed in phase 5, so it must be here. + * Continuing without it would silently skip the readiness wait and let + * attach_node proceed as if the initial COPY had finished. + */ + if (subid == NULL) + ereport(ERROR, + (errmsg("subscription %s not found on the new node", sub_name), + errdetail("It was created earlier in this attach_node run; something dropped it."))); + + for (;;) + { + int64 failed; + int64 pending; + + /* + * SYNC_STATUS_FAILED is not a transient state -- it will never become + * 'y'/'r' on its own -- so check it separately. Counting it as merely + * "pending" would spin out the whole timeout and then report a timeout + * for what is really a failure with a diagnosable cause. + */ + failed = zodan_local_count(psprintf( + "SELECT count(*) FROM spock.local_sync_status " + "WHERE sync_subid = %s AND sync_status = 'f'", subid)); + if (failed > 0) + ereport(ERROR, + (errmsg("initial synchronization failed for subscription %s", + sub_name), + errdetail(INT64_FORMAT " table(s) are in the failed sync state.", + failed), + errhint("Check the subscriber and provider logs for the sync worker error, then retry with spock.detach_node() followed by spock.attach_node()."))); + + pending = zodan_local_count(psprintf( + "SELECT count(*) FROM spock.local_sync_status " + "WHERE sync_subid = %s AND sync_status NOT IN ('y','r','f')", subid)); + + if (pending == 0) + { + ZNOTICE(" - subscription %s is READY", sub_name); + break; + } + if (TimestampDifferenceExceeds(start, GetCurrentTimestamp(), + ctx->timeout_sec * 1000)) + ereport(ERROR, + (errmsg("timed out after %d seconds waiting for subscription %s " + "to finish initial synchronization", + ctx->timeout_sec, sub_name))); + zodan_sleep_ms(1000); + } + pfree(subid); +} + +/* + * Phase 7: wait for the source->new subscription to become READY. Slot/origin + * advancement is handled by the apply worker for the active subscription; we + * only advance the inactive "other node" slots defensively. + */ +static void +zodan_wait_ready_and_advance(ZodanAddCtx *ctx) +{ + char *sub_name; + int i; + + ZNOTICE("Phase 7: Waiting for initial sync and advancing slots"); + + sub_name = zodan_gen_sub_name(ctx->src_node_name, ctx->new_node_name); + zodan_wait_sub_ready(ctx, sub_name); + + if (zodan_num_other_nodes(ctx) == 0) + return; + + /* Advance each other-node slot/origin to the resume point. */ + for (i = 0; i < ctx->nnodes; i++) + { + ZNode *rec = &ctx->nodes[i]; + char *slot_name; + char *cur_lsn; + char *target_lsn; + PGconn *conn; + + if (strcmp(rec->name, ctx->src_node_name) == 0 || + strcmp(rec->name, ctx->new_node_name) == 0) + continue; + + slot_name = zodan_gen_slot_name(rec->dsn, rec->name, + zodan_gen_sub_name(rec->name, ctx->new_node_name)); + + conn = zodan_connect(rec->dsn, "advance"); + cur_lsn = zodan_remote_scalar(conn, psprintf( + "SELECT restart_lsn FROM pg_replication_slots WHERE slot_name = %s", + quote_literal_cstr(slot_name))); + if (cur_lsn == NULL) + { + zodan_disconnect(conn); + ZNOTICE(" - slot %s does not exist, skipping advance", slot_name); + continue; + } + + /* Advance to the last commit from rec that the source had applied. */ + target_lsn = zodan_local_scalar(psprintf( + "SELECT p.remote_commit_lsn::text " + "FROM spock.progress p JOIN spock.node n ON n.node_id = p.remote_node_id " + "WHERE p.node_id = (SELECT node_id FROM spock.node_info()) " + "AND n.node_name = %s", quote_literal_cstr(rec->name))); + + if (target_lsn != NULL) + { + if (zodan_parse_lsn(target_lsn) > zodan_parse_lsn(cur_lsn)) + { + zodan_remote_command(conn, psprintf( + "SELECT pg_replication_slot_advance(%s, %s::pg_lsn)", + quote_literal_cstr(slot_name), quote_literal_cstr(target_lsn))); + + /* Advance the origin locally (it lives on the new node). */ + zodan_local_command(psprintf( + "DO $x$ BEGIN " + "IF NOT EXISTS (SELECT 1 FROM pg_replication_origin WHERE roname = %s) THEN " + "PERFORM pg_replication_origin_create(%s); END IF; " + "PERFORM pg_replication_origin_advance(%s, %s::pg_lsn); END $x$", + quote_literal_cstr(slot_name), quote_literal_cstr(slot_name), + quote_literal_cstr(slot_name), quote_literal_cstr(target_lsn))); + ZNOTICE(" OK: advanced slot/origin %s to %s", + slot_name, target_lsn); + } + pfree(target_lsn); + } + zodan_disconnect(conn); + pfree(cur_lsn); + } + SPI_commit(); +} + +/* + * Phase 8: make the new node's replication sets an exact copy of the source + * node's: the set definitions, the table memberships (including column lists + * and row filters) and the sequence memberships. + * + * This has to run after the structure and data sync, because the tables must + * exist locally before they can be added to a set, and before the new node + * starts acting as a provider in phase 10. + * + * Nothing in Spock copies replication set membership. What actually populates + * the sets on a freshly joined node is AutoDDL firing while pg_restore replays + * the structure dump, which routes each table by policy (primary key or replica + * identity to 'default', otherwise 'default_insert_only') rather than by what + * the source says. That derived membership is wiped here and replaced by the + * source's, so a table the user deliberately removed from a set on the source + * does not reappear on the new node, and extension-owned tables such as + * lolor's (which AutoDDL skips entirely, because it ignores anything created by + * CREATE EXTENSION) no longer have to be added by hand. + */ +static void +zodan_sync_repsets_from_source(ZodanAddCtx *ctx) +{ + PGconn *conn; + PGresult *res; + MemoryContext mcxt; + ZPair *local; + int nlocal; + int i; + int cleared = 0; + int added = 0; + int added_seqs = 0; + int dropped = 0; + char *src_sets; + char *src_digest; + char *local_digest; + StringInfoData missing; + + ZNOTICE("Phase 8: Mirroring replication sets from the source node"); + + mcxt = AllocSetContextCreate(CurrentMemoryContext, "zodan_repsets", + ALLOCSET_DEFAULT_SIZES); + initStringInfo(&missing); + + /* + * 1. Drop the membership AutoDDL derived during the structure restore. The + * source's memberships are replayed below; keeping both would leave two + * sources of truth. include_partitions is false throughout so the source's + * rows are mirrored one for one: if the source has a partition in a set, it + * has its own row for it. + */ + local = zodan_local_pairs( + "SELECT rs.set_name, rst.set_reloid::text " + "FROM spock.replication_set_table rst " + "JOIN spock.replication_set rs ON rs.set_id = rst.set_id " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id", + mcxt, &nlocal); + for (i = 0; i < nlocal; i++) + { + zodan_local_command(psprintf( + "SELECT spock.repset_remove_table(%s, %s::regclass, false)", + quote_literal_cstr(local[i].a), quote_literal_cstr(local[i].b))); + cleared++; + } + + local = zodan_local_pairs( + "SELECT rs.set_name, rss.set_seqoid::text " + "FROM spock.replication_set_seq rss " + "JOIN spock.replication_set rs ON rs.set_id = rss.set_id " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id", + mcxt, &nlocal); + for (i = 0; i < nlocal; i++) + { + zodan_local_command(psprintf( + "SELECT spock.repset_remove_seq(%s, %s::regclass)", + quote_literal_cstr(local[i].a), quote_literal_cstr(local[i].b))); + cleared++; + } + if (cleared > 0) + ZNOTICE(" OK: cleared %d auto-derived replication set entries", cleared); + + /* + * 2. Replication set definitions. The three built-ins already exist here + * (spock.node_create() creates them), so those are altered into shape and + * the rest created. + */ + conn = zodan_connect(ctx->src_dsn, "repsetdefs"); + res = zodan_remote_query(conn, + "SELECT rs.set_name, rs.replicate_insert, " + "rs.replicate_update, rs.replicate_delete, " + "rs.replicate_truncate " + "FROM spock.replication_set rs " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id"); + for (i = 0; i < PQntuples(res); i++) + { + const char *name = PQgetvalue(res, i, 0); + + /* libpq renders booleans as "t"/"f", which is not a SQL literal. */ + const char *ins = ZBOOL(res, i, 1); + const char *upd = ZBOOL(res, i, 2); + const char *del = ZBOOL(res, i, 3); + const char *trunc = ZBOOL(res, i, 4); + + zodan_local_command(psprintf( + "SELECT CASE WHEN EXISTS (SELECT 1 FROM spock.replication_set rs " + " JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id " + " WHERE rs.set_name = %s) " + " THEN spock.repset_alter(%s, %s, %s, %s, %s) " + " ELSE spock.repset_create(%s, %s, %s, %s, %s) END", + quote_literal_cstr(name), + quote_literal_cstr(name), ins, upd, del, trunc, + quote_literal_cstr(name), ins, upd, del, trunc)); + } + + /* + * Any set the source does not have should not exist here either. The + * comparison list comes from the result already in hand: re-reading the + * source would let a set created there in between be created above and + * then immediately dropped again. + */ + src_sets = zodan_repset_array(res); + zodan_clear(res); + zodan_disconnect(conn); + + local = zodan_local_pairs(psprintf( + "SELECT rs.set_name, NULL::text " + "FROM spock.replication_set rs " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id " + "WHERE rs.set_name <> ALL (%s::name[])", src_sets), + mcxt, &nlocal); + for (i = 0; i < nlocal; i++) + { + zodan_local_command(psprintf("SELECT spock.repset_drop(%s, true)", + quote_literal_cstr(local[i].a))); + dropped++; + ZNOTICE(" OK: dropped replication set %s (absent on source)", local[i].a); + } + + /* 3. Table membership, with column lists and row filters. */ + conn = zodan_connect(ctx->src_dsn, "repsettables"); + res = zodan_remote_query(conn, + "SELECT rs.set_name, " + "quote_ident(n.nspname) || '.' || quote_ident(c.relname), " + "rst.set_att_list::text, " + "pg_get_expr(rst.set_row_filter, rst.set_reloid) " + "FROM spock.replication_set_table rst " + "JOIN spock.replication_set rs ON rs.set_id = rst.set_id " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id " + "JOIN pg_catalog.pg_class c ON c.oid = rst.set_reloid " + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace"); + for (i = 0; i < PQntuples(res); i++) + { + const char *name = PQgetvalue(res, i, 0); + const char *rel = PQgetvalue(res, i, 1); + char *atts = PQgetisnull(res, i, 2) ? pstrdup("NULL") : + psprintf("%s::text[]", quote_literal_cstr(PQgetvalue(res, i, 2))); + char *filter = PQgetisnull(res, i, 3) ? pstrdup("NULL") : + quote_literal_cstr(PQgetvalue(res, i, 3)); + char *present; + + present = zodan_local_scalar(psprintf( + "SELECT to_regclass(%s) IS NOT NULL", quote_literal_cstr(rel))); + if (present == NULL || present[0] != 't') + { + appendStringInfo(&missing, "%s%s", missing.len > 0 ? ", " : "", rel); + continue; + } + pfree(present); + + zodan_local_command(psprintf( + "SELECT spock.repset_add_table(%s, %s::regclass, false, %s, %s, false)", + quote_literal_cstr(name), quote_literal_cstr(rel), atts, filter)); + added++; + } + zodan_clear(res); + zodan_disconnect(conn); + + /* 4. Sequence membership. */ + conn = zodan_connect(ctx->src_dsn, "repsetseqs"); + res = zodan_remote_query(conn, + "SELECT rs.set_name, " + "quote_ident(n.nspname) || '.' || quote_ident(c.relname) " + "FROM spock.replication_set_seq rss " + "JOIN spock.replication_set rs ON rs.set_id = rss.set_id " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id " + "JOIN pg_catalog.pg_class c ON c.oid = rss.set_seqoid " + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace"); + for (i = 0; i < PQntuples(res); i++) + { + const char *name = PQgetvalue(res, i, 0); + const char *rel = PQgetvalue(res, i, 1); + char *present; + + present = zodan_local_scalar(psprintf( + "SELECT to_regclass(%s) IS NOT NULL", quote_literal_cstr(rel))); + if (present == NULL || present[0] != 't') + { + appendStringInfo(&missing, "%s%s", missing.len > 0 ? ", " : "", rel); + continue; + } + pfree(present); + + zodan_local_command(psprintf( + "SELECT spock.repset_add_seq(%s, %s::regclass, false)", + quote_literal_cstr(name), quote_literal_cstr(rel))); + added_seqs++; + } + zodan_clear(res); + zodan_disconnect(conn); + + if (missing.len > 0) + ereport(ERROR, + (errmsg("the source node replicates relations that do not exist on \"%s\"", + ctx->new_node_name), + errdetail("Missing: %s", missing.data), + errhint("The new node cannot join until those relations are present."))); + + ZNOTICE(" OK: replication sets: %d dropped; membership: %d tables, %d sequences copied from source", + dropped, added, added_seqs); + + /* + * 5. The source is a moving target: somebody running repset_add_table() + * there while the join is in progress would leave the two nodes out of step. + * Re-read the source and fail loudly rather than finish with a node that + * quietly replicates a different set of tables. + * + * Comparing counts would miss a swap (one table removed, another added + * keeps the count the same) and would not notice a changed column list or + * row filter at all, so compare an ordered digest of everything that was + * copied: set definitions, table membership and sequences. + */ + conn = zodan_connect(ctx->src_dsn, "repsetcheck"); + res = zodan_remote_query(conn, ZODAN_REPSET_DIGEST_SQL); + src_digest = pstrdup(PQgetvalue(res, 0, 0)); + zodan_clear(res); + zodan_disconnect(conn); + + local_digest = zodan_local_scalar(ZODAN_REPSET_DIGEST_SQL); + + if (local_digest == NULL || strcmp(src_digest, local_digest) != 0) + ereport(ERROR, + (errmsg("the source node's replication sets changed while \"%s\" was joining", + ctx->new_node_name), + errdetail("Source digest %s, %s digest %s.", + src_digest, ctx->new_node_name, + local_digest ? local_digest : "(none)"), + errhint("Re-run attach_node once the source is quiescent."))); + + pfree(missing.data); + MemoryContextDelete(mcxt); + SPI_commit(); + + ZNOTICE(" OK: replication sets on %s match the source", ctx->new_node_name); +} + +/* + * Phase 9: enable the previously disabled subscriptions (source->new for the + * 2-node case, other->new for the multi-node case), waiting for each stored + * sync event first. + */ +static void +zodan_enable_disabled_subs(ZodanAddCtx *ctx) +{ + int i; + + ZNOTICE("Phase 9: Enabling disabled subscriptions"); + + if (zodan_num_other_nodes(ctx) == 0) + { + char *sub_name = zodan_gen_sub_name(ctx->src_node_name, ctx->new_node_name); + char *lsn; + + zodan_local_command(psprintf( + "SELECT spock.sub_enable(subscription_name := %s, immediate := true)", + quote_literal_cstr(sub_name))); + SPI_commit(); + + lsn = zodan_local_scalar(psprintf( + "SELECT sync_lsn FROM temp_sync_lsns WHERE origin_node = %s", + quote_literal_cstr(ctx->src_node_name))); + if (lsn != NULL) + { + zodan_local_wait_for_sync_event(ctx, ctx->src_node_name, lsn); + pfree(lsn); + } + ZNOTICE(" OK: enabled subscription %s", sub_name); + return; + } + + for (i = 0; i < ctx->nnodes; i++) + { + ZNode *rec = &ctx->nodes[i]; + char *sub_name; + char *lsn; + + if (strcmp(rec->name, ctx->src_node_name) == 0 || + strcmp(rec->name, ctx->new_node_name) == 0) + continue; + + sub_name = zodan_gen_sub_name(rec->name, ctx->new_node_name); + zodan_local_command(psprintf( + "SELECT spock.sub_enable(subscription_name := %s, immediate := true)", + quote_literal_cstr(sub_name))); + SPI_commit(); + + lsn = zodan_local_scalar(psprintf( + "SELECT sync_lsn FROM temp_sync_lsns WHERE origin_node = %s", + quote_literal_cstr(rec->name))); + if (lsn != NULL) + { + zodan_local_wait_for_sync_event(ctx, rec->name, lsn); + pfree(lsn); + } + ZNOTICE(" OK: enabled subscription %s", sub_name); + } +} + +/* + * Phase 10: create subscriptions from every "other" existing node to the new node + * (so the rest of the cluster receives the new node's changes). These are + * created on the remote nodes, with the new node as provider. + * + * The source node is deliberately skipped here and handled by phase 11, which + * reaches it over the caller-supplied src_dsn -- verified reachable in phase 0 -- + * rather than the if_dsn registered in the source's own spock.node_interface, + * which nothing in this orchestration ever verifies. Note that ctx->nodes is + * read from the source and does contain the source itself, so without this skip + * phase 10 would create the new->source subscription and make phase 11 dead code. + */ +static void +zodan_create_subs_to_new_node(ZodanAddCtx *ctx) +{ + int i; + + ZNOTICE("Phase 10: Creating subscriptions from other nodes to the new node"); + + for (i = 0; i < ctx->nnodes; i++) + { + ZNode *rec = &ctx->nodes[i]; + char *sub_name; + + if (strcmp(rec->name, ctx->new_node_name) == 0 || + strcmp(rec->name, ctx->src_node_name) == 0) + continue; + + sub_name = zodan_gen_sub_name(ctx->new_node_name, rec->name); + zodan_create_sub(ctx, rec->dsn, sub_name, ctx->new_node_dsn, false, false, true); + ZNOTICE(" OK: created subscription %s on node %s", sub_name, rec->name); + } +} + +/* + * Phase 11: create the enabled new->source subscription, completing + * bidirectional replication with the source node. Skipped if it somehow already + * exists, so a re-run does not fail at the very last step. + */ +static void +zodan_create_new_to_source_sub(ZodanAddCtx *ctx) +{ + char *sub_name = zodan_gen_sub_name(ctx->new_node_name, ctx->src_node_name); + int64 exists; + PGconn *conn; + + ZNOTICE("Phase 11: Creating new to source node subscription"); + + conn = zodan_connect(ctx->src_dsn, "subcheck"); + exists = 0; + { + char *c = zodan_remote_scalar(conn, psprintf( + "SELECT count(*) FROM spock.subscription WHERE sub_name = %s", + quote_literal_cstr(sub_name))); + + if (c != NULL) + { + exists = pg_strtoint64(c); + pfree(c); + } + } + zodan_disconnect(conn); + + if (exists > 0) + { + ZNOTICE(" - subscription %s already exists, skipping", sub_name); + return; + } + + zodan_create_sub(ctx, ctx->src_dsn, sub_name, ctx->new_node_dsn, false, false, true); + ZNOTICE(" OK: created subscription %s", sub_name); +} + +/* ------------------------------------------------------------------------ + * detach_node phases + * ------------------------------------------------------------------------ */ + +/* + * Drop the inbound subscriptions (target as provider) on every surviving node, + * then drop every subscription local to the target node. Slot and origin + * cleanup is implicit in spock.sub_drop(). Per-node errors are logged and + * tolerated, matching the old zodremove.sql behavior. + */ +static void +zodan_remove_subscriptions(const char *target_node_name) +{ + int i; + int nnodes; + ZNode *others; + MemoryContext mcxt; + MemoryContext old; + + /* Snapshot every node locally; the target is filtered out in C below. */ + if (SPI_execute( + "SELECT n.node_name, i.if_dsn FROM spock.node n " + "JOIN spock.node_interface i ON n.node_id = i.if_nodeid", + true, 0) != SPI_OK_SELECT) + elog(ERROR, "failed to enumerate cluster nodes"); + + nnodes = SPI_processed; + mcxt = AllocSetContextCreate(CurrentMemoryContext, "zodan_remove", + ALLOCSET_DEFAULT_SIZES); + old = MemoryContextSwitchTo(mcxt); + others = (ZNode *) palloc0(sizeof(ZNode) * Max(nnodes, 1)); + for (i = 0; i < nnodes; i++) + { + char *name = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1); + char *dsn = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 2); + + others[i].name = name ? pstrdup(name) : NULL; + others[i].dsn = dsn ? pstrdup(dsn) : NULL; + } + MemoryContextSwitchTo(old); + + /* + * Remote: drop each surviving node's inbound subscription from the target. + * + * Each node is handled inside its own internal subtransaction so that a + * failure to reach or update one node (for example a node that is down) is + * tolerated and does not abort the whole removal, matching the old + * zodremove.sql behavior. Without the subtransaction, catching the error + * would leave the surrounding transaction in an aborted state. + */ + for (i = 0; i < nnodes; i++) + { + MemoryContext subctx = CurrentMemoryContext; + ResourceOwner subowner = CurrentResourceOwner; + + /* + * volatile: assigned inside PG_TRY and read in PG_CATCH, so the compiler + * must not keep it in a register clobbered by the longjmp. + */ + PGconn *volatile conn = NULL; + + if (others[i].name == NULL || others[i].dsn == NULL || + strcmp(others[i].name, target_node_name) == 0) + continue; + + BeginInternalSubTransaction(NULL); + MemoryContextSwitchTo(subctx); + + PG_TRY(); + { + char *sub_name; + + conn = zodan_connect(others[i].dsn, "rmsub"); + sub_name = zodan_remote_scalar(conn, psprintf( + "SELECT s.sub_name FROM spock.subscription s " + "JOIN spock.node n ON n.node_id = s.sub_origin " + "WHERE n.node_name = %s", quote_literal_cstr(target_node_name))); + if (sub_name != NULL) + { + zodan_remote_command(conn, psprintf( + "SELECT spock.sub_drop(%s, true)", quote_literal_cstr(sub_name))); + ZNOTICE(" OK: dropped subscription %s on node %s", + sub_name, others[i].name); + } + zodan_disconnect(conn); + conn = NULL; + + ReleaseCurrentSubTransaction(); + MemoryContextSwitchTo(subctx); + CurrentResourceOwner = subowner; + } + PG_CATCH(); + { + MemoryContextSwitchTo(subctx); + if (conn != NULL) + zodan_disconnect(conn); + RollbackAndReleaseCurrentSubTransaction(); + MemoryContextSwitchTo(subctx); + CurrentResourceOwner = subowner; + FlushErrorState(); + ereport(WARNING, + (errmsg("could not drop inbound subscription on node %s; continuing", + others[i].name))); + } + PG_END_TRY(); + } + + /* Local: drop every subscription on the target node. */ + if (SPI_execute("SELECT sub_name FROM spock.subscription", true, 0) != SPI_OK_SELECT) + elog(ERROR, "failed to enumerate local subscriptions"); + { + int nlocal = SPI_processed; + char **names = (char **) palloc0(sizeof(char *) * Max(nlocal, 1)); + + for (i = 0; i < nlocal; i++) + { + char *nm = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1); + + names[i] = nm ? pstrdup(nm) : NULL; + } + for (i = 0; i < nlocal; i++) + { + if (names[i] == NULL) + continue; + zodan_local_command(psprintf( + "SELECT spock.sub_drop(%s, true)", quote_literal_cstr(names[i]))); + ZNOTICE(" OK: dropped local subscription %s", names[i]); + } + } + + MemoryContextDelete(mcxt); +} + +/* ------------------------------------------------------------------------ + * SQL entry points + * ------------------------------------------------------------------------ */ + +/* + * spock.attach_node(src_node_name, src_dsn, new_node_name, new_node_dsn, + * verb, new_node_location, new_node_country, new_node_info, + * timeout_sec) + * + * Run on the NEW node. + */ +Datum +spock_attach_node(PG_FUNCTION_ARGS) +{ + ZodanAddCtx *ctx; + MemoryContext mcxt; + MemoryContext old; + bool nonatomic; + + /* We must be able to commit between phases: require a top-level CALL. */ + nonatomic = fcinfo->context != NULL && IsA(fcinfo->context, CallContext) && + !((CallContext *) fcinfo->context)->atomic; + if (!nonatomic) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TRANSACTION_TERMINATION), + errmsg("spock.attach_node() cannot run inside a transaction block"), + errhint("Invoke it with a top-level CALL, not inside BEGIN/COMMIT."))); + + /* + * LANGUAGE c procedures cannot be declared STRICT, so guard the required + * text arguments explicitly: a SQL NULL would otherwise reach + * text_to_cstring() and detoast a NULL pointer, crashing the backend. + */ + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2) || PG_ARGISNULL(3)) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("src_node_name, src_dsn, new_node_name and new_node_dsn must not be NULL"))); + + /* Every previous call must have drained its registries. */ + Assert(zodan_nconns == 0 && zodan_nresults == 0); + + /* Connect first, so nothing between the context creation and the PG_TRY + * below can throw and orphan the context. */ + SPI_connect_ext(SPI_OPT_NONATOMIC); + + /* + * ctx and everything it points at must survive the SPI_commit() calls + * between phases, so the context cannot live under the SPI procedure + * context. PortalContext is the right parent: a non-atomic SPI connection + * parents its own contexts there (see spi.c), PreCommit_Portals leaves the + * active portal alone across the commits, and the portal's cleanup releases + * this context for us if argument parsing below or a later phase raises + * before the PG_FINALLY. + */ + mcxt = AllocSetContextCreate(PortalContext != NULL ? PortalContext + : TopMemoryContext, + "zodan_add", ALLOCSET_DEFAULT_SIZES); + old = MemoryContextSwitchTo(mcxt); + ctx = (ZodanAddCtx *) palloc0(sizeof(ZodanAddCtx)); + ctx->mcxt = mcxt; + ctx->src_node_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + ctx->src_dsn = text_to_cstring(PG_GETARG_TEXT_PP(1)); + ctx->new_node_name = text_to_cstring(PG_GETARG_TEXT_PP(2)); + ctx->new_node_dsn = text_to_cstring(PG_GETARG_TEXT_PP(3)); + ctx->verb = PG_ARGISNULL(4) ? false : PG_GETARG_BOOL(4); + ctx->new_node_location = PG_ARGISNULL(5) ? pstrdup("NY") : text_to_cstring(PG_GETARG_TEXT_PP(5)); + ctx->new_node_country = PG_ARGISNULL(6) ? pstrdup("USA") : text_to_cstring(PG_GETARG_TEXT_PP(6)); + ctx->new_node_info = PG_ARGISNULL(7) ? pstrdup("{}") : + DatumGetCString(DirectFunctionCall1(jsonb_out, PG_GETARG_DATUM(7))); + ctx->timeout_sec = PG_ARGISNULL(8) ? 180 : PG_GETARG_INT32(8); + if (ctx->timeout_sec <= 0) + ctx->timeout_sec = 180; + /* Cap so the "* 1000" millisecond conversions cannot overflow int. */ + if (ctx->timeout_sec > INT_MAX / 1000) + ctx->timeout_sec = INT_MAX / 1000; + MemoryContextSwitchTo(old); + + zodan_verbose = ctx->verb; + + PG_TRY(); + { + zodan_fetch_cluster_nodes(ctx); + + zodan_check_versions(ctx); + zodan_verify_prerequisites(ctx); + zodan_create_nodes(ctx); + /* Cluster snapshot may be stale after node creation; refresh. */ + zodan_fetch_cluster_nodes(ctx); + zodan_create_disabled_subs_and_slots(ctx); + zodan_sync_other_nodes_wait_source(ctx); + zodan_create_source_to_new_sub(ctx); + zodan_source_sync_wait_new(ctx); + zodan_wait_ready_and_advance(ctx); + zodan_sync_repsets_from_source(ctx); + zodan_enable_disabled_subs(ctx); + zodan_create_subs_to_new_node(ctx); + zodan_create_new_to_source_sub(ctx); + + ereport(NOTICE, + (errmsg("attach_node: node \"%s\" added to the cluster", + ctx->new_node_name))); + } + PG_FINALLY(); + { + /* + * Release anything a failed phase left open before unwinding: neither + * PGconn nor PGresult is owned by a resource owner. + */ + zodan_release_all(); + MemoryContextDelete(mcxt); + } + PG_END_TRY(); + + SPI_finish(); + PG_RETURN_VOID(); +} + +/* + * Verify that detach_node is running on the node it is being asked to remove, by + * comparing the local system identifier and database name against what + * target_node_dsn points at. This is the mirror of the check attach_node makes + * against new_node_dsn, and it is what target_node_dsn is for. + * + * The check matters because everything after it operates on the local node: the + * local subscriptions and replication sets are dropped wholesale. Run from the + * wrong node, detach_node would tear down that node's replication instead of the + * target's. + */ +static void +zodan_verify_detach_target(const char *target_node_name, const char *target_dsn) +{ + char *local_sysid; + char *local_dbname; + char *remote_sysid; + char *remote_dbname; + PGconn *conn; + + local_sysid = zodan_local_scalar("SELECT system_identifier::text FROM pg_control_system()"); + local_dbname = zodan_local_scalar("SELECT current_database()"); + + conn = zodan_connect(target_dsn, "detach"); + remote_sysid = zodan_remote_scalar(conn, + "SELECT system_identifier::text FROM pg_control_system()"); + remote_dbname = zodan_remote_scalar(conn, "SELECT current_database()"); + zodan_disconnect(conn); + + if (local_sysid == NULL || remote_sysid == NULL || + strcmp(local_sysid, remote_sysid) != 0 || + local_dbname == NULL || remote_dbname == NULL || + strcmp(local_dbname, remote_dbname) != 0) + ereport(ERROR, + (errmsg("detach_node must be run on the node being removed"), + errdetail("target_node_dsn (%s) does not match the current database connection", + target_dsn), + errhint("Connect to node %s and re-run detach_node.", + target_node_name))); + + ZNOTICE(" OK: detach_node is running on node %s", target_node_name); +} + +/* + * spock.detach_node(target_node_name, target_node_dsn, verbose_mode) + * + * Run on the node being removed. Order: subscriptions (incl. implicit slot and + * origin cleanup) -> replication sets -> node. + * + * Like attach_node, this is not atomic across the cluster; see the file header. + * Unlike attach_node, a node that cannot be reached is tolerated with a WARNING + * (see zodan_remove_subscriptions), so a surviving node that was down during the + * detach keeps a stale inbound subscription from the removed node and needs + * spock.sub_drop() run on it by hand. + */ +Datum +spock_detach_node(PG_FUNCTION_ARGS) +{ + char *target_node_name; + char *target_node_dsn; + bool verb = PG_ARGISNULL(2) ? true : PG_GETARG_BOOL(2); + bool nonatomic; + int64 exists; + + /* See spock_attach_node(): LANGUAGE c procedures cannot be STRICT. */ + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("target_node_name and target_node_dsn must not be NULL"))); + + target_node_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + target_node_dsn = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + zodan_verbose = verb; + + nonatomic = fcinfo->context != NULL && IsA(fcinfo->context, CallContext) && + !((CallContext *) fcinfo->context)->atomic; + if (!nonatomic) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TRANSACTION_TERMINATION), + errmsg("spock.detach_node() cannot run inside a transaction block"), + errhint("Invoke it with a top-level CALL, not inside BEGIN/COMMIT."))); + + /* Every previous call must have drained its registries. */ + Assert(zodan_nconns == 0 && zodan_nresults == 0); + + SPI_connect_ext(SPI_OPT_NONATOMIC); + + PG_TRY(); + { + /* Phase 1: validate the target node exists and that we are it. */ + exists = zodan_local_count(psprintf( + "SELECT count(*) FROM spock.node WHERE node_name = %s", + quote_literal_cstr(target_node_name))); + if (exists == 0) + ereport(ERROR, (errmsg("node %s does not exist", target_node_name))); + + zodan_verify_detach_target(target_node_name, target_node_dsn); + + ZNOTICE("Removing node %s from the cluster", target_node_name); + + /* Phase 2/3: drop subscriptions (remote inbound + all local). */ + zodan_remove_subscriptions(target_node_name); + SPI_commit(); + + /* Phase 4: drop replication sets local to the target node. */ + { + int i; + int nsets; + char **sets; + + if (SPI_execute("SELECT set_name FROM spock.replication_set", true, 0) != SPI_OK_SELECT) + elog(ERROR, "failed to enumerate replication sets"); + nsets = SPI_processed; + sets = (char **) palloc0(sizeof(char *) * Max(nsets, 1)); + for (i = 0; i < nsets; i++) + { + char *nm = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1); + + sets[i] = nm ? pstrdup(nm) : NULL; + } + for (i = 0; i < nsets; i++) + { + if (sets[i] == NULL) + continue; + zodan_local_command(psprintf( + "SELECT spock.repset_drop(%s, true)", quote_literal_cstr(sets[i]))); + ZNOTICE(" OK: dropped replication set %s", sets[i]); + } + } + SPI_commit(); + + /* Phase 5: drop the node itself. */ + exists = zodan_local_count(psprintf( + "SELECT count(*) FROM spock.node WHERE node_name = %s", + quote_literal_cstr(target_node_name))); + if (exists > 0) + { + zodan_local_command(psprintf( + "SELECT spock.node_drop(%s, true)", quote_literal_cstr(target_node_name))); + ZNOTICE(" OK: dropped node %s", target_node_name); + } + SPI_commit(); + + ereport(NOTICE, + (errmsg("detach_node: node \"%s\" removed from the cluster", + target_node_name))); + } + PG_FINALLY(); + { + /* Same reasoning as spock_attach_node(). */ + zodan_release_all(); + } + PG_END_TRY(); + + SPI_finish(); + PG_RETURN_VOID(); +} diff --git a/tests/docker/entrypoint.sh b/tests/docker/entrypoint.sh index d3fa324e..7c42b404 100644 --- a/tests/docker/entrypoint.sh +++ b/tests/docker/entrypoint.sh @@ -131,10 +131,8 @@ EOF country := 'ESP', location := 'Madrid', info := '{\"tiebreaker\" : \"1\"}')" else - # Add node to the existing cluster using Z0DAN - psql -h /tmp -c "CREATE EXTENSION dblink" - psql -h /tmp -f ${SPOCK_SOURCE_DIR}/samples/Z0DAN/zodan.sql - psql -h /tmp -c "CALL spock.add_node( + # Add node to the existing cluster (attach_node ships with the extension) + psql -h /tmp -c "CALL spock.attach_node( src_node_name := 'n1', src_dsn := 'host=n1 port=${PGPORT} dbname=${PGDATABASE} user=${PGUSER}', new_node_name := '${HOSTNAME}', diff --git a/tests/tap/schedule b/tests/tap/schedule index 88f3e757..c9756db3 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -15,13 +15,18 @@ test: 004_non_default_repset # test: 005_daylight_savings # test: 006_sync_during_write test: 008_rmgr +# ZODAN add/remove node: SQL/dblink orchestration (samples/Z0DAN/zodan.sql) test: 009_zodan_add_remove_nodes +# ZODAN add/remove node: in-core C orchestration (spock.attach_node/detach_node) +test: 034_attach_detach_node_incore test: 013_origin_change_restore test: 014_pgdump_restore_conflict # Tests, consuming too much time to be launched on each check: +# (SQL/dblink orchestration, and its in-core C counterpart) #test: 011_zodan_sync_third +#test: 035_attach_node_sync_third_incore # # Use GitHub Actions to launch them (see workflows/zodan_sync.yml for an example # Also, it may be run locally by a bash script like the following: diff --git a/tests/tap/schedule-nightly b/tests/tap/schedule-nightly index c193644e..8ed89547 100644 --- a/tests/tap/schedule-nightly +++ b/tests/tap/schedule-nightly @@ -11,6 +11,9 @@ # test: 012_zodan_basics +# ZODAN basics/timeout: in-core C orchestration (spock.attach_node/detach_node) +test: 036_attach_node_basics_incore +test: 037_attach_node_3n_timeout_incore # test: 016_crash_recovery_progress test: 017_zodan_3n_timeout diff --git a/tests/tap/t/033_zodan_lolor_add_node.pl b/tests/tap/t/033_zodan_lolor_add_node.pl index de8e93f4..c0d6941c 100644 --- a/tests/tap/t/033_zodan_lolor_add_node.pl +++ b/tests/tap/t/033_zodan_lolor_add_node.pl @@ -6,9 +6,9 @@ use SpockTest qw(create_cluster destroy_cluster system_or_bail system_maybe get_test_config cross_wire scalar_query ensure_lolor); -# Zodan add_node with lolor large objects. The source cluster replicates the +# Zodan attach_node with lolor large objects. The source cluster replicates the # lolor tables (lolor.pg_largeobject, lolor.pg_largeobject_metadata) in the -# default replication set. Adding a node through zodan's add_node() must: +# default replication set. Adding a node through zodan's attach_node() must: # - reject a new node that lacks the lolor extension (data sync would fail), # - reject a new node whose lolor tables already contain data, # - accept a new node with lolor installed and empty, and copy the large @@ -75,9 +75,10 @@ sub dsn { cross_wire(2, ['n1', 'n2'], 'Cross-wire nodes n1 and n2'); # create_cluster registered a spock node on n3; drop it so n3 looks like a -# freshly prepared instance (spock + dblink installed, no node/repsets). +# freshly prepared instance (spock installed, no node/repsets). attach_node is a +# C-native procedure shipped with the extension, so no dblink or zodan.sql is +# needed on the new node. ok(psql_ok(3, "SELECT spock.node_drop('n3')"), 'n3 spock node registration dropped'); -ok(psql_ok(3, "CREATE EXTENSION IF NOT EXISTS dblink"), 'dblink installed on n3'); # lolor on the source cluster, its tables in the default replication set. # n1 and n2 are cross-wired with automatic DDL replication, so CREATE @@ -98,19 +99,14 @@ sub dsn { ok(wait_for_scalar(2, "SELECT count(*) FROM lolor.pg_largeobject WHERE encode(data, 'hex') = 'deadbeefcafe'", '1'), 'large object replicated from n1 to n2'); -# Load the zodan procedures on the node being added. -system_or_bail("$PG/psql", '-X', '-p', $cfg->{node_ports}[2], '-d', $DB, - '-v', 'ON_ERROR_STOP=1', '-f', '../../samples/Z0DAN/zodan.sql'); -pass('zodan procedures loaded on n3'); - -my $add_node_sql = - "CALL spock.add_node('n1', '" . dsn(1) . "', 'n3', '" . dsn(3) . "', " . +my $attach_node_sql = + "CALL spock.attach_node('n1', '" . dsn(1) . "', 'n3', '" . dsn(3) . "', " . "true, 'CA', 'USA', '{}'::jsonb)"; # --- Negative: source replicates lolor but n3 has no lolor extension -------- -my ($rc, $out) = psql_capture(3, $add_node_sql); -ok($rc != 0, 'add_node rejected while n3 lacks the lolor extension'); +my ($rc, $out) = psql_capture(3, $attach_node_sql); +ok($rc != 0, 'attach_node rejected while n3 lacks the lolor extension'); like($out, qr/does not have the lolor extension installed/, 'rejection message asks for CREATE EXTENSION lolor'); @@ -120,24 +116,18 @@ sub dsn { ok(psql_ok(3, "SET lolor.node=3; SELECT lo_from_bytea(0, '\\x0bad0bad')"), 'pre-existing large object created on n3'); -($rc, $out) = psql_capture(3, $add_node_sql); -ok($rc != 0, 'add_node rejected while n3 has pre-existing lolor data'); +($rc, $out) = psql_capture(3, $attach_node_sql); +ok($rc != 0, 'attach_node rejected while n3 has pre-existing lolor data'); like($out, qr/pre-existing large object data/, 'rejection message mentions pre-existing large object data'); -# health_check 'pre' must report the same problem without raising. -($rc, $out) = psql_capture(3, - "CALL spock.health_check('n1', '" . dsn(1) . "', 'n3', '" . dsn(3) . "', 'pre', false)"); -like($out, qr/FAIL: Destination database has pre-existing large object data/, - 'health_check pre-check flags pre-existing lolor data'); - -# --- Positive: empty lolor tables, add_node copies the data ----------------- +# --- Positive: empty lolor tables, attach_node copies the data ----------------- ok(psql_ok(3, "DELETE FROM lolor.pg_largeobject; DELETE FROM lolor.pg_largeobject_metadata"), 'pre-existing lolor data cleared on n3'); -($rc, $out) = psql_capture(3, $add_node_sql); -is($rc, 0, 'add_node succeeded with lolor installed and empty') or diag($out); +($rc, $out) = psql_capture(3, $attach_node_sql); +is($rc, 0, 'attach_node succeeded with lolor installed and empty') or diag($out); ok(wait_for_scalar(3, "SELECT count(*) FROM lolor.pg_largeobject WHERE encode(data, 'hex') = 'deadbeefcafe'", '1'), 'existing large object data copied to n3 by initial sync'); diff --git a/tests/tap/t/034_attach_detach_node_incore.pl b/tests/tap/t/034_attach_detach_node_incore.pl new file mode 100755 index 00000000..5d7e97e7 --- /dev/null +++ b/tests/tap/t/034_attach_detach_node_incore.pl @@ -0,0 +1,350 @@ +use strict; +use warnings; +use Test::More; +use lib '.'; +use lib 't'; +use SpockTest qw(create_cluster destroy_cluster system_or_bail command_ok get_test_config cross_wire system_maybe); + +# ============================================================================= +# Test: 034_attach_detach_node_incore.pl - Test Zodan Node Addition and Removal +# ============================================================================= +# This test follows the sequence: +# 1. Create 2-node cluster and cross-wire them +# 2. Create test data and replication sets +# 3. Load ZODAN procedures on n1 +# 4. Add node n3 using ZODAN attach_node procedure +# 5. Verify n3 is properly integrated +# 6. Load ZODREMOVE procedures on n3 +# 7. Remove node n3 using ZODREMOVE detach_node procedure +# 8. Verify n3 is properly removed +# 9. Clean up + +# Step 1: Create a 2-node cluster initially +create_cluster(2, 'Create initial 2-node Spock test cluster'); + +# Get cluster configuration +my $config = get_test_config(); +my $node_count = $config->{node_count}; +my $node_ports = $config->{node_ports}; +my $host = $config->{host}; +my $dbname = $config->{db_name}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +# Step 1a: Cross-wire the 2 nodes (n1 and n2) +cross_wire(2, ['n1', 'n2'], 'Cross-wire nodes n1 and n2'); + +pass('2-node cluster created and cross-wired'); + +# Step 2: Create test data and replication sets +pass('Creating test data and replication sets'); + +# Create test table on n1 +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', " + CREATE TABLE test_zodan_table ( + id SERIAL PRIMARY KEY, + name VARCHAR(50), + value INTEGER, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) +"; + +# Create a custom replication set on n1 +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', " + SELECT spock.repset_create('zodan_test_set', true, true, true, true) +"; + +# Add table to the custom replication set +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', " + SELECT spock.repset_add_table('zodan_test_set', 'test_zodan_table') +"; + +# Insert test data on n1 +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', " + INSERT INTO test_zodan_table (name, value) VALUES + ('test_data_1', 100), + ('test_data_2', 200), + ('test_data_3', 300) +"; + +pass('Test table created and data inserted on n1'); + +# Wait for replication to n2 +system_or_bail 'sleep', '2'; + +# Verify data replicated to n2 +my $data_on_n2 = `$pg_bin/psql -p $node_ports->[1] -d $dbname -t -c "SELECT COUNT(*) FROM test_zodan_table"`; +chomp($data_on_n2); +$data_on_n2 =~ s/\s+//g; +if ($data_on_n2 eq '3') { + pass('Data successfully replicated from n1 to n2 (3 rows)'); +} else { + fail("Data replication failed: expected 3 rows, got $data_on_n2"); +} + +# Step 3: Create n3 database instance +pass('Creating n3 database instance'); + +# Create n3 data directory and start PostgreSQL instance +my $n3_datadir = "/tmp/tmp_spock_node_3_datadir"; +my $n3_port = $node_ports->[0] + 2; # Use next available port + +# Clean up any existing n3 directory +system_or_bail 'rm', '-rf', $n3_datadir; + +# Initialize n3 data directory +system_or_bail "$pg_bin/initdb", '-A', 'trust', '-D', $n3_datadir; + +# Copy configuration files if they exist +if (-f 'regress-pg_hba.conf') { + system_or_bail 'cp', 'regress-pg_hba.conf', "$n3_datadir/pg_hba.conf"; +} + +# Create PostgreSQL configuration for n3 +open(my $conf, '>>', "$n3_datadir/postgresql.conf") or die "Cannot open config file: $!"; +print $conf "shared_buffers=1GB\n"; +print $conf "shared_preload_libraries='spock'\n"; +print $conf "wal_level=logical\n"; +print $conf "spock.enable_ddl_replication=on\n"; +print $conf "spock.include_ddl_repset=on\n"; +print $conf "spock.allow_ddl_from_functions=on\n"; +print $conf "spock.exception_behaviour=sub_disable\n"; +print $conf "spock.conflict_resolution=last_update_wins\n"; +print $conf "track_commit_timestamp=on\n"; +print $conf "spock.exception_replay_queue_size=1MB\n"; +print $conf "spock.enable_spill=on\n"; +print $conf "port=$n3_port\n"; +print $conf "listen_addresses='*'\n"; +print $conf "logging_collector=on\n"; +print $conf "log_directory='/tmp/logs'\n"; +print $conf "log_filename='00$n3_port.log'\n"; +close($conf); + +# Start n3 PostgreSQL instance +system("$pg_bin/postgres -D $n3_datadir >> '$config->{log_file}' 2>&1 &"); + +# Allow n3 to startup +system_or_bail 'sleep', '10'; + +# Create database and user for testing on n3 +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', 'postgres', '-c', "CREATE DATABASE $dbname"; +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', "CREATE USER $db_user SUPERUSER"; +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', "CREATE USER super SUPERUSER"; + +# Install Spock extension on n3 +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', "CREATE EXTENSION IF NOT EXISTS spock"; +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', "ALTER EXTENSION spock UPDATE"; + +pass('n3 database instance created and configured'); + +# attach_node/detach_node ship with the spock extension; nothing to load. +my $pipe; + +print "=== STARTING ADD_NODE PROCEDURE ===\n"; + +my $attach_node_cmd = "$pg_bin/psql -p $n3_port -d $dbname -c \" + CALL spock.attach_node( + 'n1', + 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + 'n3', + 'host=$host dbname=$dbname port=$n3_port user=$db_user password=$db_password', + true, + 'CA', + 'USA', + '{}'::jsonb + ) +\""; + +print "Executing: $attach_node_cmd\n"; +print "---\n"; + +open($pipe, "$attach_node_cmd 2>&1 |") or die "Cannot open pipe: $!"; +while (my $line = <$pipe>) { + print $line; +} +close($pipe); +my $attach_node_result = $? >> 8; + +print "---\n"; +print "=== ADD_NODE PROCEDURE COMPLETED (exit code: $attach_node_result) ===\n"; + +if ($attach_node_result == 0) { + pass('attach_node procedure executed successfully'); +} else { + fail("attach_node procedure failed with exit code: $attach_node_result"); +} + +# Step 6: Verify n3 is properly integrated +pass('Verifying n3 integration'); + +# Wait for replication to complete +system_or_bail 'sleep', '3'; + +# Check if test table exists on n3 +my $table_exists_n3 = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'test_zodan_table')"`; +chomp($table_exists_n3); +$table_exists_n3 =~ s/\s+//g; +if ($table_exists_n3 eq 't') { + pass('Test table exists on n3'); + + # Check data count on n3 + my $data_on_n3 = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM test_zodan_table"`; + chomp($data_on_n3); + $data_on_n3 =~ s/\s+//g; + if ($data_on_n3 eq '3') { + pass('Data successfully replicated to n3 (3 rows)'); + } else { + fail("Data replication to n3 failed: expected 3 rows, got $data_on_n3"); + } +} else { + fail('Test table does not exist on n3'); +} + +# Verify n3 has subscriptions that reference the default replication sets +my $sub_count_n3 = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM spock.subscription"`; +chomp($sub_count_n3); +$sub_count_n3 =~ s/\s+//g; +if ($sub_count_n3 >= 2) { + pass('n3 has subscriptions referencing default replication sets (ZODAN behavior)'); +} else { + # Debug: List all subscriptions on n3 + my $debug_subs = `$pg_bin/psql -p $n3_port -d $dbname -c "SELECT sub_name FROM spock.subscription ORDER BY sub_name"`; + print "Debug: Subscriptions on n3:\n$debug_subs\n"; + fail('n3 does not have expected subscriptions'); +} + +# Verify n3 is in the node list on all nodes +my $node_count_n1 = `$pg_bin/psql -p $node_ports->[0] -d $dbname -t -c "SELECT COUNT(*) FROM spock.node"`; +chomp($node_count_n1); +$node_count_n1 =~ s/\s+//g; +if ($node_count_n1 eq '3') { + pass('n1 shows 3 nodes in cluster'); +} else { + fail("n1 node count incorrect: expected 3, got $node_count_n1"); +} + +my $node_count_n2 = `$pg_bin/psql -p $node_ports->[1] -d $dbname -t -c "SELECT COUNT(*) FROM spock.node"`; +chomp($node_count_n2); +$node_count_n2 =~ s/\s+//g; +if ($node_count_n2 eq '3') { + pass('n2 shows 3 nodes in cluster'); +} else { + fail("n2 node count incorrect: expected 3, got $node_count_n2"); +} + +my $node_count_n3 = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM spock.node"`; +chomp($node_count_n3); +$node_count_n3 =~ s/\s+//g; +if ($node_count_n3 eq '3') { + pass('n3 shows 3 nodes in cluster'); +} else { + fail("n3 node count incorrect: expected 3, got $node_count_n3"); +} + +# Step 7: Remove node n3 using the in-core detach_node procedure +pass('Removing node n3 using detach_node procedure'); + +print "=== STARTING REMOVE_NODE PROCEDURE ===\n"; + +my $detach_node_cmd = "$pg_bin/psql -p $n3_port -d $dbname -c \" + CALL spock.detach_node( + 'n3', + 'host=$host dbname=$dbname port=$n3_port user=$db_user password=$db_password', + true + ) +\""; + +print "Executing: $detach_node_cmd\n"; +print "---\n"; + +open($pipe, "$detach_node_cmd 2>&1 |") or die "Cannot open pipe: $!"; +while (my $line = <$pipe>) { + print $line; +} +close($pipe); +my $detach_node_result = $? >> 8; + +print "---\n"; +print "=== REMOVE_NODE PROCEDURE COMPLETED (exit code: $detach_node_result) ===\n"; + +if ($detach_node_result == 0) { + pass('detach_node procedure executed successfully'); +} else { + fail("detach_node procedure failed with exit code: $detach_node_result"); +} + +# Step 9: Verify n3 is properly removed +pass('Verifying n3 removal'); + +# Wait for cleanup to complete +system_or_bail 'sleep', '2'; + +# Verify n3 is no longer in the node list on n1 and n2 +my $node_count_n1_after = `$pg_bin/psql -p $node_ports->[0] -d $dbname -t -c "SELECT COUNT(*) FROM spock.node"`; +chomp($node_count_n1_after); +$node_count_n1_after =~ s/\s+//g; +if ($node_count_n1_after eq '2') { + pass('n1 shows 2 nodes in cluster after removal'); +} else { + fail("n1 node count incorrect after removal: expected 2, got $node_count_n1_after"); +} + +my $node_count_n2_after = `$pg_bin/psql -p $node_ports->[1] -d $dbname -t -c "SELECT COUNT(*) FROM spock.node"`; +chomp($node_count_n2_after); +$node_count_n2_after =~ s/\s+//g; +if ($node_count_n2_after eq '2') { + pass('n2 shows 2 nodes in cluster after removal'); +} else { + fail("n2 node count incorrect after removal: expected 2, got $node_count_n2_after"); +} + +# Verify n3 no longer exists in node list +my $n3_exists_n1 = `$pg_bin/psql -p $node_ports->[0] -d $dbname -t -c "SELECT EXISTS (SELECT 1 FROM spock.node WHERE node_name = 'n3')"`; +chomp($n3_exists_n1); +$n3_exists_n1 =~ s/\s+//g; +if ($n3_exists_n1 eq 'f') { + pass('n3 no longer exists in n1 node list'); +} else { + fail('n3 still exists in n1 node list after removal'); +} + +my $n3_exists_n2 = `$pg_bin/psql -p $node_ports->[1] -d $dbname -t -c "SELECT EXISTS (SELECT 1 FROM spock.node WHERE node_name = 'n3')"`; +chomp($n3_exists_n2); +$n3_exists_n2 =~ s/\s+//g; +if ($n3_exists_n2 eq 'f') { + pass('n3 no longer exists in n2 node list'); +} else { + fail('n3 still exists in n2 node list after removal'); +} + +# Step 10: Clean up n3 +pass('Cleaning up n3'); + +# Stop n3 PostgreSQL instance +system("$pg_bin/pg_ctl stop -D $n3_datadir -m immediate >> '$config->{log_file}' 2>&1 &"); +system_or_bail 'sleep', '5'; +system_or_bail 'rm', '-rf', $n3_datadir; + +pass('n3 database instance cleaned up'); + +# Final verification +pass('ZODAN node addition and removal test completed successfully'); + +# Test summary +print "\n=== TEST SUMMARY ===\n"; +print "✓ Created 2-node cluster\n"; +print "✓ Cross-wired nodes n1 and n2\n"; +print "✓ Created test data and replication sets\n"; +print "✓ Added node n3 using ZODAN attach_node\n"; +print "✓ Verified n3 integration\n"; +print "✓ Removed node n3 using ZODREMOVE detach_node\n"; +print "✓ Verified n3 removal\n"; +print "✓ Cleaned up n3\n"; +print "========================\n"; + +# Tear down n1/n2 explicitly (n3 was already cleaned up above) so the +# destroy_cluster pass() is counted within the plan rather than firing from the +# END block after done_testing(). +destroy_cluster('Destroy ZODAN add/remove test cluster'); +done_testing(); diff --git a/tests/tap/t/035_attach_node_sync_third_incore.pl b/tests/tap/t/035_attach_node_sync_third_incore.pl new file mode 100755 index 00000000..25a3a6e4 --- /dev/null +++ b/tests/tap/t/035_attach_node_sync_third_incore.pl @@ -0,0 +1,450 @@ +use strict; +use warnings; +use Test::More tests => 30; +use IPC::Run; +use lib '.'; +use lib 't'; +use SpockTest qw(create_cluster destroy_cluster system_or_bail get_test_config cross_wire psql_or_bail scalar_query); + +# ============================================================================= +# Test: Add third node (N3) to the configuration of highly loaded (N1 and N2) +# by non-intersecting DMLs. +# ============================================================================= +# This test follows the sequence: +# 1. Create nodes N1 and N2 +# 2. Init pgbench database +# 3. CHECK: database replicated and we see a 'zero' lag +# 4. Load N1 and N2 with a custom non-intersecting UPDATE load +# 5. Call attach_node() on N3 +# 6. Check that pgbench load still exists after the end of the Z0DAN protocol +# 7. Wait for the end of the test and final data sync. +# 8. Check consistency of the data on each node. +# 9. Clean up + +create_cluster(3, 'Create initial 2-node Spock test cluster'); + + +my ($ret1, $ret2, $ret3, $lsn1, $lsn2, $lsn3); + +# Get cluster configuration +my $config = get_test_config(); +my $node_count = $config->{node_count}; +my $node_ports = $config->{node_ports}; +my $host = $config->{host}; +my $dbname = $config->{db_name}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +cross_wire(2, ['n1', 'n2'], 'Cross-wire nodes N1 and N2'); + +print STDERR "Install the helper functions and do other preparatory stuff\n"; +my $helper_sql = '../../samples/Z0DAN/wait_subscription.sql'; +psql_or_bail(1, "\\i $helper_sql"); +psql_or_bail(2, "\\i $helper_sql"); +psql_or_bail(3, "SELECT spock.node_drop('n3')"); + +# Reduce the logfile size +psql_or_bail(1, "ALTER SYSTEM SET log_min_messages TO LOG"); +psql_or_bail(1, "ALTER SYSTEM SET log_statement TO none"); +psql_or_bail(1, "ALTER SYSTEM SET log_checkpoints TO off"); +psql_or_bail(1, "ALTER SYSTEM SET log_connections TO off"); +psql_or_bail(1, "ALTER SYSTEM SET log_disconnections TO off"); +psql_or_bail(1, "ALTER SYSTEM SET log_lock_waits TO off"); +psql_or_bail(1, "ALTER SYSTEM SET log_statement_stats TO off"); +psql_or_bail(1, "SELECT pg_reload_conf()"); + +psql_or_bail(2, "ALTER SYSTEM SET log_min_messages TO LOG"); +psql_or_bail(2, "ALTER SYSTEM SET log_statement TO none"); +psql_or_bail(2, "ALTER SYSTEM SET log_checkpoints TO off"); +psql_or_bail(2, "ALTER SYSTEM SET log_connections TO off"); +psql_or_bail(2, "ALTER SYSTEM SET log_disconnections TO off"); +psql_or_bail(2, "ALTER SYSTEM SET log_lock_waits TO off"); +psql_or_bail(2, "ALTER SYSTEM SET log_statement_stats TO off"); +psql_or_bail(2, "SELECT pg_reload_conf()"); + +psql_or_bail(3, "ALTER SYSTEM SET log_min_messages TO LOG"); +psql_or_bail(3, "ALTER SYSTEM SET log_statement TO none"); +psql_or_bail(3, "ALTER SYSTEM SET log_checkpoints TO off"); +psql_or_bail(3, "ALTER SYSTEM SET log_connections TO off"); +psql_or_bail(3, "ALTER SYSTEM SET log_disconnections TO off"); +psql_or_bail(3, "ALTER SYSTEM SET log_lock_waits TO off"); +psql_or_bail(3, "ALTER SYSTEM SET log_statement_stats TO off"); +psql_or_bail(3, "SELECT pg_reload_conf()"); + +print STDERR "Initialize pgbench database and wait for initial sync on N1 and N2 ...\n"; +system_or_bail "$pg_bin/pgbench", '-i', '-s', 1, '-h', $host, + '-p', $node_ports->[0], '-U', $db_user, $dbname; +# Wait until tables and data will be sent to N2 +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +# Test 1: after the end of replication process we should be able to see a 'zero' +# lag between the nodes. +my $lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); + +ok($lag <= 0, "Initial replication has been successful"); + +# Create non-intersecting load for nodes N1 and N2. +# Test duration should be enough to cover all the Z0DAN stages. We will kill +# pgbench immediately after the N3 is attached. +my $load1 = '../../samples/Z0DAN/n1.pgb'; +my $load2 = '../../samples/Z0DAN/n2.pgb'; +my $pgbench_stdout1=''; +my $pgbench_stderr1=''; +my $pgbench_stdout2=''; +my $pgbench_stderr2=''; +my $pgbench_handle1 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load1, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[0], '-U', $db_user, $dbname], + '>', \$pgbench_stdout1, '2>', \$pgbench_stderr1); +my $pgbench_handle2 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load2, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[1], '-U', $db_user, $dbname], + '>', \$pgbench_stdout2, '2>', \$pgbench_stderr2); +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); + +# Warming up ... +print STDERR "warming up pgbench for 30s\n"; +sleep(30); +print STDERR "done warmup\n"; + +print STDERR "Add N3 into highly loaded configuration of N1 and N2 ...\n"; +# Use transdiscard on N3 so that any "row not found" errors during catch-up +# (from transactions whose effects are already in the COPY snapshot) are +# gracefully discarded instead of disabling the subscription. +psql_or_bail(3, "ALTER SYSTEM SET spock.exception_behaviour = 'transdiscard'"); +psql_or_bail(3, "SELECT pg_reload_conf()"); + +# Drain replication backlog before attach_node so apply workers are less busy +# during slot creation, increasing the chance of an idle inter-commit gap. +print STDERR "Draining replication before attach_node ...\n"; +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); + +psql_or_bail(3, + "CALL spock.attach_node(src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user', + new_node_name := 'n3', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user', + verb := false);"); + +# Wait for replication to stabilize after attach_node before checking. +# pgbench is still running so lag won't reach zero; just drain current backlog. +print STDERR "Waiting for replication to settle after attach_node ...\n"; +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +sleep(5); + +# Ensure that pgbench load lasts longer than the Z0DAN protocol. +my $pid = $pgbench_handle1->{KIDS}[0]{PID}; +my $alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N1 still exists"); +$pid = $pgbench_handle2->{KIDS}[0]{PID}; +$alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N2 still exists"); + +print STDERR "Kill pgbench process to reduce test time\n"; +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); +$pgbench_handle1->kill_kill; +$pgbench_handle2->kill_kill; + +print STDERR "Check if pgbench finalised correctly\n"; +$pgbench_handle1->finish; +$pgbench_handle2->finish; +print STDERR "##### output of pgbench #####\n"; +print STDERR $pgbench_stdout1; +print STDERR $pgbench_stdout2; +print STDERR "##### end of output #####\n"; + +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); + +print STDERR "Wait until the end of replication ..\n"; +$lag = scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N2 => N1 has been finished successfully"); +$lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N1 => N2 has been finished successfully"); +$lag = scalar_query(3, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N1 => N3 has been finished successfully"); +$lag = scalar_query(3, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N2 => N3 has been finished successfully"); + +print STDERR "Check the data consistency.\n"; +$ret1 = scalar_query(1, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N1's pgbench_accounts aggregates: $ret1\n"; +$ret2 = scalar_query(2, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N2's pgbench_accounts aggregates: $ret2\n"; +$ret3 = scalar_query(3, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); + +print STDERR "The N3's pgbench_accounts aggregates: $ret3\n"; +ok($ret1 eq $ret3, "Equality of the data on N1 and N3 is confirmed"); +ok($ret2 eq $ret3, "Equality of the data on N2 and N3 is confirmed"); + +# Before we finish this test and destroy the cluster, we need to ensure that +# the nodes are not stuck in some work. N1 and N2 have done their job and are +# ready to switch off. However, after receiving a large amount of WAL during +# catch-up, N3 may be decoding WAL to determine a proper LSN for N3->N1 and +# N3->N2 replication. +# Being in this process, the walsender doesn't send anything valuable except +# a 'keepalive' message. Hence, we can't clearly detect the end of the process. +# So, nudge it, employing the sync_event machinery. +psql_or_bail(3, "SELECT spock.sync_event()"); +print STDERR "Wait for the end of N3->N1, N3->N2 decoding process that means the actual start of LR\n"; +psql_or_bail(3, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); + +# Nothing sensitive should be awaited here, just to be sure. +$lag = scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n3', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N3 => N1 has been finished successfully"); +$lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n3', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N3 => N2 has been finished successfully"); +# 2n congiguration. With non-intersecting load we don't anticipate any issues +# with this test. It is written to prepare infrastructure and for demonstration +# purposes. +# +# ############################################################################## + +psql_or_bail(3, "CALL spock.detach_node( + target_node_name := 'n3', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user', + verbose_mode := true)"); +system_or_bail "$pg_bin/pgbench", '-i', '-I', 'd', '-h', $host, '-p', $node_ports->[2], '-U', $db_user, $dbname; +psql_or_bail(3, 'DROP FUNCTION wait_subscription'); +psql_or_bail(3, 'VACUUM FULL'); + +# Let the cluster settle after detach_node before starting the next cycle. +print STDERR "Waiting for cluster to settle after detach_node ...\n"; +scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + timeout := '3 minutes', delay := 0.5)"); + +# To improve TPS +psql_or_bail(1, "CREATE UNIQUE INDEX ON pgbench_accounts(abs(aid))"); +$lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Wait replication of the CREATE INDEX"); + +# Create non-intersecting load for nodes N1 and N2. +# Test duration should be enough to cover all the Z0DAN stages. We will kill +# pgbench immediately after the N3 is attached. +$load1 = '../../samples/Z0DAN/n1_1.pgb'; +$load2 = '../../samples/Z0DAN/n2_1.pgb'; +$pgbench_stdout1=''; +$pgbench_stderr1=''; +$pgbench_stdout2=''; +$pgbench_stderr2=''; +$pgbench_handle1 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load1, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[0], '-U', $db_user, $dbname], + '>', \$pgbench_stdout1, '2>', \$pgbench_stderr1); +$pgbench_handle2 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load2, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[1], '-U', $db_user, $dbname], + '>', \$pgbench_stdout2, '2>', \$pgbench_stderr2); +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); + +# Warming up ... +print STDERR "warming up pgbench for 30s\n"; +sleep(30); +print STDERR "done warmup\n"; + +# Ensure that pgbench load lasts longer than the Z0DAN protocol. +$pid = $pgbench_handle1->{KIDS}[0]{PID}; +$alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N1 still exists"); +$pid = $pgbench_handle2->{KIDS}[0]{PID}; +$alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N2 still exists"); + +print STDERR "Kill pgbench process to reduce test time\n"; +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); +$pgbench_handle1->kill_kill; +$pgbench_handle2->kill_kill; + +print STDERR "Check if pgbench finalised correctly\n"; +$pgbench_handle1->finish; +$pgbench_handle2->finish; +print STDERR "##### output of pgbench #####\n"; +print STDERR "$pgbench_stdout1"; +print STDERR "$pgbench_stdout2"; +print STDERR "##### end of output #####\n"; + +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); + +print STDERR "Wait until the end of replication ..\n"; +$lag = scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N2 => N1 has been finished successfully"); +$lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N1 => N2 has been finished successfully"); + +print STDERR "Check the data consistency.\n"; +$ret1 = scalar_query(1, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N1's pgbench_accounts aggregates: $ret1\n"; +$ret2 = scalar_query(2, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N2's pgbench_accounts aggregates: $ret2\n"; + +ok($ret1 eq $ret2, "Equality of the data on N1 and N2 is confirmed"); + +# ############################################################################## +# +# Try to update an IDENTITY column in case of 3n configuration. +# It works precisely like the previous one, but node 3 should sync its state +# with loaded nodes in real time under the pgbench load. +# Here, we also inderectly test how the Z0DAN add/remove protocol works in case +# of multiple adding cycles. +# +# ############################################################################## + +$pgbench_stdout1=''; +$pgbench_stderr1=''; +$pgbench_stdout2=''; +$pgbench_stderr2=''; +$pgbench_handle1 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load1, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[0], '-U', $db_user, $dbname], + '>', \$pgbench_stdout1, '2>', \$pgbench_stderr1); +$pgbench_handle2 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load2, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[1], '-U', $db_user, $dbname], + '>', \$pgbench_stdout2, '2>', \$pgbench_stderr2); +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); + +# Warming up ... +print STDERR "warming up pgbench for 30s\n"; +sleep(30); +print STDERR "done warmup\n"; + +print STDERR "Add N3 into highly loaded configuration of N1 and N2 ..."; +psql_or_bail(3, "ALTER SYSTEM SET spock.exception_behaviour = 'transdiscard'"); +psql_or_bail(3, "SELECT pg_reload_conf()"); + +# Drain replication backlog before second attach_node. +print STDERR "Draining replication before second attach_node ...\n"; +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); + +psql_or_bail(3, + "CALL spock.attach_node(src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user', + new_node_name := 'n3', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user', + verb := false);"); + +# Wait for replication to stabilize after second attach_node. +# pgbench is still running so lag won't reach zero; just drain current backlog. +print STDERR "Waiting for replication to settle after attach_node ...\n"; +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +sleep(5); + +# Ensure that pgbench load lasts longer than the Z0DAN protocol. +$pid = $pgbench_handle1->{KIDS}[0]{PID}; +$alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N1 still exists"); +$pid = $pgbench_handle2->{KIDS}[0]{PID}; +$alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N2 still exists"); + +print STDERR "Kill pgbench process to reduce test time\n"; +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); +$pgbench_handle1->kill_kill; +$pgbench_handle2->kill_kill; + +print STDERR "Check if pgbench finalised correctly\n"; +$pgbench_handle1->finish; +$pgbench_handle2->finish; +print STDERR "##### output of pgbench #####\n"; +print STDERR $pgbench_stdout1; +print STDERR $pgbench_stdout2; +print STDERR "##### end of output #####\n"; + +# +# Wait for the end of apply process +# +print STDERR "Wait for the end of LR caused by the pgbench load\n"; +$lsn1 = scalar_query(1, "SELECT spock.sync_event()"); +$lsn2 = scalar_query(2, "SELECT spock.sync_event()"); +$lsn3 = scalar_query(3, "SELECT spock.sync_event()"); +print STDERR "DEBUGGING. LSNs: N1: $lsn1, N2: $lsn2, N3: $lsn3\n"; + +print STDERR "Wait for the N2 -> N1 sync message ...\n"; +psql_or_bail(1, "CALL spock.wait_for_sync_event(true, 'n2', '$lsn2'::pg_lsn, 1200, true)"); +print STDERR "Wait for the N1 -> N2 sync message ...\n"; +psql_or_bail(2, "CALL spock.wait_for_sync_event(true, 'n1', '$lsn1'::pg_lsn, 1200, true)"); +print STDERR "Wait for the N1 -> N3 sync message ...\n"; +psql_or_bail(3, "CALL spock.wait_for_sync_event(true, 'n1', '$lsn1'::pg_lsn, 1200, true)"); +print STDERR "Wait for the N2 -> N3 sync message ...\n"; +psql_or_bail(3, "CALL spock.wait_for_sync_event(true, 'n2', '$lsn2'::pg_lsn, 1200, true)"); +print STDERR "LR messages from active nodes has arrived to the new one\n"; + +print STDERR "Wait for the N3 -> N1 sync message ...\n"; +psql_or_bail(1, "CALL spock.wait_for_sync_event(true, 'n3', '$lsn3'::pg_lsn, 1200, true)"); +print STDERR "Wait for the N3 -> N2 sync message ...\n"; +psql_or_bail(2, "CALL spock.wait_for_sync_event(true, 'n3', '$lsn3'::pg_lsn, 1200, true)"); +print STDERR "First LR transaction has arrived from new node to the active ones\n"; + +# Wait for all replication directions to fully catch up. +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(3, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n3', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n3', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(3, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(3, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + timeout := '3 minutes', delay := 0.5)"); + +print STDERR "Check the data consistency.\n"; +$ret1 = scalar_query(1, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N1's pgbench_accounts aggregates: $ret1\n"; +$ret2 = scalar_query(2, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N2's pgbench_accounts aggregates: $ret2\n"; +$ret3 = scalar_query(3, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N3's pgbench_accounts aggregates: $ret3\n"; + +ok($ret1 eq $ret2, "Equality of the data on N1 and N2 is confirmed"); +ok($ret1 eq $ret3, "Equality of the data on N1 and N3 is confirmed"); + +# Cleanup will be handled by SpockTest.pm END block +# No need for done_testing() when using a test plan diff --git a/tests/tap/t/036_attach_node_basics_incore.pl b/tests/tap/t/036_attach_node_basics_incore.pl new file mode 100755 index 00000000..5a385ff7 --- /dev/null +++ b/tests/tap/t/036_attach_node_basics_incore.pl @@ -0,0 +1,119 @@ +use strict; +use warnings; +use Test::More; +use lib '.'; +use lib 't'; +use SpockTest qw(create_cluster destroy_cluster get_test_config psql_or_bail scalar_query); + +my ($result); + +create_cluster(3, 'Create basic Spock test cluster'); + +# Get cluster configuration +my $config = get_test_config(); +my $node_count = $config->{node_count}; +my $node_ports = $config->{node_ports}; +my $host = $config->{host}; +my $dbname = $config->{db_name}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +psql_or_bail(2, "SELECT spock.node_drop('n2')"); +psql_or_bail(3, "SELECT spock.node_drop('n3')"); +psql_or_bail(1, "CREATE EXTENSION amcheck"); +psql_or_bail(1, "CREATE TABLE test(x serial PRIMARY KEY)"); +psql_or_bail(1, "INSERT INTO test DEFAULT VALUES"); + +print STDERR "All supporting stuff has been installed successfully\n"; + +# ############################################################################## +# +# Basic check that Z0DAN correctly add node to the single-node cluster +# +# ############################################################################## + +print STDERR "Call Z0DAN: n2 => n1\n"; +psql_or_bail(2, " + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n2', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verb := false + )"); +print STDERR "Z0DAN (n2 => n1) has finished the attach process\n"; +$result = scalar_query(2, "SELECT x FROM test"); +print STDERR "Check result: $result\n"; +ok($result eq '1', "Check state of the test table after the attachment"); + +psql_or_bail(1, "SELECT spock.sub_disable('sub_n2_n1')"); + +# ############################################################################## +# +# Z0DAN reject node addition if some subscriptions are disabled +# +# ############################################################################## + +print STDERR "Call Z0DAN: n3 => n2\n"; +scalar_query(3, " + CALL spock.attach_node( + src_node_name := 'n2', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + new_node_name := 'n3', new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := false)"); + +$result = scalar_query(3, "SELECT count(*) FROM spock.local_node"); +ok($result eq '0', "N3 is not in the cluster yet"); +print STDERR "Z0DAN should fail because of a disabled subscription\n"; + +psql_or_bail(1, "SELECT spock.sub_enable('sub_n2_n1')"); +psql_or_bail(3, " + CALL spock.attach_node( + src_node_name := 'n2', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + new_node_name := 'n3', new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := true)"); + +$result = scalar_query(3, "SELECT count(*) FROM spock.local_node"); +ok($result eq '1', "N3 is in the cluster"); +$result = scalar_query(3, "SELECT x FROM test"); +print STDERR "Check result: $result\n"; +ok($result eq '1', "Check state of the test table on N3 after the attachment"); +print STDERR "Z0DAN should add N3 to the cluster\n"; + +# ############################################################################## +# +# Test that Z0DAN correctly doesn't add node to the cluster if something happens +# during the SYNC process. +# +# ############################################################################## + +# Remove node from the cluster and data leftovers. +psql_or_bail(3, "CALL spock.detach_node(target_node_name := 'n3', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verbose_mode := true)"); +psql_or_bail(3, "DROP TABLE test"); + +psql_or_bail(1, "CREATE FUNCTION fake_fn() RETURNS integer LANGUAGE sql AS \$\$ SELECT 1\$\$"); +psql_or_bail(3, "CREATE FUNCTION fake_fn() RETURNS integer LANGUAGE sql AS \$\$ SELECT 1\$\$"); +scalar_query(3, " + CALL spock.attach_node( + src_node_name := 'n2', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + new_node_name := 'n3', new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := true)"); + +# TODO: +# It seems that attach_node keeps remnants after unsuccessful execution. It is +# happened because we have commited some intermediate results before. +# It would be better to keep remote transaction opened until the end of the +# operation or just remove these remnants at the end pretending to be a +# distributed transaction. +# +# $result = scalar_query(3, "SELECT count(*) FROM spock.local_node"); +# ok($result eq '0', "N3 is not in the cluster"); + +# Clean up +destroy_cluster('Destroy test cluster'); +done_testing(); diff --git a/tests/tap/t/037_attach_node_3n_timeout_incore.pl b/tests/tap/t/037_attach_node_3n_timeout_incore.pl new file mode 100755 index 00000000..66567b49 --- /dev/null +++ b/tests/tap/t/037_attach_node_3n_timeout_incore.pl @@ -0,0 +1,197 @@ +use strict; +use warnings; +use Test::More; +use lib '.'; +use lib 't'; +use SpockTest qw(create_cluster destroy_cluster get_test_config psql_or_bail scalar_query); +use Time::HiRes qw(time); + +# ============================================================================= +# Test: Verify that attach_node fails immediately with an error when sync_event +# function is missing on the source node, rather than waiting for timeout. +# ============================================================================= +# attach_node raises an error (rather than looping until timeout) when a required +# helper such as sync_event is missing on the source node. The test: +# 1. Creates a 3-node cluster (n1, n2 active; n3 dropped and re-added) +# 2. Drops n3 from the cluster +# 3. Renames spock.sync_event() on node 1 to simulate it being missing +# 4. Attempts attach_node from n3 to join the cluster +# 5. Verifies the call fails quickly with an error (not timeout) + +create_cluster(3, 'Create 3-node Spock test cluster'); + +# Get cluster configuration +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $host = $config->{host}; +my $dbname = $config->{db_name}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +# Prepare node 2: drop from cluster (attach_node/detach_node ship with the extension) +print STDERR "Prepare N2: drop local node\n"; +psql_or_bail(2, "SELECT spock.node_drop('n2')"); +psql_or_bail(3, "SELECT spock.node_drop('n3')"); + +# Rename sync_event function on node 1 to simulate it being missing +print STDERR "Rename spock.sync_event() on N1 to simulate missing function\n"; +psql_or_bail(1, "ALTER FUNCTION spock.sync_event(boolean) RENAME TO sync_event_renamed"); + +# Attempt attach_node - this should fail quickly with an error, not timeout +print STDERR "Attempt attach_node from N2 to N1 (should fail quickly with error)\n"; +my $start_time = time(); + +# scalar_query uses backticks which don't throw exceptions - check $? for exit code +my $result = scalar_query(2, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n2', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verb := false, + timeout_sec := 30 + )}); +my $exit_code = $? >> 8; + +my $elapsed_time = time() - $start_time; +print STDERR "attach_node call completed in $elapsed_time seconds (exit code: $exit_code)\n"; + +# The call should fail quickly, well under the 30s timeout_sec above. The bound +# must stay safely below that timeout, otherwise a call that loops to the full +# timeout would still pass this assertion and defeat its purpose. +ok($elapsed_time < 25, "attach_node failed quickly (${elapsed_time}s < 25s), not waiting for timeout"); +ok($exit_code != 0, "attach_node failed as expected when sync_event is missing (exit code: $exit_code)"); + +# Restore sync_event function on N1 for cleanup +print STDERR "Restore spock.sync_event() on N1\n"; +psql_or_bail(1, "ALTER FUNCTION spock.sync_event_renamed(boolean) RENAME TO sync_event"); + +# Clean leftovers in the Spock cluster caused by unsuccessful addition +scalar_query(2, qq{ + CALL spock.detach_node( + target_node_name := 'n2', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verbose_mode := true) +}); + +print STDERR "Check: an error during pg_replication_slot_advance should stop node addition\n"; + +psql_or_bail(2, "ALTER FUNCTION pg_replication_slot_advance RENAME TO pg_replication_slot_advance_renamed"); + +# Should be OK, because of 2-n configuration, no advance needed. This join is +# expected to succeed and can take tens of seconds, so it must not be inside the +# window the "failed quickly" assertion below measures. +psql_or_bail(2, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n2', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verb := false + )}); + +# Time only the n3 attach, which is the call expected to fail quickly. +$start_time = time(); + +# Should fail quickly +$result = scalar_query(3, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n3', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := false, + timeout_sec := 30 + )}); +$exit_code = $? >> 8; + +$elapsed_time = time() - $start_time; +print STDERR "attach_node call completed in $elapsed_time seconds (exit code: $exit_code)\n"; + +ok($elapsed_time < 25, "attach_node on n3 failed quickly (${elapsed_time}s < 25s), not waiting for timeout"); +ok($exit_code != 0, "attach_node failed as expected when pg_replication_slot_advance is missing (exit code: $exit_code)"); + +psql_or_bail(2, "ALTER FUNCTION pg_replication_slot_advance_renamed RENAME TO pg_replication_slot_advance"); + +# Clean leftovers of node-3 in the Spock cluster caused by unsuccessful addition +scalar_query(3, qq{ + CALL spock.detach_node( + target_node_name := 'n3', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verbose_mode := true) +}); + +print STDERR "Check: quick fail if something happens during subscription creation on source node\n"; + +psql_or_bail(1, "ALTER FUNCTION spock.sub_create RENAME TO sub_create_renamed"); +$start_time = time(); + +$result = scalar_query(3, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n3', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := false, + timeout_sec := 30 + )}); +$exit_code = $? >> 8; + +$elapsed_time = time() - $start_time; +print STDERR "attach_node call completed in $elapsed_time seconds (exit code: $exit_code)\n"; + +ok($elapsed_time < 25, "attach_node on n3 failed quickly (${elapsed_time}s < 25s), not waiting for timeout"); +ok($exit_code != 0, "attach_node failed as expected when sub_create is missing (exit code: $exit_code)"); + +psql_or_bail(1, "ALTER FUNCTION spock.sub_create_renamed RENAME TO sub_create"); + +# Clean leftovers of node-3 in the Spock cluster caused by unsuccessful addition +scalar_query(2, qq{ + CALL spock.detach_node( + target_node_name := 'n2', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verbose_mode := true) +}); +scalar_query(3, qq{ + CALL spock.detach_node( + target_node_name := 'n3', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verbose_mode := true) +}); + +print STDERR "Final check: node-3 adds to the cluster successfully\n"; + +psql_or_bail(2, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n2', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verb := false + )}); +psql_or_bail(3, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n3', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := false + )}); + +# Verify cluster has 3 nodes on each node +for my $node (1, 2, 3) { + my $node_count = scalar_query($node, "SELECT count(*) FROM spock.node"); + ok($node_count == 3, "Node $node sees 3 nodes in cluster (got $node_count)"); +} + +# Verify each node has 2 non-disabled subscriptions +for my $node (1, 2, 3) { + my $sub_count = scalar_query($node, + "SELECT count(*) FROM spock.subscription WHERE sub_enabled = true"); + ok($sub_count == 2, "Node $node has 2 enabled subscriptions (got $sub_count)"); +} + +# Cleanup will be handled by SpockTest.pm END block +destroy_cluster('Destroy test cluster'); +done_testing();