From ecf27cb5980301ba33a861e8e74f8062fc735ef7 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 29 Jul 2026 11:59:38 +0000 Subject: [PATCH 01/12] fix(ENGKNOW-3691): parse rda db source from APPSERVER_RDA_* environment variables --- .../gorpipe/gor/model/DbConnectionCache.java | 75 ++++++++++++++++++ .../org/gorpipe/gor/model/UTestDbSource.java | 78 +++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java index f70db082..5ffa42dd 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java @@ -27,6 +27,12 @@ public class DbConnectionCache { private final ConcurrentHashMap mapSources = new ConcurrentHashMap<>(); public String defaultDbSource = "rda"; + static final String ENV_RDA_SOURCE_NAME = "rda"; + static final String ENV_RDA_URL = "APPSERVER_RDA_URL"; + static final String ENV_RDA_USERNAME = "APPSERVER_RDA_USERNAME"; + static final String ENV_RDA_PASSWORD = "APPSERVER_RDA_PASSWORD"; + static final String ENV_RDA_DRIVER = "APPSERVER_RDA_DRIVER"; + public DbConnectionCache() { } @@ -111,6 +117,75 @@ public static List parseLinesForDbSourceInstallation(String credpath, return partsList; } + /** + * Build db source definitions from environment variables. + * + * Returns entries in the same {name, driver, url, user[, pwd]} shape as + * parseLinesForDbSourceInstallation, so both paths share the install code. + * Never logs credential values, only variable names. + * + * @param env the environment to read, normally System.getenv() + * @return zero or one db source definition + */ + public static List parseEnvForDbSourceInstallation(Map env) { + List partsList = new ArrayList<>(); + if (env == null) { + return partsList; + } + + String url = trimToNull(env.get(ENV_RDA_URL)); + String user = trimToNull(env.get(ENV_RDA_USERNAME)); + String pwd = env.get(ENV_RDA_PASSWORD); + + if (url == null && user == null) { + return partsList; + } + if (url == null || user == null) { + log.warn("Incomplete db source configuration in environment for source {}: {} is not set. Ignoring it.", + ENV_RDA_SOURCE_NAME, url == null ? ENV_RDA_URL : ENV_RDA_USERNAME); + return partsList; + } + + String driver = trimToNull(env.get(ENV_RDA_DRIVER)); + if (driver == null) { + driver = driverClassForUrl(url); + } + if (driver == null) { + log.warn("Could not derive a jdbc driver for db source {} from its url, and {} is not set. Ignoring it.", + ENV_RDA_SOURCE_NAME, ENV_RDA_DRIVER); + return partsList; + } + + if (pwd == null) { + partsList.add(new String[]{ENV_RDA_SOURCE_NAME, driver, url, user}); + } else { + partsList.add(new String[]{ENV_RDA_SOURCE_NAME, driver, url, user, pwd}); + } + return partsList; + } + + /** + * @param url a jdbc url + * @return the matching driver class name, or null if the prefix is not recognized + */ + public static String driverClassForUrl(String url) { + if (url.startsWith("jdbc:postgresql:")) { + return "org.postgresql.Driver"; + } + if (url.startsWith("jdbc:oracle:")) { + return "oracle.jdbc.driver.OracleDriver"; + } + return null; + } + + private static String trimToNull(String value) { + if (value == null) { + return null; + } + String trimmed = value.trim(); + return trimmed.isEmpty() ? null : trimmed; + } + private void installDbSourceFromParts(String[] parts) throws ClassNotFoundException { Class.forName(parts[1]); // Just load the driver once and for all final DbConnection source = new DbConnection(parts[0], parts[2], parts[3], parts.length > 4 ? parts[4] : null); diff --git a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java index bc199e6d..48013ed0 100644 --- a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java +++ b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java @@ -26,7 +26,9 @@ import org.junit.Test; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class UTestDbSource { @@ -71,4 +73,80 @@ public void parseLinesForDbSourceInstallationWithRealTabsAndNewlinesAndInvalidLi Assert.assertEquals(1, partsList.size()); Assert.assertEquals(5, partsList.get(0).length); } + + private static Map rdaEnv(String url, String username, String password) { + Map env = new HashMap<>(); + if (url != null) env.put("APPSERVER_RDA_URL", url); + if (username != null) env.put("APPSERVER_RDA_USERNAME", username); + if (password != null) env.put("APPSERVER_RDA_PASSWORD", password); + return env; + } + + @Test + public void parseEnvBuildsRdaPartsInFileParserShape() { + Map env = rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret"); + List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); + Assert.assertEquals(1, partsList.size()); + String[] parts = partsList.get(0); + Assert.assertEquals(5, parts.length); + Assert.assertEquals("rda", parts[0]); + Assert.assertEquals("org.postgresql.Driver", parts[1]); + Assert.assertEquals("jdbc:postgresql://db:5432/csa", parts[2]); + Assert.assertEquals("gregor_reader", parts[3]); + Assert.assertEquals("secret", parts[4]); + } + + @Test + public void parseEnvOmitsPasswordFieldWhenPasswordUnset() { + Map env = rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", null); + List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); + Assert.assertEquals(1, partsList.size()); + Assert.assertEquals(4, partsList.get(0).length); + } + + @Test + public void parseEnvReturnsNothingWhenNoRdaVarsPresent() { + Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(new HashMap<>()).isEmpty()); + } + + @Test + public void parseEnvSkipsSourceWhenUsernameMissing() { + Map env = rdaEnv("jdbc:postgresql://db:5432/csa", null, "secret"); + Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(env).isEmpty()); + } + + @Test + public void parseEnvSkipsSourceWhenUrlMissing() { + Map env = rdaEnv(null, "gregor_reader", "secret"); + Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(env).isEmpty()); + } + + @Test + public void parseEnvTreatsBlankValuesAsUnset() { + Map env = rdaEnv("jdbc:postgresql://db:5432/csa", " ", "secret"); + Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(env).isEmpty()); + } + + @Test + public void parseEnvDerivesOracleDriverFromUrlPrefix() { + Map env = rdaEnv("jdbc:oracle:thin:@db:1521:XE", "gregor_reader", "secret"); + List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); + Assert.assertEquals(1, partsList.size()); + Assert.assertEquals("oracle.jdbc.driver.OracleDriver", partsList.get(0)[1]); + } + + @Test + public void parseEnvHonoursExplicitDriverOverride() { + Map env = rdaEnv("jdbc:whatever://db/x", "gregor_reader", "secret"); + env.put("APPSERVER_RDA_DRIVER", "com.example.Driver"); + List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); + Assert.assertEquals(1, partsList.size()); + Assert.assertEquals("com.example.Driver", partsList.get(0)[1]); + } + + @Test + public void parseEnvSkipsSourceWhenDriverCannotBeDerived() { + Map env = rdaEnv("jdbc:whatever://db/x", "gregor_reader", "secret"); + Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(env).isEmpty()); + } } \ No newline at end of file From 187369391253125fbdd8d1ac8ca31f1618c6899d Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 29 Jul 2026 12:05:50 +0000 Subject: [PATCH 02/12] fix(ENGKNOW-3691): install env db sources before file sources, tolerate missing file --- .../gorpipe/gor/model/DbConnectionCache.java | 58 ++++++++---- .../org/gorpipe/gor/model/UTestDbSource.java | 94 +++++++++++++++++++ 2 files changed, 136 insertions(+), 16 deletions(-) diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java index 5ffa42dd..c5359ac1 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java @@ -12,6 +12,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -59,27 +60,52 @@ public DbConnection lookup(String source) { */ @SuppressWarnings("WeakerAccess") // Used from gor-services public void initializeDbSources(String credpath) throws IOException { + initializeDbSources(credpath, System.getenv()); + } + + /** + * Read database sources from the environment and from the configuration file. + * + * Env-defined sources are installed first so that a file row with the same name + * takes precedence over it. + * + * @param credpath The path to the configuration file + * @param env The environment to read env-defined sources from + */ + void initializeDbSources(String credpath, Map env) throws IOException { clearDbSources(); - if (credpath != null && credpath.trim().length() > 0) { - final Path path = Paths.get(credpath); - if (Files.notExists(path)) { - throw new FileNotFoundException("Specified db credentials file (" + credpath + ") is not found"); - } + List envParts = parseEnvForDbSourceInstallation(env); + installAllFromParts(envParts); + installAllFromParts(readFileForDbSourceInstallation(credpath, !envParts.isEmpty())); + } - final List lines = Files.readAllLines(path, StandardCharsets.UTF_8); + private List readFileForDbSourceInstallation(String credpath, boolean haveEnvSources) throws IOException { + if (credpath == null || credpath.trim().length() == 0) { + log.info("No db credential path specified"); + return Collections.emptyList(); + } + + final Path path = Paths.get(credpath); + if (Files.notExists(path)) { + if (haveEnvSources) { + log.warn("Specified db credentials file ({}) is not found, continuing with db sources from the environment", credpath); + return Collections.emptyList(); + } + throw new FileNotFoundException("Specified db credentials file (" + credpath + ") is not found"); + } - List partsList = parseLinesForDbSourceInstallation(credpath, lines); + final List lines = Files.readAllLines(path, StandardCharsets.UTF_8); + return parseLinesForDbSourceInstallation(credpath, lines); + } - for (String[] parts : partsList) { - try { - installDbSourceFromParts(parts); - } catch (ClassNotFoundException e) { - log.error("Failed to load driver class {} for db source {}. Please ensure the driver is in the classpath.", - parts[1], parts[0], e); - } + private void installAllFromParts(List partsList) { + for (String[] parts : partsList) { + try { + installDbSourceFromParts(parts); + } catch (ClassNotFoundException e) { + log.error("Failed to load driver class {} for db source {}. Please ensure the driver is in the classpath.", + parts[1], parts[0], e); } - } else { - log.info("No db credential path specified"); } } diff --git a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java index 48013ed0..870694cd 100644 --- a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java +++ b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java @@ -23,8 +23,14 @@ package org.gorpipe.gor.model; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.File; +import java.io.FileNotFoundException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -149,4 +155,92 @@ public void parseEnvSkipsSourceWhenDriverCannotBeDerived() { Map env = rdaEnv("jdbc:whatever://db/x", "gregor_reader", "secret"); Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(env).isEmpty()); } + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private String writeCredentialsFile(String... lines) throws Exception { + File file = tempFolder.newFile("gor.db.credentials"); + Files.write(file.toPath(), String.join("\n", lines).getBytes(StandardCharsets.UTF_8)); + return file.getAbsolutePath(); + } + + @Test + public void envOnlyInstallsRdaSource() throws Exception { + DbConnectionCache cache = new DbConnectionCache(); + cache.initializeDbSources(null, rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret")); + + DbConnection rda = cache.lookup("rda"); + Assert.assertNotNull("rda source should be installed from the environment", rda); + Assert.assertEquals("jdbc:postgresql://db:5432/csa", rda.url); + Assert.assertEquals("gregor_reader", rda.user); + Assert.assertEquals("secret", rda.pwd); + } + + @Test + public void fileRowOverridesEnvSource() throws Exception { + String credpath = writeCredentialsFile( + "name\tdriver\turl\tuser\tpwd", + "rda\torg.postgresql.Driver\tjdbc:postgresql://filehost:5432/csa\tfileuser\tfilepwd"); + + DbConnectionCache cache = new DbConnectionCache(); + cache.initializeDbSources(credpath, rdaEnv("jdbc:postgresql://envhost:5432/csa", "envuser", "envpwd")); + + DbConnection rda = cache.lookup("rda"); + Assert.assertNotNull(rda); + Assert.assertEquals("jdbc:postgresql://filehost:5432/csa", rda.url); + Assert.assertEquals("fileuser", rda.user); + } + + @Test + public void fileSuppliesAuxiliarySourceAlongsideEnv() throws Exception { + String credpath = writeCredentialsFile( + "name\tdriver\turl\tuser\tpwd", + "aux\torg.postgresql.Driver\tjdbc:postgresql://auxhost:5432/aux\tauxuser\tauxpwd"); + + DbConnectionCache cache = new DbConnectionCache(); + cache.initializeDbSources(credpath, rdaEnv("jdbc:postgresql://envhost:5432/csa", "envuser", "envpwd")); + + Assert.assertNotNull("env source should survive", cache.lookup("rda")); + Assert.assertEquals("jdbc:postgresql://envhost:5432/csa", cache.lookup("rda").url); + Assert.assertNotNull("file source should also be installed", cache.lookup("aux")); + Assert.assertEquals("auxuser", cache.lookup("aux").user); + } + + @Test + public void missingFileToleratedWhenEnvSourcePresent() throws Exception { + String missing = new File(tempFolder.getRoot(), "does-not-exist.credentials").getAbsolutePath(); + + DbConnectionCache cache = new DbConnectionCache(); + cache.initializeDbSources(missing, rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret")); + + Assert.assertNotNull(cache.lookup("rda")); + } + + @Test(expected = FileNotFoundException.class) + public void missingFileStillThrowsWithoutEnvSources() throws Exception { + String missing = new File(tempFolder.getRoot(), "does-not-exist.credentials").getAbsolutePath(); + new DbConnectionCache().initializeDbSources(missing, new HashMap<>()); + } + + @Test + public void partialEnvSkipsSourceWithoutThrowing() throws Exception { + DbConnectionCache cache = new DbConnectionCache(); + cache.initializeDbSources(null, rdaEnv("jdbc:postgresql://db:5432/csa", null, "secret")); + + Assert.assertNull(cache.lookup("rda")); + } + + @Test + public void fileOnlyBehaviourIsUnchanged() throws Exception { + String credpath = writeCredentialsFile( + "name\tdriver\turl\tuser\tpwd", + "rda\torg.postgresql.Driver\tjdbc:postgresql://filehost:5432/csa\tfileuser\tfilepwd"); + + DbConnectionCache cache = new DbConnectionCache(); + cache.initializeDbSources(credpath, new HashMap<>()); + + Assert.assertNotNull(cache.lookup("rda")); + Assert.assertEquals("fileuser", cache.lookup("rda").user); + } } \ No newline at end of file From f27262505f180e2dacee7635472ee13070a9c695 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 29 Jul 2026 13:15:11 +0000 Subject: [PATCH 03/12] fix(ENGKNOW-3691): narrow rda env-source helpers to package-private The design spec calls for no public signature changes beyond the new package-private overload of initializeDbSources. parseEnvForDbSourceInstallation and driverClassForUrl were left public despite having no callers outside org.gorpipe.gor.model (only DbConnectionCache itself and UTestDbSource, both in-package, use them). Drop the public modifier on both, keeping them static; parseLinesForDbSourceInstallation is untouched since it is genuine public API used elsewhere. Co-Authored-By: Claude Opus 5 (1M context) --- .../main/java/org/gorpipe/gor/model/DbConnectionCache.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java index c5359ac1..bf7eee7b 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java @@ -153,7 +153,7 @@ public static List parseLinesForDbSourceInstallation(String credpath, * @param env the environment to read, normally System.getenv() * @return zero or one db source definition */ - public static List parseEnvForDbSourceInstallation(Map env) { + static List parseEnvForDbSourceInstallation(Map env) { List partsList = new ArrayList<>(); if (env == null) { return partsList; @@ -194,7 +194,7 @@ public static List parseEnvForDbSourceInstallation(Map * @param url a jdbc url * @return the matching driver class name, or null if the prefix is not recognized */ - public static String driverClassForUrl(String url) { + static String driverClassForUrl(String url) { if (url.startsWith("jdbc:postgresql:")) { return "org.postgresql.Driver"; } From 78650f4b2206d8cf3585c5eba9bd55d2c353df2f Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 29 Jul 2026 14:41:43 +0000 Subject: [PATCH 04/12] fix(ENGKNOW-3691): bump version to 5.11.2 Co-Authored-By: Claude Opus 5 (1M context) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 32447ce3..a87467fb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.11.1 +5.11.2 From de813528ccc331c55a410afe9add8166e4e29c06 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 29 Jul 2026 15:18:46 +0000 Subject: [PATCH 05/12] fix(ENGKNOW-3691): load user connections from gor.db.credentials gor.sql.credentials is retired: gor.db.credentials now carries the additional db resources it was intended for. resolveSqlCredPath falls back to the db credentials path when gor.sql.credentials is unset, so userConnections keeps loading without it. Without this, dropping the property would silently empty userConnections on gor-worker and break every sql:// source. An explicitly configured gor.sql.credentials still wins, so existing setups are unaffected. Also documents which cache each credential source feeds: rotating credentials arrive through APPSERVER_RDA_* for systemConnections, while the credentials file supplies the additional databases reached through userConnections. Co-Authored-By: Claude Opus 5 (1M context) --- .../org/gorpipe/gor/model/DbConnection.java | 31 ++++++++++-- .../gorpipe/gor/model/DbConnectionCache.java | 17 +++++-- .../org/gorpipe/gor/model/UTestDbSource.java | 48 +++++++++++++++++++ 3 files changed, 87 insertions(+), 9 deletions(-) diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java index aa7c8d73..14253dc3 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java @@ -245,9 +245,7 @@ public static void initInConsoleApp() throws ClassNotFoundException, IOException final String dbCredpath = homeDbCredFile.exists() ? homeDbCredFile.getCanonicalPath() : System.getProperty("gor.db.credentials"); systemConnections.initializeDbSources(dbCredpath); - File homeSqlCredFile = new File(System.getProperty("user.home"), "gor.sql.credentials"); - final String sqlCredpath = homeSqlCredFile.exists() ? homeSqlCredFile.getCanonicalPath() : System.getProperty("gor.sql.credentials"); - userConnections.initializeDbSources(sqlCredpath); + userConnections.initializeDbSources(resolveSqlCredPath(dbCredpath)); setPasswordCallback(s -> { final Console console = System.console(); @@ -258,6 +256,11 @@ public static void initInConsoleApp() throws ClassNotFoundException, IOException /** * Initialize DbSource to be used in a server. * + * Both caches are fed from gor.db.credentials plus the environment. In a deployment that + * supplies rotating credentials through APPSERVER_RDA_*, those reach {@link #systemConnections} + * (used for db:// sources), while the credentials file supplies the additional databases + * reached through {@link #userConnections} (sql:// sources). + * * @throws ClassNotFoundException * @throws IOException */ @@ -268,12 +271,32 @@ public static void initInServer() throws ClassNotFoundException, IOException { systemConnections.initializeDbSources(dbCredpath); } - final String sqlCredpath = System.getProperty("gor.sql.credentials"); + final String sqlCredpath = resolveSqlCredPath(dbCredpath); if (sqlCredpath != null) { userConnections.initializeDbSources(sqlCredpath); } } + /** + * Resolve the credentials file backing {@link #userConnections}. + * + * The dedicated gor.sql.credentials source is deprecated: gor.db.credentials now carries the + * additional db resources it was intended for. An explicit gor.sql.credentials still wins where + * one is configured, so existing setups keep working; otherwise the db credentials are used for + * both caches. Returns null only when neither is configured. + * + * @param dbCredpath the already-resolved gor.db.credentials path, may be null + * @return the path to load user connections from, or null if none is configured + */ + static String resolveSqlCredPath(String dbCredpath) throws IOException { + File homeSqlCredFile = new File(System.getProperty("user.home"), "gor.sql.credentials"); + if (homeSqlCredFile.exists()) { + return homeSqlCredFile.getCanonicalPath(); + } + final String sqlCredpath = System.getProperty("gor.sql.credentials"); + return sqlCredpath != null ? sqlCredpath : dbCredpath; + } + /** * Register a callback object to perform password prompting when needed * diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java index bf7eee7b..2c631e15 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java @@ -52,11 +52,16 @@ public DbConnection lookup(String source) { } /** - * Read Database sources from configuration file + * Read database sources from the environment and from the configuration file. + * + * Sources may come from two places. Environment variables carry credentials that rotate and so + * cannot be baked into a file — see {@link #parseEnvForDbSourceInstallation}. The credentials + * file carries the static set of additional databases gor can reach. A deployment typically + * supplies the rotating system credentials through the environment and the remaining resources + * through the file. * * @param credpath The path to the configuration file - * @throws ClassNotFoundException - * @throws IOException + * @throws IOException if the credentials file is configured but missing, and no env source was installed */ @SuppressWarnings("WeakerAccess") // Used from gor-services public void initializeDbSources(String credpath) throws IOException { @@ -66,8 +71,10 @@ public void initializeDbSources(String credpath) throws IOException { /** * Read database sources from the environment and from the configuration file. * - * Env-defined sources are installed first so that a file row with the same name - * takes precedence over it. + * Env-defined sources are installed first so that a file row with the same name takes + * precedence over it. A deployment that wants the environment to be authoritative for a given + * source must therefore keep a row of that name out of the credentials file — otherwise the + * file's static copy shadows the rotating one. * * @param credpath The path to the configuration file * @param env The environment to read env-defined sources from diff --git a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java index 870694cd..a0972138 100644 --- a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java +++ b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java @@ -243,4 +243,52 @@ public void fileOnlyBehaviourIsUnchanged() throws Exception { Assert.assertNotNull(cache.lookup("rda")); Assert.assertEquals("fileuser", cache.lookup("rda").user); } + + /** + * Runs body with user.home pointed at an empty dir, so the ~/gor.sql.credentials branch + * of resolveSqlCredPath is deterministically absent, and with gor.sql.credentials set as given. + */ + private void withSqlCredProperties(String sqlCredProperty, ThrowingRunnable body) throws Exception { + String originalHome = System.getProperty("user.home"); + String originalSqlCred = System.getProperty("gor.sql.credentials"); + try { + System.setProperty("user.home", tempFolder.newFolder("home").getAbsolutePath()); + if (sqlCredProperty == null) { + System.clearProperty("gor.sql.credentials"); + } else { + System.setProperty("gor.sql.credentials", sqlCredProperty); + } + body.run(); + } finally { + System.setProperty("user.home", originalHome); + if (originalSqlCred == null) { + System.clearProperty("gor.sql.credentials"); + } else { + System.setProperty("gor.sql.credentials", originalSqlCred); + } + } + } + + private interface ThrowingRunnable { + void run() throws Exception; + } + + @Test + public void userConnectionsFallBackToDbCredentialsWhenSqlCredentialsUnset() throws Exception { + withSqlCredProperties(null, () -> + Assert.assertEquals("/app/config/gor.db.credentials", + DbConnection.resolveSqlCredPath("/app/config/gor.db.credentials"))); + } + + @Test + public void explicitSqlCredentialsStillWinsOverDbCredentials() throws Exception { + withSqlCredProperties("/custom/gor.sql.credentials", () -> + Assert.assertEquals("/custom/gor.sql.credentials", + DbConnection.resolveSqlCredPath("/app/config/gor.db.credentials"))); + } + + @Test + public void sqlCredPathIsNullWhenNeitherIsConfigured() throws Exception { + withSqlCredProperties(null, () -> Assert.assertNull(DbConnection.resolveSqlCredPath(null))); + } } \ No newline at end of file From d8c23340e104f62a074863ea3f3d8481157b7d90 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 29 Jul 2026 16:03:10 +0000 Subject: [PATCH 06/12] fix(ENGKNOW-3691): remove gor.sql.credentials The db credentials file now carries the additional databases gor can reach, which is exactly what gor.sql.credentials existed for, so the separate source is redundant. Both initInConsoleApp and initInServer now load systemConnections and userConnections from gor.db.credentials. Removes the property, the ~/gor.sql.credentials home-file lookup, and the resolveSqlCredPath fallback introduced earlier on this branch. This is a breaking change for anyone setting the property. Migrates the three tests that configured it (UTestSQLInputSource, UTestInputSourceParsing, UTestSqlSource) to gor.db.credentials, and updates the DbConnection class docs, the database design note, and the SQL/GORSQL/NORSQL command docs. Also treats a blank APPSERVER_RDA_PASSWORD as unset, matching how url and username are already handled. A secret manager or template rendering an empty value would otherwise install it as a real empty password. Covered at both the parse and install level. Co-Authored-By: Claude Opus 5 (1M context) --- documentation/design/datbase.md | 22 +++--- documentation/src/command/GORSQL.rst | 4 +- documentation/src/command/NORSQL.rst | 2 +- documentation/src/command/SQL.rst | 4 +- .../test/java/gorsat/UTestSQLInputSource.java | 4 +- .../gorsat/UTestInputSourceParsing.scala | 2 +- .../org/gorpipe/gor/model/DbConnection.java | 61 +++++++--------- .../gorpipe/gor/model/DbConnectionCache.java | 4 +- .../org/gorpipe/gor/model/UTestDbSource.java | 69 ++++++------------- .../org/gorpipe/gor/model/UTestSqlSource.java | 1 - 10 files changed, 69 insertions(+), 104 deletions(-) diff --git a/documentation/design/datbase.md b/documentation/design/datbase.md index 2c21184a..03cd832d 100644 --- a/documentation/design/datbase.md +++ b/documentation/design/datbase.md @@ -83,12 +83,18 @@ TBD We have two different configuration files: * -*
  • gor.db.credentials - contains the system databases, which are used by the system internally (e.g. session management), -* and by operation where we have strict access controls (db://, //db:). These credentials -* typically would grant full access to the database. -*
  • gor.sql.credentials - contains the user databases, which are used by the user available commands/sources (SQL, -* GORSQL, NORSQL, sql://). These credentials typically would grant limited/read-only access to -* the database. +*
  • gor.db.credentials - contains all the databases gor can reach. These feed both the system connections, +* used internally (e.g. session management) and by operations with strict access +* controls (db://, //db:), and the user connections behind the user available +* commands/sources (SQL, GORSQL, NORSQL, sql://). +*


    +* Credentials that rotate can instead be supplied through the environment, as APPSERVER_RDA_URL, +* APPSERVER_RDA_USERNAME and APPSERVER_RDA_PASSWORD, which install a source named "rda". A row in the +* credentials file overrides an env-defined source of the same name, so a deployment that wants the +* environment to be authoritative must keep that name out of the file. +*


    +* The separate gor.sql.credentials file has been removed. It previously held the user databases; those now +* live in gor.db.credentials alongside the rest. *


    * The format for these files is: * 

    @@ -97,5 +103,5 @@ We have two different configuration files:
 *    ...
 *
    *

    -* The location of the files defaults to the config directory but can be specified by the system properties -* gor.db.credentials and gor.sql.credentials. \ No newline at end of file +* The location of the file defaults to the config directory but can be specified by the system property +* gor.db.credentials. \ No newline at end of file diff --git a/documentation/src/command/GORSQL.rst b/documentation/src/command/GORSQL.rst index a2085790..1684c3db 100644 --- a/documentation/src/command/GORSQL.rst +++ b/documentation/src/command/GORSQL.rst @@ -7,7 +7,7 @@ ====== GORSQL ====== -The :ref:`GORSQL` command allows you to run arbitrary SQL commands against "the database" (the database here being defined by the content of a file called gor.sql.credentials in the config directory). +The :ref:`GORSQL` command allows you to run arbitrary SQL commands against "the database" (the database here being defined by the content of a file called gor.db.credentials in the config directory). For :ref:`GORSQL` to work properly, the defined query must return Chrom-POS information as first two columns. @@ -33,7 +33,7 @@ Options | ``-ff File`` | Read tags from a tag file and filter files and file contents on. Also accepts a nested query. | | | The following place holders are provided: #{TAGS}. The necessary quoting is done automatically. | +----------------------+---------------------------------------------------------------------------------------------------+ -| ``-db database`` | Database alias as defied in ``gor.sql.credentials``. | +| ``-db database`` | Database alias as defied in ``gor.db.credentials``. | +----------------------+---------------------------------------------------------------------------------------------------+ diff --git a/documentation/src/command/NORSQL.rst b/documentation/src/command/NORSQL.rst index 2d6449ec..1b129c24 100644 --- a/documentation/src/command/NORSQL.rst +++ b/documentation/src/command/NORSQL.rst @@ -7,7 +7,7 @@ ====== NORSQL ====== -The :ref:`NORSQL` command allows you to run arbitrary SQL commands against "the database" (the database here being defined by the content of a file called gor.sql.credentials in the config directory). +The :ref:`NORSQL` command allows you to run arbitrary SQL commands against "the database" (the database here being defined by the content of a file called gor.db.credentials in the config directory). :ref:`NORSQL` can be run against any database table. diff --git a/documentation/src/command/SQL.rst b/documentation/src/command/SQL.rst index bd71cdf5..89407668 100644 --- a/documentation/src/command/SQL.rst +++ b/documentation/src/command/SQL.rst @@ -7,7 +7,7 @@ === SQL === -The SQL command allows you to run arbitrary SQL commands against "the database" (the database here being defined by the content of a file called gor.sql.credentials in the config directory). +The SQL command allows you to run arbitrary SQL commands against "the database" (the database here being defined by the content of a file called gor.db.credentials in the config directory). SQL statements need to be encapsulated between curly brackets, e.g. sql {sql_query}. @@ -37,7 +37,7 @@ Options | ``-ff File`` | Read tags from a tag file and filter files and file contents on. Also accepts a nested query. | | | The following place holders are provided: #{TAGS}. The necessary quoting is done automatically. | +----------------------+---------------------------------------------------------------------------------------------------+ -| ``-db database`` | Database alias as defied in ``gor.sql.credentials``. | +| ``-db database`` | Database alias as defied in ``gor.db.credentials``. | +----------------------+---------------------------------------------------------------------------------------------------+ diff --git a/gortools/src/test/java/gorsat/UTestSQLInputSource.java b/gortools/src/test/java/gorsat/UTestSQLInputSource.java index 649ec791..46b1eb31 100644 --- a/gortools/src/test/java/gorsat/UTestSQLInputSource.java +++ b/gortools/src/test/java/gorsat/UTestSQLInputSource.java @@ -58,9 +58,9 @@ public static void initDb() throws IOException, ClassNotFoundException, SQLExcep List credFileLines = Files.readAllLines(Path.of(rdaPaths[2])); credFileLines.add(Files.readAllLines(Path.of(avasPaths[2])).get(1)); - File credFile = FileTestUtils.createTempFile(new File(rdaPaths[0]), "gor.sql.credentials", + File credFile = FileTestUtils.createTempFile(new File(rdaPaths[0]), "gor.db.credentials", credFileLines.stream().collect(Collectors.joining("\n"))); - System.setProperty("gor.sql.credentials", credFile.getAbsolutePath()); + System.setProperty("gor.db.credentials", credFile.getAbsolutePath()); DbConnection.initInConsoleApp(); } diff --git a/gortools/src/test/scala/gorsat/UTestInputSourceParsing.scala b/gortools/src/test/scala/gorsat/UTestInputSourceParsing.scala index db304ac8..31a20597 100644 --- a/gortools/src/test/scala/gorsat/UTestInputSourceParsing.scala +++ b/gortools/src/test/scala/gorsat/UTestInputSourceParsing.scala @@ -119,7 +119,7 @@ class UTestInputSourceParsing extends AnyFunSuite with BeforeAndAfter with Mocki Class.forName("org.apache.derby.jdbc.EmbeddedDriver") val paths = createTestDataBase_Derby - System.setProperty("gor.sql.credentials", paths(2)) + System.setProperty("gor.db.credentials", paths(2)) DbConnection.initInConsoleApp() var tempDirectory = FileTestUtils.createTempDirectory(this.getClass.getName) diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java index 14253dc3..d882862c 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java @@ -43,24 +43,30 @@ * The static part creates a cache of DbConnection objects that can be used to access all databases defined in the * database configuration files. This part is located here for backward compatibility reasons. *

    - * We have two different configuration files: + * We have one configuration file: * - *

  • gor.db.credentials - contains the system databases, which are used by the system internally (e.g. session management), - * and by operation where we have strict access controls (db://, //db:). These credentials - * typically would grant full access to the database. - *
  • gor.sql.credentials - contains the user databases, which are used by the user available commands/sources (SQL, - * GORSQL, NORSQL, sql://). These credentials typically would grant limited/read-only access to - * the database. + *
  • gor.db.credentials - contains all the databases gor can reach. It feeds both the system connections, + * used internally (e.g. session management) and by operations with strict access + * controls (db://, //db:), and the user connections behind the user available + * commands/sources (SQL, GORSQL, NORSQL, sql://). *


    - * The format for these files is: + * Credentials that rotate can instead be supplied through the environment, as APPSERVER_RDA_URL, + * APPSERVER_RDA_USERNAME and APPSERVER_RDA_PASSWORD, which install a source named "rda". A row in the + * credentials file overrides an env-defined source of the same name, so a deployment that wants the + * environment to be authoritative must keep that name out of the file. + *


    + * The separate gor.sql.credentials file has been removed. It previously held the user databases; those now + * live in gor.db.credentials alongside the rest. + *


    + * The format for this file is: * 

      *    name\tdriver\turl\tuser\tpwd
  *    rda\torg.postgresql.Driver\tjdbc:postgresql://myurl.com:5432/csa\trda\tmypass
  *    ...
  *
    *

    - * The location of the files defaults to the config directory but can be specified by the system properties - * gor.db.credentials and gor.sql.credentials. + * The location of the file defaults to the config directory but can be specified by the system property + * gor.db.credentials. *


    * TODO: This class needs some refactoring to handle the different databases more gracefully. We should use some kind of * plugin mechanism (or Guice) instead of if/else statements. @@ -237,6 +243,9 @@ public boolean queryTableExists(String tableName) { * Initialize DbConnections to be used in a console app. i.e. search for config files in the in user home dir * or be specified by system properties and set up a console based login for missing db passwords * + * Both caches load from gor.db.credentials. The separate gor.sql.credentials source has been + * removed — the db credentials file now carries the additional databases it was intended for. + * * @throws ClassNotFoundException * @throws IOException */ @@ -244,8 +253,7 @@ public static void initInConsoleApp() throws ClassNotFoundException, IOException File homeDbCredFile = new File(System.getProperty("user.home"), "gor.db.credentials"); final String dbCredpath = homeDbCredFile.exists() ? homeDbCredFile.getCanonicalPath() : System.getProperty("gor.db.credentials"); systemConnections.initializeDbSources(dbCredpath); - - userConnections.initializeDbSources(resolveSqlCredPath(dbCredpath)); + userConnections.initializeDbSources(dbCredpath); setPasswordCallback(s -> { final Console console = System.console(); @@ -261,6 +269,9 @@ public static void initInConsoleApp() throws ClassNotFoundException, IOException * (used for db:// sources), while the credentials file supplies the additional databases * reached through {@link #userConnections} (sql:// sources). * + * The separate gor.sql.credentials source has been removed; it existed for exactly the + * resources the db credentials file now carries. + * * @throws ClassNotFoundException * @throws IOException */ @@ -269,32 +280,8 @@ public static void initInServer() throws ClassNotFoundException, IOException { final String dbCredpath = System.getProperty("gor.db.credentials"); if (dbCredpath != null) { systemConnections.initializeDbSources(dbCredpath); + userConnections.initializeDbSources(dbCredpath); } - - final String sqlCredpath = resolveSqlCredPath(dbCredpath); - if (sqlCredpath != null) { - userConnections.initializeDbSources(sqlCredpath); - } - } - - /** - * Resolve the credentials file backing {@link #userConnections}. - * - * The dedicated gor.sql.credentials source is deprecated: gor.db.credentials now carries the - * additional db resources it was intended for. An explicit gor.sql.credentials still wins where - * one is configured, so existing setups keep working; otherwise the db credentials are used for - * both caches. Returns null only when neither is configured. - * - * @param dbCredpath the already-resolved gor.db.credentials path, may be null - * @return the path to load user connections from, or null if none is configured - */ - static String resolveSqlCredPath(String dbCredpath) throws IOException { - File homeSqlCredFile = new File(System.getProperty("user.home"), "gor.sql.credentials"); - if (homeSqlCredFile.exists()) { - return homeSqlCredFile.getCanonicalPath(); - } - final String sqlCredpath = System.getProperty("gor.sql.credentials"); - return sqlCredpath != null ? sqlCredpath : dbCredpath; } /** diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java index 2c631e15..0dc3ff77 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java @@ -168,7 +168,9 @@ static List parseEnvForDbSourceInstallation(Map env) { String url = trimToNull(env.get(ENV_RDA_URL)); String user = trimToNull(env.get(ENV_RDA_USERNAME)); - String pwd = env.get(ENV_RDA_PASSWORD); + // Blank counts as unset here, as it does for url and username: an env var that is present but + // empty is a missing value, not a real empty password. + String pwd = trimToNull(env.get(ENV_RDA_PASSWORD)); if (url == null && user == null) { return partsList; diff --git a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java index a0972138..5ee9b4f8 100644 --- a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java +++ b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java @@ -110,6 +110,15 @@ public void parseEnvOmitsPasswordFieldWhenPasswordUnset() { Assert.assertEquals(4, partsList.get(0).length); } + @Test + public void parseEnvTreatsBlankPasswordAsUnset() { + Map env = rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", " "); + List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); + Assert.assertEquals(1, partsList.size()); + Assert.assertEquals("blank password should be omitted, not installed as an empty password", + 4, partsList.get(0).length); + } + @Test public void parseEnvReturnsNothingWhenNoRdaVarsPresent() { Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(new HashMap<>()).isEmpty()); @@ -177,6 +186,16 @@ public void envOnlyInstallsRdaSource() throws Exception { Assert.assertEquals("secret", rda.pwd); } + @Test + public void blankEnvPasswordInstallsSourceWithNullPassword() throws Exception { + DbConnectionCache cache = new DbConnectionCache(); + cache.initializeDbSources(null, rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", " ")); + + DbConnection rda = cache.lookup("rda"); + Assert.assertNotNull("source should still install, only the password is unset", rda); + Assert.assertNull("whitespace password must not reach the connection as a real password", rda.pwd); + } + @Test public void fileRowOverridesEnvSource() throws Exception { String credpath = writeCredentialsFile( @@ -243,52 +262,4 @@ public void fileOnlyBehaviourIsUnchanged() throws Exception { Assert.assertNotNull(cache.lookup("rda")); Assert.assertEquals("fileuser", cache.lookup("rda").user); } - - /** - * Runs body with user.home pointed at an empty dir, so the ~/gor.sql.credentials branch - * of resolveSqlCredPath is deterministically absent, and with gor.sql.credentials set as given. - */ - private void withSqlCredProperties(String sqlCredProperty, ThrowingRunnable body) throws Exception { - String originalHome = System.getProperty("user.home"); - String originalSqlCred = System.getProperty("gor.sql.credentials"); - try { - System.setProperty("user.home", tempFolder.newFolder("home").getAbsolutePath()); - if (sqlCredProperty == null) { - System.clearProperty("gor.sql.credentials"); - } else { - System.setProperty("gor.sql.credentials", sqlCredProperty); - } - body.run(); - } finally { - System.setProperty("user.home", originalHome); - if (originalSqlCred == null) { - System.clearProperty("gor.sql.credentials"); - } else { - System.setProperty("gor.sql.credentials", originalSqlCred); - } - } - } - - private interface ThrowingRunnable { - void run() throws Exception; - } - - @Test - public void userConnectionsFallBackToDbCredentialsWhenSqlCredentialsUnset() throws Exception { - withSqlCredProperties(null, () -> - Assert.assertEquals("/app/config/gor.db.credentials", - DbConnection.resolveSqlCredPath("/app/config/gor.db.credentials"))); - } - - @Test - public void explicitSqlCredentialsStillWinsOverDbCredentials() throws Exception { - withSqlCredProperties("/custom/gor.sql.credentials", () -> - Assert.assertEquals("/custom/gor.sql.credentials", - DbConnection.resolveSqlCredPath("/app/config/gor.db.credentials"))); - } - - @Test - public void sqlCredPathIsNullWhenNeitherIsConfigured() throws Exception { - withSqlCredProperties(null, () -> Assert.assertNull(DbConnection.resolveSqlCredPath(null))); - } -} \ No newline at end of file +} diff --git a/model/src/test/java/org/gorpipe/gor/model/UTestSqlSource.java b/model/src/test/java/org/gorpipe/gor/model/UTestSqlSource.java index 032dd0c8..392c3850 100644 --- a/model/src/test/java/org/gorpipe/gor/model/UTestSqlSource.java +++ b/model/src/test/java/org/gorpipe/gor/model/UTestSqlSource.java @@ -26,7 +26,6 @@ public class UTestSqlSource { public static void setup() throws IOException, ClassNotFoundException, SQLException { paths = DatabaseHelper.createRdaDatabase(); System.setProperty("gor.db.credentials", paths[2]); - System.setProperty("gor.sql.credentials", paths[2]); DbConnection.initInConsoleApp(); } From 1fb5947ead6c12c4c38d1a0a93dc6cb4bdd03520 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 29 Jul 2026 16:24:15 +0000 Subject: [PATCH 07/12] fix(ENGKNOW-3691): rename env credential vars to GREGOR_DB_* APPSERVER_RDA_URL/USERNAME/PASSWORD/DRIVER become GREGOR_DB_URL/USERNAME/ PASSWORD/DRIVER. The old names were borrowed from SM, which GOR does not otherwise depend on; the GREGOR_ prefix keeps them distinct from generic DB_* vars that a base image, sidecar, or platform default might inject. The installed source is still named "rda". LORD_DB_* is unrelated and untouched. Co-Authored-By: Claude Opus 5 (1M context) --- documentation/design/datbase.md | 4 +-- .../org/gorpipe/gor/model/DbConnection.java | 6 ++--- .../gorpipe/gor/model/DbConnectionCache.java | 26 +++++++++---------- .../org/gorpipe/gor/model/UTestDbSource.java | 8 +++--- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/documentation/design/datbase.md b/documentation/design/datbase.md index 03cd832d..9150820e 100644 --- a/documentation/design/datbase.md +++ b/documentation/design/datbase.md @@ -88,8 +88,8 @@ We have two different configuration files: * controls (db://, //db:), and the user connections behind the user available * commands/sources (SQL, GORSQL, NORSQL, sql://). *


    -* Credentials that rotate can instead be supplied through the environment, as APPSERVER_RDA_URL, -* APPSERVER_RDA_USERNAME and APPSERVER_RDA_PASSWORD, which install a source named "rda". A row in the +* Credentials that rotate can instead be supplied through the environment, as GREGOR_DB_URL, +* GREGOR_DB_USERNAME and GREGOR_DB_PASSWORD, which install a source named "rda". A row in the * credentials file overrides an env-defined source of the same name, so a deployment that wants the * environment to be authoritative must keep that name out of the file. *


    diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java index d882862c..b1d50326 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java @@ -50,8 +50,8 @@ * controls (db://, //db:), and the user connections behind the user available * commands/sources (SQL, GORSQL, NORSQL, sql://). *


    - * Credentials that rotate can instead be supplied through the environment, as APPSERVER_RDA_URL, - * APPSERVER_RDA_USERNAME and APPSERVER_RDA_PASSWORD, which install a source named "rda". A row in the + * Credentials that rotate can instead be supplied through the environment, as GREGOR_DB_URL, + * GREGOR_DB_USERNAME and GREGOR_DB_PASSWORD, which install a source named "rda". A row in the * credentials file overrides an env-defined source of the same name, so a deployment that wants the * environment to be authoritative must keep that name out of the file. *


    @@ -265,7 +265,7 @@ public static void initInConsoleApp() throws ClassNotFoundException, IOException * Initialize DbSource to be used in a server. * * Both caches are fed from gor.db.credentials plus the environment. In a deployment that - * supplies rotating credentials through APPSERVER_RDA_*, those reach {@link #systemConnections} + * supplies rotating credentials through GREGOR_DB_*, those reach {@link #systemConnections} * (used for db:// sources), while the credentials file supplies the additional databases * reached through {@link #userConnections} (sql:// sources). * diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java index 0dc3ff77..2ec9e575 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java @@ -28,11 +28,11 @@ public class DbConnectionCache { private final ConcurrentHashMap mapSources = new ConcurrentHashMap<>(); public String defaultDbSource = "rda"; - static final String ENV_RDA_SOURCE_NAME = "rda"; - static final String ENV_RDA_URL = "APPSERVER_RDA_URL"; - static final String ENV_RDA_USERNAME = "APPSERVER_RDA_USERNAME"; - static final String ENV_RDA_PASSWORD = "APPSERVER_RDA_PASSWORD"; - static final String ENV_RDA_DRIVER = "APPSERVER_RDA_DRIVER"; + static final String ENV_GREGOR_DB_SOURCE_NAME = "rda"; + static final String ENV_GREGOR_DB_URL = "GREGOR_DB_URL"; + static final String ENV_GREGOR_DB_USERNAME = "GREGOR_DB_USERNAME"; + static final String ENV_GREGOR_DB_PASSWORD = "GREGOR_DB_PASSWORD"; + static final String ENV_GREGOR_DB_DRIVER = "GREGOR_DB_DRIVER"; public DbConnectionCache() { } @@ -166,35 +166,35 @@ static List parseEnvForDbSourceInstallation(Map env) { return partsList; } - String url = trimToNull(env.get(ENV_RDA_URL)); - String user = trimToNull(env.get(ENV_RDA_USERNAME)); + String url = trimToNull(env.get(ENV_GREGOR_DB_URL)); + String user = trimToNull(env.get(ENV_GREGOR_DB_USERNAME)); // Blank counts as unset here, as it does for url and username: an env var that is present but // empty is a missing value, not a real empty password. - String pwd = trimToNull(env.get(ENV_RDA_PASSWORD)); + String pwd = trimToNull(env.get(ENV_GREGOR_DB_PASSWORD)); if (url == null && user == null) { return partsList; } if (url == null || user == null) { log.warn("Incomplete db source configuration in environment for source {}: {} is not set. Ignoring it.", - ENV_RDA_SOURCE_NAME, url == null ? ENV_RDA_URL : ENV_RDA_USERNAME); + ENV_GREGOR_DB_SOURCE_NAME, url == null ? ENV_GREGOR_DB_URL : ENV_GREGOR_DB_USERNAME); return partsList; } - String driver = trimToNull(env.get(ENV_RDA_DRIVER)); + String driver = trimToNull(env.get(ENV_GREGOR_DB_DRIVER)); if (driver == null) { driver = driverClassForUrl(url); } if (driver == null) { log.warn("Could not derive a jdbc driver for db source {} from its url, and {} is not set. Ignoring it.", - ENV_RDA_SOURCE_NAME, ENV_RDA_DRIVER); + ENV_GREGOR_DB_SOURCE_NAME, ENV_GREGOR_DB_DRIVER); return partsList; } if (pwd == null) { - partsList.add(new String[]{ENV_RDA_SOURCE_NAME, driver, url, user}); + partsList.add(new String[]{ENV_GREGOR_DB_SOURCE_NAME, driver, url, user}); } else { - partsList.add(new String[]{ENV_RDA_SOURCE_NAME, driver, url, user, pwd}); + partsList.add(new String[]{ENV_GREGOR_DB_SOURCE_NAME, driver, url, user, pwd}); } return partsList; } diff --git a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java index 5ee9b4f8..b6558eb8 100644 --- a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java +++ b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java @@ -82,9 +82,9 @@ public void parseLinesForDbSourceInstallationWithRealTabsAndNewlinesAndInvalidLi private static Map rdaEnv(String url, String username, String password) { Map env = new HashMap<>(); - if (url != null) env.put("APPSERVER_RDA_URL", url); - if (username != null) env.put("APPSERVER_RDA_USERNAME", username); - if (password != null) env.put("APPSERVER_RDA_PASSWORD", password); + if (url != null) env.put("GREGOR_DB_URL", url); + if (username != null) env.put("GREGOR_DB_USERNAME", username); + if (password != null) env.put("GREGOR_DB_PASSWORD", password); return env; } @@ -153,7 +153,7 @@ public void parseEnvDerivesOracleDriverFromUrlPrefix() { @Test public void parseEnvHonoursExplicitDriverOverride() { Map env = rdaEnv("jdbc:whatever://db/x", "gregor_reader", "secret"); - env.put("APPSERVER_RDA_DRIVER", "com.example.Driver"); + env.put("GREGOR_DB_DRIVER", "com.example.Driver"); List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); Assert.assertEquals(1, partsList.size()); Assert.assertEquals("com.example.Driver", partsList.get(0)[1]); From f9ecc327cc0719c02c4c0b81db830321ff22772a Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 29 Jul 2026 16:45:43 +0000 Subject: [PATCH 08/12] fix(ENGKNOW-3691): take db credentials from the caller, not the environment Gor no longer reads credentials from the environment. Instead the host application passes them in as DbCredentials, a new public record, via DbConnectionCache.initializeDbSources(String, List) or DbConnection.initInServer(List). Supplied credentials are installed before the file is read, so a file row of the same name still takes precedence. This keeps deployment-specific env var naming out of the library: gor does not care whether the host sourced them from its own config, a secret manager, or environment variables it names itself. toPartsForInstallation converts supplied credentials into the same shape the file parser produces, so installDbSourceFromParts remains the single place that loads drivers and constructs DbConnection. Driver derivation from the url prefix, blank-as-unset handling, and skip-with-warning on incomplete input are unchanged in behaviour, just moved off the env path. Co-Authored-By: Claude Opus 5 (1M context) --- documentation/design/datbase.md | 11 +- .../org/gorpipe/gor/model/DbConnection.java | 40 ++++-- .../gorpipe/gor/model/DbConnectionCache.java | 126 +++++++++--------- .../org/gorpipe/gor/model/DbCredentials.java | 51 +++++++ .../org/gorpipe/gor/model/UTestDbSource.java | 106 ++++++++------- 5 files changed, 201 insertions(+), 133 deletions(-) create mode 100644 model/src/main/java/org/gorpipe/gor/model/DbCredentials.java diff --git a/documentation/design/datbase.md b/documentation/design/datbase.md index 9150820e..5af32536 100644 --- a/documentation/design/datbase.md +++ b/documentation/design/datbase.md @@ -88,10 +88,13 @@ We have two different configuration files: * controls (db://, //db:), and the user connections behind the user available * commands/sources (SQL, GORSQL, NORSQL, sql://). *


    -* Credentials that rotate can instead be supplied through the environment, as GREGOR_DB_URL, -* GREGOR_DB_USERNAME and GREGOR_DB_PASSWORD, which install a source named "rda". A row in the -* credentials file overrides an env-defined source of the same name, so a deployment that wants the -* environment to be authoritative must keep that name out of the file. +* Credentials that rotate, and so cannot be baked into a file, can instead be passed in by the host +* application as DbCredentials objects, via DbConnection.initInServer(List) or +* DbConnectionCache.initializeDbSources(String, List). Gor does not care where the host got them — +* its own configuration, a secret manager, or environment variables the host owns the naming of — and +* never reads credentials from the environment itself. A row in the credentials file overrides a +* supplied source of the same name, so a host that wants its supplied credentials to be authoritative +* must keep that name out of the file. *


    * The separate gor.sql.credentials file has been removed. It previously held the user databases; those now * live in gor.db.credentials alongside the rest. diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java index b1d50326..279cff4d 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.lang.reflect.UndeclaredThrowableException; import java.sql.*; +import java.util.List; /** * DbConnection abstract access to database source that are configured on installation time. @@ -50,10 +51,12 @@ * controls (db://, //db:), and the user connections behind the user available * commands/sources (SQL, GORSQL, NORSQL, sql://). *


    - * Credentials that rotate can instead be supplied through the environment, as GREGOR_DB_URL, - * GREGOR_DB_USERNAME and GREGOR_DB_PASSWORD, which install a source named "rda". A row in the - * credentials file overrides an env-defined source of the same name, so a deployment that wants the - * environment to be authoritative must keep that name out of the file. + * Credentials that rotate, and so cannot be baked into a file, can instead be passed in by the host + * application as {@link DbCredentials} — see {@link #initInServer(java.util.List)} and + * {@link DbConnectionCache#initializeDbSources(String, java.util.List)}. Gor does not care where the + * host got them; it never reads credentials from the environment itself. A row in the credentials + * file overrides a supplied source of the same name, so a host that wants its supplied credentials to + * be authoritative must keep that name out of the file. *


    * The separate gor.sql.credentials file has been removed. It previously held the user databases; those now * live in gor.db.credentials alongside the rest. @@ -264,23 +267,36 @@ public static void initInConsoleApp() throws ClassNotFoundException, IOException /** * Initialize DbSource to be used in a server. * - * Both caches are fed from gor.db.credentials plus the environment. In a deployment that - * supplies rotating credentials through GREGOR_DB_*, those reach {@link #systemConnections} - * (used for db:// sources), while the credentials file supplies the additional databases - * reached through {@link #userConnections} (sql:// sources). + * @throws ClassNotFoundException + * @throws IOException + */ + @SuppressWarnings("unused") // Called from gor-services + public static void initInServer() throws ClassNotFoundException, IOException { + initInServer(List.of()); + } + + /** + * Initialize DbSource to be used in a server, with credentials supplied by the host application. + * + * Both caches are fed from the supplied credentials plus gor.db.credentials. A host that holds + * rotating credentials — from its own configuration, a secret manager, or environment variables + * it owns the naming of — passes them here rather than gor reading them itself. The credentials + * file supplies the additional databases gor can reach. A file row takes precedence over a + * supplied credential of the same name. * * The separate gor.sql.credentials source has been removed; it existed for exactly the * resources the db credentials file now carries. * + * @param credentials Credentials to install before reading the file. May be empty. * @throws ClassNotFoundException * @throws IOException */ @SuppressWarnings("unused") // Called from GorQueryTask in gor-services - public static void initInServer() throws ClassNotFoundException, IOException { + public static void initInServer(List credentials) throws ClassNotFoundException, IOException { final String dbCredpath = System.getProperty("gor.db.credentials"); - if (dbCredpath != null) { - systemConnections.initializeDbSources(dbCredpath); - userConnections.initializeDbSources(dbCredpath); + if (dbCredpath != null || !credentials.isEmpty()) { + systemConnections.initializeDbSources(dbCredpath, credentials); + userConnections.initializeDbSources(dbCredpath, credentials); } } diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java index 2ec9e575..3aa507af 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java @@ -28,12 +28,6 @@ public class DbConnectionCache { private final ConcurrentHashMap mapSources = new ConcurrentHashMap<>(); public String defaultDbSource = "rda"; - static final String ENV_GREGOR_DB_SOURCE_NAME = "rda"; - static final String ENV_GREGOR_DB_URL = "GREGOR_DB_URL"; - static final String ENV_GREGOR_DB_USERNAME = "GREGOR_DB_USERNAME"; - static final String ENV_GREGOR_DB_PASSWORD = "GREGOR_DB_PASSWORD"; - static final String ENV_GREGOR_DB_DRIVER = "GREGOR_DB_DRIVER"; - public DbConnectionCache() { } @@ -52,41 +46,41 @@ public DbConnection lookup(String source) { } /** - * Read database sources from the environment and from the configuration file. - * - * Sources may come from two places. Environment variables carry credentials that rotate and so - * cannot be baked into a file — see {@link #parseEnvForDbSourceInstallation}. The credentials - * file carries the static set of additional databases gor can reach. A deployment typically - * supplies the rotating system credentials through the environment and the remaining resources - * through the file. + * Read database sources from the configuration file. * * @param credpath The path to the configuration file - * @throws IOException if the credentials file is configured but missing, and no env source was installed + * @throws IOException if the credentials file is configured but missing */ @SuppressWarnings("WeakerAccess") // Used from gor-services public void initializeDbSources(String credpath) throws IOException { - initializeDbSources(credpath, System.getenv()); + initializeDbSources(credpath, List.of()); } /** - * Read database sources from the environment and from the configuration file. + * Read database sources from caller-supplied credentials and from the configuration file. * - * Env-defined sources are installed first so that a file row with the same name takes - * precedence over it. A deployment that wants the environment to be authoritative for a given - * source must therefore keep a row of that name out of the credentials file — otherwise the - * file's static copy shadows the rotating one. + * Sources may come from two places. The caller can pass credentials directly — useful for + * credentials that rotate and so cannot be baked into a file, which the host application may + * source from its own configuration or a secret manager. The credentials file carries the + * static set of additional databases gor can reach. * - * @param credpath The path to the configuration file - * @param env The environment to read env-defined sources from + * Supplied credentials are installed first, so a file row with the same name takes precedence. + * A deployment that wants its supplied credentials to be authoritative for a given source must + * therefore keep a row of that name out of the credentials file — otherwise the file's static + * copy shadows the rotating one. + * + * @param credpath The path to the configuration file + * @param credentials Credentials to install before reading the file. May be empty. + * @throws IOException if the credentials file is configured but missing, and no credentials were installed */ - void initializeDbSources(String credpath, Map env) throws IOException { + public void initializeDbSources(String credpath, List credentials) throws IOException { clearDbSources(); - List envParts = parseEnvForDbSourceInstallation(env); - installAllFromParts(envParts); - installAllFromParts(readFileForDbSourceInstallation(credpath, !envParts.isEmpty())); + List suppliedParts = toPartsForInstallation(credentials); + installAllFromParts(suppliedParts); + installAllFromParts(readFileForDbSourceInstallation(credpath, !suppliedParts.isEmpty())); } - private List readFileForDbSourceInstallation(String credpath, boolean haveEnvSources) throws IOException { + private List readFileForDbSourceInstallation(String credpath, boolean haveSuppliedSources) throws IOException { if (credpath == null || credpath.trim().length() == 0) { log.info("No db credential path specified"); return Collections.emptyList(); @@ -94,8 +88,8 @@ private List readFileForDbSourceInstallation(String credpath, boolean final Path path = Paths.get(credpath); if (Files.notExists(path)) { - if (haveEnvSources) { - log.warn("Specified db credentials file ({}) is not found, continuing with db sources from the environment", credpath); + if (haveSuppliedSources) { + log.warn("Specified db credentials file ({}) is not found, continuing with the supplied db sources", credpath); return Collections.emptyList(); } throw new FileNotFoundException("Specified db credentials file (" + credpath + ") is not found"); @@ -151,50 +145,56 @@ public static List parseLinesForDbSourceInstallation(String credpath, } /** - * Build db source definitions from environment variables. + * Convert caller-supplied credentials into the {name, driver, url, user[, pwd]} shape that + * parseLinesForDbSourceInstallation produces, so both paths share the install code. + * + * Incomplete credentials are skipped with a warning rather than failing initialization. Blank + * values count as unset, including the password: a secret manager or template that renders an + * empty value is expressing a missing value, not a real empty password. * - * Returns entries in the same {name, driver, url, user[, pwd]} shape as - * parseLinesForDbSourceInstallation, so both paths share the install code. - * Never logs credential values, only variable names. + * Never logs credential values, only the source name and which field was missing. * - * @param env the environment to read, normally System.getenv() - * @return zero or one db source definition + * @param credentials the credentials to convert, may be null + * @return one entry per usable credential */ - static List parseEnvForDbSourceInstallation(Map env) { + static List toPartsForInstallation(List credentials) { List partsList = new ArrayList<>(); - if (env == null) { + if (credentials == null) { return partsList; } - String url = trimToNull(env.get(ENV_GREGOR_DB_URL)); - String user = trimToNull(env.get(ENV_GREGOR_DB_USERNAME)); - // Blank counts as unset here, as it does for url and username: an env var that is present but - // empty is a missing value, not a real empty password. - String pwd = trimToNull(env.get(ENV_GREGOR_DB_PASSWORD)); + for (DbCredentials cred : credentials) { + if (cred == null) { + continue; + } - if (url == null && user == null) { - return partsList; - } - if (url == null || user == null) { - log.warn("Incomplete db source configuration in environment for source {}: {} is not set. Ignoring it.", - ENV_GREGOR_DB_SOURCE_NAME, url == null ? ENV_GREGOR_DB_URL : ENV_GREGOR_DB_USERNAME); - return partsList; - } + String name = trimToNull(cred.name()); + String url = trimToNull(cred.url()); + String user = trimToNull(cred.user()); + String pwd = trimToNull(cred.pwd()); - String driver = trimToNull(env.get(ENV_GREGOR_DB_DRIVER)); - if (driver == null) { - driver = driverClassForUrl(url); - } - if (driver == null) { - log.warn("Could not derive a jdbc driver for db source {} from its url, and {} is not set. Ignoring it.", - ENV_GREGOR_DB_SOURCE_NAME, ENV_GREGOR_DB_DRIVER); - return partsList; - } + if (name == null || url == null || user == null) { + log.warn("Incomplete db source credentials for source {}: {} is not set. Ignoring it.", + name == null ? "" : name, + name == null ? "name" : (url == null ? "url" : "user")); + continue; + } - if (pwd == null) { - partsList.add(new String[]{ENV_GREGOR_DB_SOURCE_NAME, driver, url, user}); - } else { - partsList.add(new String[]{ENV_GREGOR_DB_SOURCE_NAME, driver, url, user, pwd}); + String driver = trimToNull(cred.driver()); + if (driver == null) { + driver = driverClassForUrl(url); + } + if (driver == null) { + log.warn("Could not derive a jdbc driver for db source {} from its url, and no driver was given. Ignoring it.", + name); + continue; + } + + if (pwd == null) { + partsList.add(new String[]{name, driver, url, user}); + } else { + partsList.add(new String[]{name, driver, url, user, pwd}); + } } return partsList; } diff --git a/model/src/main/java/org/gorpipe/gor/model/DbCredentials.java b/model/src/main/java/org/gorpipe/gor/model/DbCredentials.java new file mode 100644 index 00000000..51a986c6 --- /dev/null +++ b/model/src/main/java/org/gorpipe/gor/model/DbCredentials.java @@ -0,0 +1,51 @@ +/* + * BEGIN_COPYRIGHT + * + * Copyright (C) 2011-2013 deCODE genetics Inc. + * Copyright (C) 2013-2019 WuXi NextCode Inc. + * All Rights Reserved. + * + * GORpipe is free software: you can redistribute it and/or modify + * it under the terms of the AFFERO GNU General Public License as published by + * the Free Software Foundation. + * + * GORpipe is distributed "AS-IS" AND WITHOUT ANY WARRANTY OF ANY KIND, + * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, + * NON-INFRINGEMENT, OR FITNESS FOR A PARTICULAR PURPOSE. See + * the AFFERO GNU General Public License for the complete license terms. + * + * You should have received a copy of the AFFERO GNU General Public License + * along with GORpipe. If not, see + * + * END_COPYRIGHT + */ + +package org.gorpipe.gor.model; + +/** + * Credentials for a single database source, supplied programmatically rather than through the + * credentials file. + * + * This exists so that a host application can source credentials however it likes — from its own + * configuration, a secret manager, or environment variables it owns the naming of — and hand them to + * {@link DbConnectionCache#initializeDbSources(String, java.util.List)} without gor needing to know + * where they came from. + * + * Credentials passed this way are installed before the credentials file is read, so a file row with + * the same name takes precedence. + * + * @param name the source name, e.g. "rda". Required. + * @param url the jdbc url. Required. + * @param user the database user. Required. + * @param pwd the password. May be null. + * @param driver the jdbc driver class. May be null, in which case it is derived from the url prefix. + */ +public record DbCredentials(String name, String url, String user, String pwd, String driver) { + + /** + * Credentials with the driver derived from the url prefix. + */ + public DbCredentials(String name, String url, String user, String pwd) { + this(name, url, user, pwd, null); + } +} diff --git a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java index b6558eb8..0584c769 100644 --- a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java +++ b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java @@ -32,7 +32,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -80,18 +79,18 @@ public void parseLinesForDbSourceInstallationWithRealTabsAndNewlinesAndInvalidLi Assert.assertEquals(5, partsList.get(0).length); } - private static Map rdaEnv(String url, String username, String password) { - Map env = new HashMap<>(); - if (url != null) env.put("GREGOR_DB_URL", url); - if (username != null) env.put("GREGOR_DB_USERNAME", username); - if (password != null) env.put("GREGOR_DB_PASSWORD", password); - return env; + private static List rdaCreds(String url, String username, String password) { + return List.of(new DbCredentials("rda", url, username, password)); + } + + private static List rdaCreds(String url, String username, String password, String driver) { + return List.of(new DbCredentials("rda", url, username, password, driver)); } @Test - public void parseEnvBuildsRdaPartsInFileParserShape() { - Map env = rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret"); - List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); + public void credentialsBuildPartsInFileParserShape() { + List creds = rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret"); + List partsList = DbConnectionCache.toPartsForInstallation(creds); Assert.assertEquals(1, partsList.size()); String[] parts = partsList.get(0); Assert.assertEquals(5, parts.length); @@ -103,66 +102,65 @@ public void parseEnvBuildsRdaPartsInFileParserShape() { } @Test - public void parseEnvOmitsPasswordFieldWhenPasswordUnset() { - Map env = rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", null); - List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); + public void credentialsOmitPasswordFieldWhenPasswordUnset() { + List creds = rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", null); + List partsList = DbConnectionCache.toPartsForInstallation(creds); Assert.assertEquals(1, partsList.size()); Assert.assertEquals(4, partsList.get(0).length); } @Test - public void parseEnvTreatsBlankPasswordAsUnset() { - Map env = rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", " "); - List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); + public void credentialsTreatBlankPasswordAsUnset() { + List creds = rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", " "); + List partsList = DbConnectionCache.toPartsForInstallation(creds); Assert.assertEquals(1, partsList.size()); Assert.assertEquals("blank password should be omitted, not installed as an empty password", 4, partsList.get(0).length); } @Test - public void parseEnvReturnsNothingWhenNoRdaVarsPresent() { - Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(new HashMap<>()).isEmpty()); + public void noCredentialsProducesNoParts() { + Assert.assertTrue(DbConnectionCache.toPartsForInstallation(List.of()).isEmpty()); } @Test - public void parseEnvSkipsSourceWhenUsernameMissing() { - Map env = rdaEnv("jdbc:postgresql://db:5432/csa", null, "secret"); - Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(env).isEmpty()); + public void credentialsSkippedWhenUsernameMissing() { + List creds = rdaCreds("jdbc:postgresql://db:5432/csa", null, "secret"); + Assert.assertTrue(DbConnectionCache.toPartsForInstallation(creds).isEmpty()); } @Test - public void parseEnvSkipsSourceWhenUrlMissing() { - Map env = rdaEnv(null, "gregor_reader", "secret"); - Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(env).isEmpty()); + public void credentialsSkippedWhenUrlMissing() { + List creds = rdaCreds(null, "gregor_reader", "secret"); + Assert.assertTrue(DbConnectionCache.toPartsForInstallation(creds).isEmpty()); } @Test - public void parseEnvTreatsBlankValuesAsUnset() { - Map env = rdaEnv("jdbc:postgresql://db:5432/csa", " ", "secret"); - Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(env).isEmpty()); + public void credentialsTreatBlankValuesAsUnset() { + List creds = rdaCreds("jdbc:postgresql://db:5432/csa", " ", "secret"); + Assert.assertTrue(DbConnectionCache.toPartsForInstallation(creds).isEmpty()); } @Test - public void parseEnvDerivesOracleDriverFromUrlPrefix() { - Map env = rdaEnv("jdbc:oracle:thin:@db:1521:XE", "gregor_reader", "secret"); - List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); + public void credentialsDeriveOracleDriverFromUrlPrefix() { + List creds = rdaCreds("jdbc:oracle:thin:@db:1521:XE", "gregor_reader", "secret"); + List partsList = DbConnectionCache.toPartsForInstallation(creds); Assert.assertEquals(1, partsList.size()); Assert.assertEquals("oracle.jdbc.driver.OracleDriver", partsList.get(0)[1]); } @Test - public void parseEnvHonoursExplicitDriverOverride() { - Map env = rdaEnv("jdbc:whatever://db/x", "gregor_reader", "secret"); - env.put("GREGOR_DB_DRIVER", "com.example.Driver"); - List partsList = DbConnectionCache.parseEnvForDbSourceInstallation(env); + public void credentialsHonourExplicitDriverOverride() { + List creds = rdaCreds("jdbc:whatever://db/x", "gregor_reader", "secret", "com.example.Driver"); + List partsList = DbConnectionCache.toPartsForInstallation(creds); Assert.assertEquals(1, partsList.size()); Assert.assertEquals("com.example.Driver", partsList.get(0)[1]); } @Test - public void parseEnvSkipsSourceWhenDriverCannotBeDerived() { - Map env = rdaEnv("jdbc:whatever://db/x", "gregor_reader", "secret"); - Assert.assertTrue(DbConnectionCache.parseEnvForDbSourceInstallation(env).isEmpty()); + public void credentialsSkippedWhenDriverCannotBeDerived() { + List creds = rdaCreds("jdbc:whatever://db/x", "gregor_reader", "secret"); + Assert.assertTrue(DbConnectionCache.toPartsForInstallation(creds).isEmpty()); } @Rule @@ -175,9 +173,9 @@ private String writeCredentialsFile(String... lines) throws Exception { } @Test - public void envOnlyInstallsRdaSource() throws Exception { + public void suppliedCredentialsOnlyInstallRdaSource() throws Exception { DbConnectionCache cache = new DbConnectionCache(); - cache.initializeDbSources(null, rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret")); + cache.initializeDbSources(null, rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret")); DbConnection rda = cache.lookup("rda"); Assert.assertNotNull("rda source should be installed from the environment", rda); @@ -187,9 +185,9 @@ public void envOnlyInstallsRdaSource() throws Exception { } @Test - public void blankEnvPasswordInstallsSourceWithNullPassword() throws Exception { + public void blankSuppliedPasswordInstallsSourceWithNullPassword() throws Exception { DbConnectionCache cache = new DbConnectionCache(); - cache.initializeDbSources(null, rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", " ")); + cache.initializeDbSources(null, rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", " ")); DbConnection rda = cache.lookup("rda"); Assert.assertNotNull("source should still install, only the password is unset", rda); @@ -197,13 +195,13 @@ public void blankEnvPasswordInstallsSourceWithNullPassword() throws Exception { } @Test - public void fileRowOverridesEnvSource() throws Exception { + public void fileRowOverridesSuppliedSource() throws Exception { String credpath = writeCredentialsFile( "name\tdriver\turl\tuser\tpwd", "rda\torg.postgresql.Driver\tjdbc:postgresql://filehost:5432/csa\tfileuser\tfilepwd"); DbConnectionCache cache = new DbConnectionCache(); - cache.initializeDbSources(credpath, rdaEnv("jdbc:postgresql://envhost:5432/csa", "envuser", "envpwd")); + cache.initializeDbSources(credpath, rdaCreds("jdbc:postgresql://suppliedhost:5432/csa", "supplieduser", "suppliedpwd")); DbConnection rda = cache.lookup("rda"); Assert.assertNotNull(rda); @@ -212,40 +210,40 @@ public void fileRowOverridesEnvSource() throws Exception { } @Test - public void fileSuppliesAuxiliarySourceAlongsideEnv() throws Exception { + public void fileSuppliesAuxiliarySourceAlongsideSupplied() throws Exception { String credpath = writeCredentialsFile( "name\tdriver\turl\tuser\tpwd", "aux\torg.postgresql.Driver\tjdbc:postgresql://auxhost:5432/aux\tauxuser\tauxpwd"); DbConnectionCache cache = new DbConnectionCache(); - cache.initializeDbSources(credpath, rdaEnv("jdbc:postgresql://envhost:5432/csa", "envuser", "envpwd")); + cache.initializeDbSources(credpath, rdaCreds("jdbc:postgresql://suppliedhost:5432/csa", "supplieduser", "suppliedpwd")); - Assert.assertNotNull("env source should survive", cache.lookup("rda")); - Assert.assertEquals("jdbc:postgresql://envhost:5432/csa", cache.lookup("rda").url); + Assert.assertNotNull("supplied source should survive", cache.lookup("rda")); + Assert.assertEquals("jdbc:postgresql://suppliedhost:5432/csa", cache.lookup("rda").url); Assert.assertNotNull("file source should also be installed", cache.lookup("aux")); Assert.assertEquals("auxuser", cache.lookup("aux").user); } @Test - public void missingFileToleratedWhenEnvSourcePresent() throws Exception { + public void missingFileToleratedWhenSuppliedSourcePresent() throws Exception { String missing = new File(tempFolder.getRoot(), "does-not-exist.credentials").getAbsolutePath(); DbConnectionCache cache = new DbConnectionCache(); - cache.initializeDbSources(missing, rdaEnv("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret")); + cache.initializeDbSources(missing, rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret")); Assert.assertNotNull(cache.lookup("rda")); } @Test(expected = FileNotFoundException.class) - public void missingFileStillThrowsWithoutEnvSources() throws Exception { + public void missingFileStillThrowsWithoutSuppliedSources() throws Exception { String missing = new File(tempFolder.getRoot(), "does-not-exist.credentials").getAbsolutePath(); - new DbConnectionCache().initializeDbSources(missing, new HashMap<>()); + new DbConnectionCache().initializeDbSources(missing, List.of()); } @Test - public void partialEnvSkipsSourceWithoutThrowing() throws Exception { + public void partialCredentialsSkippedWithoutThrowing() throws Exception { DbConnectionCache cache = new DbConnectionCache(); - cache.initializeDbSources(null, rdaEnv("jdbc:postgresql://db:5432/csa", null, "secret")); + cache.initializeDbSources(null, rdaCreds("jdbc:postgresql://db:5432/csa", null, "secret")); Assert.assertNull(cache.lookup("rda")); } @@ -257,7 +255,7 @@ public void fileOnlyBehaviourIsUnchanged() throws Exception { "rda\torg.postgresql.Driver\tjdbc:postgresql://filehost:5432/csa\tfileuser\tfilepwd"); DbConnectionCache cache = new DbConnectionCache(); - cache.initializeDbSources(credpath, new HashMap<>()); + cache.initializeDbSources(credpath, List.of()); Assert.assertNotNull(cache.lookup("rda")); Assert.assertEquals("fileuser", cache.lookup("rda").user); From 63f41b390483fad188eeafa351b1fab991b09b12 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 29 Jul 2026 16:55:47 +0000 Subject: [PATCH 09/12] docs(ENGKNOW-3691): rewrite the database configuration section The Configuration section was a block of javadoc pasted into markdown - li tags, pre blocks, and a TBD - and still described two credentials files. Rewrites it as markdown covering the two sources that exist now: the gor.db.credentials file and credentials the host passes in as DbCredentials, with the precedence rule between them and the missing-file behaviour. Co-Authored-By: Claude Opus 5 (1M context) --- documentation/design/datbase.md | 85 ++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/documentation/design/datbase.md b/documentation/design/datbase.md index 5af32536..128e8398 100644 --- a/documentation/design/datbase.md +++ b/documentation/design/datbase.md @@ -79,32 +79,61 @@ TBD ### Configuration -TBD +Database sources come from two places: a credentials file, and credentials the host application passes +in programmatically. + +#### The credentials file + +`gor.db.credentials` contains the databases gor can reach. It feeds both connection caches: + +- **system connections** — used internally (e.g. session management) and by the access-controlled + operations (`db://`, `//db:`) +- **user connections** — behind the user-available commands and sources (`SQL`, `GORSQL`, `NORSQL`, + `sql://`) + +The file is tab-separated, with a header line: + +``` +name driver url user pwd +rda org.postgresql.Driver jdbc:postgresql://myurl.com:5432/csa rda mypass +``` + +The password column is optional. Lines starting with `#` are ignored. + +Its location defaults to the config directory and can be set with the `gor.db.credentials` system +property. + +> The separate `gor.sql.credentials` file has been removed. It previously held the user databases; +> those now live in `gor.db.credentials` alongside the rest. + +#### Credentials passed in by the host + +Credentials that rotate cannot be baked into a file — a rotation would leave it stale. The host +application passes those in as `DbCredentials`: + +```java +var credentials = List.of(new DbCredentials("rda", url, user, password)); + +DbConnection.initInServer(credentials); +// or, per cache: +DbConnection.systemConnections.initializeDbSources(credpath, credentials); +``` + +Gor never reads credentials from the environment itself, and does not care where the host got them — +its own configuration, a secret manager, or environment variables the host owns the naming of. This +keeps deployment-specific naming out of the library. + +`DbCredentials` takes `name`, `url`, `user`, `pwd`, and an optional `driver`. When `driver` is null it +is derived from the url prefix (`jdbc:postgresql:`, `jdbc:oracle:`). Blank values count as unset, +including the password, so a secret manager rendering an empty string does not become a real empty +password. Incomplete credentials are logged — naming the missing field, never a value — and skipped, +rather than failing startup. + +#### Precedence + +Supplied credentials are installed first, then the file is read on top, so **a file row overrides a +supplied source of the same name**. A host that wants its supplied credentials to be authoritative for +a source must keep a row of that name out of the file. -We have two different configuration files: -* -*

  • gor.db.credentials - contains all the databases gor can reach. These feed both the system connections, -* used internally (e.g. session management) and by operations with strict access -* controls (db://, //db:), and the user connections behind the user available -* commands/sources (SQL, GORSQL, NORSQL, sql://). -*


    -* Credentials that rotate, and so cannot be baked into a file, can instead be passed in by the host -* application as DbCredentials objects, via DbConnection.initInServer(List) or -* DbConnectionCache.initializeDbSources(String, List). Gor does not care where the host got them — -* its own configuration, a secret manager, or environment variables the host owns the naming of — and -* never reads credentials from the environment itself. A row in the credentials file overrides a -* supplied source of the same name, so a host that wants its supplied credentials to be authoritative -* must keep that name out of the file. -*


    -* The separate gor.sql.credentials file has been removed. It previously held the user databases; those now -* live in gor.db.credentials alongside the rest. -*


    -* The format for these files is: -* 

    -*    name\tdriver\turl\tuser\tpwd
-*    rda\torg.postgresql.Driver\tjdbc:postgresql://myurl.com:5432/csa\trda\tmypass
-*    ...
-*
    -*

    -* The location of the file defaults to the config directory but can be specified by the system property -* gor.db.credentials. \ No newline at end of file +If the configured credentials file is missing, that is an error — unless credentials were supplied, in +which case gor logs a warning and continues with those. \ No newline at end of file From 62c742a86c33d2810be936a5c873d8569a2ae5a7 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Wed, 29 Jul 2026 17:21:21 +0000 Subject: [PATCH 10/12] fix(ENGKNOW-3691): Update docs. --- documentation/design/datbase.md | 3 --- .../main/java/org/gorpipe/gor/model/DbConnection.java | 9 --------- 2 files changed, 12 deletions(-) diff --git a/documentation/design/datbase.md b/documentation/design/datbase.md index 128e8398..fbc1e7ad 100644 --- a/documentation/design/datbase.md +++ b/documentation/design/datbase.md @@ -103,9 +103,6 @@ The password column is optional. Lines starting with `#` are ignored. Its location defaults to the config directory and can be set with the `gor.db.credentials` system property. -> The separate `gor.sql.credentials` file has been removed. It previously held the user databases; -> those now live in `gor.db.credentials` alongside the rest. - #### Credentials passed in by the host Credentials that rotate cannot be baked into a file — a rotation would leave it stale. The host diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java index 279cff4d..930afa21 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java +++ b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java @@ -58,9 +58,6 @@ * file overrides a supplied source of the same name, so a host that wants its supplied credentials to * be authoritative must keep that name out of the file. *


    - * The separate gor.sql.credentials file has been removed. It previously held the user databases; those now - * live in gor.db.credentials alongside the rest. - *


    * The format for this file is: * 

      *    name\tdriver\turl\tuser\tpwd
@@ -246,9 +243,6 @@ public boolean queryTableExists(String tableName) {
      * Initialize DbConnections to be used in a console app. i.e. search for config files in the in user home dir
      * or be specified by system properties and set up a console based login for missing db passwords
      *
-     * Both caches load from gor.db.credentials. The separate gor.sql.credentials source has been
-     * removed — the db credentials file now carries the additional databases it was intended for.
-     *
      * @throws ClassNotFoundException
      * @throws IOException
      */
@@ -284,9 +278,6 @@ public static void initInServer() throws ClassNotFoundException, IOException {
      * file supplies the additional databases gor can reach. A file row takes precedence over a
      * supplied credential of the same name.
      *
-     * The separate gor.sql.credentials source has been removed; it existed for exactly the
-     * resources the db credentials file now carries.
-     *
      * @param credentials Credentials to install before reading the file. May be empty.
      * @throws ClassNotFoundException
      * @throws IOException

From 55dfd3d2f80b99c8d52d47e2bfe63bd3b5acd05c Mon Sep 17 00:00:00 2001
From: Gisli Magnusson 
Date: Wed, 29 Jul 2026 22:39:23 +0000
Subject: [PATCH 11/12] fix(ENGKNOW-3691): feed the system and user caches from
 separate sources

systemConnections now comes from the credentials the host passes in, and
userConnections from gor.db.credentials. Previously both caches were fed from
both sources with the file taking precedence.

Keeping them apart matches what each is for: system credentials rotate and
cannot be baked into a file without going stale, and the rotating system
credentials are not the ones user queries should reach. It also removes the
collision between the two, so there is no precedence rule left to get wrong.

The two initializeDbSources overloads are now alternatives rather than a merge -
initializeDbSources(String) reads the file, initializeDbSources(List) installs
supplied credentials, and each clears the cache first. initInServer's parameter
is renamed systemCredentials to say which cache it feeds.

A console app has no host to supply credentials, so initInConsoleApp still loads
both caches from the file.

Note passing no credentials now leaves the system cache empty rather than
falling back to the file, so db:// will not resolve in a server that supplies
none.

Co-Authored-By: Claude Opus 5 (1M context) 
---
 documentation/design/datbase.md               | 46 ++++++------
 .../org/gorpipe/gor/model/DbConnection.java   | 46 +++++++-----
 .../gorpipe/gor/model/DbConnectionCache.java  | 36 ++++------
 .../org/gorpipe/gor/model/UTestDbSource.java  | 71 ++++++++-----------
 4 files changed, 91 insertions(+), 108 deletions(-)

diff --git a/documentation/design/datbase.md b/documentation/design/datbase.md
index fbc1e7ad..49d2b2ab 100644
--- a/documentation/design/datbase.md
+++ b/documentation/design/datbase.md
@@ -79,19 +79,22 @@ TBD
 
 ### Configuration
 
-Database sources come from two places: a credentials file, and credentials the host application passes
-in programmatically.
+There are two connection caches, and in a server they are fed from **separate** sources:
 
-#### The credentials file
+| Cache | Used by | Fed from |
+|---|---|---|
+| system connections | internal use (e.g. session management) and the access-controlled operations `db://`, `//db:` | credentials the host passes in |
+| user connections | the user-available commands and sources `SQL`, `GORSQL`, `NORSQL`, `sql://` | the `gor.db.credentials` file |
+
+They are kept apart deliberately. System credentials typically rotate, and a file cannot hold rotating
+credentials without going stale; equally, the rotating system credentials are not the ones user queries
+should reach.
 
-`gor.db.credentials` contains the databases gor can reach. It feeds both connection caches:
+In a console app there is no host to supply credentials, so both caches load from the file.
 
-- **system connections** — used internally (e.g. session management) and by the access-controlled
-  operations (`db://`, `//db:`)
-- **user connections** — behind the user-available commands and sources (`SQL`, `GORSQL`, `NORSQL`,
-  `sql://`)
+#### The credentials file
 
-The file is tab-separated, with a header line:
+`gor.db.credentials` is tab-separated, with a header line:
 
 ```
 name	driver	url	user	pwd
@@ -103,17 +106,19 @@ The password column is optional. Lines starting with `#` are ignored.
 Its location defaults to the config directory and can be set with the `gor.db.credentials` system
 property.
 
+A missing credentials file is an error.
+
 #### Credentials passed in by the host
 
-Credentials that rotate cannot be baked into a file — a rotation would leave it stale. The host
-application passes those in as `DbCredentials`:
+The host application supplies the system credentials as `DbCredentials`:
 
 ```java
-var credentials = List.of(new DbCredentials("rda", url, user, password));
+var systemCredentials = List.of(new DbCredentials("rda", url, user, password));
 
-DbConnection.initInServer(credentials);
-// or, per cache:
-DbConnection.systemConnections.initializeDbSources(credpath, credentials);
+DbConnection.initInServer(systemCredentials);
+// which is, per cache:
+DbConnection.systemConnections.initializeDbSources(systemCredentials);
+DbConnection.userConnections.initializeDbSources(credpath);
 ```
 
 Gor never reads credentials from the environment itself, and does not care where the host got them —
@@ -126,11 +131,6 @@ including the password, so a secret manager rendering an empty string does not b
 password. Incomplete credentials are logged — naming the missing field, never a value — and skipped,
 rather than failing startup.
 
-#### Precedence
-
-Supplied credentials are installed first, then the file is read on top, so **a file row overrides a
-supplied source of the same name**. A host that wants its supplied credentials to be authoritative for
-a source must keep a row of that name out of the file.
-
-If the configured credentials file is missing, that is an error — unless credentials were supplied, in
-which case gor logs a warning and continues with those.
\ No newline at end of file
+The two `initializeDbSources` overloads are alternatives, not additive: each clears the cache first.
+That is what keeps a cache fed from exactly one source. Passing no credentials therefore leaves the
+system cache empty, and `db://` sources will not resolve.
\ No newline at end of file
diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java
index 930afa21..eed8ea8a 100644
--- a/model/src/main/java/org/gorpipe/gor/model/DbConnection.java
+++ b/model/src/main/java/org/gorpipe/gor/model/DbConnection.java
@@ -51,12 +51,13 @@
  *                         controls (db://, //db:), and the user connections behind the user available
  *                         commands/sources (SQL, GORSQL, NORSQL, sql://).
  * 

    
- * Credentials that rotate, and so cannot be baked into a file, can instead be passed in by the host
- * application as {@link DbCredentials} — see {@link #initInServer(java.util.List)} and
- * {@link DbConnectionCache#initializeDbSources(String, java.util.List)}.  Gor does not care where the
- * host got them; it never reads credentials from the environment itself.  A row in the credentials
- * file overrides a supplied source of the same name, so a host that wants its supplied credentials to
- * be authoritative must keep that name out of the file.
+ * In a server, the two caches are fed from separate sources.  {@link #userConnections} comes from the
+ * gor.db.credentials file, while {@link #systemConnections} comes from credentials the host passes in
+ * as {@link DbCredentials} — see {@link #initInServer(java.util.List)}.  System credentials typically
+ * rotate, and so cannot be baked into a file without going stale; gor does not care where the host got
+ * them and never reads them from the environment itself.
+ * 

    
+ * In a console app there is no host to supply credentials, so both caches load from the file.
  * 

    
  * The format for this file is:
  * 
    @@ -270,24 +271,33 @@ public static void initInServer() throws ClassNotFoundException, IOException {
     }
 
     /**
-     * Initialize DbSource to be used in a server, with credentials supplied by the host application.
+     * Initialize DbSource to be used in a server, with the system credentials supplied by the host
+     * application.
+     *
+     * The two caches are fed from separate sources:
      *
-     * Both caches are fed from the supplied credentials plus gor.db.credentials. A host that holds
-     * rotating credentials — from its own configuration, a secret manager, or environment variables
-     * it owns the naming of — passes them here rather than gor reading them itself. The credentials
-     * file supplies the additional databases gor can reach. A file row takes precedence over a
-     * supplied credential of the same name.
+     * 
  •  {@link #systemConnections}, used internally and by the access-controlled operations
+     *      (db://, //db:), comes from the supplied credentials. These typically rotate, so a host
+     *      holding them — in its own configuration, a secret manager, or environment variables it
+     *      owns the naming of — passes them here rather than gor reading them itself.
+     * 
  •  {@link #userConnections}, behind the user-available commands and sources (SQL, GORSQL,
+     *      NORSQL, sql://), comes from the gor.db.credentials file.
      *
-     * @param credentials Credentials to install before reading the file. May be empty.
+     * The sources are kept apart deliberately: the file cannot hold credentials that rotate without
+     * going stale, and the rotating system credentials are not the ones user queries should reach.
+     *
+     * @param systemCredentials Credentials for {@link #systemConnections}. May be empty, which leaves
+     *                          that cache empty.
      * @throws ClassNotFoundException
      * @throws IOException
      */
     @SuppressWarnings("unused") // Called from GorQueryTask in gor-services
-    public static void initInServer(List credentials) throws ClassNotFoundException, IOException {
-        final String dbCredpath = System.getProperty("gor.db.credentials");
-        if (dbCredpath != null || !credentials.isEmpty()) {
-            systemConnections.initializeDbSources(dbCredpath, credentials);
-            userConnections.initializeDbSources(dbCredpath, credentials);
+    public static void initInServer(List systemCredentials) throws ClassNotFoundException, IOException {
+        systemConnections.initializeDbSources(systemCredentials);
+
+        final String userCredpath = System.getProperty("gor.db.credentials");
+        if (userCredpath != null) {
+            userConnections.initializeDbSources(userCredpath);
         }
     }
 
diff --git a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java
index 3aa507af..440652d5 100644
--- a/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java
+++ b/model/src/main/java/org/gorpipe/gor/model/DbConnectionCache.java
@@ -46,41 +46,33 @@ public DbConnection lookup(String source) {
     }
 
     /**
-     * Read database sources from the configuration file.
+     * Read database sources from the configuration file, replacing any sources already installed.
      *
      * @param credpath The path to the configuration file
      * @throws IOException if the credentials file is configured but missing
      */
     @SuppressWarnings("WeakerAccess") // Used from gor-services
     public void initializeDbSources(String credpath) throws IOException {
-        initializeDbSources(credpath, List.of());
+        clearDbSources();
+        installAllFromParts(readFileForDbSourceInstallation(credpath));
     }
 
     /**
-     * Read database sources from caller-supplied credentials and from the configuration file.
-     *
-     * Sources may come from two places. The caller can pass credentials directly — useful for
-     * credentials that rotate and so cannot be baked into a file, which the host application may
-     * source from its own configuration or a secret manager. The credentials file carries the
-     * static set of additional databases gor can reach.
+     * Install database sources from credentials supplied by the host application, replacing any
+     * sources already installed.
      *
-     * Supplied credentials are installed first, so a file row with the same name takes precedence.
-     * A deployment that wants its supplied credentials to be authoritative for a given source must
-     * therefore keep a row of that name out of the credentials file — otherwise the file's static
-     * copy shadows the rotating one.
+     * This is the counterpart to {@link #initializeDbSources(String)} for credentials that rotate and
+     * so cannot be baked into a file. The host may source them from its own configuration, a secret
+     * manager, or environment variables it owns the naming of — gor does not read them itself.
      *
-     * @param credpath    The path to the configuration file
-     * @param credentials Credentials to install before reading the file. May be empty.
-     * @throws IOException if the credentials file is configured but missing, and no credentials were installed
+     * @param credentials The credentials to install. May be empty, which leaves the cache empty.
      */
-    public void initializeDbSources(String credpath, List credentials) throws IOException {
+    public void initializeDbSources(List credentials) {
         clearDbSources();
-        List suppliedParts = toPartsForInstallation(credentials);
-        installAllFromParts(suppliedParts);
-        installAllFromParts(readFileForDbSourceInstallation(credpath, !suppliedParts.isEmpty()));
+        installAllFromParts(toPartsForInstallation(credentials));
     }
 
-    private List readFileForDbSourceInstallation(String credpath, boolean haveSuppliedSources) throws IOException {
+    private List readFileForDbSourceInstallation(String credpath) throws IOException {
         if (credpath == null || credpath.trim().length() == 0) {
             log.info("No db credential path specified");
             return Collections.emptyList();
@@ -88,10 +80,6 @@ private List readFileForDbSourceInstallation(String credpath, boolean
 
         final Path path = Paths.get(credpath);
         if (Files.notExists(path)) {
-            if (haveSuppliedSources) {
-                log.warn("Specified db credentials file ({}) is not found, continuing with the supplied db sources", credpath);
-                return Collections.emptyList();
-            }
             throw new FileNotFoundException("Specified db credentials file (" + credpath + ") is not found");
         }
 
diff --git a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java
index 0584c769..42edae10 100644
--- a/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java
+++ b/model/src/test/java/org/gorpipe/gor/model/UTestDbSource.java
@@ -33,7 +33,6 @@
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 
 public class UTestDbSource {
 
@@ -173,21 +172,21 @@ private String writeCredentialsFile(String... lines) throws Exception {
     }
 
     @Test
-    public void suppliedCredentialsOnlyInstallRdaSource() throws Exception {
+    public void suppliedCredentialsOnlyInstallRdaSource() {
         DbConnectionCache cache = new DbConnectionCache();
-        cache.initializeDbSources(null, rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret"));
+        cache.initializeDbSources(rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret"));
 
         DbConnection rda = cache.lookup("rda");
-        Assert.assertNotNull("rda source should be installed from the environment", rda);
+        Assert.assertNotNull("rda source should be installed from the supplied credentials", rda);
         Assert.assertEquals("jdbc:postgresql://db:5432/csa", rda.url);
         Assert.assertEquals("gregor_reader", rda.user);
         Assert.assertEquals("secret", rda.pwd);
     }
 
     @Test
-    public void blankSuppliedPasswordInstallsSourceWithNullPassword() throws Exception {
+    public void blankSuppliedPasswordInstallsSourceWithNullPassword() {
         DbConnectionCache cache = new DbConnectionCache();
-        cache.initializeDbSources(null, rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", "   "));
+        cache.initializeDbSources(rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", "   "));
 
         DbConnection rda = cache.lookup("rda");
         Assert.assertNotNull("source should still install, only the password is unset", rda);
@@ -195,69 +194,55 @@ public void blankSuppliedPasswordInstallsSourceWithNullPassword() throws Excepti
     }
 
     @Test
-    public void fileRowOverridesSuppliedSource() throws Exception {
-        String credpath = writeCredentialsFile(
-                "name\tdriver\turl\tuser\tpwd",
-                "rda\torg.postgresql.Driver\tjdbc:postgresql://filehost:5432/csa\tfileuser\tfilepwd");
-
+    public void partialCredentialsSkippedWithoutThrowing() {
         DbConnectionCache cache = new DbConnectionCache();
-        cache.initializeDbSources(credpath, rdaCreds("jdbc:postgresql://suppliedhost:5432/csa", "supplieduser", "suppliedpwd"));
+        cache.initializeDbSources(rdaCreds("jdbc:postgresql://db:5432/csa", null, "secret"));
 
-        DbConnection rda = cache.lookup("rda");
-        Assert.assertNotNull(rda);
-        Assert.assertEquals("jdbc:postgresql://filehost:5432/csa", rda.url);
-        Assert.assertEquals("fileuser", rda.user);
+        Assert.assertNull(cache.lookup("rda"));
     }
 
     @Test
-    public void fileSuppliesAuxiliarySourceAlongsideSupplied() throws Exception {
+    public void suppliedCredentialsReplaceAnythingAlreadyInstalled() throws Exception {
         String credpath = writeCredentialsFile(
                 "name\tdriver\turl\tuser\tpwd",
                 "aux\torg.postgresql.Driver\tjdbc:postgresql://auxhost:5432/aux\tauxuser\tauxpwd");
 
         DbConnectionCache cache = new DbConnectionCache();
-        cache.initializeDbSources(credpath, rdaCreds("jdbc:postgresql://suppliedhost:5432/csa", "supplieduser", "suppliedpwd"));
+        cache.initializeDbSources(credpath);
+        Assert.assertNotNull(cache.lookup("aux"));
 
-        Assert.assertNotNull("supplied source should survive", cache.lookup("rda"));
-        Assert.assertEquals("jdbc:postgresql://suppliedhost:5432/csa", cache.lookup("rda").url);
-        Assert.assertNotNull("file source should also be installed", cache.lookup("aux"));
-        Assert.assertEquals("auxuser", cache.lookup("aux").user);
+        // The two initializers are alternatives, not additive - each clears the cache first, which is
+        // what keeps the system and user caches fed from exactly one source each.
+        cache.initializeDbSources(rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret"));
+        Assert.assertNotNull(cache.lookup("rda"));
+        Assert.assertNull("file sources should not survive a credentials load", cache.lookup("aux"));
     }
 
     @Test
-    public void missingFileToleratedWhenSuppliedSourcePresent() throws Exception {
-        String missing = new File(tempFolder.getRoot(), "does-not-exist.credentials").getAbsolutePath();
+    public void fileInstallsEveryRow() throws Exception {
+        String credpath = writeCredentialsFile(
+                "name\tdriver\turl\tuser\tpwd",
+                "rda\torg.postgresql.Driver\tjdbc:postgresql://filehost:5432/csa\tfileuser\tfilepwd",
+                "aux\torg.postgresql.Driver\tjdbc:postgresql://auxhost:5432/aux\tauxuser\tauxpwd");
 
         DbConnectionCache cache = new DbConnectionCache();
-        cache.initializeDbSources(missing, rdaCreds("jdbc:postgresql://db:5432/csa", "gregor_reader", "secret"));
+        cache.initializeDbSources(credpath);
 
-        Assert.assertNotNull(cache.lookup("rda"));
+        Assert.assertEquals("fileuser", cache.lookup("rda").user);
+        Assert.assertEquals("auxuser", cache.lookup("aux").user);
     }
 
     @Test(expected = FileNotFoundException.class)
-    public void missingFileStillThrowsWithoutSuppliedSources() throws Exception {
+    public void missingFileThrows() throws Exception {
         String missing = new File(tempFolder.getRoot(), "does-not-exist.credentials").getAbsolutePath();
-        new DbConnectionCache().initializeDbSources(missing, List.of());
+        new DbConnectionCache().initializeDbSources(missing);
     }
 
     @Test
-    public void partialCredentialsSkippedWithoutThrowing() throws Exception {
+    public void noCredentialPathLeavesCacheEmpty() throws Exception {
         DbConnectionCache cache = new DbConnectionCache();
-        cache.initializeDbSources(null, rdaCreds("jdbc:postgresql://db:5432/csa", null, "secret"));
+        cache.initializeDbSources((String) null);
 
         Assert.assertNull(cache.lookup("rda"));
     }
-
-    @Test
-    public void fileOnlyBehaviourIsUnchanged() throws Exception {
-        String credpath = writeCredentialsFile(
-                "name\tdriver\turl\tuser\tpwd",
-                "rda\torg.postgresql.Driver\tjdbc:postgresql://filehost:5432/csa\tfileuser\tfilepwd");
-
-        DbConnectionCache cache = new DbConnectionCache();
-        cache.initializeDbSources(credpath, List.of());
-
-        Assert.assertNotNull(cache.lookup("rda"));
-        Assert.assertEquals("fileuser", cache.lookup("rda").user);
-    }
 }

From fab5421d49536fef5ee0df967211efa8f7cf69ee Mon Sep 17 00:00:00 2001
From: Gisli Magnusson 
Date: Wed, 29 Jul 2026 22:43:21 +0000
Subject: [PATCH 12/12] docs(ENGKNOW-3691): say which connection cache each
 access method uses

Now that the system and user caches are fed from different sources, which cache
an access method resolves against determines which credentials it gets - so the
document has to say.

Adds that to the notes for each of the four methods: sql commands and sql://
resolve against the user connections, db:// and //db: against the system
connections. The SQL, GORSQL and NORSQL command pages now say the databases they
reach are the user connections rather than just "the database".

Co-Authored-By: Claude Opus 5 (1M context) 
---
 documentation/design/datbase.md      | 6 +++++-
 documentation/src/command/GORSQL.rst | 2 +-
 documentation/src/command/NORSQL.rst | 2 +-
 documentation/src/command/SQL.rst    | 2 +-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/documentation/design/datbase.md b/documentation/design/datbase.md
index 49d2b2ab..29be8a15 100644
--- a/documentation/design/datbase.md
+++ b/documentation/design/datbase.md
@@ -23,11 +23,13 @@ These come in two flavors:
 
 Notes:
 1. Uses DBNorIterator.
+2. Resolves against the **user** connections, i.e. the `gor.db.credentials` file. See [Configuration](#configuration).
 
 #### **sql:// URIs**
 
 Notes:
 1. Uses DBNorIterator (and SQLSource).
+2. Resolves against the **user** connections, i.e. the `gor.db.credentials` file. See [Configuration](#configuration).
 
 ### **Limited SQL Commands**
 These come in  flavors:
@@ -50,6 +52,7 @@ gor db://rda:variant_annotations | top 10
 
 Notes:
 1. Uses DBSource and DbGenomicIterator.
+2. Resolves against the **system** connections, i.e. the credentials the host passes in. See [Configuration](#configuration).
 
 #### **//db/ Paths**
 The //db/ paths are arbitrary SELECT statements that can be used to selected from a given database.  
@@ -68,7 +71,8 @@ Their limit is the can ONLY be executed from a link file  but not from a GOR que
 
 Notes:  
 1. Uses DBNorIterator.
-2. This is the old style of doing SQL access in GOR.  The new preferred way is to use the `sql`, `norsql`, `gorsql` commands.  This is likely to be deprecated.
+2. Resolves against the **system** connections, i.e. the credentials the host passes in. See [Configuration](#configuration).
+3. This is the old style of doing SQL access in GOR.  The new preferred way is to use the `sql`, `norsql`, `gorsql` commands.  This is likely to be deprecated.
 
 ### Sepcial Variables
 
diff --git a/documentation/src/command/GORSQL.rst b/documentation/src/command/GORSQL.rst
index 1684c3db..a1b7f120 100644
--- a/documentation/src/command/GORSQL.rst
+++ b/documentation/src/command/GORSQL.rst
@@ -7,7 +7,7 @@
 ======
 GORSQL
 ======
-The :ref:`GORSQL` command allows you to run arbitrary SQL commands against "the database" (the database here being defined by the content of a file called gor.db.credentials in the config directory).
+The :ref:`GORSQL` command allows you to run arbitrary SQL commands against "the database" (the databases available here are the user connections, defined by the content of a file called gor.db.credentials in the config directory).
 
 For :ref:`GORSQL` to work properly, the defined query must return Chrom-POS information as first two columns.
 
diff --git a/documentation/src/command/NORSQL.rst b/documentation/src/command/NORSQL.rst
index 1b129c24..7d80d017 100644
--- a/documentation/src/command/NORSQL.rst
+++ b/documentation/src/command/NORSQL.rst
@@ -7,7 +7,7 @@
 ======
 NORSQL
 ======
-The :ref:`NORSQL` command allows you to run arbitrary SQL commands against "the database" (the database here being defined by the content of a file called gor.db.credentials in the config directory).
+The :ref:`NORSQL` command allows you to run arbitrary SQL commands against "the database" (the databases available here are the user connections, defined by the content of a file called gor.db.credentials in the config directory).
 
 :ref:`NORSQL` can be run against any database table.
 
diff --git a/documentation/src/command/SQL.rst b/documentation/src/command/SQL.rst
index 89407668..257143d4 100644
--- a/documentation/src/command/SQL.rst
+++ b/documentation/src/command/SQL.rst
@@ -7,7 +7,7 @@
 ===
 SQL
 ===
-The SQL command allows you to run arbitrary SQL commands against "the database" (the database here being defined by the content of a file called gor.db.credentials in the config directory).
+The SQL command allows you to run arbitrary SQL commands against "the database" (the databases available here are the user connections, defined by the content of a file called gor.db.credentials in the config directory).
 
 SQL statements need to be encapsulated between curly brackets, e.g. sql {sql_query}.