Fail over to a different Mullvad/IVPN relay when the current one stays dead

app/src/main/java/eu/faircode/netguard/ServiceSinkhole.java

@@ -1807,7 +1807,8 @@ private boolean startNative(final ParcelFileDescriptor vpn, List<Rule> listAllow
         // It also handles the disable case internally — no need to gate.
         net.kollnig.missioncontrol.wg.WgEgress.INSTANCE.setRecoveryCallbacks(
                 () -> ServiceSinkhole.reload("wireguard connectivity repair", ServiceSinkhole.this, false),
-                () -> showWireGuardErrorNotification(getString(R.string.msg_wg_recovery_failed)));
+                () -> showWireGuardErrorNotification(getString(R.string.msg_wg_recovery_failed)),
+                () -> net.kollnig.missioncontrol.wg.WgRelayFailover.attemptFailover(ServiceSinkhole.this));
         jni_wireguard_required(prefs.getBoolean("wg_enabled", false)
                 && !TextUtils.isEmpty(prefs.getString("wg_config", "")));
         boolean wgOk = net.kollnig.missioncontrol.wg.WgEgress.INSTANCE.startOrUpdate(

app/src/main/java/net/kollnig/missioncontrol/wg/IvpnProfileGenerator.java

@@ -115,6 +115,19 @@ public GeneratedProfile generate(String accountNumber, String requestedCountryCo
                                      WgProfileManager.IvpnSession reusableSession,
                                      String captchaId, String captchaValue)
             throws Exception {
+        return generate(accountNumber, requestedCountryCode, reusableSession, captchaId, captchaValue, null);
+    }
+
+    /**
+     * @param excludeHostname a relay hostname to avoid re-picking when another
+     *                        candidate is available in the chosen pool, e.g. a
+     *                        relay a caller just failed over away from.
+     */
+    public GeneratedProfile generate(String accountNumber, String requestedCountryCode,
+                                     WgProfileManager.IvpnSession reusableSession,
+                                     String captchaId, String captchaValue,
+                                     String excludeHostname)
+            throws Exception {
         String account = accountNumber == null ? "" : accountNumber.trim();
         if (account.isEmpty())
             throw new IllegalArgumentException("IVPN account number is required");
@@ -126,7 +139,7 @@ public GeneratedProfile generate(String accountNumber, String requestedCountryCo
             session = createSession(account, privateKey, publicKey, captchaId, captchaValue);
         }
 
-        Relay relay = chooseRelay(fetchRelays(), requestedCountryCode);
+        Relay relay = chooseRelay(fetchRelays(), requestedCountryCode, excludeHostname);
         String config = buildConfig(session.privateKey, session.address, relay);
         return new GeneratedProfile("IVPN - " + relay.countryName, config, account,
                 relay.countryCode, relay.countryName, relay.hostname, session);
@@ -245,6 +258,10 @@ private List<Relay> fetchRelays() throws Exception {
     }
 
     private Relay chooseRelay(List<Relay> relays, String requestedCountryCode) {
+        return chooseRelay(relays, requestedCountryCode, null);
+    }
+
+    private Relay chooseRelay(List<Relay> relays, String requestedCountryCode, String excludeHostname) {
         if (relays.isEmpty())
             throw new IllegalStateException("No IVPN WireGuard relays found");
 
@@ -254,6 +271,17 @@ private Relay chooseRelay(List<Relay> relays, String requestedCountryCode) {
         if (candidates.isEmpty())
             candidates = new ArrayList<>(relays);
 
+        if (!TextUtils.isEmpty(excludeHostname)) {
+            List<Relay> withoutExcluded = new ArrayList<>();
+            for (Relay relay : candidates)
+                if (!excludeHostname.equals(relay.hostname))
+                    withoutExcluded.add(relay);
+            // Only apply the exclusion if it leaves a choice — a single-relay
+            // country should still return that relay rather than throw.
+            if (!withoutExcluded.isEmpty())
+                candidates = withoutExcluded;
+        }
+
         Collections.sort(candidates, Comparator.comparingDouble(relay -> relay.load));
         int bestPool = Math.min(3, candidates.size());
         return candidates.get(random.nextInt(Math.max(1, bestPool)));

app/src/main/java/net/kollnig/missioncontrol/wg/MullvadProfileGenerator.java

@@ -110,6 +110,16 @@ public GeneratedProfile generate(String accountNumber, String requestedCountryCo
     }
 
     public GeneratedProfile generate(String accountNumber, String requestedCountryCode, String reusableConfig) throws Exception {
+        return generate(accountNumber, requestedCountryCode, reusableConfig, null);
+    }
+
+    /**
+     * @param excludeHostname a relay hostname to avoid re-picking when another
+     *                        candidate is available in the chosen pool, e.g. a
+     *                        relay a caller just failed over away from.
+     */
+    public GeneratedProfile generate(String accountNumber, String requestedCountryCode, String reusableConfig,
+                                     String excludeHostname) throws Exception {
         String account = accountNumber == null ? "" : accountNumber.trim();
         if (account.isEmpty())
             throw new IllegalArgumentException("Mullvad account number is required");
@@ -126,7 +136,7 @@ public GeneratedProfile generate(String accountNumber, String requestedCountryCo
             privateKey = reusable.getPrivateKey();
             device = deviceFromConfig(reusable);
         }
-        Relay relay = chooseRelay(fetchRelays(), requestedCountryCode);
+        Relay relay = chooseRelay(fetchRelays(), requestedCountryCode, excludeHostname);
 
         String config = buildConfig(privateKey, device, relay);
         return new GeneratedProfile("Mullvad - " + relay.countryName, config, account,
@@ -278,6 +288,10 @@ private List<Relay> fetchRelays() throws Exception {
     }
 
     private Relay chooseRelay(List<Relay> relays, String requestedCountryCode) {
+        return chooseRelay(relays, requestedCountryCode, null);
+    }
+
+    private Relay chooseRelay(List<Relay> relays, String requestedCountryCode, String excludeHostname) {
         if (relays.isEmpty())
             throw new IllegalStateException("No active Mullvad WireGuard relays found");
 
@@ -287,6 +301,17 @@ private Relay chooseRelay(List<Relay> relays, String requestedCountryCode) {
         if (candidates.isEmpty())
             candidates = new ArrayList<>(relays);
 
+        if (!TextUtils.isEmpty(excludeHostname)) {
+            List<Relay> withoutExcluded = new ArrayList<>();
+            for (Relay relay : candidates)
+                if (!excludeHostname.equals(relay.hostname))
+                    withoutExcluded.add(relay);
+            // Only apply the exclusion if it leaves a choice — a single-relay
+            // country should still return that relay rather than throw.
+            if (!withoutExcluded.isEmpty())
+                candidates = withoutExcluded;
+        }
+
         List<Relay> stboot = new ArrayList<>();
         for (Relay relay : candidates)
             if (relay.stboot)

app/src/main/java/net/kollnig/missioncontrol/wg/WgEgress.kt

@@ -42,6 +42,20 @@ object WgEgress {
     // indefinitely, each doing DNS + handshake + monitor polling under a wakelock.
     private const val RESTART_BACKOFF_BASE_MS = 20_000L
     private const val RESTART_BACKOFF_MAX_MS = 5 * 60_000L
+    // A dead relay server looks identical to any other broken-tunnel cause
+    // from in here (no rx, handshake stalls). After this many full-restart
+    // cycles against the SAME endpoint have all failed to recover, and on
+    // every further multiple of it, ask the provider-aware callback to move
+    // the active profile to a different relay before continuing the restart
+    // loop, instead of retrying the one dead server forever.
+    private const val RELAY_FAILOVER_AFTER_ATTEMPTS = 3
+    // Bounds the relay-list fetch + config rewrite so a stalled network call
+    // can never withhold a restart for longer than the caller would have
+    // waited anyway. forceRestartPending is already set by the time this
+    // runs, which makes onMonitorBroken/onUnderlyingNetworkChanged no-op
+    // until a restart is scheduled — without a bound, a hung call would
+    // silently stall recovery instead of merely skipping the relay switch.
+    private const val FAILOVER_TIMEOUT_MS = 15_000L
 
     @Volatile private var tunnel: WgTunnel? = null
     // Monotonically identifies the tunnel instance published in [tunnel].
@@ -112,6 +126,16 @@ object WgEgress {
 
     @Volatile private var requestReloadCb: Runnable? = null
     @Volatile private var notifyBrokenCb: Runnable? = null
+    // Provider-aware hook: tries to move the active profile to a different
+    // relay server (same provider/account/country, reused key material) and
+    // returns whether it actually switched. Null, or a profile with no
+    // provider (self-hosted / manually imported single-server configs), means
+    // there is nothing to fail over to.
+    @Volatile private var regenerateEndpointCb: (() -> Boolean)? = null
+    private val failoverInFlight = java.util.concurrent.atomic.AtomicBoolean(false)
+    private val failoverExecutor = java.util.concurrent.Executors.newSingleThreadExecutor {
+        Thread(it, "wg-failover").apply { isDaemon = true }
+    }
     // Guarded by monitorLock: started/stopped from the vpn handler thread,
     // the wg-rebind thread, and the dying monitor thread itself. Unsynchronized
     // access could leak a second polling thread (double prods, double restarts).
@@ -134,9 +158,14 @@ object WgEgress {
      * [notifyBroken] surfaces a user-facing notification. Registered once
      * when the service starts the egress.
      */
-    fun setRecoveryCallbacks(requestReload: Runnable, notifyBroken: Runnable) {
+    fun setRecoveryCallbacks(
+        requestReload: Runnable,
+        notifyBroken: Runnable,
+        regenerateEndpoint: (() -> Boolean)? = null
+    ) {
         requestReloadCb = requestReload
         notifyBrokenCb = notifyBroken
+        regenerateEndpointCb = regenerateEndpoint
     }
 
     fun addStateListener(l: Runnable) { listeners.add(l) }
@@ -334,28 +363,45 @@ object WgEgress {
         requestFullRestart(
             "connectivity monitor: tunnel still broken after cheap recovery",
             notify = true,
-            expected = expected
+            expected = expected,
+            // Only a monitor-observed break (no rx/handshake) is evidence the
+            // relay itself is unreachable; a rebind failure below is not.
+            eligibleForFailover = true
         )
     }
 
-    private fun requestFullRestart(reason: String, notify: Boolean, expected: TunnelSnapshot) {
+    private fun requestFullRestart(
+        reason: String,
+        notify: Boolean,
+        expected: TunnelSnapshot,
+        eligibleForFailover: Boolean
+    ) {
         val attempt: Int
-        val delay: Long
         synchronized(tunnelLifecycleLock) {
             // This check and installation of pending state must be atomic with
             // tunnel replacement. Otherwise a replacement can land between
             // them and inherit forceRestartPending from an obsolete failure.
             if (!isCurrentLocked(expected)) return
             clearEndpointCache()
             attempt = restartAttempts++
-            delay = if (attempt == 0) 0L
-            else minOf(RESTART_BACKOFF_MAX_MS, RESTART_BACKOFF_BASE_MS shl (attempt - 1).coerceAtMost(10))
             forceRestartPending = true
             pendingRestartTunnel = expected.tunnel
             pendingRestartTunnelGeneration = expected.generation
         }
-        Log.w(TAG, "$reason; forcing restart (attempt=${attempt + 1}, delay=${delay}ms)")
+        Log.w(TAG, "$reason; forcing restart (attempt=${attempt + 1})")
         if (notify) scheduleRecoveryNotificationCheck()
+
+        val regenerate = regenerateEndpointCb
+        if (eligibleForFailover && attempt > 0 && attempt % RELAY_FAILOVER_AFTER_ATTEMPTS == 0 && regenerate != null) {
+            tryRelayFailover(regenerate, expected, attempt)
+            return
+        }
+        scheduleRestart(attempt)
+    }
+
+    private fun scheduleRestart(attempt: Int) {
+        val delay = if (attempt == 0) 0L
+        else minOf(RESTART_BACKOFF_MAX_MS, RESTART_BACKOFF_BASE_MS shl (attempt - 1).coerceAtMost(10))
         verifyHandler.removeCallbacks(pendingRestartRunnable)
         if (delay <= 0L) {
             restartScheduled = false
@@ -366,6 +412,65 @@ object WgEgress {
         }
     }
 
+    /**
+     * The same relay has now failed to recover across several full-restart
+     * cycles — likely the server itself is down, not a local network blip.
+     * Runs the (network-bound) [regenerate] callback off-thread; on success
+     * it has already rewritten the active profile's config, so the queued
+     * restart picks up the new peer and the backoff resets, since the new
+     * relay hasn't earned any of the old one's penalty. On failure or
+     * [FAILOVER_TIMEOUT_MS] timeout, falls back to the normal same-config
+     * backoff so we never restart less often than before this existed.
+     *
+     * [resolved] guards against the timeout and the completed callback both
+     * trying to schedule a restart for the same attempt: whichever of the
+     * two runs first (main thread for the timeout, [failoverExecutor] for
+     * the callback) wins the race, the other is a no-op.
+     */
+    private fun tryRelayFailover(regenerate: () -> Boolean, expected: TunnelSnapshot, attempt: Int) {
+        if (!failoverInFlight.compareAndSet(false, true)) {
+            scheduleRestart(attempt)
+            return
+        }
+        val resolved = java.util.concurrent.atomic.AtomicBoolean(false)
+        val timeoutRunnable = Runnable {
+            if (resolved.compareAndSet(false, true)) {
+                failoverInFlight.set(false)
+                if (isCurrent(expected)) {
+                    Log.w(TAG, "relay failover timed out after ${FAILOVER_TIMEOUT_MS}ms; falling back to backoff")
+                    scheduleRestart(attempt)
+                }
+            }
+        }
+        verifyHandler.postDelayed(timeoutRunnable, FAILOVER_TIMEOUT_MS)
+        try {
+            failoverExecutor.execute {
+                val switched = try {
+                    regenerate()
+                } catch (e: Throwable) {
+                    Log.w(TAG, "relay failover callback threw", e)
+                    false
+                } finally {
+                    failoverInFlight.set(false)
+                }
+                verifyHandler.removeCallbacks(timeoutRunnable)
+                if (!resolved.compareAndSet(false, true)) return@execute
+                if (!isCurrent(expected)) return@execute
+                if (switched) {
+                    Log.w(TAG, "relay failover: switched the active profile to a different server")
+                    restartAttempts = 0
+                    scheduleRestart(0)
+                } else {
+                    scheduleRestart(attempt)
+                }
+            }
+        } catch (e: java.util.concurrent.RejectedExecutionException) {
+            verifyHandler.removeCallbacks(timeoutRunnable)
+            failoverInFlight.set(false)
+            if (resolved.compareAndSet(false, true)) scheduleRestart(attempt)
+        }
+    }
+
     private fun scheduleRecoveryNotificationCheck() {
         val gen = ++recoveryNotificationGeneration
         verifyHandler.postDelayed({
@@ -517,7 +622,12 @@ object WgEgress {
                                 requestFullRestart(
                                     "WG rebind failed after network change",
                                     notify = false,
-                                    expected = expected
+                                    expected = expected,
+                                    // A rebind failure means the local network
+                                    // changed under us, not that the relay is
+                                    // dead — don't let it count toward
+                                    // switching relays.
+                                    eligibleForFailover = false
                                 )
                             }
                         }

app/src/main/java/net/kollnig/missioncontrol/wg/WgProfileManager.java

@@ -229,17 +229,36 @@ public void updateActiveProfileConfig(String config) {
         String active = getActiveProfileId();
         if (TextUtils.isEmpty(active))
             return;
+        updateProfileConfig(active, config);
+    }
+
+    /**
+     * Like {@link #updateActiveProfileConfig}, but only writes if {@code profileId}
+     * is still the active profile. For callers (e.g. background relay failover)
+     * that resolved which profile to update before doing slow work: without this
+     * check, a profile switch that lands during that work would silently write
+     * the stale profile's regenerated config over whatever the user switched to.
+     * Returns whether the write happened.
+     */
+    public boolean updateProfileConfigIfActive(String profileId, String config) {
+        if (TextUtils.isEmpty(profileId) || !profileId.equals(getActiveProfileId()))
+            return false;
+        updateProfileConfig(profileId, config);
+        return true;
+    }
 
+    private void updateProfileConfig(String profileId, String config) {
         JSONArray profiles = readProfilesJson();
-        JSONObject profile = findJsonProfile(profiles, active);
+        JSONObject profile = findJsonProfile(profiles, profileId);
         if (profile == null)
             return;
 
         try {
             profile.put("config", config == null ? "" : config);
             SharedPreferences.Editor editor = prefs.edit();
             writeProfilesJson(editor, profiles);
-            editor.putString(PREF_WG_CONFIG, config == null ? "" : config);
+            if (profileId.equals(getActiveProfileId()))
+                editor.putString(PREF_WG_CONFIG, config == null ? "" : config);
             editor.apply();
         } catch (JSONException ex) {
             Log.w(TAG, "Update WireGuard profile failed: " + ex.getMessage());